Log: /mnt/jenkins/workspace/cloud-pxc-operator_PR-2525/e2e-tests/logs/monitoring-2-0-8-0.log Warning: version difference between client (1.36) and server (1.33) exceeds the supported minor version skew of +/-1 Warning: version difference between client (1.36) and server (1.33) exceeds the supported minor version skew of +/-1 + cluster=monitoring + create_infra monitoring-2-0-9920 + local ns=monitoring-2-0-9920 + '[' -n pxc-operator ']' + read -r pns name + kubectl_bin get pxc --all-namespaces -o 'jsonpath={range .items[*]}{.metadata.namespace} {.metadata.name}{"\n"}{end}' + [[ -z monitoring-2-0-30530 ]] + [[ -z monitoring ]] + kubectl_bin patch pxc -n monitoring-2-0-30530 monitoring --type=merge -p '{"metadata":{"finalizers":[]}}' ++ mktemp + local LAST_OUT=/tmp/tmp.Ek8ev1UabT ++ mktemp + local LAST_ERR=/tmp/tmp.d4L6mJxUdN + local exit_status=0 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl patch pxc -n monitoring-2-0-30530 monitoring --type=merge -p '{"metadata":{"finalizers":[]}}' + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.Ek8ev1UabT perconaxtradbcluster.pxc.percona.com/monitoring patched (no change) + cat /tmp/tmp.d4L6mJxUdN + rm /tmp/tmp.Ek8ev1UabT /tmp/tmp.d4L6mJxUdN + return 0 + read -r pns name + read -r pns name + kubectl_bin get pxc-backup --all-namespaces -o 'jsonpath={range .items[*]}{.metadata.namespace} {.metadata.name}{"\n"}{end}' + timeout 300 kubectl delete pxc --all --all-namespaces --wait=false --ignore-not-found perconaxtradbcluster.pxc.percona.com "monitoring" deleted from monitoring-2-0-30530 namespace + timeout 300 kubectl delete pxc-backup --all --all-namespaces --wait=false --ignore-not-found No resources found + timeout 300 kubectl delete pxc-restore --all --all-namespaces --wait=false --ignore-not-found No resources found + create_namespace pxc-operator + local namespace=pxc-operator + local skip_clean_namespace= + [[ 1 == 1 ]] + [[ -z '' ]] + destroy_chaos_mesh ++ tail -n1 ++ helm list --all-namespaces --filter chaos-mesh ++ sed s/NAMESPACE// ++ awk '-F ' '{print $2}' + local chaos_mesh_ns= + '[' -n '' ']' ++ grep chaos-mesh ++ kubectl get MutatingWebhookConfiguration ++ awk '{print $1}' + timeout 30 kubectl delete MutatingWebhookConfiguration error: resource(s) were provided, but no name was specified + : ++ awk '{print $1}' ++ kubectl get ValidatingWebhookConfiguration ++ grep chaos-mesh + timeout 30 kubectl delete ValidatingWebhookConfiguration error: resource(s) were provided, but no name was specified + : ++ kubectl get ValidatingWebhookConfiguration ++ grep validate-auth ++ awk '{print $1}' + timeout 30 kubectl delete ValidatingWebhookConfiguration error: resource(s) were provided, but no name was specified + : ++ awk '{print $1}' ++ kubectl api-resources ++ grep chaos-mesh ++ grep chaos-mesh.org ++ kubectl get crd ++ awk '{print $1}' + timeout 30 kubectl delete crd error: resource(s) were provided, but no name was specified + : ++ kubectl get clusterrolebinding ++ grep chaos-mesh ++ awk '{print $1}' + timeout 30 kubectl delete clusterrolebinding error: resource(s) were provided, but no name was specified + : ++ awk '{print $1}' ++ grep chaos-mesh ++ kubectl get clusterrole + timeout 30 kubectl delete clusterrole error: resource(s) were provided, but no name was specified + : + desc 'cleaned up all old namespaces' + set +o xtrace ----------------------------------------------------------------------------------- cleaned up all old namespaces ----------------------------------------------------------------------------------- + '[' -n '' ']' + desc 'cleaned up old namespaces pxc-operator' + set +o xtrace ----------------------------------------------------------------------------------- cleaned up old namespaces pxc-operator ----------------------------------------------------------------------------------- + kubectl_bin delete namespace pxc-operator ++ mktemp + kubectl_bin get ns + xargs kubectl delete ns + awk '{print$1}' + local LAST_OUT=/tmp/tmp.uaLjYdH0E1 ++ mktemp + local LAST_OUT=/tmp/tmp.vcHhPeYClh + grep -E -v '^kube-|^default|Terminating|pxc-operator|openshift|^gke-|^gmp-|^NAME' ++ mktemp + local LAST_ERR=/tmp/tmp.4HfVr94xh5 + local exit_status=0 ++ mktemp ++ seq 0 2 + local LAST_ERR=/tmp/tmp.aV4jLkPFxS + local exit_status=0 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl get ns + for i in $(seq 0 2) + set +e + kubectl delete namespace pxc-operator + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.vcHhPeYClh + cat /tmp/tmp.aV4jLkPFxS + rm /tmp/tmp.vcHhPeYClh /tmp/tmp.aV4jLkPFxS + return 0 namespace "monitoring-2-0-30530" deleted + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.uaLjYdH0E1 namespace "pxc-operator" deleted + cat /tmp/tmp.4HfVr94xh5 + rm /tmp/tmp.uaLjYdH0E1 /tmp/tmp.4HfVr94xh5 + return 0 + wait_for_delete namespace/pxc-operator + local res=namespace/pxc-operator + echo -n 'waiting for namespace/pxc-operator to be deleted' waiting for namespace/pxc-operator to be deleted+ set +o xtrace Error from server (NotFound): namespaces "pxc-operator" not found + desc 'create namespace pxc-operator' + set +o xtrace ----------------------------------------------------------------------------------- create namespace pxc-operator ----------------------------------------------------------------------------------- + kubectl_bin create namespace pxc-operator ++ mktemp + local LAST_OUT=/tmp/tmp.v88KPZLFdv ++ mktemp + local LAST_ERR=/tmp/tmp.53wG9jSNTI + local exit_status=0 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl create namespace pxc-operator + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.v88KPZLFdv namespace/pxc-operator created + cat /tmp/tmp.53wG9jSNTI + rm /tmp/tmp.v88KPZLFdv /tmp/tmp.53wG9jSNTI + return 0 ++ kubectl_bin config current-context +++ mktemp ++ local LAST_OUT=/tmp/tmp.nwmBzDT3Os +++ mktemp ++ local LAST_ERR=/tmp/tmp.Fe1ns44taU ++ local exit_status=0 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl config current-context ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.nwmBzDT3Os ++ cat /tmp/tmp.Fe1ns44taU ++ rm /tmp/tmp.nwmBzDT3Os /tmp/tmp.Fe1ns44taU ++ return 0 + kubectl_bin config set-context gke_cloud-dev-112233_us-central1-a_jen-pxc-2525-c81e47f0-4-cluster13 --namespace=pxc-operator ++ mktemp + local LAST_OUT=/tmp/tmp.i7XTmrTnVi ++ mktemp + local LAST_ERR=/tmp/tmp.x0Yr8NLMGu + local exit_status=0 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl config set-context gke_cloud-dev-112233_us-central1-a_jen-pxc-2525-c81e47f0-4-cluster13 --namespace=pxc-operator + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.i7XTmrTnVi Context "gke_cloud-dev-112233_us-central1-a_jen-pxc-2525-c81e47f0-4-cluster13" modified. + cat /tmp/tmp.x0Yr8NLMGu + rm /tmp/tmp.i7XTmrTnVi /tmp/tmp.x0Yr8NLMGu + return 0 + deploy_operator + desc 'start PXC operator' + set +o xtrace ----------------------------------------------------------------------------------- start PXC operator ----------------------------------------------------------------------------------- + kubectl_bin apply --server-side --force-conflicts -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2525/deploy/crd.yaml ++ mktemp + local LAST_OUT=/tmp/tmp.uda2QQqUnV ++ mktemp + local LAST_ERR=/tmp/tmp.Pl9laipQGc + local exit_status=0 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl apply --server-side --force-conflicts -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2525/deploy/crd.yaml + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.uda2QQqUnV customresourcedefinition.apiextensions.k8s.io/perconaxtradbclusterbackups.pxc.percona.com serverside-applied customresourcedefinition.apiextensions.k8s.io/perconaxtradbclusterrestores.pxc.percona.com serverside-applied customresourcedefinition.apiextensions.k8s.io/perconaxtradbclusters.pxc.percona.com serverside-applied + cat /tmp/tmp.Pl9laipQGc + rm /tmp/tmp.uda2QQqUnV /tmp/tmp.Pl9laipQGc + return 0 + kubectl_bin wait --for=condition=Established --timeout=120s crd/perconaxtradbclusters.pxc.percona.com ++ mktemp + local LAST_OUT=/tmp/tmp.8qfZjEkouo ++ mktemp + local LAST_ERR=/tmp/tmp.YbsXF4MD1h + local exit_status=0 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl wait --for=condition=Established --timeout=120s crd/perconaxtradbclusters.pxc.percona.com + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.8qfZjEkouo customresourcedefinition.apiextensions.k8s.io/perconaxtradbclusters.pxc.percona.com condition met + cat /tmp/tmp.YbsXF4MD1h E0806 03:00:58.766922 763909 reflector.go:227] "Failed to watch" err="Get \"https://34.31.71.55/apis/apiextensions.k8s.io/v1/customresourcedefinitions?allowWatchBookmarks=true&fieldSelector=metadata.name%3Dperconaxtradbclusters.pxc.percona.com&resourceVersion=1785985253457386000&timeoutSeconds=588&watch=true\": context canceled" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" type="*unstructured.Unstructured" + rm /tmp/tmp.8qfZjEkouo /tmp/tmp.YbsXF4MD1h + return 0 + kubectl_bin wait --for=condition=Established --timeout=120s crd/perconaxtradbclusterbackups.pxc.percona.com ++ mktemp + local LAST_OUT=/tmp/tmp.dZro111Xei ++ mktemp + local LAST_ERR=/tmp/tmp.uJmvv2pvOT + local exit_status=0 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl wait --for=condition=Established --timeout=120s crd/perconaxtradbclusterbackups.pxc.percona.com + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.dZro111Xei customresourcedefinition.apiextensions.k8s.io/perconaxtradbclusterbackups.pxc.percona.com condition met + cat /tmp/tmp.uJmvv2pvOT E0806 03:01:01.292337 764396 reflector.go:227] "Failed to watch" err="Get \"https://34.31.71.55/apis/apiextensions.k8s.io/v1/customresourcedefinitions?allowWatchBookmarks=true&fieldSelector=metadata.name%3Dperconaxtradbclusterbackups.pxc.percona.com&resourceVersion=1785985258457882000&timeoutSeconds=344&watch=true\": context canceled" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" type="*unstructured.Unstructured" + rm /tmp/tmp.dZro111Xei /tmp/tmp.uJmvv2pvOT + return 0 + kubectl_bin wait --for=condition=Established --timeout=120s crd/perconaxtradbclusterrestores.pxc.percona.com ++ mktemp + local LAST_OUT=/tmp/tmp.nn3IoRHzlO ++ mktemp + local LAST_ERR=/tmp/tmp.NWMKWkUM19 + local exit_status=0 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl wait --for=condition=Established --timeout=120s crd/perconaxtradbclusterrestores.pxc.percona.com + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.nn3IoRHzlO customresourcedefinition.apiextensions.k8s.io/perconaxtradbclusterrestores.pxc.percona.com condition met + cat /tmp/tmp.NWMKWkUM19 E0806 03:01:04.286370 764760 reflector.go:227] "Failed to watch" err="Get \"https://34.31.71.55/apis/apiextensions.k8s.io/v1/customresourcedefinitions?allowWatchBookmarks=true&fieldSelector=metadata.name%3Dperconaxtradbclusterrestores.pxc.percona.com&resourceVersion=1785985263458020000&timeoutSeconds=457&watch=true\": context canceled" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" type="*unstructured.Unstructured" + rm /tmp/tmp.nn3IoRHzlO /tmp/tmp.NWMKWkUM19 + return 0 + '[' -n pxc-operator ']' + apply_rbac cw-rbac + local operator_namespace=pxc-operator + local rbac=cw-rbac + cat /mnt/jenkins/workspace/cloud-pxc-operator_PR-2525/deploy/cw-rbac.yaml + kubectl_bin apply -f - + sed -e 's^namespace: .*^namespace: pxc-operator^' ++ mktemp + local LAST_OUT=/tmp/tmp.s9fQbmD3Z0 ++ mktemp + local LAST_ERR=/tmp/tmp.TLGkoQO1Jh + local exit_status=0 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.s9fQbmD3Z0 clusterrole.rbac.authorization.k8s.io/percona-xtradb-cluster-operator unchanged serviceaccount/percona-xtradb-cluster-operator created clusterrolebinding.rbac.authorization.k8s.io/service-account-percona-xtradb-cluster-operator unchanged + cat /tmp/tmp.TLGkoQO1Jh + rm /tmp/tmp.s9fQbmD3Z0 /tmp/tmp.TLGkoQO1Jh + return 0 + kubectl_bin apply -f - + sed -e 's^image: .*^image: perconalab/percona-xtradb-cluster-operator:PR-2525-c81e47f0^' + sed -e 's^failureThreshold: .*^failureThreshold: 10^' + yq eval '(select(.kind == "Deployment").spec.template.spec.containers[] | select(.name == "percona-xtradb-cluster-operator").env[] | select(.name == "LOG_LEVEL").value) = "VERBOSE"' - + yq eval '(select(.kind == "Deployment").spec.template.spec.containers[] | select(.name == "percona-xtradb-cluster-operator").env[] | select(.name == "DISABLE_TELEMETRY").value) = "true"' - + cat /mnt/jenkins/workspace/cloud-pxc-operator_PR-2525/deploy/cw-operator.yaml + yq eval '(select(.kind == "Deployment").spec.template.spec.containers[] | select(.name == "percona-xtradb-cluster-operator").env[] | select(.name == "PXCO_FEATURE_GATES").value) = ""' - ++ mktemp + local LAST_OUT=/tmp/tmp.BZOo0TDri6 ++ mktemp + local LAST_ERR=/tmp/tmp.ywyfsYaZ2A + local exit_status=0 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.BZOo0TDri6 deployment.apps/percona-xtradb-cluster-operator created service/percona-xtradb-cluster-operator created + cat /tmp/tmp.ywyfsYaZ2A + rm /tmp/tmp.BZOo0TDri6 /tmp/tmp.ywyfsYaZ2A + return 0 + sleep 10 + kubectl_bin wait --for=condition=Ready pods -l app.kubernetes.io/component=operator,app.kubernetes.io/instance=percona-xtradb-cluster-operator,app.kubernetes.io/name=percona-xtradb-cluster-operator --timeout=30s ++ mktemp + local LAST_OUT=/tmp/tmp.9hwlLK6lq0 ++ mktemp + local LAST_ERR=/tmp/tmp.WBNvKSkcYL + local exit_status=0 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl wait --for=condition=Ready pods -l app.kubernetes.io/component=operator,app.kubernetes.io/instance=percona-xtradb-cluster-operator,app.kubernetes.io/name=percona-xtradb-cluster-operator --timeout=30s + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.9hwlLK6lq0 pod/percona-xtradb-cluster-operator-6699755d74-jq8cr condition met + cat /tmp/tmp.WBNvKSkcYL E0806 03:01:27.128018 767543 reflector.go:227] "Failed to watch" err="Get \"https://34.31.71.55/api/v1/namespaces/pxc-operator/pods?allowWatchBookmarks=true&fieldSelector=metadata.name%3Dpercona-xtradb-cluster-operator-6699755d74-jq8cr&resourceVersion=1785985286576229000&timeoutSeconds=584&watch=true\": context canceled" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" type="*unstructured.Unstructured" + rm /tmp/tmp.9hwlLK6lq0 /tmp/tmp.WBNvKSkcYL + return 0 ++ get_operator_pod ++ local label_prefix=app.kubernetes.io/ +++ grep -c percona-xtradb-cluster-operator +++ kubectl get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator -n pxc-operator ++ local check_label=1 ++ [[ 1 -eq 0 ]] ++ kubectl_bin get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator --field-selector=status.phase=Running -o json -n pxc-operator ++ jq -r '.items[] | select(.metadata.deletionTimestamp == null) | .metadata.name' ++ head -1 +++ mktemp ++ local LAST_OUT=/tmp/tmp.9lKz0IHTBZ +++ mktemp ++ local LAST_ERR=/tmp/tmp.xyYFK1n8AB ++ local exit_status=0 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator --field-selector=status.phase=Running -o json -n pxc-operator ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.9lKz0IHTBZ ++ cat /tmp/tmp.xyYFK1n8AB ++ rm /tmp/tmp.9lKz0IHTBZ /tmp/tmp.xyYFK1n8AB ++ return 0 + wait_pod percona-xtradb-cluster-operator-6699755d74-jq8cr 480 pxc-operator + local pod=percona-xtradb-cluster-operator-6699755d74-jq8cr + local max_retry=480 + local ns=pxc-operator ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ grep -E '^(pxc|proxysql)$' ++ echo percona-xtradb-cluster-operator-6699755d74-jq8cr + local container= + set +o xtrace pod/percona-xtradb-cluster-operator-6699755d74-jq8cr condition met E0806 03:01:31.908048 768298 reflector.go:227] "Failed to watch" err="Get \"https://34.31.71.55/api/v1/namespaces/pxc-operator/pods?allowWatchBookmarks=true&fieldSelector=metadata.name%3Dpercona-xtradb-cluster-operator-6699755d74-jq8cr&resourceVersion=1785985289522741000&timeoutSeconds=318&watch=true\": context canceled" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" type="*unstructured.Unstructured" waiting for pod/percona-xtradb-cluster-operator-6699755d74-jq8cr to become Ready.Ok + sleep 3 + create_namespace monitoring-2-0-9920 + local namespace=monitoring-2-0-9920 + local skip_clean_namespace= + [[ 1 == 1 ]] + [[ -z '' ]] + destroy_chaos_mesh ++ awk '-F ' '{print $2}' ++ helm list --all-namespaces --filter chaos-mesh ++ tail -n1 ++ sed s/NAMESPACE// + local chaos_mesh_ns= + '[' -n '' ']' ++ grep chaos-mesh ++ awk '{print $1}' ++ kubectl get MutatingWebhookConfiguration + timeout 30 kubectl delete MutatingWebhookConfiguration error: resource(s) were provided, but no name was specified + : ++ grep chaos-mesh ++ kubectl get ValidatingWebhookConfiguration ++ awk '{print $1}' + timeout 30 kubectl delete ValidatingWebhookConfiguration error: resource(s) were provided, but no name was specified + : ++ kubectl get ValidatingWebhookConfiguration ++ awk '{print $1}' ++ grep validate-auth + timeout 30 kubectl delete ValidatingWebhookConfiguration error: resource(s) were provided, but no name was specified + : ++ awk '{print $1}' ++ grep chaos-mesh ++ kubectl api-resources ++ awk '{print $1}' ++ kubectl get crd ++ grep chaos-mesh.org + timeout 30 kubectl delete crd error: resource(s) were provided, but no name was specified + : ++ kubectl get clusterrolebinding ++ awk '{print $1}' ++ grep chaos-mesh + timeout 30 kubectl delete clusterrolebinding error: resource(s) were provided, but no name was specified + : ++ awk '{print $1}' ++ kubectl get clusterrole ++ grep chaos-mesh + timeout 30 kubectl delete clusterrole error: resource(s) were provided, but no name was specified + : + desc 'cleaned up all old namespaces' + set +o xtrace ----------------------------------------------------------------------------------- cleaned up all old namespaces ----------------------------------------------------------------------------------- + '[' -n '' ']' + desc 'cleaned up old namespaces monitoring-2-0-9920' + set +o xtrace ----------------------------------------------------------------------------------- cleaned up old namespaces monitoring-2-0-9920 ----------------------------------------------------------------------------------- + kubectl_bin delete namespace monitoring-2-0-9920 + kubectl_bin get ns ++ mktemp + local LAST_OUT=/tmp/tmp.ptv2CAJMTH + awk '{print$1}' + grep -E -v '^kube-|^default|Terminating|pxc-operator|openshift|^gke-|^gmp-|^NAME' ++ mktemp + local LAST_OUT=/tmp/tmp.BDcaJh6hnN + xargs kubectl delete ns ++ mktemp + local LAST_ERR=/tmp/tmp.EkSVBDqpgW + local exit_status=0 ++ mktemp + local LAST_ERR=/tmp/tmp.X5QRtfOlOB + local exit_status=0 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl get ns ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl delete namespace monitoring-2-0-9920 + exit_status=1 + set -e + '[' 1 '!=' 0 ']' + '[' 1 == 1 ']' + sleep 0 + for i in $(seq 0 2) + set +e + kubectl delete namespace monitoring-2-0-9920 + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.ptv2CAJMTH + cat /tmp/tmp.EkSVBDqpgW + rm /tmp/tmp.ptv2CAJMTH /tmp/tmp.EkSVBDqpgW + return 0 + exit_status=1 + set -e + '[' 1 '!=' 0 ']' + '[' 1 == 1 ']' + sleep 0 + for i in $(seq 0 2) + set +e + kubectl delete namespace monitoring-2-0-9920 error: resource(s) were provided, but no name was specified + exit_status=1 + set -e + '[' 1 '!=' 0 ']' + '[' 1 == 1 ']' + sleep 0 + cat /tmp/tmp.BDcaJh6hnN + cat /tmp/tmp.X5QRtfOlOB Error from server (NotFound): namespaces "monitoring-2-0-9920" not found + rm /tmp/tmp.BDcaJh6hnN /tmp/tmp.X5QRtfOlOB + return 1 + : + wait_for_delete namespace/monitoring-2-0-9920 + local res=namespace/monitoring-2-0-9920 + echo -n 'waiting for namespace/monitoring-2-0-9920 to be deleted' waiting for namespace/monitoring-2-0-9920 to be deleted+ set +o xtrace Error from server (NotFound): namespaces "monitoring-2-0-9920" not found + desc 'create namespace monitoring-2-0-9920' + set +o xtrace ----------------------------------------------------------------------------------- create namespace monitoring-2-0-9920 ----------------------------------------------------------------------------------- + kubectl_bin create namespace monitoring-2-0-9920 ++ mktemp + local LAST_OUT=/tmp/tmp.2BqiaYQZPn ++ mktemp + local LAST_ERR=/tmp/tmp.M7E9viTBYN + local exit_status=0 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl create namespace monitoring-2-0-9920 + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.2BqiaYQZPn namespace/monitoring-2-0-9920 created + cat /tmp/tmp.M7E9viTBYN + rm /tmp/tmp.2BqiaYQZPn /tmp/tmp.M7E9viTBYN + return 0 ++ kubectl_bin config current-context +++ mktemp ++ local LAST_OUT=/tmp/tmp.KOmDMhkll9 +++ mktemp ++ local LAST_ERR=/tmp/tmp.PQwqnfQ87F ++ local exit_status=0 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl config current-context ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.KOmDMhkll9 ++ cat /tmp/tmp.PQwqnfQ87F ++ rm /tmp/tmp.KOmDMhkll9 /tmp/tmp.PQwqnfQ87F ++ return 0 + kubectl_bin config set-context gke_cloud-dev-112233_us-central1-a_jen-pxc-2525-c81e47f0-4-cluster13 --namespace=monitoring-2-0-9920 ++ mktemp + local LAST_OUT=/tmp/tmp.cdXtmpC7Gq ++ mktemp + local LAST_ERR=/tmp/tmp.Hz6gMTyPfc + local exit_status=0 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl config set-context gke_cloud-dev-112233_us-central1-a_jen-pxc-2525-c81e47f0-4-cluster13 --namespace=monitoring-2-0-9920 + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.cdXtmpC7Gq Context "gke_cloud-dev-112233_us-central1-a_jen-pxc-2525-c81e47f0-4-cluster13" modified. + cat /tmp/tmp.Hz6gMTyPfc + rm /tmp/tmp.cdXtmpC7Gq /tmp/tmp.Hz6gMTyPfc + return 0 + apply_secrets + desc 'create secrets for cloud storages' + set +o xtrace ----------------------------------------------------------------------------------- create secrets for cloud storages ----------------------------------------------------------------------------------- + '[' -z '' ']' + kubectl_bin apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2525/e2e-tests/conf/minio-secret.yml -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2525/e2e-tests/conf/cloud-secret.yml ++ mktemp + local LAST_OUT=/tmp/tmp.8oTXZbY1hS ++ mktemp + local LAST_ERR=/tmp/tmp.O0oh6x9uwj + local exit_status=0 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2525/e2e-tests/conf/minio-secret.yml -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2525/e2e-tests/conf/cloud-secret.yml + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.8oTXZbY1hS secret/minio-secret created secret/aws-s3-secret created secret/do-spaces-secret created secret/gcp-cs-secret created secret/azure-secret created + cat /tmp/tmp.O0oh6x9uwj + rm /tmp/tmp.8oTXZbY1hS /tmp/tmp.O0oh6x9uwj + return 0 + deploy_helm monitoring-2-0-9920 + helm repo add hashicorp https://helm.releases.hashicorp.com "hashicorp" already exists with the same configuration, skipping + helm repo add minio https://charts.min.io/ "minio" already exists with the same configuration, skipping + helm repo update Hang tight while we grab the latest from your chart repositories... ...Successfully got an update from the "minio" chart repository ...Successfully got an update from the "chaos-mesh" chart repository ...Successfully got an update from the "hashicorp" chart repository ...Successfully got an update from the "percona" chart repository Update Complete. ⎈Happy Helming!⎈ + desc 'install PMM Server' + set +o xtrace ----------------------------------------------------------------------------------- install PMM Server ----------------------------------------------------------------------------------- + deploy_pmm_server + '[' '!' -z '' ']' + helm repo add percona https://percona.github.io/percona-helm-charts/ "percona" already exists with the same configuration, skipping + helm repo update Hang tight while we grab the latest from your chart repositories... ...Successfully got an update from the "chaos-mesh" chart repository ...Successfully got an update from the "minio" chart repository ...Successfully got an update from the "hashicorp" chart repository ...Successfully got an update from the "percona" chart repository Update Complete. ⎈Happy Helming!⎈ + helm uninstall monitoring Error: uninstall: Release not loaded: monitoring: release: not found + : + retry 10 60 helm install monitoring --set imageRepo=perconalab/pmm-server --set imageTag=dev-latest https://percona-charts.storage.googleapis.com/pmm-server-9.9.9.tgz + local max=10 + local delay=60 + shift 2 + local n=1 + helm install monitoring --set imageRepo=perconalab/pmm-server --set imageTag=dev-latest https://percona-charts.storage.googleapis.com/pmm-server-9.9.9.tgz NAME: monitoring LAST DEPLOYED: Thu Aug 6 03:02:17 2026 NAMESPACE: monitoring-2-0-9920 STATUS: deployed REVISION: 1 TEST SUITE: None NOTES: PMM server can be accessed via HTTPS (port 443) on the following DNS name from within your cluster: endpoint: https://monitoring-service.monitoring-2-0-9920.svc.cluster.local:443 login: admin password: admin + wait_for_pmm_service + local timeout=420 ++ date +%s + local start=1785985339 + kubectl_bin get svc monitoring-service -o 'jsonpath={.status.loadBalancer.ingress[0]}' + grep -q . ++ date +%s + (( 1785985340 - start >= timeout )) + sleep 2 + grep -q . + kubectl_bin get svc monitoring-service -o 'jsonpath={.status.loadBalancer.ingress[0]}' ++ date +%s + (( 1785985343 - start >= timeout )) + sleep 2 + kubectl_bin get svc monitoring-service -o 'jsonpath={.status.loadBalancer.ingress[0]}' + grep -q . ++ date +%s + (( 1785985346 - start >= timeout )) + sleep 2 + grep -q . + kubectl_bin get svc monitoring-service -o 'jsonpath={.status.loadBalancer.ingress[0]}' ++ date +%s + (( 1785985350 - start >= timeout )) + sleep 2 + kubectl_bin get svc monitoring-service -o 'jsonpath={.status.loadBalancer.ingress[0]}' + grep -q . ++ date +%s + (( 1785985354 - start >= timeout )) + sleep 2 + grep -q . + kubectl_bin get svc monitoring-service -o 'jsonpath={.status.loadBalancer.ingress[0]}' ++ date +%s + (( 1785985357 - start >= timeout )) + sleep 2 + kubectl_bin get svc monitoring-service -o 'jsonpath={.status.loadBalancer.ingress[0]}' + grep -q . ++ date +%s + (( 1785985360 - start >= timeout )) + sleep 2 + kubectl_bin get svc monitoring-service -o 'jsonpath={.status.loadBalancer.ingress[0]}' + grep -q . ++ date +%s + (( 1785985364 - start >= timeout )) + sleep 2 + kubectl_bin get svc monitoring-service -o 'jsonpath={.status.loadBalancer.ingress[0]}' + grep -q . ++ date +%s + (( 1785985368 - start >= timeout )) + sleep 2 + kubectl_bin get svc monitoring-service -o 'jsonpath={.status.loadBalancer.ingress[0]}' + grep -q . ++ date +%s + (( 1785985371 - start >= timeout )) + sleep 2 + kubectl_bin get svc monitoring-service -o 'jsonpath={.status.loadBalancer.ingress[0]}' + grep -q . ++ date +%s + (( 1785985375 - start >= timeout )) + sleep 2 + grep -q . + kubectl_bin get svc monitoring-service -o 'jsonpath={.status.loadBalancer.ingress[0]}' + kubectl_bin wait sts/monitoring '--for=jsonpath={.status.readyReplicas}=1' --timeout=420s ++ mktemp + local LAST_OUT=/tmp/tmp.HbGQPpUfX0 ++ mktemp + local LAST_ERR=/tmp/tmp.YRMlrablgz + local exit_status=0 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl wait sts/monitoring '--for=jsonpath={.status.readyReplicas}=1' --timeout=420s + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.HbGQPpUfX0 statefulset.apps/monitoring condition met + cat /tmp/tmp.YRMlrablgz E0806 03:03:01.188906 779813 reflector.go:227] "Failed to watch" err="Get \"https://34.31.71.55/apis/apps/v1/namespaces/monitoring-2-0-9920/statefulsets?allowWatchBookmarks=true&fieldSelector=metadata.name%3Dmonitoring&resourceVersion=1785985378674950000&timeoutSeconds=475&watch=true\": context canceled" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" type="*unstructured.Unstructured" + rm /tmp/tmp.HbGQPpUfX0 /tmp/tmp.YRMlrablgz + return 0 ++ kubectl_bin exec monitoring-0 -- bash -c 'printenv | grep ADMIN_PASSWORD | cut -d '\''='\'' -f2' +++ mktemp ++ local LAST_OUT=/tmp/tmp.kkWYT1ouIw +++ mktemp ++ local LAST_ERR=/tmp/tmp.UNAkbKcZlG ++ local exit_status=0 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl exec monitoring-0 -- bash -c 'printenv | grep ADMIN_PASSWORD | cut -d '\''='\'' -f2' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.kkWYT1ouIw ++ cat /tmp/tmp.UNAkbKcZlG ++ rm /tmp/tmp.kkWYT1ouIw /tmp/tmp.UNAkbKcZlG ++ return 0 + ADMIN_PASSWORD=admin + sleep 5 + kubectl_bin exec monitoring-0 -- bash -c 'grafana-cli --homepath=/usr/share/grafana --config=/etc/grafana/grafana.ini admin reset-admin-password admin' ++ mktemp + local LAST_OUT=/tmp/tmp.wWl9DEaiYX ++ mktemp + local LAST_ERR=/tmp/tmp.45RIr6dyNa + local exit_status=0 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl exec monitoring-0 -- bash -c 'grafana-cli --homepath=/usr/share/grafana --config=/etc/grafana/grafana.ini admin reset-admin-password admin' + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.wWl9DEaiYX logger=settings t=2026-08-06T03:03:10.176769352Z level=info msg="Starting Grafana" version= commit= branch= compiled=1970-01-01T00:00:00Z logger=settings t=2026-08-06T03:03:10.176936572Z level=info msg="Config loaded from" file=/usr/share/grafana/conf/defaults.ini logger=settings t=2026-08-06T03:03:10.176949232Z level=info msg="Config loaded from" file=/etc/grafana/grafana.ini logger=settings t=2026-08-06T03:03:10.176954782Z level=info msg="Path Home" path=/usr/share/grafana logger=settings t=2026-08-06T03:03:10.176960142Z level=info msg="Path Data" path=/srv/grafana logger=settings t=2026-08-06T03:03:10.176965042Z level=info msg="Path Logs" path=/srv/logs logger=settings t=2026-08-06T03:03:10.176969862Z level=info msg="Path Plugins" path=/srv/grafana/plugins logger=settings t=2026-08-06T03:03:10.176974472Z level=info msg="Path Provisioning" path=/usr/share/grafana/conf/provisioning logger=settings t=2026-08-06T03:03:10.176978812Z level=info msg="App mode production" logger=sqlstore t=2026-08-06T03:03:10.177047612Z level=info msg="Connecting to DB" dbtype=postgres logger=migrator t=2026-08-06T03:03:10.189704651Z level=info msg="Starting DB migrations" logger=migrator t=2026-08-06T03:03:10.192803311Z level=info msg="migrations completed" performed=0 skipped=452 duration=284.65µs logger=secrets t=2026-08-06T03:03:10.193993591Z level=info msg="Envelope encryption state" enabled=true currentprovider=secretKey.v1 logger=plugin.finder t=2026-08-06T03:03:10.222295778Z level=warn msg="Skipping finding plugins as directory does not exist" path=/usr/share/grafana/plugins-bundled logger=plugin.signature.validator t=2026-08-06T03:03:10.344139828Z level=warn msg="Permitting unsigned plugin. This is not recommended" pluginID=pmm-pt-summary-panel pluginDir=/srv/grafana/plugins/pmm-app/dist/pmm-pt-summary/panel logger=plugin.signature.validator t=2026-08-06T03:03:10.344178088Z level=warn msg="Permitting unsigned plugin. This is not recommended" pluginID=grafana-polystat-panel pluginDir=/srv/grafana/plugins/grafana-polystat-panel logger=plugin.signature.validator t=2026-08-06T03:03:10.344190879Z level=warn msg="Permitting unsigned plugin. This is not recommended" pluginID=pmm-pt-summary-datasource pluginDir=/srv/grafana/plugins/pmm-app/dist/pmm-pt-summary/datasource logger=plugin.signature.validator t=2026-08-06T03:03:10.344206739Z level=warn msg="Permitting unsigned plugin. This is not recommended" pluginID=pmm-qan-app-panel pluginDir=/srv/grafana/plugins/pmm-app/dist/pmm-qan logger=plugin.signature.validator t=2026-08-06T03:03:10.344261388Z level=warn msg="Permitting unsigned plugin. This is not recommended" pluginID=pmm-app pluginDir=/srv/grafana/plugins/pmm-app/dist logger=plugin.pmm-app t=2026-08-06T03:03:10.349775418Z level=warn msg="Included dashboard is missing a UID field" logger=plugin.loader t=2026-08-06T03:03:10.350040888Z level=info msg="Plugin registered" pluginID=pmm-pt-summary-panel logger=plugin.loader t=2026-08-06T03:03:10.350052288Z level=info msg="Plugin registered" pluginID=grafana-piechart-panel logger=plugin.loader t=2026-08-06T03:03:10.350058288Z level=info msg="Plugin registered" pluginID=grafana-clickhouse-datasource logger=plugin.grafana-clickhouse-datasource t=2026-08-06T03:03:10.359799077Z level=warn msg="Plugin process is running with elevated privileges. This is not recommended" logger=plugin.loader t=2026-08-06T03:03:10.359830987Z level=info msg="Plugin registered" pluginID=grafana-polystat-panel logger=plugin.loader t=2026-08-06T03:03:10.359839567Z level=info msg="Plugin registered" pluginID=pmm-pt-summary-datasource logger=plugin.loader t=2026-08-06T03:03:10.359845847Z level=info msg="Plugin registered" pluginID=pmm-qan-app-panel logger=plugin.loader t=2026-08-06T03:03:10.359851257Z level=info msg="Plugin registered" pluginID=jdbranham-diagram-panel logger=plugin.loader t=2026-08-06T03:03:10.359855607Z level=info msg="Plugin registered" pluginID=petrslavotinek-carpetplot-panel logger=plugin.loader t=2026-08-06T03:03:10.359858947Z level=info msg="Plugin registered" pluginID=camptocamp-prometheus-alertmanager-datasource logger=plugin.loader t=2026-08-06T03:03:10.359863407Z level=info msg="Plugin registered" pluginID=grafana-worldmap-panel logger=plugin.loader t=2026-08-06T03:03:10.359867667Z level=info msg="Plugin registered" pluginID=natel-discrete-panel logger=plugin.loader t=2026-08-06T03:03:10.359871337Z level=info msg="Plugin registered" pluginID=pmm-app Admin password changed successfully ✔ + cat /tmp/tmp.45RIr6dyNa + rm /tmp/tmp.wWl9DEaiYX /tmp/tmp.45RIr6dyNa + return 0 + desc 'create PXC cluster' + set +o xtrace ----------------------------------------------------------------------------------- create PXC cluster ----------------------------------------------------------------------------------- + spinup_pxc monitoring /mnt/jenkins/workspace/cloud-pxc-operator_PR-2525/e2e-tests/monitoring-2-0/conf/monitoring.yml 3 120 /mnt/jenkins/workspace/cloud-pxc-operator_PR-2525/e2e-tests/monitoring-2-0/conf/secrets.yaml + local cluster=monitoring + local config=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2525/e2e-tests/monitoring-2-0/conf/monitoring.yml + local size=3 + local sleep=120 + local secretsFile=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2525/e2e-tests/monitoring-2-0/conf/secrets.yaml + local pxcClientFile=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2525/e2e-tests/conf/client.yml + local port=3306 + desc 'create first PXC cluster' + set +o xtrace ----------------------------------------------------------------------------------- create first PXC cluster ----------------------------------------------------------------------------------- + kubectl_bin apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2525/e2e-tests/monitoring-2-0/conf/secrets.yaml ++ mktemp + local LAST_OUT=/tmp/tmp.EBF7E3qitp ++ mktemp + local LAST_ERR=/tmp/tmp.8JQdTjTyOE + local exit_status=0 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2525/e2e-tests/monitoring-2-0/conf/secrets.yaml + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.EBF7E3qitp secret/my-cluster-secrets created + cat /tmp/tmp.8JQdTjTyOE + rm /tmp/tmp.EBF7E3qitp /tmp/tmp.8JQdTjTyOE + return 0 + apply_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-2525/e2e-tests/conf/client.yml + local config_file=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2525/e2e-tests/conf/client.yml + local pvc_name= + '[' -z '' ']' + cat_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-2525/e2e-tests/conf/client.yml '' + local input_file=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2525/e2e-tests/conf/client.yml + local pvc_name= + kubectl_bin apply -f - ++ mktemp + local LAST_OUT=/tmp/tmp.4rZAa2jaYg + /usr/bin/sed -e 's#image:.*-init$#image: perconalab/percona-xtradb-cluster-operator:PR-2525-c81e47f0#' + /usr/bin/sed -e 's#image:.*-proxysql$#image: perconalab/percona-xtradb-cluster-operator:main-proxysql#' + /usr/bin/sed -e 's#image:.*-pxc\([0-9]*.[0-9]*\)\{0,1\}$#image: perconalab/percona-xtradb-cluster-operator:main-pxc8.0#' + /usr/bin/sed -e s~minio-service.#namespace~minio-service.monitoring-2-0-9920~ + /usr/bin/sed -e 's#image:.*-logcollector$#image: perconalab/fluentbit:main-logcollector#' + /usr/bin/sed -e 's#apiVersion: pxc.percona.com/v.*$#apiVersion: pxc.percona.com/v1#' ++ mktemp + /usr/bin/sed -e 's#apply:.*#apply: Never#' + /usr/bin/sed -e 's#claimName:..*-backup-pvc$#claimName: #' + cat /mnt/jenkins/workspace/cloud-pxc-operator_PR-2525/e2e-tests/conf/client.yml + local LAST_ERR=/tmp/tmp.MKpEjv8jOf + local exit_status=0 ++ seq 0 2 + /usr/bin/sed -e 's#image:.*\/percona-xtradb-cluster:.*$#image: perconalab/percona-xtradb-cluster-operator:main-pxc8.0#' + /usr/bin/sed -e 's#image:.*-backup$#image: perconalab/percona-xtradb-cluster-operator:main-pxc8.0-backup#' + for i in $(seq 0 2) + set +e + kubectl apply -f - + /usr/bin/sed -e 's#image:.*-pmm$#image: perconalab/pmm-client:dev-latest#' + /usr/bin/sed -e 's#image:.*-haproxy$#image: perconalab/percona-xtradb-cluster-operator:main-haproxy#' + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.4rZAa2jaYg deployment.apps/pxc-client created + cat /tmp/tmp.MKpEjv8jOf + rm /tmp/tmp.4rZAa2jaYg /tmp/tmp.MKpEjv8jOf + return 0 + [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 5\.7 ]] + apply_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-2525/e2e-tests/monitoring-2-0/conf/monitoring.yml + local config_file=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2525/e2e-tests/monitoring-2-0/conf/monitoring.yml + local pvc_name= + '[' -z '' ']' + cat_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-2525/e2e-tests/monitoring-2-0/conf/monitoring.yml '' + local input_file=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2525/e2e-tests/monitoring-2-0/conf/monitoring.yml + local pvc_name= + kubectl_bin apply -f - + cat /mnt/jenkins/workspace/cloud-pxc-operator_PR-2525/e2e-tests/monitoring-2-0/conf/monitoring.yml ++ mktemp + /usr/bin/sed -e 's#image:.*-init$#image: perconalab/percona-xtradb-cluster-operator:PR-2525-c81e47f0#' + local LAST_OUT=/tmp/tmp.68xGkXgBX2 + /usr/bin/sed -e 's#image:.*-pmm$#image: perconalab/pmm-client:dev-latest#' + /usr/bin/sed -e 's#image:.*\/percona-xtradb-cluster:.*$#image: perconalab/percona-xtradb-cluster-operator:main-pxc8.0#' + /usr/bin/sed -e 's#image:.*-backup$#image: perconalab/percona-xtradb-cluster-operator:main-pxc8.0-backup#' + /usr/bin/sed -e s~minio-service.#namespace~minio-service.monitoring-2-0-9920~ + /usr/bin/sed -e 's#image:.*-pxc\([0-9]*.[0-9]*\)\{0,1\}$#image: perconalab/percona-xtradb-cluster-operator:main-pxc8.0#' + /usr/bin/sed -e 's#image:.*-proxysql$#image: perconalab/percona-xtradb-cluster-operator:main-proxysql#' + /usr/bin/sed -e 's#apiVersion: pxc.percona.com/v.*$#apiVersion: pxc.percona.com/v1#' + /usr/bin/sed -e 's#image:.*-haproxy$#image: perconalab/percona-xtradb-cluster-operator:main-haproxy#' + /usr/bin/sed -e 's#image:.*-logcollector$#image: perconalab/fluentbit:main-logcollector#' ++ mktemp + /usr/bin/sed -e 's#apply:.*#apply: Never#' + /usr/bin/sed -e 's#claimName:..*-backup-pvc$#claimName: #' + local LAST_ERR=/tmp/tmp.aLwuTdJpyL + local exit_status=0 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.68xGkXgBX2 perconaxtradbcluster.pxc.percona.com/monitoring created + cat /tmp/tmp.aLwuTdJpyL + rm /tmp/tmp.68xGkXgBX2 /tmp/tmp.aLwuTdJpyL + return 0 + desc 'waiting pods to be running for monitoring' + set +o xtrace ----------------------------------------------------------------------------------- waiting pods to be running for monitoring ----------------------------------------------------------------------------------- ++ get_proxy monitoring ++ local target_cluster=monitoring +++ kubectl_bin get pxc monitoring -o 'jsonpath={.spec.haproxy.enabled}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.uZKYj1k3ea ++++ mktemp +++ local LAST_ERR=/tmp/tmp.rMxDjwTNsi +++ local exit_status=0 ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl get pxc monitoring -o 'jsonpath={.spec.haproxy.enabled}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 ']' +++ break +++ cat /tmp/tmp.uZKYj1k3ea +++ cat /tmp/tmp.rMxDjwTNsi +++ rm /tmp/tmp.uZKYj1k3ea /tmp/tmp.rMxDjwTNsi +++ return 0 ++ [[ true == \t\r\u\e ]] ++ echo monitoring-haproxy ++ return + local proxy=monitoring-haproxy ++ get_proxy_size monitoring ++ local cluster=monitoring ++ local size=0 +++ kubectl_bin get pxc monitoring -o 'jsonpath={.spec.haproxy.enabled}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.VspN90JWlK ++++ mktemp +++ local LAST_ERR=/tmp/tmp.7v49mv1Lng +++ local exit_status=0 ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl get pxc monitoring -o 'jsonpath={.spec.haproxy.enabled}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 ']' +++ break +++ cat /tmp/tmp.VspN90JWlK +++ cat /tmp/tmp.7v49mv1Lng +++ rm /tmp/tmp.VspN90JWlK /tmp/tmp.7v49mv1Lng +++ return 0 ++ [[ true == \t\r\u\e ]] +++ kubectl_bin get pxc monitoring -o 'jsonpath={.spec.haproxy.size}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.5M5Osjstaq ++++ mktemp +++ local LAST_ERR=/tmp/tmp.pI6znkibtc +++ local exit_status=0 ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl get pxc monitoring -o 'jsonpath={.spec.haproxy.size}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 ']' +++ break +++ cat /tmp/tmp.5M5Osjstaq +++ cat /tmp/tmp.pI6znkibtc +++ rm /tmp/tmp.5M5Osjstaq /tmp/tmp.pI6znkibtc +++ return 0 ++ size=2 ++ echo 2 + local proxy_size=2 + wait_for_running monitoring-pxc 3 + local name=monitoring-pxc + local size=3 + local last_pod + local max_retry=480 + [[ -z monitoring-pxc ]] + [[ 3 -le 0 ]] + last_pod=2 + desc 'Waiting for 3 running pods for monitoring-pxc' + set +o xtrace ----------------------------------------------------------------------------------- Waiting for 3 running pods for monitoring-pxc ----------------------------------------------------------------------------------- ++ seq 0 2 + for i in $(seq 0 $last_pod) + wait_pod monitoring-pxc-0 480 + local pod=monitoring-pxc-0 + local max_retry=480 + local ns= ++ grep -E '^(pxc|proxysql)$' ++ echo monitoring-pxc-0 ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' + local container=pxc + set +o xtrace pod/monitoring-pxc-0 condition met waiting for pod/monitoring-pxc-0 to become Ready.Ok + for i in $(seq 0 $last_pod) + wait_pod monitoring-pxc-1 480 + local pod=monitoring-pxc-1 + local max_retry=480 + local ns= ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ echo monitoring-pxc-1 ++ grep -E '^(pxc|proxysql)$' + local container=pxc + set +o xtrace pod/monitoring-pxc-1 condition met waiting for pod/monitoring-pxc-1 to become Ready.Ok + for i in $(seq 0 $last_pod) + wait_pod monitoring-pxc-2 480 + local pod=monitoring-pxc-2 + local max_retry=480 + local ns= ++ echo monitoring-pxc-2 ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ grep -E '^(pxc|proxysql)$' + local container=pxc + set +o xtrace pod/monitoring-pxc-2 condition met waiting for pod/monitoring-pxc-2 to become Ready.Ok + wait_for_running monitoring-haproxy 2 + local name=monitoring-haproxy + local size=2 + local last_pod + local max_retry=480 + [[ -z monitoring-haproxy ]] + [[ 2 -le 0 ]] + last_pod=1 + desc 'Waiting for 2 running pods for monitoring-haproxy' + set +o xtrace ----------------------------------------------------------------------------------- Waiting for 2 running pods for monitoring-haproxy ----------------------------------------------------------------------------------- ++ seq 0 1 + for i in $(seq 0 $last_pod) + wait_pod monitoring-haproxy-0 480 + local pod=monitoring-haproxy-0 + local max_retry=480 + local ns= ++ grep -E '^(pxc|proxysql)$' ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ echo monitoring-haproxy-0 + local container= + set +o xtrace pod/monitoring-haproxy-0 condition met E0806 03:07:03.063554 810983 reflector.go:227] "Failed to watch" err="Get \"https://34.31.71.55/api/v1/namespaces/monitoring-2-0-9920/pods?allowWatchBookmarks=true&fieldSelector=metadata.name%3Dmonitoring-haproxy-0&resourceVersion=1785985621560242000&timeoutSeconds=386&watch=true\": context canceled" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" type="*unstructured.Unstructured" waiting for pod/monitoring-haproxy-0 to become Ready.Ok + for i in $(seq 0 $last_pod) + wait_pod monitoring-haproxy-1 480 + local pod=monitoring-haproxy-1 + local max_retry=480 + local ns= ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ echo monitoring-haproxy-1 ++ grep -E '^(pxc|proxysql)$' + local container= + set +o xtrace pod/monitoring-haproxy-1 condition met E0806 03:07:13.841324 812415 reflector.go:227] "Failed to watch" err="Get \"https://34.31.71.55/api/v1/namespaces/monitoring-2-0-9920/pods?allowWatchBookmarks=true&fieldSelector=metadata.name%3Dmonitoring-haproxy-1&resourceVersion=1785985631560439000&timeoutSeconds=470&watch=true\": context canceled" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" type="*unstructured.Unstructured" waiting for pod/monitoring-haproxy-1 to become Ready.Ok + wait_cluster_consistency monitoring 3 2 + local cluster_name=monitoring + local pxc_size=3 + local proxy_size=2 + desc 'wait cluster consistency' + set +o xtrace ----------------------------------------------------------------------------------- wait cluster consistency ----------------------------------------------------------------------------------- + local max_seconds=1500 + local delay=5 + local retries=300 + local pxc_ready + local proxy_ready + sleep 7 + echo -n 'waiting for pxc/monitoring to be ready' waiting for pxc/monitoring to be ready++ seq 1 300 + for _ in $(seq 1 "${retries}") ++ kubectl_bin get pxc monitoring -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.PnMuhOU9LE +++ mktemp ++ local LAST_ERR=/tmp/tmp.3SICBCprKb ++ local exit_status=0 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pxc monitoring -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.PnMuhOU9LE ++ cat /tmp/tmp.3SICBCprKb ++ rm /tmp/tmp.PnMuhOU9LE /tmp/tmp.3SICBCprKb ++ return 0 + state=ready ++ kubectl_bin get pxc monitoring -o 'jsonpath={.status.pxc.ready}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.qRZkNRoCZj +++ mktemp ++ local LAST_ERR=/tmp/tmp.tzgPQL2oTo ++ local exit_status=0 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pxc monitoring -o 'jsonpath={.status.pxc.ready}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.qRZkNRoCZj ++ cat /tmp/tmp.tzgPQL2oTo ++ rm /tmp/tmp.qRZkNRoCZj /tmp/tmp.tzgPQL2oTo ++ return 0 + pxc_ready=3 + proxy_ready=0 + [[ 2 -gt 0 ]] +++ get_proxy_engine monitoring +++ local cluster_name=monitoring ++++ get_proxy monitoring ++++ local target_cluster=monitoring +++++ kubectl_bin get pxc monitoring -o 'jsonpath={.spec.haproxy.enabled}' ++++++ mktemp +++++ local LAST_OUT=/tmp/tmp.7xBgASGRqt ++++++ mktemp +++++ local LAST_ERR=/tmp/tmp.NTYmjvWEeu +++++ local exit_status=0 ++++++ seq 0 2 +++++ for i in $(seq 0 2) +++++ set +e +++++ kubectl get pxc monitoring -o 'jsonpath={.spec.haproxy.enabled}' +++++ exit_status=0 +++++ set -e +++++ '[' 0 '!=' 0 ']' +++++ break +++++ cat /tmp/tmp.7xBgASGRqt +++++ cat /tmp/tmp.NTYmjvWEeu +++++ rm /tmp/tmp.7xBgASGRqt /tmp/tmp.NTYmjvWEeu +++++ return 0 ++++ [[ true == \t\r\u\e ]] ++++ echo monitoring-haproxy ++++ return +++ local cluster_proxy=monitoring-haproxy +++ echo haproxy ++ kubectl_bin get pxc monitoring -o 'jsonpath={.status.haproxy.ready}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.9ffv6ORCYK +++ mktemp ++ local LAST_ERR=/tmp/tmp.ZCKL8GEfle ++ local exit_status=0 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pxc monitoring -o 'jsonpath={.status.haproxy.ready}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.9ffv6ORCYK ++ cat /tmp/tmp.ZCKL8GEfle ++ rm /tmp/tmp.9ffv6ORCYK /tmp/tmp.ZCKL8GEfle ++ return 0 + proxy_ready=2 + [[ ready == \r\e\a\d\y ]] + [[ 3 == \3 ]] + [[ 2 == \2 ]] + echo + return + sleep 120 ++ kubectl get pxc monitoring -o 'jsonpath={.spec.secretsName}' + local secret_name=my-cluster-secrets ++ getSecretData my-cluster-secrets root ++ local secretName=my-cluster-secrets ++ local dataKey=root ++ base64 --decode ++ kubectl_bin get secrets/my-cluster-secrets '--template={{.data.root}}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.svbiG5RU3T +++ mktemp ++ local LAST_ERR=/tmp/tmp.CnyqsLEVTi ++ local exit_status=0 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get secrets/my-cluster-secrets '--template={{.data.root}}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.svbiG5RU3T ++ cat /tmp/tmp.CnyqsLEVTi ++ rm /tmp/tmp.svbiG5RU3T /tmp/tmp.CnyqsLEVTi ++ return 0 + local root_pass=root_password + desc 'write data' + set +o xtrace ----------------------------------------------------------------------------------- write data ----------------------------------------------------------------------------------- + [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 5\.7 ]] + run_mysql 'CREATE DATABASE IF NOT EXISTS myApp; use myApp; CREATE TABLE IF NOT EXISTS myApp (id int PRIMARY KEY) ;' '-h monitoring-pxc-0.monitoring-pxc -uroot -p'\''root_password'\'' -P3306' + local 'command=CREATE DATABASE IF NOT EXISTS myApp; use myApp; CREATE TABLE IF NOT EXISTS myApp (id int PRIMARY KEY) ;' + local 'uri=-h monitoring-pxc-0.monitoring-pxc -uroot -p'\''root_password'\'' -P3306' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.2rAuxrcMZr +++ mktemp ++ local LAST_ERR=/tmp/tmp.4IuFHdC1wc ++ local exit_status=0 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.2rAuxrcMZr ++ cat /tmp/tmp.4IuFHdC1wc ++ rm /tmp/tmp.2rAuxrcMZr /tmp/tmp.4IuFHdC1wc ++ return 0 + client_pod=pxc-client-67fc4995bb-fmmwr + wait_pod pxc-client-67fc4995bb-fmmwr + local pod=pxc-client-67fc4995bb-fmmwr + local max_retry=480 + local ns= ++ echo pxc-client-67fc4995bb-fmmwr ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ grep -E '^(pxc|proxysql)$' + local container= + set +o xtrace pod/pxc-client-67fc4995bb-fmmwr condition met E0806 03:09:46.705260 831841 reflector.go:227] "Failed to watch" err="Get \"https://34.31.71.55/api/v1/namespaces/monitoring-2-0-9920/pods?allowWatchBookmarks=true&fieldSelector=metadata.name%3Dpxc-client-67fc4995bb-fmmwr&resourceVersion=1785985784462981000&timeoutSeconds=403&watch=true\": context canceled" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" type="*unstructured.Unstructured" waiting for pod/pxc-client-67fc4995bb-fmmwr to become ReadyDefaulted container "pxc-client" out of: pxc-client, backup .Ok + set +o xtrace + write_data monitoring 100500 no-proxy -P3306 + local cluster=monitoring + local id=100500 + local use_proxy=no-proxy + local port=-P3306 + local database=myApp + local table=myApp + local root_pass ++ get_password monitoring root ++ local cluster=monitoring ++ local user=root +++ kubectl_bin get pxc monitoring -o 'jsonpath={.spec.secretsName}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.UclbwwUYwL ++++ mktemp +++ local LAST_ERR=/tmp/tmp.yhvr1USOa5 +++ local exit_status=0 ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl get pxc monitoring -o 'jsonpath={.spec.secretsName}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 ']' +++ break +++ cat /tmp/tmp.UclbwwUYwL +++ cat /tmp/tmp.yhvr1USOa5 +++ rm /tmp/tmp.UclbwwUYwL /tmp/tmp.yhvr1USOa5 +++ return 0 ++ getSecretData my-cluster-secrets root ++ local secretName=my-cluster-secrets ++ local dataKey=root ++ base64 --decode ++ kubectl_bin get secrets/my-cluster-secrets '--template={{.data.root}}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.1285HufJ1Z +++ mktemp ++ local LAST_ERR=/tmp/tmp.OtMNj1OicG ++ local exit_status=0 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get secrets/my-cluster-secrets '--template={{.data.root}}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.1285HufJ1Z ++ cat /tmp/tmp.OtMNj1OicG ++ rm /tmp/tmp.1285HufJ1Z /tmp/tmp.OtMNj1OicG ++ return 0 + root_pass=root_password ++ get_proxy monitoring ++ local target_cluster=monitoring +++ kubectl_bin get pxc monitoring -o 'jsonpath={.spec.haproxy.enabled}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.8AWLT8tf3g ++++ mktemp +++ local LAST_ERR=/tmp/tmp.eL8G8oHP2a +++ local exit_status=0 ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl get pxc monitoring -o 'jsonpath={.spec.haproxy.enabled}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 ']' +++ break +++ cat /tmp/tmp.8AWLT8tf3g +++ cat /tmp/tmp.eL8G8oHP2a +++ rm /tmp/tmp.8AWLT8tf3g /tmp/tmp.eL8G8oHP2a +++ return 0 ++ [[ true == \t\r\u\e ]] ++ echo monitoring-haproxy ++ return + proxy=monitoring-haproxy + [[ no-proxy == true ]] + [[ no-proxy == \u\s\e\-\p\r\o\x\y ]] + host='-h monitoring-pxc-0.monitoring-pxc' + run_mysql 'INSERT IGNORE myApp.myApp (id) VALUES (100500)' '-h monitoring-pxc-0.monitoring-pxc -P3306 -uroot -p'\''root_password'\''' + local 'command=INSERT IGNORE myApp.myApp (id) VALUES (100500)' + local 'uri=-h monitoring-pxc-0.monitoring-pxc -P3306 -uroot -p'\''root_password'\''' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.WRPG9gr1Si +++ mktemp ++ local LAST_ERR=/tmp/tmp.6LKpnwyrRL ++ local exit_status=0 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.WRPG9gr1Si ++ cat /tmp/tmp.6LKpnwyrRL ++ rm /tmp/tmp.WRPG9gr1Si /tmp/tmp.6LKpnwyrRL ++ return 0 + client_pod=pxc-client-67fc4995bb-fmmwr + wait_pod pxc-client-67fc4995bb-fmmwr + local pod=pxc-client-67fc4995bb-fmmwr + local max_retry=480 + local ns= ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ grep -E '^(pxc|proxysql)$' ++ echo pxc-client-67fc4995bb-fmmwr + local container= + set +o xtrace pod/pxc-client-67fc4995bb-fmmwr condition met E0806 03:10:05.029708 834103 reflector.go:227] "Failed to watch" err="Get \"https://34.31.71.55/api/v1/namespaces/monitoring-2-0-9920/pods?allowWatchBookmarks=true&fieldSelector=metadata.name%3Dpxc-client-67fc4995bb-fmmwr&resourceVersion=1785985801704264000&timeoutSeconds=578&watch=true\": context canceled" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" type="*unstructured.Unstructured" waiting for pod/pxc-client-67fc4995bb-fmmwr to become ReadyDefaulted container "pxc-client" out of: pxc-client, backup .Ok + set +o xtrace + sleep 30 + desc 'compare initial data' + set +o xtrace ----------------------------------------------------------------------------------- compare initial data ----------------------------------------------------------------------------------- + compare_data monitoring select-1 no-proxy -P3306 + local cluster=monitoring + local compare=select-1 + local use_proxy=no-proxy + local port=-P3306 + local database=myApp + local table=myApp + local size + local root_pass ++ get_password monitoring root ++ local cluster=monitoring ++ local user=root +++ kubectl_bin get pxc monitoring -o 'jsonpath={.spec.secretsName}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.yAIlYv5rc9 ++++ mktemp +++ local LAST_ERR=/tmp/tmp.jsOjDBFuVO +++ local exit_status=0 ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl get pxc monitoring -o 'jsonpath={.spec.secretsName}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 ']' +++ break +++ cat /tmp/tmp.yAIlYv5rc9 +++ cat /tmp/tmp.jsOjDBFuVO +++ rm /tmp/tmp.yAIlYv5rc9 /tmp/tmp.jsOjDBFuVO +++ return 0 ++ getSecretData my-cluster-secrets root ++ local secretName=my-cluster-secrets ++ local dataKey=root ++ base64 --decode ++ kubectl_bin get secrets/my-cluster-secrets '--template={{.data.root}}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.Nf4JZLpE6c +++ mktemp ++ local LAST_ERR=/tmp/tmp.gS6eeiJcwr ++ local exit_status=0 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get secrets/my-cluster-secrets '--template={{.data.root}}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.Nf4JZLpE6c ++ cat /tmp/tmp.gS6eeiJcwr ++ rm /tmp/tmp.Nf4JZLpE6c /tmp/tmp.gS6eeiJcwr ++ return 0 + root_pass=root_password + [[ no-proxy == true ]] + [[ no-proxy == \u\s\e\-\p\r\o\x\y ]] ++ get_pxc_size monitoring ++ local cluster=monitoring ++ local size=0 +++ kubectl_bin get pxc monitoring -o 'jsonpath={.spec.pxc.size}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.hpCubx4b8w ++++ mktemp +++ local LAST_ERR=/tmp/tmp.YELSu2Vdam +++ local exit_status=0 ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl get pxc monitoring -o 'jsonpath={.spec.pxc.size}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 ']' +++ break +++ cat /tmp/tmp.hpCubx4b8w +++ cat /tmp/tmp.YELSu2Vdam +++ rm /tmp/tmp.hpCubx4b8w /tmp/tmp.YELSu2Vdam +++ return 0 ++ size=3 ++ echo 3 + size=3 ++ seq 0 2 + for i in $(seq 0 $((size - 1))) + compare_mysql_cmd select-1 'SELECT id FROM myApp.myApp ORDER BY id;' '-h monitoring-pxc-0.monitoring-pxc -P3306 -uroot -p'\''root_password'\''' + local command_id=select-1 + local 'command=SELECT id FROM myApp.myApp ORDER BY id;' + local 'uri=-h monitoring-pxc-0.monitoring-pxc -P3306 -uroot -p'\''root_password'\''' + local postfix= + local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2525/e2e-tests/monitoring-2-0/compare/select-1.sql + [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 8\.4 ]] + [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 8\.0 ]] + [[ -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2525/e2e-tests/monitoring-2-0/compare/select-1-80.sql ]] + [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 5\.7 ]] + run_mysql 'SELECT id FROM myApp.myApp ORDER BY id;' '-h monitoring-pxc-0.monitoring-pxc -P3306 -uroot -p'\''root_password'\''' + local 'command=SELECT id FROM myApp.myApp ORDER BY id;' + local 'uri=-h monitoring-pxc-0.monitoring-pxc -P3306 -uroot -p'\''root_password'\''' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.KW7XrrOZCD +++ mktemp ++ local LAST_ERR=/tmp/tmp.K3zhwItmTx ++ local exit_status=0 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.KW7XrrOZCD ++ cat /tmp/tmp.K3zhwItmTx ++ rm /tmp/tmp.KW7XrrOZCD /tmp/tmp.K3zhwItmTx ++ return 0 + client_pod=pxc-client-67fc4995bb-fmmwr + wait_pod pxc-client-67fc4995bb-fmmwr + local pod=pxc-client-67fc4995bb-fmmwr + local max_retry=480 + local ns= ++ echo pxc-client-67fc4995bb-fmmwr ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ grep -E '^(pxc|proxysql)$' + local container= + set +o xtrace pod/pxc-client-67fc4995bb-fmmwr condition met E0806 03:11:03.042363 841740 reflector.go:227] "Failed to watch" err="Get \"https://34.31.71.55/api/v1/namespaces/monitoring-2-0-9920/pods?allowWatchBookmarks=true&fieldSelector=metadata.name%3Dpxc-client-67fc4995bb-fmmwr&resourceVersion=1785985860128457000&timeoutSeconds=328&watch=true\": context canceled" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" type="*unstructured.Unstructured" waiting for pod/pxc-client-67fc4995bb-fmmwr to become ReadyDefaulted container "pxc-client" out of: pxc-client, backup .Ok + set +o xtrace + [[ ! -s /tmp/tmp.gvyFfLErHq/select-1.sql ]] ++ grep 'Unknown MySQL server host' /tmp/tmp.gvyFfLErHq/select-1.sql + [[ -n '' ]] + [[ 0 -eq 0 ]] + diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-2525/e2e-tests/monitoring-2-0/compare/select-1.sql /tmp/tmp.gvyFfLErHq/select-1.sql + for i in $(seq 0 $((size - 1))) + compare_mysql_cmd select-1 'SELECT id FROM myApp.myApp ORDER BY id;' '-h monitoring-pxc-1.monitoring-pxc -P3306 -uroot -p'\''root_password'\''' + local command_id=select-1 + local 'command=SELECT id FROM myApp.myApp ORDER BY id;' + local 'uri=-h monitoring-pxc-1.monitoring-pxc -P3306 -uroot -p'\''root_password'\''' + local postfix= + local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2525/e2e-tests/monitoring-2-0/compare/select-1.sql + [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 8\.4 ]] + [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 8\.0 ]] + [[ -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2525/e2e-tests/monitoring-2-0/compare/select-1-80.sql ]] + [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 5\.7 ]] + run_mysql 'SELECT id FROM myApp.myApp ORDER BY id;' '-h monitoring-pxc-1.monitoring-pxc -P3306 -uroot -p'\''root_password'\''' + local 'command=SELECT id FROM myApp.myApp ORDER BY id;' + local 'uri=-h monitoring-pxc-1.monitoring-pxc -P3306 -uroot -p'\''root_password'\''' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.P1t2bmZDMH +++ mktemp ++ local LAST_ERR=/tmp/tmp.zmrojWor2G ++ local exit_status=0 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.P1t2bmZDMH ++ cat /tmp/tmp.zmrojWor2G ++ rm /tmp/tmp.P1t2bmZDMH /tmp/tmp.zmrojWor2G ++ return 0 + client_pod=pxc-client-67fc4995bb-fmmwr + wait_pod pxc-client-67fc4995bb-fmmwr + local pod=pxc-client-67fc4995bb-fmmwr + local max_retry=480 + local ns= ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ grep -E '^(pxc|proxysql)$' ++ echo pxc-client-67fc4995bb-fmmwr + local container= + set +o xtrace pod/pxc-client-67fc4995bb-fmmwr condition met E0806 03:11:19.223667 844013 reflector.go:227] "Failed to watch" err="Get \"https://34.31.71.55/api/v1/namespaces/monitoring-2-0-9920/pods?allowWatchBookmarks=true&fieldSelector=metadata.name%3Dpxc-client-67fc4995bb-fmmwr&resourceVersion=1785985877458682000&timeoutSeconds=308&watch=true\": context canceled" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" type="*unstructured.Unstructured" waiting for pod/pxc-client-67fc4995bb-fmmwr to become ReadyDefaulted container "pxc-client" out of: pxc-client, backup .Ok + set +o xtrace + [[ ! -s /tmp/tmp.gvyFfLErHq/select-1.sql ]] ++ grep 'Unknown MySQL server host' /tmp/tmp.gvyFfLErHq/select-1.sql + [[ -n '' ]] + [[ 0 -eq 0 ]] + diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-2525/e2e-tests/monitoring-2-0/compare/select-1.sql /tmp/tmp.gvyFfLErHq/select-1.sql + for i in $(seq 0 $((size - 1))) + compare_mysql_cmd select-1 'SELECT id FROM myApp.myApp ORDER BY id;' '-h monitoring-pxc-2.monitoring-pxc -P3306 -uroot -p'\''root_password'\''' + local command_id=select-1 + local 'command=SELECT id FROM myApp.myApp ORDER BY id;' + local 'uri=-h monitoring-pxc-2.monitoring-pxc -P3306 -uroot -p'\''root_password'\''' + local postfix= + local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2525/e2e-tests/monitoring-2-0/compare/select-1.sql + [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 8\.4 ]] + [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 8\.0 ]] + [[ -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2525/e2e-tests/monitoring-2-0/compare/select-1-80.sql ]] + [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 5\.7 ]] + run_mysql 'SELECT id FROM myApp.myApp ORDER BY id;' '-h monitoring-pxc-2.monitoring-pxc -P3306 -uroot -p'\''root_password'\''' + local 'command=SELECT id FROM myApp.myApp ORDER BY id;' + local 'uri=-h monitoring-pxc-2.monitoring-pxc -P3306 -uroot -p'\''root_password'\''' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.0PHzd0VNG8 +++ mktemp ++ local LAST_ERR=/tmp/tmp.WJYdC0tlB5 ++ local exit_status=0 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.0PHzd0VNG8 ++ cat /tmp/tmp.WJYdC0tlB5 ++ rm /tmp/tmp.0PHzd0VNG8 /tmp/tmp.WJYdC0tlB5 ++ return 0 + client_pod=pxc-client-67fc4995bb-fmmwr + wait_pod pxc-client-67fc4995bb-fmmwr + local pod=pxc-client-67fc4995bb-fmmwr + local max_retry=480 + local ns= ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ echo pxc-client-67fc4995bb-fmmwr ++ grep -E '^(pxc|proxysql)$' + local container= + set +o xtrace pod/pxc-client-67fc4995bb-fmmwr condition met E0806 03:11:30.901632 845532 reflector.go:227] "Failed to watch" err="Get \"https://34.31.71.55/api/v1/namespaces/monitoring-2-0-9920/pods?allowWatchBookmarks=true&fieldSelector=metadata.name%3Dpxc-client-67fc4995bb-fmmwr&resourceVersion=1785985888930230000&timeoutSeconds=357&watch=true\": context canceled" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" type="*unstructured.Unstructured" waiting for pod/pxc-client-67fc4995bb-fmmwr to become ReadyDefaulted container "pxc-client" out of: pxc-client, backup .Ok + set +o xtrace + [[ ! -s /tmp/tmp.gvyFfLErHq/select-1.sql ]] ++ grep 'Unknown MySQL server host' /tmp/tmp.gvyFfLErHq/select-1.sql + [[ -n '' ]] + [[ 0 -eq 0 ]] + diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-2525/e2e-tests/monitoring-2-0/compare/select-1.sql /tmp/tmp.gvyFfLErHq/select-1.sql + is_keyring_plugin_in_use monitoring + local cluster=monitoring + [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 8\.4 ]] + grep -E -o 'early-plugin-load=keyring_\w+.so' + kubectl exec monitoring-pxc-0 -c pxc -- bash -c 'cat /etc/mysql/node.cnf' + return 1 + desc 'add PMM API key to secret' + set +o xtrace ----------------------------------------------------------------------------------- add PMM API key to secret ----------------------------------------------------------------------------------- ++ jq .key +++ get_service_endpoint monitoring-service +++ local service=monitoring-service ++++ kubectl_bin get service/monitoring-service -o 'jsonpath={.status.loadBalancer.ingress[].hostname}' +++++ mktemp ++++ local LAST_OUT=/tmp/tmp.FIqHDat0IQ +++++ mktemp ++++ local LAST_ERR=/tmp/tmp.RLA2MIBXtv ++++ local exit_status=0 +++++ seq 0 2 ++++ for i in $(seq 0 2) ++++ set +e ++++ kubectl get service/monitoring-service -o 'jsonpath={.status.loadBalancer.ingress[].hostname}' ++++ exit_status=0 ++++ set -e ++++ '[' 0 '!=' 0 ']' ++++ break ++++ cat /tmp/tmp.FIqHDat0IQ ++++ cat /tmp/tmp.RLA2MIBXtv ++++ rm /tmp/tmp.FIqHDat0IQ /tmp/tmp.RLA2MIBXtv ++++ return 0 +++ local endpoint= +++ '[' -z '' ']' ++++ kubectl_bin get service/monitoring-service -o 'jsonpath={.status.loadBalancer.ingress[].ip}' +++++ mktemp ++++ local LAST_OUT=/tmp/tmp.TOMq2XB0Pc +++++ mktemp ++++ local LAST_ERR=/tmp/tmp.GneX679uSk ++++ local exit_status=0 +++++ seq 0 2 ++++ for i in $(seq 0 2) ++++ set +e ++++ kubectl get service/monitoring-service -o 'jsonpath={.status.loadBalancer.ingress[].ip}' ++++ exit_status=0 ++++ set -e ++++ '[' 0 '!=' 0 ']' ++++ break ++++ cat /tmp/tmp.TOMq2XB0Pc ++++ cat /tmp/tmp.GneX679uSk ++++ rm /tmp/tmp.TOMq2XB0Pc /tmp/tmp.GneX679uSk ++++ return 0 +++ endpoint=34.44.51.100 +++ '[' -n 34.44.51.100 ']' +++ '[' 34.44.51.100 '!=' null ']' +++ head -n 1 +++ echo 34.44.51.100 +++ sed -e 's/^"//; s/"$//;' +++ return ++ curl --insecure -X POST -H 'Content-Type: application/json' -d '{"name":"operator", "role": "Admin"}' https://admin:admin@34.44.51.100/graph/api/auth/keys % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 36 0 0 100 36 0 112 --:--:-- --:--:-- --:--:-- 112 100 155 100 119 100 36 355 107 --:--:-- --:--:-- --:--:-- 462 + API_KEY='"eyJrIjoibHdXWVpkWUZGc1NpM2JHbjU0ZmE5NmFLY3cxN3FGT2kiLCJuIjoib3BlcmF0b3IiLCJpZCI6MX0="' + kubectl_bin patch secret my-cluster-secrets --type merge --patch '{"stringData": {"pmmserverkey": "eyJrIjoibHdXWVpkWUZGc1NpM2JHbjU0ZmE5NmFLY3cxN3FGT2kiLCJuIjoib3BlcmF0b3IiLCJpZCI6MX0="}}' ++ mktemp + local LAST_OUT=/tmp/tmp.QMJqYM7CoK ++ mktemp + local LAST_ERR=/tmp/tmp.rkX0qYNVLR + local exit_status=0 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl patch secret my-cluster-secrets --type merge --patch '{"stringData": {"pmmserverkey": "eyJrIjoibHdXWVpkWUZGc1NpM2JHbjU0ZmE5NmFLY3cxN3FGT2kiLCJuIjoib3BlcmF0b3IiLCJpZCI6MX0="}}' + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.QMJqYM7CoK secret/my-cluster-secrets patched + cat /tmp/tmp.rkX0qYNVLR + rm /tmp/tmp.QMJqYM7CoK /tmp/tmp.rkX0qYNVLR + return 0 + wait_for_generation sts/monitoring-pxc 2 + local resource=sts/monitoring-pxc + local target_generation=2 + echo 'Waiting for sts/monitoring-pxc to reach generation 2...' Waiting for sts/monitoring-pxc to reach generation 2... + true ++ kubectl get sts/monitoring-pxc -o 'jsonpath={.metadata.generation}' + current_generation=1 + '[' 1 -eq 2 ']' + echo 'Resource sts/monitoring-pxc is at generation 1. Waiting...' Resource sts/monitoring-pxc is at generation 1. Waiting... + sleep 5 + true ++ kubectl get sts/monitoring-pxc -o 'jsonpath={.metadata.generation}' + current_generation=2 + '[' 2 -eq 2 ']' + echo 'Resource sts/monitoring-pxc has reached generation 2.' Resource sts/monitoring-pxc has reached generation 2. + break + wait_for_generation sts/monitoring-haproxy 2 + local resource=sts/monitoring-haproxy + local target_generation=2 + echo 'Waiting for sts/monitoring-haproxy to reach generation 2...' Waiting for sts/monitoring-haproxy to reach generation 2... + true ++ kubectl get sts/monitoring-haproxy -o 'jsonpath={.metadata.generation}' + current_generation=2 + '[' 2 -eq 2 ']' + echo 'Resource sts/monitoring-haproxy has reached generation 2.' Resource sts/monitoring-haproxy has reached generation 2. + break + sleep 10 + kubectl wait pod -l app.kubernetes.io/managed-by=percona-xtradb-cluster-operator --for=condition=ready --timeout=600s E0806 03:12:06.185133 850009 reflector.go:227] "Failed to watch" err="Get \"https://34.31.71.55/api/v1/namespaces/monitoring-2-0-9920/pods?allowWatchBookmarks=true&fieldSelector=metadata.name%3Dmonitoring-haproxy-0&resourceVersion=1785985925489153000&timeoutSeconds=510&watch=true\": context canceled" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" type="*unstructured.Unstructured" pod/monitoring-haproxy-0 condition met pod/monitoring-haproxy-1 condition met E0806 03:12:14.964246 850009 reflector.go:227] "Failed to watch" err="Get \"https://34.31.71.55/api/v1/namespaces/monitoring-2-0-9920/pods?allowWatchBookmarks=true&fieldSelector=metadata.name%3Dmonitoring-pxc-0&resourceVersion=1785985934666719013&timeoutSeconds=467&watch=true\": context canceled" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" type="*unstructured.Unstructured" pod/monitoring-pxc-0 condition met E0806 03:12:15.266032 850009 reflector.go:227] "Failed to watch" err="Get \"https://34.31.71.55/api/v1/namespaces/monitoring-2-0-9920/pods?allowWatchBookmarks=true&fieldSelector=metadata.name%3Dmonitoring-pxc-1&resourceVersion=1785985934997679013&timeoutSeconds=322&watch=true\": context canceled" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" type="*unstructured.Unstructured" pod/monitoring-pxc-1 condition met pod/monitoring-pxc-2 condition met + wait_cluster_consistency monitoring 3 2 + local cluster_name=monitoring + local pxc_size=3 + local proxy_size=2 + desc 'wait cluster consistency' + set +o xtrace ----------------------------------------------------------------------------------- wait cluster consistency ----------------------------------------------------------------------------------- + local max_seconds=1500 + local delay=5 + local retries=300 + local pxc_ready + local proxy_ready + sleep 7 + echo -n 'waiting for pxc/monitoring to be ready' waiting for pxc/monitoring to be ready++ seq 1 300 + for _ in $(seq 1 "${retries}") ++ kubectl_bin get pxc monitoring -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.UXuNAwAUy3 +++ mktemp ++ local LAST_ERR=/tmp/tmp.aS0UZSkwKq ++ local exit_status=0 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pxc monitoring -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.UXuNAwAUy3 ++ cat /tmp/tmp.aS0UZSkwKq ++ rm /tmp/tmp.UXuNAwAUy3 /tmp/tmp.aS0UZSkwKq ++ return 0 + state=initializing ++ kubectl_bin get pxc monitoring -o 'jsonpath={.status.pxc.ready}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.y1LIbfyqP9 +++ mktemp ++ local LAST_ERR=/tmp/tmp.Hgzqr2cySD ++ local exit_status=0 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pxc monitoring -o 'jsonpath={.status.pxc.ready}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.y1LIbfyqP9 ++ cat /tmp/tmp.Hgzqr2cySD ++ rm /tmp/tmp.y1LIbfyqP9 /tmp/tmp.Hgzqr2cySD ++ return 0 + pxc_ready=1 + proxy_ready=0 + [[ 2 -gt 0 ]] +++ get_proxy_engine monitoring +++ local cluster_name=monitoring ++++ get_proxy monitoring ++++ local target_cluster=monitoring +++++ kubectl_bin get pxc monitoring -o 'jsonpath={.spec.haproxy.enabled}' ++++++ mktemp +++++ local LAST_OUT=/tmp/tmp.fzrEhRW3cS ++++++ mktemp +++++ local LAST_ERR=/tmp/tmp.35KsYFl3hv +++++ local exit_status=0 ++++++ seq 0 2 +++++ for i in $(seq 0 2) +++++ set +e +++++ kubectl get pxc monitoring -o 'jsonpath={.spec.haproxy.enabled}' +++++ exit_status=0 +++++ set -e +++++ '[' 0 '!=' 0 ']' +++++ break +++++ cat /tmp/tmp.fzrEhRW3cS +++++ cat /tmp/tmp.35KsYFl3hv +++++ rm /tmp/tmp.fzrEhRW3cS /tmp/tmp.35KsYFl3hv +++++ return 0 ++++ [[ true == \t\r\u\e ]] ++++ echo monitoring-haproxy ++++ return +++ local cluster_proxy=monitoring-haproxy +++ echo haproxy ++ kubectl_bin get pxc monitoring -o 'jsonpath={.status.haproxy.ready}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.apGCsxNYI7 +++ mktemp ++ local LAST_ERR=/tmp/tmp.ewAJIjTYkv ++ local exit_status=0 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pxc monitoring -o 'jsonpath={.status.haproxy.ready}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.apGCsxNYI7 ++ cat /tmp/tmp.ewAJIjTYkv ++ rm /tmp/tmp.apGCsxNYI7 /tmp/tmp.ewAJIjTYkv ++ return 0 + proxy_ready=2 + [[ initializing == \r\e\a\d\y ]] + echo -n . .+ sleep 5 + for _ in $(seq 1 "${retries}") ++ kubectl_bin get pxc monitoring -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.fUThRkv2lS +++ mktemp ++ local LAST_ERR=/tmp/tmp.1CojV0A91v ++ local exit_status=0 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pxc monitoring -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.fUThRkv2lS ++ cat /tmp/tmp.1CojV0A91v ++ rm /tmp/tmp.fUThRkv2lS /tmp/tmp.1CojV0A91v ++ return 0 + state=initializing ++ kubectl_bin get pxc monitoring -o 'jsonpath={.status.pxc.ready}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.CpFIUabILS +++ mktemp ++ local LAST_ERR=/tmp/tmp.ssA272wes8 ++ local exit_status=0 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pxc monitoring -o 'jsonpath={.status.pxc.ready}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.CpFIUabILS ++ cat /tmp/tmp.ssA272wes8 ++ rm /tmp/tmp.CpFIUabILS /tmp/tmp.ssA272wes8 ++ return 0 + pxc_ready=1 + proxy_ready=0 + [[ 2 -gt 0 ]] +++ get_proxy_engine monitoring +++ local cluster_name=monitoring ++++ get_proxy monitoring ++++ local target_cluster=monitoring +++++ kubectl_bin get pxc monitoring -o 'jsonpath={.spec.haproxy.enabled}' ++++++ mktemp +++++ local LAST_OUT=/tmp/tmp.lrF927mf3H ++++++ mktemp +++++ local LAST_ERR=/tmp/tmp.DZr57RSW8d +++++ local exit_status=0 ++++++ seq 0 2 +++++ for i in $(seq 0 2) +++++ set +e +++++ kubectl get pxc monitoring -o 'jsonpath={.spec.haproxy.enabled}' +++++ exit_status=0 +++++ set -e +++++ '[' 0 '!=' 0 ']' +++++ break +++++ cat /tmp/tmp.lrF927mf3H +++++ cat /tmp/tmp.DZr57RSW8d +++++ rm /tmp/tmp.lrF927mf3H /tmp/tmp.DZr57RSW8d +++++ return 0 ++++ [[ true == \t\r\u\e ]] ++++ echo monitoring-haproxy ++++ return +++ local cluster_proxy=monitoring-haproxy +++ echo haproxy ++ kubectl_bin get pxc monitoring -o 'jsonpath={.status.haproxy.ready}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.IJKCndng9I +++ mktemp ++ local LAST_ERR=/tmp/tmp.bfVAyCpG6M ++ local exit_status=0 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pxc monitoring -o 'jsonpath={.status.haproxy.ready}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.IJKCndng9I ++ cat /tmp/tmp.bfVAyCpG6M ++ rm /tmp/tmp.IJKCndng9I /tmp/tmp.bfVAyCpG6M ++ return 0 + proxy_ready=2 + [[ initializing == \r\e\a\d\y ]] + echo -n . .+ sleep 5 + for _ in $(seq 1 "${retries}") ++ kubectl_bin get pxc monitoring -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.n8ahA8IaSA +++ mktemp ++ local LAST_ERR=/tmp/tmp.32s6tqy61g ++ local exit_status=0 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pxc monitoring -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.n8ahA8IaSA ++ cat /tmp/tmp.32s6tqy61g ++ rm /tmp/tmp.n8ahA8IaSA /tmp/tmp.32s6tqy61g ++ return 0 + state=initializing ++ kubectl_bin get pxc monitoring -o 'jsonpath={.status.pxc.ready}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.6HQgxodqz8 +++ mktemp ++ local LAST_ERR=/tmp/tmp.zga9ZYQ7qb ++ local exit_status=0 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pxc monitoring -o 'jsonpath={.status.pxc.ready}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.6HQgxodqz8 ++ cat /tmp/tmp.zga9ZYQ7qb ++ rm /tmp/tmp.6HQgxodqz8 /tmp/tmp.zga9ZYQ7qb ++ return 0 + pxc_ready=1 + proxy_ready=0 + [[ 2 -gt 0 ]] +++ get_proxy_engine monitoring +++ local cluster_name=monitoring ++++ get_proxy monitoring ++++ local target_cluster=monitoring +++++ kubectl_bin get pxc monitoring -o 'jsonpath={.spec.haproxy.enabled}' ++++++ mktemp +++++ local LAST_OUT=/tmp/tmp.j7UZVTAsNl ++++++ mktemp +++++ local LAST_ERR=/tmp/tmp.g0XikC34PA +++++ local exit_status=0 ++++++ seq 0 2 +++++ for i in $(seq 0 2) +++++ set +e +++++ kubectl get pxc monitoring -o 'jsonpath={.spec.haproxy.enabled}' +++++ exit_status=0 +++++ set -e +++++ '[' 0 '!=' 0 ']' +++++ break +++++ cat /tmp/tmp.j7UZVTAsNl +++++ cat /tmp/tmp.g0XikC34PA +++++ rm /tmp/tmp.j7UZVTAsNl /tmp/tmp.g0XikC34PA +++++ return 0 ++++ [[ true == \t\r\u\e ]] ++++ echo monitoring-haproxy ++++ return +++ local cluster_proxy=monitoring-haproxy +++ echo haproxy ++ kubectl_bin get pxc monitoring -o 'jsonpath={.status.haproxy.ready}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.sLNJyJkk7F +++ mktemp ++ local LAST_ERR=/tmp/tmp.22rTKArx4z ++ local exit_status=0 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pxc monitoring -o 'jsonpath={.status.haproxy.ready}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.sLNJyJkk7F ++ cat /tmp/tmp.22rTKArx4z ++ rm /tmp/tmp.sLNJyJkk7F /tmp/tmp.22rTKArx4z ++ return 0 + proxy_ready=2 + [[ initializing == \r\e\a\d\y ]] + echo -n . .+ sleep 5 + for _ in $(seq 1 "${retries}") ++ kubectl_bin get pxc monitoring -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.xpdFyLOAEW +++ mktemp ++ local LAST_ERR=/tmp/tmp.N6RNHvO7QK ++ local exit_status=0 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pxc monitoring -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.xpdFyLOAEW ++ cat /tmp/tmp.N6RNHvO7QK ++ rm /tmp/tmp.xpdFyLOAEW /tmp/tmp.N6RNHvO7QK ++ return 0 + state=initializing ++ kubectl_bin get pxc monitoring -o 'jsonpath={.status.pxc.ready}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.Eabkm5Pu9D +++ mktemp ++ local LAST_ERR=/tmp/tmp.402tvrlUCM ++ local exit_status=0 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pxc monitoring -o 'jsonpath={.status.pxc.ready}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.Eabkm5Pu9D ++ cat /tmp/tmp.402tvrlUCM ++ rm /tmp/tmp.Eabkm5Pu9D /tmp/tmp.402tvrlUCM ++ return 0 + pxc_ready=1 + proxy_ready=0 + [[ 2 -gt 0 ]] +++ get_proxy_engine monitoring +++ local cluster_name=monitoring ++++ get_proxy monitoring ++++ local target_cluster=monitoring +++++ kubectl_bin get pxc monitoring -o 'jsonpath={.spec.haproxy.enabled}' ++++++ mktemp +++++ local LAST_OUT=/tmp/tmp.mjT93Sxqoi ++++++ mktemp +++++ local LAST_ERR=/tmp/tmp.4H3YCGhEcw +++++ local exit_status=0 ++++++ seq 0 2 +++++ for i in $(seq 0 2) +++++ set +e +++++ kubectl get pxc monitoring -o 'jsonpath={.spec.haproxy.enabled}' +++++ exit_status=0 +++++ set -e +++++ '[' 0 '!=' 0 ']' +++++ break +++++ cat /tmp/tmp.mjT93Sxqoi +++++ cat /tmp/tmp.4H3YCGhEcw +++++ rm /tmp/tmp.mjT93Sxqoi /tmp/tmp.4H3YCGhEcw +++++ return 0 ++++ [[ true == \t\r\u\e ]] ++++ echo monitoring-haproxy ++++ return +++ local cluster_proxy=monitoring-haproxy +++ echo haproxy ++ kubectl_bin get pxc monitoring -o 'jsonpath={.status.haproxy.ready}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.fyLFY3ktWA +++ mktemp ++ local LAST_ERR=/tmp/tmp.wkVQEyn2uZ ++ local exit_status=0 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pxc monitoring -o 'jsonpath={.status.haproxy.ready}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.fyLFY3ktWA ++ cat /tmp/tmp.wkVQEyn2uZ ++ rm /tmp/tmp.fyLFY3ktWA /tmp/tmp.wkVQEyn2uZ ++ return 0 + proxy_ready=2 + [[ initializing == \r\e\a\d\y ]] + echo -n . .+ sleep 5 + for _ in $(seq 1 "${retries}") ++ kubectl_bin get pxc monitoring -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.xuJ8bH8skw +++ mktemp ++ local LAST_ERR=/tmp/tmp.JLVhIU7VS4 ++ local exit_status=0 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pxc monitoring -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.xuJ8bH8skw ++ cat /tmp/tmp.JLVhIU7VS4 ++ rm /tmp/tmp.xuJ8bH8skw /tmp/tmp.JLVhIU7VS4 ++ return 0 + state=initializing ++ kubectl_bin get pxc monitoring -o 'jsonpath={.status.pxc.ready}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.XgxK5KtimC +++ mktemp ++ local LAST_ERR=/tmp/tmp.ogEBikcYNW ++ local exit_status=0 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pxc monitoring -o 'jsonpath={.status.pxc.ready}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.XgxK5KtimC ++ cat /tmp/tmp.ogEBikcYNW ++ rm /tmp/tmp.XgxK5KtimC /tmp/tmp.ogEBikcYNW ++ return 0 + pxc_ready=2 + proxy_ready=0 + [[ 2 -gt 0 ]] +++ get_proxy_engine monitoring +++ local cluster_name=monitoring ++++ get_proxy monitoring ++++ local target_cluster=monitoring +++++ kubectl_bin get pxc monitoring -o 'jsonpath={.spec.haproxy.enabled}' ++++++ mktemp +++++ local LAST_OUT=/tmp/tmp.kvEPmTBLKW ++++++ mktemp +++++ local LAST_ERR=/tmp/tmp.9AGeINRLNt +++++ local exit_status=0 ++++++ seq 0 2 +++++ for i in $(seq 0 2) +++++ set +e +++++ kubectl get pxc monitoring -o 'jsonpath={.spec.haproxy.enabled}' +++++ exit_status=0 +++++ set -e +++++ '[' 0 '!=' 0 ']' +++++ break +++++ cat /tmp/tmp.kvEPmTBLKW +++++ cat /tmp/tmp.9AGeINRLNt +++++ rm /tmp/tmp.kvEPmTBLKW /tmp/tmp.9AGeINRLNt +++++ return 0 ++++ [[ true == \t\r\u\e ]] ++++ echo monitoring-haproxy ++++ return +++ local cluster_proxy=monitoring-haproxy +++ echo haproxy ++ kubectl_bin get pxc monitoring -o 'jsonpath={.status.haproxy.ready}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.rTitRCNwQA +++ mktemp ++ local LAST_ERR=/tmp/tmp.fToDbfHIvX ++ local exit_status=0 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pxc monitoring -o 'jsonpath={.status.haproxy.ready}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.rTitRCNwQA ++ cat /tmp/tmp.fToDbfHIvX ++ rm /tmp/tmp.rTitRCNwQA /tmp/tmp.fToDbfHIvX ++ return 0 + proxy_ready=2 + [[ initializing == \r\e\a\d\y ]] + echo -n . .+ sleep 5 + for _ in $(seq 1 "${retries}") ++ kubectl_bin get pxc monitoring -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.P68aSAMIDy +++ mktemp ++ local LAST_ERR=/tmp/tmp.gy4TxGPe6p ++ local exit_status=0 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pxc monitoring -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.P68aSAMIDy ++ cat /tmp/tmp.gy4TxGPe6p ++ rm /tmp/tmp.P68aSAMIDy /tmp/tmp.gy4TxGPe6p ++ return 0 + state=initializing ++ kubectl_bin get pxc monitoring -o 'jsonpath={.status.pxc.ready}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.z0IkCfcTbK +++ mktemp ++ local LAST_ERR=/tmp/tmp.w9sXeAObwt ++ local exit_status=0 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pxc monitoring -o 'jsonpath={.status.pxc.ready}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.z0IkCfcTbK ++ cat /tmp/tmp.w9sXeAObwt ++ rm /tmp/tmp.z0IkCfcTbK /tmp/tmp.w9sXeAObwt ++ return 0 + pxc_ready=2 + proxy_ready=0 + [[ 2 -gt 0 ]] +++ get_proxy_engine monitoring +++ local cluster_name=monitoring ++++ get_proxy monitoring ++++ local target_cluster=monitoring +++++ kubectl_bin get pxc monitoring -o 'jsonpath={.spec.haproxy.enabled}' ++++++ mktemp +++++ local LAST_OUT=/tmp/tmp.Q8O4DGINr7 ++++++ mktemp +++++ local LAST_ERR=/tmp/tmp.mw7rbeQAYT +++++ local exit_status=0 ++++++ seq 0 2 +++++ for i in $(seq 0 2) +++++ set +e +++++ kubectl get pxc monitoring -o 'jsonpath={.spec.haproxy.enabled}' +++++ exit_status=0 +++++ set -e +++++ '[' 0 '!=' 0 ']' +++++ break +++++ cat /tmp/tmp.Q8O4DGINr7 +++++ cat /tmp/tmp.mw7rbeQAYT +++++ rm /tmp/tmp.Q8O4DGINr7 /tmp/tmp.mw7rbeQAYT +++++ return 0 ++++ [[ true == \t\r\u\e ]] ++++ echo monitoring-haproxy ++++ return +++ local cluster_proxy=monitoring-haproxy +++ echo haproxy ++ kubectl_bin get pxc monitoring -o 'jsonpath={.status.haproxy.ready}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.5HJ6p1CMvE +++ mktemp ++ local LAST_ERR=/tmp/tmp.O6NshyIQvr ++ local exit_status=0 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pxc monitoring -o 'jsonpath={.status.haproxy.ready}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.5HJ6p1CMvE ++ cat /tmp/tmp.O6NshyIQvr ++ rm /tmp/tmp.5HJ6p1CMvE /tmp/tmp.O6NshyIQvr ++ return 0 + proxy_ready=2 + [[ initializing == \r\e\a\d\y ]] + echo -n . .+ sleep 5 + for _ in $(seq 1 "${retries}") ++ kubectl_bin get pxc monitoring -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.MqKnNQh8XX +++ mktemp ++ local LAST_ERR=/tmp/tmp.eu9rVPZNKD ++ local exit_status=0 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pxc monitoring -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.MqKnNQh8XX ++ cat /tmp/tmp.eu9rVPZNKD ++ rm /tmp/tmp.MqKnNQh8XX /tmp/tmp.eu9rVPZNKD ++ return 0 + state=initializing ++ kubectl_bin get pxc monitoring -o 'jsonpath={.status.pxc.ready}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.rF3azVvspV +++ mktemp ++ local LAST_ERR=/tmp/tmp.baRw29Gub0 ++ local exit_status=0 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pxc monitoring -o 'jsonpath={.status.pxc.ready}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.rF3azVvspV ++ cat /tmp/tmp.baRw29Gub0 ++ rm /tmp/tmp.rF3azVvspV /tmp/tmp.baRw29Gub0 ++ return 0 + pxc_ready=2 + proxy_ready=0 + [[ 2 -gt 0 ]] +++ get_proxy_engine monitoring +++ local cluster_name=monitoring ++++ get_proxy monitoring ++++ local target_cluster=monitoring +++++ kubectl_bin get pxc monitoring -o 'jsonpath={.spec.haproxy.enabled}' ++++++ mktemp +++++ local LAST_OUT=/tmp/tmp.MwJ9DVXNZ7 ++++++ mktemp +++++ local LAST_ERR=/tmp/tmp.059QptDl3B +++++ local exit_status=0 ++++++ seq 0 2 +++++ for i in $(seq 0 2) +++++ set +e +++++ kubectl get pxc monitoring -o 'jsonpath={.spec.haproxy.enabled}' +++++ exit_status=0 +++++ set -e +++++ '[' 0 '!=' 0 ']' +++++ break +++++ cat /tmp/tmp.MwJ9DVXNZ7 +++++ cat /tmp/tmp.059QptDl3B +++++ rm /tmp/tmp.MwJ9DVXNZ7 /tmp/tmp.059QptDl3B +++++ return 0 ++++ [[ true == \t\r\u\e ]] ++++ echo monitoring-haproxy ++++ return +++ local cluster_proxy=monitoring-haproxy +++ echo haproxy ++ kubectl_bin get pxc monitoring -o 'jsonpath={.status.haproxy.ready}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.BtQxZNDeMw +++ mktemp ++ local LAST_ERR=/tmp/tmp.Ba5CXAhvey ++ local exit_status=0 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pxc monitoring -o 'jsonpath={.status.haproxy.ready}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.BtQxZNDeMw ++ cat /tmp/tmp.Ba5CXAhvey ++ rm /tmp/tmp.BtQxZNDeMw /tmp/tmp.Ba5CXAhvey ++ return 0 + proxy_ready=2 + [[ initializing == \r\e\a\d\y ]] + echo -n . .+ sleep 5 + for _ in $(seq 1 "${retries}") ++ kubectl_bin get pxc monitoring -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.rZu1hTGxWz +++ mktemp ++ local LAST_ERR=/tmp/tmp.itPVzs2a79 ++ local exit_status=0 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pxc monitoring -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.rZu1hTGxWz ++ cat /tmp/tmp.itPVzs2a79 ++ rm /tmp/tmp.rZu1hTGxWz /tmp/tmp.itPVzs2a79 ++ return 0 + state=initializing ++ kubectl_bin get pxc monitoring -o 'jsonpath={.status.pxc.ready}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.SdG6Iq1rBM +++ mktemp ++ local LAST_ERR=/tmp/tmp.BJo9zk8e8w ++ local exit_status=0 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pxc monitoring -o 'jsonpath={.status.pxc.ready}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.SdG6Iq1rBM ++ cat /tmp/tmp.BJo9zk8e8w ++ rm /tmp/tmp.SdG6Iq1rBM /tmp/tmp.BJo9zk8e8w ++ return 0 + pxc_ready=2 + proxy_ready=0 + [[ 2 -gt 0 ]] +++ get_proxy_engine monitoring +++ local cluster_name=monitoring ++++ get_proxy monitoring ++++ local target_cluster=monitoring +++++ kubectl_bin get pxc monitoring -o 'jsonpath={.spec.haproxy.enabled}' ++++++ mktemp +++++ local LAST_OUT=/tmp/tmp.RKxFioQCDy ++++++ mktemp +++++ local LAST_ERR=/tmp/tmp.sLHGTS7mYL +++++ local exit_status=0 ++++++ seq 0 2 +++++ for i in $(seq 0 2) +++++ set +e +++++ kubectl get pxc monitoring -o 'jsonpath={.spec.haproxy.enabled}' +++++ exit_status=0 +++++ set -e +++++ '[' 0 '!=' 0 ']' +++++ break +++++ cat /tmp/tmp.RKxFioQCDy +++++ cat /tmp/tmp.sLHGTS7mYL +++++ rm /tmp/tmp.RKxFioQCDy /tmp/tmp.sLHGTS7mYL +++++ return 0 ++++ [[ true == \t\r\u\e ]] ++++ echo monitoring-haproxy ++++ return +++ local cluster_proxy=monitoring-haproxy +++ echo haproxy ++ kubectl_bin get pxc monitoring -o 'jsonpath={.status.haproxy.ready}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.73jIM42CzC +++ mktemp ++ local LAST_ERR=/tmp/tmp.GSaxGV1rtT ++ local exit_status=0 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pxc monitoring -o 'jsonpath={.status.haproxy.ready}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.73jIM42CzC ++ cat /tmp/tmp.GSaxGV1rtT ++ rm /tmp/tmp.73jIM42CzC /tmp/tmp.GSaxGV1rtT ++ return 0 + proxy_ready=2 + [[ initializing == \r\e\a\d\y ]] + echo -n . .+ sleep 5 + for _ in $(seq 1 "${retries}") ++ kubectl_bin get pxc monitoring -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.YAFqmLfCte +++ mktemp ++ local LAST_ERR=/tmp/tmp.oTnH0vz2Xa ++ local exit_status=0 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pxc monitoring -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.YAFqmLfCte ++ cat /tmp/tmp.oTnH0vz2Xa ++ rm /tmp/tmp.YAFqmLfCte /tmp/tmp.oTnH0vz2Xa ++ return 0 + state=initializing ++ kubectl_bin get pxc monitoring -o 'jsonpath={.status.pxc.ready}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.7Pc91X0uCb +++ mktemp ++ local LAST_ERR=/tmp/tmp.R4j3jWmhS7 ++ local exit_status=0 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pxc monitoring -o 'jsonpath={.status.pxc.ready}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.7Pc91X0uCb ++ cat /tmp/tmp.R4j3jWmhS7 ++ rm /tmp/tmp.7Pc91X0uCb /tmp/tmp.R4j3jWmhS7 ++ return 0 + pxc_ready=2 + proxy_ready=0 + [[ 2 -gt 0 ]] +++ get_proxy_engine monitoring +++ local cluster_name=monitoring ++++ get_proxy monitoring ++++ local target_cluster=monitoring +++++ kubectl_bin get pxc monitoring -o 'jsonpath={.spec.haproxy.enabled}' ++++++ mktemp +++++ local LAST_OUT=/tmp/tmp.RtoXYsMQeF ++++++ mktemp +++++ local LAST_ERR=/tmp/tmp.aqil4SadBg +++++ local exit_status=0 ++++++ seq 0 2 +++++ for i in $(seq 0 2) +++++ set +e +++++ kubectl get pxc monitoring -o 'jsonpath={.spec.haproxy.enabled}' +++++ exit_status=0 +++++ set -e +++++ '[' 0 '!=' 0 ']' +++++ break +++++ cat /tmp/tmp.RtoXYsMQeF +++++ cat /tmp/tmp.aqil4SadBg +++++ rm /tmp/tmp.RtoXYsMQeF /tmp/tmp.aqil4SadBg +++++ return 0 ++++ [[ true == \t\r\u\e ]] ++++ echo monitoring-haproxy ++++ return +++ local cluster_proxy=monitoring-haproxy +++ echo haproxy ++ kubectl_bin get pxc monitoring -o 'jsonpath={.status.haproxy.ready}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.TqH6sp1xKx +++ mktemp ++ local LAST_ERR=/tmp/tmp.6oL3KB33dG ++ local exit_status=0 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pxc monitoring -o 'jsonpath={.status.haproxy.ready}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.TqH6sp1xKx ++ cat /tmp/tmp.6oL3KB33dG ++ rm /tmp/tmp.TqH6sp1xKx /tmp/tmp.6oL3KB33dG ++ return 0 + proxy_ready=2 + [[ initializing == \r\e\a\d\y ]] + echo -n . .+ sleep 5 + for _ in $(seq 1 "${retries}") ++ kubectl_bin get pxc monitoring -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.LkJeU6LHmi +++ mktemp ++ local LAST_ERR=/tmp/tmp.EfqkRcMn5G ++ local exit_status=0 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pxc monitoring -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.LkJeU6LHmi ++ cat /tmp/tmp.EfqkRcMn5G ++ rm /tmp/tmp.LkJeU6LHmi /tmp/tmp.EfqkRcMn5G ++ return 0 + state=ready ++ kubectl_bin get pxc monitoring -o 'jsonpath={.status.pxc.ready}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.Wn9hpZeoRi +++ mktemp ++ local LAST_ERR=/tmp/tmp.hQq6pBm0aB ++ local exit_status=0 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pxc monitoring -o 'jsonpath={.status.pxc.ready}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.Wn9hpZeoRi ++ cat /tmp/tmp.hQq6pBm0aB ++ rm /tmp/tmp.Wn9hpZeoRi /tmp/tmp.hQq6pBm0aB ++ return 0 + pxc_ready=3 + proxy_ready=0 + [[ 2 -gt 0 ]] +++ get_proxy_engine monitoring +++ local cluster_name=monitoring ++++ get_proxy monitoring ++++ local target_cluster=monitoring +++++ kubectl_bin get pxc monitoring -o 'jsonpath={.spec.haproxy.enabled}' ++++++ mktemp +++++ local LAST_OUT=/tmp/tmp.kJmDXPELVe ++++++ mktemp +++++ local LAST_ERR=/tmp/tmp.e3ROCcIwhh +++++ local exit_status=0 ++++++ seq 0 2 +++++ for i in $(seq 0 2) +++++ set +e +++++ kubectl get pxc monitoring -o 'jsonpath={.spec.haproxy.enabled}' +++++ exit_status=0 +++++ set -e +++++ '[' 0 '!=' 0 ']' +++++ break +++++ cat /tmp/tmp.kJmDXPELVe +++++ cat /tmp/tmp.e3ROCcIwhh +++++ rm /tmp/tmp.kJmDXPELVe /tmp/tmp.e3ROCcIwhh +++++ return 0 ++++ [[ true == \t\r\u\e ]] ++++ echo monitoring-haproxy ++++ return +++ local cluster_proxy=monitoring-haproxy +++ echo haproxy ++ kubectl_bin get pxc monitoring -o 'jsonpath={.status.haproxy.ready}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.IQe6pmDj7k +++ mktemp ++ local LAST_ERR=/tmp/tmp.fFN5XdxcOJ ++ local exit_status=0 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pxc monitoring -o 'jsonpath={.status.haproxy.ready}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.IQe6pmDj7k ++ cat /tmp/tmp.fFN5XdxcOJ ++ rm /tmp/tmp.IQe6pmDj7k /tmp/tmp.fFN5XdxcOJ ++ return 0 + proxy_ready=2 + [[ ready == \r\e\a\d\y ]] + [[ 3 == \3 ]] + [[ 2 == \2 ]] + echo + return + compare_kubectl statefulset/monitoring-pxc -no-prefix + local resource=statefulset/monitoring-pxc + local postfix=-no-prefix + local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2525/e2e-tests/monitoring-2-0/compare/statefulset_monitoring-pxc-no-prefix.yml + local new_result=/tmp/tmp.gvyFfLErHq/statefulset_monitoring-pxc.yml + desc 'compare statefulset/monitoring-pxc--no-prefix' + set +o xtrace ----------------------------------------------------------------------------------- compare statefulset/monitoring-pxc--no-prefix ----------------------------------------------------------------------------------- + '[' 0 = 1 -a -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2525/e2e-tests/monitoring-2-0/compare/statefulset_monitoring-pxc-no-prefix-eks.yml ']' + [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 8\.0 ]] + '[' -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2525/e2e-tests/monitoring-2-0/compare/statefulset_monitoring-pxc-no-prefix-80.yml ']' + [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 8\.4 ]] + version_gt 1.33 ++ bc -l ++ echo '1.33 >= 1.33' + '[' 1 -eq 1 ']' + return 0 + '[' -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2525/e2e-tests/monitoring-2-0/compare/statefulset_monitoring-pxc-no-prefix-k133.yml ']' + version_gt 1.29 ++ echo '1.33 >= 1.29' ++ bc -l + '[' 1 -eq 1 ']' + return 0 + '[' -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2525/e2e-tests/monitoring-2-0/compare/statefulset_monitoring-pxc-no-prefix-k129.yml ']' + version_gt 1.27 ++ bc -l ++ echo '1.33 >= 1.27' + '[' 1 -eq 1 ']' + return 0 + '[' -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2525/e2e-tests/monitoring-2-0/compare/statefulset_monitoring-pxc-no-prefix-k127.yml ']' + expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2525/e2e-tests/monitoring-2-0/compare/statefulset_monitoring-pxc-no-prefix-k127.yml + '[' '!' -z '' -a -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2525/e2e-tests/monitoring-2-0/compare/statefulset_monitoring-pxc-no-prefix-k127-oc.yml ']' + version_gt 1.29 ++ echo '1.33 >= 1.29' ++ bc -l + '[' 1 -eq 1 ']' + return 0 + '[' '!' -z '' -a -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2525/e2e-tests/monitoring-2-0/compare/statefulset_monitoring-pxc-no-prefix-k127-k129-oc.yml ']' + '[' 0 = 1 -a -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2525/e2e-tests/monitoring-2-0/compare/statefulset_monitoring-pxc-no-prefix-k127-eks.yml ']' + '[' 0 = 1 -a -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2525/e2e-tests/monitoring-2-0/compare/statefulset_monitoring-pxc-no-prefix-k127-aks.yml ']' + yq eval ' del(.metadata.managedFields) | del(.. | select(has("creationTimestamp")).creationTimestamp) | del(.. | select(has("namespace")).namespace) | del(.. | select(has("uid")).uid) | del(.metadata.resourceVersion) | del(.spec.template.spec.containers[].env[] | select(.name == "CLUSTER_HASH")) | del(.spec.template.spec.containers[].env[] | select(.name == "S3_BUCKET_PATH")) | del(.spec.template.spec.containers[].env[] | select(.name == "BACKUP_PATH")) | del(.spec.template.spec.containers[].env[] | select(.name == "S3_BUCKET_URL")) | del(.spec.template.spec.containers[].env[] | select(.name == "AZURE_CONTAINER_NAME")) | del(.spec.template.spec.containers[].env[] | select(.name == "XTRABACKUP_ENABLED")) | del(.metadata.selfLink) | del(.metadata.deletionTimestamp) | del(.metadata.labels."batch.kubernetes.io/job-name") | del(.metadata.labels."job-name") | del(.metadata.labels."topology.kubernetes.io/region") | del(.metadata.labels."topology.kubernetes.io/zone") | del(.metadata.annotations."kubectl.kubernetes.io/last-applied-configuration") | del(.metadata.annotations."kubernetes.io/psp") | del(.metadata.annotations."batch.kubernetes.io/job-tracking") | del(.metadata.annotations."networking.gke.io/target-pool") | del(.metadata.annotations."cloud.google.com/neg") | del(.metadata.annotations."k8s.v1.cni.cncf.io*") | del(.metadata.annotations."k8s.ovn.org/pod-networks") | del(.metadata.annotations."kubernetes.digitalocean.com/load-balancer-id") | del(.metadata.annotations."service.beta.kubernetes.io/do-loadbalancer-type") | del(.spec.template.metadata.annotations."last-applied-secret") | del(.spec.template.metadata.labels."batch.kubernetes.io/job-name") | del(.spec.template.metadata.labels."job-name") | del(.. | select(has("batch.kubernetes.io/controller-uid"))."batch.kubernetes.io/controller-uid") | del(.. | select(has("image")).image) | del(.. | select(has("clusterIP")).clusterIP) | del(.. | select(has("clusterIPs")).clusterIPs) | del(.. | select(has("dataSource")).dataSource) | del(.. | select(has("procMount")).procMount) | del(.. | select(has("storageClassName")).storageClassName) | del(.. | select(has("finalizers")).finalizers) | del(.. | select(has("kubernetes.io/pvc-protection"))."kubernetes.io/pvc-protection") | del(.. | select(has("volumeName")).volumeName) | del(.. | select(has("volume.beta.kubernetes.io/storage-provisioner"))."volume.beta.kubernetes.io/storage-provisioner") | del(.. | select(has("volume.kubernetes.io/storage-provisioner"))."volume.kubernetes.io/storage-provisioner") | del(.spec.volumeMode) | del(.spec.nodeName) | del(.. | select(has("volume.kubernetes.io/selected-node"))."volume.kubernetes.io/selected-node") | del(.. | select(has("percona.com/last-config-hash"))."percona.com/last-config-hash") | del(.. | select(has("percona.com/configuration-hash"))."percona.com/configuration-hash") | del(.. | select(has("percona.com/env-secret-config-hash"))."percona.com/env-secret-config-hash") | del(.. | select(has("percona.com/ssl-hash"))."percona.com/ssl-hash") | del(.. | select(has("percona.com/ssl-internal-hash"))."percona.com/ssl-internal-hash") | del(.. | select(has("kubectl.kubernetes.io/default-container"))."kubectl.kubernetes.io/default-container") | del(.spec.volumeClaimTemplates[].spec.volumeMode | select(. == "Filesystem")) | del(.. | select(has("healthCheckNodePort")).healthCheckNodePort) | del(.. | select(has("nodePort")).nodePort) | del(.. | select(has("imagePullSecrets")).imagePullSecrets) | del(.. | select(has("enableServiceLinks")).enableServiceLinks) | del(.status) | del(.spec.volumeClaimTemplates[].apiVersion) | del(.spec.volumeClaimTemplates[].kind) | del(.metadata.ownerReferences[].apiVersion) | del(.. | select(has("controller-uid")).controller-uid) | del(.. | select(has("preemptionPolicy")).preemptionPolicy) | del(.spec.ipFamilies) | del(.spec.ipFamilyPolicy) | del(.spec.updateStrategy.rollingUpdate.maxUnavailable) | (.. | select(. == "policy/v1beta1")) = "policy/v1" | del(.. | select(has("kubernetes.io/hostname"))."kubernetes.io/hostname") | (.. | select(tag == "!!str")) |= sub("monitoring-2-0-9920", "namespace") | (.. | select(tag == "!!str")) |= sub("kube-api-access-.*", "kube-api-access") | del(.. | select(has("annotations")).annotations | select(length==0)) | del(.spec.crVersion) | del(.. | select(.[] == "percona-xtradb-cluster-operator-workload-token*"))' - + kubectl_bin get -o yaml statefulset/monitoring-pxc ++ mktemp + local LAST_OUT=/tmp/tmp.fFIGbvCmGo ++ mktemp + local LAST_ERR=/tmp/tmp.HyLz0s4cpU + local exit_status=0 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl get -o yaml statefulset/monitoring-pxc + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.fFIGbvCmGo + cat /tmp/tmp.HyLz0s4cpU + rm /tmp/tmp.fFIGbvCmGo /tmp/tmp.HyLz0s4cpU + return 0 + [[ 0 -eq 0 ]] + diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-2525/e2e-tests/monitoring-2-0/compare/statefulset_monitoring-pxc-no-prefix-k127.yml /tmp/tmp.gvyFfLErHq/statefulset_monitoring-pxc.yml + log 'compare_kubectl: statefulset/monitoring-pxc OK' ++ date +%Y-%m-%dT%H:%M:%S%z + echo '[2026-08-06T03:14:51+0000]' compare_kubectl: statefulset/monitoring-pxc OK [2026-08-06T03:14:51+0000] compare_kubectl: statefulset/monitoring-pxc OK + compare_kubectl statefulset/monitoring-haproxy -no-prefix + local resource=statefulset/monitoring-haproxy + local postfix=-no-prefix + local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2525/e2e-tests/monitoring-2-0/compare/statefulset_monitoring-haproxy-no-prefix.yml + local new_result=/tmp/tmp.gvyFfLErHq/statefulset_monitoring-haproxy.yml + desc 'compare statefulset/monitoring-haproxy--no-prefix' + set +o xtrace ----------------------------------------------------------------------------------- compare statefulset/monitoring-haproxy--no-prefix ----------------------------------------------------------------------------------- + '[' 0 = 1 -a -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2525/e2e-tests/monitoring-2-0/compare/statefulset_monitoring-haproxy-no-prefix-eks.yml ']' + [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 8\.0 ]] + '[' -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2525/e2e-tests/monitoring-2-0/compare/statefulset_monitoring-haproxy-no-prefix-80.yml ']' + [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 8\.4 ]] + version_gt 1.33 ++ echo '1.33 >= 1.33' ++ bc -l + '[' 1 -eq 1 ']' + return 0 + '[' -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2525/e2e-tests/monitoring-2-0/compare/statefulset_monitoring-haproxy-no-prefix-k133.yml ']' + version_gt 1.29 ++ bc -l ++ echo '1.33 >= 1.29' + '[' 1 -eq 1 ']' + return 0 + '[' -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2525/e2e-tests/monitoring-2-0/compare/statefulset_monitoring-haproxy-no-prefix-k129.yml ']' + version_gt 1.27 ++ bc -l ++ echo '1.33 >= 1.27' + '[' 1 -eq 1 ']' + return 0 + '[' -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2525/e2e-tests/monitoring-2-0/compare/statefulset_monitoring-haproxy-no-prefix-k127.yml ']' + expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2525/e2e-tests/monitoring-2-0/compare/statefulset_monitoring-haproxy-no-prefix-k127.yml + '[' '!' -z '' -a -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2525/e2e-tests/monitoring-2-0/compare/statefulset_monitoring-haproxy-no-prefix-k127-oc.yml ']' + version_gt 1.29 ++ echo '1.33 >= 1.29' ++ bc -l + '[' 1 -eq 1 ']' + return 0 + '[' '!' -z '' -a -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2525/e2e-tests/monitoring-2-0/compare/statefulset_monitoring-haproxy-no-prefix-k127-k129-oc.yml ']' + '[' 0 = 1 -a -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2525/e2e-tests/monitoring-2-0/compare/statefulset_monitoring-haproxy-no-prefix-k127-eks.yml ']' + '[' 0 = 1 -a -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2525/e2e-tests/monitoring-2-0/compare/statefulset_monitoring-haproxy-no-prefix-k127-aks.yml ']' + kubectl_bin get -o yaml statefulset/monitoring-haproxy + yq eval ' del(.metadata.managedFields) | del(.. | select(has("creationTimestamp")).creationTimestamp) | del(.. | select(has("namespace")).namespace) | del(.. | select(has("uid")).uid) | del(.metadata.resourceVersion) | del(.spec.template.spec.containers[].env[] | select(.name == "CLUSTER_HASH")) | del(.spec.template.spec.containers[].env[] | select(.name == "S3_BUCKET_PATH")) | del(.spec.template.spec.containers[].env[] | select(.name == "BACKUP_PATH")) | del(.spec.template.spec.containers[].env[] | select(.name == "S3_BUCKET_URL")) | del(.spec.template.spec.containers[].env[] | select(.name == "AZURE_CONTAINER_NAME")) | del(.spec.template.spec.containers[].env[] | select(.name == "XTRABACKUP_ENABLED")) | del(.metadata.selfLink) | del(.metadata.deletionTimestamp) | del(.metadata.labels."batch.kubernetes.io/job-name") | del(.metadata.labels."job-name") | del(.metadata.labels."topology.kubernetes.io/region") | del(.metadata.labels."topology.kubernetes.io/zone") | del(.metadata.annotations."kubectl.kubernetes.io/last-applied-configuration") | del(.metadata.annotations."kubernetes.io/psp") | del(.metadata.annotations."batch.kubernetes.io/job-tracking") | del(.metadata.annotations."networking.gke.io/target-pool") | del(.metadata.annotations."cloud.google.com/neg") | del(.metadata.annotations."k8s.v1.cni.cncf.io*") | del(.metadata.annotations."k8s.ovn.org/pod-networks") | del(.metadata.annotations."kubernetes.digitalocean.com/load-balancer-id") | del(.metadata.annotations."service.beta.kubernetes.io/do-loadbalancer-type") | del(.spec.template.metadata.annotations."last-applied-secret") | del(.spec.template.metadata.labels."batch.kubernetes.io/job-name") | del(.spec.template.metadata.labels."job-name") | del(.. | select(has("batch.kubernetes.io/controller-uid"))."batch.kubernetes.io/controller-uid") | del(.. | select(has("image")).image) | del(.. | select(has("clusterIP")).clusterIP) | del(.. | select(has("clusterIPs")).clusterIPs) | del(.. | select(has("dataSource")).dataSource) | del(.. | select(has("procMount")).procMount) | del(.. | select(has("storageClassName")).storageClassName) | del(.. | select(has("finalizers")).finalizers) | del(.. | select(has("kubernetes.io/pvc-protection"))."kubernetes.io/pvc-protection") | del(.. | select(has("volumeName")).volumeName) | del(.. | select(has("volume.beta.kubernetes.io/storage-provisioner"))."volume.beta.kubernetes.io/storage-provisioner") | del(.. | select(has("volume.kubernetes.io/storage-provisioner"))."volume.kubernetes.io/storage-provisioner") | del(.spec.volumeMode) | del(.spec.nodeName) | del(.. | select(has("volume.kubernetes.io/selected-node"))."volume.kubernetes.io/selected-node") | del(.. | select(has("percona.com/last-config-hash"))."percona.com/last-config-hash") | del(.. | select(has("percona.com/configuration-hash"))."percona.com/configuration-hash") | del(.. | select(has("percona.com/env-secret-config-hash"))."percona.com/env-secret-config-hash") | del(.. | select(has("percona.com/ssl-hash"))."percona.com/ssl-hash") | del(.. | select(has("percona.com/ssl-internal-hash"))."percona.com/ssl-internal-hash") | del(.. | select(has("kubectl.kubernetes.io/default-container"))."kubectl.kubernetes.io/default-container") | del(.spec.volumeClaimTemplates[].spec.volumeMode | select(. == "Filesystem")) | del(.. | select(has("healthCheckNodePort")).healthCheckNodePort) | del(.. | select(has("nodePort")).nodePort) | del(.. | select(has("imagePullSecrets")).imagePullSecrets) | del(.. | select(has("enableServiceLinks")).enableServiceLinks) | del(.status) | del(.spec.volumeClaimTemplates[].apiVersion) | del(.spec.volumeClaimTemplates[].kind) | del(.metadata.ownerReferences[].apiVersion) | del(.. | select(has("controller-uid")).controller-uid) | del(.. | select(has("preemptionPolicy")).preemptionPolicy) | del(.spec.ipFamilies) | del(.spec.ipFamilyPolicy) | del(.spec.updateStrategy.rollingUpdate.maxUnavailable) | (.. | select(. == "policy/v1beta1")) = "policy/v1" | del(.. | select(has("kubernetes.io/hostname"))."kubernetes.io/hostname") | (.. | select(tag == "!!str")) |= sub("monitoring-2-0-9920", "namespace") | (.. | select(tag == "!!str")) |= sub("kube-api-access-.*", "kube-api-access") | del(.. | select(has("annotations")).annotations | select(length==0)) | del(.spec.crVersion) | del(.. | select(.[] == "percona-xtradb-cluster-operator-workload-token*"))' - ++ mktemp + local LAST_OUT=/tmp/tmp.xjiTbHlPO8 ++ mktemp + local LAST_ERR=/tmp/tmp.krgVQeT0WX + local exit_status=0 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl get -o yaml statefulset/monitoring-haproxy + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.xjiTbHlPO8 + cat /tmp/tmp.krgVQeT0WX + rm /tmp/tmp.xjiTbHlPO8 /tmp/tmp.krgVQeT0WX + return 0 + [[ 0 -eq 0 ]] + diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-2525/e2e-tests/monitoring-2-0/compare/statefulset_monitoring-haproxy-no-prefix-k127.yml /tmp/tmp.gvyFfLErHq/statefulset_monitoring-haproxy.yml + log 'compare_kubectl: statefulset/monitoring-haproxy OK' ++ date +%Y-%m-%dT%H:%M:%S%z + echo '[2026-08-06T03:14:54+0000]' compare_kubectl: statefulset/monitoring-haproxy OK [2026-08-06T03:14:54+0000] compare_kubectl: statefulset/monitoring-haproxy OK + desc 'apply my-env-var-secrets to add PMM_PREFIX' + set +o xtrace ----------------------------------------------------------------------------------- apply my-env-var-secrets to add PMM_PREFIX ----------------------------------------------------------------------------------- + kubectl_bin apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2525/e2e-tests/monitoring-2-0/conf/envsecrets.yaml ++ mktemp + local LAST_OUT=/tmp/tmp.McRq1aheIb ++ mktemp + local LAST_ERR=/tmp/tmp.OcfaopEZs5 + local exit_status=0 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2525/e2e-tests/monitoring-2-0/conf/envsecrets.yaml + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.McRq1aheIb secret/my-env-var-secrets created + cat /tmp/tmp.OcfaopEZs5 + rm /tmp/tmp.McRq1aheIb /tmp/tmp.OcfaopEZs5 + return 0 + wait_for_generation sts/monitoring-pxc 3 + local resource=sts/monitoring-pxc + local target_generation=3 + echo 'Waiting for sts/monitoring-pxc to reach generation 3...' Waiting for sts/monitoring-pxc to reach generation 3... + true ++ kubectl get sts/monitoring-pxc -o 'jsonpath={.metadata.generation}' + current_generation=3 + '[' 3 -eq 3 ']' + echo 'Resource sts/monitoring-pxc has reached generation 3.' Resource sts/monitoring-pxc has reached generation 3. + break + wait_for_generation sts/monitoring-haproxy 3 + local resource=sts/monitoring-haproxy + local target_generation=3 + echo 'Waiting for sts/monitoring-haproxy to reach generation 3...' Waiting for sts/monitoring-haproxy to reach generation 3... + true ++ kubectl get sts/monitoring-haproxy -o 'jsonpath={.metadata.generation}' + current_generation=3 + '[' 3 -eq 3 ']' + echo 'Resource sts/monitoring-haproxy has reached generation 3.' Resource sts/monitoring-haproxy has reached generation 3. + break + desc 'add new PMM API key to secret' + set +o xtrace ----------------------------------------------------------------------------------- add new PMM API key to secret ----------------------------------------------------------------------------------- ++ jq .key +++ get_service_endpoint monitoring-service +++ local service=monitoring-service ++++ kubectl_bin get service/monitoring-service -o 'jsonpath={.status.loadBalancer.ingress[].hostname}' +++++ mktemp ++++ local LAST_OUT=/tmp/tmp.jgwonUP0gx +++++ mktemp ++++ local LAST_ERR=/tmp/tmp.VYpvwumFnk ++++ local exit_status=0 +++++ seq 0 2 ++++ for i in $(seq 0 2) ++++ set +e ++++ kubectl get service/monitoring-service -o 'jsonpath={.status.loadBalancer.ingress[].hostname}' ++++ exit_status=0 ++++ set -e ++++ '[' 0 '!=' 0 ']' ++++ break ++++ cat /tmp/tmp.jgwonUP0gx ++++ cat /tmp/tmp.VYpvwumFnk ++++ rm /tmp/tmp.jgwonUP0gx /tmp/tmp.VYpvwumFnk ++++ return 0 +++ local endpoint= +++ '[' -z '' ']' ++++ kubectl_bin get service/monitoring-service -o 'jsonpath={.status.loadBalancer.ingress[].ip}' +++++ mktemp ++++ local LAST_OUT=/tmp/tmp.wjudgEUKG5 +++++ mktemp ++++ local LAST_ERR=/tmp/tmp.fGXgBhxi8j ++++ local exit_status=0 +++++ seq 0 2 ++++ for i in $(seq 0 2) ++++ set +e ++++ kubectl get service/monitoring-service -o 'jsonpath={.status.loadBalancer.ingress[].ip}' ++++ exit_status=0 ++++ set -e ++++ '[' 0 '!=' 0 ']' ++++ break ++++ cat /tmp/tmp.wjudgEUKG5 ++++ cat /tmp/tmp.fGXgBhxi8j ++++ rm /tmp/tmp.wjudgEUKG5 /tmp/tmp.fGXgBhxi8j ++++ return 0 +++ endpoint=34.44.51.100 +++ '[' -n 34.44.51.100 ']' +++ '[' 34.44.51.100 '!=' null ']' +++ head -n 1 +++ sed -e 's/^"//; s/"$//;' +++ echo 34.44.51.100 +++ return ++ curl --insecure -X POST -H 'Content-Type: application/json' -d '{"name":"operator-new", "role": "Admin"}' https://admin:admin@34.44.51.100/graph/api/auth/keys % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 167 100 127 100 40 380 119 --:--:-- --:--:-- --:--:-- 501 + API_KEY_NEW='"eyJrIjoiYlNUU2ZSb1NwUEtkeUxOcjhhV0JDeEhwVDNpU0dqQ0kiLCJuIjoib3BlcmF0b3ItbmV3IiwiaWQiOjF9"' + kubectl_bin patch secret my-cluster-secrets --type merge --patch '{"stringData": {"pmmserverkey": "eyJrIjoiYlNUU2ZSb1NwUEtkeUxOcjhhV0JDeEhwVDNpU0dqQ0kiLCJuIjoib3BlcmF0b3ItbmV3IiwiaWQiOjF9"}}' ++ mktemp + local LAST_OUT=/tmp/tmp.ce51EkE0BY ++ mktemp + local LAST_ERR=/tmp/tmp.fOx3W3Loin + local exit_status=0 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl patch secret my-cluster-secrets --type merge --patch '{"stringData": {"pmmserverkey": "eyJrIjoiYlNUU2ZSb1NwUEtkeUxOcjhhV0JDeEhwVDNpU0dqQ0kiLCJuIjoib3BlcmF0b3ItbmV3IiwiaWQiOjF9"}}' + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.ce51EkE0BY secret/my-cluster-secrets patched + cat /tmp/tmp.fOx3W3Loin + rm /tmp/tmp.ce51EkE0BY /tmp/tmp.fOx3W3Loin + return 0 + desc 'delete old PMM key' + set +o xtrace ----------------------------------------------------------------------------------- delete old PMM key ----------------------------------------------------------------------------------- ++ jq '.[] | select( .name == "operator").id' +++ get_service_endpoint monitoring-service +++ local service=monitoring-service ++++ kubectl_bin get service/monitoring-service -o 'jsonpath={.status.loadBalancer.ingress[].hostname}' +++++ mktemp ++++ local LAST_OUT=/tmp/tmp.02vpcYHQ8j +++++ mktemp ++++ local LAST_ERR=/tmp/tmp.ky8pqg4L8C ++++ local exit_status=0 +++++ seq 0 2 ++++ for i in $(seq 0 2) ++++ set +e ++++ kubectl get service/monitoring-service -o 'jsonpath={.status.loadBalancer.ingress[].hostname}' ++++ exit_status=0 ++++ set -e ++++ '[' 0 '!=' 0 ']' ++++ break ++++ cat /tmp/tmp.02vpcYHQ8j ++++ cat /tmp/tmp.ky8pqg4L8C ++++ rm /tmp/tmp.02vpcYHQ8j /tmp/tmp.ky8pqg4L8C ++++ return 0 +++ local endpoint= +++ '[' -z '' ']' ++++ kubectl_bin get service/monitoring-service -o 'jsonpath={.status.loadBalancer.ingress[].ip}' +++++ mktemp ++++ local LAST_OUT=/tmp/tmp.aylzVrpVBg +++++ mktemp ++++ local LAST_ERR=/tmp/tmp.lIPfvjOkoT ++++ local exit_status=0 +++++ seq 0 2 ++++ for i in $(seq 0 2) ++++ set +e ++++ kubectl get service/monitoring-service -o 'jsonpath={.status.loadBalancer.ingress[].ip}' ++++ exit_status=0 ++++ set -e ++++ '[' 0 '!=' 0 ']' ++++ break ++++ cat /tmp/tmp.aylzVrpVBg ++++ cat /tmp/tmp.lIPfvjOkoT ++++ rm /tmp/tmp.aylzVrpVBg /tmp/tmp.lIPfvjOkoT ++++ return 0 +++ endpoint=34.44.51.100 +++ '[' -n 34.44.51.100 ']' +++ '[' 34.44.51.100 '!=' null ']' +++ echo 34.44.51.100 +++ head -n 1 +++ sed -e 's/^"//; s/"$//;' +++ return ++ curl --insecure -X GET https://admin:admin@34.44.51.100/graph/api/auth/keys % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 597 100 597 0 0 1821 0 --:--:-- --:--:-- --:--:-- 1820 + ID_API_KEY_OLD=6 ++ get_service_endpoint monitoring-service ++ local service=monitoring-service +++ kubectl_bin get service/monitoring-service -o 'jsonpath={.status.loadBalancer.ingress[].hostname}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.oOVRYokvL6 ++++ mktemp +++ local LAST_ERR=/tmp/tmp.eipAoZmGvd +++ local exit_status=0 ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl get service/monitoring-service -o 'jsonpath={.status.loadBalancer.ingress[].hostname}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 ']' +++ break +++ cat /tmp/tmp.oOVRYokvL6 +++ cat /tmp/tmp.eipAoZmGvd +++ rm /tmp/tmp.oOVRYokvL6 /tmp/tmp.eipAoZmGvd +++ return 0 ++ local endpoint= ++ '[' -z '' ']' +++ kubectl_bin get service/monitoring-service -o 'jsonpath={.status.loadBalancer.ingress[].ip}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.Wt0xbxMu5s ++++ mktemp +++ local LAST_ERR=/tmp/tmp.BjAtJU3BmN +++ local exit_status=0 ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl get service/monitoring-service -o 'jsonpath={.status.loadBalancer.ingress[].ip}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 ']' +++ break +++ cat /tmp/tmp.Wt0xbxMu5s +++ cat /tmp/tmp.BjAtJU3BmN +++ rm /tmp/tmp.Wt0xbxMu5s /tmp/tmp.BjAtJU3BmN +++ return 0 ++ endpoint=34.44.51.100 ++ '[' -n 34.44.51.100 ']' ++ '[' 34.44.51.100 '!=' null ']' ++ echo 34.44.51.100 ++ head -n 1 ++ sed -e 's/^"//; s/"$//;' ++ return + curl --insecure -X DELETE https://admin:admin@34.44.51.100/graph/api/auth/keys/6 % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 29 100 29 0 0 87 0 --:--:-- --:--:-- --:--:-- 87 {"message":"API key deleted"}+ wait_for_generation sts/monitoring-pxc 4 + local resource=sts/monitoring-pxc + local target_generation=4 + echo 'Waiting for sts/monitoring-pxc to reach generation 4...' Waiting for sts/monitoring-pxc to reach generation 4... + true ++ kubectl get sts/monitoring-pxc -o 'jsonpath={.metadata.generation}' + current_generation=3 + '[' 3 -eq 4 ']' + echo 'Resource sts/monitoring-pxc is at generation 3. Waiting...' Resource sts/monitoring-pxc is at generation 3. Waiting... + sleep 5 + true ++ kubectl get sts/monitoring-pxc -o 'jsonpath={.metadata.generation}' + current_generation=3 + '[' 3 -eq 4 ']' + echo 'Resource sts/monitoring-pxc is at generation 3. Waiting...' Resource sts/monitoring-pxc is at generation 3. Waiting... + sleep 5 + true ++ kubectl get sts/monitoring-pxc -o 'jsonpath={.metadata.generation}' + current_generation=3 + '[' 3 -eq 4 ']' + echo 'Resource sts/monitoring-pxc is at generation 3. Waiting...' Resource sts/monitoring-pxc is at generation 3. Waiting... + sleep 5 + true ++ kubectl get sts/monitoring-pxc -o 'jsonpath={.metadata.generation}' + current_generation=3 + '[' 3 -eq 4 ']' + echo 'Resource sts/monitoring-pxc is at generation 3. Waiting...' Resource sts/monitoring-pxc is at generation 3. Waiting... + sleep 5 + true ++ kubectl get sts/monitoring-pxc -o 'jsonpath={.metadata.generation}' + current_generation=3 + '[' 3 -eq 4 ']' + echo 'Resource sts/monitoring-pxc is at generation 3. Waiting...' Resource sts/monitoring-pxc is at generation 3. Waiting... + sleep 5 + true ++ kubectl get sts/monitoring-pxc -o 'jsonpath={.metadata.generation}' + current_generation=3 + '[' 3 -eq 4 ']' + echo 'Resource sts/monitoring-pxc is at generation 3. Waiting...' Resource sts/monitoring-pxc is at generation 3. Waiting... + sleep 5 + true ++ kubectl get sts/monitoring-pxc -o 'jsonpath={.metadata.generation}' + current_generation=3 + '[' 3 -eq 4 ']' + echo 'Resource sts/monitoring-pxc is at generation 3. Waiting...' Resource sts/monitoring-pxc is at generation 3. Waiting... + sleep 5 + true ++ kubectl get sts/monitoring-pxc -o 'jsonpath={.metadata.generation}' + current_generation=3 + '[' 3 -eq 4 ']' + echo 'Resource sts/monitoring-pxc is at generation 3. Waiting...' Resource sts/monitoring-pxc is at generation 3. Waiting... + sleep 5 + true ++ kubectl get sts/monitoring-pxc -o 'jsonpath={.metadata.generation}' + current_generation=3 + '[' 3 -eq 4 ']' + echo 'Resource sts/monitoring-pxc is at generation 3. Waiting...' Resource sts/monitoring-pxc is at generation 3. Waiting... + sleep 5 + true ++ kubectl get sts/monitoring-pxc -o 'jsonpath={.metadata.generation}' + current_generation=3 + '[' 3 -eq 4 ']' + echo 'Resource sts/monitoring-pxc is at generation 3. Waiting...' Resource sts/monitoring-pxc is at generation 3. Waiting... + sleep 5 + true ++ kubectl get sts/monitoring-pxc -o 'jsonpath={.metadata.generation}' + current_generation=3 + '[' 3 -eq 4 ']' + echo 'Resource sts/monitoring-pxc is at generation 3. Waiting...' Resource sts/monitoring-pxc is at generation 3. Waiting... + sleep 5 + true ++ kubectl get sts/monitoring-pxc -o 'jsonpath={.metadata.generation}' + current_generation=3 + '[' 3 -eq 4 ']' + echo 'Resource sts/monitoring-pxc is at generation 3. Waiting...' Resource sts/monitoring-pxc is at generation 3. Waiting... + sleep 5 + true ++ kubectl get sts/monitoring-pxc -o 'jsonpath={.metadata.generation}' + current_generation=3 + '[' 3 -eq 4 ']' + echo 'Resource sts/monitoring-pxc is at generation 3. Waiting...' Resource sts/monitoring-pxc is at generation 3. Waiting... + sleep 5 + true ++ kubectl get sts/monitoring-pxc -o 'jsonpath={.metadata.generation}' + current_generation=3 + '[' 3 -eq 4 ']' + echo 'Resource sts/monitoring-pxc is at generation 3. Waiting...' Resource sts/monitoring-pxc is at generation 3. Waiting... + sleep 5 + true ++ kubectl get sts/monitoring-pxc -o 'jsonpath={.metadata.generation}' + current_generation=3 + '[' 3 -eq 4 ']' + echo 'Resource sts/monitoring-pxc is at generation 3. Waiting...' Resource sts/monitoring-pxc is at generation 3. Waiting... + sleep 5 + true ++ kubectl get sts/monitoring-pxc -o 'jsonpath={.metadata.generation}' + current_generation=4 + '[' 4 -eq 4 ']' + echo 'Resource sts/monitoring-pxc has reached generation 4.' Resource sts/monitoring-pxc has reached generation 4. + break + wait_for_generation sts/monitoring-haproxy 4 + local resource=sts/monitoring-haproxy + local target_generation=4 + echo 'Waiting for sts/monitoring-haproxy to reach generation 4...' Waiting for sts/monitoring-haproxy to reach generation 4... + true ++ kubectl get sts/monitoring-haproxy -o 'jsonpath={.metadata.generation}' + current_generation=4 + '[' 4 -eq 4 ']' + echo 'Resource sts/monitoring-haproxy has reached generation 4.' Resource sts/monitoring-haproxy has reached generation 4. + break + sleep 10 + kubectl wait pod -l app.kubernetes.io/managed-by=percona-xtradb-cluster-operator --for=condition=ready --timeout=600s E0806 03:17:58.789175 893510 reflector.go:227] "Failed to watch" err="Get \"https://34.31.71.55/api/v1/namespaces/monitoring-2-0-9920/pods?allowWatchBookmarks=true&fieldSelector=metadata.name%3Dmonitoring-haproxy-0&resourceVersion=1785986278068257000&timeoutSeconds=538&watch=true\": context canceled" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" type="*unstructured.Unstructured" pod/monitoring-haproxy-0 condition met pod/monitoring-haproxy-1 condition met E0806 03:18:08.784204 893510 reflector.go:227] "Failed to watch" err="Get \"https://34.31.71.55/api/v1/namespaces/monitoring-2-0-9920/pods?allowWatchBookmarks=true&fieldSelector=metadata.name%3Dmonitoring-pxc-0&resourceVersion=1785986288405631013&timeoutSeconds=582&watch=true\": context canceled" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" type="*unstructured.Unstructured" pod/monitoring-pxc-0 condition met E0806 03:18:09.088169 893510 reflector.go:227] "Failed to watch" err="Get \"https://34.31.71.55/api/v1/namespaces/monitoring-2-0-9920/pods?allowWatchBookmarks=true&fieldSelector=metadata.name%3Dmonitoring-pxc-1&resourceVersion=1785986288810159013&timeoutSeconds=568&watch=true\": context canceled" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" type="*unstructured.Unstructured" pod/monitoring-pxc-1 condition met pod/monitoring-pxc-2 condition met + desc 'check if pmm-client container enabled' + set +o xtrace ----------------------------------------------------------------------------------- check if pmm-client container enabled ----------------------------------------------------------------------------------- + compare_kubectl statefulset/monitoring-pxc + local resource=statefulset/monitoring-pxc + local postfix= + local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2525/e2e-tests/monitoring-2-0/compare/statefulset_monitoring-pxc.yml + local new_result=/tmp/tmp.gvyFfLErHq/statefulset_monitoring-pxc.yml + desc 'compare statefulset/monitoring-pxc-' + set +o xtrace ----------------------------------------------------------------------------------- compare statefulset/monitoring-pxc- ----------------------------------------------------------------------------------- + '[' 0 = 1 -a -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2525/e2e-tests/monitoring-2-0/compare/statefulset_monitoring-pxc-eks.yml ']' + [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 8\.0 ]] + '[' -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2525/e2e-tests/monitoring-2-0/compare/statefulset_monitoring-pxc-80.yml ']' + [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 8\.4 ]] + version_gt 1.33 ++ bc -l ++ echo '1.33 >= 1.33' + '[' 1 -eq 1 ']' + return 0 + '[' -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2525/e2e-tests/monitoring-2-0/compare/statefulset_monitoring-pxc-k133.yml ']' + version_gt 1.29 ++ bc -l ++ echo '1.33 >= 1.29' + '[' 1 -eq 1 ']' + return 0 + '[' -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2525/e2e-tests/monitoring-2-0/compare/statefulset_monitoring-pxc-k129.yml ']' + version_gt 1.27 ++ echo '1.33 >= 1.27' ++ bc -l + '[' 1 -eq 1 ']' + return 0 + '[' -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2525/e2e-tests/monitoring-2-0/compare/statefulset_monitoring-pxc-k127.yml ']' + expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2525/e2e-tests/monitoring-2-0/compare/statefulset_monitoring-pxc-k127.yml + '[' '!' -z '' -a -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2525/e2e-tests/monitoring-2-0/compare/statefulset_monitoring-pxc-k127-oc.yml ']' + version_gt 1.29 ++ bc -l ++ echo '1.33 >= 1.29' + '[' 1 -eq 1 ']' + return 0 + '[' '!' -z '' -a -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2525/e2e-tests/monitoring-2-0/compare/statefulset_monitoring-pxc-k127-k129-oc.yml ']' + '[' 0 = 1 -a -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2525/e2e-tests/monitoring-2-0/compare/statefulset_monitoring-pxc-k127-eks.yml ']' + '[' 0 = 1 -a -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2525/e2e-tests/monitoring-2-0/compare/statefulset_monitoring-pxc-k127-aks.yml ']' + yq eval ' del(.metadata.managedFields) | del(.. | select(has("creationTimestamp")).creationTimestamp) | del(.. | select(has("namespace")).namespace) | del(.. | select(has("uid")).uid) | del(.metadata.resourceVersion) | del(.spec.template.spec.containers[].env[] | select(.name == "CLUSTER_HASH")) | del(.spec.template.spec.containers[].env[] | select(.name == "S3_BUCKET_PATH")) | del(.spec.template.spec.containers[].env[] | select(.name == "BACKUP_PATH")) | del(.spec.template.spec.containers[].env[] | select(.name == "S3_BUCKET_URL")) | del(.spec.template.spec.containers[].env[] | select(.name == "AZURE_CONTAINER_NAME")) | del(.spec.template.spec.containers[].env[] | select(.name == "XTRABACKUP_ENABLED")) | del(.metadata.selfLink) | del(.metadata.deletionTimestamp) | del(.metadata.labels."batch.kubernetes.io/job-name") | del(.metadata.labels."job-name") | del(.metadata.labels."topology.kubernetes.io/region") | del(.metadata.labels."topology.kubernetes.io/zone") | del(.metadata.annotations."kubectl.kubernetes.io/last-applied-configuration") | del(.metadata.annotations."kubernetes.io/psp") | del(.metadata.annotations."batch.kubernetes.io/job-tracking") | del(.metadata.annotations."networking.gke.io/target-pool") | del(.metadata.annotations."cloud.google.com/neg") | del(.metadata.annotations."k8s.v1.cni.cncf.io*") | del(.metadata.annotations."k8s.ovn.org/pod-networks") | del(.metadata.annotations."kubernetes.digitalocean.com/load-balancer-id") | del(.metadata.annotations."service.beta.kubernetes.io/do-loadbalancer-type") | del(.spec.template.metadata.annotations."last-applied-secret") | del(.spec.template.metadata.labels."batch.kubernetes.io/job-name") | del(.spec.template.metadata.labels."job-name") | del(.. | select(has("batch.kubernetes.io/controller-uid"))."batch.kubernetes.io/controller-uid") | del(.. | select(has("image")).image) | del(.. | select(has("clusterIP")).clusterIP) | del(.. | select(has("clusterIPs")).clusterIPs) | del(.. | select(has("dataSource")).dataSource) | del(.. | select(has("procMount")).procMount) | del(.. | select(has("storageClassName")).storageClassName) | del(.. | select(has("finalizers")).finalizers) | del(.. | select(has("kubernetes.io/pvc-protection"))."kubernetes.io/pvc-protection") | del(.. | select(has("volumeName")).volumeName) | del(.. | select(has("volume.beta.kubernetes.io/storage-provisioner"))."volume.beta.kubernetes.io/storage-provisioner") | del(.. | select(has("volume.kubernetes.io/storage-provisioner"))."volume.kubernetes.io/storage-provisioner") | del(.spec.volumeMode) | del(.spec.nodeName) | del(.. | select(has("volume.kubernetes.io/selected-node"))."volume.kubernetes.io/selected-node") | del(.. | select(has("percona.com/last-config-hash"))."percona.com/last-config-hash") | del(.. | select(has("percona.com/configuration-hash"))."percona.com/configuration-hash") | del(.. | select(has("percona.com/env-secret-config-hash"))."percona.com/env-secret-config-hash") | del(.. | select(has("percona.com/ssl-hash"))."percona.com/ssl-hash") | del(.. | select(has("percona.com/ssl-internal-hash"))."percona.com/ssl-internal-hash") | del(.. | select(has("kubectl.kubernetes.io/default-container"))."kubectl.kubernetes.io/default-container") | del(.spec.volumeClaimTemplates[].spec.volumeMode | select(. == "Filesystem")) | del(.. | select(has("healthCheckNodePort")).healthCheckNodePort) | del(.. | select(has("nodePort")).nodePort) | del(.. | select(has("imagePullSecrets")).imagePullSecrets) | del(.. | select(has("enableServiceLinks")).enableServiceLinks) | del(.status) | del(.spec.volumeClaimTemplates[].apiVersion) | del(.spec.volumeClaimTemplates[].kind) | del(.metadata.ownerReferences[].apiVersion) | del(.. | select(has("controller-uid")).controller-uid) | del(.. | select(has("preemptionPolicy")).preemptionPolicy) | del(.spec.ipFamilies) | del(.spec.ipFamilyPolicy) | del(.spec.updateStrategy.rollingUpdate.maxUnavailable) | (.. | select(. == "policy/v1beta1")) = "policy/v1" | del(.. | select(has("kubernetes.io/hostname"))."kubernetes.io/hostname") | (.. | select(tag == "!!str")) |= sub("monitoring-2-0-9920", "namespace") | (.. | select(tag == "!!str")) |= sub("kube-api-access-.*", "kube-api-access") | del(.. | select(has("annotations")).annotations | select(length==0)) | del(.spec.crVersion) | del(.. | select(.[] == "percona-xtradb-cluster-operator-workload-token*"))' - + kubectl_bin get -o yaml statefulset/monitoring-pxc ++ mktemp + local LAST_OUT=/tmp/tmp.PzWNELQqXt ++ mktemp + local LAST_ERR=/tmp/tmp.QKE8rrt0Yb + local exit_status=0 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl get -o yaml statefulset/monitoring-pxc + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.PzWNELQqXt + cat /tmp/tmp.QKE8rrt0Yb + rm /tmp/tmp.PzWNELQqXt /tmp/tmp.QKE8rrt0Yb + return 0 + [[ 0 -eq 0 ]] + diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-2525/e2e-tests/monitoring-2-0/compare/statefulset_monitoring-pxc-k127.yml /tmp/tmp.gvyFfLErHq/statefulset_monitoring-pxc.yml + log 'compare_kubectl: statefulset/monitoring-pxc OK' ++ date +%Y-%m-%dT%H:%M:%S%z + echo '[2026-08-06T03:18:39+0000]' compare_kubectl: statefulset/monitoring-pxc OK [2026-08-06T03:18:39+0000] compare_kubectl: statefulset/monitoring-pxc OK + compare_kubectl statefulset/monitoring-haproxy + local resource=statefulset/monitoring-haproxy + local postfix= + local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2525/e2e-tests/monitoring-2-0/compare/statefulset_monitoring-haproxy.yml + local new_result=/tmp/tmp.gvyFfLErHq/statefulset_monitoring-haproxy.yml + desc 'compare statefulset/monitoring-haproxy-' + set +o xtrace ----------------------------------------------------------------------------------- compare statefulset/monitoring-haproxy- ----------------------------------------------------------------------------------- + '[' 0 = 1 -a -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2525/e2e-tests/monitoring-2-0/compare/statefulset_monitoring-haproxy-eks.yml ']' + [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 8\.0 ]] + '[' -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2525/e2e-tests/monitoring-2-0/compare/statefulset_monitoring-haproxy-80.yml ']' + [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 8\.4 ]] + version_gt 1.33 ++ echo '1.33 >= 1.33' ++ bc -l + '[' 1 -eq 1 ']' + return 0 + '[' -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2525/e2e-tests/monitoring-2-0/compare/statefulset_monitoring-haproxy-k133.yml ']' + version_gt 1.29 ++ bc -l ++ echo '1.33 >= 1.29' + '[' 1 -eq 1 ']' + return 0 + '[' -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2525/e2e-tests/monitoring-2-0/compare/statefulset_monitoring-haproxy-k129.yml ']' + version_gt 1.27 ++ bc -l ++ echo '1.33 >= 1.27' + '[' 1 -eq 1 ']' + return 0 + '[' -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2525/e2e-tests/monitoring-2-0/compare/statefulset_monitoring-haproxy-k127.yml ']' + expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2525/e2e-tests/monitoring-2-0/compare/statefulset_monitoring-haproxy-k127.yml + '[' '!' -z '' -a -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2525/e2e-tests/monitoring-2-0/compare/statefulset_monitoring-haproxy-k127-oc.yml ']' + version_gt 1.29 ++ bc -l ++ echo '1.33 >= 1.29' + '[' 1 -eq 1 ']' + return 0 + '[' '!' -z '' -a -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2525/e2e-tests/monitoring-2-0/compare/statefulset_monitoring-haproxy-k127-k129-oc.yml ']' + '[' 0 = 1 -a -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2525/e2e-tests/monitoring-2-0/compare/statefulset_monitoring-haproxy-k127-eks.yml ']' + '[' 0 = 1 -a -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2525/e2e-tests/monitoring-2-0/compare/statefulset_monitoring-haproxy-k127-aks.yml ']' + kubectl_bin get -o yaml statefulset/monitoring-haproxy ++ mktemp + local LAST_OUT=/tmp/tmp.00a4ifIeiG + yq eval ' del(.metadata.managedFields) | del(.. | select(has("creationTimestamp")).creationTimestamp) | del(.. | select(has("namespace")).namespace) | del(.. | select(has("uid")).uid) | del(.metadata.resourceVersion) | del(.spec.template.spec.containers[].env[] | select(.name == "CLUSTER_HASH")) | del(.spec.template.spec.containers[].env[] | select(.name == "S3_BUCKET_PATH")) | del(.spec.template.spec.containers[].env[] | select(.name == "BACKUP_PATH")) | del(.spec.template.spec.containers[].env[] | select(.name == "S3_BUCKET_URL")) | del(.spec.template.spec.containers[].env[] | select(.name == "AZURE_CONTAINER_NAME")) | del(.spec.template.spec.containers[].env[] | select(.name == "XTRABACKUP_ENABLED")) | del(.metadata.selfLink) | del(.metadata.deletionTimestamp) | del(.metadata.labels."batch.kubernetes.io/job-name") | del(.metadata.labels."job-name") | del(.metadata.labels."topology.kubernetes.io/region") | del(.metadata.labels."topology.kubernetes.io/zone") | del(.metadata.annotations."kubectl.kubernetes.io/last-applied-configuration") | del(.metadata.annotations."kubernetes.io/psp") | del(.metadata.annotations."batch.kubernetes.io/job-tracking") | del(.metadata.annotations."networking.gke.io/target-pool") | del(.metadata.annotations."cloud.google.com/neg") | del(.metadata.annotations."k8s.v1.cni.cncf.io*") | del(.metadata.annotations."k8s.ovn.org/pod-networks") | del(.metadata.annotations."kubernetes.digitalocean.com/load-balancer-id") | del(.metadata.annotations."service.beta.kubernetes.io/do-loadbalancer-type") | del(.spec.template.metadata.annotations."last-applied-secret") | del(.spec.template.metadata.labels."batch.kubernetes.io/job-name") | del(.spec.template.metadata.labels."job-name") | del(.. | select(has("batch.kubernetes.io/controller-uid"))."batch.kubernetes.io/controller-uid") | del(.. | select(has("image")).image) | del(.. | select(has("clusterIP")).clusterIP) | del(.. | select(has("clusterIPs")).clusterIPs) | del(.. | select(has("dataSource")).dataSource) | del(.. | select(has("procMount")).procMount) | del(.. | select(has("storageClassName")).storageClassName) | del(.. | select(has("finalizers")).finalizers) | del(.. | select(has("kubernetes.io/pvc-protection"))."kubernetes.io/pvc-protection") | del(.. | select(has("volumeName")).volumeName) | del(.. | select(has("volume.beta.kubernetes.io/storage-provisioner"))."volume.beta.kubernetes.io/storage-provisioner") | del(.. | select(has("volume.kubernetes.io/storage-provisioner"))."volume.kubernetes.io/storage-provisioner") | del(.spec.volumeMode) | del(.spec.nodeName) | del(.. | select(has("volume.kubernetes.io/selected-node"))."volume.kubernetes.io/selected-node") | del(.. | select(has("percona.com/last-config-hash"))."percona.com/last-config-hash") | del(.. | select(has("percona.com/configuration-hash"))."percona.com/configuration-hash") | del(.. | select(has("percona.com/env-secret-config-hash"))."percona.com/env-secret-config-hash") | del(.. | select(has("percona.com/ssl-hash"))."percona.com/ssl-hash") | del(.. | select(has("percona.com/ssl-internal-hash"))."percona.com/ssl-internal-hash") | del(.. | select(has("kubectl.kubernetes.io/default-container"))."kubectl.kubernetes.io/default-container") | del(.spec.volumeClaimTemplates[].spec.volumeMode | select(. == "Filesystem")) | del(.. | select(has("healthCheckNodePort")).healthCheckNodePort) | del(.. | select(has("nodePort")).nodePort) | del(.. | select(has("imagePullSecrets")).imagePullSecrets) | del(.. | select(has("enableServiceLinks")).enableServiceLinks) | del(.status) | del(.spec.volumeClaimTemplates[].apiVersion) | del(.spec.volumeClaimTemplates[].kind) | del(.metadata.ownerReferences[].apiVersion) | del(.. | select(has("controller-uid")).controller-uid) | del(.. | select(has("preemptionPolicy")).preemptionPolicy) | del(.spec.ipFamilies) | del(.spec.ipFamilyPolicy) | del(.spec.updateStrategy.rollingUpdate.maxUnavailable) | (.. | select(. == "policy/v1beta1")) = "policy/v1" | del(.. | select(has("kubernetes.io/hostname"))."kubernetes.io/hostname") | (.. | select(tag == "!!str")) |= sub("monitoring-2-0-9920", "namespace") | (.. | select(tag == "!!str")) |= sub("kube-api-access-.*", "kube-api-access") | del(.. | select(has("annotations")).annotations | select(length==0)) | del(.spec.crVersion) | del(.. | select(.[] == "percona-xtradb-cluster-operator-workload-token*"))' - ++ mktemp + local LAST_ERR=/tmp/tmp.IwdDX7CAzQ + local exit_status=0 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl get -o yaml statefulset/monitoring-haproxy + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.00a4ifIeiG + cat /tmp/tmp.IwdDX7CAzQ + rm /tmp/tmp.00a4ifIeiG /tmp/tmp.IwdDX7CAzQ + return 0 + [[ 0 -eq 0 ]] + diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-2525/e2e-tests/monitoring-2-0/compare/statefulset_monitoring-haproxy-k127.yml /tmp/tmp.gvyFfLErHq/statefulset_monitoring-haproxy.yml + log 'compare_kubectl: statefulset/monitoring-haproxy OK' ++ date +%Y-%m-%dT%H:%M:%S%z + echo '[2026-08-06T03:18:42+0000]' compare_kubectl: statefulset/monitoring-haproxy OK [2026-08-06T03:18:42+0000] compare_kubectl: statefulset/monitoring-haproxy OK + desc 'verify clients agents statuses' + set +o xtrace ----------------------------------------------------------------------------------- verify clients agents statuses ----------------------------------------------------------------------------------- + sleep 300 ++ getSecretData my-cluster-secrets pmmserverkey ++ local secretName=my-cluster-secrets ++ local dataKey=pmmserverkey ++ kubectl_bin get secrets/my-cluster-secrets '--template={{.data.pmmserverkey}}' ++ base64 --decode +++ mktemp ++ local LAST_OUT=/tmp/tmp.LcZQpaIze9 +++ mktemp ++ local LAST_ERR=/tmp/tmp.dc3Y8OI80A ++ local exit_status=0 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get secrets/my-cluster-secrets '--template={{.data.pmmserverkey}}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.LcZQpaIze9 ++ cat /tmp/tmp.dc3Y8OI80A ++ rm /tmp/tmp.LcZQpaIze9 /tmp/tmp.dc3Y8OI80A ++ return 0 + API_KEY=eyJrIjoiYlNUU2ZSb1NwUEtkeUxOcjhhV0JDeEhwVDNpU0dqQ0kiLCJuIjoib3BlcmF0b3ItbmV3IiwiaWQiOjF9 +++ get_service_endpoint monitoring-service +++ local service=monitoring-service ++++ kubectl_bin get service/monitoring-service -o 'jsonpath={.status.loadBalancer.ingress[].hostname}' +++++ mktemp ++++ local LAST_OUT=/tmp/tmp.tYRspHKA1c +++++ mktemp ++++ local LAST_ERR=/tmp/tmp.RAd3TAaTbi ++++ local exit_status=0 +++++ seq 0 2 ++++ for i in $(seq 0 2) ++++ set +e ++++ kubectl get service/monitoring-service -o 'jsonpath={.status.loadBalancer.ingress[].hostname}' ++++ exit_status=0 ++++ set -e ++++ '[' 0 '!=' 0 ']' ++++ break ++++ cat /tmp/tmp.tYRspHKA1c ++++ cat /tmp/tmp.RAd3TAaTbi ++++ rm /tmp/tmp.tYRspHKA1c /tmp/tmp.RAd3TAaTbi ++++ return 0 +++ local endpoint= +++ '[' -z '' ']' ++++ kubectl_bin get service/monitoring-service -o 'jsonpath={.status.loadBalancer.ingress[].ip}' +++++ mktemp ++++ local LAST_OUT=/tmp/tmp.s59fOS9FE6 +++++ mktemp ++++ local LAST_ERR=/tmp/tmp.nCxByCgr0f ++++ local exit_status=0 +++++ seq 0 2 ++++ for i in $(seq 0 2) ++++ set +e ++++ kubectl get service/monitoring-service -o 'jsonpath={.status.loadBalancer.ingress[].ip}' ++++ exit_status=0 ++++ set -e ++++ '[' 0 '!=' 0 ']' ++++ break ++++ cat /tmp/tmp.s59fOS9FE6 ++++ cat /tmp/tmp.nCxByCgr0f ++++ rm /tmp/tmp.s59fOS9FE6 /tmp/tmp.nCxByCgr0f ++++ return 0 +++ endpoint=34.44.51.100 +++ '[' -n 34.44.51.100 ']' +++ '[' 34.44.51.100 '!=' null ']' +++ sed -e 's/^"//; s/"$//;' +++ head -n 1 +++ echo 34.44.51.100 +++ return ++ get_mgmnt_service_list eyJrIjoiYlNUU2ZSb1NwUEtkeUxOcjhhV0JDeEhwVDNpU0dqQ0kiLCJuIjoib3BlcmF0b3ItbmV3IiwiaWQiOjF9 34.44.51.100 monitoring-2-0-9920 ++ local api_key=eyJrIjoiYlNUU2ZSb1NwUEtkeUxOcjhhV0JDeEhwVDNpU0dqQ0kiLCJuIjoib3BlcmF0b3ItbmV3IiwiaWQiOjF9 ++ local endpoint=34.44.51.100 ++ local namespace=monitoring-2-0-9920 ++ jq 'walk(if type == "array" then sort_by(.agent_type) else . end)' ++ jq 'walk(if type=="object" then with_entries(select(.key | test("service_id|node_id|agent_id|created_at|updated_at") | not)) else . end)' ++ curl -s -k -H 'Authorization: Bearer eyJrIjoiYlNUU2ZSb1NwUEtkeUxOcjhhV0JDeEhwVDNpU0dqQ0kiLCJuIjoib3BlcmF0b3ItbmV3IiwiaWQiOjF9' -X POST https://34.44.51.100/v1/management/Service/List ++ /usr/bin/sed -i s/monitoring-2-0-9920-//g /tmp/tmp.gvyFfLErHq/active_pmm_agents.json ++ jq '.services | sort_by(.node_name)' ++ cat /tmp/tmp.gvyFfLErHq/active_pmm_agents.json ++ echo /tmp/tmp.gvyFfLErHq/active_pmm_agents_sorted.json + diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-2525/e2e-tests/monitoring-2-0/compare/agents-list.json /tmp/tmp.gvyFfLErHq/active_pmm_agents_sorted.json + desc 'check mysql metrics' + set +o xtrace ----------------------------------------------------------------------------------- check mysql metrics ----------------------------------------------------------------------------------- + get_metric_values node_boot_time_seconds pxc-prefix-monitoring-2-0-9920-monitoring-pxc-0 admin:admin + local metric=node_boot_time_seconds + local instance=pxc-prefix-monitoring-2-0-9920-monitoring-pxc-0 + local user_pass=admin:admin ++ /usr/bin/date -u +%s -d '-1 minute' + local start=1785986570 ++ /usr/bin/date -u +%s + local end=1785986630 ++ get_service_endpoint monitoring-service ++ local service=monitoring-service +++ kubectl_bin get service/monitoring-service -o 'jsonpath={.status.loadBalancer.ingress[].hostname}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.PNlD8AUHzX ++++ mktemp +++ local LAST_ERR=/tmp/tmp.5g6j9ltmqg +++ local exit_status=0 ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl get service/monitoring-service -o 'jsonpath={.status.loadBalancer.ingress[].hostname}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 ']' +++ break +++ cat /tmp/tmp.PNlD8AUHzX +++ cat /tmp/tmp.5g6j9ltmqg +++ rm /tmp/tmp.PNlD8AUHzX /tmp/tmp.5g6j9ltmqg +++ return 0 ++ local endpoint= ++ '[' -z '' ']' +++ kubectl_bin get service/monitoring-service -o 'jsonpath={.status.loadBalancer.ingress[].ip}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.22oCHJ3eLP ++++ mktemp +++ local LAST_ERR=/tmp/tmp.JpbYKixoi6 +++ local exit_status=0 ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl get service/monitoring-service -o 'jsonpath={.status.loadBalancer.ingress[].ip}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 ']' +++ break +++ cat /tmp/tmp.22oCHJ3eLP +++ cat /tmp/tmp.JpbYKixoi6 +++ rm /tmp/tmp.22oCHJ3eLP /tmp/tmp.JpbYKixoi6 +++ return 0 ++ endpoint=34.44.51.100 ++ '[' -n 34.44.51.100 ']' ++ '[' 34.44.51.100 '!=' null ']' ++ echo 34.44.51.100 ++ head -n 1 ++ sed -e 's/^"//; s/"$//;' ++ return + local endpoint=34.44.51.100 ++ curl -s -k 'https://admin:admin@34.44.51.100/graph/api/datasources/proxy/1/api/v1/query_range?query=min%28node_boot_time_seconds%7Bnode_name%3D%7E%22pxc-prefix-monitoring-2-0-9920-monitoring-pxc-0%22%7d%20or%20node_boot_time_seconds%7Bnode_name%3D%7E%22pxc-prefix-monitoring-2-0-9920-monitoring-pxc-0%22%7D%29&start=1785986570&end=1785986630&step=60' ++ jq '.data.result[0]' + local 'result={ "metric": {}, "values": [ [ 1785986570, "1785979322" ], [ 1785986630, "1785979322" ] ] }' + '[' '{ "metric": {}, "values": [ [ 1785986570, "1785979322" ], [ 1785986630, "1785979322" ] ] }' = null ']' + echo -n '{ "metric": {}, "values": [ [ 1785986570, "1785979322" ], [ 1785986630, "1785979322" ] ] }' + grep '^"[0-9]' + jq '.values[][1]' "1785979322" "1785979322" + get_metric_values mysql_global_status_uptime pxc-prefix-monitoring-2-0-9920-monitoring-pxc-0 admin:admin + local metric=mysql_global_status_uptime + local instance=pxc-prefix-monitoring-2-0-9920-monitoring-pxc-0 + local user_pass=admin:admin ++ /usr/bin/date -u +%s -d '-1 minute' + local start=1785986575 ++ /usr/bin/date -u +%s + local end=1785986635 ++ get_service_endpoint monitoring-service ++ local service=monitoring-service +++ kubectl_bin get service/monitoring-service -o 'jsonpath={.status.loadBalancer.ingress[].hostname}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.Pc3YZzEHzt ++++ mktemp +++ local LAST_ERR=/tmp/tmp.5jlvtYHGra +++ local exit_status=0 ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl get service/monitoring-service -o 'jsonpath={.status.loadBalancer.ingress[].hostname}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 ']' +++ break +++ cat /tmp/tmp.Pc3YZzEHzt +++ cat /tmp/tmp.5jlvtYHGra +++ rm /tmp/tmp.Pc3YZzEHzt /tmp/tmp.5jlvtYHGra +++ return 0 ++ local endpoint= ++ '[' -z '' ']' +++ kubectl_bin get service/monitoring-service -o 'jsonpath={.status.loadBalancer.ingress[].ip}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.MGa0TopC7P ++++ mktemp +++ local LAST_ERR=/tmp/tmp.fGshqzZic6 +++ local exit_status=0 ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl get service/monitoring-service -o 'jsonpath={.status.loadBalancer.ingress[].ip}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 ']' +++ break +++ cat /tmp/tmp.MGa0TopC7P +++ cat /tmp/tmp.fGshqzZic6 +++ rm /tmp/tmp.MGa0TopC7P /tmp/tmp.fGshqzZic6 +++ return 0 ++ endpoint=34.44.51.100 ++ '[' -n 34.44.51.100 ']' ++ '[' 34.44.51.100 '!=' null ']' ++ sed -e 's/^"//; s/"$//;' ++ echo 34.44.51.100 ++ head -n 1 ++ return + local endpoint=34.44.51.100 ++ curl -s -k 'https://admin:admin@34.44.51.100/graph/api/datasources/proxy/1/api/v1/query_range?query=min%28mysql_global_status_uptime%7Bnode_name%3D%7E%22pxc-prefix-monitoring-2-0-9920-monitoring-pxc-0%22%7d%20or%20mysql_global_status_uptime%7Bnode_name%3D%7E%22pxc-prefix-monitoring-2-0-9920-monitoring-pxc-0%22%7D%29&start=1785986575&end=1785986635&step=60' ++ jq '.data.result[0]' + local 'result={ "metric": {}, "values": [ [ 1785986575, "173" ], [ 1785986635, "233" ] ] }' + '[' '{ "metric": {}, "values": [ [ 1785986575, "173" ], [ 1785986635, "233" ] ] }' = null ']' + jq '.values[][1]' + echo -n '{ "metric": {}, "values": [ [ 1785986575, "173" ], [ 1785986635, "233" ] ] }' + grep '^"[0-9]' "173" "233" + desc 'check haproxy metrics' + set +o xtrace ----------------------------------------------------------------------------------- check haproxy metrics ----------------------------------------------------------------------------------- + get_metric_values haproxy_backend_status pxc-prefix-monitoring-2-0-9920-monitoring-haproxy-0 admin:admin + local metric=haproxy_backend_status + local instance=pxc-prefix-monitoring-2-0-9920-monitoring-haproxy-0 + local user_pass=admin:admin ++ /usr/bin/date -u +%s -d '-1 minute' + local start=1785986580 ++ /usr/bin/date -u +%s + local end=1785986640 ++ get_service_endpoint monitoring-service ++ local service=monitoring-service +++ kubectl_bin get service/monitoring-service -o 'jsonpath={.status.loadBalancer.ingress[].hostname}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.91ZjwciMVH ++++ mktemp +++ local LAST_ERR=/tmp/tmp.wO0DLP9nE6 +++ local exit_status=0 ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl get service/monitoring-service -o 'jsonpath={.status.loadBalancer.ingress[].hostname}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 ']' +++ break +++ cat /tmp/tmp.91ZjwciMVH +++ cat /tmp/tmp.wO0DLP9nE6 +++ rm /tmp/tmp.91ZjwciMVH /tmp/tmp.wO0DLP9nE6 +++ return 0 ++ local endpoint= ++ '[' -z '' ']' +++ kubectl_bin get service/monitoring-service -o 'jsonpath={.status.loadBalancer.ingress[].ip}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.QFGNMXbopy ++++ mktemp +++ local LAST_ERR=/tmp/tmp.LzsGPiiSJY +++ local exit_status=0 ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl get service/monitoring-service -o 'jsonpath={.status.loadBalancer.ingress[].ip}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 ']' +++ break +++ cat /tmp/tmp.QFGNMXbopy +++ cat /tmp/tmp.LzsGPiiSJY +++ rm /tmp/tmp.QFGNMXbopy /tmp/tmp.LzsGPiiSJY +++ return 0 ++ endpoint=34.44.51.100 ++ '[' -n 34.44.51.100 ']' ++ '[' 34.44.51.100 '!=' null ']' ++ echo 34.44.51.100 ++ head -n 1 ++ sed -e 's/^"//; s/"$//;' ++ return + local endpoint=34.44.51.100 ++ curl -s -k 'https://admin:admin@34.44.51.100/graph/api/datasources/proxy/1/api/v1/query_range?query=min%28haproxy_backend_status%7Bnode_name%3D%7E%22pxc-prefix-monitoring-2-0-9920-monitoring-haproxy-0%22%7d%20or%20haproxy_backend_status%7Bnode_name%3D%7E%22pxc-prefix-monitoring-2-0-9920-monitoring-haproxy-0%22%7D%29&start=1785986580&end=1785986640&step=60' ++ jq '.data.result[0]' + local 'result={ "metric": {}, "values": [ [ 1785986580, "0" ], [ 1785986640, "0" ] ] }' + '[' '{ "metric": {}, "values": [ [ 1785986580, "0" ], [ 1785986640, "0" ] ] }' = null ']' + echo -n '{ "metric": {}, "values": [ [ 1785986580, "0" ], [ 1785986640, "0" ] ] }' + jq '.values[][1]' + grep '^"[0-9]' "0" "0" + get_metric_values haproxy_backend_active_servers pxc-prefix-monitoring-2-0-9920-monitoring-haproxy-0 admin:admin + local metric=haproxy_backend_active_servers + local instance=pxc-prefix-monitoring-2-0-9920-monitoring-haproxy-0 + local user_pass=admin:admin ++ /usr/bin/date -u +%s -d '-1 minute' + local start=1785986585 ++ /usr/bin/date -u +%s + local end=1785986645 ++ get_service_endpoint monitoring-service ++ local service=monitoring-service +++ kubectl_bin get service/monitoring-service -o 'jsonpath={.status.loadBalancer.ingress[].hostname}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.QNfc3DDNWQ ++++ mktemp +++ local LAST_ERR=/tmp/tmp.wjKVtkgUzm +++ local exit_status=0 ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl get service/monitoring-service -o 'jsonpath={.status.loadBalancer.ingress[].hostname}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 ']' +++ break +++ cat /tmp/tmp.QNfc3DDNWQ +++ cat /tmp/tmp.wjKVtkgUzm +++ rm /tmp/tmp.QNfc3DDNWQ /tmp/tmp.wjKVtkgUzm +++ return 0 ++ local endpoint= ++ '[' -z '' ']' +++ kubectl_bin get service/monitoring-service -o 'jsonpath={.status.loadBalancer.ingress[].ip}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.1BkLDGd6FM ++++ mktemp +++ local LAST_ERR=/tmp/tmp.gs3Xs818yd +++ local exit_status=0 ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl get service/monitoring-service -o 'jsonpath={.status.loadBalancer.ingress[].ip}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 ']' +++ break +++ cat /tmp/tmp.1BkLDGd6FM +++ cat /tmp/tmp.gs3Xs818yd +++ rm /tmp/tmp.1BkLDGd6FM /tmp/tmp.gs3Xs818yd +++ return 0 ++ endpoint=34.44.51.100 ++ '[' -n 34.44.51.100 ']' ++ '[' 34.44.51.100 '!=' null ']' ++ head -n 1 ++ sed -e 's/^"//; s/"$//;' ++ echo 34.44.51.100 ++ return + local endpoint=34.44.51.100 ++ curl -s -k 'https://admin:admin@34.44.51.100/graph/api/datasources/proxy/1/api/v1/query_range?query=min%28haproxy_backend_active_servers%7Bnode_name%3D%7E%22pxc-prefix-monitoring-2-0-9920-monitoring-haproxy-0%22%7d%20or%20haproxy_backend_active_servers%7Bnode_name%3D%7E%22pxc-prefix-monitoring-2-0-9920-monitoring-haproxy-0%22%7D%29&start=1785986585&end=1785986645&step=60' ++ jq '.data.result[0]' + local 'result={ "metric": {}, "values": [ [ 1785986585, "1" ], [ 1785986645, "1" ] ] }' + '[' '{ "metric": {}, "values": [ [ 1785986585, "1" ], [ 1785986645, "1" ] ] }' = null ']' + echo -n '{ "metric": {}, "values": [ [ 1785986585, "1" ], [ 1785986645, "1" ] ] }' + jq '.values[][1]' + grep '^"[0-9]' "1" "1" + desc 'check QAN data' + set +o xtrace ----------------------------------------------------------------------------------- check QAN data ----------------------------------------------------------------------------------- + get_qan20_values monitoring-pxc-0 admin:admin + local instance=monitoring-pxc-0 + local user_pass=admin:admin ++ /usr/bin/date -u +%Y-%m-%dT%H:%M:%S -d '-30 minute' + local start=2026-08-06T02:54:09 ++ /usr/bin/date -u +%Y-%m-%dT%H:%M:%S + local end=2026-08-06T03:24:09 ++ get_service_endpoint monitoring-service ++ local service=monitoring-service +++ kubectl_bin get service/monitoring-service -o 'jsonpath={.status.loadBalancer.ingress[].hostname}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.trg81ZJUjH ++++ mktemp +++ local LAST_ERR=/tmp/tmp.ya7dHnCPWc +++ local exit_status=0 ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl get service/monitoring-service -o 'jsonpath={.status.loadBalancer.ingress[].hostname}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 ']' +++ break +++ cat /tmp/tmp.trg81ZJUjH +++ cat /tmp/tmp.ya7dHnCPWc +++ rm /tmp/tmp.trg81ZJUjH /tmp/tmp.ya7dHnCPWc +++ return 0 ++ local endpoint= ++ '[' -z '' ']' +++ kubectl_bin get service/monitoring-service -o 'jsonpath={.status.loadBalancer.ingress[].ip}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.qp0aMuEPaJ ++++ mktemp +++ local LAST_ERR=/tmp/tmp.Vt6zaZZp3l +++ local exit_status=0 ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl get service/monitoring-service -o 'jsonpath={.status.loadBalancer.ingress[].ip}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 ']' +++ break +++ cat /tmp/tmp.qp0aMuEPaJ +++ cat /tmp/tmp.Vt6zaZZp3l +++ rm /tmp/tmp.qp0aMuEPaJ /tmp/tmp.Vt6zaZZp3l +++ return 0 ++ endpoint=34.44.51.100 ++ '[' -n 34.44.51.100 ']' ++ '[' 34.44.51.100 '!=' null ']' ++ echo 34.44.51.100 ++ sed -e 's/^"//; s/"$//;' ++ head -n 1 ++ return + local endpoint=34.44.51.100 + cat ++ /usr/bin/date -u -d '-12 hour' +%Y-%m-%dT%H:%M:%S%:z ++ /usr/bin/date -u +%Y-%m-%dT%H:%M:%S%:z + curl -s -k -XPOST -d @payload.json https://admin:admin@34.44.51.100/v0/qan/GetReport + jq '.rows[].fingerprint' null + rm -f payload.json + nodeList=($(get_node_id_from_pmm)) ++ get_node_id_from_pmm ++ nodeList=() ++ local -a nodeList +++ kubectl_bin get pods --no-headers -l app.kubernetes.io/component=pxc --output=custom-columns=NAME:.metadata.name ++++ mktemp +++ local LAST_OUT=/tmp/tmp.sAKp6otDAF ++++ mktemp +++ local LAST_ERR=/tmp/tmp.mmvK0v5pFJ +++ local exit_status=0 ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl get pods --no-headers -l app.kubernetes.io/component=pxc --output=custom-columns=NAME:.metadata.name +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 ']' +++ break +++ cat /tmp/tmp.sAKp6otDAF +++ cat /tmp/tmp.mmvK0v5pFJ +++ rm /tmp/tmp.sAKp6otDAF /tmp/tmp.mmvK0v5pFJ +++ return 0 ++ for instance in $(kubectl_bin get pods --no-headers -l app.kubernetes.io/component=pxc --output=custom-columns='NAME:.metadata.name') ++ nodeList+=($(kubectl_bin exec -n "$namespace" $instance -c pmm-client -- pmm-admin status --json | jq -r '.pmm_agent_status.node_id')) +++ jq -r .pmm_agent_status.node_id +++ kubectl_bin exec -n monitoring-2-0-9920 monitoring-pxc-0 -c pmm-client -- pmm-admin status --json ++++ mktemp +++ local LAST_OUT=/tmp/tmp.YV8ylQdPrP ++++ mktemp +++ local LAST_ERR=/tmp/tmp.kZlcOr8FUc +++ local exit_status=0 ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl exec -n monitoring-2-0-9920 monitoring-pxc-0 -c pmm-client -- pmm-admin status --json +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 ']' +++ break +++ cat /tmp/tmp.YV8ylQdPrP +++ cat /tmp/tmp.kZlcOr8FUc +++ rm /tmp/tmp.YV8ylQdPrP /tmp/tmp.kZlcOr8FUc +++ return 0 ++ for instance in $(kubectl_bin get pods --no-headers -l app.kubernetes.io/component=pxc --output=custom-columns='NAME:.metadata.name') ++ nodeList+=($(kubectl_bin exec -n "$namespace" $instance -c pmm-client -- pmm-admin status --json | jq -r '.pmm_agent_status.node_id')) +++ jq -r .pmm_agent_status.node_id +++ kubectl_bin exec -n monitoring-2-0-9920 monitoring-pxc-1 -c pmm-client -- pmm-admin status --json ++++ mktemp +++ local LAST_OUT=/tmp/tmp.O80eGiq6op ++++ mktemp +++ local LAST_ERR=/tmp/tmp.139EWGSnYJ +++ local exit_status=0 ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl exec -n monitoring-2-0-9920 monitoring-pxc-1 -c pmm-client -- pmm-admin status --json +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 ']' +++ break +++ cat /tmp/tmp.O80eGiq6op +++ cat /tmp/tmp.139EWGSnYJ +++ rm /tmp/tmp.O80eGiq6op /tmp/tmp.139EWGSnYJ +++ return 0 ++ for instance in $(kubectl_bin get pods --no-headers -l app.kubernetes.io/component=pxc --output=custom-columns='NAME:.metadata.name') ++ nodeList+=($(kubectl_bin exec -n "$namespace" $instance -c pmm-client -- pmm-admin status --json | jq -r '.pmm_agent_status.node_id')) +++ jq -r .pmm_agent_status.node_id +++ kubectl_bin exec -n monitoring-2-0-9920 monitoring-pxc-2 -c pmm-client -- pmm-admin status --json ++++ mktemp +++ local LAST_OUT=/tmp/tmp.UzAc6FfV2P ++++ mktemp +++ local LAST_ERR=/tmp/tmp.RJOEkBUTia +++ local exit_status=0 ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl exec -n monitoring-2-0-9920 monitoring-pxc-2 -c pmm-client -- pmm-admin status --json +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 ']' +++ break +++ cat /tmp/tmp.UzAc6FfV2P +++ cat /tmp/tmp.RJOEkBUTia +++ rm /tmp/tmp.UzAc6FfV2P /tmp/tmp.RJOEkBUTia +++ return 0 ++ echo /node_id/71c7e1c4-0913-4076-a6a6-a154e87ee9d1 /node_id/178cc489-10d0-46db-810b-a9905f251e2c /node_id/9f299829-013d-4fb7-8500-d0c56e233ad8 + nodeList_from_pmm=($(does_node_id_exists "${nodeList[@]}")) ++ does_node_id_exists /node_id/71c7e1c4-0913-4076-a6a6-a154e87ee9d1 /node_id/178cc489-10d0-46db-810b-a9905f251e2c /node_id/9f299829-013d-4fb7-8500-d0c56e233ad8 ++ nodeList=('/node_id/71c7e1c4-0913-4076-a6a6-a154e87ee9d1' '/node_id/178cc489-10d0-46db-810b-a9905f251e2c' '/node_id/9f299829-013d-4fb7-8500-d0c56e233ad8') ++ local -a nodeList ++ nodeList_from_pmm=() ++ local -a nodeList_from_pmm ++ for node_id in "${nodeList[@]}" ++ nodeList_from_pmm+=($(kubectl_bin exec -n "${namespace}" monitoring-0 -- pmm-admin --server-url=https://admin:admin@$(get_service_ip monitoring-service)/ --server-insecure-tls inventory list nodes --node-type=CONTAINER_NODE | grep $node_id | awk '{print $4}')) +++ awk '{print $4}' ++++ get_service_ip monitoring-service ++++ local service=monitoring-service ++++ grep -q NotFound ++++ kubectl_bin get service/monitoring-service -o 'jsonpath={.spec.type}' +++ grep /node_id/71c7e1c4-0913-4076-a6a6-a154e87ee9d1 +++++ kubectl_bin get service/monitoring-service -o 'jsonpath={.spec.type}' ++++++ mktemp +++++ local LAST_OUT=/tmp/tmp.McwOkGh74L ++++++ mktemp +++++ local LAST_ERR=/tmp/tmp.Yj096oVXvE +++++ local exit_status=0 ++++++ seq 0 2 +++++ for i in $(seq 0 2) +++++ set +e +++++ kubectl get service/monitoring-service -o 'jsonpath={.spec.type}' +++++ exit_status=0 +++++ set -e +++++ '[' 0 '!=' 0 ']' +++++ break +++++ cat /tmp/tmp.McwOkGh74L +++++ cat /tmp/tmp.Yj096oVXvE +++++ rm /tmp/tmp.McwOkGh74L /tmp/tmp.Yj096oVXvE +++++ return 0 ++++ '[' LoadBalancer = ClusterIP ']' ++++ kubectl_bin get service/monitoring-service -o 'jsonpath={.status.loadBalancer.ingress[]}' ++++ grep -E -q 'hostname|ip' ++++ kubectl_bin get service/monitoring-service -o 'jsonpath={.status.loadBalancer.ingress[].ip}' +++++ mktemp ++++ local LAST_OUT=/tmp/tmp.Jb56AeTjDU +++++ mktemp ++++ local LAST_ERR=/tmp/tmp.4aXSRBAlQM ++++ local exit_status=0 +++++ seq 0 2 ++++ for i in $(seq 0 2) ++++ set +e ++++ kubectl get service/monitoring-service -o 'jsonpath={.status.loadBalancer.ingress[].ip}' ++++ exit_status=0 ++++ set -e ++++ '[' 0 '!=' 0 ']' ++++ break ++++ cat /tmp/tmp.Jb56AeTjDU ++++ cat /tmp/tmp.4aXSRBAlQM ++++ rm /tmp/tmp.Jb56AeTjDU /tmp/tmp.4aXSRBAlQM ++++ return 0 ++++ kubectl_bin get service/monitoring-service -o 'jsonpath={.status.loadBalancer.ingress[].hostname}' +++++ mktemp ++++ local LAST_OUT=/tmp/tmp.KCGVv3qTOK +++++ mktemp ++++ local LAST_ERR=/tmp/tmp.G4xFfCVi6X ++++ local exit_status=0 +++++ seq 0 2 ++++ for i in $(seq 0 2) ++++ set +e ++++ kubectl get service/monitoring-service -o 'jsonpath={.status.loadBalancer.ingress[].hostname}' ++++ exit_status=0 ++++ set -e ++++ '[' 0 '!=' 0 ']' ++++ break ++++ cat /tmp/tmp.KCGVv3qTOK ++++ cat /tmp/tmp.G4xFfCVi6X ++++ rm /tmp/tmp.KCGVv3qTOK /tmp/tmp.G4xFfCVi6X ++++ return 0 +++ kubectl_bin exec -n monitoring-2-0-9920 monitoring-0 -- pmm-admin --server-url=https://admin:admin@34.44.51.100/ --server-insecure-tls inventory list nodes --node-type=CONTAINER_NODE ++++ mktemp +++ local LAST_OUT=/tmp/tmp.pvapZP6057 ++++ mktemp +++ local LAST_ERR=/tmp/tmp.F1FlSlBlHa +++ local exit_status=0 ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl exec -n monitoring-2-0-9920 monitoring-0 -- pmm-admin --server-url=https://admin:admin@34.44.51.100/ --server-insecure-tls inventory list nodes --node-type=CONTAINER_NODE +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 ']' +++ break +++ cat /tmp/tmp.pvapZP6057 +++ cat /tmp/tmp.F1FlSlBlHa +++ rm /tmp/tmp.pvapZP6057 /tmp/tmp.F1FlSlBlHa +++ return 0 ++ for node_id in "${nodeList[@]}" ++ nodeList_from_pmm+=($(kubectl_bin exec -n "${namespace}" monitoring-0 -- pmm-admin --server-url=https://admin:admin@$(get_service_ip monitoring-service)/ --server-insecure-tls inventory list nodes --node-type=CONTAINER_NODE | grep $node_id | awk '{print $4}')) +++ awk '{print $4}' ++++ get_service_ip monitoring-service ++++ local service=monitoring-service +++ grep /node_id/178cc489-10d0-46db-810b-a9905f251e2c ++++ grep -q NotFound ++++ kubectl_bin get service/monitoring-service -o 'jsonpath={.spec.type}' +++++ kubectl_bin get service/monitoring-service -o 'jsonpath={.spec.type}' ++++++ mktemp +++++ local LAST_OUT=/tmp/tmp.fYouiMdUCz ++++++ mktemp +++++ local LAST_ERR=/tmp/tmp.OFGUe63BBt +++++ local exit_status=0 ++++++ seq 0 2 +++++ for i in $(seq 0 2) +++++ set +e +++++ kubectl get service/monitoring-service -o 'jsonpath={.spec.type}' +++++ exit_status=0 +++++ set -e +++++ '[' 0 '!=' 0 ']' +++++ break +++++ cat /tmp/tmp.fYouiMdUCz +++++ cat /tmp/tmp.OFGUe63BBt +++++ rm /tmp/tmp.fYouiMdUCz /tmp/tmp.OFGUe63BBt +++++ return 0 ++++ '[' LoadBalancer = ClusterIP ']' ++++ grep -E -q 'hostname|ip' ++++ kubectl_bin get service/monitoring-service -o 'jsonpath={.status.loadBalancer.ingress[]}' ++++ kubectl_bin get service/monitoring-service -o 'jsonpath={.status.loadBalancer.ingress[].ip}' +++++ mktemp ++++ local LAST_OUT=/tmp/tmp.GOeImnIMJD +++++ mktemp ++++ local LAST_ERR=/tmp/tmp.icHMr71oqr ++++ local exit_status=0 +++++ seq 0 2 ++++ for i in $(seq 0 2) ++++ set +e ++++ kubectl get service/monitoring-service -o 'jsonpath={.status.loadBalancer.ingress[].ip}' ++++ exit_status=0 ++++ set -e ++++ '[' 0 '!=' 0 ']' ++++ break ++++ cat /tmp/tmp.GOeImnIMJD ++++ cat /tmp/tmp.icHMr71oqr ++++ rm /tmp/tmp.GOeImnIMJD /tmp/tmp.icHMr71oqr ++++ return 0 ++++ kubectl_bin get service/monitoring-service -o 'jsonpath={.status.loadBalancer.ingress[].hostname}' +++++ mktemp ++++ local LAST_OUT=/tmp/tmp.6CTkJgyjQm +++++ mktemp ++++ local LAST_ERR=/tmp/tmp.Pn9AHThwEC ++++ local exit_status=0 +++++ seq 0 2 ++++ for i in $(seq 0 2) ++++ set +e ++++ kubectl get service/monitoring-service -o 'jsonpath={.status.loadBalancer.ingress[].hostname}' ++++ exit_status=0 ++++ set -e ++++ '[' 0 '!=' 0 ']' ++++ break ++++ cat /tmp/tmp.6CTkJgyjQm ++++ cat /tmp/tmp.Pn9AHThwEC ++++ rm /tmp/tmp.6CTkJgyjQm /tmp/tmp.Pn9AHThwEC ++++ return 0 +++ kubectl_bin exec -n monitoring-2-0-9920 monitoring-0 -- pmm-admin --server-url=https://admin:admin@34.44.51.100/ --server-insecure-tls inventory list nodes --node-type=CONTAINER_NODE ++++ mktemp +++ local LAST_OUT=/tmp/tmp.ypFZShhjtv ++++ mktemp +++ local LAST_ERR=/tmp/tmp.FvebwKeJ77 +++ local exit_status=0 ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl exec -n monitoring-2-0-9920 monitoring-0 -- pmm-admin --server-url=https://admin:admin@34.44.51.100/ --server-insecure-tls inventory list nodes --node-type=CONTAINER_NODE +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 ']' +++ break +++ cat /tmp/tmp.ypFZShhjtv +++ cat /tmp/tmp.FvebwKeJ77 +++ rm /tmp/tmp.ypFZShhjtv /tmp/tmp.FvebwKeJ77 +++ return 0 ++ for node_id in "${nodeList[@]}" ++ nodeList_from_pmm+=($(kubectl_bin exec -n "${namespace}" monitoring-0 -- pmm-admin --server-url=https://admin:admin@$(get_service_ip monitoring-service)/ --server-insecure-tls inventory list nodes --node-type=CONTAINER_NODE | grep $node_id | awk '{print $4}')) +++ awk '{print $4}' +++ grep /node_id/9f299829-013d-4fb7-8500-d0c56e233ad8 ++++ get_service_ip monitoring-service ++++ local service=monitoring-service ++++ kubectl_bin get service/monitoring-service -o 'jsonpath={.spec.type}' ++++ grep -q NotFound +++++ kubectl_bin get service/monitoring-service -o 'jsonpath={.spec.type}' ++++++ mktemp +++++ local LAST_OUT=/tmp/tmp.1ws5565EQb ++++++ mktemp +++++ local LAST_ERR=/tmp/tmp.0vXLhUbMIm +++++ local exit_status=0 ++++++ seq 0 2 +++++ for i in $(seq 0 2) +++++ set +e +++++ kubectl get service/monitoring-service -o 'jsonpath={.spec.type}' +++++ exit_status=0 +++++ set -e +++++ '[' 0 '!=' 0 ']' +++++ break +++++ cat /tmp/tmp.1ws5565EQb +++++ cat /tmp/tmp.0vXLhUbMIm +++++ rm /tmp/tmp.1ws5565EQb /tmp/tmp.0vXLhUbMIm +++++ return 0 ++++ '[' LoadBalancer = ClusterIP ']' ++++ grep -E -q 'hostname|ip' ++++ kubectl_bin get service/monitoring-service -o 'jsonpath={.status.loadBalancer.ingress[]}' ++++ kubectl_bin get service/monitoring-service -o 'jsonpath={.status.loadBalancer.ingress[].ip}' +++++ mktemp ++++ local LAST_OUT=/tmp/tmp.bwUrLCdSxx +++++ mktemp ++++ local LAST_ERR=/tmp/tmp.JLUW8G9s4e ++++ local exit_status=0 +++++ seq 0 2 ++++ for i in $(seq 0 2) ++++ set +e ++++ kubectl get service/monitoring-service -o 'jsonpath={.status.loadBalancer.ingress[].ip}' ++++ exit_status=0 ++++ set -e ++++ '[' 0 '!=' 0 ']' ++++ break ++++ cat /tmp/tmp.bwUrLCdSxx ++++ cat /tmp/tmp.JLUW8G9s4e ++++ rm /tmp/tmp.bwUrLCdSxx /tmp/tmp.JLUW8G9s4e ++++ return 0 ++++ kubectl_bin get service/monitoring-service -o 'jsonpath={.status.loadBalancer.ingress[].hostname}' +++++ mktemp ++++ local LAST_OUT=/tmp/tmp.HJQMkAkbtQ +++++ mktemp ++++ local LAST_ERR=/tmp/tmp.dOKAiBTX3A ++++ local exit_status=0 +++++ seq 0 2 ++++ for i in $(seq 0 2) ++++ set +e ++++ kubectl get service/monitoring-service -o 'jsonpath={.status.loadBalancer.ingress[].hostname}' ++++ exit_status=0 ++++ set -e ++++ '[' 0 '!=' 0 ']' ++++ break ++++ cat /tmp/tmp.HJQMkAkbtQ ++++ cat /tmp/tmp.dOKAiBTX3A ++++ rm /tmp/tmp.HJQMkAkbtQ /tmp/tmp.dOKAiBTX3A ++++ return 0 +++ kubectl_bin exec -n monitoring-2-0-9920 monitoring-0 -- pmm-admin --server-url=https://admin:admin@34.44.51.100/ --server-insecure-tls inventory list nodes --node-type=CONTAINER_NODE ++++ mktemp +++ local LAST_OUT=/tmp/tmp.yS9ZV8kbVF ++++ mktemp +++ local LAST_ERR=/tmp/tmp.7TWy3B5RCT +++ local exit_status=0 ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl exec -n monitoring-2-0-9920 monitoring-0 -- pmm-admin --server-url=https://admin:admin@34.44.51.100/ --server-insecure-tls inventory list nodes --node-type=CONTAINER_NODE +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 ']' +++ break +++ cat /tmp/tmp.yS9ZV8kbVF +++ cat /tmp/tmp.7TWy3B5RCT +++ rm /tmp/tmp.yS9ZV8kbVF /tmp/tmp.7TWy3B5RCT +++ return 0 ++ echo /node_id/71c7e1c4-0913-4076-a6a6-a154e87ee9d1 /node_id/178cc489-10d0-46db-810b-a9905f251e2c /node_id/9f299829-013d-4fb7-8500-d0c56e233ad8 + for node_id in "${nodeList_from_pmm[@]}" + '[' -z /node_id/71c7e1c4-0913-4076-a6a6-a154e87ee9d1 ']' + for node_id in "${nodeList_from_pmm[@]}" + '[' -z /node_id/178cc489-10d0-46db-810b-a9905f251e2c ']' + for node_id in "${nodeList_from_pmm[@]}" + '[' -z /node_id/9f299829-013d-4fb7-8500-d0c56e233ad8 ']' + kubectl_bin patch pxc monitoring --type json '-p=[{"op":"add","path":"/spec/pause","value":true}]' ++ mktemp + local LAST_OUT=/tmp/tmp.CGLbxfcIcb ++ mktemp + local LAST_ERR=/tmp/tmp.01vaOzwNmu + local exit_status=0 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl patch pxc monitoring --type json '-p=[{"op":"add","path":"/spec/pause","value":true}]' + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.CGLbxfcIcb perconaxtradbcluster.pxc.percona.com/monitoring patched + cat /tmp/tmp.01vaOzwNmu + rm /tmp/tmp.CGLbxfcIcb /tmp/tmp.01vaOzwNmu + return 0 + wait_for_delete pod/monitoring-pxc-0 + local res=pod/monitoring-pxc-0 + echo -n 'waiting for pod/monitoring-pxc-0 to be deleted' waiting for pod/monitoring-pxc-0 to be deleted+ set +o xtrace .........Error from server (NotFound): pods "monitoring-pxc-0" not found + does_node_id_exists_in_pmm=($(does_node_id_exists "${nodeList[@]}")) ++ does_node_id_exists /node_id/71c7e1c4-0913-4076-a6a6-a154e87ee9d1 /node_id/178cc489-10d0-46db-810b-a9905f251e2c /node_id/9f299829-013d-4fb7-8500-d0c56e233ad8 ++ nodeList=('/node_id/71c7e1c4-0913-4076-a6a6-a154e87ee9d1' '/node_id/178cc489-10d0-46db-810b-a9905f251e2c' '/node_id/9f299829-013d-4fb7-8500-d0c56e233ad8') ++ local -a nodeList ++ nodeList_from_pmm=() ++ local -a nodeList_from_pmm ++ for node_id in "${nodeList[@]}" ++ nodeList_from_pmm+=($(kubectl_bin exec -n "${namespace}" monitoring-0 -- pmm-admin --server-url=https://admin:admin@$(get_service_ip monitoring-service)/ --server-insecure-tls inventory list nodes --node-type=CONTAINER_NODE | grep $node_id | awk '{print $4}')) +++ awk '{print $4}' +++ grep /node_id/71c7e1c4-0913-4076-a6a6-a154e87ee9d1 ++++ get_service_ip monitoring-service ++++ local service=monitoring-service ++++ grep -q NotFound ++++ kubectl_bin get service/monitoring-service -o 'jsonpath={.spec.type}' +++++ kubectl_bin get service/monitoring-service -o 'jsonpath={.spec.type}' ++++++ mktemp +++++ local LAST_OUT=/tmp/tmp.fniz8Kevw8 ++++++ mktemp +++++ local LAST_ERR=/tmp/tmp.R1j2Vri0n0 +++++ local exit_status=0 ++++++ seq 0 2 +++++ for i in $(seq 0 2) +++++ set +e +++++ kubectl get service/monitoring-service -o 'jsonpath={.spec.type}' +++++ exit_status=0 +++++ set -e +++++ '[' 0 '!=' 0 ']' +++++ break +++++ cat /tmp/tmp.fniz8Kevw8 +++++ cat /tmp/tmp.R1j2Vri0n0 +++++ rm /tmp/tmp.fniz8Kevw8 /tmp/tmp.R1j2Vri0n0 +++++ return 0 ++++ '[' LoadBalancer = ClusterIP ']' ++++ kubectl_bin get service/monitoring-service -o 'jsonpath={.status.loadBalancer.ingress[]}' ++++ grep -E -q 'hostname|ip' ++++ kubectl_bin get service/monitoring-service -o 'jsonpath={.status.loadBalancer.ingress[].ip}' +++++ mktemp ++++ local LAST_OUT=/tmp/tmp.dPItFfxiWP +++++ mktemp ++++ local LAST_ERR=/tmp/tmp.Putg6IvN3d ++++ local exit_status=0 +++++ seq 0 2 ++++ for i in $(seq 0 2) ++++ set +e ++++ kubectl get service/monitoring-service -o 'jsonpath={.status.loadBalancer.ingress[].ip}' ++++ exit_status=0 ++++ set -e ++++ '[' 0 '!=' 0 ']' ++++ break ++++ cat /tmp/tmp.dPItFfxiWP ++++ cat /tmp/tmp.Putg6IvN3d ++++ rm /tmp/tmp.dPItFfxiWP /tmp/tmp.Putg6IvN3d ++++ return 0 ++++ kubectl_bin get service/monitoring-service -o 'jsonpath={.status.loadBalancer.ingress[].hostname}' +++++ mktemp ++++ local LAST_OUT=/tmp/tmp.WvYRldtFU6 +++++ mktemp ++++ local LAST_ERR=/tmp/tmp.70lwGTMHhL ++++ local exit_status=0 +++++ seq 0 2 ++++ for i in $(seq 0 2) ++++ set +e ++++ kubectl get service/monitoring-service -o 'jsonpath={.status.loadBalancer.ingress[].hostname}' ++++ exit_status=0 ++++ set -e ++++ '[' 0 '!=' 0 ']' ++++ break ++++ cat /tmp/tmp.WvYRldtFU6 ++++ cat /tmp/tmp.70lwGTMHhL ++++ rm /tmp/tmp.WvYRldtFU6 /tmp/tmp.70lwGTMHhL ++++ return 0 +++ kubectl_bin exec -n monitoring-2-0-9920 monitoring-0 -- pmm-admin --server-url=https://admin:admin@34.44.51.100/ --server-insecure-tls inventory list nodes --node-type=CONTAINER_NODE ++++ mktemp +++ local LAST_OUT=/tmp/tmp.s3UL4gPasn ++++ mktemp +++ local LAST_ERR=/tmp/tmp.EF7qbGkxok +++ local exit_status=0 ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl exec -n monitoring-2-0-9920 monitoring-0 -- pmm-admin --server-url=https://admin:admin@34.44.51.100/ --server-insecure-tls inventory list nodes --node-type=CONTAINER_NODE +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 ']' +++ break +++ cat /tmp/tmp.s3UL4gPasn +++ cat /tmp/tmp.EF7qbGkxok +++ rm /tmp/tmp.s3UL4gPasn /tmp/tmp.EF7qbGkxok +++ return 0 ++ for node_id in "${nodeList[@]}" ++ nodeList_from_pmm+=($(kubectl_bin exec -n "${namespace}" monitoring-0 -- pmm-admin --server-url=https://admin:admin@$(get_service_ip monitoring-service)/ --server-insecure-tls inventory list nodes --node-type=CONTAINER_NODE | grep $node_id | awk '{print $4}')) +++ awk '{print $4}' +++ grep /node_id/178cc489-10d0-46db-810b-a9905f251e2c ++++ get_service_ip monitoring-service ++++ local service=monitoring-service ++++ grep -q NotFound ++++ kubectl_bin get service/monitoring-service -o 'jsonpath={.spec.type}' +++++ kubectl_bin get service/monitoring-service -o 'jsonpath={.spec.type}' ++++++ mktemp +++++ local LAST_OUT=/tmp/tmp.Yxwl6gJssN ++++++ mktemp +++++ local LAST_ERR=/tmp/tmp.gQ2tWIgZY6 +++++ local exit_status=0 ++++++ seq 0 2 +++++ for i in $(seq 0 2) +++++ set +e +++++ kubectl get service/monitoring-service -o 'jsonpath={.spec.type}' +++++ exit_status=0 +++++ set -e +++++ '[' 0 '!=' 0 ']' +++++ break +++++ cat /tmp/tmp.Yxwl6gJssN +++++ cat /tmp/tmp.gQ2tWIgZY6 +++++ rm /tmp/tmp.Yxwl6gJssN /tmp/tmp.gQ2tWIgZY6 +++++ return 0 ++++ '[' LoadBalancer = ClusterIP ']' ++++ grep -E -q 'hostname|ip' ++++ kubectl_bin get service/monitoring-service -o 'jsonpath={.status.loadBalancer.ingress[]}' ++++ kubectl_bin get service/monitoring-service -o 'jsonpath={.status.loadBalancer.ingress[].ip}' +++++ mktemp ++++ local LAST_OUT=/tmp/tmp.vRxQN7ALbI +++++ mktemp ++++ local LAST_ERR=/tmp/tmp.46DWLvMOO6 ++++ local exit_status=0 +++++ seq 0 2 ++++ for i in $(seq 0 2) ++++ set +e ++++ kubectl get service/monitoring-service -o 'jsonpath={.status.loadBalancer.ingress[].ip}' ++++ exit_status=0 ++++ set -e ++++ '[' 0 '!=' 0 ']' ++++ break ++++ cat /tmp/tmp.vRxQN7ALbI ++++ cat /tmp/tmp.46DWLvMOO6 ++++ rm /tmp/tmp.vRxQN7ALbI /tmp/tmp.46DWLvMOO6 ++++ return 0 ++++ kubectl_bin get service/monitoring-service -o 'jsonpath={.status.loadBalancer.ingress[].hostname}' +++++ mktemp ++++ local LAST_OUT=/tmp/tmp.ZVtgZ5vA8r +++++ mktemp ++++ local LAST_ERR=/tmp/tmp.ooFb0pSD8D ++++ local exit_status=0 +++++ seq 0 2 ++++ for i in $(seq 0 2) ++++ set +e ++++ kubectl get service/monitoring-service -o 'jsonpath={.status.loadBalancer.ingress[].hostname}' ++++ exit_status=0 ++++ set -e ++++ '[' 0 '!=' 0 ']' ++++ break ++++ cat /tmp/tmp.ZVtgZ5vA8r ++++ cat /tmp/tmp.ooFb0pSD8D ++++ rm /tmp/tmp.ZVtgZ5vA8r /tmp/tmp.ooFb0pSD8D ++++ return 0 +++ kubectl_bin exec -n monitoring-2-0-9920 monitoring-0 -- pmm-admin --server-url=https://admin:admin@34.44.51.100/ --server-insecure-tls inventory list nodes --node-type=CONTAINER_NODE ++++ mktemp +++ local LAST_OUT=/tmp/tmp.V7zGpcdsym ++++ mktemp +++ local LAST_ERR=/tmp/tmp.4eGiDsW0yo +++ local exit_status=0 ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl exec -n monitoring-2-0-9920 monitoring-0 -- pmm-admin --server-url=https://admin:admin@34.44.51.100/ --server-insecure-tls inventory list nodes --node-type=CONTAINER_NODE +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 ']' +++ break +++ cat /tmp/tmp.V7zGpcdsym +++ cat /tmp/tmp.4eGiDsW0yo +++ rm /tmp/tmp.V7zGpcdsym /tmp/tmp.4eGiDsW0yo +++ return 0 ++ for node_id in "${nodeList[@]}" ++ nodeList_from_pmm+=($(kubectl_bin exec -n "${namespace}" monitoring-0 -- pmm-admin --server-url=https://admin:admin@$(get_service_ip monitoring-service)/ --server-insecure-tls inventory list nodes --node-type=CONTAINER_NODE | grep $node_id | awk '{print $4}')) +++ awk '{print $4}' +++ grep /node_id/9f299829-013d-4fb7-8500-d0c56e233ad8 ++++ get_service_ip monitoring-service ++++ local service=monitoring-service ++++ grep -q NotFound ++++ kubectl_bin get service/monitoring-service -o 'jsonpath={.spec.type}' +++++ kubectl_bin get service/monitoring-service -o 'jsonpath={.spec.type}' ++++++ mktemp +++++ local LAST_OUT=/tmp/tmp.uI49oZiREE ++++++ mktemp +++++ local LAST_ERR=/tmp/tmp.IWmdCeMumy +++++ local exit_status=0 ++++++ seq 0 2 +++++ for i in $(seq 0 2) +++++ set +e +++++ kubectl get service/monitoring-service -o 'jsonpath={.spec.type}' +++++ exit_status=0 +++++ set -e +++++ '[' 0 '!=' 0 ']' +++++ break +++++ cat /tmp/tmp.uI49oZiREE +++++ cat /tmp/tmp.IWmdCeMumy +++++ rm /tmp/tmp.uI49oZiREE /tmp/tmp.IWmdCeMumy +++++ return 0 ++++ '[' LoadBalancer = ClusterIP ']' ++++ kubectl_bin get service/monitoring-service -o 'jsonpath={.status.loadBalancer.ingress[]}' ++++ grep -E -q 'hostname|ip' ++++ kubectl_bin get service/monitoring-service -o 'jsonpath={.status.loadBalancer.ingress[].ip}' +++++ mktemp ++++ local LAST_OUT=/tmp/tmp.0z4HWB3Tvx +++++ mktemp ++++ local LAST_ERR=/tmp/tmp.2eNOt5DrJu ++++ local exit_status=0 +++++ seq 0 2 ++++ for i in $(seq 0 2) ++++ set +e ++++ kubectl get service/monitoring-service -o 'jsonpath={.status.loadBalancer.ingress[].ip}' ++++ exit_status=0 ++++ set -e ++++ '[' 0 '!=' 0 ']' ++++ break ++++ cat /tmp/tmp.0z4HWB3Tvx ++++ cat /tmp/tmp.2eNOt5DrJu ++++ rm /tmp/tmp.0z4HWB3Tvx /tmp/tmp.2eNOt5DrJu ++++ return 0 ++++ kubectl_bin get service/monitoring-service -o 'jsonpath={.status.loadBalancer.ingress[].hostname}' +++++ mktemp ++++ local LAST_OUT=/tmp/tmp.5Tkqn8Bvcr +++++ mktemp ++++ local LAST_ERR=/tmp/tmp.dY0zFuY5CK ++++ local exit_status=0 +++++ seq 0 2 ++++ for i in $(seq 0 2) ++++ set +e ++++ kubectl get service/monitoring-service -o 'jsonpath={.status.loadBalancer.ingress[].hostname}' ++++ exit_status=0 ++++ set -e ++++ '[' 0 '!=' 0 ']' ++++ break ++++ cat /tmp/tmp.5Tkqn8Bvcr ++++ cat /tmp/tmp.dY0zFuY5CK ++++ rm /tmp/tmp.5Tkqn8Bvcr /tmp/tmp.dY0zFuY5CK ++++ return 0 +++ kubectl_bin exec -n monitoring-2-0-9920 monitoring-0 -- pmm-admin --server-url=https://admin:admin@34.44.51.100/ --server-insecure-tls inventory list nodes --node-type=CONTAINER_NODE ++++ mktemp +++ local LAST_OUT=/tmp/tmp.4WU8bhqjFb ++++ mktemp +++ local LAST_ERR=/tmp/tmp.gfh60F0lQz +++ local exit_status=0 ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl exec -n monitoring-2-0-9920 monitoring-0 -- pmm-admin --server-url=https://admin:admin@34.44.51.100/ --server-insecure-tls inventory list nodes --node-type=CONTAINER_NODE +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 ']' +++ break +++ cat /tmp/tmp.4WU8bhqjFb +++ cat /tmp/tmp.gfh60F0lQz +++ rm /tmp/tmp.4WU8bhqjFb /tmp/tmp.gfh60F0lQz +++ return 0 ++ echo + [[ -n '' ]] + helm uninstall monitoring release "monitoring" uninstalled + destroy monitoring-2-0-9920 + local namespace=monitoring-2-0-9920 + local ignore_logs=true + [[ 0 == 1 ]] + desc 'destroy cluster/operator and all other resources' + set +o xtrace ----------------------------------------------------------------------------------- destroy cluster/operator and all other resources ----------------------------------------------------------------------------------- + '[' true == false -o 1 == 1 ']' + grep -v 'the object has been modified' + grep -v 'get backup status: Job.batch' + /usr/bin/sed -r 's/"ts":[0-9.]+//; s^limits-[0-9.]+/^^g' + sort -u + tee /tmp/tmp.gvyFfLErHq/operator.log + grep -v level=info ++ get_operator_pod ++ local label_prefix=app.kubernetes.io/ +++ grep -c percona-xtradb-cluster-operator +++ kubectl get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator -n pxc-operator ++ local check_label=1 ++ [[ 1 -eq 0 ]] ++ kubectl_bin get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator --field-selector=status.phase=Running -o json -n pxc-operator ++ jq -r '.items[] | select(.metadata.deletionTimestamp == null) | .metadata.name' ++ head -1 +++ mktemp ++ local LAST_OUT=/tmp/tmp.iEsh5DdXe8 +++ mktemp ++ local LAST_ERR=/tmp/tmp.qnudV7q6J5 ++ local exit_status=0 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator --field-selector=status.phase=Running -o json -n pxc-operator ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.iEsh5DdXe8 ++ cat /tmp/tmp.qnudV7q6J5 ++ rm /tmp/tmp.iEsh5DdXe8 /tmp/tmp.qnudV7q6J5 ++ return 0 + kubectl_bin logs -n pxc-operator percona-xtradb-cluster-operator-6699755d74-jq8cr ++ mktemp + local LAST_OUT=/tmp/tmp.Rc8oOfYuw3 ++ mktemp + local LAST_ERR=/tmp/tmp.7r76oiOYeg + local exit_status=0 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl logs -n pxc-operator percona-xtradb-cluster-operator-6699755d74-jq8cr + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.Rc8oOfYuw3 + cat /tmp/tmp.7r76oiOYeg + rm /tmp/tmp.Rc8oOfYuw3 /tmp/tmp.7r76oiOYeg + return 0 FieldPath: "metadata.name", FieldPath: "metadata.namespace", LocalObjectReference: {Name: "internal-monitoring"}, Optional: nil, ... // 2 identical fields ConfigMapKeyRef: nil, FieldRef: nil, FieldRef: &v1.ObjectFieldSelector{ FileKeyRef: nil, HTTPHeaders: nil, Host: "", Path: "/local/Status", Port: {IntVal: 7777}, ResourceFieldRef: nil, SecretKeyRef: &v1.SecretKeySelector{ }, ContainerPort: 30100, ContainerPort: 30101, ContainerPort: 30102, ContainerPort: 30103, ContainerPort: 30104, ContainerPort: 30105, ContainerPort: 3306, ContainerPort: 33060, ContainerPort: 33062, ContainerPort: 3307, ContainerPort: 3309, ContainerPort: 4444, ContainerPort: 4567, ContainerPort: 4568, ContainerPort: 7777, ContainerPort: 8404, Exec: nil, GRPC: nil, HTTPGet: &v1.HTTPGetAction{ HostIP: "", HostPort: 0, Items: nil, Items: nil, LocalObjectReference: {Name: "auto-monitoring-pxc"}, LocalObjectReference: {Name: "monitoring-haproxy"}, LocalObjectReference: {Name: "monitoring-pxc"}, Name: "", Name: "ist", Name: "mysql", Name: "mysql-admin", Name: "mysql-replicas", Name: "mysqlx", Name: "proxy-protocol", Name: "sst", Name: "stats", Name: "write-set", Name: "PMM_AGENT_SERVER_USERNAME", Name: "PMM_AGENT_SETUP_NODE_NAME", Name: "PMM_USER", Name: "PMM_AGENT_SERVER_PASSWORD", Name: "PMM_PASSWORD", Name: "POD_NAME", Name: "POD_NAMESPASE", Optional: &true, Optional: &false, Optional: &true, SecretName: "internal-monitoring", SecretName: "monitoring-mysql-init", SecretName: "monitoring-ssl-internal", SecretName: "monitoring-vault", SecretName: "my-env-var-secrets", SecretName: "some-name-ssl", TCPSocket: nil, Value: "", ValueFrom: &v1.EnvVarSource{ ValueFrom: nil, }, ... // 11 identical elements ... // 12 identical elements ... // 16 identical fields ... // 22 identical elements ... // 22 identical fields ... // 23 identical elements ... // 25 identical elements ... // 26 identical elements ... // 3 identical fields ... // 4 identical elements ... // 4 identical fields ... // 9 identical elements ... // 9 identical fields AWSElasticBlockStore: nil, AzureFile: nil, ConfigMap: &v1.ConfigMapVolumeSource{ FC: nil, FailureThreshold: 3, GitRepo: nil, ISCSI: nil, InitialDelaySeconds: 300, InitialDelaySeconds: 15, InitialDelaySeconds: 300, NFS: nil, ProbeHandler: {Exec: &{Command: {"/var/lib/mysql/liveness-check.sh"}}}, ProbeHandler: v1.ProbeHandler{ Quobyte: nil, Secret: &v1.SecretVolumeSource{ SuccessThreshold: 1, TerminationGracePeriodSeconds: nil, TimeoutSeconds: 5, TimeoutSeconds: 15, TimeoutSeconds: 5, VsphereVolume: nil, { {Name: "CLIENT_PORT_LISTEN", Value: "7777"}, {Name: "CLIENT_PORT_MAX", Value: "30105"}, {Name: "CLIENT_PORT_MIN", Value: "30100"}, {Name: "DB_TYPE", Value: "haproxy"}, {Name: "DB_TYPE", Value: "mysql"}, {Name: "DB_USER", Value: "monitor"}, {Name: "MONITOR_USER", Value: "monitor"}, {Name: "PMM_AGENT_LISTEN_PORT", Value: "7777"}, {Name: "PMM_AGENT_PORTS_MIN", Value: "30100"}, {Name: "PMM_AGENT_SERVER_ADDRESS", Value: "monitoring-service"}, {Name: "PMM_AGENT_SERVER_USERNAME", Value: "api_key"}, {Name: "PMM_AGENT_SETUP_FORCE", Value: "1"}, {Name: "PMM_AGENT_SETUP_NODE_TYPE", Value: "container"}, {Name: "PMM_SERVER", Value: "monitoring-service"}, }, ... // 16 identical fields ... // 2 identical fields ... // 3 identical fields ... // 4 identical fields ... // 8 identical fields Args: nil, Args: {"haproxy"}, Args: {"mysqld"}, Env: {{Name: "PXC_SERVICE", Value: "monitoring-pxc"}, {Name: "LIVENESS_CHECK_TIMEOUT", Value: "5"}, {Name: "READINESS_CHECK_TIMEOUT", Value: "1"}}, Env: {{Name: "PXC_SERVICE", Value: "monitoring-pxc-unready"}, {Name: "MONITOR_HOST", Value: "%"}, {Name: "MYSQL_ROOT_PASSWORD", ValueFrom: &{SecretKeyRef: &{LocalObjectReference: {Name: "internal-monitoring"}, Key: "root"}}}, {Name: "XTRABACKUP_PASSWORD", ValueFrom: &{SecretKeyRef: &{LocalObjectReference: {Name: "internal-monitoring"}, Key: "xtrabackup"}}}, ...}, Env: []v1.EnvVar{ EnvFrom: {{SecretRef: &{LocalObjectReference: {Name: "my-env-var-secrets"}, Optional: &true}}}, ImagePullPolicy: "Always", Lifecycle: &{PreStop: &{Exec: &{Command: {"bash", "-c", "pmm-admin unregister --force"}}}}, Lifecycle: nil, LivenessProbe: &v1.Probe{ Name: "auto-config", Name: "config", Name: "haproxy-custom", Name: "my-env-var-secrets", Name: "mysql-init-file", Name: "mysql-users-secret-file", Name: "ssl", Name: "ssl-internal", Name: "vault-keyring-secret", Ports: []v1.ContainerPort{ ReadinessProbe: &{ProbeHandler: {Exec: &{Command: {"/var/lib/mysql/readiness-check.sh"}}}, InitialDelaySeconds: 15, TimeoutSeconds: 15, PeriodSeconds: 30, ...}, ReadinessProbe: &v1.Probe{ ResizePolicy: nil, Resources: {Limits: {s"cpu": {i: {...}, Format: "DecimalSI"}, s"memory": {i: {...}, Format: "DecimalSI"}}, Requests: {s"cpu": {i: {...}, s: "308m", Format: "DecimalSI"}, s"memory": {i: {...}, s: "508M", Format: "DecimalSI"}}}, SecurityContext: nil, StartupProbe: nil, VolumeDevices: nil, VolumeMounts: {{Name: "bin", MountPath: "/var/lib/mysql"}}, VolumeMounts: {{Name: "datadir", MountPath: "/var/lib/mysql"}, {Name: "config", MountPath: "/etc/percona-xtradb-cluster.conf.d"}, {Name: "tmp", MountPath: "/tmp"}, {Name: "ssl", MountPath: "/etc/mysql/ssl"}, ...}, VolumeMounts: {{Name: "datadir", MountPath: "/var/lib/mysql"}}, VolumeSource: v1.VolumeSource{ WorkingDir: "", }, "kubectl.kubernetes.io/default-container": "haproxy", "kubectl.kubernetes.io/default-container": "pxc", "percona.com/configuration-hash": "d41d8cd98f00b204e9800998ecf8427e", "percona.com/env-secret-config-hash": "5bc403a8d44324c1bc704f02e74876d0", "percona.com/ssl-hash": "56707adad7f1492782ebe9aaa617d08e", "percona.com/ssl-internal-hash": "e75870291841ff78939e8fc9350618a4", ... // 2 identical entries ... // 3 identical fields ... // 5 identical fields AccessModes: nil, Capacity: nil, DataSource: nil, DataSourceRef: nil, StorageClassName: nil, VolumeAttributesClassName: nil, VolumeName: "", { {Name: "bin", VolumeSource: {EmptyDir: &{}}}, {Name: "haproxy-auto", VolumeSource: {EmptyDir: &{}}}, {Name: "tmp", VolumeSource: {EmptyDir: &{}}}, }, ... // 2 identical fields ... // 6 identical fields ... // 7 identical fields ... // 9 identical fields ActiveDeadlineSeconds: nil, Affinity: nil, Affinity: nil, Annotations: map[string]string{ AutomountServiceAccountToken: nil, Containers: []v1.Container{ DeletionGracePeriodSeconds: nil, EphemeralContainers: nil, Finalizers: nil, HostAliases: nil, HostAliases: nil, HostIPC: false, Hostname: "", ImagePullSecrets: nil, InitContainers: []v1.Container{ Labels: {"app.kubernetes.io/component": "haproxy", "app.kubernetes.io/instance": "monitoring", "app.kubernetes.io/managed-by": "percona-xtradb-cluster-operator", "app.kubernetes.io/name": "percona-xtradb-cluster", ...}, Labels: {"app.kubernetes.io/component": "pxc", "app.kubernetes.io/instance": "monitoring", "app.kubernetes.io/managed-by": "percona-xtradb-cluster-operator", "app.kubernetes.io/name": "percona-xtradb-cluster", ...}, ManagedFields: nil, NodeName: "", NodeSelector: nil, OS: nil, ObjectMeta: {Name: "datadir", Labels: {"app.kubernetes.io/component": "pxc", "app.kubernetes.io/instance": "monitoring", "app.kubernetes.io/managed-by": "percona-xtradb-cluster-operator", "app.kubernetes.io/name": "percona-xtradb-cluster", ...}}, Overhead: nil, OwnerReferences: nil, PreemptionPolicy: nil, ServiceAccountName: "default", SetHostnameAsFQDN: nil, ShareProcessNamespace: nil, Spec: v1.PersistentVolumeClaimSpec{ Status: v1.PersistentVolumeClaimStatus{ Subdomain: "", Subdomain: "", TerminationGracePeriodSeconds: &30, TerminationGracePeriodSeconds: &600, Tolerations: nil, Tolerations: nil, TypeMeta: {}, Volumes: []v1.Volume{ }, ObjectMeta: v1.ObjectMeta{ ObjectMeta: {Labels: {"app.kubernetes.io/component": "haproxy", "app.kubernetes.io/instance": "monitoring", "app.kubernetes.io/managed-by": "percona-xtradb-cluster-operator", "app.kubernetes.io/name": "percona-xtradb-cluster", ...}, Annotations: {"kubectl.kubernetes.io/default-container": "haproxy", "last-applied-secret": "efd87259fad0d07e7d06001e143e1653756066643d7b4398610e759bb4c15110", "percona.com/configuration-hash": "d41d8cd98f00b204e9800998ecf8427e", "percona.com/env-secret-config-hash": "5bc403a8d44324c1bc704f02e74876d0"}}, ObjectMeta: {Labels: {"app.kubernetes.io/component": "haproxy", "app.kubernetes.io/instance": "monitoring", "app.kubernetes.io/managed-by": "percona-xtradb-cluster-operator", "app.kubernetes.io/name": "percona-xtradb-cluster", ...}, Annotations: {"kubectl.kubernetes.io/default-container": "haproxy", "percona.com/configuration-hash": "d41d8cd98f00b204e9800998ecf8427e"}}, ObjectMeta: {Labels: {"app.kubernetes.io/component": "pxc", "app.kubernetes.io/instance": "monitoring", "app.kubernetes.io/managed-by": "percona-xtradb-cluster-operator", "app.kubernetes.io/name": "percona-xtradb-cluster", ...}, Annotations: {"kubectl.kubernetes.io/default-container": "pxc", "last-applied-secret": "efd87259fad0d07e7d06001e143e1653756066643d7b4398610e759bb4c15110", "percona.com/configuration-hash": "d41d8cd98f00b204e9800998ecf8427e", "percona.com/env-secret-config-hash": "5bc403a8d44324c1bc704f02e74876d0", ...}}, ObjectMeta: {Labels: {"app.kubernetes.io/component": "pxc", "app.kubernetes.io/instance": "monitoring", "app.kubernetes.io/managed-by": "percona-xtradb-cluster-operator", "app.kubernetes.io/name": "percona-xtradb-cluster", ...}, Annotations: {"kubectl.kubernetes.io/default-container": "pxc", "percona.com/configuration-hash": "d41d8cd98f00b204e9800998ecf8427e", "percona.com/ssl-hash": "56707adad7f1492782ebe9aaa617d08e", "percona.com/ssl-internal-hash": "e75870291841ff78939e8fc9350618a4"}}, Spec: v1.PodSpec{ { }, ... // 2 identical fields Conditions: nil, DeletionGracePeriodSeconds: nil, DeletionTimestamp: nil, Finalizers: nil, Labels: nil, MinReadySeconds: 0, Namespace: "monitoring-2-0-9920", Ordinals: nil, OwnerReferences: {{APIVersion: "pxc.percona.com/v1", Kind: "PerconaXtraDBCluster", Name: "monitoring", UID: "d5f5f44f-a98f-4642-8fcd-2ce8d89c1555", ...}}, Replicas: &2, Replicas: &3, Selector: &{MatchLabels: {"app.kubernetes.io/component": "haproxy", "app.kubernetes.io/instance": "monitoring", "app.kubernetes.io/managed-by": "percona-xtradb-cluster-operator", "app.kubernetes.io/name": "percona-xtradb-cluster", ...}}, Selector: &{MatchLabels: {"app.kubernetes.io/component": "pxc", "app.kubernetes.io/instance": "monitoring", "app.kubernetes.io/managed-by": "percona-xtradb-cluster-operator", "app.kubernetes.io/name": "percona-xtradb-cluster", ...}}, SelfLink: "", ServiceName: "monitoring-haproxy", ServiceName: "monitoring-pxc", Template: v1.PodTemplateSpec{ UpdateStrategy: {Type: "RollingUpdate", RollingUpdate: &{Partition: &0}}, VolumeClaimTemplates: nil, VolumeClaimTemplates: []v1.PersistentVolumeClaim{ }, ObjectMeta: v1.ObjectMeta{ Spec: v1.StatefulSetSpec{ Status: v1.StatefulSetStatus{ TypeMeta: {Kind: "StatefulSet", APIVersion: "apps/v1"}, }, &v1.StatefulSet{ } + APIVersion: "", + Key: "pmmserverkey", + Scheme: "", + DefaultMode: nil, + DefaultMode: nil, + Protocol: "", + Value: "$(PMM_PREFIX)$(POD_NAMESPASE)-$(POD_NAME)", + Value: "api_key", + PeriodSeconds: 0, + TerminationMessagePath: "", + TerminationMessagePolicy: "", + "last-applied-secret": "efd87259fad0d07e7d06001e143e1653756066643d7b4398610e759bb4c15110", + "percona.com/env-secret-config-hash": "5bc403a8d44324c1bc704f02e74876d0", + Phase: "", + VolumeMode: nil, + DNSPolicy: "", + DeprecatedServiceAccount: "", + RestartPolicy: "", + SchedulerName: "", + SchedulerName: "", + SecurityContext: nil, + TopologySpreadConstraints: []v1.TopologySpreadConstraint{}, + "percona.com/last-config-hash": "eyJyZXBsaWNhcyI6MCwic2VsZWN0b3IiOnsibWF0Y2hMYWJlbHMiOnsiYXBwLmt1YmVybmV0ZXMuaW8vY29tcG9uZW50IjoiaGFwcm94eSIsImFwcC5rdWJlcm5ldGVzLmlvL2luc3RhbmNlIjoibW9uaXRvcmluZyIsImFwcC5rdWJlcm5ldGVzLmlvL21hbmFnZWQtYnkiOiJwZXJjb25hLXh0cmFkYi1jbHVzdGVyLW9wZXJhdG9yIiwiYXBw"..., + "percona.com/last-config-hash": "eyJyZXBsaWNhcyI6MCwic2VsZWN0b3IiOnsibWF0Y2hMYWJlbHMiOnsiYXBwLmt1YmVybmV0ZXMuaW8vY29tcG9uZW50IjoicHhjIiwiYXBwLmt1YmVybmV0ZXMuaW8vaW5zdGFuY2UiOiJtb25pdG9yaW5nIiwiYXBwLmt1YmVybmV0ZXMuaW8vbWFuYWdlZC1ieSI6InBlcmNvbmEteHRyYWRiLWNsdXN0ZXItb3BlcmF0b3IiLCJhcHAua3Vi"..., + "percona.com/last-config-hash": "eyJyZXBsaWNhcyI6Miwic2VsZWN0b3IiOnsibWF0Y2hMYWJlbHMiOnsiYXBwLmt1YmVybmV0ZXMuaW8vY29tcG9uZW50IjoiaGFwcm94eSIsImFwcC5rdWJlcm5ldGVzLmlvL2luc3RhbmNlIjoibW9uaXRvcmluZyIsImFwcC5rdWJlcm5ldGVzLmlvL21hbmFnZWQtYnkiOiJwZXJjb25hLXh0cmFkYi1jbHVzdGVyLW9wZXJhdG9yIiwiYXBwLmt1YmVybmV0ZXMuaW8vbmFtZSI6InBlcmNvbmEteHRyYWRiLWNsdXN0ZXIiLCJhcHAua3ViZXJuZXRlcy5pby9wYXJ0LW9mIjoicGVyY29uYS14dHJhZGItY2x1c3RlciJ9fSwidGVtcGxhdGUiOnsibWV0YWRhdGEiOnsibGFiZWxzIjp7ImFwcC5rdWJlcm5ldGVzLmlvL2NvbXBvbmVudCI6ImhhcHJveHkiLCJhcHAua3ViZXJuZXRlcy5pby9pbnN0YW5jZSI6Im1vbml0b3JpbmciLCJhcHAua3ViZXJuZXRlcy5pby9tYW5hZ2VkLWJ5IjoicGVyY29uYS14dHJhZGItY2x1c3Rlci1vcGVyYXRvciIsImFwcC5rdWJlcm5ldGVzLmlvL25hbWUiOiJwZXJjb25hLXh0cmFkYi1jbHVzdGVyIiwiYXBwLmt1YmVybmV0ZXMuaW8vcGFydC1vZiI6InBlcmNvbmEteHRyYWRiLWNsdXN0ZXIifSwiYW5ub3RhdGlvbnMiOnsia3ViZWN0bC5rdWJlcm5ldGVzLmlvL2RlZmF1bHQtY29udGFpbmVyIjoiaGFwcm94eSIsImxhc3QtYXBwbGllZC1zZWNyZXQiOiJlZmQ4NzI1OWZhZDBkMDdlN2QwNjAwMWUxNDNlMTY1Mzc1NjA2NjY0M2Q3YjQzOTg2MTBlNzU5YmI0YzE1MTEwIiwicGVyY29uYS5jb20vY29uZmlndXJhdGlvbi1oYXNoIjoi"..., + "percona.com/last-config-hash": "eyJyZXBsaWNhcyI6Miwic2VsZWN0b3IiOnsibWF0Y2hMYWJlbHMiOnsiYXBwLmt1YmVybmV0ZXMuaW8vY29tcG9uZW50IjoiaGFwcm94eSIsImFwcC5rdWJlcm5ldGVzLmlvL2luc3RhbmNlIjoibW9uaXRvcmluZyIsImFwcC5rdWJlcm5ldGVzLmlvL21hbmFnZWQtYnkiOiJwZXJjb25hLXh0cmFkYi1jbHVzdGVyLW9wZXJhdG9yIiwiYXBwLmt1YmVybmV0ZXMuaW8vbmFtZSI6InBlcmNvbmEteHRyYWRiLWNsdXN0ZXIiLCJhcHAua3ViZXJuZXRlcy5pby9wYXJ0LW9mIjoicGVyY29uYS14dHJhZGItY2x1c3RlciJ9fSwidGVtcGxhdGUiOnsibWV0YWRhdGEiOnsibGFiZWxzIjp7ImFwcC5rdWJlcm5ldGVzLmlvL2NvbXBvbmVudCI6ImhhcHJveHkiLCJhcHAua3ViZXJuZXRlcy5pby9pbnN0YW5jZSI6Im1vbml0b3JpbmciLCJhcHAua3ViZXJuZXRlcy5pby9tYW5hZ2VkLWJ5IjoicGVyY29uYS14dHJhZGItY2x1c3Rlci1vcGVyYXRvciIsImFwcC5rdWJlcm5ldGVzLmlvL25hbWUiOiJwZXJjb25hLXh0cmFkYi1jbHVzdGVyIiwiYXBwLmt1YmVybmV0ZXMuaW8vcGFydC1vZiI6InBlcmNvbmEteHRyYWRiLWNsdXN0ZXIifSwiYW5ub3RhdGlvbnMiOnsia3ViZWN0bC5rdWJlcm5ldGVzLmlvL2RlZmF1bHQtY29udGFpbmVyIjoiaGFwcm94eSIsInBlcmNvbmEuY29tL2NvbmZpZ3VyYXRpb24taGFzaCI6ImQ0MWQ4Y2Q5OGYwMGIyMDRlOTgwMDk5OGVjZjg0MjdlIiwicGVyY29uYS5jb20vZW52LXNlY3JldC1jb25maWctaGFzaCI6IjViYzQwM2E4ZDQ0MzI0YzFi"..., + "percona.com/last-config-hash": "eyJyZXBsaWNhcyI6Miwic2VsZWN0b3IiOnsibWF0Y2hMYWJlbHMiOnsiYXBwLmt1YmVybmV0ZXMuaW8vY29tcG9uZW50IjoiaGFwcm94eSIsImFwcC5rdWJlcm5ldGVzLmlvL2luc3RhbmNlIjoibW9uaXRvcmluZyIsImFwcC5rdWJlcm5ldGVzLmlvL21hbmFnZWQtYnkiOiJwZXJjb25hLXh0cmFkYi1jbHVzdGVyLW9wZXJhdG9yIiwiYXBwLmt1YmVybmV0ZXMuaW8vbmFtZSI6InBlcmNvbmEteHRyYWRiLWNsdXN0ZXIiLCJhcHAua3ViZXJuZXRlcy5pby9wYXJ0LW9mIjoicGVyY29uYS14dHJhZGItY2x1c3RlciJ9fSwidGVtcGxhdGUiOnsibWV0YWRhdGEiOnsibGFiZWxzIjp7ImFwcC5rdWJlcm5ldGVzLmlvL2NvbXBvbmVudCI6ImhhcHJveHkiLCJhcHAua3ViZXJuZXRlcy5pby9pbnN0YW5jZSI6Im1vbml0b3JpbmciLCJhcHAua3ViZXJuZXRlcy5pby9tYW5hZ2VkLWJ5IjoicGVyY29uYS14dHJhZGItY2x1c3Rlci1vcGVyYXRvciIsImFwcC5rdWJlcm5ldGVzLmlvL25hbWUiOiJwZXJjb25hLXh0cmFkYi1jbHVzdGVyIiwiYXBwLmt1YmVybmV0ZXMuaW8vcGFydC1vZiI6InBlcmNvbmEteHRyYWRiLWNsdXN0ZXIifSwiYW5ub3RhdGlvbnMiOnsia3ViZWN0bC5rdWJlcm5ldGVzLmlvL2RlZmF1bHQtY29udGFpbmVyIjoiaGFwcm94eSIsInBlcmNvbmEuY29tL2NvbmZpZ3VyYXRpb24taGFzaCI6ImQ0MWQ4Y2Q5OGYwMGIyMDRlOTgwMDk5OGVjZjg0MjdlIn19LCJzcGVjIjp7InZvbHVtZXMiOlt7Im5hbWUiOiJoYXByb3h5LWN1c3RvbSIsImNvbmZpZ01hcCI6eyJuYW1lIjoibW9uaXRvcmluZy1oYXByb3h5Iiwib3B0aW9uYWwiOnRydWV9fSx7Im5hbWUiOiJoYXByb3h5LWF1dG8iLCJlbXB0eURpciI6e319LHsibmFtZSI6Im15c3FsLXVzZXJzLXNlY3JldC1maWxlIiwic2VjcmV0Ijp7InNlY3JldE5hbWUiOiJpbnRlcm5hbC1tb25pdG9yaW5nIiwib3B0aW9uYWwiOmZhbHNlfX0seyJuYW1lIjoibXktZW52LXZhci1zZWNyZXRzIiwic2VjcmV0Ijp7InNlY3JldE5hbWUiOiJteS1lbnYtdmFyLXNlY3JldHMiLCJvcHRpb25hbCI6dHJ1ZX19LHsibmFtZSI6ImJpbiIsImVtcHR5RGlyIjp7fX1dLCJpbml0Q29udGFpbmVycyI6W3sibmFtZSI6InB4Yy1pbml0IiwiaW1hZ2UiOiJwZXJjb25hbGFiL3BlcmNvbmEteHRyYWRiLWNsdXN0ZXItb3BlcmF0b3I6UFItMjUyNS1jODFlNDdmMCIsImNvbW1hbmQiOlsiL3B4Yy1pbml0LWVudHJ5cG9pbnQuc2giXSwicmVzb3VyY2VzIjp7ImxpbWl0cyI6eyJjcHUiOiI1MG0iLCJtZW1vcnkiOiI1ME0ifX0sInZvbHVtZU1vdW50cyI6W3sibmFtZSI6ImJpbiIsIm1vdW50UGF0aCI6Ii92YXIvbGliL215c3FsIn1dLCJpbWFnZVB1bGxQb2xpY3kiOiJBbHdheXMifSx7Im5hbWUiOiJoYXByb3h5LWluaXQiLCJpbWFnZSI6InBlcmNvbmFsYWIvcGVyY29uYS14dHJhZGItY2x1c3Rlci1vcGVyYXRvcjpQUi0yNTI1LWM4MWU0N2YwIiwiY29tbWFuZCI6WyIvaGFwcm94eS1pbml0LWVudHJ5cG9pbnQuc2giXSwicmVzb3VyY2VzIjp7ImxpbWl0cyI6eyJjcHUiOiI1MG0iLCJtZW1vcnkiOiI1ME0ifX0sInZvbHVtZU1vdW50cyI6W3sibmFtZSI6ImJpbiIsIm1vdW50UGF0aCI6Ii9vcHQvcGVyY29uYSJ9XSwiaW1hZ2VQdWxsUG9saWN5IjoiQWx3YXlzIn1dLCJjb250YWluZXJzIjpbeyJuYW1lIjoicG1tLWNsaWVudCIsImltYWdlIjoicGVyY29uYWxhYi9wbW0tY2xpZW50OmRldi1sYXRlc3QiLCJwb3J0cyI6W3siY29udGFpbmVyUG9ydCI6Nzc3N30seyJjb250YWluZXJQb3J0IjozMDEwMH0seyJjb250YWluZXJQb3J0IjozMDEwMX0seyJjb250YWluZXJQb3J0IjozMDEwMn0seyJjb250YWluZXJQb3J0IjozMDEwM30seyJjb250YWluZXJQb3J0IjozMDEwNH0seyJjb250YWluZXJQb3J0IjozMDEwNX1dLCJlbnZGcm9tIjpbeyJzZWNyZXRSZWYiOnsibmFtZSI6Im15LWVudi12YXItc2VjcmV0cyIsIm9wdGlvbmFsIjp0cnVlfX1dLCJlbnYiOlt7Im5hbWUiOiJQTU1fU0VSVkVSIiwidmFsdWUiOiJtb25pdG9yaW5nLXNlcnZpY2UifSx7Im5hbWUiOiJQTU1fVVNFUiIsInZhbHVlIjoiYXBpX2tleSJ9LHsibmFtZSI6IlBNTV9QQVNTV09SRCIsInZhbHVlRnJvbSI6eyJzZWNyZXRLZXlSZWYiOnsibmFtZSI6ImludGVybmFsLW1vbml0b3JpbmciLCJrZXkiOiJwbW1zZXJ2ZXJrZXkifX19LHsibmFtZSI6IkNMSUVOVF9QT1JUX0xJU1RFTiIsInZhbHVlIjoiNzc3NyJ9LHsibmFtZSI6IkNMSUVOVF9QT1JUX01JTiIsInZhbHVlIjoiMzAxMDAifSx7Im5hbWUiOiJDTElFTlRfUE9SVF9NQVgiLCJ2YWx1ZSI6IjMwMTA1In0seyJuYW1lIjoiUE9EX05BTUUiLCJ2YWx1ZUZyb20iOnsiZmllbGRSZWYiOnsiZmllbGRQYXRoIjoibWV0YWRhdGEubmFtZSJ9fX0seyJuYW1lIjoiUE9EX05BTUVTUEFTRSIsInZhbHVlRnJvbSI6eyJmaWVsZFJlZiI6eyJmaWVsZFBhdGgiOiJtZXRhZGF0YS5uYW1lc3BhY2UifX19LHsibmFtZSI6IlBNTV9BR0VOVF9TRVJWRVJfQUREUkVTUyIsInZhbHVlIjoibW9uaXRvcmluZy1zZXJ2aWNlIn0seyJuYW1lIjoiUE1NX0FHRU5UX1NFUlZFUl9VU0VSTkFNRSIsInZhbHVlIjoiYXBpX2tleSJ9LHsibmFtZSI6IlBNTV9BR0VOVF9TRVJWRVJfUEFTU1dPUkQiLCJ2YWx1ZUZyb20iOnsic2VjcmV0S2V5UmVmIjp7Im5hbWUiOiJpbnRlcm5hbC1tb25pdG9yaW5nIiwia2V5IjoicG1tc2VydmVya2V5In19fSx7Im5hbWUiOiJQTU1fQUdFTlRfTElTVEVOX1BPUlQiLCJ2YWx1ZSI6Ijc3NzcifSx7Im5hbWUiOiJQTU1fQUdFTlRfUE9SVFNfTUlOIiwidmFsdWUiOiIzMDEwMCJ9LHsibmFtZSI6IlBNTV9BR0VOVF9QT1JUU19NQVgiLCJ2YWx1ZSI6IjMwMTA1In0seyJuYW1lIjoiUE1NX0FHRU5UX0NPTkZJR19GSUxFIiwidmFsdWUiOiIvdXNyL2xvY2FsL3BlcmNvbmEvcG1tMi9jb25maWcvcG1tLWFnZW50LnlhbWwifSx7Im5hbWUiOiJQTU1fQUdFTlRfU0VSVkVSX0lOU0VDVVJFX1RMUyIsInZhbHVlIjoiMSJ9LHsibmFtZSI6IlBNTV9BR0VOVF9MSVNURU5fQUREUkVTUyIsInZhbHVlIjoiMC4wLjAuMCJ9LHsibmFtZSI6IlBNTV9BR0VOVF9TRVRVUF9NRVRSSUNTX01PREUiLCJ2"..., + "percona.com/last-config-hash": "eyJyZXBsaWNhcyI6Mywic2VsZWN0b3IiOnsibWF0Y2hMYWJlbHMiOnsiYXBwLmt1YmVybmV0ZXMuaW8vY29tcG9uZW50IjoicHhjIiwiYXBwLmt1YmVybmV0ZXMuaW8vaW5zdGFuY2UiOiJtb25pdG9yaW5nIiwiYXBwLmt1YmVybmV0ZXMuaW8vbWFuYWdlZC1ieSI6InBlcmNvbmEteHRyYWRiLWNsdXN0ZXItb3BlcmF0b3IiLCJhcHAua3ViZXJuZXRlcy5pby9uYW1lIjoicGVyY29uYS14dHJhZGItY2x1c3RlciIsImFwcC5rdWJlcm5ldGVzLmlvL3BhcnQtb2YiOiJwZXJjb25hLXh0cmFkYi1jbHVzdGVyIn19LCJ0ZW1wbGF0ZSI6eyJtZXRhZGF0YSI6eyJsYWJlbHMiOnsiYXBwLmt1YmVybmV0ZXMuaW8vY29tcG9uZW50IjoicHhjIiwiYXBwLmt1YmVybmV0ZXMuaW8vaW5zdGFuY2UiOiJtb25pdG9yaW5nIiwiYXBwLmt1YmVybmV0ZXMuaW8vbWFuYWdlZC1ieSI6InBlcmNvbmEteHRyYWRiLWNsdXN0ZXItb3BlcmF0b3IiLCJhcHAua3ViZXJuZXRlcy5pby9uYW1lIjoicGVyY29uYS14dHJhZGItY2x1c3RlciIsImFwcC5rdWJlcm5ldGVzLmlvL3BhcnQtb2YiOiJwZXJjb25hLXh0cmFkYi1jbHVzdGVyIn0sImFubm90YXRpb25zIjp7Imt1YmVjdGwua3ViZXJuZXRlcy5pby9kZWZhdWx0LWNvbnRhaW5lciI6InB4YyIsImxhc3QtYXBwbGllZC1zZWNyZXQiOiJlZmQ4NzI1OWZhZDBkMDdlN2QwNjAwMWUxNDNlMTY1Mzc1NjA2NjY0M2Q3YjQzOTg2MTBlNzU5YmI0YzE1MTEwIiwicGVyY29uYS5jb20vY29uZmlndXJhdGlvbi1oYXNoIjoiZDQxZDhjZDk4ZjAw"..., + "percona.com/last-config-hash": "eyJyZXBsaWNhcyI6Mywic2VsZWN0b3IiOnsibWF0Y2hMYWJlbHMiOnsiYXBwLmt1YmVybmV0ZXMuaW8vY29tcG9uZW50IjoicHhjIiwiYXBwLmt1YmVybmV0ZXMuaW8vaW5zdGFuY2UiOiJtb25pdG9yaW5nIiwiYXBwLmt1YmVybmV0ZXMuaW8vbWFuYWdlZC1ieSI6InBlcmNvbmEteHRyYWRiLWNsdXN0ZXItb3BlcmF0b3IiLCJhcHAua3ViZXJuZXRlcy5pby9uYW1lIjoicGVyY29uYS14dHJhZGItY2x1c3RlciIsImFwcC5rdWJlcm5ldGVzLmlvL3BhcnQtb2YiOiJwZXJjb25hLXh0cmFkYi1jbHVzdGVyIn19LCJ0ZW1wbGF0ZSI6eyJtZXRhZGF0YSI6eyJsYWJlbHMiOnsiYXBwLmt1YmVybmV0ZXMuaW8vY29tcG9uZW50IjoicHhjIiwiYXBwLmt1YmVybmV0ZXMuaW8vaW5zdGFuY2UiOiJtb25pdG9yaW5nIiwiYXBwLmt1YmVybmV0ZXMuaW8vbWFuYWdlZC1ieSI6InBlcmNvbmEteHRyYWRiLWNsdXN0ZXItb3BlcmF0b3IiLCJhcHAua3ViZXJuZXRlcy5pby9uYW1lIjoicGVyY29uYS14dHJhZGItY2x1c3RlciIsImFwcC5rdWJlcm5ldGVzLmlvL3BhcnQtb2YiOiJwZXJjb25hLXh0cmFkYi1jbHVzdGVyIn0sImFubm90YXRpb25zIjp7Imt1YmVjdGwua3ViZXJuZXRlcy5pby9kZWZhdWx0LWNvbnRhaW5lciI6InB4YyIsInBlcmNvbmEuY29tL2NvbmZpZ3VyYXRpb24taGFzaCI6ImQ0MWQ4Y2Q5OGYwMGIyMDRlOTgwMDk5OGVjZjg0MjdlIiwicGVyY29uYS5jb20vZW52LXNlY3JldC1jb25maWctaGFzaCI6IjViYzQwM2E4ZDQ0MzI0YzFiYzcwNGYwMmU3NDg3"..., + "percona.com/last-config-hash": "eyJyZXBsaWNhcyI6Mywic2VsZWN0b3IiOnsibWF0Y2hMYWJlbHMiOnsiYXBwLmt1YmVybmV0ZXMuaW8vY29tcG9uZW50IjoicHhjIiwiYXBwLmt1YmVybmV0ZXMuaW8vaW5zdGFuY2UiOiJtb25pdG9yaW5nIiwiYXBwLmt1YmVybmV0ZXMuaW8vbWFuYWdlZC1ieSI6InBlcmNvbmEteHRyYWRiLWNsdXN0ZXItb3BlcmF0b3IiLCJhcHAua3ViZXJuZXRlcy5pby9uYW1lIjoicGVyY29uYS14dHJhZGItY2x1c3RlciIsImFwcC5rdWJlcm5ldGVzLmlvL3BhcnQtb2YiOiJwZXJjb25hLXh0cmFkYi1jbHVzdGVyIn19LCJ0ZW1wbGF0ZSI6eyJtZXRhZGF0YSI6eyJsYWJlbHMiOnsiYXBwLmt1YmVybmV0ZXMuaW8vY29tcG9uZW50IjoicHhjIiwiYXBwLmt1YmVybmV0ZXMuaW8vaW5zdGFuY2UiOiJtb25pdG9yaW5nIiwiYXBwLmt1YmVybmV0ZXMuaW8vbWFuYWdlZC1ieSI6InBlcmNvbmEteHRyYWRiLWNsdXN0ZXItb3BlcmF0b3IiLCJhcHAua3ViZXJuZXRlcy5pby9uYW1lIjoicGVyY29uYS14dHJhZGItY2x1c3RlciIsImFwcC5rdWJlcm5ldGVzLmlvL3BhcnQtb2YiOiJwZXJjb25hLXh0cmFkYi1jbHVzdGVyIn0sImFubm90YXRpb25zIjp7Imt1YmVjdGwua3ViZXJuZXRlcy5pby9kZWZhdWx0LWNvbnRhaW5lciI6InB4YyIsInBlcmNvbmEuY29tL2NvbmZpZ3VyYXRpb24taGFzaCI6ImQ0MWQ4Y2Q5OGYwMGIyMDRlOTgwMDk5OGVjZjg0MjdlIiwicGVyY29uYS5jb20vc3NsLWhhc2giOiI1NjcwN2FkYWQ3ZjE0OTI3ODJlYmU5YWFhNjE3ZDA4ZSIsInBlcmNvbmEuY29tL3NzbC1pbnRlcm5hbC1oYXNoIjoiZTc1ODcwMjkxODQxZmY3ODkzOWU4ZmM5MzUwNjE4YTQifX0sInNwZWMiOnsidm9sdW1lcyI6W3sibmFtZSI6InRtcCIsImVtcHR5RGlyIjp7fX0seyJuYW1lIjoiY29uZmlnIiwiY29uZmlnTWFwIjp7Im5hbWUiOiJtb25pdG9yaW5nLXB4YyIsIm9wdGlvbmFsIjp0cnVlfX0seyJuYW1lIjoic3NsLWludGVybmFsIiwic2VjcmV0Ijp7InNlY3JldE5hbWUiOiJtb25pdG9yaW5nLXNzbC1pbnRlcm5hbCIsIm9wdGlvbmFsIjp0cnVlfX0seyJuYW1lIjoic3NsIiwic2VjcmV0Ijp7InNlY3JldE5hbWUiOiJzb21lLW5hbWUtc3NsIiwib3B0aW9uYWwiOmZhbHNlfX0seyJuYW1lIjoiYXV0by1jb25maWciLCJjb25maWdNYXAiOnsibmFtZSI6ImF1dG8tbW9uaXRvcmluZy1weGMiLCJvcHRpb25hbCI6dHJ1ZX19LHsibmFtZSI6InZhdWx0LWtleXJpbmctc2VjcmV0Iiwic2VjcmV0Ijp7InNlY3JldE5hbWUiOiJtb25pdG9yaW5nLXZhdWx0Iiwib3B0aW9uYWwiOnRydWV9fSx7Im5hbWUiOiJteXNxbC11c2Vycy1zZWNyZXQtZmlsZSIsInNlY3JldCI6eyJzZWNyZXROYW1lIjoiaW50ZXJuYWwtbW9uaXRvcmluZyIsIm9wdGlvbmFsIjpmYWxzZX19LHsibmFtZSI6Im15c3FsLWluaXQtZmlsZSIsInNlY3JldCI6eyJzZWNyZXROYW1lIjoibW9uaXRvcmluZy1teXNxbC1pbml0Iiwib3B0aW9uYWwiOnRydWV9fSx7Im5hbWUiOiJiaW4iLCJlbXB0eURpciI6e319XSwiaW5pdENvbnRhaW5lcnMiOlt7Im5hbWUiOiJweGMtaW5pdCIsImltYWdlIjoicGVyY29uYWxhYi9wZXJjb25hLXh0cmFkYi1jbHVzdGVyLW9wZXJhdG9yOlBSLTI1MjUtYzgxZTQ3ZjAiLCJjb21tYW5kIjpbIi9weGMtaW5pdC1lbnRyeXBvaW50LnNoIl0sInJlc291cmNlcyI6eyJsaW1pdHMiOnsiY3B1IjoiNTBtIiwibWVtb3J5IjoiNTBNIn19LCJ2b2x1bWVNb3VudHMiOlt7Im5hbWUiOiJkYXRhZGlyIiwibW91bnRQYXRoIjoiL3Zhci9saWIvbXlzcWwifSx7Im5hbWUiOiJiaW4iLCJtb3VudFBhdGgiOiIvb3B0L3BlcmNvbmEifV0sImltYWdlUHVsbFBvbGljeSI6IkFsd2F5cyJ9XSwiY29udGFpbmVycyI6W3sibmFtZSI6InBtbS1jbGllbnQiLCJpbWFnZSI6InBlcmNvbmFsYWIvcG1tLWNsaWVudDpkZXYtbGF0ZXN0IiwicG9ydHMiOlt7ImNvbnRhaW5lclBvcnQiOjc3Nzd9LHsiY29udGFpbmVyUG9ydCI6MzAxMDB9LHsiY29udGFpbmVyUG9ydCI6MzAxMDF9LHsiY29udGFpbmVyUG9ydCI6MzAxMDJ9LHsiY29udGFpbmVyUG9ydCI6MzAxMDN9LHsiY29udGFpbmVyUG9ydCI6MzAxMDR9LHsiY29udGFpbmVyUG9ydCI6MzAxMDV9XSwiZW52RnJvbSI6W3sic2VjcmV0UmVmIjp7Im5hbWUiOiJteS1lbnYtdmFyLXNlY3JldHMiLCJvcHRpb25hbCI6dHJ1ZX19XSwiZW52IjpbeyJuYW1lIjoiUE1NX1NFUlZFUiIsInZhbHVlIjoibW9uaXRvcmluZy1zZXJ2aWNlIn0seyJuYW1lIjoiUE1NX1VTRVIiLCJ2YWx1ZSI6ImFwaV9rZXkifSx7Im5hbWUiOiJQTU1fUEFTU1dPUkQiLCJ2YWx1ZUZyb20iOnsic2VjcmV0S2V5UmVmIjp7Im5hbWUiOiJpbnRlcm5hbC1tb25pdG9yaW5nIiwia2V5IjoicG1tc2VydmVya2V5In19fSx7Im5hbWUiOiJDTElFTlRfUE9SVF9MSVNURU4iLCJ2YWx1ZSI6Ijc3NzcifSx7Im5hbWUiOiJDTElFTlRfUE9SVF9NSU4iLCJ2YWx1ZSI6IjMwMTAwIn0seyJuYW1lIjoiQ0xJRU5UX1BPUlRfTUFYIiwidmFsdWUiOiIzMDEwNSJ9LHsibmFtZSI6IlBPRF9OQU1FIiwidmFsdWVGcm9tIjp7ImZpZWxkUmVmIjp7ImZpZWxkUGF0aCI6Im1ldGFkYXRhLm5hbWUifX19LHsibmFtZSI6IlBPRF9OQU1FU1BBU0UiLCJ2YWx1ZUZyb20iOnsiZmllbGRSZWYiOnsiZmllbGRQYXRoIjoibWV0YWRhdGEubmFtZXNwYWNlIn19fSx7Im5hbWUiOiJQTU1fQUdFTlRfU0VSVkVSX0FERFJFU1MiLCJ2YWx1ZSI6Im1vbml0b3Jpbmctc2VydmljZSJ9LHsibmFtZSI6IlBNTV9BR0VOVF9TRVJWRVJfVVNFUk5BTUUiLCJ2YWx1ZSI6ImFwaV9rZXkifSx7Im5hbWUiOiJQTU1fQUdFTlRfU0VSVkVSX1BBU1NXT1JEIiwidmFsdWVGcm9tIjp7InNlY3JldEtleVJlZiI6eyJuYW1lIjoiaW50ZXJuYWwtbW9uaXRvcmluZyIsImtleSI6InBtbXNlcnZlcmtleSJ9fX0seyJuYW1lIjoiUE1NX0FHRU5UX0xJU1RFTl9QT1JUIiwidmFsdWUiOiI3Nzc3In0seyJuYW1lIjoiUE1NX0FHRU5UX1BPUlRTX01JTiIsInZhbHVlIjoiMzAxMDAifSx7Im5hbWUiOiJQTU1fQUdFTlRfUE9SVFNfTUFYIiwidmFsdWUiOiIzMDEwNSJ9LHsibmFtZSI6IlBNTV9BR0VOVF9DT05GSUdfRklMRSIsInZhbHVl"..., + Annotations: map[string]string{ + AvailableReplicas: 0, + CollisionCount: nil, + CreationTimestamp: v1.Time{}, + CurrentReplicas: 0, + CurrentRevision: "", + Generation: 0, + ManagedFields: nil, + ObservedGeneration: 0, + PersistentVolumeClaimRetentionPolicy: nil, + PodManagementPolicy: "", + ReadyReplicas: 0, + Replicas: 0, + Replicas: &0, + ResourceVersion: "", + RevisionHistoryLimit: nil, + UID: "", + UpdateRevision: "", + UpdatedReplicas: 0, + }, - APIVersion: "v1", - Key: "pmmserver", - Scheme: "HTTP", - DefaultMode: &420, - DefaultMode: &420, - Protocol: "TCP", - Value: "$(POD_NAMESPASE)-$(POD_NAME)", - Value: "admin", - PeriodSeconds: 10, - TerminationMessagePath: "/dev/termination-log", - TerminationMessagePolicy: "File", - Phase: "Pending", - VolumeMode: &"Filesystem", - APIVersion: "apps/v1", - APIVersion: "apps/v1", - DNSPolicy: "ClusterFirst", - DeprecatedServiceAccount: "default", - FieldsType: "FieldsV1", - FieldsType: "FieldsV1", - FieldsV1: s`{"f:status":{"f:availableReplicas":{},"f:collisionCount":{},"f:c`..., - FieldsV1: s`{"f:metadata":{"f:annotations":{".":{},"f:percona.com/last-confi`..., - Manager: "kube-controller-manager", - Manager: "percona-xtradb-cluster-operator", - Operation: "Update", - Operation: "Update", - RestartPolicy: "Always", - SchedulerName: "default-scheduler", - SchedulerName: "default-scheduler", - SecurityContext: s"&PodSecurityContext{SELinuxOptions:nil,RunAsUser:nil,RunAsNonRoot:nil,SupplementalGroups:[],FSGroup:nil,RunAsGroup:nil,Sysctls:[]Sysctl{},WindowsOptions:nil,FSGroupChangePolicy:nil,SeccompProfile:nil,AppArmorProfile:nil,SupplementalGroupsPolicy:nil,SELinux"..., - Subresource: "status", - Time: s"2026-08-06 03:04:38 +0000 UTC", - Time: s"2026-08-06 03:06:51 +0000 UTC", - Time: s"2026-08-06 03:12:40 +0000 UTC", - Time: s"2026-08-06 03:14:38 +0000 UTC", - Time: s"2026-08-06 03:15:53 +0000 UTC", - Time: s"2026-08-06 03:17:39 +0000 UTC", - Time: s"2026-08-06 03:18:34 +0000 UTC", - Time: s"2026-08-06 03:20:21 +0000 UTC", - Time: s"2026-08-06 03:03:19 +0000 UTC", - Time: s"2026-08-06 03:11:48 +0000 UTC", - Time: s"2026-08-06 03:14:57 +0000 UTC", - Time: s"2026-08-06 03:17:40 +0000 UTC", - Time: s"2026-08-06 03:17:41 +0000 UTC", - TopologySpreadConstraints: nil, - "percona.com/last-config-hash": "eyJyZXBsaWNhcyI6Miwic2VsZWN0b3IiOnsibWF0Y2hMYWJlbHMiOnsiYXBwLmt1YmVybmV0ZXMuaW8vY29tcG9uZW50IjoiaGFwcm94eSIsImFwcC5rdWJlcm5ldGVzLmlvL2luc3RhbmNlIjoibW9uaXRvcmluZyIsImFwcC5rdWJlcm5ldGVzLmlvL21hbmFnZWQtYnkiOiJwZXJjb25hLXh0cmFkYi1jbHVzdGVyLW9wZXJhdG9yIiwiYXBw"..., - "percona.com/last-config-hash": "eyJyZXBsaWNhcyI6Miwic2VsZWN0b3IiOnsibWF0Y2hMYWJlbHMiOnsiYXBwLmt1YmVybmV0ZXMuaW8vY29tcG9uZW50IjoiaGFwcm94eSIsImFwcC5rdWJlcm5ldGVzLmlvL2luc3RhbmNlIjoibW9uaXRvcmluZyIsImFwcC5rdWJlcm5ldGVzLmlvL21hbmFnZWQtYnkiOiJwZXJjb25hLXh0cmFkYi1jbHVzdGVyLW9wZXJhdG9yIiwiYXBwLmt1YmVybmV0ZXMuaW8vbmFtZSI6InBlcmNvbmEteHRyYWRiLWNsdXN0ZXIiLCJhcHAua3ViZXJuZXRlcy5pby9wYXJ0LW9mIjoicGVyY29uYS14dHJhZGItY2x1c3RlciJ9fSwidGVtcGxhdGUiOnsibWV0YWRhdGEiOnsibGFiZWxzIjp7ImFwcC5rdWJlcm5ldGVzLmlvL2NvbXBvbmVudCI6ImhhcHJveHkiLCJhcHAua3ViZXJuZXRlcy5pby9pbnN0YW5jZSI6Im1vbml0b3JpbmciLCJhcHAua3ViZXJuZXRlcy5pby9tYW5hZ2VkLWJ5IjoicGVyY29uYS14dHJhZGItY2x1c3Rlci1vcGVyYXRvciIsImFwcC5rdWJlcm5ldGVzLmlvL25hbWUiOiJwZXJjb25hLXh0cmFkYi1jbHVzdGVyIiwiYXBwLmt1YmVybmV0ZXMuaW8vcGFydC1vZiI6InBlcmNvbmEteHRyYWRiLWNsdXN0ZXIifSwiYW5ub3RhdGlvbnMiOnsia3ViZWN0bC5rdWJlcm5ldGVzLmlvL2RlZmF1bHQtY29udGFpbmVyIjoiaGFwcm94eSIsInBlcmNvbmEuY29tL2NvbmZpZ3VyYXRpb24taGFzaCI6ImQ0MWQ4Y2Q5OGYwMGIyMDRlOTgwMDk5OGVjZjg0MjdlIiwicGVyY29uYS5jb20vZW52LXNlY3JldC1jb25maWctaGFzaCI6IjViYzQwM2E4ZDQ0MzI0YzFi"..., - "percona.com/last-config-hash": "eyJyZXBsaWNhcyI6Miwic2VsZWN0b3IiOnsibWF0Y2hMYWJlbHMiOnsiYXBwLmt1YmVybmV0ZXMuaW8vY29tcG9uZW50IjoiaGFwcm94eSIsImFwcC5rdWJlcm5ldGVzLmlvL2luc3RhbmNlIjoibW9uaXRvcmluZyIsImFwcC5rdWJlcm5ldGVzLmlvL21hbmFnZWQtYnkiOiJwZXJjb25hLXh0cmFkYi1jbHVzdGVyLW9wZXJhdG9yIiwiYXBwLmt1YmVybmV0ZXMuaW8vbmFtZSI6InBlcmNvbmEteHRyYWRiLWNsdXN0ZXIiLCJhcHAua3ViZXJuZXRlcy5pby9wYXJ0LW9mIjoicGVyY29uYS14dHJhZGItY2x1c3RlciJ9fSwidGVtcGxhdGUiOnsibWV0YWRhdGEiOnsibGFiZWxzIjp7ImFwcC5rdWJlcm5ldGVzLmlvL2NvbXBvbmVudCI6ImhhcHJveHkiLCJhcHAua3ViZXJuZXRlcy5pby9pbnN0YW5jZSI6Im1vbml0b3JpbmciLCJhcHAua3ViZXJuZXRlcy5pby9tYW5hZ2VkLWJ5IjoicGVyY29uYS14dHJhZGItY2x1c3Rlci1vcGVyYXRvciIsImFwcC5rdWJlcm5ldGVzLmlvL25hbWUiOiJwZXJjb25hLXh0cmFkYi1jbHVzdGVyIiwiYXBwLmt1YmVybmV0ZXMuaW8vcGFydC1vZiI6InBlcmNvbmEteHRyYWRiLWNsdXN0ZXIifSwiYW5ub3RhdGlvbnMiOnsia3ViZWN0bC5rdWJlcm5ldGVzLmlvL2RlZmF1bHQtY29udGFpbmVyIjoiaGFwcm94eSIsInBlcmNvbmEuY29tL2NvbmZpZ3VyYXRpb24taGFzaCI6ImQ0MWQ4Y2Q5OGYwMGIyMDRlOTgwMDk5OGVjZjg0MjdlIn19LCJzcGVjIjp7InZvbHVtZXMiOlt7Im5hbWUiOiJoYXByb3h5LWN1c3RvbSIsImNvbmZpZ01h"..., - "percona.com/last-config-hash": "eyJyZXBsaWNhcyI6Miwic2VsZWN0b3IiOnsibWF0Y2hMYWJlbHMiOnsiYXBwLmt1YmVybmV0ZXMuaW8vY29tcG9uZW50IjoiaGFwcm94eSIsImFwcC5rdWJlcm5ldGVzLmlvL2luc3RhbmNlIjoibW9uaXRvcmluZyIsImFwcC5rdWJlcm5ldGVzLmlvL21hbmFnZWQtYnkiOiJwZXJjb25hLXh0cmFkYi1jbHVzdGVyLW9wZXJhdG9yIiwiYXBwLmt1YmVybmV0ZXMuaW8vbmFtZSI6InBlcmNvbmEteHRyYWRiLWNsdXN0ZXIiLCJhcHAua3ViZXJuZXRlcy5pby9wYXJ0LW9mIjoicGVyY29uYS14dHJhZGItY2x1c3RlciJ9fSwidGVtcGxhdGUiOnsibWV0YWRhdGEiOnsibGFiZWxzIjp7ImFwcC5rdWJlcm5ldGVzLmlvL2NvbXBvbmVudCI6ImhhcHJveHkiLCJhcHAua3ViZXJuZXRlcy5pby9pbnN0YW5jZSI6Im1vbml0b3JpbmciLCJhcHAua3ViZXJuZXRlcy5pby9tYW5hZ2VkLWJ5IjoicGVyY29uYS14dHJhZGItY2x1c3Rlci1vcGVyYXRvciIsImFwcC5rdWJlcm5ldGVzLmlvL25hbWUiOiJwZXJjb25hLXh0cmFkYi1jbHVzdGVyIiwiYXBwLmt1YmVybmV0ZXMuaW8vcGFydC1vZiI6InBlcmNvbmEteHRyYWRiLWNsdXN0ZXIifSwiYW5ub3RhdGlvbnMiOnsia3ViZWN0bC5rdWJlcm5ldGVzLmlvL2RlZmF1bHQtY29udGFpbmVyIjoiaGFwcm94eSIsInBlcmNvbmEuY29tL2NvbmZpZ3VyYXRpb24taGFzaCI6ImQ0MWQ4Y2Q5OGYwMGIyMDRlOTgwMDk5OGVjZjg0MjdlIn19LCJzcGVjIjp7InZvbHVtZXMiOlt7Im5hbWUiOiJoYXByb3h5LWN1c3RvbSIsImNvbmZpZ01hcCI6eyJuYW1lIjoibW9uaXRvcmluZy1oYXByb3h5Iiwib3B0aW9uYWwiOnRydWV9fSx7Im5hbWUiOiJoYXByb3h5LWF1dG8iLCJlbXB0eURpciI6e319LHsibmFtZSI6Im15c3FsLXVzZXJzLXNlY3JldC1maWxlIiwic2VjcmV0Ijp7InNlY3JldE5hbWUiOiJpbnRlcm5hbC1tb25pdG9yaW5nIiwib3B0aW9uYWwiOmZhbHNlfX0seyJuYW1lIjoibXktZW52LXZhci1zZWNyZXRzIiwic2VjcmV0Ijp7InNlY3JldE5hbWUiOiJteS1lbnYtdmFyLXNlY3JldHMiLCJvcHRpb25hbCI6dHJ1ZX19LHsibmFtZSI6ImJpbiIsImVtcHR5RGlyIjp7fX1dLCJpbml0Q29udGFpbmVycyI6W3sibmFtZSI6InB4Yy1pbml0IiwiaW1hZ2UiOiJwZXJjb25hbGFiL3BlcmNvbmEteHRyYWRiLWNsdXN0ZXItb3BlcmF0b3I6UFItMjUyNS1jODFlNDdmMCIsImNvbW1hbmQiOlsiL3B4Yy1pbml0LWVudHJ5cG9pbnQuc2giXSwicmVzb3VyY2VzIjp7ImxpbWl0cyI6eyJjcHUiOiI1MG0iLCJtZW1vcnkiOiI1ME0ifX0sInZvbHVtZU1vdW50cyI6W3sibmFtZSI6ImJpbiIsIm1vdW50UGF0aCI6Ii92YXIvbGliL215c3FsIn1dLCJpbWFnZVB1bGxQb2xpY3kiOiJBbHdheXMifSx7Im5hbWUiOiJoYXByb3h5LWluaXQiLCJpbWFnZSI6InBlcmNvbmFsYWIvcGVyY29uYS14dHJhZGItY2x1c3Rlci1vcGVyYXRvcjpQUi0yNTI1LWM4MWU0N2YwIiwiY29tbWFuZCI6WyIvaGFwcm94eS1pbml0LWVudHJ5cG9pbnQuc2giXSwicmVzb3VyY2VzIjp7ImxpbWl0cyI6eyJjcHUiOiI1MG0iLCJtZW1vcnkiOiI1ME0ifX0sInZvbHVtZU1vdW50cyI6W3sibmFtZSI6ImJpbiIsIm1vdW50UGF0aCI6Ii9vcHQvcGVyY29uYSJ9XSwiaW1hZ2VQdWxsUG9saWN5IjoiQWx3YXlzIn1dLCJjb250YWluZXJzIjpbeyJuYW1lIjoicG1tLWNsaWVudCIsImltYWdlIjoicGVyY29uYWxhYi9wbW0tY2xpZW50OmRldi1sYXRlc3QiLCJwb3J0cyI6W3siY29udGFpbmVyUG9ydCI6Nzc3N30seyJjb250YWluZXJQb3J0IjozMDEwMH0seyJjb250YWluZXJQb3J0IjozMDEwMX0seyJjb250YWluZXJQb3J0IjozMDEwMn0seyJjb250YWluZXJQb3J0IjozMDEwM30seyJjb250YWluZXJQb3J0IjozMDEwNH0seyJjb250YWluZXJQb3J0IjozMDEwNX1dLCJlbnZGcm9tIjpbeyJzZWNyZXRSZWYiOnsibmFtZSI6Im15LWVudi12YXItc2VjcmV0cyIsIm9wdGlvbmFsIjp0cnVlfX1dLCJlbnYiOlt7Im5hbWUiOiJQTU1fU0VSVkVSIiwidmFsdWUiOiJtb25pdG9yaW5nLXNlcnZpY2UifSx7Im5hbWUiOiJQTU1fVVNFUiIsInZhbHVlIjoiYWRtaW4ifSx7Im5hbWUiOiJQTU1fUEFTU1dPUkQiLCJ2YWx1ZUZyb20iOnsic2VjcmV0S2V5UmVmIjp7Im5hbWUiOiJpbnRlcm5hbC1tb25pdG9yaW5nIiwia2V5IjoicG1tc2VydmVyIn19fSx7Im5hbWUiOiJDTElFTlRfUE9SVF9MSVNURU4iLCJ2YWx1ZSI6Ijc3NzcifSx7Im5hbWUiOiJDTElFTlRfUE9SVF9NSU4iLCJ2YWx1ZSI6IjMwMTAwIn0seyJuYW1lIjoiQ0xJRU5UX1BPUlRfTUFYIiwidmFsdWUiOiIzMDEwNSJ9LHsibmFtZSI6IlBPRF9OQU1FIiwidmFsdWVGcm9tIjp7ImZpZWxkUmVmIjp7ImZpZWxkUGF0aCI6Im1ldGFkYXRhLm5hbWUifX19LHsibmFtZSI6IlBPRF9OQU1FU1BBU0UiLCJ2YWx1ZUZyb20iOnsiZmllbGRSZWYiOnsiZmllbGRQYXRoIjoibWV0YWRhdGEubmFtZXNwYWNlIn19fSx7Im5hbWUiOiJQTU1fQUdFTlRfU0VSVkVSX0FERFJFU1MiLCJ2YWx1ZSI6Im1vbml0b3Jpbmctc2VydmljZSJ9LHsibmFtZSI6IlBNTV9BR0VOVF9TRVJWRVJfVVNFUk5BTUUiLCJ2YWx1ZSI6ImFkbWluIn0seyJuYW1lIjoiUE1NX0FHRU5UX1NFUlZFUl9QQVNTV09SRCIsInZhbHVlRnJvbSI6eyJzZWNyZXRLZXlSZWYiOnsibmFtZSI6ImludGVybmFsLW1vbml0b3JpbmciLCJrZXkiOiJwbW1zZXJ2ZXIifX19LHsibmFtZSI6IlBNTV9BR0VOVF9MSVNURU5fUE9SVCIsInZhbHVlIjoiNzc3NyJ9LHsibmFtZSI6IlBNTV9BR0VOVF9QT1JUU19NSU4iLCJ2YWx1ZSI6IjMwMTAwIn0seyJuYW1lIjoiUE1NX0FHRU5UX1BPUlRTX01BWCIsInZhbHVlIjoiMzAxMDUifSx7Im5hbWUiOiJQTU1fQUdFTlRfQ09ORklHX0ZJTEUiLCJ2YWx1ZSI6Ii91c3IvbG9jYWwvcGVyY29uYS9wbW0yL2NvbmZpZy9wbW0tYWdlbnQueWFtbCJ9LHsibmFtZSI6IlBNTV9BR0VOVF9TRVJWRVJfSU5TRUNVUkVfVExTIiwidmFsdWUiOiIxIn0seyJuYW1lIjoiUE1NX0FHRU5UX0xJU1RFTl9BRERSRVNTIiwidmFsdWUiOiIwLjAuMC4wIn0seyJuYW1lIjoiUE1NX0FHRU5UX1NFVFVQX01FVFJJQ1NfTU9ERSIsInZhbHVlIjoicHVz"..., - "percona.com/last-config-hash": "eyJyZXBsaWNhcyI6Mywic2VsZWN0b3IiOnsibWF0Y2hMYWJlbHMiOnsiYXBwLmt1YmVybmV0ZXMuaW8vY29tcG9uZW50IjoicHhjIiwiYXBwLmt1YmVybmV0ZXMuaW8vaW5zdGFuY2UiOiJtb25pdG9yaW5nIiwiYXBwLmt1YmVybmV0ZXMuaW8vbWFuYWdlZC1ieSI6InBlcmNvbmEteHRyYWRiLWNsdXN0ZXItb3BlcmF0b3IiLCJhcHAua3Vi"..., - "percona.com/last-config-hash": "eyJyZXBsaWNhcyI6Mywic2VsZWN0b3IiOnsibWF0Y2hMYWJlbHMiOnsiYXBwLmt1YmVybmV0ZXMuaW8vY29tcG9uZW50IjoicHhjIiwiYXBwLmt1YmVybmV0ZXMuaW8vaW5zdGFuY2UiOiJtb25pdG9yaW5nIiwiYXBwLmt1YmVybmV0ZXMuaW8vbWFuYWdlZC1ieSI6InBlcmNvbmEteHRyYWRiLWNsdXN0ZXItb3BlcmF0b3IiLCJhcHAua3ViZXJuZXRlcy5pby9uYW1lIjoicGVyY29uYS14dHJhZGItY2x1c3RlciIsImFwcC5rdWJlcm5ldGVzLmlvL3BhcnQtb2YiOiJwZXJjb25hLXh0cmFkYi1jbHVzdGVyIn19LCJ0ZW1wbGF0ZSI6eyJtZXRhZGF0YSI6eyJsYWJlbHMiOnsiYXBwLmt1YmVybmV0ZXMuaW8vY29tcG9uZW50IjoicHhjIiwiYXBwLmt1YmVybmV0ZXMuaW8vaW5zdGFuY2UiOiJtb25pdG9yaW5nIiwiYXBwLmt1YmVybmV0ZXMuaW8vbWFuYWdlZC1ieSI6InBlcmNvbmEteHRyYWRiLWNsdXN0ZXItb3BlcmF0b3IiLCJhcHAua3ViZXJuZXRlcy5pby9uYW1lIjoicGVyY29uYS14dHJhZGItY2x1c3RlciIsImFwcC5rdWJlcm5ldGVzLmlvL3BhcnQtb2YiOiJwZXJjb25hLXh0cmFkYi1jbHVzdGVyIn0sImFubm90YXRpb25zIjp7Imt1YmVjdGwua3ViZXJuZXRlcy5pby9kZWZhdWx0LWNvbnRhaW5lciI6InB4YyIsInBlcmNvbmEuY29tL2NvbmZpZ3VyYXRpb24taGFzaCI6ImQ0MWQ4Y2Q5OGYwMGIyMDRlOTgwMDk5OGVjZjg0MjdlIiwicGVyY29uYS5jb20vZW52LXNlY3JldC1jb25maWctaGFzaCI6IjViYzQwM2E4ZDQ0MzI0YzFiYzcwNGYwMmU3NDg3"..., - "percona.com/last-config-hash": "eyJyZXBsaWNhcyI6Mywic2VsZWN0b3IiOnsibWF0Y2hMYWJlbHMiOnsiYXBwLmt1YmVybmV0ZXMuaW8vY29tcG9uZW50IjoicHhjIiwiYXBwLmt1YmVybmV0ZXMuaW8vaW5zdGFuY2UiOiJtb25pdG9yaW5nIiwiYXBwLmt1YmVybmV0ZXMuaW8vbWFuYWdlZC1ieSI6InBlcmNvbmEteHRyYWRiLWNsdXN0ZXItb3BlcmF0b3IiLCJhcHAua3ViZXJuZXRlcy5pby9uYW1lIjoicGVyY29uYS14dHJhZGItY2x1c3RlciIsImFwcC5rdWJlcm5ldGVzLmlvL3BhcnQtb2YiOiJwZXJjb25hLXh0cmFkYi1jbHVzdGVyIn19LCJ0ZW1wbGF0ZSI6eyJtZXRhZGF0YSI6eyJsYWJlbHMiOnsiYXBwLmt1YmVybmV0ZXMuaW8vY29tcG9uZW50IjoicHhjIiwiYXBwLmt1YmVybmV0ZXMuaW8vaW5zdGFuY2UiOiJtb25pdG9yaW5nIiwiYXBwLmt1YmVybmV0ZXMuaW8vbWFuYWdlZC1ieSI6InBlcmNvbmEteHRyYWRiLWNsdXN0ZXItb3BlcmF0b3IiLCJhcHAua3ViZXJuZXRlcy5pby9uYW1lIjoicGVyY29uYS14dHJhZGItY2x1c3RlciIsImFwcC5rdWJlcm5ldGVzLmlvL3BhcnQtb2YiOiJwZXJjb25hLXh0cmFkYi1jbHVzdGVyIn0sImFubm90YXRpb25zIjp7Imt1YmVjdGwua3ViZXJuZXRlcy5pby9kZWZhdWx0LWNvbnRhaW5lciI6InB4YyIsInBlcmNvbmEuY29tL2NvbmZpZ3VyYXRpb24taGFzaCI6ImQ0MWQ4Y2Q5OGYwMGIyMDRlOTgwMDk5OGVjZjg0MjdlIiwicGVyY29uYS5jb20vc3NsLWhhc2giOiI1NjcwN2FkYWQ3ZjE0OTI3ODJlYmU5YWFhNjE3ZDA4ZSIsInBlcmNvbmEu"..., - "percona.com/last-config-hash": "eyJyZXBsaWNhcyI6Mywic2VsZWN0b3IiOnsibWF0Y2hMYWJlbHMiOnsiYXBwLmt1YmVybmV0ZXMuaW8vY29tcG9uZW50IjoicHhjIiwiYXBwLmt1YmVybmV0ZXMuaW8vaW5zdGFuY2UiOiJtb25pdG9yaW5nIiwiYXBwLmt1YmVybmV0ZXMuaW8vbWFuYWdlZC1ieSI6InBlcmNvbmEteHRyYWRiLWNsdXN0ZXItb3BlcmF0b3IiLCJhcHAua3ViZXJuZXRlcy5pby9uYW1lIjoicGVyY29uYS14dHJhZGItY2x1c3RlciIsImFwcC5rdWJlcm5ldGVzLmlvL3BhcnQtb2YiOiJwZXJjb25hLXh0cmFkYi1jbHVzdGVyIn19LCJ0ZW1wbGF0ZSI6eyJtZXRhZGF0YSI6eyJsYWJlbHMiOnsiYXBwLmt1YmVybmV0ZXMuaW8vY29tcG9uZW50IjoicHhjIiwiYXBwLmt1YmVybmV0ZXMuaW8vaW5zdGFuY2UiOiJtb25pdG9yaW5nIiwiYXBwLmt1YmVybmV0ZXMuaW8vbWFuYWdlZC1ieSI6InBlcmNvbmEteHRyYWRiLWNsdXN0ZXItb3BlcmF0b3IiLCJhcHAua3ViZXJuZXRlcy5pby9uYW1lIjoicGVyY29uYS14dHJhZGItY2x1c3RlciIsImFwcC5rdWJlcm5ldGVzLmlvL3BhcnQtb2YiOiJwZXJjb25hLXh0cmFkYi1jbHVzdGVyIn0sImFubm90YXRpb25zIjp7Imt1YmVjdGwua3ViZXJuZXRlcy5pby9kZWZhdWx0LWNvbnRhaW5lciI6InB4YyIsInBlcmNvbmEuY29tL2NvbmZpZ3VyYXRpb24taGFzaCI6ImQ0MWQ4Y2Q5OGYwMGIyMDRlOTgwMDk5OGVjZjg0MjdlIiwicGVyY29uYS5jb20vc3NsLWhhc2giOiI1NjcwN2FkYWQ3ZjE0OTI3ODJlYmU5YWFhNjE3ZDA4ZSIsInBlcmNvbmEuY29tL3NzbC1pbnRlcm5hbC1oYXNoIjoiZTc1ODcwMjkxODQxZmY3ODkzOWU4ZmM5MzUwNjE4YTQifX0sInNwZWMiOnsidm9sdW1lcyI6W3sibmFtZSI6InRtcCIsImVtcHR5RGlyIjp7fX0seyJuYW1lIjoiY29uZmlnIiwiY29uZmlnTWFwIjp7Im5hbWUiOiJtb25pdG9yaW5nLXB4YyIsIm9wdGlvbmFsIjp0cnVlfX0seyJuYW1lIjoic3NsLWludGVybmFsIiwic2VjcmV0Ijp7InNlY3JldE5hbWUiOiJtb25pdG9yaW5nLXNzbC1pbnRlcm5hbCIsIm9wdGlvbmFsIjp0cnVlfX0seyJuYW1lIjoic3NsIiwic2VjcmV0Ijp7InNlY3JldE5hbWUiOiJzb21lLW5hbWUtc3NsIiwib3B0aW9uYWwiOmZhbHNlfX0seyJuYW1lIjoiYXV0by1jb25maWciLCJjb25maWdNYXAiOnsibmFtZSI6ImF1dG8tbW9uaXRvcmluZy1weGMiLCJvcHRpb25hbCI6dHJ1ZX19LHsibmFtZSI6InZhdWx0LWtleXJpbmctc2VjcmV0Iiwic2VjcmV0Ijp7InNlY3JldE5hbWUiOiJtb25pdG9yaW5nLXZhdWx0Iiwib3B0aW9uYWwiOnRydWV9fSx7Im5hbWUiOiJteXNxbC11c2Vycy1zZWNyZXQtZmlsZSIsInNlY3JldCI6eyJzZWNyZXROYW1lIjoiaW50ZXJuYWwtbW9uaXRvcmluZyIsIm9wdGlvbmFsIjpmYWxzZX19LHsibmFtZSI6Im15c3FsLWluaXQtZmlsZSIsInNlY3JldCI6eyJzZWNyZXROYW1lIjoibW9uaXRvcmluZy1teXNxbC1pbml0Iiwib3B0aW9uYWwiOnRydWV9fSx7Im5hbWUiOiJiaW4iLCJlbXB0eURpciI6e319XSwiaW5pdENvbnRhaW5lcnMiOlt7Im5hbWUiOiJweGMtaW5pdCIsImltYWdlIjoicGVyY29uYWxhYi9wZXJjb25hLXh0cmFkYi1jbHVzdGVyLW9wZXJhdG9yOlBSLTI1MjUtYzgxZTQ3ZjAiLCJjb21tYW5kIjpbIi9weGMtaW5pdC1lbnRyeXBvaW50LnNoIl0sInJlc291cmNlcyI6eyJsaW1pdHMiOnsiY3B1IjoiNTBtIiwibWVtb3J5IjoiNTBNIn19LCJ2b2x1bWVNb3VudHMiOlt7Im5hbWUiOiJkYXRhZGlyIiwibW91bnRQYXRoIjoiL3Zhci9saWIvbXlzcWwifSx7Im5hbWUiOiJiaW4iLCJtb3VudFBhdGgiOiIvb3B0L3BlcmNvbmEifV0sImltYWdlUHVsbFBvbGljeSI6IkFsd2F5cyJ9XSwiY29udGFpbmVycyI6W3sibmFtZSI6InBtbS1jbGllbnQiLCJpbWFnZSI6InBlcmNvbmFsYWIvcG1tLWNsaWVudDpkZXYtbGF0ZXN0IiwicG9ydHMiOlt7ImNvbnRhaW5lclBvcnQiOjc3Nzd9LHsiY29udGFpbmVyUG9ydCI6MzAxMDB9LHsiY29udGFpbmVyUG9ydCI6MzAxMDF9LHsiY29udGFpbmVyUG9ydCI6MzAxMDJ9LHsiY29udGFpbmVyUG9ydCI6MzAxMDN9LHsiY29udGFpbmVyUG9ydCI6MzAxMDR9LHsiY29udGFpbmVyUG9ydCI6MzAxMDV9XSwiZW52RnJvbSI6W3sic2VjcmV0UmVmIjp7Im5hbWUiOiJteS1lbnYtdmFyLXNlY3JldHMiLCJvcHRpb25hbCI6dHJ1ZX19XSwiZW52IjpbeyJuYW1lIjoiUE1NX1NFUlZFUiIsInZhbHVlIjoibW9uaXRvcmluZy1zZXJ2aWNlIn0seyJuYW1lIjoiUE1NX1VTRVIiLCJ2YWx1ZSI6ImFkbWluIn0seyJuYW1lIjoiUE1NX1BBU1NXT1JEIiwidmFsdWVGcm9tIjp7InNlY3JldEtleVJlZiI6eyJuYW1lIjoiaW50ZXJuYWwtbW9uaXRvcmluZyIsImtleSI6InBtbXNlcnZlciJ9fX0seyJuYW1lIjoiQ0xJRU5UX1BPUlRfTElTVEVOIiwidmFsdWUiOiI3Nzc3In0seyJuYW1lIjoiQ0xJRU5UX1BPUlRfTUlOIiwidmFsdWUiOiIzMDEwMCJ9LHsibmFtZSI6IkNMSUVOVF9QT1JUX01BWCIsInZhbHVlIjoiMzAxMDUifSx7Im5hbWUiOiJQT0RfTkFNRSIsInZhbHVlRnJvbSI6eyJmaWVsZFJlZiI6eyJmaWVsZFBhdGgiOiJtZXRhZGF0YS5uYW1lIn19fSx7Im5hbWUiOiJQT0RfTkFNRVNQQVNFIiwidmFsdWVGcm9tIjp7ImZpZWxkUmVmIjp7ImZpZWxkUGF0aCI6Im1ldGFkYXRhLm5hbWVzcGFjZSJ9fX0seyJuYW1lIjoiUE1NX0FHRU5UX1NFUlZFUl9BRERSRVNTIiwidmFsdWUiOiJtb25pdG9yaW5nLXNlcnZpY2UifSx7Im5hbWUiOiJQTU1fQUdFTlRfU0VSVkVSX1VTRVJOQU1FIiwidmFsdWUiOiJhZG1pbiJ9LHsibmFtZSI6IlBNTV9BR0VOVF9TRVJWRVJfUEFTU1dPUkQiLCJ2YWx1ZUZyb20iOnsic2VjcmV0S2V5UmVmIjp7Im5hbWUiOiJpbnRlcm5hbC1tb25pdG9yaW5nIiwia2V5IjoicG1tc2VydmVyIn19fSx7Im5hbWUiOiJQTU1fQUdFTlRfTElTVEVOX1BPUlQiLCJ2YWx1ZSI6Ijc3NzcifSx7Im5hbWUiOiJQTU1fQUdFTlRfUE9SVFNfTUlOIiwidmFsdWUiOiIzMDEwMCJ9LHsibmFtZSI6IlBNTV9BR0VOVF9QT1JUU19NQVgiLCJ2YWx1ZSI6IjMwMTA1In0seyJuYW1lIjoiUE1NX0FHRU5UX0NPTkZJR19GSUxFIiwidmFsdWUiOiIvdXNyL2xv"..., - { - }, - Annotations: map[string]string{ - AvailableReplicas: 2, - AvailableReplicas: 3, - CollisionCount: &0, - CreationTimestamp: v1.Time{Time: s"2026-08-06 03:03:19 +0000 UTC"}, - CurrentReplicas: 2, - CurrentReplicas: 3, - CurrentRevision: "monitoring-haproxy-6b767554d5", - CurrentRevision: "monitoring-haproxy-76cd687c56", - CurrentRevision: "monitoring-haproxy-7bc4d8df8d", - CurrentRevision: "monitoring-haproxy-7fddc6b894", - CurrentRevision: "monitoring-pxc-6789465c7", - CurrentRevision: "monitoring-pxc-6c4f55d55", - CurrentRevision: "monitoring-pxc-787fc9f6cc", - CurrentRevision: "monitoring-pxc-7b94777f8b", - Generation: 1, - Generation: 2, - Generation: 3, - Generation: 4, - ManagedFields: []v1.ManagedFieldsEntry{ - ObservedGeneration: 1, - ObservedGeneration: 2, - ObservedGeneration: 3, - ObservedGeneration: 4, - PersistentVolumeClaimRetentionPolicy: s"&StatefulSetPersistentVolumeClaimRetentionPolicy{WhenDeleted:Retain,WhenScaled:Retain,}", - PodManagementPolicy: "OrderedReady", - ReadyReplicas: 2, - ReadyReplicas: 3, - Replicas: 2, - Replicas: 3, - Replicas: &2, - Replicas: &3, - ResourceVersion: "1785985478875583001", - ResourceVersion: "1785985611686847014", - ResourceVersion: "1785985960387839001", - ResourceVersion: "1785986078908319014", - ResourceVersion: "1785986153734047001", - ResourceVersion: "1785986259244639014", - ResourceVersion: "1785986314051183001", - ResourceVersion: "1785986421482143014", - RevisionHistoryLimit: &10, - UID: "37cae3fb-3443-4e14-816d-e52712c25590", - UID: "caab1560-bf70-4edc-adb2-6724645d7633", - UpdateRevision: "monitoring-haproxy-6b767554d5", - UpdateRevision: "monitoring-haproxy-76cd687c56", - UpdateRevision: "monitoring-haproxy-7bc4d8df8d", - UpdateRevision: "monitoring-haproxy-7fddc6b894", - UpdateRevision: "monitoring-pxc-6789465c7", - UpdateRevision: "monitoring-pxc-6c4f55d55", - UpdateRevision: "monitoring-pxc-787fc9f6cc", - UpdateRevision: "monitoring-pxc-7b94777f8b", - UpdatedReplicas: 2, - UpdatedReplicas: 3, - }, 2026-08-06T03:01:14.912Z INFO setup Manager starting up {"gitCommit": "c81e47f0baf68eddc8c1664c722d106d38b61d55", "gitBranch": "PR-2525-c81e47f0", "buildTime": "2026-08-06T01:12:58Z", "goVersion": "go1.26.5", "os": "linux", "arch": "amd64"} 2026-08-06T03:01:14.912Z INFO setup Runs on {"platform": "kubernetes", "version": "v1.33.12-gke.1270000"} 2026-08-06T03:01:14.913Z INFO setup Feature gates {"PXCO_FEATURE_GATES": "", "enabled": ""} 2026-08-06T03:01:14.916Z INFO setup Registering Components. 2026-08-06T03:01:15.405Z INFO controller-runtime.certwatcher Updated current TLS certificate {"cert": "/tmp/k8s-webhook-server/serving-certs/tls.crt", "key": "/tmp/k8s-webhook-server/serving-certs/tls.key"} 2026-08-06T03:01:15.405Z INFO controller-runtime.metrics Serving metrics server {"bindAddress": ":8080", "secure": false} 2026-08-06T03:01:15.405Z INFO controller-runtime.metrics Starting metrics server 2026-08-06T03:01:15.405Z INFO controller-runtime.webhook Registering webhook {"path": "/validate-percona-xtradbcluster"} 2026-08-06T03:01:15.405Z INFO controller-runtime.webhook Starting webhook server 2026-08-06T03:01:15.405Z INFO setup Starting the Cmd. 2026-08-06T03:01:15.405Z INFO starting server {"name": "health probe", "addr": "[::]:8081"} 2026-08-06T03:01:15.406Z INFO controller-runtime.certwatcher Starting certificate poll+watcher {"cert": "/tmp/k8s-webhook-server/serving-certs/tls.crt", "key": "/tmp/k8s-webhook-server/serving-certs/tls.key", "interval": "10s"} 2026-08-06T03:01:15.406Z INFO controller-runtime.webhook Serving webhook server {"host": "", "port": 9443} 2026-08-06T03:01:15.506Z INFO Attempting to acquire leader lease... {"lock": "pxc-operator/08db1feb.percona.com"} 2026-08-06T03:01:15.565Z DEBUG events percona-xtradb-cluster-operator-6699755d74-jq8cr_a5aefafa-12ae-444d-af87-fb1dcb326371 became leader {"type": "Normal", "object": {"kind":"Lease","namespace":"pxc-operator","name":"08db1feb.percona.com","uid":"f6c264df-c001-4b1b-ab4c-7615f73ffa74","apiVersion":"coordination.k8s.io/v1","resourceVersion":"1785985275558319009"}, "reason": "LeaderElection"} 2026-08-06T03:01:15.565Z INFO Starting EventSource {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "source": "kind source: *v1.PerconaXtraDBCluster"} 2026-08-06T03:01:15.565Z INFO Starting EventSource {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "source": "kind source: *v1.Secret"} 2026-08-06T03:01:15.565Z INFO Starting EventSource {"controller": "pxcrestore-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBClusterRestore", "source": "kind source: *v1.PerconaXtraDBClusterRestore"} 2026-08-06T03:01:15.565Z INFO Successfully acquired lease {"lock": "pxc-operator/08db1feb.percona.com"} 2026-08-06T03:01:15.566Z INFO Starting EventSource {"controller": "pxcbackup-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBClusterBackup", "source": "kind source: *v1.PerconaXtraDBClusterBackup"} 2026-08-06T03:01:15.666Z INFO Starting Controller {"controller": "pxcbackup-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBClusterBackup"} 2026-08-06T03:01:15.666Z INFO Starting workers {"controller": "pxcbackup-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBClusterBackup", "worker count": 1} 2026-08-06T03:01:15.767Z INFO Starting Controller {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster"} 2026-08-06T03:01:15.767Z INFO Starting Controller {"controller": "pxcrestore-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBClusterRestore"} 2026-08-06T03:01:15.767Z INFO Starting workers {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "worker count": 1} 2026-08-06T03:01:15.767Z INFO Starting workers {"controller": "pxcrestore-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBClusterRestore", "worker count": 1} 2026-08-06T03:03:18.425Z INFO Set CR version {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"monitoring","namespace":"monitoring-2-0-9920"}, "namespace": "monitoring-2-0-9920", "name": "monitoring", "reconcileID": "011e7185-c800-4cd7-8373-c65bd613c221", "version": "1.21.0"} 2026-08-06T03:03:19.801Z DEBUG Creating object {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"monitoring","namespace":"monitoring-2-0-9920"}, "namespace": "monitoring-2-0-9920", "name": "monitoring", "reconcileID": "011e7185-c800-4cd7-8373-c65bd613c221", "object": "monitoring-pxc", "kind": "&TypeMeta{Kind:StatefulSet,APIVersion:apps/v1,}"} 2026-08-06T03:03:19.872Z DEBUG Creating object {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"monitoring","namespace":"monitoring-2-0-9920"}, "namespace": "monitoring-2-0-9920", "name": "monitoring", "reconcileID": "011e7185-c800-4cd7-8373-c65bd613c221", "object": "monitoring-haproxy", "kind": "&TypeMeta{Kind:StatefulSet,APIVersion:apps/v1,}"} 2026-08-06T03:03:19.973Z DEBUG Creating object {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"monitoring","namespace":"monitoring-2-0-9920"}, "namespace": "monitoring-2-0-9920", "name": "monitoring", "reconcileID": "011e7185-c800-4cd7-8373-c65bd613c221", "object": "monitoring-pxc", "kind": "&TypeMeta{Kind:Service,APIVersion:v1,}"} 2026-08-06T03:03:20.049Z DEBUG Creating object {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"monitoring","namespace":"monitoring-2-0-9920"}, "namespace": "monitoring-2-0-9920", "name": "monitoring", "reconcileID": "011e7185-c800-4cd7-8373-c65bd613c221", "object": "monitoring-pxc-unready", "kind": "&TypeMeta{Kind:Service,APIVersion:v1,}"} 2026-08-06T03:03:20.116Z DEBUG Creating object {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"monitoring","namespace":"monitoring-2-0-9920"}, "namespace": "monitoring-2-0-9920", "name": "monitoring", "reconcileID": "011e7185-c800-4cd7-8373-c65bd613c221", "object": "monitoring-haproxy", "kind": "&TypeMeta{Kind:Service,APIVersion:v1,}"} 2026-08-06T03:03:20.322Z DEBUG Creating object {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"monitoring","namespace":"monitoring-2-0-9920"}, "namespace": "monitoring-2-0-9920", "name": "monitoring", "reconcileID": "011e7185-c800-4cd7-8373-c65bd613c221", "object": "monitoring-haproxy-replicas", "kind": "&TypeMeta{Kind:Service,APIVersion:v1,}"} 2026-08-06T03:03:21.469Z DEBUG Creating object {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"monitoring","namespace":"monitoring-2-0-9920"}, "namespace": "monitoring-2-0-9920", "name": "monitoring", "reconcileID": "c2e2022e-dbb6-4227-91a6-45afd3bbe9c1", "object": "monitoring-pxc", "kind": "&TypeMeta{Kind:PodDisruptionBudget,APIVersion:policy/v1,}"} 2026-08-06T03:03:21.498Z DEBUG Creating object {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"monitoring","namespace":"monitoring-2-0-9920"}, "namespace": "monitoring-2-0-9920", "name": "monitoring", "reconcileID": "c2e2022e-dbb6-4227-91a6-45afd3bbe9c1", "object": "monitoring-haproxy", "kind": "&TypeMeta{Kind:PodDisruptionBudget,APIVersion:policy/v1,}"} 2026-08-06T03:04:12.400Z INFO Password expiration policy updated {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"monitoring","namespace":"monitoring-2-0-9920"}, "namespace": "monitoring-2-0-9920", "name": "monitoring", "reconcileID": "55e676d8-1265-4b0e-9781-5c114ad3d0da", "user": "operator"} 2026-08-06T03:04:12.486Z INFO Password expiration policy updated {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"monitoring","namespace":"monitoring-2-0-9920"}, "namespace": "monitoring-2-0-9920", "name": "monitoring", "reconcileID": "55e676d8-1265-4b0e-9781-5c114ad3d0da", "user": "monitor"} 2026-08-06T03:04:12.539Z INFO User monitor: granted privileges {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"monitoring","namespace":"monitoring-2-0-9920"}, "namespace": "monitoring-2-0-9920", "name": "monitoring", "reconcileID": "55e676d8-1265-4b0e-9781-5c114ad3d0da"} 2026-08-06T03:04:12.570Z INFO monitor user privileges granted {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"monitoring","namespace":"monitoring-2-0-9920"}, "namespace": "monitoring-2-0-9920", "name": "monitoring", "reconcileID": "55e676d8-1265-4b0e-9781-5c114ad3d0da"} 2026-08-06T03:04:12.605Z INFO Password expiration policy updated {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"monitoring","namespace":"monitoring-2-0-9920"}, "namespace": "monitoring-2-0-9920", "name": "monitoring", "reconcileID": "55e676d8-1265-4b0e-9781-5c114ad3d0da", "user": "xtrabackup"} 2026-08-06T03:04:12.647Z INFO User xtrabackup: granted privileges {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"monitoring","namespace":"monitoring-2-0-9920"}, "namespace": "monitoring-2-0-9920", "name": "monitoring", "reconcileID": "55e676d8-1265-4b0e-9781-5c114ad3d0da"} 2026-08-06T03:04:12.682Z INFO Password expiration policy updated {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"monitoring","namespace":"monitoring-2-0-9920"}, "namespace": "monitoring-2-0-9920", "name": "monitoring", "reconcileID": "55e676d8-1265-4b0e-9781-5c114ad3d0da", "user": "replication"} 2026-08-06T03:06:57.537Z INFO Password expiration policy updated {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"monitoring","namespace":"monitoring-2-0-9920"}, "namespace": "monitoring-2-0-9920", "name": "monitoring", "reconcileID": "ad0ae5a1-1c1f-4429-ae6f-ddb24accc3e2", "user": "root"} 2026-08-06T03:06:57.623Z INFO update PXC version (fetched from db) {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"monitoring","namespace":"monitoring-2-0-9920"}, "namespace": "monitoring-2-0-9920", "name": "monitoring", "reconcileID": "ad0ae5a1-1c1f-4429-ae6f-ddb24accc3e2", "new version": "8.0.43-34.1"} 2026-08-06T03:11:43.034Z INFO Internal secrets updated {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"monitoring","namespace":"monitoring-2-0-9920"}, "namespace": "monitoring-2-0-9920", "name": "monitoring", "reconcileID": "27af049e-dff4-4141-ab78-3635eb065d8d", "user": "pmmserverkey"} 2026-08-06T03:11:48.628Z DEBUG Updating object {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"monitoring","namespace":"monitoring-2-0-9920"}, "namespace": "monitoring-2-0-9920", "name": "monitoring", "reconcileID": "0d42b74a-7503-4a07-b14d-13f38b562b06", "object": "monitoring-pxc", "kind": "&TypeMeta{Kind:StatefulSet,APIVersion:apps/v1,}", "hashChanged": true, "metaChanged": true} 2026-08-06T03:11:48.758Z DEBUG Updating object {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"monitoring","namespace":"monitoring-2-0-9920"}, "namespace": "monitoring-2-0-9920", "name": "monitoring", "reconcileID": "0d42b74a-7503-4a07-b14d-13f38b562b06", "object": "monitoring-haproxy", "kind": "&TypeMeta{Kind:StatefulSet,APIVersion:apps/v1,}", "hashChanged": true, "metaChanged": true} 2026-08-06T03:11:48.870Z DEBUG Updating object {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"monitoring","namespace":"monitoring-2-0-9920"}, "namespace": "monitoring-2-0-9920", "name": "monitoring", "reconcileID": "0d42b74a-7503-4a07-b14d-13f38b562b06", "object": "monitoring-haproxy", "kind": "&TypeMeta{Kind:StatefulSet,APIVersion:apps/v1,}", "hashChanged": true, "metaChanged": true} 2026-08-06T03:11:49.363Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"monitoring","namespace":"monitoring-2-0-9920"}, "namespace": "monitoring-2-0-9920", "name": "monitoring", "reconcileID": "0d42b74a-7503-4a07-b14d-13f38b562b06", "err": "failed to ensure cluster readonly status: connect to pod monitoring-pxc-2: dial tcp: lookup monitoring-pxc-2.monitoring-pxc.monitoring-2-0-9920 on 34.118.224.10:53: no such host"} 2026-08-06T03:13:00.786Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"monitoring","namespace":"monitoring-2-0-9920"}, "namespace": "monitoring-2-0-9920", "name": "monitoring", "reconcileID": "044d581d-2cc0-421b-8615-f28e89445bb4", "err": "failed to ensure cluster readonly status: connect to pod monitoring-pxc-1: dial tcp 10.32.32.28:33062: connect: connection refused"} 2026-08-06T03:13:43.154Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"monitoring","namespace":"monitoring-2-0-9920"}, "namespace": "monitoring-2-0-9920", "name": "monitoring", "reconcileID": "521045bf-ae34-480d-a261-d30944d3bca3", "err": "failed to connect to pod monitoring-pxc-0: dial tcp: lookup monitoring-pxc-0.monitoring-pxc.monitoring-2-0-9920 on 34.118.224.10:53: no such host"} 2026-08-06T03:13:43.960Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"monitoring","namespace":"monitoring-2-0-9920"}, "namespace": "monitoring-2-0-9920", "name": "monitoring", "reconcileID": "4ba1f406-145f-4381-aa6a-15cdf6c78ad6", "err": "failed to connect to pod monitoring-pxc-0: dial tcp: lookup monitoring-pxc-0.monitoring-pxc.monitoring-2-0-9920 on 34.118.224.10:53: no such host"} 2026-08-06T03:13:49.250Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"monitoring","namespace":"monitoring-2-0-9920"}, "namespace": "monitoring-2-0-9920", "name": "monitoring", "reconcileID": "7c51066d-420a-4530-aa1a-acb582c841b3", "err": "failed to ensure cluster readonly status: connect to pod monitoring-pxc-0: dial tcp: lookup monitoring-pxc-0.monitoring-pxc.monitoring-2-0-9920 on 34.118.224.10:53: no such host"} 2026-08-06T03:14:57.869Z DEBUG Updating object {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"monitoring","namespace":"monitoring-2-0-9920"}, "namespace": "monitoring-2-0-9920", "name": "monitoring", "reconcileID": "e59a3cbc-4104-4eea-bcbe-2422ffb978e0", "object": "monitoring-pxc", "kind": "&TypeMeta{Kind:StatefulSet,APIVersion:apps/v1,}", "hashChanged": true, "metaChanged": true} 2026-08-06T03:14:57.948Z DEBUG Updating object {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"monitoring","namespace":"monitoring-2-0-9920"}, "namespace": "monitoring-2-0-9920", "name": "monitoring", "reconcileID": "e59a3cbc-4104-4eea-bcbe-2422ffb978e0", "object": "monitoring-haproxy", "kind": "&TypeMeta{Kind:StatefulSet,APIVersion:apps/v1,}", "hashChanged": true, "metaChanged": true} 2026-08-06T03:14:58.621Z DEBUG Updating object {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"monitoring","namespace":"monitoring-2-0-9920"}, "namespace": "monitoring-2-0-9920", "name": "monitoring", "reconcileID": "e59a3cbc-4104-4eea-bcbe-2422ffb978e0", "object": "monitoring-haproxy", "kind": "&TypeMeta{Kind:StatefulSet,APIVersion:apps/v1,}", "hashChanged": true, "metaChanged": true} 2026-08-06T03:16:05.418Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"monitoring","namespace":"monitoring-2-0-9920"}, "namespace": "monitoring-2-0-9920", "name": "monitoring", "reconcileID": "3f2d393d-287d-4e3d-80f6-9d5cdf47b19c", "err": "failed to ensure cluster readonly status: connect to pod monitoring-pxc-1: dial tcp: lookup monitoring-pxc-1.monitoring-pxc.monitoring-2-0-9920 on 34.118.224.10:53: no such host"} 2026-08-06T03:16:58.703Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"monitoring","namespace":"monitoring-2-0-9920"}, "namespace": "monitoring-2-0-9920", "name": "monitoring", "reconcileID": "e28baaf4-7f59-4275-81a3-5374a9a0fd3d", "err": "failed to ensure cluster readonly status: connect to pod monitoring-pxc-0: dial tcp 10.32.33.31:33062: connect: connection refused"} 2026-08-06T03:17:40.938Z INFO Password changed, updating user {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"monitoring","namespace":"monitoring-2-0-9920"}, "namespace": "monitoring-2-0-9920", "name": "monitoring", "reconcileID": "aa343a08-48e2-4759-b47d-1255b0805835", "user": "pmmserverkey"} 2026-08-06T03:17:40.961Z INFO HAProxy pods will be restarted {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"monitoring","namespace":"monitoring-2-0-9920"}, "namespace": "monitoring-2-0-9920", "name": "monitoring", "reconcileID": "aa343a08-48e2-4759-b47d-1255b0805835", "last-applied-secret": "efd87259fad0d07e7d06001e143e1653756066643d7b4398610e759bb4c15110"} 2026-08-06T03:17:40.961Z INFO Internal secrets updated {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"monitoring","namespace":"monitoring-2-0-9920"}, "namespace": "monitoring-2-0-9920", "name": "monitoring", "reconcileID": "aa343a08-48e2-4759-b47d-1255b0805835", "user": "pmmserverkey"} 2026-08-06T03:17:40.961Z INFO PXC pods will be restarted {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"monitoring","namespace":"monitoring-2-0-9920"}, "namespace": "monitoring-2-0-9920", "name": "monitoring", "reconcileID": "aa343a08-48e2-4759-b47d-1255b0805835", "last-applied-secret": "efd87259fad0d07e7d06001e143e1653756066643d7b4398610e759bb4c15110"} 2026-08-06T03:17:40.964Z DEBUG Updating object {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"monitoring","namespace":"monitoring-2-0-9920"}, "namespace": "monitoring-2-0-9920", "name": "monitoring", "reconcileID": "aa343a08-48e2-4759-b47d-1255b0805835", "object": "monitoring-pxc", "kind": "&TypeMeta{Kind:StatefulSet,APIVersion:apps/v1,}", "hashChanged": true, "metaChanged": true} 2026-08-06T03:17:41.085Z DEBUG Updating object {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"monitoring","namespace":"monitoring-2-0-9920"}, "namespace": "monitoring-2-0-9920", "name": "monitoring", "reconcileID": "aa343a08-48e2-4759-b47d-1255b0805835", "object": "monitoring-haproxy", "kind": "&TypeMeta{Kind:StatefulSet,APIVersion:apps/v1,}", "hashChanged": true, "metaChanged": true} 2026-08-06T03:18:47.788Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"monitoring","namespace":"monitoring-2-0-9920"}, "namespace": "monitoring-2-0-9920", "name": "monitoring", "reconcileID": "28817d7a-fe63-41dc-b68f-e8cdf0585bab", "err": "failed to ensure cluster readonly status: connect to pod monitoring-pxc-1: dial tcp: lookup monitoring-pxc-1.monitoring-pxc.monitoring-2-0-9920 on 34.118.224.10:53: no such host"} 2026-08-06T03:19:56.755Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"monitoring","namespace":"monitoring-2-0-9920"}, "namespace": "monitoring-2-0-9920", "name": "monitoring", "reconcileID": "4c58c604-1b00-4196-9681-fa9d35a62a1c", "err": "failed to ensure cluster readonly status: connect to pod monitoring-pxc-0: dial tcp: lookup monitoring-pxc-0.monitoring-pxc.monitoring-2-0-9920 on 34.118.224.10:53: no such host"} 2026-08-06T03:24:55.449Z DEBUG Updating object {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"monitoring","namespace":"monitoring-2-0-9920"}, "namespace": "monitoring-2-0-9920", "name": "monitoring", "reconcileID": "9037ffbb-84b6-49cd-9019-2e177ae0467b", "object": "monitoring-pxc", "kind": "&TypeMeta{Kind:StatefulSet,APIVersion:apps/v1,}", "hashChanged": true, "metaChanged": true} 2026-08-06T03:24:55.531Z DEBUG Updating object {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"monitoring","namespace":"monitoring-2-0-9920"}, "namespace": "monitoring-2-0-9920", "name": "monitoring", "reconcileID": "9037ffbb-84b6-49cd-9019-2e177ae0467b", "object": "monitoring-haproxy", "kind": "&TypeMeta{Kind:StatefulSet,APIVersion:apps/v1,}", "hashChanged": true, "metaChanged": true} 2026-08-06T03:24:55.696Z DEBUG Updating object {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"monitoring","namespace":"monitoring-2-0-9920"}, "namespace": "monitoring-2-0-9920", "name": "monitoring", "reconcileID": "9037ffbb-84b6-49cd-9019-2e177ae0467b", "object": "monitoring-haproxy", "kind": "&TypeMeta{Kind:StatefulSet,APIVersion:apps/v1,}", "hashChanged": true, "metaChanged": true} [mysql] 2026/08/06 03:19:55 packets.go:58 read tcp 10.32.32.26:34036->34.118.233.244:3306: i/o timeout + local backup_names ++ kubectl_bin get pxc-backup --all-namespaces -o 'jsonpath={range .items[*]}{.metadata.namespace}/{.metadata.name}{"\n"}{end}' + backup_names= + [[ -n '' ]] + local pxc_names ++ kubectl_bin get pxc --all-namespaces -o 'jsonpath={range .items[*]}{.metadata.namespace}/{.metadata.name}{"\n"}{end}' + pxc_names=monitoring-2-0-9920/monitoring + [[ -n monitoring-2-0-9920/monitoring ]] + kubectl_bin get pxc --all-namespaces -o 'jsonpath={range .items[*]}{.metadata.namespace} {.metadata.name}{"\n"}{end}' + read -r pns name + kubectl_bin patch pxc -n monitoring-2-0-9920 monitoring --type=merge -p '{"metadata":{"finalizers":["percona.com/delete-pxc-pvc"]}}' perconaxtradbcluster.pxc.percona.com/monitoring patched + read -r pns name + timeout 300 kubectl delete pxc --all --all-namespaces --ignore-not-found perconaxtradbcluster.pxc.percona.com "monitoring" deleted from monitoring-2-0-9920 namespace ++ date +%s + local deadline=1785987090 ++ kubectl_bin get pxc --all-namespaces -o name --ignore-not-found + [[ -z '' ]] + timeout 300 kubectl delete pxc-restore --all --all-namespaces --wait=false --ignore-not-found No resources found + kubectl_bin delete ValidatingWebhookConfiguration percona-xtradbcluster-webhook --ignore-not-found ++ mktemp + local LAST_OUT=/tmp/tmp.aPPd636Xqb ++ mktemp + local LAST_ERR=/tmp/tmp.WfvN6mF4sH + local exit_status=0 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl delete ValidatingWebhookConfiguration percona-xtradbcluster-webhook --ignore-not-found + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.aPPd636Xqb validatingwebhookconfiguration.admissionregistration.k8s.io "percona-xtradbcluster-webhook" deleted + cat /tmp/tmp.WfvN6mF4sH + rm /tmp/tmp.aPPd636Xqb /tmp/tmp.WfvN6mF4sH + return 0 + kubectl_bin delete -f https://github.com/jetstack/cert-manager/releases/download/v1.20.2/cert-manager.yaml + : + '[' '!' -z '' ']' + '[' -n pxc-operator ']' ++ date +%s + kubectl_bin delete --grace-period=0 --force=true namespace monitoring-2-0-9920 + kubectl_bin delete --grace-period=0 --force=true namespace pxc-operator + local deadline=1785986936 ++ kubectl_bin get namespace monitoring-2-0-9920 --ignore-not-found -o name ++ mktemp ++ mktemp + local LAST_OUT=/tmp/tmp.6CeeKdOHaD + local LAST_OUT=/tmp/tmp.0wHF6OC5bq ++ mktemp ++ mktemp + local LAST_ERR=/tmp/tmp.RBGJ3j8IMR + local exit_status=0 + local LAST_ERR=/tmp/tmp.Miz3EwaSJu + local exit_status=0 ++ seq 0 2 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl delete --grace-period=0 --force=true namespace pxc-operator + for i in $(seq 0 2) + set +e + kubectl delete --grace-period=0 --force=true namespace monitoring-2-0-9920 + [[ -z namespace/monitoring-2-0-9920 ]] ++ date +%s + [[ 1785986818 -ge 1785986936 ]] + sleep 5 ++ kubectl_bin get namespace monitoring-2-0-9920 --ignore-not-found -o name + [[ -z namespace/monitoring-2-0-9920 ]] ++ date +%s + [[ 1785986825 -ge 1785986936 ]] + sleep 5 ++ kubectl_bin get namespace monitoring-2-0-9920 --ignore-not-found -o name + [[ -z namespace/monitoring-2-0-9920 ]] ++ date +%s + [[ 1785986833 -ge 1785986936 ]] + sleep 5 + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.6CeeKdOHaD namespace "pxc-operator" force deleted + cat /tmp/tmp.RBGJ3j8IMR Warning: Immediate deletion does not wait for confirmation that the running resource has been terminated. The resource may continue to run on the cluster indefinitely. + rm /tmp/tmp.6CeeKdOHaD /tmp/tmp.RBGJ3j8IMR + return 0 ++ kubectl_bin get namespace monitoring-2-0-9920 --ignore-not-found -o name + [[ -z namespace/monitoring-2-0-9920 ]] ++ date +%s + [[ 1785986839 -ge 1785986936 ]] + sleep 5 ++ kubectl_bin get namespace monitoring-2-0-9920 --ignore-not-found -o name + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.0wHF6OC5bq namespace "monitoring-2-0-9920" force deleted + cat /tmp/tmp.Miz3EwaSJu Warning: Immediate deletion does not wait for confirmation that the running resource has been terminated. The resource may continue to run on the cluster indefinitely. + rm /tmp/tmp.0wHF6OC5bq /tmp/tmp.Miz3EwaSJu + return 0 + [[ -z '' ]] ++ date +%s + deadline=1785986967 ++ awk '$2 != ""' ++ kubectl_bin get pv -o 'jsonpath={range .items[*]}{.metadata.name}{"\t"}{.metadata.deletionTimestamp}{"\n"}{end}' + [[ -z '' ]] + rm -rf /tmp/tmp.gvyFfLErHq + desc 'test passed' + set +o xtrace ----------------------------------------------------------------------------------- test passed -----------------------------------------------------------------------------------