Log: /mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/e2e-tests/logs/monitoring-pmm3-8-4.log Warning: version difference between client (1.37) and server (1.34) exceeds the supported minor version skew of +/-1 Warning: version difference between client (1.37) and server (1.34) exceeds the supported minor version skew of +/-1 + cluster=monitoring + create_infra monitoring-pmm3-26352 + local ns=monitoring-pmm3-26352 + '[' -n pxc-operator ']' + read -r pns name + kubectl_bin get pxc --all-namespaces -o 'jsonpath={range .items[*]}{.metadata.namespace} {.metadata.name}{"\n"}{end}' + [[ -z monitoring-pmm3-31122 ]] + [[ -z monitoring ]] + kubectl_bin patch pxc -n monitoring-pmm3-31122 monitoring --type=merge -p '{"metadata":{"finalizers":[]}}' ++ mktemp + local LAST_OUT=/tmp/tmp.oniGl2ATde ++ mktemp + local LAST_ERR=/tmp/tmp.jfv72LwJAm + local exit_status=0 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl patch pxc -n monitoring-pmm3-31122 monitoring --type=merge -p '{"metadata":{"finalizers":[]}}' + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.oniGl2ATde perconaxtradbcluster.pxc.percona.com/monitoring patched (no change) + cat /tmp/tmp.jfv72LwJAm + rm /tmp/tmp.oniGl2ATde /tmp/tmp.jfv72LwJAm + return 0 + read -r pns name + kubectl_bin get pxc-backup --all-namespaces -o 'jsonpath={range .items[*]}{.metadata.namespace} {.metadata.name}{"\n"}{end}' + read -r pns name + timeout 300 kubectl delete pxc --all --all-namespaces --wait=false --ignore-not-found perconaxtradbcluster.pxc.percona.com "monitoring" deleted from monitoring-pmm3-31122 namespace + timeout 300 kubectl delete pxc-backup --all --all-namespaces --wait=false --ignore-not-found No resources found + timeout 300 kubectl delete pxc-restore --all --all-namespaces --wait=false --ignore-not-found No resources found + create_namespace pxc-operator + local namespace=pxc-operator + local skip_clean_namespace= + [[ 1 == 1 ]] + [[ -z '' ]] + destroy_chaos_mesh ++ sed s/NAMESPACE// ++ awk '-F ' '{print $2}' ++ tail -n1 ++ helm list --all-namespaces --filter chaos-mesh + local chaos_mesh_ns= + '[' -n '' ']' ++ awk '{print $1}' ++ kubectl get MutatingWebhookConfiguration ++ grep chaos-mesh + timeout 30 kubectl delete MutatingWebhookConfiguration error: resource(s) were provided, but no name was specified + : ++ kubectl get ValidatingWebhookConfiguration ++ awk '{print $1}' ++ grep chaos-mesh + timeout 30 kubectl delete ValidatingWebhookConfiguration error: resource(s) were provided, but no name was specified + : ++ grep validate-auth ++ awk '{print $1}' ++ kubectl get ValidatingWebhookConfiguration + timeout 30 kubectl delete ValidatingWebhookConfiguration error: resource(s) were provided, but no name was specified + : ++ awk '{print $1}' ++ grep chaos-mesh ++ kubectl api-resources ++ kubectl get crd ++ grep chaos-mesh.org ++ awk '{print $1}' + timeout 30 kubectl delete crd error: resource(s) were provided, but no name was specified + : ++ kubectl get clusterrolebinding ++ grep chaos-mesh ++ awk '{print $1}' + timeout 30 kubectl delete clusterrolebinding error: resource(s) were provided, but no name was specified + : ++ awk '{print $1}' ++ grep chaos-mesh ++ kubectl get clusterrole + timeout 30 kubectl delete clusterrole error: resource(s) were provided, but no name was specified + : + desc 'cleaned up all old namespaces' + set +o xtrace ----------------------------------------------------------------------------------- cleaned up all old namespaces ----------------------------------------------------------------------------------- + '[' -n '' ']' + desc 'cleaned up old namespaces pxc-operator' + set +o xtrace ----------------------------------------------------------------------------------- cleaned up old namespaces pxc-operator ----------------------------------------------------------------------------------- + kubectl_bin delete namespace pxc-operator ++ mktemp + local LAST_OUT=/tmp/tmp.h0G46Qe402 + xargs kubectl delete ns + awk '{print$1}' + grep -E -v '^kube-|^default|Terminating|pxc-operator|openshift|^gke-|^gmp-|^NAME' ++ mktemp + kubectl_bin get ns + local LAST_ERR=/tmp/tmp.A6800kFOfm + local exit_status=0 ++ mktemp ++ seq 0 2 + local LAST_OUT=/tmp/tmp.o7pxga0rOU ++ mktemp + for i in $(seq 0 2) + set +e + kubectl delete namespace pxc-operator + local LAST_ERR=/tmp/tmp.LweGdr4GwV + local exit_status=0 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl get ns + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.o7pxga0rOU + cat /tmp/tmp.LweGdr4GwV + rm /tmp/tmp.o7pxga0rOU /tmp/tmp.LweGdr4GwV + return 0 namespace "monitoring-pmm3-31122" deleted + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.h0G46Qe402 namespace "pxc-operator" deleted + cat /tmp/tmp.A6800kFOfm + rm /tmp/tmp.h0G46Qe402 /tmp/tmp.A6800kFOfm + return 0 + wait_for_delete namespace/pxc-operator + local res=namespace/pxc-operator + echo -n 'waiting for namespace/pxc-operator to be deleted' waiting for namespace/pxc-operator to be deleted+ set +o xtrace Error from server (NotFound): namespaces "pxc-operator" not found + desc 'create namespace pxc-operator' + set +o xtrace ----------------------------------------------------------------------------------- create namespace pxc-operator ----------------------------------------------------------------------------------- + kubectl_bin create namespace pxc-operator ++ mktemp + local LAST_OUT=/tmp/tmp.ITUV4aN04k ++ mktemp + local LAST_ERR=/tmp/tmp.g8S3uW5wMC + local exit_status=0 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl create namespace pxc-operator + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.ITUV4aN04k namespace/pxc-operator created + cat /tmp/tmp.g8S3uW5wMC + rm /tmp/tmp.ITUV4aN04k /tmp/tmp.g8S3uW5wMC + return 0 ++ kubectl_bin config current-context +++ mktemp ++ local LAST_OUT=/tmp/tmp.yM7TfW6rVY +++ mktemp ++ local LAST_ERR=/tmp/tmp.LWSgUu0b5F ++ local exit_status=0 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl config current-context ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.yM7TfW6rVY ++ cat /tmp/tmp.LWSgUu0b5F ++ rm /tmp/tmp.yM7TfW6rVY /tmp/tmp.LWSgUu0b5F ++ return 0 + kubectl_bin config set-context gke_cloud-dev-112233_us-central1-a_jen-pxc-2546-caf25420-2-cluster4 --namespace=pxc-operator ++ mktemp + local LAST_OUT=/tmp/tmp.T3BwmNEtty ++ mktemp + local LAST_ERR=/tmp/tmp.N7nERt8GbP + local exit_status=0 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl config set-context gke_cloud-dev-112233_us-central1-a_jen-pxc-2546-caf25420-2-cluster4 --namespace=pxc-operator + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.T3BwmNEtty Context "gke_cloud-dev-112233_us-central1-a_jen-pxc-2546-caf25420-2-cluster4" modified. + cat /tmp/tmp.N7nERt8GbP + rm /tmp/tmp.T3BwmNEtty /tmp/tmp.N7nERt8GbP + return 0 + deploy_operator + desc 'start PXC operator' + set +o xtrace ----------------------------------------------------------------------------------- start PXC operator ----------------------------------------------------------------------------------- + kubectl_bin apply --server-side --force-conflicts -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/deploy/crd.yaml ++ mktemp + local LAST_OUT=/tmp/tmp.jNzY2UCtZx ++ mktemp + local LAST_ERR=/tmp/tmp.EEiw7IxTyv + local exit_status=0 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl apply --server-side --force-conflicts -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/deploy/crd.yaml + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.jNzY2UCtZx customresourcedefinition.apiextensions.k8s.io/perconaxtradbclusterbackups.pxc.percona.com serverside-applied customresourcedefinition.apiextensions.k8s.io/perconaxtradbclusterrestores.pxc.percona.com serverside-applied customresourcedefinition.apiextensions.k8s.io/perconaxtradbclusters.pxc.percona.com serverside-applied + cat /tmp/tmp.EEiw7IxTyv + rm /tmp/tmp.jNzY2UCtZx /tmp/tmp.EEiw7IxTyv + return 0 + kubectl_bin wait --for=condition=Established --timeout=120s crd/perconaxtradbclusters.pxc.percona.com ++ mktemp + local LAST_OUT=/tmp/tmp.X7dpBxP4Xi ++ mktemp + local LAST_ERR=/tmp/tmp.O0JcVOki8u + local exit_status=0 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl wait --for=condition=Established --timeout=120s crd/perconaxtradbclusters.pxc.percona.com + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.X7dpBxP4Xi customresourcedefinition.apiextensions.k8s.io/perconaxtradbclusters.pxc.percona.com condition met + cat /tmp/tmp.O0JcVOki8u + rm /tmp/tmp.X7dpBxP4Xi /tmp/tmp.O0JcVOki8u + return 0 + kubectl_bin wait --for=condition=Established --timeout=120s crd/perconaxtradbclusterbackups.pxc.percona.com ++ mktemp + local LAST_OUT=/tmp/tmp.niDglu60M4 ++ mktemp + local LAST_ERR=/tmp/tmp.ZeLCFnhUc7 + local exit_status=0 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl wait --for=condition=Established --timeout=120s crd/perconaxtradbclusterbackups.pxc.percona.com + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.niDglu60M4 customresourcedefinition.apiextensions.k8s.io/perconaxtradbclusterbackups.pxc.percona.com condition met + cat /tmp/tmp.ZeLCFnhUc7 + rm /tmp/tmp.niDglu60M4 /tmp/tmp.ZeLCFnhUc7 + return 0 + kubectl_bin wait --for=condition=Established --timeout=120s crd/perconaxtradbclusterrestores.pxc.percona.com ++ mktemp + local LAST_OUT=/tmp/tmp.bLkdnMFdqn ++ mktemp + local LAST_ERR=/tmp/tmp.8kUlj9mURx + local exit_status=0 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl wait --for=condition=Established --timeout=120s crd/perconaxtradbclusterrestores.pxc.percona.com + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.bLkdnMFdqn customresourcedefinition.apiextensions.k8s.io/perconaxtradbclusterrestores.pxc.percona.com condition met + cat /tmp/tmp.8kUlj9mURx + rm /tmp/tmp.bLkdnMFdqn /tmp/tmp.8kUlj9mURx + return 0 + '[' -n pxc-operator ']' + apply_rbac cw-rbac + local operator_namespace=pxc-operator + local rbac=cw-rbac + sed -e 's^namespace: .*^namespace: pxc-operator^' + cat /mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/deploy/cw-rbac.yaml + kubectl_bin apply -f - ++ mktemp + local LAST_OUT=/tmp/tmp.Oh8cB7hDwu ++ mktemp + local LAST_ERR=/tmp/tmp.PpLQWQOfSK + local exit_status=0 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.Oh8cB7hDwu clusterrole.rbac.authorization.k8s.io/percona-xtradb-cluster-operator unchanged serviceaccount/percona-xtradb-cluster-operator created clusterrolebinding.rbac.authorization.k8s.io/service-account-percona-xtradb-cluster-operator unchanged + cat /tmp/tmp.PpLQWQOfSK + rm /tmp/tmp.Oh8cB7hDwu /tmp/tmp.PpLQWQOfSK + return 0 + yq eval '(select(.kind == "Deployment").spec.template.spec.containers[] | select(.name == "percona-xtradb-cluster-operator").env[] | select(.name == "DISABLE_TELEMETRY").value) = "true"' - + kubectl_bin apply -f - + sed -e 's^failureThreshold: .*^failureThreshold: 10^' + yq eval '(select(.kind == "Deployment").spec.template.spec.containers[] | select(.name == "percona-xtradb-cluster-operator").env[] | select(.name == "LOG_LEVEL").value) = "VERBOSE"' - + sed -e 's^image: .*^image: perconalab/percona-xtradb-cluster-operator:PR-2546-caf25420^' + yq eval '(select(.kind == "Deployment").spec.template.spec.containers[] | select(.name == "percona-xtradb-cluster-operator").env[] | select(.name == "PXCO_FEATURE_GATES").value) = ""' - + cat /mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/deploy/cw-operator.yaml ++ mktemp + local LAST_OUT=/tmp/tmp.r5pOUneFCx ++ mktemp + local LAST_ERR=/tmp/tmp.JMr8OhdYaY + local exit_status=0 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.r5pOUneFCx deployment.apps/percona-xtradb-cluster-operator created service/percona-xtradb-cluster-operator created + cat /tmp/tmp.JMr8OhdYaY + rm /tmp/tmp.r5pOUneFCx /tmp/tmp.JMr8OhdYaY + return 0 + sleep 10 + kubectl_bin wait --for=condition=Ready pods -l app.kubernetes.io/component=operator,app.kubernetes.io/instance=percona-xtradb-cluster-operator,app.kubernetes.io/name=percona-xtradb-cluster-operator --timeout=30s ++ mktemp + local LAST_OUT=/tmp/tmp.mnDs9EO5e3 ++ mktemp + local LAST_ERR=/tmp/tmp.ktal7iw58o + local exit_status=0 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl wait --for=condition=Ready pods -l app.kubernetes.io/component=operator,app.kubernetes.io/instance=percona-xtradb-cluster-operator,app.kubernetes.io/name=percona-xtradb-cluster-operator --timeout=30s + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.mnDs9EO5e3 pod/percona-xtradb-cluster-operator-658fcf4b98-jtpl5 condition met + cat /tmp/tmp.ktal7iw58o + rm /tmp/tmp.mnDs9EO5e3 /tmp/tmp.ktal7iw58o + return 0 ++ get_operator_pod ++ local label_prefix=app.kubernetes.io/ +++ grep -c percona-xtradb-cluster-operator +++ kubectl get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator -n pxc-operator ++ local check_label=1 ++ [[ 1 -eq 0 ]] ++ head -1 ++ jq -r '.items[] | select(.metadata.deletionTimestamp == null) | .metadata.name' ++ kubectl_bin get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator --field-selector=status.phase=Running -o json -n pxc-operator +++ mktemp ++ local LAST_OUT=/tmp/tmp.tnDo0jhiFt +++ mktemp ++ local LAST_ERR=/tmp/tmp.eHX7ZXbfTA ++ local exit_status=0 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator --field-selector=status.phase=Running -o json -n pxc-operator ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.tnDo0jhiFt ++ cat /tmp/tmp.eHX7ZXbfTA ++ rm /tmp/tmp.tnDo0jhiFt /tmp/tmp.eHX7ZXbfTA ++ return 0 + wait_pod percona-xtradb-cluster-operator-658fcf4b98-jtpl5 480 pxc-operator + local pod=percona-xtradb-cluster-operator-658fcf4b98-jtpl5 + local max_retry=480 + local ns=pxc-operator ++ echo percona-xtradb-cluster-operator-658fcf4b98-jtpl5 ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ grep -E '^(pxc|proxysql)$' + local container= + set +o xtrace pod/percona-xtradb-cluster-operator-658fcf4b98-jtpl5 condition met waiting for pod/percona-xtradb-cluster-operator-658fcf4b98-jtpl5 to become Ready.Ok + sleep 3 + create_namespace monitoring-pmm3-26352 + local namespace=monitoring-pmm3-26352 + local skip_clean_namespace= + [[ 1 == 1 ]] + [[ -z '' ]] + destroy_chaos_mesh ++ helm list --all-namespaces --filter chaos-mesh ++ sed s/NAMESPACE// ++ awk '-F ' '{print $2}' ++ tail -n1 + local chaos_mesh_ns= + '[' -n '' ']' ++ kubectl get MutatingWebhookConfiguration ++ awk '{print $1}' ++ grep chaos-mesh + timeout 30 kubectl delete MutatingWebhookConfiguration error: resource(s) were provided, but no name was specified + : ++ grep chaos-mesh ++ awk '{print $1}' ++ kubectl get ValidatingWebhookConfiguration + timeout 30 kubectl delete ValidatingWebhookConfiguration error: resource(s) were provided, but no name was specified + : ++ kubectl get ValidatingWebhookConfiguration ++ grep validate-auth ++ awk '{print $1}' + timeout 30 kubectl delete ValidatingWebhookConfiguration error: resource(s) were provided, but no name was specified + : ++ awk '{print $1}' ++ kubectl api-resources ++ grep chaos-mesh ++ kubectl get crd ++ awk '{print $1}' ++ grep chaos-mesh.org + timeout 30 kubectl delete crd error: resource(s) were provided, but no name was specified + : ++ awk '{print $1}' ++ grep chaos-mesh ++ kubectl get clusterrolebinding + timeout 30 kubectl delete clusterrolebinding error: resource(s) were provided, but no name was specified + : ++ kubectl get clusterrole ++ grep chaos-mesh ++ awk '{print $1}' + timeout 30 kubectl delete clusterrole error: resource(s) were provided, but no name was specified + : + desc 'cleaned up all old namespaces' + set +o xtrace ----------------------------------------------------------------------------------- cleaned up all old namespaces ----------------------------------------------------------------------------------- + '[' -n '' ']' + desc 'cleaned up old namespaces monitoring-pmm3-26352' + set +o xtrace ----------------------------------------------------------------------------------- cleaned up old namespaces monitoring-pmm3-26352 ----------------------------------------------------------------------------------- + kubectl_bin delete namespace monitoring-pmm3-26352 + kubectl_bin get ns + awk '{print$1}' ++ mktemp + grep -E -v '^kube-|^default|Terminating|pxc-operator|openshift|^gke-|^gmp-|^NAME' + xargs kubectl delete ns + local LAST_OUT=/tmp/tmp.vYsI7Vvmsp ++ mktemp + local LAST_OUT=/tmp/tmp.Wu9nKwt40n ++ mktemp + local LAST_ERR=/tmp/tmp.rOiUGPAVmK + local exit_status=0 ++ mktemp ++ seq 0 2 + local LAST_ERR=/tmp/tmp.7u5YBs8eze + local exit_status=0 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl get ns + for i in $(seq 0 2) + set +e + kubectl delete namespace monitoring-pmm3-26352 + exit_status=1 + set -e + '[' 1 '!=' 0 ']' + '[' 1 == 1 ']' + sleep 0 + for i in $(seq 0 2) + set +e + kubectl delete namespace monitoring-pmm3-26352 + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.Wu9nKwt40n + cat /tmp/tmp.7u5YBs8eze + rm /tmp/tmp.Wu9nKwt40n /tmp/tmp.7u5YBs8eze + return 0 + exit_status=1 + set -e + '[' 1 '!=' 0 ']' + '[' 1 == 1 ']' + sleep 0 + for i in $(seq 0 2) + set +e + kubectl delete namespace monitoring-pmm3-26352 error: resource(s) were provided, but no name was specified + exit_status=1 + set -e + '[' 1 '!=' 0 ']' + '[' 1 == 1 ']' + sleep 0 + cat /tmp/tmp.vYsI7Vvmsp + cat /tmp/tmp.rOiUGPAVmK Error from server (NotFound): namespaces "monitoring-pmm3-26352" not found + rm /tmp/tmp.vYsI7Vvmsp /tmp/tmp.rOiUGPAVmK + return 1 + : + wait_for_delete namespace/monitoring-pmm3-26352 + local res=namespace/monitoring-pmm3-26352 + echo -n 'waiting for namespace/monitoring-pmm3-26352 to be deleted' waiting for namespace/monitoring-pmm3-26352 to be deleted+ set +o xtrace Error from server (NotFound): namespaces "monitoring-pmm3-26352" not found + desc 'create namespace monitoring-pmm3-26352' + set +o xtrace ----------------------------------------------------------------------------------- create namespace monitoring-pmm3-26352 ----------------------------------------------------------------------------------- + kubectl_bin create namespace monitoring-pmm3-26352 ++ mktemp + local LAST_OUT=/tmp/tmp.1GaPEXXRl7 ++ mktemp + local LAST_ERR=/tmp/tmp.x2WUNck4r7 + local exit_status=0 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl create namespace monitoring-pmm3-26352 + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.1GaPEXXRl7 namespace/monitoring-pmm3-26352 created + cat /tmp/tmp.x2WUNck4r7 + rm /tmp/tmp.1GaPEXXRl7 /tmp/tmp.x2WUNck4r7 + return 0 ++ kubectl_bin config current-context +++ mktemp ++ local LAST_OUT=/tmp/tmp.gw38b66B7Q +++ mktemp ++ local LAST_ERR=/tmp/tmp.vuShdn5ly3 ++ local exit_status=0 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl config current-context ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.gw38b66B7Q ++ cat /tmp/tmp.vuShdn5ly3 ++ rm /tmp/tmp.gw38b66B7Q /tmp/tmp.vuShdn5ly3 ++ return 0 + kubectl_bin config set-context gke_cloud-dev-112233_us-central1-a_jen-pxc-2546-caf25420-2-cluster4 --namespace=monitoring-pmm3-26352 ++ mktemp + local LAST_OUT=/tmp/tmp.G5dXZ9qlix ++ mktemp + local LAST_ERR=/tmp/tmp.glLNKfcXws + local exit_status=0 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl config set-context gke_cloud-dev-112233_us-central1-a_jen-pxc-2546-caf25420-2-cluster4 --namespace=monitoring-pmm3-26352 + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.G5dXZ9qlix Context "gke_cloud-dev-112233_us-central1-a_jen-pxc-2546-caf25420-2-cluster4" modified. + cat /tmp/tmp.glLNKfcXws + rm /tmp/tmp.G5dXZ9qlix /tmp/tmp.glLNKfcXws + return 0 + apply_secrets + desc 'create secrets for cloud storages' + set +o xtrace ----------------------------------------------------------------------------------- create secrets for cloud storages ----------------------------------------------------------------------------------- + '[' -z '' ']' + kubectl_bin apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/e2e-tests/conf/minio-secret.yml -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/e2e-tests/conf/cloud-secret.yml ++ mktemp + local LAST_OUT=/tmp/tmp.JrKdGj3NNG ++ mktemp + local LAST_ERR=/tmp/tmp.OTrYkDRvBl + local exit_status=0 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/e2e-tests/conf/minio-secret.yml -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/e2e-tests/conf/cloud-secret.yml + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.JrKdGj3NNG secret/minio-secret created secret/aws-s3-secret created secret/do-spaces-secret created secret/gcp-cs-secret created secret/azure-secret created + cat /tmp/tmp.OTrYkDRvBl + rm /tmp/tmp.JrKdGj3NNG /tmp/tmp.OTrYkDRvBl + return 0 + deploy_helm monitoring-pmm3-26352 + helm repo add hashicorp https://helm.releases.hashicorp.com "hashicorp" already exists with the same configuration, skipping + helm repo add minio https://charts.min.io/ "minio" already exists with the same configuration, skipping + helm repo update Hang tight while we grab the latest from your chart repositories... ...Successfully got an update from the "minio" chart repository ...Successfully got an update from the "chaos-mesh" chart repository ...Successfully got an update from the "hashicorp" chart repository ...Successfully got an update from the "percona" chart repository Update Complete. ⎈Happy Helming!⎈ + desc 'install PMM Server' + set +o xtrace ----------------------------------------------------------------------------------- install PMM Server ----------------------------------------------------------------------------------- + deploy_pmm3_server + helm uninstall -n monitoring-pmm3-26352 monitoring Error: uninstall: Release not loaded: monitoring: release: not found + : + helm repo remove percona "percona" has been removed from your repositories + kubectl delete clusterrole monitoring --ignore-not-found + kubectl delete clusterrolebinding monitoring --ignore-not-found + helm repo add percona https://percona.github.io/percona-helm-charts/ "percona" has been added to your repositories + helm repo update Hang tight while we grab the latest from your chart repositories... ...Successfully got an update from the "minio" chart repository ...Successfully got an update from the "chaos-mesh" chart repository ...Successfully got an update from the "hashicorp" chart repository ...Successfully got an update from the "percona" chart repository Update Complete. ⎈Happy Helming!⎈ + '[' '!' -z '' ']' + retry 10 60 helm install monitoring percona/pmm -n monitoring-pmm3-26352 --set fullnameOverride=monitoring --set image.tag=3-dev-latest --set image.repository=perconalab/pmm-server --set service.type=LoadBalancer --force + local max=10 + local delay=60 + shift 2 + local n=1 + helm install monitoring percona/pmm -n monitoring-pmm3-26352 --set fullnameOverride=monitoring --set image.tag=3-dev-latest --set image.repository=perconalab/pmm-server --set service.type=LoadBalancer --force NAME: monitoring LAST DEPLOYED: Mon Oct 5 14:30:35 2026 NAMESPACE: monitoring-pmm3-26352 STATUS: deployed REVISION: 1 TEST SUITE: None NOTES: Percona Monitoring and Management (PMM) An open source database monitoring, observability and management tool Check more info here: https://docs.percona.com/percona-monitoring-and-management/index.html Get the application URL: NOTE: It may take a few minutes for the LoadBalancer IP to be available. You can watch the status of by running 'kubectl get --namespace monitoring-pmm3-26352 svc -w monitoring-service' export SERVICE_IP=$(kubectl get svc --namespace monitoring-pmm3-26352 monitoring-service -o jsonpath="{.status.loadBalancer.ingress[0].ip}") echo https://$SERVICE_IP: Get password for the "admin" user: export ADMIN_PASS=$(kubectl get secret pmm-secret --namespace monitoring-pmm3-26352 -o jsonpath='{.data.PMM_ADMIN_PASSWORD}' | base64 --decode) echo $ADMIN_PASS + wait_for_pmm_service + local timeout=420 ++ date +%s + local start=1791210637 + kubectl_bin get svc monitoring-service -o 'jsonpath={.status.loadBalancer.ingress[0]}' + grep -q . ++ date +%s + (( 1791210639 - start >= timeout )) + sleep 2 + kubectl_bin get svc monitoring-service -o 'jsonpath={.status.loadBalancer.ingress[0]}' + grep -q . ++ date +%s + (( 1791210642 - start >= timeout )) + sleep 2 + grep -q . + kubectl_bin get svc monitoring-service -o 'jsonpath={.status.loadBalancer.ingress[0]}' ++ date +%s + (( 1791210646 - start >= timeout )) + sleep 2 + kubectl_bin get svc monitoring-service -o 'jsonpath={.status.loadBalancer.ingress[0]}' + grep -q . ++ date +%s + (( 1791210651 - start >= timeout )) + sleep 2 + grep -q . + kubectl_bin get svc monitoring-service -o 'jsonpath={.status.loadBalancer.ingress[0]}' ++ date +%s + (( 1791210654 - start >= timeout )) + sleep 2 + kubectl_bin get svc monitoring-service -o 'jsonpath={.status.loadBalancer.ingress[0]}' + grep -q . ++ date +%s + (( 1791210657 - start >= timeout )) + sleep 2 + kubectl_bin get svc monitoring-service -o 'jsonpath={.status.loadBalancer.ingress[0]}' + grep -q . ++ date +%s + (( 1791210661 - start >= timeout )) + sleep 2 + kubectl_bin get svc monitoring-service -o 'jsonpath={.status.loadBalancer.ingress[0]}' + grep -q . ++ date +%s + (( 1791210665 - start >= timeout )) + sleep 2 + kubectl_bin get svc monitoring-service -o 'jsonpath={.status.loadBalancer.ingress[0]}' + grep -q . ++ date +%s + (( 1791210669 - start >= timeout )) + sleep 2 + kubectl_bin get svc monitoring-service -o 'jsonpath={.status.loadBalancer.ingress[0]}' + grep -q . ++ date +%s + (( 1791210673 - start >= timeout )) + sleep 2 + grep -q . + kubectl_bin get svc monitoring-service -o 'jsonpath={.status.loadBalancer.ingress[0]}' ++ date +%s + (( 1791210677 - start >= timeout )) + sleep 2 + grep -q . + kubectl_bin get svc monitoring-service -o 'jsonpath={.status.loadBalancer.ingress[0]}' ++ date +%s + (( 1791210680 - start >= timeout )) + sleep 2 + kubectl_bin get svc monitoring-service -o 'jsonpath={.status.loadBalancer.ingress[0]}' + grep -q . + kubectl_bin wait sts/monitoring '--for=jsonpath={.status.readyReplicas}=1' --timeout=420s ++ mktemp + local LAST_OUT=/tmp/tmp.kJtNjQPhiY ++ mktemp + local LAST_ERR=/tmp/tmp.pEjwLCvYEj + local exit_status=0 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl wait sts/monitoring '--for=jsonpath={.status.readyReplicas}=1' --timeout=420s + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.kJtNjQPhiY statefulset.apps/monitoring condition met + cat /tmp/tmp.pEjwLCvYEj + rm /tmp/tmp.kJtNjQPhiY /tmp/tmp.pEjwLCvYEj + return 0 + desc 'create secret' + set +o xtrace ----------------------------------------------------------------------------------- create secret ----------------------------------------------------------------------------------- + kubectl_bin apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/e2e-tests/monitoring-pmm3/conf/secrets.yaml ++ mktemp + local LAST_OUT=/tmp/tmp.fcfkclpvXb ++ mktemp + local LAST_ERR=/tmp/tmp.m8DGM0IaIW + local exit_status=0 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/e2e-tests/monitoring-pmm3/conf/secrets.yaml + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.fcfkclpvXb secret/my-cluster-secrets created + cat /tmp/tmp.m8DGM0IaIW + rm /tmp/tmp.fcfkclpvXb /tmp/tmp.m8DGM0IaIW + return 0 + desc 'add PMM3 token to secret' + set +o xtrace ----------------------------------------------------------------------------------- add PMM3 token to secret ----------------------------------------------------------------------------------- ++ get_pmm_server_token operator ++ local key_name=operator ++ [[ -z operator ]] ++ local ADMIN_PASSWORD +++ base64 --decode +++ kubectl get secret pmm-secret -o 'jsonpath={.data.PMM_ADMIN_PASSWORD}' ++ ADMIN_PASSWORD='4 RTDc<0"()1 |=G' ++ [[ -z 4 RTDc<0"()1 |=G ]] ++ local create_response create_status_code create_json_response ++ local retry=0 ++ [[ '' == 201 ]] ++++ get_service_ip monitoring-service ++++ local service=monitoring-service ++++ kubectl_bin get service/monitoring-service -o 'jsonpath={.spec.type}' ++++ grep -q NotFound +++++ kubectl_bin get service/monitoring-service -o 'jsonpath={.spec.type}' ++++++ mktemp +++++ local LAST_OUT=/tmp/tmp.Q0C5FKBTOB ++++++ mktemp +++++ local LAST_ERR=/tmp/tmp.EosvEoODsn +++++ local exit_status=0 ++++++ seq 0 2 +++++ for i in $(seq 0 2) +++++ set +e +++++ kubectl get service/monitoring-service -o 'jsonpath={.spec.type}' +++++ exit_status=0 +++++ set -e +++++ '[' 0 '!=' 0 ']' +++++ break +++++ cat /tmp/tmp.Q0C5FKBTOB +++++ cat /tmp/tmp.EosvEoODsn +++++ rm /tmp/tmp.Q0C5FKBTOB /tmp/tmp.EosvEoODsn +++++ return 0 ++++ '[' LoadBalancer = ClusterIP ']' ++++ kubectl_bin get service/monitoring-service -o 'jsonpath={.status.loadBalancer.ingress[]}' ++++ grep -E -q 'hostname|ip' ++++ kubectl_bin get service/monitoring-service -o 'jsonpath={.status.loadBalancer.ingress[].ip}' +++++ mktemp ++++ local LAST_OUT=/tmp/tmp.nnVBzLmSjv +++++ mktemp ++++ local LAST_ERR=/tmp/tmp.4g4QcIIdGk ++++ local exit_status=0 +++++ seq 0 2 ++++ for i in $(seq 0 2) ++++ set +e ++++ kubectl get service/monitoring-service -o 'jsonpath={.status.loadBalancer.ingress[].ip}' ++++ exit_status=0 ++++ set -e ++++ '[' 0 '!=' 0 ']' ++++ break ++++ cat /tmp/tmp.nnVBzLmSjv ++++ cat /tmp/tmp.4g4QcIIdGk ++++ rm /tmp/tmp.nnVBzLmSjv /tmp/tmp.4g4QcIIdGk ++++ return 0 ++++ kubectl_bin get service/monitoring-service -o 'jsonpath={.status.loadBalancer.ingress[].hostname}' +++++ mktemp ++++ local LAST_OUT=/tmp/tmp.7J0TUpHBue +++++ mktemp ++++ local LAST_ERR=/tmp/tmp.8iwb51wHUI ++++ local exit_status=0 +++++ seq 0 2 ++++ for i in $(seq 0 2) ++++ set +e ++++ kubectl get service/monitoring-service -o 'jsonpath={.status.loadBalancer.ingress[].hostname}' ++++ exit_status=0 ++++ set -e ++++ '[' 0 '!=' 0 ']' ++++ break ++++ cat /tmp/tmp.7J0TUpHBue ++++ cat /tmp/tmp.8iwb51wHUI ++++ rm /tmp/tmp.7J0TUpHBue /tmp/tmp.8iwb51wHUI ++++ return 0 +++ curl --insecure -s -X POST -H 'Content-Type: application/json' -H 'Accept: application/json' -d '{"name":"operator", "role":"Admin", "isDisabled":false}' --user 'admin:4 RTDc<0"()1 |=G' https://104.154.140.66/graph/api/serviceaccounts -w '\n%{http_code}' ++ create_response='{"id":2,"uid":"ag0bil4m64xdse","name":"operator","login":"sa-1-operator","orgId":1,"isDisabled":false,"role":"Admin","tokens":0,"avatarUrl":""} 201' +++ tail -n1 +++ echo '{"id":2,"uid":"ag0bil4m64xdse","name":"operator","login":"sa-1-operator","orgId":1,"isDisabled":false,"role":"Admin","tokens":0,"avatarUrl":""} 201' ++ create_status_code=201 +++ sed '$ d' +++ echo '{"id":2,"uid":"ag0bil4m64xdse","name":"operator","login":"sa-1-operator","orgId":1,"isDisabled":false,"role":"Admin","tokens":0,"avatarUrl":""} 201' ++ create_json_response='{"id":2,"uid":"ag0bil4m64xdse","name":"operator","login":"sa-1-operator","orgId":1,"isDisabled":false,"role":"Admin","tokens":0,"avatarUrl":""}' ++ sleep 5 ++ let retry+=1 ++ '[' 1 -ge 24 ']' ++ [[ 201 == 201 ]] ++ local service_account_id +++ echo '{"id":2,"uid":"ag0bil4m64xdse","name":"operator","login":"sa-1-operator","orgId":1,"isDisabled":false,"role":"Admin","tokens":0,"avatarUrl":""}' +++ jq -r .id ++ service_account_id=2 ++ [[ -z 2 ]] ++ [[ 2 == \n\u\l\l ]] ++ local token_response token_status_code token_json_response ++++ get_service_ip monitoring-service ++++ local service=monitoring-service ++++ kubectl_bin get service/monitoring-service -o 'jsonpath={.spec.type}' ++++ grep -q NotFound +++++ kubectl_bin get service/monitoring-service -o 'jsonpath={.spec.type}' ++++++ mktemp +++++ local LAST_OUT=/tmp/tmp.EdRQE6gPcB ++++++ mktemp +++++ local LAST_ERR=/tmp/tmp.eRxQJlLUmE +++++ local exit_status=0 ++++++ seq 0 2 +++++ for i in $(seq 0 2) +++++ set +e +++++ kubectl get service/monitoring-service -o 'jsonpath={.spec.type}' +++++ exit_status=0 +++++ set -e +++++ '[' 0 '!=' 0 ']' +++++ break +++++ cat /tmp/tmp.EdRQE6gPcB +++++ cat /tmp/tmp.eRxQJlLUmE +++++ rm /tmp/tmp.EdRQE6gPcB /tmp/tmp.eRxQJlLUmE +++++ return 0 ++++ '[' LoadBalancer = ClusterIP ']' ++++ kubectl_bin get service/monitoring-service -o 'jsonpath={.status.loadBalancer.ingress[]}' ++++ grep -E -q 'hostname|ip' ++++ kubectl_bin get service/monitoring-service -o 'jsonpath={.status.loadBalancer.ingress[].ip}' +++++ mktemp ++++ local LAST_OUT=/tmp/tmp.vPQNj72v2F +++++ mktemp ++++ local LAST_ERR=/tmp/tmp.9Z5ircq6VO ++++ local exit_status=0 +++++ seq 0 2 ++++ for i in $(seq 0 2) ++++ set +e ++++ kubectl get service/monitoring-service -o 'jsonpath={.status.loadBalancer.ingress[].ip}' ++++ exit_status=0 ++++ set -e ++++ '[' 0 '!=' 0 ']' ++++ break ++++ cat /tmp/tmp.vPQNj72v2F ++++ cat /tmp/tmp.9Z5ircq6VO ++++ rm /tmp/tmp.vPQNj72v2F /tmp/tmp.9Z5ircq6VO ++++ return 0 ++++ kubectl_bin get service/monitoring-service -o 'jsonpath={.status.loadBalancer.ingress[].hostname}' +++++ mktemp ++++ local LAST_OUT=/tmp/tmp.qMSS8nfgny +++++ mktemp ++++ local LAST_ERR=/tmp/tmp.a6PuyXtC6I ++++ local exit_status=0 +++++ seq 0 2 ++++ for i in $(seq 0 2) ++++ set +e ++++ kubectl get service/monitoring-service -o 'jsonpath={.status.loadBalancer.ingress[].hostname}' ++++ exit_status=0 ++++ set -e ++++ '[' 0 '!=' 0 ']' ++++ break ++++ cat /tmp/tmp.qMSS8nfgny ++++ cat /tmp/tmp.a6PuyXtC6I ++++ rm /tmp/tmp.qMSS8nfgny /tmp/tmp.a6PuyXtC6I ++++ return 0 +++ curl --insecure -s -X POST -H 'Content-Type: application/json' -d '{"name":"operator"}' --user 'admin:4 RTDc<0"()1 |=G' https://104.154.140.66/graph/api/serviceaccounts/2/tokens -w '\n%{http_code}' ++ token_response='{"id":1,"name":"operator","key":"glsa_QsLu14akDuQtobBNFr7DXn1hCxzsbu1n_6290e371"} 200' +++ tail -n1 +++ echo '{"id":1,"name":"operator","key":"glsa_QsLu14akDuQtobBNFr7DXn1hCxzsbu1n_6290e371"} 200' ++ token_status_code=200 +++ echo '{"id":1,"name":"operator","key":"glsa_QsLu14akDuQtobBNFr7DXn1hCxzsbu1n_6290e371"} 200' +++ sed '$ d' ++ token_json_response='{"id":1,"name":"operator","key":"glsa_QsLu14akDuQtobBNFr7DXn1hCxzsbu1n_6290e371"}' ++ [[ 200 -ne 200 ]] ++ echo '{"id":1,"name":"operator","key":"glsa_QsLu14akDuQtobBNFr7DXn1hCxzsbu1n_6290e371"}' ++ jq -r .key + TOKEN=glsa_QsLu14akDuQtobBNFr7DXn1hCxzsbu1n_6290e371 + kubectl_bin patch secret my-cluster-secrets --type merge --patch '{"stringData": {"pmmservertoken": "glsa_QsLu14akDuQtobBNFr7DXn1hCxzsbu1n_6290e371"}}' ++ mktemp + local LAST_OUT=/tmp/tmp.GHADkYtjL5 ++ mktemp + local LAST_ERR=/tmp/tmp.xW7c6RvNEC + local exit_status=0 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl patch secret my-cluster-secrets --type merge --patch '{"stringData": {"pmmservertoken": "glsa_QsLu14akDuQtobBNFr7DXn1hCxzsbu1n_6290e371"}}' + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.GHADkYtjL5 secret/my-cluster-secrets patched + cat /tmp/tmp.xW7c6RvNEC + rm /tmp/tmp.GHADkYtjL5 /tmp/tmp.xW7c6RvNEC + return 0 + desc 'create PXC cluster' + set +o xtrace ----------------------------------------------------------------------------------- create PXC cluster ----------------------------------------------------------------------------------- + spinup_pxc monitoring /mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/e2e-tests/monitoring-pmm3/conf/monitoring.yml 3 120 + local cluster=monitoring + local config=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/e2e-tests/monitoring-pmm3/conf/monitoring.yml + local size=3 + local sleep=120 + local pxcClientFile=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/e2e-tests/conf/client.yml + local port=3306 + desc 'create first PXC cluster' + set +o xtrace ----------------------------------------------------------------------------------- create first PXC cluster ----------------------------------------------------------------------------------- + apply_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/e2e-tests/conf/client.yml + local config_file=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/e2e-tests/conf/client.yml + local pvc_name= + '[' -z '' ']' + cat_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/e2e-tests/conf/client.yml '' + local input_file=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/e2e-tests/conf/client.yml + local pvc_name= + kubectl_bin apply -f - + cat /mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/e2e-tests/conf/client.yml ++ mktemp + /usr/bin/sed -e 's#image:.*-init$#image: perconalab/percona-xtradb-cluster-operator:PR-2546-caf25420#' + local LAST_OUT=/tmp/tmp.msDPSWKsKk + /usr/bin/sed -e 's#image:.*-backup$#image: perconalab/percona-xtradb-cluster-operator:main-pxc8.4-backup#' + /usr/bin/sed -e 's#image:.*\/percona-xtradb-cluster:.*$#image: perconalab/percona-xtradb-cluster-operator:main-pxc8.4#' + /usr/bin/sed -e 's#image:.*-pmm$#image: perconalab/pmm-client:dev-latest#' + /usr/bin/sed -e 's#image:.*-logcollector$#image: perconalab/fluentbit:main-logcollector#' + /usr/bin/sed -e 's#image:.*-proxysql$#image: perconalab/percona-xtradb-cluster-operator:main-proxysql#' + /usr/bin/sed -e 's#image:.*-pxc\([0-9]*.[0-9]*\)\{0,1\}$#image: perconalab/percona-xtradb-cluster-operator:main-pxc8.4#' + /usr/bin/sed -e 's#apiVersion: pxc.percona.com/v.*$#apiVersion: pxc.percona.com/v1#' + /usr/bin/sed -e 's#image:.*-haproxy$#image: perconalab/percona-xtradb-cluster-operator:main-haproxy#' ++ mktemp + /usr/bin/sed -e s~minio-service.#namespace~minio-service.monitoring-pmm3-26352~ + /usr/bin/sed -e 's#apply:.*#apply: Never#' + local LAST_ERR=/tmp/tmp.vdbRcrL2J5 + local exit_status=0 + /usr/bin/sed -e 's#claimName:..*-backup-pvc$#claimName: #' ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.msDPSWKsKk deployment.apps/pxc-client created + cat /tmp/tmp.vdbRcrL2J5 + rm /tmp/tmp.msDPSWKsKk /tmp/tmp.vdbRcrL2J5 + return 0 + [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.4 =~ 5\.7 ]] + apply_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/e2e-tests/monitoring-pmm3/conf/monitoring.yml + local config_file=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/e2e-tests/monitoring-pmm3/conf/monitoring.yml + local pvc_name= + '[' -z '' ']' + kubectl_bin apply -f - ++ mktemp + local LAST_OUT=/tmp/tmp.TsZ1ITvtEi ++ mktemp + cat_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/e2e-tests/monitoring-pmm3/conf/monitoring.yml '' + local input_file=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/e2e-tests/monitoring-pmm3/conf/monitoring.yml + local pvc_name= + local LAST_ERR=/tmp/tmp.CiL410oKUe + local exit_status=0 + cat /mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/e2e-tests/monitoring-pmm3/conf/monitoring.yml + /usr/bin/sed -e 's#image:.*-backup$#image: perconalab/percona-xtradb-cluster-operator:main-pxc8.4-backup#' + /usr/bin/sed -e 's#apiVersion: pxc.percona.com/v.*$#apiVersion: pxc.percona.com/v1#' + /usr/bin/sed -e 's#image:.*-proxysql$#image: perconalab/percona-xtradb-cluster-operator:main-proxysql#' + /usr/bin/sed -e 's#image:.*-pxc\([0-9]*.[0-9]*\)\{0,1\}$#image: perconalab/percona-xtradb-cluster-operator:main-pxc8.4#' + /usr/bin/sed -e 's#image:.*-pmm$#image: perconalab/pmm-client:dev-latest#' + /usr/bin/sed -e 's#image:.*-haproxy$#image: perconalab/percona-xtradb-cluster-operator:main-haproxy#' + /usr/bin/sed -e 's#apply:.*#apply: Never#' + /usr/bin/sed -e 's#image:.*-logcollector$#image: perconalab/fluentbit:main-logcollector#' + /usr/bin/sed -e 's#claimName:..*-backup-pvc$#claimName: #' + /usr/bin/sed -e s~minio-service.#namespace~minio-service.monitoring-pmm3-26352~ + /usr/bin/sed -e 's#image:.*\/percona-xtradb-cluster:.*$#image: perconalab/percona-xtradb-cluster-operator:main-pxc8.4#' + /usr/bin/sed -e 's#image:.*-init$#image: perconalab/percona-xtradb-cluster-operator:PR-2546-caf25420#' ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.TsZ1ITvtEi perconaxtradbcluster.pxc.percona.com/monitoring created + cat /tmp/tmp.CiL410oKUe + rm /tmp/tmp.TsZ1ITvtEi /tmp/tmp.CiL410oKUe + return 0 + desc 'check if all 3 Pods started' + set +o xtrace ----------------------------------------------------------------------------------- check if all 3 Pods started ----------------------------------------------------------------------------------- ++ get_proxy monitoring ++ local target_cluster=monitoring +++ kubectl_bin get pxc monitoring -o 'jsonpath={.spec.haproxy.enabled}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.ozoXgLOBle ++++ mktemp +++ local LAST_ERR=/tmp/tmp.etgD4osdMW +++ local exit_status=0 ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl get pxc monitoring -o 'jsonpath={.spec.haproxy.enabled}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 ']' +++ break +++ cat /tmp/tmp.ozoXgLOBle +++ cat /tmp/tmp.etgD4osdMW +++ rm /tmp/tmp.ozoXgLOBle /tmp/tmp.etgD4osdMW +++ return 0 ++ [[ true == \t\r\u\e ]] ++ echo monitoring-haproxy ++ return + local proxy=monitoring-haproxy + kubectl_bin wait --for=condition=Ready pod -l app.kubernetes.io/instance=monitoring,app.kubernetes.io/managed-by=percona-xtradb-cluster-operator --timeout=300s -n monitoring-pmm3-26352 ++ mktemp + local LAST_OUT=/tmp/tmp.hyO79f7ncf ++ mktemp + local LAST_ERR=/tmp/tmp.joLGa72cKb + local exit_status=0 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl wait --for=condition=Ready pod -l app.kubernetes.io/instance=monitoring,app.kubernetes.io/managed-by=percona-xtradb-cluster-operator --timeout=300s -n monitoring-pmm3-26352 + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.hyO79f7ncf pod/monitoring-haproxy-0 condition met pod/monitoring-pxc-0 condition met + cat /tmp/tmp.joLGa72cKb + rm /tmp/tmp.hyO79f7ncf /tmp/tmp.joLGa72cKb + return 0 + wait_for_running monitoring-haproxy 1 + local name=monitoring-haproxy + local size=1 + local last_pod + local max_retry=480 + [[ -z monitoring-haproxy ]] + [[ 1 -le 0 ]] + last_pod=0 + desc 'Waiting for 1 running pods for monitoring-haproxy' + set +o xtrace ----------------------------------------------------------------------------------- Waiting for 1 running pods for monitoring-haproxy ----------------------------------------------------------------------------------- ++ seq 0 0 + for i in $(seq 0 $last_pod) + wait_pod monitoring-haproxy-0 480 + local pod=monitoring-haproxy-0 + local max_retry=480 + local ns= ++ grep -E '^(pxc|proxysql)$' ++ echo monitoring-haproxy-0 ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' + local container= + set +o xtrace pod/monitoring-haproxy-0 condition met waiting for pod/monitoring-haproxy-0 to become Ready.Ok + wait_for_running monitoring-pxc 3 + local name=monitoring-pxc + local size=3 + local last_pod + local max_retry=480 + [[ -z monitoring-pxc ]] + [[ 3 -le 0 ]] + last_pod=2 + desc 'Waiting for 3 running pods for monitoring-pxc' + set +o xtrace ----------------------------------------------------------------------------------- Waiting for 3 running pods for monitoring-pxc ----------------------------------------------------------------------------------- ++ seq 0 2 + for i in $(seq 0 $last_pod) + wait_pod monitoring-pxc-0 480 + local pod=monitoring-pxc-0 + local max_retry=480 + local ns= ++ echo monitoring-pxc-0 ++ grep -E '^(pxc|proxysql)$' ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' + local container=pxc + set +o xtrace pod/monitoring-pxc-0 condition met waiting for pod/monitoring-pxc-0 to become Ready.Ok + for i in $(seq 0 $last_pod) + wait_pod monitoring-pxc-1 480 + local pod=monitoring-pxc-1 + local max_retry=480 + local ns= ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ echo monitoring-pxc-1 ++ grep -E '^(pxc|proxysql)$' + local container=pxc + set +o xtrace pod/monitoring-pxc-1 condition met waiting for pod/monitoring-pxc-1 to become Ready.Ok + for i in $(seq 0 $last_pod) + wait_pod monitoring-pxc-2 480 + local pod=monitoring-pxc-2 + local max_retry=480 + local ns= ++ echo monitoring-pxc-2 ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ grep -E '^(pxc|proxysql)$' + local container=pxc + set +o xtrace pod/monitoring-pxc-2 condition met waiting for pod/monitoring-pxc-2 to become Ready.Ok + sleep 120 ++ kubectl get pxc monitoring -o 'jsonpath={.spec.secretsName}' + local secret_name=my-cluster-secrets ++ getSecretData my-cluster-secrets root ++ local secretName=my-cluster-secrets ++ local dataKey=root ++ kubectl_bin get secrets/my-cluster-secrets '--template={{.data.root}}' ++ base64 --decode +++ mktemp ++ local LAST_OUT=/tmp/tmp.Juu1EW6RoJ +++ mktemp ++ local LAST_ERR=/tmp/tmp.HYboxK0Tr0 ++ local exit_status=0 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get secrets/my-cluster-secrets '--template={{.data.root}}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.Juu1EW6RoJ ++ cat /tmp/tmp.HYboxK0Tr0 ++ rm /tmp/tmp.Juu1EW6RoJ /tmp/tmp.HYboxK0Tr0 ++ return 0 + local root_pass=root_password + desc 'write data' + set +o xtrace ----------------------------------------------------------------------------------- write data ----------------------------------------------------------------------------------- + [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.4 =~ 5\.7 ]] + run_mysql 'CREATE DATABASE IF NOT EXISTS myApp; use myApp; CREATE TABLE IF NOT EXISTS myApp (id int PRIMARY KEY) ;' '-h monitoring-haproxy -uroot -p'\''root_password'\'' -P3306' + local 'command=CREATE DATABASE IF NOT EXISTS myApp; use myApp; CREATE TABLE IF NOT EXISTS myApp (id int PRIMARY KEY) ;' + local 'uri=-h monitoring-haproxy -uroot -p'\''root_password'\'' -P3306' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.qGjspsK5wv +++ mktemp ++ local LAST_ERR=/tmp/tmp.xabdAOPteL ++ local exit_status=0 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.qGjspsK5wv ++ cat /tmp/tmp.xabdAOPteL ++ rm /tmp/tmp.qGjspsK5wv /tmp/tmp.xabdAOPteL ++ return 0 + client_pod=pxc-client-58d9f9fc48-xtxrf + wait_pod pxc-client-58d9f9fc48-xtxrf + local pod=pxc-client-58d9f9fc48-xtxrf + local max_retry=480 + local ns= ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ echo pxc-client-58d9f9fc48-xtxrf ++ grep -E '^(pxc|proxysql)$' + local container= + set +o xtrace pod/pxc-client-58d9f9fc48-xtxrf condition met waiting for pod/pxc-client-58d9f9fc48-xtxrf to become ReadyDefaulted container "pxc-client" out of: pxc-client, backup .Ok + set +o xtrace + run_mysql 'INSERT myApp.myApp (id) VALUES (100500)' '-h monitoring-haproxy -uroot -p'\''root_password'\'' -P3306' + local 'command=INSERT myApp.myApp (id) VALUES (100500)' + local 'uri=-h monitoring-haproxy -uroot -p'\''root_password'\'' -P3306' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.d1dKCsHd3J +++ mktemp ++ local LAST_ERR=/tmp/tmp.Kybr7wIr34 ++ local exit_status=0 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.d1dKCsHd3J ++ cat /tmp/tmp.Kybr7wIr34 ++ rm /tmp/tmp.d1dKCsHd3J /tmp/tmp.Kybr7wIr34 ++ return 0 + client_pod=pxc-client-58d9f9fc48-xtxrf + wait_pod pxc-client-58d9f9fc48-xtxrf + local pod=pxc-client-58d9f9fc48-xtxrf + local max_retry=480 + local ns= ++ echo pxc-client-58d9f9fc48-xtxrf ++ grep -E '^(pxc|proxysql)$' ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' + local container= + set +o xtrace pod/pxc-client-58d9f9fc48-xtxrf condition met waiting for pod/pxc-client-58d9f9fc48-xtxrf to become ReadyDefaulted container "pxc-client" out of: pxc-client, backup .Ok + set +o xtrace + sleep 30 ++ seq 0 2 + for i in $(seq 0 $((size - 1))) + compare_mysql_cmd select-1 'SELECT * from myApp.myApp;' '-h monitoring-pxc-0.monitoring-pxc -uroot -p'\''root_password'\'' -P3306' + local command_id=select-1 + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h monitoring-pxc-0.monitoring-pxc -uroot -p'\''root_password'\'' -P3306' + local postfix= + local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/e2e-tests/monitoring-pmm3/compare/select-1.sql + [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.4 =~ 8\.4 ]] + [[ -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/e2e-tests/monitoring-pmm3/compare/select-1-84.sql ]] + [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.4 =~ 8\.0 ]] + [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.4 =~ 5\.7 ]] + run_mysql 'SELECT * from myApp.myApp;' '-h monitoring-pxc-0.monitoring-pxc -uroot -p'\''root_password'\'' -P3306' + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h monitoring-pxc-0.monitoring-pxc -uroot -p'\''root_password'\'' -P3306' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.w5BTfFmmQX +++ mktemp ++ local LAST_ERR=/tmp/tmp.GAN8Wrpcdm ++ local exit_status=0 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.w5BTfFmmQX ++ cat /tmp/tmp.GAN8Wrpcdm ++ rm /tmp/tmp.w5BTfFmmQX /tmp/tmp.GAN8Wrpcdm ++ return 0 + client_pod=pxc-client-58d9f9fc48-xtxrf + wait_pod pxc-client-58d9f9fc48-xtxrf + local pod=pxc-client-58d9f9fc48-xtxrf + local max_retry=480 + local ns= ++ echo pxc-client-58d9f9fc48-xtxrf ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ grep -E '^(pxc|proxysql)$' + local container= + set +o xtrace pod/pxc-client-58d9f9fc48-xtxrf condition met waiting for pod/pxc-client-58d9f9fc48-xtxrf to become ReadyDefaulted container "pxc-client" out of: pxc-client, backup .Ok + set +o xtrace + [[ ! -s /tmp/tmp.mnnOTDKNTG/select-1.sql ]] ++ grep 'Unknown MySQL server host' /tmp/tmp.mnnOTDKNTG/select-1.sql + [[ -n '' ]] + [[ 0 -eq 0 ]] + diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/e2e-tests/monitoring-pmm3/compare/select-1.sql /tmp/tmp.mnnOTDKNTG/select-1.sql + for i in $(seq 0 $((size - 1))) + compare_mysql_cmd select-1 'SELECT * from myApp.myApp;' '-h monitoring-pxc-1.monitoring-pxc -uroot -p'\''root_password'\'' -P3306' + local command_id=select-1 + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h monitoring-pxc-1.monitoring-pxc -uroot -p'\''root_password'\'' -P3306' + local postfix= + local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/e2e-tests/monitoring-pmm3/compare/select-1.sql + [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.4 =~ 8\.4 ]] + [[ -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/e2e-tests/monitoring-pmm3/compare/select-1-84.sql ]] + [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.4 =~ 8\.0 ]] + [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.4 =~ 5\.7 ]] + run_mysql 'SELECT * from myApp.myApp;' '-h monitoring-pxc-1.monitoring-pxc -uroot -p'\''root_password'\'' -P3306' + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h monitoring-pxc-1.monitoring-pxc -uroot -p'\''root_password'\'' -P3306' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.NoOF6JQF6J +++ mktemp ++ local LAST_ERR=/tmp/tmp.CSKxvLrmGp ++ local exit_status=0 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.NoOF6JQF6J ++ cat /tmp/tmp.CSKxvLrmGp ++ rm /tmp/tmp.NoOF6JQF6J /tmp/tmp.CSKxvLrmGp ++ return 0 + client_pod=pxc-client-58d9f9fc48-xtxrf + wait_pod pxc-client-58d9f9fc48-xtxrf + local pod=pxc-client-58d9f9fc48-xtxrf + local max_retry=480 + local ns= ++ grep -E '^(pxc|proxysql)$' ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ echo pxc-client-58d9f9fc48-xtxrf + local container= + set +o xtrace pod/pxc-client-58d9f9fc48-xtxrf condition met waiting for pod/pxc-client-58d9f9fc48-xtxrf to become ReadyDefaulted container "pxc-client" out of: pxc-client, backup .Ok + set +o xtrace + [[ ! -s /tmp/tmp.mnnOTDKNTG/select-1.sql ]] ++ grep 'Unknown MySQL server host' /tmp/tmp.mnnOTDKNTG/select-1.sql + [[ -n '' ]] + [[ 0 -eq 0 ]] + diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/e2e-tests/monitoring-pmm3/compare/select-1.sql /tmp/tmp.mnnOTDKNTG/select-1.sql + for i in $(seq 0 $((size - 1))) + compare_mysql_cmd select-1 'SELECT * from myApp.myApp;' '-h monitoring-pxc-2.monitoring-pxc -uroot -p'\''root_password'\'' -P3306' + local command_id=select-1 + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h monitoring-pxc-2.monitoring-pxc -uroot -p'\''root_password'\'' -P3306' + local postfix= + local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/e2e-tests/monitoring-pmm3/compare/select-1.sql + [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.4 =~ 8\.4 ]] + [[ -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/e2e-tests/monitoring-pmm3/compare/select-1-84.sql ]] + [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.4 =~ 8\.0 ]] + [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.4 =~ 5\.7 ]] + run_mysql 'SELECT * from myApp.myApp;' '-h monitoring-pxc-2.monitoring-pxc -uroot -p'\''root_password'\'' -P3306' + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h monitoring-pxc-2.monitoring-pxc -uroot -p'\''root_password'\'' -P3306' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.8x0nPgTPui +++ mktemp ++ local LAST_ERR=/tmp/tmp.xbbhOEGs7s ++ local exit_status=0 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.8x0nPgTPui ++ cat /tmp/tmp.xbbhOEGs7s ++ rm /tmp/tmp.8x0nPgTPui /tmp/tmp.xbbhOEGs7s ++ return 0 + client_pod=pxc-client-58d9f9fc48-xtxrf + wait_pod pxc-client-58d9f9fc48-xtxrf + local pod=pxc-client-58d9f9fc48-xtxrf + local max_retry=480 + local ns= ++ grep -E '^(pxc|proxysql)$' ++ echo pxc-client-58d9f9fc48-xtxrf ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' + local container= + set +o xtrace pod/pxc-client-58d9f9fc48-xtxrf condition met waiting for pod/pxc-client-58d9f9fc48-xtxrf to become ReadyDefaulted container "pxc-client" out of: pxc-client, backup .Ok + set +o xtrace + [[ ! -s /tmp/tmp.mnnOTDKNTG/select-1.sql ]] ++ grep 'Unknown MySQL server host' /tmp/tmp.mnnOTDKNTG/select-1.sql + [[ -n '' ]] + [[ 0 -eq 0 ]] + diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/e2e-tests/monitoring-pmm3/compare/select-1.sql /tmp/tmp.mnnOTDKNTG/select-1.sql ++ is_keyring_plugin_in_use monitoring ++ local cluster=monitoring ++ [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.4 =~ 8\.4 ]] ++ kubectl exec monitoring-pxc-0 -c pxc -- ls /var/lib/mysql/mysqld.my ++ return 1 + '[' '' ']' + wait_for_generation sts/monitoring-pxc 1 + local resource=sts/monitoring-pxc + local target_generation=1 + echo 'Waiting for sts/monitoring-pxc to reach generation 1...' Waiting for sts/monitoring-pxc to reach generation 1... + true ++ kubectl get sts/monitoring-pxc -o 'jsonpath={.metadata.generation}' + current_generation=1 + '[' 1 -eq 1 ']' + echo 'Resource sts/monitoring-pxc has reached generation 1.' Resource sts/monitoring-pxc has reached generation 1. + break + wait_for_generation sts/monitoring-haproxy 1 + local resource=sts/monitoring-haproxy + local target_generation=1 + echo 'Waiting for sts/monitoring-haproxy to reach generation 1...' Waiting for sts/monitoring-haproxy to reach generation 1... + true ++ kubectl get sts/monitoring-haproxy -o 'jsonpath={.metadata.generation}' + current_generation=1 + '[' 1 -eq 1 ']' + echo 'Resource sts/monitoring-haproxy has reached generation 1.' Resource sts/monitoring-haproxy has reached generation 1. + break + sleep 10 + kubectl wait pod -l app.kubernetes.io/managed-by=percona-xtradb-cluster-operator --for=condition=ready --timeout=600s pod/monitoring-haproxy-0 condition met pod/monitoring-haproxy-1 condition met pod/monitoring-pxc-0 condition met pod/monitoring-pxc-1 condition met pod/monitoring-pxc-2 condition met + wait_cluster_consistency monitoring 3 2 + local cluster_name=monitoring + local pxc_size=3 + local proxy_size=2 + desc 'wait cluster consistency' + set +o xtrace ----------------------------------------------------------------------------------- wait cluster consistency ----------------------------------------------------------------------------------- + local max_seconds=1500 + local delay=5 + local retries=300 + local pxc_ready + local proxy_ready + sleep 7 + echo -n 'waiting for pxc/monitoring to be ready' waiting for pxc/monitoring to be ready++ seq 1 300 + for _ in $(seq 1 "${retries}") ++ kubectl_bin get pxc monitoring -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.nqCtyKLqyf +++ mktemp ++ local LAST_ERR=/tmp/tmp.Be85LvdzhV ++ local exit_status=0 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pxc monitoring -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.nqCtyKLqyf ++ cat /tmp/tmp.Be85LvdzhV ++ rm /tmp/tmp.nqCtyKLqyf /tmp/tmp.Be85LvdzhV ++ return 0 + state=ready ++ kubectl_bin get pxc monitoring -o 'jsonpath={.status.pxc.ready}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.Fr2TdQ601M +++ mktemp ++ local LAST_ERR=/tmp/tmp.TRs8Svy3hw ++ local exit_status=0 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pxc monitoring -o 'jsonpath={.status.pxc.ready}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.Fr2TdQ601M ++ cat /tmp/tmp.TRs8Svy3hw ++ rm /tmp/tmp.Fr2TdQ601M /tmp/tmp.TRs8Svy3hw ++ return 0 + pxc_ready=3 + proxy_ready=0 + [[ 2 -gt 0 ]] +++ get_proxy_engine monitoring +++ local cluster_name=monitoring ++++ get_proxy monitoring ++++ local target_cluster=monitoring +++++ kubectl_bin get pxc monitoring -o 'jsonpath={.spec.haproxy.enabled}' ++++++ mktemp +++++ local LAST_OUT=/tmp/tmp.Al7jiCC0m7 ++++++ mktemp +++++ local LAST_ERR=/tmp/tmp.IPgpz9yzGz +++++ local exit_status=0 ++++++ seq 0 2 +++++ for i in $(seq 0 2) +++++ set +e +++++ kubectl get pxc monitoring -o 'jsonpath={.spec.haproxy.enabled}' +++++ exit_status=0 +++++ set -e +++++ '[' 0 '!=' 0 ']' +++++ break +++++ cat /tmp/tmp.Al7jiCC0m7 +++++ cat /tmp/tmp.IPgpz9yzGz +++++ rm /tmp/tmp.Al7jiCC0m7 /tmp/tmp.IPgpz9yzGz +++++ return 0 ++++ [[ true == \t\r\u\e ]] ++++ echo monitoring-haproxy ++++ return +++ local cluster_proxy=monitoring-haproxy +++ echo haproxy ++ kubectl_bin get pxc monitoring -o 'jsonpath={.status.haproxy.ready}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.srM89XDUu9 +++ mktemp ++ local LAST_ERR=/tmp/tmp.bDAGDKkC39 ++ local exit_status=0 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pxc monitoring -o 'jsonpath={.status.haproxy.ready}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.srM89XDUu9 ++ cat /tmp/tmp.bDAGDKkC39 ++ rm /tmp/tmp.srM89XDUu9 /tmp/tmp.bDAGDKkC39 ++ return 0 + proxy_ready=2 + [[ ready == \r\e\a\d\y ]] + [[ 3 == \3 ]] + [[ 2 == \2 ]] + echo + return + compare_kubectl statefulset/monitoring-pxc -no-prefix + local resource=statefulset/monitoring-pxc + local postfix=-no-prefix + local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/e2e-tests/monitoring-pmm3/compare/statefulset_monitoring-pxc-no-prefix.yml + local new_result=/tmp/tmp.mnnOTDKNTG/statefulset_monitoring-pxc.yml + desc 'compare statefulset/monitoring-pxc--no-prefix' + set +o xtrace ----------------------------------------------------------------------------------- compare statefulset/monitoring-pxc--no-prefix ----------------------------------------------------------------------------------- + '[' 0 = 1 -a -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/e2e-tests/monitoring-pmm3/compare/statefulset_monitoring-pxc-no-prefix-eks.yml ']' + [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.4 =~ 8\.0 ]] + [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.4 =~ 8\.4 ]] + '[' -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/e2e-tests/monitoring-pmm3/compare/statefulset_monitoring-pxc-no-prefix-84.yml ']' + version_gt 1.33 ++ echo '1.34 >= 1.33' ++ bc -l + '[' 1 -eq 1 ']' + return 0 + '[' -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/e2e-tests/monitoring-pmm3/compare/statefulset_monitoring-pxc-no-prefix-k133.yml ']' + version_gt 1.29 ++ bc -l ++ echo '1.34 >= 1.29' + '[' 1 -eq 1 ']' + return 0 + '[' -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/e2e-tests/monitoring-pmm3/compare/statefulset_monitoring-pxc-no-prefix-k129.yml ']' + version_gt 1.27 ++ bc -l ++ echo '1.34 >= 1.27' + '[' 1 -eq 1 ']' + return 0 + '[' -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/e2e-tests/monitoring-pmm3/compare/statefulset_monitoring-pxc-no-prefix-k127.yml ']' + expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/e2e-tests/monitoring-pmm3/compare/statefulset_monitoring-pxc-no-prefix-k127.yml + '[' '!' -z '' -a -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/e2e-tests/monitoring-pmm3/compare/statefulset_monitoring-pxc-no-prefix-k127-oc.yml ']' + version_gt 1.29 ++ echo '1.34 >= 1.29' ++ bc -l + '[' 1 -eq 1 ']' + return 0 + '[' '!' -z '' -a -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/e2e-tests/monitoring-pmm3/compare/statefulset_monitoring-pxc-no-prefix-k127-k129-oc.yml ']' + '[' 0 = 1 -a -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/e2e-tests/monitoring-pmm3/compare/statefulset_monitoring-pxc-no-prefix-k127-eks.yml ']' + '[' 0 = 1 -a -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/e2e-tests/monitoring-pmm3/compare/statefulset_monitoring-pxc-no-prefix-k127-aks.yml ']' + yq eval ' del(.metadata.managedFields) | del(.. | select(has("creationTimestamp")).creationTimestamp) | del(.. | select(has("namespace")).namespace) | del(.. | select(has("uid")).uid) | del(.metadata.resourceVersion) | del(.spec.template.spec.containers[].env[] | select(.name == "CLUSTER_HASH")) | del(.spec.template.spec.containers[].env[] | select(.name == "S3_BUCKET_PATH")) | del(.spec.template.spec.containers[].env[] | select(.name == "BACKUP_PATH")) | del(.spec.template.spec.containers[].env[] | select(.name == "S3_BUCKET_URL")) | del(.spec.template.spec.containers[].env[] | select(.name == "AZURE_CONTAINER_NAME")) | del(.spec.template.spec.containers[].env[] | select(.name == "XTRABACKUP_ENABLED")) | del(.metadata.selfLink) | del(.metadata.deletionTimestamp) | del(.metadata.labels."batch.kubernetes.io/job-name") | del(.metadata.labels."job-name") | del(.metadata.labels."topology.kubernetes.io/region") | del(.metadata.labels."topology.kubernetes.io/zone") | del(.metadata.annotations."kubectl.kubernetes.io/last-applied-configuration") | del(.metadata.annotations."kubernetes.io/psp") | del(.metadata.annotations."batch.kubernetes.io/job-tracking") | del(.metadata.annotations."networking.gke.io/target-pool") | del(.metadata.annotations."cloud.google.com/neg") | del(.metadata.annotations."k8s.v1.cni.cncf.io*") | del(.metadata.annotations."k8s.ovn.org/pod-networks") | del(.metadata.annotations."kubernetes.digitalocean.com/load-balancer-id") | del(.metadata.annotations."service.beta.kubernetes.io/do-loadbalancer-type") | del(.spec.template.metadata.annotations."last-applied-secret") | del(.spec.template.metadata.labels."batch.kubernetes.io/job-name") | del(.spec.template.metadata.labels."job-name") | del(.. | select(has("batch.kubernetes.io/controller-uid"))."batch.kubernetes.io/controller-uid") | del(.. | select(has("image")).image) | del(.. | select(has("clusterIP")).clusterIP) | del(.. | select(has("clusterIPs")).clusterIPs) | del(.. | select(has("dataSource")).dataSource) | del(.. | select(has("procMount")).procMount) | del(.. | select(has("storageClassName")).storageClassName) | del(.. | select(has("finalizers")).finalizers) | del(.. | select(has("kubernetes.io/pvc-protection"))."kubernetes.io/pvc-protection") | del(.. | select(has("volumeName")).volumeName) | del(.. | select(has("volume.beta.kubernetes.io/storage-provisioner"))."volume.beta.kubernetes.io/storage-provisioner") | del(.. | select(has("volume.kubernetes.io/storage-provisioner"))."volume.kubernetes.io/storage-provisioner") | del(.spec.volumeMode) | del(.spec.nodeName) | del(.. | select(has("volume.kubernetes.io/selected-node"))."volume.kubernetes.io/selected-node") | del(.. | select(has("percona.com/last-config-hash"))."percona.com/last-config-hash") | del(.. | select(has("percona.com/configuration-hash"))."percona.com/configuration-hash") | del(.. | select(has("percona.com/env-secret-config-hash"))."percona.com/env-secret-config-hash") | del(.. | select(has("percona.com/ssl-hash"))."percona.com/ssl-hash") | del(.. | select(has("percona.com/ssl-internal-hash"))."percona.com/ssl-internal-hash") | del(.. | select(has("kubectl.kubernetes.io/default-container"))."kubectl.kubernetes.io/default-container") | del(.spec.volumeClaimTemplates[].spec.volumeMode | select(. == "Filesystem")) | del(.. | select(has("healthCheckNodePort")).healthCheckNodePort) | del(.. | select(has("nodePort")).nodePort) | del(.. | select(has("imagePullSecrets")).imagePullSecrets) | del(.. | select(has("enableServiceLinks")).enableServiceLinks) | del(.status) | del(.spec.volumeClaimTemplates[].apiVersion) | del(.spec.volumeClaimTemplates[].kind) | del(.metadata.ownerReferences[].apiVersion) | del(.. | select(has("controller-uid")).controller-uid) | del(.. | select(has("preemptionPolicy")).preemptionPolicy) | del(.spec.ipFamilies) | del(.spec.ipFamilyPolicy) | del(.spec.updateStrategy.rollingUpdate.+ kubectl_bin get -o yaml statefulset/monitoring-pxc maxUnavailable) | (.. | select(. == "policy/v1beta1")) = "policy/v1" | del(.. | select(has("kubernetes.io/hostname"))."kubernetes.io/hostname") | (.. | select(tag == "!!str")) |= sub("monitoring-pmm3-26352", "namespace") | (.. | select(tag == "!!str")) |= sub("kube-api-access-.*", "kube-api-access") | del(.. | select(has("annotations")).annotations | select(length==0)) | del(.spec.crVersion) | del(.. | select(.[] == "percona-xtradb-cluster-operator-workload-token*"))' - ++ mktemp + local LAST_OUT=/tmp/tmp.v1gGQwsOJz ++ mktemp + local LAST_ERR=/tmp/tmp.e5yIMKv3l7 + local exit_status=0 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl get -o yaml statefulset/monitoring-pxc + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.v1gGQwsOJz + cat /tmp/tmp.e5yIMKv3l7 + rm /tmp/tmp.v1gGQwsOJz /tmp/tmp.e5yIMKv3l7 + return 0 + [[ 0 -eq 0 ]] + diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/e2e-tests/monitoring-pmm3/compare/statefulset_monitoring-pxc-no-prefix-k127.yml /tmp/tmp.mnnOTDKNTG/statefulset_monitoring-pxc.yml + log 'compare_kubectl: statefulset/monitoring-pxc OK' ++ date +%Y-%m-%dT%H:%M:%S%z + echo '[2026-10-05T14:42:24+0000]' compare_kubectl: statefulset/monitoring-pxc OK [2026-10-05T14:42:24+0000] compare_kubectl: statefulset/monitoring-pxc OK + compare_kubectl statefulset/monitoring-haproxy -no-prefix + local resource=statefulset/monitoring-haproxy + local postfix=-no-prefix + local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/e2e-tests/monitoring-pmm3/compare/statefulset_monitoring-haproxy-no-prefix.yml + local new_result=/tmp/tmp.mnnOTDKNTG/statefulset_monitoring-haproxy.yml + desc 'compare statefulset/monitoring-haproxy--no-prefix' + set +o xtrace ----------------------------------------------------------------------------------- compare statefulset/monitoring-haproxy--no-prefix ----------------------------------------------------------------------------------- + '[' 0 = 1 -a -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/e2e-tests/monitoring-pmm3/compare/statefulset_monitoring-haproxy-no-prefix-eks.yml ']' + [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.4 =~ 8\.0 ]] + [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.4 =~ 8\.4 ]] + '[' -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/e2e-tests/monitoring-pmm3/compare/statefulset_monitoring-haproxy-no-prefix-84.yml ']' + version_gt 1.33 ++ echo '1.34 >= 1.33' ++ bc -l + '[' 1 -eq 1 ']' + return 0 + '[' -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/e2e-tests/monitoring-pmm3/compare/statefulset_monitoring-haproxy-no-prefix-k133.yml ']' + version_gt 1.29 ++ bc -l ++ echo '1.34 >= 1.29' + '[' 1 -eq 1 ']' + return 0 + '[' -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/e2e-tests/monitoring-pmm3/compare/statefulset_monitoring-haproxy-no-prefix-k129.yml ']' + version_gt 1.27 ++ bc -l ++ echo '1.34 >= 1.27' + '[' 1 -eq 1 ']' + return 0 + '[' -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/e2e-tests/monitoring-pmm3/compare/statefulset_monitoring-haproxy-no-prefix-k127.yml ']' + expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/e2e-tests/monitoring-pmm3/compare/statefulset_monitoring-haproxy-no-prefix-k127.yml + '[' '!' -z '' -a -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/e2e-tests/monitoring-pmm3/compare/statefulset_monitoring-haproxy-no-prefix-k127-oc.yml ']' + version_gt 1.29 ++ bc -l ++ echo '1.34 >= 1.29' + '[' 1 -eq 1 ']' + return 0 + '[' '!' -z '' -a -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/e2e-tests/monitoring-pmm3/compare/statefulset_monitoring-haproxy-no-prefix-k127-k129-oc.yml ']' + '[' 0 = 1 -a -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/e2e-tests/monitoring-pmm3/compare/statefulset_monitoring-haproxy-no-prefix-k127-eks.yml ']' + '[' 0 = 1 -a -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/e2e-tests/monitoring-pmm3/compare/statefulset_monitoring-haproxy-no-prefix-k127-aks.yml ']' + yq eval ' del(.metadata.managedFields) | del(.. | select(has("creationTimestamp")).creationTimestamp) | del(.. | select(has("namespace")).namespace) | del(.. | select(has("uid")).uid) | del(.metadata.resourceVersion) | del(.spec.template.spec.containers[].env[] | select(.name == "CLUSTER_HASH")) | del(.spec.template.spec.containers[].env[] | select(.name == "S3_BUCKET_PATH")) | del(.spec.template.spec.containers[].env[] | select(.name == "BACKUP_PATH")) | del(.spec.template.spec.containers[].env[] | select(.name == "S3_BUCKET_URL")) | del(.spec.template.spec.containers[].env[] | select(.name == "AZURE_CONTAINER_NAME")) | del(.spec.template.spec.containers[].env[] | select(.name == "XTRABACKUP_ENABLED")) | del(.metadata.selfLink) | del(.metadata.deletionTimestamp) | del(.metadata.labels."batch.kubernetes.io/job-name") | del(.metadata.labels."job-name") | del(.metadata.labels."topology.kubernetes.io/region") | del(.metadata.labels."topology.kubernetes.io/zone") | del(.metadata.annotations."kubectl.kubernetes.io/last-applied-configuration") | del(.metadata.annotations."kubernetes.io/psp") | del(.metadata.annotations."batch.kubernetes.io/job-tracking") | del(.metadata.annotations."networking.gke.io/target-pool") | del(.metadata.annotations."cloud.google.com/neg") | del(.metadata.annotations."k8s.v1.cni.cncf.io*") | del(.metadata.annotations."k8s.ovn.org/pod-networks") | del(.metadata.annotations."kubernetes.digitalocean.com/load-balancer-id") | del(.metadata.annotations."service.beta.kubernetes.io/do-loadbalancer-type") | del(.spec.template.metadata.annotations."last-applied-secret") | del(.spec.template.metadata.labels."batch.kubernetes.io/job-name") | del(.spec.template.metadata.labels."job-name") | del(.. | select(has("batch.kubernetes.io/controller-uid"))."batch.kubernetes.io/controller-uid") | del(.. | select(has("image")).image) | del(.. | select(has("clusterIP")).clusterIP) | del(.. | select(has("clusterIPs")).clusterIPs) | del(.. | select(has("dataSource")).dataSource) | del(.. | select(has("procMount")).procMount) | del(.. | select(has("storageClassName")).storageClassName) | del(.. | select(has("finalizers")).finalizers) | del(.. | select(has("kubernetes.io/pvc-protection"))."kubernetes.io/pvc-protection") | del(.. | select(has("volumeName")).volumeName) | del(.. | select(has("volume.beta.kubernetes.io/storage-provisioner"))."volume.beta.kubernetes.io/storage-provisioner") | del(.. | select(has("volume.kubernetes.io/storage-provisioner"))."volume.kubernetes.io/storage-provisioner") | del(.spec.volumeMode) | del(.spec.nodeName) | del(.. | select(has("volume.kubernetes.io/selected-node"))."volume.kubernetes.io/selected-node") | del(.. | select(has("percona.com/last-config-hash"))."percona.com/last-config-hash") | del(.. | select(has("percona.com/configuration-hash"))."percona.com/configuration-hash") | del(.. | select(has("percona.com/env-secret-config-hash"))."percona.com/env-secret-config-hash") | del(.. | select(has("percona.com/ssl-hash"))."percona.com/ssl-hash") | del(.. | select(has("percona.com/ssl-internal-hash"))."percona.com/ssl-internal-hash") | del(.. | select(has("kubectl.kubernetes.io/default-container"))."kubectl.kubernetes.io/default-container") | del(.spec.volumeClaimTemplates[].spec.volumeMode | select(. == "Filesystem")) | del(.. | select(has("healthCheckNodePort")).healthCheckNodePort) | del(.. | select(has("nodePort")).nodePort) | del(.. | select(has("imagePullSecrets")).imagePullSecrets) | del(.. | select(has("enableServiceLinks")).enableServiceLinks) | del(.status) | del(.spec.volumeClaimTemplates[].apiVersion) | del(.spec.volumeClaimTemplates[].kind) | del(.metadata.ownerReferences[].apiVersion) | del(.. | select(has("controller-uid")).controller-uid) | del(.. | select(has("preemptionPolicy")).preemptionPolicy) | del(.spec.ipFamilies) | del(.spec.ipFamilyPolicy) | del(.spec.updateStrategy.rollingUpdate.maxUnavailable) | (.. | select(. == "policy/v1beta1")) = "policy/v1" | del(.. | select(has("kubernetes.io/hostname"))."kubernetes.io/hostname") | (.. | select(tag == "!!str")) |= sub("monitoring-pmm3-26352", "namespace") | (.. | select(tag == "!!str")) |= sub("kube-api-access-.*", "kube-api-access") | del(.. | select(has("annotations")).annotations | select(length==0)) | del(.spec.crVersion) | del(.. | select(.[] == "percona-xtradb-cluster-operator-workload-token*"))' - + kubectl_bin get -o yaml statefulset/monitoring-haproxy ++ mktemp + local LAST_OUT=/tmp/tmp.axdqpVZClj ++ mktemp + local LAST_ERR=/tmp/tmp.d3IbnbJIo2 + local exit_status=0 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl get -o yaml statefulset/monitoring-haproxy + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.axdqpVZClj + cat /tmp/tmp.d3IbnbJIo2 + rm /tmp/tmp.axdqpVZClj /tmp/tmp.d3IbnbJIo2 + return 0 + [[ 0 -eq 0 ]] + diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/e2e-tests/monitoring-pmm3/compare/statefulset_monitoring-haproxy-no-prefix-k127.yml /tmp/tmp.mnnOTDKNTG/statefulset_monitoring-haproxy.yml + log 'compare_kubectl: statefulset/monitoring-haproxy OK' ++ date +%Y-%m-%dT%H:%M:%S%z + echo '[2026-10-05T14:42:28+0000]' compare_kubectl: statefulset/monitoring-haproxy OK [2026-10-05T14:42:28+0000] compare_kubectl: statefulset/monitoring-haproxy OK + desc 'apply my-env-var-secrets to add PMM_PREFIX' + set +o xtrace ----------------------------------------------------------------------------------- apply my-env-var-secrets to add PMM_PREFIX ----------------------------------------------------------------------------------- + kubectl_bin apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/e2e-tests/monitoring-pmm3/conf/envsecrets.yaml ++ mktemp + local LAST_OUT=/tmp/tmp.dz7n9oMsms ++ mktemp + local LAST_ERR=/tmp/tmp.DujzbMMM8f + local exit_status=0 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/e2e-tests/monitoring-pmm3/conf/envsecrets.yaml + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.dz7n9oMsms secret/my-env-var-secrets created + cat /tmp/tmp.DujzbMMM8f + rm /tmp/tmp.dz7n9oMsms /tmp/tmp.DujzbMMM8f + return 0 + wait_for_generation sts/monitoring-pxc 2 + local resource=sts/monitoring-pxc + local target_generation=2 + echo 'Waiting for sts/monitoring-pxc to reach generation 2...' Waiting for sts/monitoring-pxc to reach generation 2... + true ++ kubectl get sts/monitoring-pxc -o 'jsonpath={.metadata.generation}' + current_generation=1 + '[' 1 -eq 2 ']' + echo 'Resource sts/monitoring-pxc is at generation 1. Waiting...' Resource sts/monitoring-pxc is at generation 1. Waiting... + sleep 5 + true ++ kubectl get sts/monitoring-pxc -o 'jsonpath={.metadata.generation}' + current_generation=2 + '[' 2 -eq 2 ']' + echo 'Resource sts/monitoring-pxc has reached generation 2.' Resource sts/monitoring-pxc has reached generation 2. + break + wait_for_generation sts/monitoring-haproxy 2 + local resource=sts/monitoring-haproxy + local target_generation=2 + echo 'Waiting for sts/monitoring-haproxy to reach generation 2...' Waiting for sts/monitoring-haproxy to reach generation 2... + true ++ kubectl get sts/monitoring-haproxy -o 'jsonpath={.metadata.generation}' + current_generation=2 + '[' 2 -eq 2 ']' + echo 'Resource sts/monitoring-haproxy has reached generation 2.' Resource sts/monitoring-haproxy has reached generation 2. + break + desc 'create new PMM token and add it to the secret' + set +o xtrace ----------------------------------------------------------------------------------- create new PMM token and add it to the secret ----------------------------------------------------------------------------------- ++ get_pmm_server_token operator_new ++ local key_name=operator_new ++ [[ -z operator_new ]] ++ local ADMIN_PASSWORD +++ base64 --decode +++ kubectl get secret pmm-secret -o 'jsonpath={.data.PMM_ADMIN_PASSWORD}' ++ ADMIN_PASSWORD='4 RTDc<0"()1 |=G' ++ [[ -z 4 RTDc<0"()1 |=G ]] ++ local create_response create_status_code create_json_response ++ local retry=0 ++ [[ '' == 201 ]] ++++ get_service_ip monitoring-service ++++ local service=monitoring-service ++++ grep -q NotFound ++++ kubectl_bin get service/monitoring-service -o 'jsonpath={.spec.type}' +++++ kubectl_bin get service/monitoring-service -o 'jsonpath={.spec.type}' ++++++ mktemp +++++ local LAST_OUT=/tmp/tmp.aUNIgSSeQa ++++++ mktemp +++++ local LAST_ERR=/tmp/tmp.ntpWZXNcli +++++ local exit_status=0 ++++++ seq 0 2 +++++ for i in $(seq 0 2) +++++ set +e +++++ kubectl get service/monitoring-service -o 'jsonpath={.spec.type}' +++++ exit_status=0 +++++ set -e +++++ '[' 0 '!=' 0 ']' +++++ break +++++ cat /tmp/tmp.aUNIgSSeQa +++++ cat /tmp/tmp.ntpWZXNcli +++++ rm /tmp/tmp.aUNIgSSeQa /tmp/tmp.ntpWZXNcli +++++ return 0 ++++ '[' LoadBalancer = ClusterIP ']' ++++ grep -E -q 'hostname|ip' ++++ kubectl_bin get service/monitoring-service -o 'jsonpath={.status.loadBalancer.ingress[]}' ++++ kubectl_bin get service/monitoring-service -o 'jsonpath={.status.loadBalancer.ingress[].ip}' +++++ mktemp ++++ local LAST_OUT=/tmp/tmp.gJcYSEdWGn +++++ mktemp ++++ local LAST_ERR=/tmp/tmp.Q9AZjVdZGW ++++ local exit_status=0 +++++ seq 0 2 ++++ for i in $(seq 0 2) ++++ set +e ++++ kubectl get service/monitoring-service -o 'jsonpath={.status.loadBalancer.ingress[].ip}' ++++ exit_status=0 ++++ set -e ++++ '[' 0 '!=' 0 ']' ++++ break ++++ cat /tmp/tmp.gJcYSEdWGn ++++ cat /tmp/tmp.Q9AZjVdZGW ++++ rm /tmp/tmp.gJcYSEdWGn /tmp/tmp.Q9AZjVdZGW ++++ return 0 ++++ kubectl_bin get service/monitoring-service -o 'jsonpath={.status.loadBalancer.ingress[].hostname}' +++++ mktemp ++++ local LAST_OUT=/tmp/tmp.qgseGeYBcD +++++ mktemp ++++ local LAST_ERR=/tmp/tmp.J8WR3o58Fg ++++ local exit_status=0 +++++ seq 0 2 ++++ for i in $(seq 0 2) ++++ set +e ++++ kubectl get service/monitoring-service -o 'jsonpath={.status.loadBalancer.ingress[].hostname}' ++++ exit_status=0 ++++ set -e ++++ '[' 0 '!=' 0 ']' ++++ break ++++ cat /tmp/tmp.qgseGeYBcD ++++ cat /tmp/tmp.J8WR3o58Fg ++++ rm /tmp/tmp.qgseGeYBcD /tmp/tmp.J8WR3o58Fg ++++ return 0 +++ curl --insecure -s -X POST -H 'Content-Type: application/json' -H 'Accept: application/json' -d '{"name":"operator_new", "role":"Admin", "isDisabled":false}' --user 'admin:4 RTDc<0"()1 |=G' https://104.154.140.66/graph/api/serviceaccounts -w '\n%{http_code}' ++ create_response='{"id":3,"uid":"eg0bjhptbsw00c","name":"operator_new","login":"sa-1-operator_new","orgId":1,"isDisabled":false,"role":"Admin","tokens":0,"avatarUrl":""} 201' +++ echo '{"id":3,"uid":"eg0bjhptbsw00c","name":"operator_new","login":"sa-1-operator_new","orgId":1,"isDisabled":false,"role":"Admin","tokens":0,"avatarUrl":""} 201' +++ tail -n1 ++ create_status_code=201 +++ sed '$ d' +++ echo '{"id":3,"uid":"eg0bjhptbsw00c","name":"operator_new","login":"sa-1-operator_new","orgId":1,"isDisabled":false,"role":"Admin","tokens":0,"avatarUrl":""} 201' ++ create_json_response='{"id":3,"uid":"eg0bjhptbsw00c","name":"operator_new","login":"sa-1-operator_new","orgId":1,"isDisabled":false,"role":"Admin","tokens":0,"avatarUrl":""}' ++ sleep 5 ++ let retry+=1 ++ '[' 1 -ge 24 ']' ++ [[ 201 == 201 ]] ++ local service_account_id +++ echo '{"id":3,"uid":"eg0bjhptbsw00c","name":"operator_new","login":"sa-1-operator_new","orgId":1,"isDisabled":false,"role":"Admin","tokens":0,"avatarUrl":""}' +++ jq -r .id ++ service_account_id=3 ++ [[ -z 3 ]] ++ [[ 3 == \n\u\l\l ]] ++ local token_response token_status_code token_json_response ++++ get_service_ip monitoring-service ++++ local service=monitoring-service ++++ grep -q NotFound ++++ kubectl_bin get service/monitoring-service -o 'jsonpath={.spec.type}' +++++ kubectl_bin get service/monitoring-service -o 'jsonpath={.spec.type}' ++++++ mktemp +++++ local LAST_OUT=/tmp/tmp.NgrQqkK8Rn ++++++ mktemp +++++ local LAST_ERR=/tmp/tmp.Dif4tRDrbE +++++ local exit_status=0 ++++++ seq 0 2 +++++ for i in $(seq 0 2) +++++ set +e +++++ kubectl get service/monitoring-service -o 'jsonpath={.spec.type}' +++++ exit_status=0 +++++ set -e +++++ '[' 0 '!=' 0 ']' +++++ break +++++ cat /tmp/tmp.NgrQqkK8Rn +++++ cat /tmp/tmp.Dif4tRDrbE +++++ rm /tmp/tmp.NgrQqkK8Rn /tmp/tmp.Dif4tRDrbE +++++ return 0 ++++ '[' LoadBalancer = ClusterIP ']' ++++ grep -E -q 'hostname|ip' ++++ kubectl_bin get service/monitoring-service -o 'jsonpath={.status.loadBalancer.ingress[]}' ++++ kubectl_bin get service/monitoring-service -o 'jsonpath={.status.loadBalancer.ingress[].ip}' +++++ mktemp ++++ local LAST_OUT=/tmp/tmp.GBq5FjhmT9 +++++ mktemp ++++ local LAST_ERR=/tmp/tmp.NtPAnRZujK ++++ local exit_status=0 +++++ seq 0 2 ++++ for i in $(seq 0 2) ++++ set +e ++++ kubectl get service/monitoring-service -o 'jsonpath={.status.loadBalancer.ingress[].ip}' ++++ exit_status=0 ++++ set -e ++++ '[' 0 '!=' 0 ']' ++++ break ++++ cat /tmp/tmp.GBq5FjhmT9 ++++ cat /tmp/tmp.NtPAnRZujK ++++ rm /tmp/tmp.GBq5FjhmT9 /tmp/tmp.NtPAnRZujK ++++ return 0 ++++ kubectl_bin get service/monitoring-service -o 'jsonpath={.status.loadBalancer.ingress[].hostname}' +++++ mktemp ++++ local LAST_OUT=/tmp/tmp.ZrWVIf7Cvt +++++ mktemp ++++ local LAST_ERR=/tmp/tmp.L75RzQc6ss ++++ local exit_status=0 +++++ seq 0 2 ++++ for i in $(seq 0 2) ++++ set +e ++++ kubectl get service/monitoring-service -o 'jsonpath={.status.loadBalancer.ingress[].hostname}' ++++ exit_status=0 ++++ set -e ++++ '[' 0 '!=' 0 ']' ++++ break ++++ cat /tmp/tmp.ZrWVIf7Cvt ++++ cat /tmp/tmp.L75RzQc6ss ++++ rm /tmp/tmp.ZrWVIf7Cvt /tmp/tmp.L75RzQc6ss ++++ return 0 +++ curl --insecure -s -X POST -H 'Content-Type: application/json' -d '{"name":"operator_new"}' --user 'admin:4 RTDc<0"()1 |=G' https://104.154.140.66/graph/api/serviceaccounts/3/tokens -w '\n%{http_code}' ++ token_response='{"id":2,"name":"operator_new","key":"glsa_5JP1O3YUnBKgF4WW4DX6ZplIWMPMaXmo_18baf097"} 200' +++ echo '{"id":2,"name":"operator_new","key":"glsa_5JP1O3YUnBKgF4WW4DX6ZplIWMPMaXmo_18baf097"} 200' +++ tail -n1 ++ token_status_code=200 +++ sed '$ d' +++ echo '{"id":2,"name":"operator_new","key":"glsa_5JP1O3YUnBKgF4WW4DX6ZplIWMPMaXmo_18baf097"} 200' ++ token_json_response='{"id":2,"name":"operator_new","key":"glsa_5JP1O3YUnBKgF4WW4DX6ZplIWMPMaXmo_18baf097"}' ++ [[ 200 -ne 200 ]] ++ jq -r .key ++ echo '{"id":2,"name":"operator_new","key":"glsa_5JP1O3YUnBKgF4WW4DX6ZplIWMPMaXmo_18baf097"}' + NEW_TOKEN=glsa_5JP1O3YUnBKgF4WW4DX6ZplIWMPMaXmo_18baf097 + kubectl_bin patch secret my-cluster-secrets --type merge --patch '{"stringData": {"pmmservertoken": "glsa_5JP1O3YUnBKgF4WW4DX6ZplIWMPMaXmo_18baf097"}}' ++ mktemp + local LAST_OUT=/tmp/tmp.wmRuUPKzgn ++ mktemp + local LAST_ERR=/tmp/tmp.Nqmi4XszDH + local exit_status=0 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl patch secret my-cluster-secrets --type merge --patch '{"stringData": {"pmmservertoken": "glsa_5JP1O3YUnBKgF4WW4DX6ZplIWMPMaXmo_18baf097"}}' + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.wmRuUPKzgn secret/my-cluster-secrets patched + cat /tmp/tmp.Nqmi4XszDH + rm /tmp/tmp.wmRuUPKzgn /tmp/tmp.Nqmi4XszDH + return 0 + desc 'delete old PMM token' + set +o xtrace ----------------------------------------------------------------------------------- delete old PMM token ----------------------------------------------------------------------------------- + delete_pmm_server_token operator + local key_name=operator + [[ -z operator ]] + local ADMIN_PASSWORD ++ base64 --decode ++ kubectl get secret pmm-secret -o 'jsonpath={.data.PMM_ADMIN_PASSWORD}' + ADMIN_PASSWORD='4 RTDc<0"()1 |=G' + [[ -z 4 RTDc<0"()1 |=G ]] + local 'user_credentials=admin:4 RTDc<0"()1 |=G' + local service_accounts_response service_accounts_status +++ get_service_ip monitoring-service +++ local service=monitoring-service +++ kubectl_bin get service/monitoring-service -o 'jsonpath={.spec.type}' +++ grep -q NotFound ++++ kubectl_bin get service/monitoring-service -o 'jsonpath={.spec.type}' +++++ mktemp ++++ local LAST_OUT=/tmp/tmp.lGPL7MIq2I +++++ mktemp ++++ local LAST_ERR=/tmp/tmp.0JXjCiwvya ++++ local exit_status=0 +++++ seq 0 2 ++++ for i in $(seq 0 2) ++++ set +e ++++ kubectl get service/monitoring-service -o 'jsonpath={.spec.type}' ++++ exit_status=0 ++++ set -e ++++ '[' 0 '!=' 0 ']' ++++ break ++++ cat /tmp/tmp.lGPL7MIq2I ++++ cat /tmp/tmp.0JXjCiwvya ++++ rm /tmp/tmp.lGPL7MIq2I /tmp/tmp.0JXjCiwvya ++++ return 0 +++ '[' LoadBalancer = ClusterIP ']' +++ kubectl_bin get service/monitoring-service -o 'jsonpath={.status.loadBalancer.ingress[]}' +++ grep -E -q 'hostname|ip' +++ kubectl_bin get service/monitoring-service -o 'jsonpath={.status.loadBalancer.ingress[].ip}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.LMVKUmlbNW ++++ mktemp +++ local LAST_ERR=/tmp/tmp.cl9ELdVrTE +++ local exit_status=0 ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl get service/monitoring-service -o 'jsonpath={.status.loadBalancer.ingress[].ip}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 ']' +++ break +++ cat /tmp/tmp.LMVKUmlbNW +++ cat /tmp/tmp.cl9ELdVrTE +++ rm /tmp/tmp.LMVKUmlbNW /tmp/tmp.cl9ELdVrTE +++ return 0 +++ kubectl_bin get service/monitoring-service -o 'jsonpath={.status.loadBalancer.ingress[].hostname}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.O6x8jYZUTV ++++ mktemp +++ local LAST_ERR=/tmp/tmp.bnOzCXzHog +++ local exit_status=0 ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl get service/monitoring-service -o 'jsonpath={.status.loadBalancer.ingress[].hostname}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 ']' +++ break +++ cat /tmp/tmp.O6x8jYZUTV +++ cat /tmp/tmp.bnOzCXzHog +++ rm /tmp/tmp.O6x8jYZUTV /tmp/tmp.bnOzCXzHog +++ return 0 ++ curl --insecure -s -X GET --user 'admin:4 RTDc<0"()1 |=G' https://104.154.140.66/graph/api/serviceaccounts/search -w '\n%{http_code}' + service_accounts_response='{"totalCount":2,"serviceAccounts":[{"id":2,"uid":"ag0bil4m64xdse","name":"operator","login":"sa-1-operator","orgId":1,"isDisabled":false,"role":"Admin","tokens":1,"avatarUrl":"/graph/avatar/f741e7abdd8def4ed30a37f602429df3"},{"id":3,"uid":"eg0bjhptbsw00c","name":"operator_new","login":"sa-1-operator_new","orgId":1,"isDisabled":false,"role":"Admin","tokens":1,"avatarUrl":"/graph/avatar/6d1b2db67a8f3e06db4ae076406fece5"}],"page":1,"perPage":1000} 200' ++ tail -n1 ++ echo '{"totalCount":2,"serviceAccounts":[{"id":2,"uid":"ag0bil4m64xdse","name":"operator","login":"sa-1-operator","orgId":1,"isDisabled":false,"role":"Admin","tokens":1,"avatarUrl":"/graph/avatar/f741e7abdd8def4ed30a37f602429df3"},{"id":3,"uid":"eg0bjhptbsw00c","name":"operator_new","login":"sa-1-operator_new","orgId":1,"isDisabled":false,"role":"Admin","tokens":1,"avatarUrl":"/graph/avatar/6d1b2db67a8f3e06db4ae076406fece5"}],"page":1,"perPage":1000} 200' + service_accounts_status=200 ++ echo '{"totalCount":2,"serviceAccounts":[{"id":2,"uid":"ag0bil4m64xdse","name":"operator","login":"sa-1-operator","orgId":1,"isDisabled":false,"role":"Admin","tokens":1,"avatarUrl":"/graph/avatar/f741e7abdd8def4ed30a37f602429df3"},{"id":3,"uid":"eg0bjhptbsw00c","name":"operator_new","login":"sa-1-operator_new","orgId":1,"isDisabled":false,"role":"Admin","tokens":1,"avatarUrl":"/graph/avatar/6d1b2db67a8f3e06db4ae076406fece5"}],"page":1,"perPage":1000} ++ sed '$ d' 200' + service_accounts_json='{"totalCount":2,"serviceAccounts":[{"id":2,"uid":"ag0bil4m64xdse","name":"operator","login":"sa-1-operator","orgId":1,"isDisabled":false,"role":"Admin","tokens":1,"avatarUrl":"/graph/avatar/f741e7abdd8def4ed30a37f602429df3"},{"id":3,"uid":"eg0bjhptbsw00c","name":"operator_new","login":"sa-1-operator_new","orgId":1,"isDisabled":false,"role":"Admin","tokens":1,"avatarUrl":"/graph/avatar/6d1b2db67a8f3e06db4ae076406fece5"}],"page":1,"perPage":1000}' + [[ 200 -ne 200 ]] + local service_account_id ++ jq -r '.serviceAccounts[] | select(.name == "operator").id' ++ echo '{"totalCount":2,"serviceAccounts":[{"id":2,"uid":"ag0bil4m64xdse","name":"operator","login":"sa-1-operator","orgId":1,"isDisabled":false,"role":"Admin","tokens":1,"avatarUrl":"/graph/avatar/f741e7abdd8def4ed30a37f602429df3"},{"id":3,"uid":"eg0bjhptbsw00c","name":"operator_new","login":"sa-1-operator_new","orgId":1,"isDisabled":false,"role":"Admin","tokens":1,"avatarUrl":"/graph/avatar/6d1b2db67a8f3e06db4ae076406fece5"}],"page":1,"perPage":1000}' + service_account_id=2 + [[ -z 2 ]] + [[ 2 == \n\u\l\l ]] + local tokens_response tokens_status tokens_json +++ get_service_ip monitoring-service +++ local service=monitoring-service +++ kubectl_bin get service/monitoring-service -o 'jsonpath={.spec.type}' +++ grep -q NotFound ++++ kubectl_bin get service/monitoring-service -o 'jsonpath={.spec.type}' +++++ mktemp ++++ local LAST_OUT=/tmp/tmp.4qrWPwN5ml +++++ mktemp ++++ local LAST_ERR=/tmp/tmp.oNAmvCdjTj ++++ local exit_status=0 +++++ seq 0 2 ++++ for i in $(seq 0 2) ++++ set +e ++++ kubectl get service/monitoring-service -o 'jsonpath={.spec.type}' ++++ exit_status=0 ++++ set -e ++++ '[' 0 '!=' 0 ']' ++++ break ++++ cat /tmp/tmp.4qrWPwN5ml ++++ cat /tmp/tmp.oNAmvCdjTj ++++ rm /tmp/tmp.4qrWPwN5ml /tmp/tmp.oNAmvCdjTj ++++ return 0 +++ '[' LoadBalancer = ClusterIP ']' +++ grep -E -q 'hostname|ip' +++ kubectl_bin get service/monitoring-service -o 'jsonpath={.status.loadBalancer.ingress[]}' +++ kubectl_bin get service/monitoring-service -o 'jsonpath={.status.loadBalancer.ingress[].ip}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.TcGkh09gTf ++++ mktemp +++ local LAST_ERR=/tmp/tmp.fz6cXEJKHa +++ local exit_status=0 ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl get service/monitoring-service -o 'jsonpath={.status.loadBalancer.ingress[].ip}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 ']' +++ break +++ cat /tmp/tmp.TcGkh09gTf +++ cat /tmp/tmp.fz6cXEJKHa +++ rm /tmp/tmp.TcGkh09gTf /tmp/tmp.fz6cXEJKHa +++ return 0 +++ kubectl_bin get service/monitoring-service -o 'jsonpath={.status.loadBalancer.ingress[].hostname}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.k1CZ8HvPVD ++++ mktemp +++ local LAST_ERR=/tmp/tmp.fegw3m7SNX +++ local exit_status=0 ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl get service/monitoring-service -o 'jsonpath={.status.loadBalancer.ingress[].hostname}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 ']' +++ break +++ cat /tmp/tmp.k1CZ8HvPVD +++ cat /tmp/tmp.fegw3m7SNX +++ rm /tmp/tmp.k1CZ8HvPVD /tmp/tmp.fegw3m7SNX +++ return 0 ++ curl --insecure -s -X GET --user 'admin:4 RTDc<0"()1 |=G' https://104.154.140.66/graph/api/serviceaccounts/2/tokens -w '\n%{http_code}' + tokens_response='[{"id":1,"name":"operator","created":"2026-10-05T14:33:16Z","lastUsedAt":"2026-10-05T14:43:39Z","expiration":null,"secondsUntilExpiration":0,"hasExpired":false,"isRevoked":false}] 200' ++ tail -n1 ++ echo '[{"id":1,"name":"operator","created":"2026-10-05T14:33:16Z","lastUsedAt":"2026-10-05T14:43:39Z","expiration":null,"secondsUntilExpiration":0,"hasExpired":false,"isRevoked":false}] 200' + tokens_status=200 ++ sed '$ d' ++ echo '[{"id":1,"name":"operator","created":"2026-10-05T14:33:16Z","lastUsedAt":"2026-10-05T14:43:39Z","expiration":null,"secondsUntilExpiration":0,"hasExpired":false,"isRevoked":false}] 200' + tokens_json='[{"id":1,"name":"operator","created":"2026-10-05T14:33:16Z","lastUsedAt":"2026-10-05T14:43:39Z","expiration":null,"secondsUntilExpiration":0,"hasExpired":false,"isRevoked":false}]' + [[ 200 -ne 200 ]] + local token_id ++ echo '[{"id":1,"name":"operator","created":"2026-10-05T14:33:16Z","lastUsedAt":"2026-10-05T14:43:39Z","expiration":null,"secondsUntilExpiration":0,"hasExpired":false,"isRevoked":false}]' ++ jq -r '.[] | select(.name == "operator").id' + token_id=1 + [[ -z 1 ]] + [[ 1 == \n\u\l\l ]] + local delete_response delete_status +++ get_service_ip monitoring-service +++ local service=monitoring-service +++ kubectl_bin get service/monitoring-service -o 'jsonpath={.spec.type}' +++ grep -q NotFound ++++ kubectl_bin get service/monitoring-service -o 'jsonpath={.spec.type}' +++++ mktemp ++++ local LAST_OUT=/tmp/tmp.NqiMaXXMWe +++++ mktemp ++++ local LAST_ERR=/tmp/tmp.EWJkRbJjvk ++++ local exit_status=0 +++++ seq 0 2 ++++ for i in $(seq 0 2) ++++ set +e ++++ kubectl get service/monitoring-service -o 'jsonpath={.spec.type}' ++++ exit_status=0 ++++ set -e ++++ '[' 0 '!=' 0 ']' ++++ break ++++ cat /tmp/tmp.NqiMaXXMWe ++++ cat /tmp/tmp.EWJkRbJjvk ++++ rm /tmp/tmp.NqiMaXXMWe /tmp/tmp.EWJkRbJjvk ++++ return 0 +++ '[' LoadBalancer = ClusterIP ']' +++ grep -E -q 'hostname|ip' +++ kubectl_bin get service/monitoring-service -o 'jsonpath={.status.loadBalancer.ingress[]}' +++ kubectl_bin get service/monitoring-service -o 'jsonpath={.status.loadBalancer.ingress[].ip}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.5CgHBGdkmx ++++ mktemp +++ local LAST_ERR=/tmp/tmp.GoYWEq5on3 +++ local exit_status=0 ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl get service/monitoring-service -o 'jsonpath={.status.loadBalancer.ingress[].ip}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 ']' +++ break +++ cat /tmp/tmp.5CgHBGdkmx +++ cat /tmp/tmp.GoYWEq5on3 +++ rm /tmp/tmp.5CgHBGdkmx /tmp/tmp.GoYWEq5on3 +++ return 0 +++ kubectl_bin get service/monitoring-service -o 'jsonpath={.status.loadBalancer.ingress[].hostname}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.OjXrPmFIBA ++++ mktemp +++ local LAST_ERR=/tmp/tmp.TyUo8Y1Wc0 +++ local exit_status=0 ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl get service/monitoring-service -o 'jsonpath={.status.loadBalancer.ingress[].hostname}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 ']' +++ break +++ cat /tmp/tmp.OjXrPmFIBA +++ cat /tmp/tmp.TyUo8Y1Wc0 +++ rm /tmp/tmp.OjXrPmFIBA /tmp/tmp.TyUo8Y1Wc0 +++ return 0 ++ curl --insecure -s -X DELETE --user 'admin:4 RTDc<0"()1 |=G' https://104.154.140.66/graph/api/serviceaccounts/2/tokens/1 -w '\n%{http_code}' + delete_response='{"message":"Service account token deleted"} 200' ++ echo '{"message":"Service account token deleted"} 200' ++ tail -n1 + delete_status=200 + [[ 200 -ne 200 ]] + wait_for_generation sts/monitoring-pxc 3 + local resource=sts/monitoring-pxc + local target_generation=3 + echo 'Waiting for sts/monitoring-pxc to reach generation 3...' Waiting for sts/monitoring-pxc to reach generation 3... + true ++ kubectl get sts/monitoring-pxc -o 'jsonpath={.metadata.generation}' + current_generation=2 + '[' 2 -eq 3 ']' + echo 'Resource sts/monitoring-pxc is at generation 2. Waiting...' Resource sts/monitoring-pxc is at generation 2. Waiting... + sleep 5 + true ++ kubectl get sts/monitoring-pxc -o 'jsonpath={.metadata.generation}' + current_generation=2 + '[' 2 -eq 3 ']' + echo 'Resource sts/monitoring-pxc is at generation 2. Waiting...' Resource sts/monitoring-pxc is at generation 2. Waiting... + sleep 5 + true ++ kubectl get sts/monitoring-pxc -o 'jsonpath={.metadata.generation}' + current_generation=2 + '[' 2 -eq 3 ']' + echo 'Resource sts/monitoring-pxc is at generation 2. Waiting...' Resource sts/monitoring-pxc is at generation 2. Waiting... + sleep 5 + true ++ kubectl get sts/monitoring-pxc -o 'jsonpath={.metadata.generation}' + current_generation=2 + '[' 2 -eq 3 ']' + echo 'Resource sts/monitoring-pxc is at generation 2. Waiting...' Resource sts/monitoring-pxc is at generation 2. Waiting... + sleep 5 + true ++ kubectl get sts/monitoring-pxc -o 'jsonpath={.metadata.generation}' + current_generation=2 + '[' 2 -eq 3 ']' + echo 'Resource sts/monitoring-pxc is at generation 2. Waiting...' Resource sts/monitoring-pxc is at generation 2. Waiting... + sleep 5 + true ++ kubectl get sts/monitoring-pxc -o 'jsonpath={.metadata.generation}' + current_generation=2 + '[' 2 -eq 3 ']' + echo 'Resource sts/monitoring-pxc is at generation 2. Waiting...' Resource sts/monitoring-pxc is at generation 2. Waiting... + sleep 5 + true ++ kubectl get sts/monitoring-pxc -o 'jsonpath={.metadata.generation}' + current_generation=2 + '[' 2 -eq 3 ']' + echo 'Resource sts/monitoring-pxc is at generation 2. Waiting...' Resource sts/monitoring-pxc is at generation 2. Waiting... + sleep 5 + true ++ kubectl get sts/monitoring-pxc -o 'jsonpath={.metadata.generation}' + current_generation=3 + '[' 3 -eq 3 ']' + echo 'Resource sts/monitoring-pxc has reached generation 3.' Resource sts/monitoring-pxc has reached generation 3. + break + wait_for_generation sts/monitoring-haproxy 3 + local resource=sts/monitoring-haproxy + local target_generation=3 + echo 'Waiting for sts/monitoring-haproxy to reach generation 3...' Waiting for sts/monitoring-haproxy to reach generation 3... + true ++ kubectl get sts/monitoring-haproxy -o 'jsonpath={.metadata.generation}' + current_generation=3 + '[' 3 -eq 3 ']' + echo 'Resource sts/monitoring-haproxy has reached generation 3.' Resource sts/monitoring-haproxy has reached generation 3. + break + sleep 10 + kubectl wait pod -l app.kubernetes.io/managed-by=percona-xtradb-cluster-operator --for=condition=ready --timeout=600s pod/monitoring-haproxy-0 condition met pod/monitoring-haproxy-1 condition met pod/monitoring-pxc-0 condition met pod/monitoring-pxc-1 condition met pod/monitoring-pxc-2 condition met + desc 'check if pmm-client container enabled' + set +o xtrace ----------------------------------------------------------------------------------- check if pmm-client container enabled ----------------------------------------------------------------------------------- + compare_kubectl statefulset/monitoring-pxc + local resource=statefulset/monitoring-pxc + local postfix= + local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/e2e-tests/monitoring-pmm3/compare/statefulset_monitoring-pxc.yml + local new_result=/tmp/tmp.mnnOTDKNTG/statefulset_monitoring-pxc.yml + desc 'compare statefulset/monitoring-pxc-' + set +o xtrace ----------------------------------------------------------------------------------- compare statefulset/monitoring-pxc- ----------------------------------------------------------------------------------- + '[' 0 = 1 -a -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/e2e-tests/monitoring-pmm3/compare/statefulset_monitoring-pxc-eks.yml ']' + [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.4 =~ 8\.0 ]] + [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.4 =~ 8\.4 ]] + '[' -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/e2e-tests/monitoring-pmm3/compare/statefulset_monitoring-pxc-84.yml ']' + version_gt 1.33 ++ echo '1.34 >= 1.33' ++ bc -l + '[' 1 -eq 1 ']' + return 0 + '[' -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/e2e-tests/monitoring-pmm3/compare/statefulset_monitoring-pxc-k133.yml ']' + version_gt 1.29 ++ echo '1.34 >= 1.29' ++ bc -l + '[' 1 -eq 1 ']' + return 0 + '[' -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/e2e-tests/monitoring-pmm3/compare/statefulset_monitoring-pxc-k129.yml ']' + version_gt 1.27 ++ bc -l ++ echo '1.34 >= 1.27' + '[' 1 -eq 1 ']' + return 0 + '[' -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/e2e-tests/monitoring-pmm3/compare/statefulset_monitoring-pxc-k127.yml ']' + expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/e2e-tests/monitoring-pmm3/compare/statefulset_monitoring-pxc-k127.yml + '[' '!' -z '' -a -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/e2e-tests/monitoring-pmm3/compare/statefulset_monitoring-pxc-k127-oc.yml ']' + version_gt 1.29 ++ bc -l ++ echo '1.34 >= 1.29' + '[' 1 -eq 1 ']' + return 0 + '[' '!' -z '' -a -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/e2e-tests/monitoring-pmm3/compare/statefulset_monitoring-pxc-k127-k129-oc.yml ']' + '[' 0 = 1 -a -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/e2e-tests/monitoring-pmm3/compare/statefulset_monitoring-pxc-k127-eks.yml ']' + '[' 0 = 1 -a -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/e2e-tests/monitoring-pmm3/compare/statefulset_monitoring-pxc-k127-aks.yml ']' + kubectl_bin get -o yaml statefulset/monitoring-pxc + yq eval ' del(.metadata.managedFields) | del(.. | select(has("creationTimestamp")).creationTimestamp) | del(.. | select(has("namespace")).namespace) | del(.. | select(has("uid")).uid) | del(.metadata.resourceVersion) | del(.spec.template.spec.containers[].env[] | select(.name == "CLUSTER_HASH")) | del(.spec.template.spec.containers[].env[] | select(.name == "S3_BUCKET_PATH")) | del(.spec.template.spec.containers[].env[] | select(.name == "BACKUP_PATH")) | del(.spec.template.spec.containers[].env[] | select(.name == "S3_BUCKET_URL")) | del(.spec.template.spec.containers[].env[] | select(.name == "AZURE_CONTAINER_NAME")) | del(.spec.template.spec.containers[].env[] | select(.name == "XTRABACKUP_ENABLED")) | del(.metadata.selfLink) | del(.metadata.deletionTimestamp) | del(.metadata.labels."batch.kubernetes.io/job-name") | del(.metadata.labels."job-name") | del(.metadata.labels."topology.kubernetes.io/region") | del(.metadata.labels."topology.kubernetes.io/zone") | del(.metadata.annotations."kubectl.kubernetes.io/last-applied-configuration") | del(.metadata.annotations."kubernetes.io/psp") | del(.metadata.annotations."batch.kubernetes.io/job-tracking") | del(.metadata.annotations."networking.gke.io/target-pool") | del(.metadata.annotations."cloud.google.com/neg") | del(.metadata.annotations."k8s.v1.cni.cncf.io*") | del(.metadata.annotations."k8s.ovn.org/pod-networks") | del(.metadata.annotations."kubernetes.digitalocean.com/load-balancer-id") | del(.metadata.annotations."service.beta.kubernetes.io/do-loadbalancer-type") | del(.spec.template.metadata.annotations."last-applied-secret") | del(.spec.template.metadata.labels."batch.kubernetes.io/job-name") | del(.spec.template.metadata.labels."job-name") | del(.. | select(has("batch.kubernetes.io/controller-uid"))."batch.kubernetes.io/controller-uid") | del(.. | select(has("image")).image) | del(.. | select(has("clusterIP")).clusterIP) | del(.. | select(has("clusterIPs")).clusterIPs) | del(.. | select(has("dataSource")).dataSource) | del(.. | select(has("procMount")).procMount) | del(.. | select(has("storageClassName")).storageClassName) | del(.. | select(has("finalizers")).finalizers) | del(.. | select(has("kubernetes.io/pvc-protection"))."kubernetes.io/pvc-protection") | del(.. | select(has("volumeName")).volumeName) | del(.. | select(has("volume.beta.kubernetes.io/storage-provisioner"))."volume.beta.kubernetes.io/storage-provisioner") | del(.. | select(has("volume.kubernetes.io/storage-provisioner"))."volume.kubernetes.io/storage-provisioner") | del(.spec.volumeMode) | del(.spec.nodeName) | del(.. | select(has("volume.kubernetes.io/selected-node"))."volume.kubernetes.io/selected-node") | del(.. | select(has("percona.com/last-config-hash"))."percona.com/last-config-hash") | del(.. | select(has("percona.com/configuration-hash"))."percona.com/configuration-hash") | del(.. | select(has("percona.com/env-secret-config-hash"))."percona.com/env-secret-config-hash") | del(.. | select(has("percona.com/ssl-hash"))."percona.com/ssl-hash") | del(.. | select(has("percona.com/ssl-internal-hash"))."percona.com/ssl-internal-hash") | del(.. | select(has("kubectl.kubernetes.io/default-container"))."kubectl.kubernetes.io/default-container") | del(.spec.volumeClaimTemplates[].spec.volumeMode | select(. == "Filesystem")) | del(.. | select(has("healthCheckNodePort")).healthCheckNodePort) | del(.. | select(has("nodePort")).nodePort) | del(.. | select(has("imagePullSecrets")).imagePullSecrets) | del(.. | select(has("enableServiceLinks")).enableServiceLinks) | del(.status) | del(.spec.volumeClaimTemplates[].apiVersion) | del(.spec.volumeClaimTemplates[].kind) | del(.metadata.ownerReferences[].apiVersion) | del(.. | select(has("controller-uid")).controller-uid) | del(.. | select(has("preemptionPolicy")).preemptionPolicy) | del(.spec.ipFamilies) | del(.spec.ipFamilyPolicy) | del(.spec.updateStrategy.rollingUpdate.maxUnavailable) | (.. | select(. == "policy/v1beta1")) = "policy/v1" | del(.. | select(has("kubernetes.io/hostname"))."kubernetes.io/hostname") | (.. | select(tag == "!!str")) |= sub("monitoring-pmm3-26352", "namespace") | (.. | select(tag == "!!str")) |= sub("kube-api-access-.*", "kube-api-access") | del(.. | select(has("annotations")).annotations | select(length==0)) | del(.spec.crVersion) | del(.. | select(.[] == "percona-xtradb-cluster-operator-workload-token*"))' - ++ mktemp + local LAST_OUT=/tmp/tmp.QuOf2aSmFg ++ mktemp + local LAST_ERR=/tmp/tmp.pqOTqXVLLj + local exit_status=0 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl get -o yaml statefulset/monitoring-pxc + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.QuOf2aSmFg + cat /tmp/tmp.pqOTqXVLLj + rm /tmp/tmp.QuOf2aSmFg /tmp/tmp.pqOTqXVLLj + return 0 + [[ 0 -eq 0 ]] + diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/e2e-tests/monitoring-pmm3/compare/statefulset_monitoring-pxc-k127.yml /tmp/tmp.mnnOTDKNTG/statefulset_monitoring-pxc.yml + log 'compare_kubectl: statefulset/monitoring-pxc OK' ++ date +%Y-%m-%dT%H:%M:%S%z + echo '[2026-10-05T14:46:14+0000]' compare_kubectl: statefulset/monitoring-pxc OK [2026-10-05T14:46:14+0000] compare_kubectl: statefulset/monitoring-pxc OK + compare_kubectl statefulset/monitoring-haproxy + local resource=statefulset/monitoring-haproxy + local postfix= + local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/e2e-tests/monitoring-pmm3/compare/statefulset_monitoring-haproxy.yml + local new_result=/tmp/tmp.mnnOTDKNTG/statefulset_monitoring-haproxy.yml + desc 'compare statefulset/monitoring-haproxy-' + set +o xtrace ----------------------------------------------------------------------------------- compare statefulset/monitoring-haproxy- ----------------------------------------------------------------------------------- + '[' 0 = 1 -a -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/e2e-tests/monitoring-pmm3/compare/statefulset_monitoring-haproxy-eks.yml ']' + [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.4 =~ 8\.0 ]] + [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.4 =~ 8\.4 ]] + '[' -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/e2e-tests/monitoring-pmm3/compare/statefulset_monitoring-haproxy-84.yml ']' + version_gt 1.33 ++ bc -l ++ echo '1.34 >= 1.33' + '[' 1 -eq 1 ']' + return 0 + '[' -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/e2e-tests/monitoring-pmm3/compare/statefulset_monitoring-haproxy-k133.yml ']' + version_gt 1.29 ++ bc -l ++ echo '1.34 >= 1.29' + '[' 1 -eq 1 ']' + return 0 + '[' -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/e2e-tests/monitoring-pmm3/compare/statefulset_monitoring-haproxy-k129.yml ']' + version_gt 1.27 ++ echo '1.34 >= 1.27' ++ bc -l + '[' 1 -eq 1 ']' + return 0 + '[' -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/e2e-tests/monitoring-pmm3/compare/statefulset_monitoring-haproxy-k127.yml ']' + expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/e2e-tests/monitoring-pmm3/compare/statefulset_monitoring-haproxy-k127.yml + '[' '!' -z '' -a -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/e2e-tests/monitoring-pmm3/compare/statefulset_monitoring-haproxy-k127-oc.yml ']' + version_gt 1.29 ++ bc -l ++ echo '1.34 >= 1.29' + '[' 1 -eq 1 ']' + return 0 + '[' '!' -z '' -a -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/e2e-tests/monitoring-pmm3/compare/statefulset_monitoring-haproxy-k127-k129-oc.yml ']' + '[' 0 = 1 -a -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/e2e-tests/monitoring-pmm3/compare/statefulset_monitoring-haproxy-k127-eks.yml ']' + '[' 0 = 1 -a -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/e2e-tests/monitoring-pmm3/compare/statefulset_monitoring-haproxy-k127-aks.yml ']' + yq eval ' del(.metadata.managedFields) | del(.. | select(has("creationTimestamp")).creationTimestamp) | del(.. | select(has("namespace")).namespace) | del(.. | select(has("uid")).uid) | del(.metadata.resourceVersion) | del(.spec.template.spec.containers[].env[] | select(.name == "CLUSTER_HASH")) | del(.spec.template.spec.containers[].env[] | select(.name == "S3_BUCKET_PATH")) | del(.spec.template.spec.containers[].env[] | select(.name == "BACKUP_PATH")) | del(.spec.template.spec.containers[].env[] | select(.name == "S3_BUCKET_URL")) | del(.spec.template.spec.containers[].env[] | select(.name == "AZURE_CONTAINER_NAME")) | del(.spec.template.spec.containers[].env[] | select(.name == "XTRABACKUP_ENABLED")) | del(.metadata.selfLink) | del(.metadata.deletionTimestamp) | del(.metadata.labels."batch.kubernetes.io/job-name") | del(.metadata.labels."job-name") | del(.metadata.labels."topology.kubernetes.io/region") | del(.metadata.labels."topology.kubernetes.io/zone") | del(.metadata.annotations."kubectl.kubernetes.io/last-applied-configuration") | del(.metadata.annotations."kubernetes.io/psp") | del(.metadata.annotations."batch.kubernetes.io/job-tracking") | del(.metadata.annotations."networking.gke.io/target-pool") | del(.metadata.annotations."cloud.google.com/neg") | del(.metadata.annotations."k8s.v1.cni.cncf.io*") | del(.metadata.annotations."k8s.ovn.org/pod-networks") | del(.metadata.annotations."kubernetes.digitalocean.com/load-balancer-id") | del(.metadata.annotations."service.beta.kubernetes.io/do-loadbalancer-type") | del(.spec.template.metadata.annotations."last-applied-secret") | del(.spec.template.metadata.labels."batch.kubernetes.io/job-name") | del(.spec.template.metadata.labels."job-name") | del(.. | select(has("batch.kubernetes.io/controller-uid"))."batch.kubernetes.io/controller-uid") | del(.. | select(has("image")).image) | del(.. | select(has("clusterIP")).clusterIP) | del(.. | select(has("clusterIPs")).clusterIPs) | del(.. | select(has("dataSource")).dataSource) | del(.. | select(has("procMount")).procMount) | del(.. | select(has("storageClassName")).storageClassName) | del(.. | select(has("finalizers")).finalizers) | del(.. | select(has("kubernetes.io/pvc-protection"))."kubernetes.io/pvc-protection") | del(.. | select(has("volumeName")).volumeName) | del(.. | select(has("volume.beta.kubernetes.io/storage-provisioner"))."volume.beta.kubernetes.io/storage-provisioner") | del(.. | select(has("volume.kubernetes.io/storage-provisioner"))."volume.kubernetes.io/storage-provisioner") | del(.spec.volumeMode) | del(.spec.nodeName) | del(.. | select(has("volume.kubernetes.io/selected-node"))."volume.kubernetes.io/selected-node") | del(.. | select(has("percona.com/last-config-hash"))."percona.com/last-config-hash") | del(.. | select(has("percona.com/configuration-hash"))."percona.com/configuration-hash") | del(.. | select(has("percona.com/env-secret-config-hash"))."percona.com/env-secret-config-hash") | del(.. | select(has("percona.com/ssl-hash"))."percona.com/ssl-hash") | del(.. | select(has("percona.com/ssl-internal-hash"))."percona.com/ssl-internal-hash") | del(.. | select(has("kubectl.kubernetes.io/default-container"))."kubectl.kubernetes.io/default-container") | del(.spec.volumeClaimTemplates[].spec.volumeMode | select(. == "Filesystem")) | del(.. | select(has("healthCheckNodePort")).healthCheckNodePort) | del(.. | select(has("nodePort")).nodePort) | del(.. | select(has("imagePullSecrets")).imagePullSecrets) | del(.. | select(has("enableServiceLinks")).enableServiceLinks) | del(.status) | del(.spec.volumeClaimTemplates[].apiVersion) | del(.spec.volumeClaimTemplates[].kind) | del(.metadata.ownerReferences[].apiVersion) | del(.. | select(has("controller-uid")).controller-uid) | del(.. | select(has("preemptionPolicy")).preemptionPolicy) | del(.spec.ipFamilies) | del(.spec.ipFamilyPolicy) | del(.spec.updateStrategy.rollingUpdate.maxUnavailable) | (.. | select(. == "policy/v1beta1")) = "policy/v1" | del(.. | select(has("kubernetes.io/hostname"))."kubernetes.io/hostname") | (.. | select(tag == "!!str")) |= sub("monitoring-pmm3-26352", "namespace") | (.. | select(tag == "!!str")) |= sub("kube-api-access-.*", "kube-api-access") | del(.. | select(has("annotations")).annotations | select(length==0)) | del(.spec.crVersion) | del(.. | select(.[] == "percona-xtradb-cluster-operator-workload-token*"))' - + kubectl_bin get -o yaml statefulset/monitoring-haproxy ++ mktemp + local LAST_OUT=/tmp/tmp.xvg5b1niKJ ++ mktemp + local LAST_ERR=/tmp/tmp.yIXW4fETBd + local exit_status=0 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl get -o yaml statefulset/monitoring-haproxy + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.xvg5b1niKJ + cat /tmp/tmp.yIXW4fETBd + rm /tmp/tmp.xvg5b1niKJ /tmp/tmp.yIXW4fETBd + return 0 + [[ 0 -eq 0 ]] + diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/e2e-tests/monitoring-pmm3/compare/statefulset_monitoring-haproxy-k127.yml /tmp/tmp.mnnOTDKNTG/statefulset_monitoring-haproxy.yml + log 'compare_kubectl: statefulset/monitoring-haproxy OK' ++ date +%Y-%m-%dT%H:%M:%S%z + echo '[2026-10-05T14:46:17+0000]' compare_kubectl: statefulset/monitoring-haproxy OK [2026-10-05T14:46:17+0000] compare_kubectl: statefulset/monitoring-haproxy OK + desc 'check mysql metrics' + set +o xtrace ----------------------------------------------------------------------------------- check mysql metrics ----------------------------------------------------------------------------------- + sleep 60 + get_metric_values_pmm3 node_boot_time_seconds pxc-prefix-monitoring-pmm3-26352-monitoring-pxc-0 glsa_5JP1O3YUnBKgF4WW4DX6ZplIWMPMaXmo_18baf097 + local metric=node_boot_time_seconds + local instance=pxc-prefix-monitoring-pmm3-26352-monitoring-pxc-0 + local token=glsa_5JP1O3YUnBKgF4WW4DX6ZplIWMPMaXmo_18baf097 ++ /usr/bin/date -u +%s -d '-1 minute' + local start=1791211577 ++ /usr/bin/date -u +%s + local end=1791211637 ++ get_service_endpoint monitoring-service ++ local service=monitoring-service +++ kubectl_bin get service/monitoring-service -o 'jsonpath={.status.loadBalancer.ingress[].hostname}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.RZPU4qcPFm ++++ mktemp +++ local LAST_ERR=/tmp/tmp.NYenxbqeCr +++ local exit_status=0 ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl get service/monitoring-service -o 'jsonpath={.status.loadBalancer.ingress[].hostname}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 ']' +++ break +++ cat /tmp/tmp.RZPU4qcPFm +++ cat /tmp/tmp.NYenxbqeCr +++ rm /tmp/tmp.RZPU4qcPFm /tmp/tmp.NYenxbqeCr +++ return 0 ++ local endpoint= ++ '[' -z '' ']' +++ kubectl_bin get service/monitoring-service -o 'jsonpath={.status.loadBalancer.ingress[].ip}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.cKah5O51mD ++++ mktemp +++ local LAST_ERR=/tmp/tmp.UZVKVgMdB3 +++ local exit_status=0 ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl get service/monitoring-service -o 'jsonpath={.status.loadBalancer.ingress[].ip}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 ']' +++ break +++ cat /tmp/tmp.cKah5O51mD +++ cat /tmp/tmp.UZVKVgMdB3 +++ rm /tmp/tmp.cKah5O51mD /tmp/tmp.UZVKVgMdB3 +++ return 0 ++ endpoint=104.154.140.66 ++ '[' -n 104.154.140.66 ']' ++ '[' 104.154.140.66 '!=' null ']' ++ echo 104.154.140.66 ++ head -n 1 ++ sed -e 's/^"//; s/"$//;' ++ return + local endpoint=104.154.140.66 + '[' -z node_boot_time_seconds ']' + '[' -z glsa_5JP1O3YUnBKgF4WW4DX6ZplIWMPMaXmo_18baf097 ']' + local wait_count=30 + local retry=0 ++ grep '^"[0-9]' ++ curl -s -k -H 'Authorization: Bearer glsa_5JP1O3YUnBKgF4WW4DX6ZplIWMPMaXmo_18baf097' 'https://104.154.140.66/graph/api/datasources/proxy/1/api/v1/query_range?query=min%28node_boot_time_seconds%7Bnode_name%3D%7E%22pxc-prefix-monitoring-pmm3-26352-monitoring-pxc-0%22%7d%20or%20node_boot_time_seconds%7Bnode_name%3D%7E%22pxc-prefix-monitoring-pmm3-26352-monitoring-pxc-0%22%7D%29&start=1791211577&end=1791211637&step=60' ++ jq '.data.result[0].values[][1]' jq: error (at :0): Cannot iterate over null (null) + [[ -n '' ]] + sleep 2 ++ /usr/bin/date -u +%s -d '-1 minute' + local start=1791211583 ++ /usr/bin/date -u +%s + local end=1791211643 + let retry+=1 + [[ 1 -ge 30 ]] ++ grep '^"[0-9]' ++ jq '.data.result[0].values[][1]' ++ curl -s -k -H 'Authorization: Bearer glsa_5JP1O3YUnBKgF4WW4DX6ZplIWMPMaXmo_18baf097' 'https://104.154.140.66/graph/api/datasources/proxy/1/api/v1/query_range?query=min%28node_boot_time_seconds%7Bnode_name%3D%7E%22pxc-prefix-monitoring-pmm3-26352-monitoring-pxc-0%22%7d%20or%20node_boot_time_seconds%7Bnode_name%3D%7E%22pxc-prefix-monitoring-pmm3-26352-monitoring-pxc-0%22%7D%29&start=1791211583&end=1791211643&step=60' jq: error (at :0): Cannot iterate over null (null) + [[ -n '' ]] + sleep 2 ++ /usr/bin/date -u +%s -d '-1 minute' + local start=1791211585 ++ /usr/bin/date -u +%s + local end=1791211645 + let retry+=1 + [[ 2 -ge 30 ]] ++ grep '^"[0-9]' ++ jq '.data.result[0].values[][1]' ++ curl -s -k -H 'Authorization: Bearer glsa_5JP1O3YUnBKgF4WW4DX6ZplIWMPMaXmo_18baf097' 'https://104.154.140.66/graph/api/datasources/proxy/1/api/v1/query_range?query=min%28node_boot_time_seconds%7Bnode_name%3D%7E%22pxc-prefix-monitoring-pmm3-26352-monitoring-pxc-0%22%7d%20or%20node_boot_time_seconds%7Bnode_name%3D%7E%22pxc-prefix-monitoring-pmm3-26352-monitoring-pxc-0%22%7D%29&start=1791211585&end=1791211645&step=60' jq: error (at :0): Cannot iterate over null (null) + [[ -n '' ]] + sleep 2 ++ /usr/bin/date -u +%s -d '-1 minute' + local start=1791211588 ++ /usr/bin/date -u +%s + local end=1791211648 + let retry+=1 + [[ 3 -ge 30 ]] ++ jq '.data.result[0].values[][1]' ++ grep '^"[0-9]' ++ curl -s -k -H 'Authorization: Bearer glsa_5JP1O3YUnBKgF4WW4DX6ZplIWMPMaXmo_18baf097' 'https://104.154.140.66/graph/api/datasources/proxy/1/api/v1/query_range?query=min%28node_boot_time_seconds%7Bnode_name%3D%7E%22pxc-prefix-monitoring-pmm3-26352-monitoring-pxc-0%22%7d%20or%20node_boot_time_seconds%7Bnode_name%3D%7E%22pxc-prefix-monitoring-pmm3-26352-monitoring-pxc-0%22%7D%29&start=1791211588&end=1791211648&step=60' jq: error (at :0): Cannot iterate over null (null) + [[ -n '' ]] + sleep 2 ++ /usr/bin/date -u +%s -d '-1 minute' + local start=1791211590 ++ /usr/bin/date -u +%s + local end=1791211650 + let retry+=1 + [[ 4 -ge 30 ]] ++ jq '.data.result[0].values[][1]' ++ grep '^"[0-9]' ++ curl -s -k -H 'Authorization: Bearer glsa_5JP1O3YUnBKgF4WW4DX6ZplIWMPMaXmo_18baf097' 'https://104.154.140.66/graph/api/datasources/proxy/1/api/v1/query_range?query=min%28node_boot_time_seconds%7Bnode_name%3D%7E%22pxc-prefix-monitoring-pmm3-26352-monitoring-pxc-0%22%7d%20or%20node_boot_time_seconds%7Bnode_name%3D%7E%22pxc-prefix-monitoring-pmm3-26352-monitoring-pxc-0%22%7D%29&start=1791211590&end=1791211650&step=60' jq: error (at :0): Cannot iterate over null (null) + [[ -n '' ]] + sleep 2 ++ /usr/bin/date -u +%s -d '-1 minute' + local start=1791211593 ++ /usr/bin/date -u +%s + local end=1791211653 + let retry+=1 + [[ 5 -ge 30 ]] ++ jq '.data.result[0].values[][1]' ++ curl -s -k -H 'Authorization: Bearer glsa_5JP1O3YUnBKgF4WW4DX6ZplIWMPMaXmo_18baf097' 'https://104.154.140.66/graph/api/datasources/proxy/1/api/v1/query_range?query=min%28node_boot_time_seconds%7Bnode_name%3D%7E%22pxc-prefix-monitoring-pmm3-26352-monitoring-pxc-0%22%7d%20or%20node_boot_time_seconds%7Bnode_name%3D%7E%22pxc-prefix-monitoring-pmm3-26352-monitoring-pxc-0%22%7D%29&start=1791211593&end=1791211653&step=60' ++ grep '^"[0-9]' jq: error (at :0): Cannot iterate over null (null) + [[ -n '' ]] + sleep 2 ++ /usr/bin/date -u +%s -d '-1 minute' + local start=1791211595 ++ /usr/bin/date -u +%s + local end=1791211655 + let retry+=1 + [[ 6 -ge 30 ]] ++ jq '.data.result[0].values[][1]' ++ grep '^"[0-9]' ++ curl -s -k -H 'Authorization: Bearer glsa_5JP1O3YUnBKgF4WW4DX6ZplIWMPMaXmo_18baf097' 'https://104.154.140.66/graph/api/datasources/proxy/1/api/v1/query_range?query=min%28node_boot_time_seconds%7Bnode_name%3D%7E%22pxc-prefix-monitoring-pmm3-26352-monitoring-pxc-0%22%7d%20or%20node_boot_time_seconds%7Bnode_name%3D%7E%22pxc-prefix-monitoring-pmm3-26352-monitoring-pxc-0%22%7D%29&start=1791211595&end=1791211655&step=60' jq: error (at :0): Cannot iterate over null (null) + [[ -n '' ]] + sleep 2 ++ /usr/bin/date -u +%s -d '-1 minute' + local start=1791211598 ++ /usr/bin/date -u +%s + local end=1791211658 + let retry+=1 + [[ 7 -ge 30 ]] ++ jq '.data.result[0].values[][1]' ++ grep '^"[0-9]' ++ curl -s -k -H 'Authorization: Bearer glsa_5JP1O3YUnBKgF4WW4DX6ZplIWMPMaXmo_18baf097' 'https://104.154.140.66/graph/api/datasources/proxy/1/api/v1/query_range?query=min%28node_boot_time_seconds%7Bnode_name%3D%7E%22pxc-prefix-monitoring-pmm3-26352-monitoring-pxc-0%22%7d%20or%20node_boot_time_seconds%7Bnode_name%3D%7E%22pxc-prefix-monitoring-pmm3-26352-monitoring-pxc-0%22%7D%29&start=1791211598&end=1791211658&step=60' + [[ -n "1791200722" ]] + get_metric_values_pmm3 mysql_global_status_uptime pxc-prefix-monitoring-pmm3-26352-monitoring-pxc-0 glsa_5JP1O3YUnBKgF4WW4DX6ZplIWMPMaXmo_18baf097 + local metric=mysql_global_status_uptime + local instance=pxc-prefix-monitoring-pmm3-26352-monitoring-pxc-0 + local token=glsa_5JP1O3YUnBKgF4WW4DX6ZplIWMPMaXmo_18baf097 ++ /usr/bin/date -u +%s -d '-1 minute' + local start=1791211598 ++ /usr/bin/date -u +%s + local end=1791211658 ++ get_service_endpoint monitoring-service ++ local service=monitoring-service +++ kubectl_bin get service/monitoring-service -o 'jsonpath={.status.loadBalancer.ingress[].hostname}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.wvuihgsckn ++++ mktemp +++ local LAST_ERR=/tmp/tmp.iQkzd0jyZN +++ local exit_status=0 ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl get service/monitoring-service -o 'jsonpath={.status.loadBalancer.ingress[].hostname}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 ']' +++ break +++ cat /tmp/tmp.wvuihgsckn +++ cat /tmp/tmp.iQkzd0jyZN +++ rm /tmp/tmp.wvuihgsckn /tmp/tmp.iQkzd0jyZN +++ return 0 ++ local endpoint= ++ '[' -z '' ']' +++ kubectl_bin get service/monitoring-service -o 'jsonpath={.status.loadBalancer.ingress[].ip}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.NHeM9VXTg4 ++++ mktemp +++ local LAST_ERR=/tmp/tmp.4KYRNFs1uU +++ local exit_status=0 ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl get service/monitoring-service -o 'jsonpath={.status.loadBalancer.ingress[].ip}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 ']' +++ break +++ cat /tmp/tmp.NHeM9VXTg4 +++ cat /tmp/tmp.4KYRNFs1uU +++ rm /tmp/tmp.NHeM9VXTg4 /tmp/tmp.4KYRNFs1uU +++ return 0 ++ endpoint=104.154.140.66 ++ '[' -n 104.154.140.66 ']' ++ '[' 104.154.140.66 '!=' null ']' ++ echo 104.154.140.66 ++ head -n 1 ++ sed -e 's/^"//; s/"$//;' ++ return + local endpoint=104.154.140.66 + '[' -z mysql_global_status_uptime ']' + '[' -z glsa_5JP1O3YUnBKgF4WW4DX6ZplIWMPMaXmo_18baf097 ']' + local wait_count=30 + local retry=0 ++ grep '^"[0-9]' ++ curl -s -k -H 'Authorization: Bearer glsa_5JP1O3YUnBKgF4WW4DX6ZplIWMPMaXmo_18baf097' 'https://104.154.140.66/graph/api/datasources/proxy/1/api/v1/query_range?query=min%28mysql_global_status_uptime%7Bnode_name%3D%7E%22pxc-prefix-monitoring-pmm3-26352-monitoring-pxc-0%22%7d%20or%20mysql_global_status_uptime%7Bnode_name%3D%7E%22pxc-prefix-monitoring-pmm3-26352-monitoring-pxc-0%22%7D%29&start=1791211598&end=1791211658&step=60' ++ jq '.data.result[0].values[][1]' jq: error (at :0): Cannot iterate over null (null) + [[ -n '' ]] + sleep 2 ++ /usr/bin/date -u +%s -d '-1 minute' + local start=1791211605 ++ /usr/bin/date -u +%s + local end=1791211665 + let retry+=1 + [[ 1 -ge 30 ]] ++ grep '^"[0-9]' ++ curl -s -k -H 'Authorization: Bearer glsa_5JP1O3YUnBKgF4WW4DX6ZplIWMPMaXmo_18baf097' 'https://104.154.140.66/graph/api/datasources/proxy/1/api/v1/query_range?query=min%28mysql_global_status_uptime%7Bnode_name%3D%7E%22pxc-prefix-monitoring-pmm3-26352-monitoring-pxc-0%22%7d%20or%20mysql_global_status_uptime%7Bnode_name%3D%7E%22pxc-prefix-monitoring-pmm3-26352-monitoring-pxc-0%22%7D%29&start=1791211605&end=1791211665&step=60' ++ jq '.data.result[0].values[][1]' jq: error (at :0): Cannot iterate over null (null) + [[ -n '' ]] + sleep 2 ++ /usr/bin/date -u +%s -d '-1 minute' + local start=1791211607 ++ /usr/bin/date -u +%s + local end=1791211667 + let retry+=1 + [[ 2 -ge 30 ]] ++ jq '.data.result[0].values[][1]' ++ curl -s -k -H 'Authorization: Bearer glsa_5JP1O3YUnBKgF4WW4DX6ZplIWMPMaXmo_18baf097' 'https://104.154.140.66/graph/api/datasources/proxy/1/api/v1/query_range?query=min%28mysql_global_status_uptime%7Bnode_name%3D%7E%22pxc-prefix-monitoring-pmm3-26352-monitoring-pxc-0%22%7d%20or%20mysql_global_status_uptime%7Bnode_name%3D%7E%22pxc-prefix-monitoring-pmm3-26352-monitoring-pxc-0%22%7D%29&start=1791211607&end=1791211667&step=60' ++ grep '^"[0-9]' + [[ -n "4" ]] + desc 'check haproxy metrics' + set +o xtrace ----------------------------------------------------------------------------------- check haproxy metrics ----------------------------------------------------------------------------------- + get_metric_values_pmm3 haproxy_backend_status pxc-prefix-monitoring-pmm3-26352-monitoring-haproxy-0 glsa_5JP1O3YUnBKgF4WW4DX6ZplIWMPMaXmo_18baf097 + local metric=haproxy_backend_status + local instance=pxc-prefix-monitoring-pmm3-26352-monitoring-haproxy-0 + local token=glsa_5JP1O3YUnBKgF4WW4DX6ZplIWMPMaXmo_18baf097 ++ /usr/bin/date -u +%s -d '-1 minute' + local start=1791211607 ++ /usr/bin/date -u +%s + local end=1791211667 ++ get_service_endpoint monitoring-service ++ local service=monitoring-service +++ kubectl_bin get service/monitoring-service -o 'jsonpath={.status.loadBalancer.ingress[].hostname}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.sJ2Wj5jSVx ++++ mktemp +++ local LAST_ERR=/tmp/tmp.U2dhGjrJPp +++ local exit_status=0 ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl get service/monitoring-service -o 'jsonpath={.status.loadBalancer.ingress[].hostname}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 ']' +++ break +++ cat /tmp/tmp.sJ2Wj5jSVx +++ cat /tmp/tmp.U2dhGjrJPp +++ rm /tmp/tmp.sJ2Wj5jSVx /tmp/tmp.U2dhGjrJPp +++ return 0 ++ local endpoint= ++ '[' -z '' ']' +++ kubectl_bin get service/monitoring-service -o 'jsonpath={.status.loadBalancer.ingress[].ip}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.XP65ox7Qu8 ++++ mktemp +++ local LAST_ERR=/tmp/tmp.7veSagbVxC +++ local exit_status=0 ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl get service/monitoring-service -o 'jsonpath={.status.loadBalancer.ingress[].ip}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 ']' +++ break +++ cat /tmp/tmp.XP65ox7Qu8 +++ cat /tmp/tmp.7veSagbVxC +++ rm /tmp/tmp.XP65ox7Qu8 /tmp/tmp.7veSagbVxC +++ return 0 ++ endpoint=104.154.140.66 ++ '[' -n 104.154.140.66 ']' ++ '[' 104.154.140.66 '!=' null ']' ++ sed -e 's/^"//; s/"$//;' ++ head -n 1 ++ echo 104.154.140.66 ++ return + local endpoint=104.154.140.66 + '[' -z haproxy_backend_status ']' + '[' -z glsa_5JP1O3YUnBKgF4WW4DX6ZplIWMPMaXmo_18baf097 ']' + local wait_count=30 + local retry=0 ++ grep '^"[0-9]' ++ curl -s -k -H 'Authorization: Bearer glsa_5JP1O3YUnBKgF4WW4DX6ZplIWMPMaXmo_18baf097' 'https://104.154.140.66/graph/api/datasources/proxy/1/api/v1/query_range?query=min%28haproxy_backend_status%7Bnode_name%3D%7E%22pxc-prefix-monitoring-pmm3-26352-monitoring-haproxy-0%22%7d%20or%20haproxy_backend_status%7Bnode_name%3D%7E%22pxc-prefix-monitoring-pmm3-26352-monitoring-haproxy-0%22%7D%29&start=1791211607&end=1791211667&step=60' ++ jq '.data.result[0].values[][1]' + [[ -n "0" "0" ]] + get_metric_values_pmm3 haproxy_backend_active_servers pxc-prefix-monitoring-pmm3-26352-monitoring-haproxy-0 glsa_5JP1O3YUnBKgF4WW4DX6ZplIWMPMaXmo_18baf097 + local metric=haproxy_backend_active_servers + local instance=pxc-prefix-monitoring-pmm3-26352-monitoring-haproxy-0 + local token=glsa_5JP1O3YUnBKgF4WW4DX6ZplIWMPMaXmo_18baf097 ++ /usr/bin/date -u +%s -d '-1 minute' + local start=1791211614 ++ /usr/bin/date -u +%s + local end=1791211674 ++ get_service_endpoint monitoring-service ++ local service=monitoring-service +++ kubectl_bin get service/monitoring-service -o 'jsonpath={.status.loadBalancer.ingress[].hostname}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.90byYjQIJn ++++ mktemp +++ local LAST_ERR=/tmp/tmp.WJ4ckZQ2ah +++ local exit_status=0 ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl get service/monitoring-service -o 'jsonpath={.status.loadBalancer.ingress[].hostname}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 ']' +++ break +++ cat /tmp/tmp.90byYjQIJn +++ cat /tmp/tmp.WJ4ckZQ2ah +++ rm /tmp/tmp.90byYjQIJn /tmp/tmp.WJ4ckZQ2ah +++ return 0 ++ local endpoint= ++ '[' -z '' ']' +++ kubectl_bin get service/monitoring-service -o 'jsonpath={.status.loadBalancer.ingress[].ip}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.sIRsDzu8E0 ++++ mktemp +++ local LAST_ERR=/tmp/tmp.AZDELU6XJ4 +++ local exit_status=0 ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl get service/monitoring-service -o 'jsonpath={.status.loadBalancer.ingress[].ip}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 ']' +++ break +++ cat /tmp/tmp.sIRsDzu8E0 +++ cat /tmp/tmp.AZDELU6XJ4 +++ rm /tmp/tmp.sIRsDzu8E0 /tmp/tmp.AZDELU6XJ4 +++ return 0 ++ endpoint=104.154.140.66 ++ '[' -n 104.154.140.66 ']' ++ '[' 104.154.140.66 '!=' null ']' ++ sed -e 's/^"//; s/"$//;' ++ echo 104.154.140.66 ++ head -n 1 ++ return + local endpoint=104.154.140.66 + '[' -z haproxy_backend_active_servers ']' + '[' -z glsa_5JP1O3YUnBKgF4WW4DX6ZplIWMPMaXmo_18baf097 ']' + local wait_count=30 + local retry=0 ++ curl -s -k -H 'Authorization: Bearer glsa_5JP1O3YUnBKgF4WW4DX6ZplIWMPMaXmo_18baf097' 'https://104.154.140.66/graph/api/datasources/proxy/1/api/v1/query_range?query=min%28haproxy_backend_active_servers%7Bnode_name%3D%7E%22pxc-prefix-monitoring-pmm3-26352-monitoring-haproxy-0%22%7d%20or%20haproxy_backend_active_servers%7Bnode_name%3D%7E%22pxc-prefix-monitoring-pmm3-26352-monitoring-haproxy-0%22%7D%29&start=1791211614&end=1791211674&step=60' ++ jq '.data.result[0].values[][1]' ++ grep '^"[0-9]' + [[ -n "1" "0" ]] + desc 'check QAN data' + set +o xtrace ----------------------------------------------------------------------------------- check QAN data ----------------------------------------------------------------------------------- + get_qan20_values monitoring-pxc-0 glsa_5JP1O3YUnBKgF4WW4DX6ZplIWMPMaXmo_18baf097 + local instance=monitoring-pxc-0 + local token=glsa_5JP1O3YUnBKgF4WW4DX6ZplIWMPMaXmo_18baf097 ++ /usr/bin/date -u +%Y-%m-%dT%H:%M:%S -d '-30 minute' + local start=2026-10-05T14:17:57 ++ /usr/bin/date -u +%Y-%m-%dT%H:%M:%S + local end=2026-10-05T14:47:57 ++ get_service_endpoint monitoring-service ++ local service=monitoring-service +++ kubectl_bin get service/monitoring-service -o 'jsonpath={.status.loadBalancer.ingress[].hostname}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.fGaih60Pg2 ++++ mktemp +++ local LAST_ERR=/tmp/tmp.Drmv4gfjso +++ local exit_status=0 ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl get service/monitoring-service -o 'jsonpath={.status.loadBalancer.ingress[].hostname}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 ']' +++ break +++ cat /tmp/tmp.fGaih60Pg2 +++ cat /tmp/tmp.Drmv4gfjso +++ rm /tmp/tmp.fGaih60Pg2 /tmp/tmp.Drmv4gfjso +++ return 0 ++ local endpoint= ++ '[' -z '' ']' +++ kubectl_bin get service/monitoring-service -o 'jsonpath={.status.loadBalancer.ingress[].ip}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.SsVnRwtlFX ++++ mktemp +++ local LAST_ERR=/tmp/tmp.gmKEroXOp4 +++ local exit_status=0 ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl get service/monitoring-service -o 'jsonpath={.status.loadBalancer.ingress[].ip}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 ']' +++ break +++ cat /tmp/tmp.SsVnRwtlFX +++ cat /tmp/tmp.gmKEroXOp4 +++ rm /tmp/tmp.SsVnRwtlFX /tmp/tmp.gmKEroXOp4 +++ return 0 ++ endpoint=104.154.140.66 ++ '[' -n 104.154.140.66 ']' ++ '[' 104.154.140.66 '!=' null ']' ++ sed -e 's/^"//; s/"$//;' ++ head -n 1 ++ echo 104.154.140.66 ++ return + local endpoint=104.154.140.66 + cat ++ /usr/bin/date -u -d '-12 hour' +%Y-%m-%dT%H:%M:%S%:z ++ /usr/bin/date -u +%Y-%m-%dT%H:%M:%S%:z + jq '.rows[].fingerprint' + curl -s -k -H 'Authorization: Bearer glsa_5JP1O3YUnBKgF4WW4DX6ZplIWMPMaXmo_18baf097' -XPOST -d @payload.json https://104.154.140.66/v1/qan/metrics:getReport null + rm -f payload.json + desc 'verify that the custom cluster name is configured' + set +o xtrace ----------------------------------------------------------------------------------- verify that the custom cluster name is configured ----------------------------------------------------------------------------------- + verify_custom_cluster_name pxc-prefix-foo-custom-cluster-name glsa_5JP1O3YUnBKgF4WW4DX6ZplIWMPMaXmo_18baf097 pxc-prefix-monitoring-pmm3-26352-monitoring-pxc-0 pxc-prefix-monitoring-pmm3-26352-monitoring-pxc-1 pxc-prefix-monitoring-pmm3-26352-monitoring-pxc-2 + local expected_cluster=pxc-prefix-foo-custom-cluster-name + local token=glsa_5JP1O3YUnBKgF4WW4DX6ZplIWMPMaXmo_18baf097 + shift 2 + service_names=('pxc-prefix-monitoring-pmm3-26352-monitoring-pxc-0' 'pxc-prefix-monitoring-pmm3-26352-monitoring-pxc-1' 'pxc-prefix-monitoring-pmm3-26352-monitoring-pxc-2') + local service_names + local endpoint ++ get_service_endpoint monitoring-service ++ local service=monitoring-service +++ kubectl_bin get service/monitoring-service -o 'jsonpath={.status.loadBalancer.ingress[].hostname}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.TBEsrSN6id ++++ mktemp +++ local LAST_ERR=/tmp/tmp.YRFZSbJwhz +++ local exit_status=0 ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl get service/monitoring-service -o 'jsonpath={.status.loadBalancer.ingress[].hostname}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 ']' +++ break +++ cat /tmp/tmp.TBEsrSN6id +++ cat /tmp/tmp.YRFZSbJwhz +++ rm /tmp/tmp.TBEsrSN6id /tmp/tmp.YRFZSbJwhz +++ return 0 ++ local endpoint= ++ '[' -z '' ']' +++ kubectl_bin get service/monitoring-service -o 'jsonpath={.status.loadBalancer.ingress[].ip}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.EVgroyijrm ++++ mktemp +++ local LAST_ERR=/tmp/tmp.pObBJ50IXo +++ local exit_status=0 ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl get service/monitoring-service -o 'jsonpath={.status.loadBalancer.ingress[].ip}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 ']' +++ break +++ cat /tmp/tmp.EVgroyijrm +++ cat /tmp/tmp.pObBJ50IXo +++ rm /tmp/tmp.EVgroyijrm /tmp/tmp.pObBJ50IXo +++ return 0 ++ endpoint=104.154.140.66 ++ '[' -n 104.154.140.66 ']' ++ '[' 104.154.140.66 '!=' null ']' ++ head -n 1 ++ echo 104.154.140.66 ++ sed -e 's/^"//; s/"$//;' ++ return + endpoint=104.154.140.66 + local response ++ curl -s -k -H 'Authorization: Bearer glsa_5JP1O3YUnBKgF4WW4DX6ZplIWMPMaXmo_18baf097' https://104.154.140.66/v1/inventory/services + response='{ "mysql": [ { "service_id": "82bde836-2370-4a42-a586-cab7e703e3e3", "service_name": "pxc-prefix-monitoring-pmm3-26352-monitoring-pxc-2", "node_id": "e72bd050-21a5-4908-b998-b50ee9de66a6", "address": "monitoring-pxc-2", "port": 33062, "socket": "", "environment": "", "cluster": "pxc-prefix-foo-custom-cluster-name", "replication_set": "", "custom_labels": {}, "version": "", "extra_dsn_params": {} }, { "service_id": "92b1f376-d918-4d5c-bc73-6b1e64d25239", "service_name": "pxc-prefix-monitoring-pmm3-26352-monitoring-pxc-1", "node_id": "abb35ca8-de48-45b7-bff6-7392e46fc2d5", "address": "monitoring-pxc-1", "port": 33062, "socket": "", "environment": "", "cluster": "pxc-prefix-foo-custom-cluster-name", "replication_set": "", "custom_labels": {}, "version": "", "extra_dsn_params": {} }, { "service_id": "f9757b9e-76d1-4e14-a253-9b6eca976404", "service_name": "pxc-prefix-monitoring-pmm3-26352-monitoring-pxc-0", "node_id": "1d4b2c42-91d2-4c16-9724-3626f9c05a81", "address": "monitoring-pxc-0", "port": 33062, "socket": "", "environment": "", "cluster": "pxc-prefix-foo-custom-cluster-name", "replication_set": "", "custom_labels": {}, "version": "", "extra_dsn_params": {} } ], "mongodb": [], "postgresql": [ { "service_id": "90140445-85e8-4118-9de2-ad845f41da7b", "service_name": "pmm-server-postgresql", "database_name": "postgres", "node_id": "pmm-server", "address": "127.0.0.1", "port": 5432, "socket": "", "environment": "", "cluster": "", "replication_set": "", "custom_labels": {}, "version": "", "auto_discovery_limit": 0 } ], "proxysql": [], "haproxy": [ { "service_id": "9d045f2c-7acf-415d-b0e2-a094c2cd5458", "service_name": "pxc-prefix-monitoring-pmm3-26352-monitoring-haproxy-1", "node_id": "b486e731-078f-4790-b4a2-a567a82fc146", "environment": "", "cluster": "pxc-prefix-foo-custom-cluster-name", "replication_set": "", "custom_labels": {} }, { "service_id": "af6225d4-888f-4f96-b9a2-e819ceaa317f", "service_name": "pxc-prefix-monitoring-pmm3-26352-monitoring-haproxy-0", "node_id": "da78329e-d19a-42f6-b2df-66fb4188a05c", "environment": "", "cluster": "pxc-prefix-foo-custom-cluster-name", "replication_set": "", "custom_labels": {} } ], "external": [], "valkey": [] }' + local verified=0 + for service_name in "${service_names[@]}" + local actual_cluster ++ jq -r --arg name pxc-prefix-monitoring-pmm3-26352-monitoring-pxc-0 ' .mysql[] | select(.service_name == $name) | .cluster ' ++ echo '{ "mysql": [ { "service_id": "82bde836-2370-4a42-a586-cab7e703e3e3", "service_name": "pxc-prefix-monitoring-pmm3-26352-monitoring-pxc-2", "node_id": "e72bd050-21a5-4908-b998-b50ee9de66a6", "address": "monitoring-pxc-2", "port": 33062, "socket": "", "environment": "", "cluster": "pxc-prefix-foo-custom-cluster-name", "replication_set": "", "custom_labels": {}, "version": "", "extra_dsn_params": {} }, { "service_id": "92b1f376-d918-4d5c-bc73-6b1e64d25239", "service_name": "pxc-prefix-monitoring-pmm3-26352-monitoring-pxc-1", "node_id": "abb35ca8-de48-45b7-bff6-7392e46fc2d5", "address": "monitoring-pxc-1", "port": 33062, "socket": "", "environment": "", "cluster": "pxc-prefix-foo-custom-cluster-name", "replication_set": "", "custom_labels": {}, "version": "", "extra_dsn_params": {} }, { "service_id": "f9757b9e-76d1-4e14-a253-9b6eca976404", "service_name": "pxc-prefix-monitoring-pmm3-26352-monitoring-pxc-0", "node_id": "1d4b2c42-91d2-4c16-9724-3626f9c05a81", "address": "monitoring-pxc-0", "port": 33062, "socket": "", "environment": "", "cluster": "pxc-prefix-foo-custom-cluster-name", "replication_set": "", "custom_labels": {}, "version": "", "extra_dsn_params": {} } ], "mongodb": [], "postgresql": [ { "service_id": "90140445-85e8-4118-9de2-ad845f41da7b", "service_name": "pmm-server-postgresql", "database_name": "postgres", "node_id": "pmm-server", "address": "127.0.0.1", "port": 5432, "socket": "", "environment": "", "cluster": "", "replication_set": "", "custom_labels": {}, "version": "", "auto_discovery_limit": 0 } ], "proxysql": [], "haproxy": [ { "service_id": "9d045f2c-7acf-415d-b0e2-a094c2cd5458", "service_name": "pxc-prefix-monitoring-pmm3-26352-monitoring-haproxy-1", "node_id": "b486e731-078f-4790-b4a2-a567a82fc146", "environment": "", "cluster": "pxc-prefix-foo-custom-cluster-name", "replication_set": "", "custom_labels": {} }, { "service_id": "af6225d4-888f-4f96-b9a2-e819ceaa317f", "service_name": "pxc-prefix-monitoring-pmm3-26352-monitoring-haproxy-0", "node_id": "da78329e-d19a-42f6-b2df-66fb4188a05c", "environment": "", "cluster": "pxc-prefix-foo-custom-cluster-name", "replication_set": "", "custom_labels": {} } ], "external": [], "valkey": [] }' + actual_cluster=pxc-prefix-foo-custom-cluster-name + [[ -z pxc-prefix-foo-custom-cluster-name ]] + [[ pxc-prefix-foo-custom-cluster-name == \n\u\l\l ]] + [[ pxc-prefix-foo-custom-cluster-name != \p\x\c\-\p\r\e\f\i\x\-\f\o\o\-\c\u\s\t\o\m\-\c\l\u\s\t\e\r\-\n\a\m\e ]] + for service_name in "${service_names[@]}" + local actual_cluster ++ jq -r --arg name pxc-prefix-monitoring-pmm3-26352-monitoring-pxc-1 ' .mysql[] | select(.service_name == $name) | .cluster ' ++ echo '{ "mysql": [ { "service_id": "82bde836-2370-4a42-a586-cab7e703e3e3", "service_name": "pxc-prefix-monitoring-pmm3-26352-monitoring-pxc-2", "node_id": "e72bd050-21a5-4908-b998-b50ee9de66a6", "address": "monitoring-pxc-2", "port": 33062, "socket": "", "environment": "", "cluster": "pxc-prefix-foo-custom-cluster-name", "replication_set": "", "custom_labels": {}, "version": "", "extra_dsn_params": {} }, { "service_id": "92b1f376-d918-4d5c-bc73-6b1e64d25239", "service_name": "pxc-prefix-monitoring-pmm3-26352-monitoring-pxc-1", "node_id": "abb35ca8-de48-45b7-bff6-7392e46fc2d5", "address": "monitoring-pxc-1", "port": 33062, "socket": "", "environment": "", "cluster": "pxc-prefix-foo-custom-cluster-name", "replication_set": "", "custom_labels": {}, "version": "", "extra_dsn_params": {} }, { "service_id": "f9757b9e-76d1-4e14-a253-9b6eca976404", "service_name": "pxc-prefix-monitoring-pmm3-26352-monitoring-pxc-0", "node_id": "1d4b2c42-91d2-4c16-9724-3626f9c05a81", "address": "monitoring-pxc-0", "port": 33062, "socket": "", "environment": "", "cluster": "pxc-prefix-foo-custom-cluster-name", "replication_set": "", "custom_labels": {}, "version": "", "extra_dsn_params": {} } ], "mongodb": [], "postgresql": [ { "service_id": "90140445-85e8-4118-9de2-ad845f41da7b", "service_name": "pmm-server-postgresql", "database_name": "postgres", "node_id": "pmm-server", "address": "127.0.0.1", "port": 5432, "socket": "", "environment": "", "cluster": "", "replication_set": "", "custom_labels": {}, "version": "", "auto_discovery_limit": 0 } ], "proxysql": [], "haproxy": [ { "service_id": "9d045f2c-7acf-415d-b0e2-a094c2cd5458", "service_name": "pxc-prefix-monitoring-pmm3-26352-monitoring-haproxy-1", "node_id": "b486e731-078f-4790-b4a2-a567a82fc146", "environment": "", "cluster": "pxc-prefix-foo-custom-cluster-name", "replication_set": "", "custom_labels": {} }, { "service_id": "af6225d4-888f-4f96-b9a2-e819ceaa317f", "service_name": "pxc-prefix-monitoring-pmm3-26352-monitoring-haproxy-0", "node_id": "da78329e-d19a-42f6-b2df-66fb4188a05c", "environment": "", "cluster": "pxc-prefix-foo-custom-cluster-name", "replication_set": "", "custom_labels": {} } ], "external": [], "valkey": [] }' + actual_cluster=pxc-prefix-foo-custom-cluster-name + [[ -z pxc-prefix-foo-custom-cluster-name ]] + [[ pxc-prefix-foo-custom-cluster-name == \n\u\l\l ]] + [[ pxc-prefix-foo-custom-cluster-name != \p\x\c\-\p\r\e\f\i\x\-\f\o\o\-\c\u\s\t\o\m\-\c\l\u\s\t\e\r\-\n\a\m\e ]] + for service_name in "${service_names[@]}" + local actual_cluster ++ jq -r --arg name pxc-prefix-monitoring-pmm3-26352-monitoring-pxc-2 ' .mysql[] | select(.service_name == $name) | .cluster ' ++ echo '{ "mysql": [ { "service_id": "82bde836-2370-4a42-a586-cab7e703e3e3", "service_name": "pxc-prefix-monitoring-pmm3-26352-monitoring-pxc-2", "node_id": "e72bd050-21a5-4908-b998-b50ee9de66a6", "address": "monitoring-pxc-2", "port": 33062, "socket": "", "environment": "", "cluster": "pxc-prefix-foo-custom-cluster-name", "replication_set": "", "custom_labels": {}, "version": "", "extra_dsn_params": {} }, { "service_id": "92b1f376-d918-4d5c-bc73-6b1e64d25239", "service_name": "pxc-prefix-monitoring-pmm3-26352-monitoring-pxc-1", "node_id": "abb35ca8-de48-45b7-bff6-7392e46fc2d5", "address": "monitoring-pxc-1", "port": 33062, "socket": "", "environment": "", "cluster": "pxc-prefix-foo-custom-cluster-name", "replication_set": "", "custom_labels": {}, "version": "", "extra_dsn_params": {} }, { "service_id": "f9757b9e-76d1-4e14-a253-9b6eca976404", "service_name": "pxc-prefix-monitoring-pmm3-26352-monitoring-pxc-0", "node_id": "1d4b2c42-91d2-4c16-9724-3626f9c05a81", "address": "monitoring-pxc-0", "port": 33062, "socket": "", "environment": "", "cluster": "pxc-prefix-foo-custom-cluster-name", "replication_set": "", "custom_labels": {}, "version": "", "extra_dsn_params": {} } ], "mongodb": [], "postgresql": [ { "service_id": "90140445-85e8-4118-9de2-ad845f41da7b", "service_name": "pmm-server-postgresql", "database_name": "postgres", "node_id": "pmm-server", "address": "127.0.0.1", "port": 5432, "socket": "", "environment": "", "cluster": "", "replication_set": "", "custom_labels": {}, "version": "", "auto_discovery_limit": 0 } ], "proxysql": [], "haproxy": [ { "service_id": "9d045f2c-7acf-415d-b0e2-a094c2cd5458", "service_name": "pxc-prefix-monitoring-pmm3-26352-monitoring-haproxy-1", "node_id": "b486e731-078f-4790-b4a2-a567a82fc146", "environment": "", "cluster": "pxc-prefix-foo-custom-cluster-name", "replication_set": "", "custom_labels": {} }, { "service_id": "af6225d4-888f-4f96-b9a2-e819ceaa317f", "service_name": "pxc-prefix-monitoring-pmm3-26352-monitoring-haproxy-0", "node_id": "da78329e-d19a-42f6-b2df-66fb4188a05c", "environment": "", "cluster": "pxc-prefix-foo-custom-cluster-name", "replication_set": "", "custom_labels": {} } ], "external": [], "valkey": [] }' + actual_cluster=pxc-prefix-foo-custom-cluster-name + [[ -z pxc-prefix-foo-custom-cluster-name ]] + [[ pxc-prefix-foo-custom-cluster-name == \n\u\l\l ]] + [[ pxc-prefix-foo-custom-cluster-name != \p\x\c\-\p\r\e\f\i\x\-\f\o\o\-\c\u\s\t\o\m\-\c\l\u\s\t\e\r\-\n\a\m\e ]] + return 0 + nodeList=($(get_node_id_from_pmm)) ++ get_node_id_from_pmm ++ nodeList=() ++ local -a nodeList +++ kubectl_bin get pods --no-headers -l app.kubernetes.io/component=pxc --output=custom-columns=NAME:.metadata.name ++++ mktemp +++ local LAST_OUT=/tmp/tmp.vBzXbnENCY ++++ mktemp +++ local LAST_ERR=/tmp/tmp.k7aSgAuonE +++ local exit_status=0 ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl get pods --no-headers -l app.kubernetes.io/component=pxc --output=custom-columns=NAME:.metadata.name +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 ']' +++ break +++ cat /tmp/tmp.vBzXbnENCY +++ cat /tmp/tmp.k7aSgAuonE +++ rm /tmp/tmp.vBzXbnENCY /tmp/tmp.k7aSgAuonE +++ return 0 ++ for instance in $(kubectl_bin get pods --no-headers -l app.kubernetes.io/component=pxc --output=custom-columns='NAME:.metadata.name') ++ nodeList+=($(kubectl_bin exec -n "$namespace" $instance -c pmm-client -- pmm-admin status --json | jq -r '.pmm_agent_status.node_id')) +++ kubectl_bin exec -n monitoring-pmm3-26352 monitoring-pxc-0 -c pmm-client -- pmm-admin status --json +++ jq -r .pmm_agent_status.node_id ++++ mktemp +++ local LAST_OUT=/tmp/tmp.d9ulvl6L3M ++++ mktemp +++ local LAST_ERR=/tmp/tmp.5PkVz6KSj0 +++ local exit_status=0 ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl exec -n monitoring-pmm3-26352 monitoring-pxc-0 -c pmm-client -- pmm-admin status --json +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 ']' +++ break +++ cat /tmp/tmp.d9ulvl6L3M +++ cat /tmp/tmp.5PkVz6KSj0 +++ rm /tmp/tmp.d9ulvl6L3M /tmp/tmp.5PkVz6KSj0 +++ return 0 ++ for instance in $(kubectl_bin get pods --no-headers -l app.kubernetes.io/component=pxc --output=custom-columns='NAME:.metadata.name') ++ nodeList+=($(kubectl_bin exec -n "$namespace" $instance -c pmm-client -- pmm-admin status --json | jq -r '.pmm_agent_status.node_id')) +++ kubectl_bin exec -n monitoring-pmm3-26352 monitoring-pxc-1 -c pmm-client -- pmm-admin status --json +++ jq -r .pmm_agent_status.node_id ++++ mktemp +++ local LAST_OUT=/tmp/tmp.HTJBvcrSEo ++++ mktemp +++ local LAST_ERR=/tmp/tmp.FWPvPywgGl +++ local exit_status=0 ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl exec -n monitoring-pmm3-26352 monitoring-pxc-1 -c pmm-client -- pmm-admin status --json +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 ']' +++ break +++ cat /tmp/tmp.HTJBvcrSEo +++ cat /tmp/tmp.FWPvPywgGl +++ rm /tmp/tmp.HTJBvcrSEo /tmp/tmp.FWPvPywgGl +++ return 0 ++ for instance in $(kubectl_bin get pods --no-headers -l app.kubernetes.io/component=pxc --output=custom-columns='NAME:.metadata.name') ++ nodeList+=($(kubectl_bin exec -n "$namespace" $instance -c pmm-client -- pmm-admin status --json | jq -r '.pmm_agent_status.node_id')) +++ kubectl_bin exec -n monitoring-pmm3-26352 monitoring-pxc-2 -c pmm-client -- pmm-admin status --json +++ jq -r .pmm_agent_status.node_id ++++ mktemp +++ local LAST_OUT=/tmp/tmp.k1JwGKBBhd ++++ mktemp +++ local LAST_ERR=/tmp/tmp.UxRGS8nOoX +++ local exit_status=0 ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl exec -n monitoring-pmm3-26352 monitoring-pxc-2 -c pmm-client -- pmm-admin status --json +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 ']' +++ break +++ cat /tmp/tmp.k1JwGKBBhd +++ cat /tmp/tmp.UxRGS8nOoX +++ rm /tmp/tmp.k1JwGKBBhd /tmp/tmp.UxRGS8nOoX +++ return 0 ++ echo 1d4b2c42-91d2-4c16-9724-3626f9c05a81 abb35ca8-de48-45b7-bff6-7392e46fc2d5 e72bd050-21a5-4908-b998-b50ee9de66a6 + nodeList_from_pmm=($(does_node_id_exists "${nodeList[@]}")) ++ does_node_id_exists 1d4b2c42-91d2-4c16-9724-3626f9c05a81 abb35ca8-de48-45b7-bff6-7392e46fc2d5 e72bd050-21a5-4908-b998-b50ee9de66a6 ++ nodeList=('1d4b2c42-91d2-4c16-9724-3626f9c05a81' 'abb35ca8-de48-45b7-bff6-7392e46fc2d5' 'e72bd050-21a5-4908-b998-b50ee9de66a6') ++ local -a nodeList ++ nodeList_from_pmm=() ++ local -a nodeList_from_pmm ++ for node_id in "${nodeList[@]}" ++ nodeList_from_pmm+=($(kubectl_bin exec -n "${namespace}" monitoring-0 -- pmm-admin --server-url=https://admin:admin@$(get_service_ip monitoring-service)/ --server-insecure-tls inventory list nodes --node-type=CONTAINER_NODE | grep $node_id | awk '{print $4}')) +++ grep 1d4b2c42-91d2-4c16-9724-3626f9c05a81 ++++ get_service_ip monitoring-service ++++ local service=monitoring-service ++++ kubectl_bin get service/monitoring-service -o 'jsonpath={.spec.type}' +++ awk '{print $4}' ++++ grep -q NotFound +++++ kubectl_bin get service/monitoring-service -o 'jsonpath={.spec.type}' ++++++ mktemp +++++ local LAST_OUT=/tmp/tmp.wGZJ54eQc3 ++++++ mktemp +++++ local LAST_ERR=/tmp/tmp.UKLHpFwELU +++++ local exit_status=0 ++++++ seq 0 2 +++++ for i in $(seq 0 2) +++++ set +e +++++ kubectl get service/monitoring-service -o 'jsonpath={.spec.type}' +++++ exit_status=0 +++++ set -e +++++ '[' 0 '!=' 0 ']' +++++ break +++++ cat /tmp/tmp.wGZJ54eQc3 +++++ cat /tmp/tmp.UKLHpFwELU +++++ rm /tmp/tmp.wGZJ54eQc3 /tmp/tmp.UKLHpFwELU +++++ return 0 ++++ '[' LoadBalancer = ClusterIP ']' ++++ kubectl_bin get service/monitoring-service -o 'jsonpath={.status.loadBalancer.ingress[]}' ++++ grep -E -q 'hostname|ip' ++++ kubectl_bin get service/monitoring-service -o 'jsonpath={.status.loadBalancer.ingress[].ip}' +++++ mktemp ++++ local LAST_OUT=/tmp/tmp.yyYVqcJtnP +++++ mktemp ++++ local LAST_ERR=/tmp/tmp.g7vGUuRbGe ++++ local exit_status=0 +++++ seq 0 2 ++++ for i in $(seq 0 2) ++++ set +e ++++ kubectl get service/monitoring-service -o 'jsonpath={.status.loadBalancer.ingress[].ip}' ++++ exit_status=0 ++++ set -e ++++ '[' 0 '!=' 0 ']' ++++ break ++++ cat /tmp/tmp.yyYVqcJtnP ++++ cat /tmp/tmp.g7vGUuRbGe ++++ rm /tmp/tmp.yyYVqcJtnP /tmp/tmp.g7vGUuRbGe ++++ return 0 ++++ kubectl_bin get service/monitoring-service -o 'jsonpath={.status.loadBalancer.ingress[].hostname}' +++++ mktemp ++++ local LAST_OUT=/tmp/tmp.dRzKEHEqDJ +++++ mktemp ++++ local LAST_ERR=/tmp/tmp.2FsVmTQeeV ++++ local exit_status=0 +++++ seq 0 2 ++++ for i in $(seq 0 2) ++++ set +e ++++ kubectl get service/monitoring-service -o 'jsonpath={.status.loadBalancer.ingress[].hostname}' ++++ exit_status=0 ++++ set -e ++++ '[' 0 '!=' 0 ']' ++++ break ++++ cat /tmp/tmp.dRzKEHEqDJ ++++ cat /tmp/tmp.2FsVmTQeeV ++++ rm /tmp/tmp.dRzKEHEqDJ /tmp/tmp.2FsVmTQeeV ++++ return 0 +++ kubectl_bin exec -n monitoring-pmm3-26352 monitoring-0 -- pmm-admin --server-url=https://admin:admin@104.154.140.66/ --server-insecure-tls inventory list nodes --node-type=CONTAINER_NODE ++++ mktemp +++ local LAST_OUT=/tmp/tmp.LCDxSuJKmp ++++ mktemp +++ local LAST_ERR=/tmp/tmp.fNCQ1uVrU6 +++ local exit_status=0 ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl exec -n monitoring-pmm3-26352 monitoring-0 -- pmm-admin --server-url=https://admin:admin@104.154.140.66/ --server-insecure-tls inventory list nodes --node-type=CONTAINER_NODE +++ exit_status=1 +++ set -e +++ '[' 1 '!=' 0 ']' +++ '[' 1 == 1 ']' +++ sleep 0 +++ for i in $(seq 0 2) +++ set +e +++ kubectl exec -n monitoring-pmm3-26352 monitoring-0 -- pmm-admin --server-url=https://admin:admin@104.154.140.66/ --server-insecure-tls inventory list nodes --node-type=CONTAINER_NODE +++ exit_status=1 +++ set -e +++ '[' 1 '!=' 0 ']' +++ '[' 1 == 1 ']' +++ sleep 0 +++ for i in $(seq 0 2) +++ set +e +++ kubectl exec -n monitoring-pmm3-26352 monitoring-0 -- pmm-admin --server-url=https://admin:admin@104.154.140.66/ --server-insecure-tls inventory list nodes --node-type=CONTAINER_NODE +++ exit_status=1 +++ set -e +++ '[' 1 '!=' 0 ']' +++ '[' 1 == 1 ']' +++ sleep 0 +++ cat /tmp/tmp.LCDxSuJKmp +++ cat /tmp/tmp.fNCQ1uVrU6 command terminated with exit code 1 +++ rm /tmp/tmp.LCDxSuJKmp /tmp/tmp.fNCQ1uVrU6 +++ return 1 ++ for node_id in "${nodeList[@]}" ++ nodeList_from_pmm+=($(kubectl_bin exec -n "${namespace}" monitoring-0 -- pmm-admin --server-url=https://admin:admin@$(get_service_ip monitoring-service)/ --server-insecure-tls inventory list nodes --node-type=CONTAINER_NODE | grep $node_id | awk '{print $4}')) +++ grep abb35ca8-de48-45b7-bff6-7392e46fc2d5 +++ awk '{print $4}' ++++ get_service_ip monitoring-service ++++ local service=monitoring-service ++++ grep -q NotFound ++++ kubectl_bin get service/monitoring-service -o 'jsonpath={.spec.type}' +++++ kubectl_bin get service/monitoring-service -o 'jsonpath={.spec.type}' ++++++ mktemp +++++ local LAST_OUT=/tmp/tmp.NyqBHWwOqD ++++++ mktemp +++++ local LAST_ERR=/tmp/tmp.ixuIaam6C6 +++++ local exit_status=0 ++++++ seq 0 2 +++++ for i in $(seq 0 2) +++++ set +e +++++ kubectl get service/monitoring-service -o 'jsonpath={.spec.type}' +++++ exit_status=0 +++++ set -e +++++ '[' 0 '!=' 0 ']' +++++ break +++++ cat /tmp/tmp.NyqBHWwOqD +++++ cat /tmp/tmp.ixuIaam6C6 +++++ rm /tmp/tmp.NyqBHWwOqD /tmp/tmp.ixuIaam6C6 +++++ return 0 ++++ '[' LoadBalancer = ClusterIP ']' ++++ kubectl_bin get service/monitoring-service -o 'jsonpath={.status.loadBalancer.ingress[]}' ++++ grep -E -q 'hostname|ip' ++++ kubectl_bin get service/monitoring-service -o 'jsonpath={.status.loadBalancer.ingress[].ip}' +++++ mktemp ++++ local LAST_OUT=/tmp/tmp.kYE7EHejdj +++++ mktemp ++++ local LAST_ERR=/tmp/tmp.r04W5f4bQy ++++ local exit_status=0 +++++ seq 0 2 ++++ for i in $(seq 0 2) ++++ set +e ++++ kubectl get service/monitoring-service -o 'jsonpath={.status.loadBalancer.ingress[].ip}' ++++ exit_status=0 ++++ set -e ++++ '[' 0 '!=' 0 ']' ++++ break ++++ cat /tmp/tmp.kYE7EHejdj ++++ cat /tmp/tmp.r04W5f4bQy ++++ rm /tmp/tmp.kYE7EHejdj /tmp/tmp.r04W5f4bQy ++++ return 0 ++++ kubectl_bin get service/monitoring-service -o 'jsonpath={.status.loadBalancer.ingress[].hostname}' +++++ mktemp ++++ local LAST_OUT=/tmp/tmp.LufomXbEWM +++++ mktemp ++++ local LAST_ERR=/tmp/tmp.DuaO0vkW1F ++++ local exit_status=0 +++++ seq 0 2 ++++ for i in $(seq 0 2) ++++ set +e ++++ kubectl get service/monitoring-service -o 'jsonpath={.status.loadBalancer.ingress[].hostname}' ++++ exit_status=0 ++++ set -e ++++ '[' 0 '!=' 0 ']' ++++ break ++++ cat /tmp/tmp.LufomXbEWM ++++ cat /tmp/tmp.DuaO0vkW1F ++++ rm /tmp/tmp.LufomXbEWM /tmp/tmp.DuaO0vkW1F ++++ return 0 +++ kubectl_bin exec -n monitoring-pmm3-26352 monitoring-0 -- pmm-admin --server-url=https://admin:admin@104.154.140.66/ --server-insecure-tls inventory list nodes --node-type=CONTAINER_NODE ++++ mktemp +++ local LAST_OUT=/tmp/tmp.amL8NmzaL4 ++++ mktemp +++ local LAST_ERR=/tmp/tmp.x9GSEGRFxl +++ local exit_status=0 ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl exec -n monitoring-pmm3-26352 monitoring-0 -- pmm-admin --server-url=https://admin:admin@104.154.140.66/ --server-insecure-tls inventory list nodes --node-type=CONTAINER_NODE +++ exit_status=1 +++ set -e +++ '[' 1 '!=' 0 ']' +++ '[' 1 == 1 ']' +++ sleep 0 +++ for i in $(seq 0 2) +++ set +e +++ kubectl exec -n monitoring-pmm3-26352 monitoring-0 -- pmm-admin --server-url=https://admin:admin@104.154.140.66/ --server-insecure-tls inventory list nodes --node-type=CONTAINER_NODE +++ exit_status=1 +++ set -e +++ '[' 1 '!=' 0 ']' +++ '[' 1 == 1 ']' +++ sleep 0 +++ for i in $(seq 0 2) +++ set +e +++ kubectl exec -n monitoring-pmm3-26352 monitoring-0 -- pmm-admin --server-url=https://admin:admin@104.154.140.66/ --server-insecure-tls inventory list nodes --node-type=CONTAINER_NODE +++ exit_status=1 +++ set -e +++ '[' 1 '!=' 0 ']' +++ '[' 1 == 1 ']' +++ sleep 0 +++ cat /tmp/tmp.amL8NmzaL4 +++ cat /tmp/tmp.x9GSEGRFxl command terminated with exit code 1 +++ rm /tmp/tmp.amL8NmzaL4 /tmp/tmp.x9GSEGRFxl +++ return 1 ++ for node_id in "${nodeList[@]}" ++ nodeList_from_pmm+=($(kubectl_bin exec -n "${namespace}" monitoring-0 -- pmm-admin --server-url=https://admin:admin@$(get_service_ip monitoring-service)/ --server-insecure-tls inventory list nodes --node-type=CONTAINER_NODE | grep $node_id | awk '{print $4}')) +++ awk '{print $4}' +++ grep e72bd050-21a5-4908-b998-b50ee9de66a6 ++++ get_service_ip monitoring-service ++++ local service=monitoring-service ++++ kubectl_bin get service/monitoring-service -o 'jsonpath={.spec.type}' ++++ grep -q NotFound +++++ kubectl_bin get service/monitoring-service -o 'jsonpath={.spec.type}' ++++++ mktemp +++++ local LAST_OUT=/tmp/tmp.4zQNqQnF00 ++++++ mktemp +++++ local LAST_ERR=/tmp/tmp.P16bBih3Ke +++++ local exit_status=0 ++++++ seq 0 2 +++++ for i in $(seq 0 2) +++++ set +e +++++ kubectl get service/monitoring-service -o 'jsonpath={.spec.type}' +++++ exit_status=0 +++++ set -e +++++ '[' 0 '!=' 0 ']' +++++ break +++++ cat /tmp/tmp.4zQNqQnF00 +++++ cat /tmp/tmp.P16bBih3Ke +++++ rm /tmp/tmp.4zQNqQnF00 /tmp/tmp.P16bBih3Ke +++++ return 0 ++++ '[' LoadBalancer = ClusterIP ']' ++++ kubectl_bin get service/monitoring-service -o 'jsonpath={.status.loadBalancer.ingress[]}' ++++ grep -E -q 'hostname|ip' ++++ kubectl_bin get service/monitoring-service -o 'jsonpath={.status.loadBalancer.ingress[].ip}' +++++ mktemp ++++ local LAST_OUT=/tmp/tmp.BqsfEFbbAy +++++ mktemp ++++ local LAST_ERR=/tmp/tmp.PmTNxLUZ1R ++++ local exit_status=0 +++++ seq 0 2 ++++ for i in $(seq 0 2) ++++ set +e ++++ kubectl get service/monitoring-service -o 'jsonpath={.status.loadBalancer.ingress[].ip}' ++++ exit_status=0 ++++ set -e ++++ '[' 0 '!=' 0 ']' ++++ break ++++ cat /tmp/tmp.BqsfEFbbAy ++++ cat /tmp/tmp.PmTNxLUZ1R ++++ rm /tmp/tmp.BqsfEFbbAy /tmp/tmp.PmTNxLUZ1R ++++ return 0 ++++ kubectl_bin get service/monitoring-service -o 'jsonpath={.status.loadBalancer.ingress[].hostname}' +++++ mktemp ++++ local LAST_OUT=/tmp/tmp.jlr2dKcmoF +++++ mktemp ++++ local LAST_ERR=/tmp/tmp.C8P5R7aUT1 ++++ local exit_status=0 +++++ seq 0 2 ++++ for i in $(seq 0 2) ++++ set +e ++++ kubectl get service/monitoring-service -o 'jsonpath={.status.loadBalancer.ingress[].hostname}' ++++ exit_status=0 ++++ set -e ++++ '[' 0 '!=' 0 ']' ++++ break ++++ cat /tmp/tmp.jlr2dKcmoF ++++ cat /tmp/tmp.C8P5R7aUT1 ++++ rm /tmp/tmp.jlr2dKcmoF /tmp/tmp.C8P5R7aUT1 ++++ return 0 +++ kubectl_bin exec -n monitoring-pmm3-26352 monitoring-0 -- pmm-admin --server-url=https://admin:admin@104.154.140.66/ --server-insecure-tls inventory list nodes --node-type=CONTAINER_NODE ++++ mktemp +++ local LAST_OUT=/tmp/tmp.GXqcniTbeb ++++ mktemp +++ local LAST_ERR=/tmp/tmp.HinAAbsiCq +++ local exit_status=0 ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl exec -n monitoring-pmm3-26352 monitoring-0 -- pmm-admin --server-url=https://admin:admin@104.154.140.66/ --server-insecure-tls inventory list nodes --node-type=CONTAINER_NODE +++ exit_status=1 +++ set -e +++ '[' 1 '!=' 0 ']' +++ '[' 1 == 1 ']' +++ sleep 0 +++ for i in $(seq 0 2) +++ set +e +++ kubectl exec -n monitoring-pmm3-26352 monitoring-0 -- pmm-admin --server-url=https://admin:admin@104.154.140.66/ --server-insecure-tls inventory list nodes --node-type=CONTAINER_NODE +++ exit_status=1 +++ set -e +++ '[' 1 '!=' 0 ']' +++ '[' 1 == 1 ']' +++ sleep 0 +++ for i in $(seq 0 2) +++ set +e +++ kubectl exec -n monitoring-pmm3-26352 monitoring-0 -- pmm-admin --server-url=https://admin:admin@104.154.140.66/ --server-insecure-tls inventory list nodes --node-type=CONTAINER_NODE +++ exit_status=1 +++ set -e +++ '[' 1 '!=' 0 ']' +++ '[' 1 == 1 ']' +++ sleep 0 +++ cat /tmp/tmp.GXqcniTbeb +++ cat /tmp/tmp.HinAAbsiCq command terminated with exit code 1 +++ rm /tmp/tmp.GXqcniTbeb /tmp/tmp.HinAAbsiCq +++ return 1 ++ echo + kubectl_bin patch pxc monitoring --type json '-p=[{"op":"add","path":"/spec/pause","value":true}]' ++ mktemp + local LAST_OUT=/tmp/tmp.ApUL029S2g ++ mktemp + local LAST_ERR=/tmp/tmp.fCJMzFHxPl + local exit_status=0 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl patch pxc monitoring --type json '-p=[{"op":"add","path":"/spec/pause","value":true}]' + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.ApUL029S2g perconaxtradbcluster.pxc.percona.com/monitoring patched + cat /tmp/tmp.fCJMzFHxPl + rm /tmp/tmp.ApUL029S2g /tmp/tmp.fCJMzFHxPl + return 0 + wait_for_delete pod/monitoring-pxc-0 + local res=pod/monitoring-pxc-0 + echo -n 'waiting for pod/monitoring-pxc-0 to be deleted' waiting for pod/monitoring-pxc-0 to be deleted+ set +o xtrace ..........Error from server (NotFound): pods "monitoring-pxc-0" not found + does_node_id_exists_in_pmm=($(does_node_id_exists "${nodeList[@]}")) ++ does_node_id_exists 1d4b2c42-91d2-4c16-9724-3626f9c05a81 abb35ca8-de48-45b7-bff6-7392e46fc2d5 e72bd050-21a5-4908-b998-b50ee9de66a6 ++ nodeList=('1d4b2c42-91d2-4c16-9724-3626f9c05a81' 'abb35ca8-de48-45b7-bff6-7392e46fc2d5' 'e72bd050-21a5-4908-b998-b50ee9de66a6') ++ local -a nodeList ++ nodeList_from_pmm=() ++ local -a nodeList_from_pmm ++ for node_id in "${nodeList[@]}" ++ nodeList_from_pmm+=($(kubectl_bin exec -n "${namespace}" monitoring-0 -- pmm-admin --server-url=https://admin:admin@$(get_service_ip monitoring-service)/ --server-insecure-tls inventory list nodes --node-type=CONTAINER_NODE | grep $node_id | awk '{print $4}')) +++ awk '{print $4}' +++ grep 1d4b2c42-91d2-4c16-9724-3626f9c05a81 ++++ get_service_ip monitoring-service ++++ local service=monitoring-service ++++ kubectl_bin get service/monitoring-service -o 'jsonpath={.spec.type}' ++++ grep -q NotFound +++++ kubectl_bin get service/monitoring-service -o 'jsonpath={.spec.type}' ++++++ mktemp +++++ local LAST_OUT=/tmp/tmp.rxJmsj8FIk ++++++ mktemp +++++ local LAST_ERR=/tmp/tmp.avzO91DSKC +++++ local exit_status=0 ++++++ seq 0 2 +++++ for i in $(seq 0 2) +++++ set +e +++++ kubectl get service/monitoring-service -o 'jsonpath={.spec.type}' +++++ exit_status=0 +++++ set -e +++++ '[' 0 '!=' 0 ']' +++++ break +++++ cat /tmp/tmp.rxJmsj8FIk +++++ cat /tmp/tmp.avzO91DSKC +++++ rm /tmp/tmp.rxJmsj8FIk /tmp/tmp.avzO91DSKC +++++ return 0 ++++ '[' LoadBalancer = ClusterIP ']' ++++ grep -E -q 'hostname|ip' ++++ kubectl_bin get service/monitoring-service -o 'jsonpath={.status.loadBalancer.ingress[]}' ++++ kubectl_bin get service/monitoring-service -o 'jsonpath={.status.loadBalancer.ingress[].ip}' +++++ mktemp ++++ local LAST_OUT=/tmp/tmp.GgpFigYXPt +++++ mktemp ++++ local LAST_ERR=/tmp/tmp.QhWB5afRlc ++++ local exit_status=0 +++++ seq 0 2 ++++ for i in $(seq 0 2) ++++ set +e ++++ kubectl get service/monitoring-service -o 'jsonpath={.status.loadBalancer.ingress[].ip}' ++++ exit_status=0 ++++ set -e ++++ '[' 0 '!=' 0 ']' ++++ break ++++ cat /tmp/tmp.GgpFigYXPt ++++ cat /tmp/tmp.QhWB5afRlc ++++ rm /tmp/tmp.GgpFigYXPt /tmp/tmp.QhWB5afRlc ++++ return 0 ++++ kubectl_bin get service/monitoring-service -o 'jsonpath={.status.loadBalancer.ingress[].hostname}' +++++ mktemp ++++ local LAST_OUT=/tmp/tmp.7fjhHRTdrJ +++++ mktemp ++++ local LAST_ERR=/tmp/tmp.lmILfV2kLN ++++ local exit_status=0 +++++ seq 0 2 ++++ for i in $(seq 0 2) ++++ set +e ++++ kubectl get service/monitoring-service -o 'jsonpath={.status.loadBalancer.ingress[].hostname}' ++++ exit_status=0 ++++ set -e ++++ '[' 0 '!=' 0 ']' ++++ break ++++ cat /tmp/tmp.7fjhHRTdrJ ++++ cat /tmp/tmp.lmILfV2kLN ++++ rm /tmp/tmp.7fjhHRTdrJ /tmp/tmp.lmILfV2kLN ++++ return 0 +++ kubectl_bin exec -n monitoring-pmm3-26352 monitoring-0 -- pmm-admin --server-url=https://admin:admin@104.154.140.66/ --server-insecure-tls inventory list nodes --node-type=CONTAINER_NODE ++++ mktemp +++ local LAST_OUT=/tmp/tmp.XDB4y638MF ++++ mktemp +++ local LAST_ERR=/tmp/tmp.j2TCdloBFf +++ local exit_status=0 ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl exec -n monitoring-pmm3-26352 monitoring-0 -- pmm-admin --server-url=https://admin:admin@104.154.140.66/ --server-insecure-tls inventory list nodes --node-type=CONTAINER_NODE +++ exit_status=1 +++ set -e +++ '[' 1 '!=' 0 ']' +++ '[' 1 == 1 ']' +++ sleep 0 +++ for i in $(seq 0 2) +++ set +e +++ kubectl exec -n monitoring-pmm3-26352 monitoring-0 -- pmm-admin --server-url=https://admin:admin@104.154.140.66/ --server-insecure-tls inventory list nodes --node-type=CONTAINER_NODE +++ exit_status=1 +++ set -e +++ '[' 1 '!=' 0 ']' +++ '[' 1 == 1 ']' +++ sleep 0 +++ for i in $(seq 0 2) +++ set +e +++ kubectl exec -n monitoring-pmm3-26352 monitoring-0 -- pmm-admin --server-url=https://admin:admin@104.154.140.66/ --server-insecure-tls inventory list nodes --node-type=CONTAINER_NODE +++ exit_status=1 +++ set -e +++ '[' 1 '!=' 0 ']' +++ '[' 1 == 1 ']' +++ sleep 0 +++ cat /tmp/tmp.XDB4y638MF +++ cat /tmp/tmp.j2TCdloBFf command terminated with exit code 1 +++ rm /tmp/tmp.XDB4y638MF /tmp/tmp.j2TCdloBFf +++ return 1 ++ for node_id in "${nodeList[@]}" ++ nodeList_from_pmm+=($(kubectl_bin exec -n "${namespace}" monitoring-0 -- pmm-admin --server-url=https://admin:admin@$(get_service_ip monitoring-service)/ --server-insecure-tls inventory list nodes --node-type=CONTAINER_NODE | grep $node_id | awk '{print $4}')) +++ awk '{print $4}' +++ grep abb35ca8-de48-45b7-bff6-7392e46fc2d5 ++++ get_service_ip monitoring-service ++++ local service=monitoring-service ++++ kubectl_bin get service/monitoring-service -o 'jsonpath={.spec.type}' ++++ grep -q NotFound +++++ kubectl_bin get service/monitoring-service -o 'jsonpath={.spec.type}' ++++++ mktemp +++++ local LAST_OUT=/tmp/tmp.y60X4G870k ++++++ mktemp +++++ local LAST_ERR=/tmp/tmp.FMWJa9acF7 +++++ local exit_status=0 ++++++ seq 0 2 +++++ for i in $(seq 0 2) +++++ set +e +++++ kubectl get service/monitoring-service -o 'jsonpath={.spec.type}' +++++ exit_status=0 +++++ set -e +++++ '[' 0 '!=' 0 ']' +++++ break +++++ cat /tmp/tmp.y60X4G870k +++++ cat /tmp/tmp.FMWJa9acF7 +++++ rm /tmp/tmp.y60X4G870k /tmp/tmp.FMWJa9acF7 +++++ return 0 ++++ '[' LoadBalancer = ClusterIP ']' ++++ kubectl_bin get service/monitoring-service -o 'jsonpath={.status.loadBalancer.ingress[]}' ++++ grep -E -q 'hostname|ip' ++++ kubectl_bin get service/monitoring-service -o 'jsonpath={.status.loadBalancer.ingress[].ip}' +++++ mktemp ++++ local LAST_OUT=/tmp/tmp.CePFuzXdRi +++++ mktemp ++++ local LAST_ERR=/tmp/tmp.fqBzWwENOM ++++ local exit_status=0 +++++ seq 0 2 ++++ for i in $(seq 0 2) ++++ set +e ++++ kubectl get service/monitoring-service -o 'jsonpath={.status.loadBalancer.ingress[].ip}' ++++ exit_status=0 ++++ set -e ++++ '[' 0 '!=' 0 ']' ++++ break ++++ cat /tmp/tmp.CePFuzXdRi ++++ cat /tmp/tmp.fqBzWwENOM ++++ rm /tmp/tmp.CePFuzXdRi /tmp/tmp.fqBzWwENOM ++++ return 0 ++++ kubectl_bin get service/monitoring-service -o 'jsonpath={.status.loadBalancer.ingress[].hostname}' +++++ mktemp ++++ local LAST_OUT=/tmp/tmp.BodNX84i3d +++++ mktemp ++++ local LAST_ERR=/tmp/tmp.qkseIqLBIW ++++ local exit_status=0 +++++ seq 0 2 ++++ for i in $(seq 0 2) ++++ set +e ++++ kubectl get service/monitoring-service -o 'jsonpath={.status.loadBalancer.ingress[].hostname}' ++++ exit_status=0 ++++ set -e ++++ '[' 0 '!=' 0 ']' ++++ break ++++ cat /tmp/tmp.BodNX84i3d ++++ cat /tmp/tmp.qkseIqLBIW ++++ rm /tmp/tmp.BodNX84i3d /tmp/tmp.qkseIqLBIW ++++ return 0 +++ kubectl_bin exec -n monitoring-pmm3-26352 monitoring-0 -- pmm-admin --server-url=https://admin:admin@104.154.140.66/ --server-insecure-tls inventory list nodes --node-type=CONTAINER_NODE ++++ mktemp +++ local LAST_OUT=/tmp/tmp.4RL4wPqqbK ++++ mktemp +++ local LAST_ERR=/tmp/tmp.0PtiEzIdC1 +++ local exit_status=0 ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl exec -n monitoring-pmm3-26352 monitoring-0 -- pmm-admin --server-url=https://admin:admin@104.154.140.66/ --server-insecure-tls inventory list nodes --node-type=CONTAINER_NODE +++ exit_status=1 +++ set -e +++ '[' 1 '!=' 0 ']' +++ '[' 1 == 1 ']' +++ sleep 0 +++ for i in $(seq 0 2) +++ set +e +++ kubectl exec -n monitoring-pmm3-26352 monitoring-0 -- pmm-admin --server-url=https://admin:admin@104.154.140.66/ --server-insecure-tls inventory list nodes --node-type=CONTAINER_NODE +++ exit_status=1 +++ set -e +++ '[' 1 '!=' 0 ']' +++ '[' 1 == 1 ']' +++ sleep 0 +++ for i in $(seq 0 2) +++ set +e +++ kubectl exec -n monitoring-pmm3-26352 monitoring-0 -- pmm-admin --server-url=https://admin:admin@104.154.140.66/ --server-insecure-tls inventory list nodes --node-type=CONTAINER_NODE +++ exit_status=1 +++ set -e +++ '[' 1 '!=' 0 ']' +++ '[' 1 == 1 ']' +++ sleep 0 +++ cat /tmp/tmp.4RL4wPqqbK +++ cat /tmp/tmp.0PtiEzIdC1 command terminated with exit code 1 +++ rm /tmp/tmp.4RL4wPqqbK /tmp/tmp.0PtiEzIdC1 +++ return 1 ++ for node_id in "${nodeList[@]}" ++ nodeList_from_pmm+=($(kubectl_bin exec -n "${namespace}" monitoring-0 -- pmm-admin --server-url=https://admin:admin@$(get_service_ip monitoring-service)/ --server-insecure-tls inventory list nodes --node-type=CONTAINER_NODE | grep $node_id | awk '{print $4}')) +++ awk '{print $4}' +++ grep e72bd050-21a5-4908-b998-b50ee9de66a6 ++++ get_service_ip monitoring-service ++++ local service=monitoring-service ++++ kubectl_bin get service/monitoring-service -o 'jsonpath={.spec.type}' ++++ grep -q NotFound +++++ kubectl_bin get service/monitoring-service -o 'jsonpath={.spec.type}' ++++++ mktemp +++++ local LAST_OUT=/tmp/tmp.6Wre16O1Ea ++++++ mktemp +++++ local LAST_ERR=/tmp/tmp.dc5JuVWTel +++++ local exit_status=0 ++++++ seq 0 2 +++++ for i in $(seq 0 2) +++++ set +e +++++ kubectl get service/monitoring-service -o 'jsonpath={.spec.type}' +++++ exit_status=0 +++++ set -e +++++ '[' 0 '!=' 0 ']' +++++ break +++++ cat /tmp/tmp.6Wre16O1Ea +++++ cat /tmp/tmp.dc5JuVWTel +++++ rm /tmp/tmp.6Wre16O1Ea /tmp/tmp.dc5JuVWTel +++++ return 0 ++++ '[' LoadBalancer = ClusterIP ']' ++++ grep -E -q 'hostname|ip' ++++ kubectl_bin get service/monitoring-service -o 'jsonpath={.status.loadBalancer.ingress[]}' ++++ kubectl_bin get service/monitoring-service -o 'jsonpath={.status.loadBalancer.ingress[].ip}' +++++ mktemp ++++ local LAST_OUT=/tmp/tmp.vcdjwFcRo6 +++++ mktemp ++++ local LAST_ERR=/tmp/tmp.Z0njiCItGC ++++ local exit_status=0 +++++ seq 0 2 ++++ for i in $(seq 0 2) ++++ set +e ++++ kubectl get service/monitoring-service -o 'jsonpath={.status.loadBalancer.ingress[].ip}' ++++ exit_status=0 ++++ set -e ++++ '[' 0 '!=' 0 ']' ++++ break ++++ cat /tmp/tmp.vcdjwFcRo6 ++++ cat /tmp/tmp.Z0njiCItGC ++++ rm /tmp/tmp.vcdjwFcRo6 /tmp/tmp.Z0njiCItGC ++++ return 0 ++++ kubectl_bin get service/monitoring-service -o 'jsonpath={.status.loadBalancer.ingress[].hostname}' +++++ mktemp ++++ local LAST_OUT=/tmp/tmp.jC1GeqO1jP +++++ mktemp ++++ local LAST_ERR=/tmp/tmp.IUrzxM1hxs ++++ local exit_status=0 +++++ seq 0 2 ++++ for i in $(seq 0 2) ++++ set +e ++++ kubectl get service/monitoring-service -o 'jsonpath={.status.loadBalancer.ingress[].hostname}' ++++ exit_status=0 ++++ set -e ++++ '[' 0 '!=' 0 ']' ++++ break ++++ cat /tmp/tmp.jC1GeqO1jP ++++ cat /tmp/tmp.IUrzxM1hxs ++++ rm /tmp/tmp.jC1GeqO1jP /tmp/tmp.IUrzxM1hxs ++++ return 0 +++ kubectl_bin exec -n monitoring-pmm3-26352 monitoring-0 -- pmm-admin --server-url=https://admin:admin@104.154.140.66/ --server-insecure-tls inventory list nodes --node-type=CONTAINER_NODE ++++ mktemp +++ local LAST_OUT=/tmp/tmp.UtBFuMxN44 ++++ mktemp +++ local LAST_ERR=/tmp/tmp.UU4AubyXmo +++ local exit_status=0 ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl exec -n monitoring-pmm3-26352 monitoring-0 -- pmm-admin --server-url=https://admin:admin@104.154.140.66/ --server-insecure-tls inventory list nodes --node-type=CONTAINER_NODE +++ exit_status=1 +++ set -e +++ '[' 1 '!=' 0 ']' +++ '[' 1 == 1 ']' +++ sleep 0 +++ for i in $(seq 0 2) +++ set +e +++ kubectl exec -n monitoring-pmm3-26352 monitoring-0 -- pmm-admin --server-url=https://admin:admin@104.154.140.66/ --server-insecure-tls inventory list nodes --node-type=CONTAINER_NODE +++ exit_status=1 +++ set -e +++ '[' 1 '!=' 0 ']' +++ '[' 1 == 1 ']' +++ sleep 0 +++ for i in $(seq 0 2) +++ set +e +++ kubectl exec -n monitoring-pmm3-26352 monitoring-0 -- pmm-admin --server-url=https://admin:admin@104.154.140.66/ --server-insecure-tls inventory list nodes --node-type=CONTAINER_NODE +++ exit_status=1 +++ set -e +++ '[' 1 '!=' 0 ']' +++ '[' 1 == 1 ']' +++ sleep 0 +++ cat /tmp/tmp.UtBFuMxN44 +++ cat /tmp/tmp.UU4AubyXmo command terminated with exit code 1 +++ rm /tmp/tmp.UtBFuMxN44 /tmp/tmp.UU4AubyXmo +++ return 1 ++ echo + [[ -n '' ]] + helm uninstall monitoring release "monitoring" uninstalled + destroy monitoring-pmm3-26352 + local namespace=monitoring-pmm3-26352 + local ignore_logs=true + [[ 0 == 1 ]] + desc 'destroy cluster/operator and all other resources' + set +o xtrace ----------------------------------------------------------------------------------- destroy cluster/operator and all other resources ----------------------------------------------------------------------------------- + '[' true == false -o 1 == 1 ']' + tee /tmp/tmp.mnnOTDKNTG/operator.log + sort -u + grep -v 'the object has been modified' + grep -v 'get backup status: Job.batch' + /usr/bin/sed -r 's/"ts":[0-9.]+//; s^limits-[0-9.]+/^^g' + grep -v level=info ++ get_operator_pod ++ local label_prefix=app.kubernetes.io/ +++ kubectl get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator -n pxc-operator +++ grep -c percona-xtradb-cluster-operator ++ local check_label=1 ++ [[ 1 -eq 0 ]] ++ head -1 ++ jq -r '.items[] | select(.metadata.deletionTimestamp == null) | .metadata.name' ++ kubectl_bin get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator --field-selector=status.phase=Running -o json -n pxc-operator +++ mktemp ++ local LAST_OUT=/tmp/tmp.lJysI4ufwW +++ mktemp ++ local LAST_ERR=/tmp/tmp.ezTdJXEVDe ++ local exit_status=0 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator --field-selector=status.phase=Running -o json -n pxc-operator ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.lJysI4ufwW ++ cat /tmp/tmp.ezTdJXEVDe ++ rm /tmp/tmp.lJysI4ufwW /tmp/tmp.ezTdJXEVDe ++ return 0 + kubectl_bin logs -n pxc-operator percona-xtradb-cluster-operator-658fcf4b98-jtpl5 ++ mktemp + local LAST_OUT=/tmp/tmp.N9cHfQJrO7 ++ mktemp + local LAST_ERR=/tmp/tmp.cnCaQXLX22 + local exit_status=0 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl logs -n pxc-operator percona-xtradb-cluster-operator-658fcf4b98-jtpl5 + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.N9cHfQJrO7 + cat /tmp/tmp.cnCaQXLX22 + rm /tmp/tmp.N9cHfQJrO7 /tmp/tmp.cnCaQXLX22 + return 0 /go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:221 /go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312 /go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437 /go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:478 /go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:272 /go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:509 FieldPath: "metadata.name", FieldPath: "metadata.namespace", ... // 2 identical fields ConfigMapKeyRef: nil, FieldRef: &v1.ObjectFieldSelector{ HTTPHeaders: nil, Host: "", Path: "/local/Status", Port: {IntVal: 7777}, Protocol: nil, ResourceFieldRef: nil, }, ContainerPort: 30100, ContainerPort: 30101, ContainerPort: 30102, ContainerPort: 30103, ContainerPort: 30104, ContainerPort: 30105, ContainerPort: 3306, ContainerPort: 33060, ContainerPort: 33062, ContainerPort: 3307, ContainerPort: 3309, ContainerPort: 4444, ContainerPort: 4567, ContainerPort: 4568, ContainerPort: 7777, ContainerPort: 8404, DefaultUser: nil, DefaultUser: nil, Exec: nil, GRPC: nil, HTTPGet: &v1.HTTPGetAction{ HostIP: "", HostPort: 0, Items: nil, Items: nil, LocalObjectReference: {Name: "auto-monitoring-pxc"}, LocalObjectReference: {Name: "monitoring-haproxy"}, LocalObjectReference: {Name: "monitoring-pxc"}, Name: "", Name: "ist", Name: "mysql", Name: "mysql-admin", Name: "mysql-replicas", Name: "mysqlx", Name: "proxy-protocol", Name: "sst", Name: "stats", Name: "write-set", Name: "PMM_AGENT_SETUP_NODE_NAME", Name: "POD_NAME", Name: "POD_NAMESPACE", Optional: &true, Optional: &false, Optional: &true, SecretName: "internal-monitoring", SecretName: "monitoring-mysql-init", SecretName: "monitoring-ssl-internal", SecretName: "monitoring-vault", SecretName: "my-env-var-secrets", SecretName: "some-name-ssl", TCPSocket: nil, Value: "", ValueFrom: &v1.EnvVarSource{ ValueFrom: nil, }, ... // 16 identical fields ... // 18 identical elements ... // 2 identical elements ... // 22 identical fields ... // 25 identical elements ... // 26 identical elements ... // 3 identical elements ... // 3 identical fields ... // 4 identical fields ... // 9 identical fields AWSElasticBlockStore: nil, AzureFile: nil, ConfigMap: &v1.ConfigMapVolumeSource{ FC: nil, FailureThreshold: 3, GitRepo: nil, ISCSI: nil, InitialDelaySeconds: 300, InitialDelaySeconds: 15, InitialDelaySeconds: 300, NFS: nil, ProbeHandler: {Exec: &{Command: {"/var/lib/mysql/liveness-check.sh"}}}, ProbeHandler: v1.ProbeHandler{ Quobyte: nil, Secret: &v1.SecretVolumeSource{ SuccessThreshold: 1, TerminationGracePeriodSeconds: nil, TimeoutSeconds: 5, TimeoutSeconds: 15, TimeoutSeconds: 5, VsphereVolume: nil, { {Name: "CLUSTER_NAME", Value: "foo-custom-cluster-name"}, {Name: "DB_HOST", Value: "localhost"}, {Name: "DB_PORT", Value: "33062"}, {Name: "DB_TYPE", Value: "haproxy"}, {Name: "DB_TYPE", Value: "mysql"}, {Name: "MONITOR_USER", Value: "monitor"}, {Name: "PMM_AGENT_SERVER_ADDRESS", Value: "monitoring-service"}, {Name: "PMM_AGENT_SERVER_USERNAME", Value: "service_token"}, }, ... // 16 identical fields ... // 2 identical fields ... // 3 identical fields ... // 4 identical fields ... // 8 identical fields Args: nil, Args: {"haproxy"}, Args: {"mysqld"}, Env: {{Name: "PXC_SERVICE", Value: "monitoring-pxc"}, {Name: "LIVENESS_CHECK_TIMEOUT", Value: "5"}, {Name: "READINESS_CHECK_TIMEOUT", Value: "1"}}, Env: {{Name: "PXC_SERVICE", Value: "monitoring-pxc-unready"}, {Name: "MONITOR_HOST", Value: "%"}, {Name: "MYSQL_ROOT_PASSWORD", ValueFrom: &{SecretKeyRef: &{LocalObjectReference: {Name: "internal-monitoring"}, Key: "root"}}}, {Name: "XTRABACKUP_PASSWORD", ValueFrom: &{SecretKeyRef: &{LocalObjectReference: {Name: "internal-monitoring"}, Key: "xtrabackup"}}}, ...}, Env: []v1.EnvVar{ EnvFrom: {{SecretRef: &{LocalObjectReference: {Name: "my-env-var-secrets"}, Optional: &true}}}, ImagePullPolicy: "Always", Lifecycle: &{PreStop: &{Exec: &{Command: {"bash", "-c", "pmm-admin unregister --force"}}}}, Lifecycle: nil, LivenessProbe: &v1.Probe{ Name: "auto-config", Name: "config", Name: "haproxy-custom", Name: "my-env-var-secrets", Name: "mysql-init-file", Name: "mysql-users-secret-file", Name: "ssl", Name: "ssl-internal", Name: "vault-keyring-secret", Ports: []v1.ContainerPort{ ReadinessProbe: &{ProbeHandler: {Exec: &{Command: {"/var/lib/mysql/readiness-check.sh"}}}, InitialDelaySeconds: 15, TimeoutSeconds: 15, PeriodSeconds: 30, ...}, ReadinessProbe: &v1.Probe{ ResizePolicy: nil, Resources: {Limits: {s"cpu": {i: {...}, Format: "DecimalSI"}, s"memory": {i: {...}, Format: "DecimalSI"}}, Requests: {s"cpu": {i: {...}, s: "308m", Format: "DecimalSI"}, s"memory": {i: {...}, s: "508M", Format: "DecimalSI"}}}, SecurityContext: nil, StartupProbe: nil, VolumeDevices: nil, VolumeMounts: {{Name: "bin", MountPath: "/var/lib/mysql"}}, VolumeMounts: {{Name: "datadir", MountPath: "/var/lib/mysql"}, {Name: "config", MountPath: "/etc/percona-xtradb-cluster.conf.d"}, {Name: "tmp", MountPath: "/tmp"}, {Name: "ssl", MountPath: "/etc/mysql/ssl"}, ...}, VolumeMounts: {{Name: "datadir", MountPath: "/var/lib/mysql"}}, VolumeSource: v1.VolumeSource{ WorkingDir: "", }, "kubectl.kubernetes.io/default-container": "haproxy", "kubectl.kubernetes.io/default-container": "pxc", "percona.com/configuration-hash": "d41d8cd98f00b204e9800998ecf8427e", "percona.com/env-secret-config-hash": "5bc403a8d44324c1bc704f02e74876d0", "percona.com/ssl-hash": "c30d37b3878cd26f7b7280286a238802", "percona.com/ssl-internal-hash": "6ea22ae3995e84dbd34f05beba22b99f", ... // 2 identical entries ... // 3 identical fields ... // 6 identical fields AccessModes: nil, Capacity: nil, DataSource: nil, DataSourceRef: nil, StorageClassName: nil, VolumeAttributesClassName: nil, VolumeName: "", { {Name: "bin", VolumeSource: {EmptyDir: &{}}}, {Name: "haproxy-auto", VolumeSource: {EmptyDir: &{}}}, {Name: "tmp", VolumeSource: {EmptyDir: &{}}}, }, ... // 2 identical fields ... // 6 identical fields ... // 7 identical fields ... // 9 identical fields ActiveDeadlineSeconds: nil, Affinity: nil, Affinity: nil, Annotations: map[string]string{ AutomountServiceAccountToken: nil, Containers: []v1.Container{ DeletionGracePeriodSeconds: nil, EphemeralContainers: nil, Finalizers: nil, HostAliases: nil, HostAliases: nil, HostIPC: false, Hostname: "", ImagePullSecrets: nil, InitContainers: []v1.Container{ Labels: {"app.kubernetes.io/component": "haproxy", "app.kubernetes.io/instance": "monitoring", "app.kubernetes.io/managed-by": "percona-xtradb-cluster-operator", "app.kubernetes.io/name": "percona-xtradb-cluster", ...}, Labels: {"app.kubernetes.io/component": "pxc", "app.kubernetes.io/instance": "monitoring", "app.kubernetes.io/managed-by": "percona-xtradb-cluster-operator", "app.kubernetes.io/name": "percona-xtradb-cluster", ...}, ManagedFields: nil, NodeName: "", NodeSelector: nil, OS: nil, ObjectMeta: {Name: "datadir", Labels: {"app.kubernetes.io/component": "pxc", "app.kubernetes.io/instance": "monitoring", "app.kubernetes.io/managed-by": "percona-xtradb-cluster-operator", "app.kubernetes.io/name": "percona-xtradb-cluster", ...}}, Overhead: nil, OwnerReferences: nil, PreemptionPolicy: nil, ServiceAccountName: "default", SetHostnameAsFQDN: nil, ShareProcessNamespace: nil, Spec: v1.PersistentVolumeClaimSpec{ Status: v1.PersistentVolumeClaimStatus{ Subdomain: "", Subdomain: "", TerminationGracePeriodSeconds: &30, TerminationGracePeriodSeconds: &600, Tolerations: nil, Tolerations: nil, TypeMeta: {}, Volumes: []v1.Volume{ }, ObjectMeta: v1.ObjectMeta{ ObjectMeta: {Labels: {"app.kubernetes.io/component": "haproxy", "app.kubernetes.io/instance": "monitoring", "app.kubernetes.io/managed-by": "percona-xtradb-cluster-operator", "app.kubernetes.io/name": "percona-xtradb-cluster", ...}, Annotations: {"kubectl.kubernetes.io/default-container": "haproxy", "last-applied-secret": "cefd1660654364c2ae04d98afb3937493a384850d1b8dd5cff992bb409b54bda", "percona.com/configuration-hash": "d41d8cd98f00b204e9800998ecf8427e", "percona.com/env-secret-config-hash": "5bc403a8d44324c1bc704f02e74876d0"}}, ObjectMeta: {Labels: {"app.kubernetes.io/component": "pxc", "app.kubernetes.io/instance": "monitoring", "app.kubernetes.io/managed-by": "percona-xtradb-cluster-operator", "app.kubernetes.io/name": "percona-xtradb-cluster", ...}, Annotations: {"kubectl.kubernetes.io/default-container": "pxc", "last-applied-secret": "cefd1660654364c2ae04d98afb3937493a384850d1b8dd5cff992bb409b54bda", "percona.com/configuration-hash": "d41d8cd98f00b204e9800998ecf8427e", "percona.com/env-secret-config-hash": "5bc403a8d44324c1bc704f02e74876d0", ...}}, Spec: v1.PodSpec{ { }, ... // 2 identical fields Conditions: nil, DeletionGracePeriodSeconds: nil, DeletionTimestamp: nil, Finalizers: nil, Labels: nil, MinReadySeconds: 0, Namespace: "monitoring-pmm3-26352", Ordinals: nil, OwnerReferences: {{APIVersion: "pxc.percona.com/v1", Kind: "PerconaXtraDBCluster", Name: "monitoring", UID: "140ae4c3-de35-472b-b85f-4e00c200d7ea", ...}}, Replicas: &2, Replicas: &3, Selector: &{MatchLabels: {"app.kubernetes.io/component": "haproxy", "app.kubernetes.io/instance": "monitoring", "app.kubernetes.io/managed-by": "percona-xtradb-cluster-operator", "app.kubernetes.io/name": "percona-xtradb-cluster", ...}}, Selector: &{MatchLabels: {"app.kubernetes.io/component": "pxc", "app.kubernetes.io/instance": "monitoring", "app.kubernetes.io/managed-by": "percona-xtradb-cluster-operator", "app.kubernetes.io/name": "percona-xtradb-cluster", ...}}, SelfLink: "", ServiceName: "monitoring-haproxy", ServiceName: "monitoring-pxc", Template: v1.PodTemplateSpec{ UpdateStrategy: {Type: "RollingUpdate", RollingUpdate: &{Partition: &0}}, VolumeClaimTemplates: nil, VolumeClaimTemplates: []v1.PersistentVolumeClaim{ }, ObjectMeta: v1.ObjectMeta{ Spec: v1.StatefulSetSpec{ Status: v1.StatefulSetStatus{ TypeMeta: {Kind: "StatefulSet", APIVersion: "apps/v1"}, }, &v1.StatefulSet{ } + APIVersion: "", + Scheme: "", + DefaultMode: nil, + DefaultMode: nil, + Protocol: "", + Value: "$(PMM_PREFIX)$(POD_NAMESPACE)-$(POD_NAME)", + PeriodSeconds: 0, + TerminationMessagePath: "", + TerminationMessagePolicy: "", + "last-applied-secret": "cefd1660654364c2ae04d98afb3937493a384850d1b8dd5cff992bb409b54bda", + "percona.com/env-secret-config-hash": "5bc403a8d44324c1bc704f02e74876d0", + Phase: "", + VolumeMode: nil, + DNSPolicy: "", + DeprecatedServiceAccount: "", + RestartPolicy: "", + SchedulerName: "", + SchedulerName: "", + SecurityContext: nil, + TopologySpreadConstraints: []v1.TopologySpreadConstraint{}, + "percona.com/last-config-hash": "eyJyZXBsaWNhcyI6MCwic2VsZWN0b3IiOnsibWF0Y2hMYWJlbHMiOnsiYXBwLmt1YmVybmV0ZXMuaW8vY29tcG9uZW50IjoiaGFwcm94eSIsImFwcC5rdWJlcm5ldGVzLmlvL2luc3RhbmNlIjoibW9uaXRvcmluZyIsImFwcC5rdWJlcm5ldGVzLmlvL21hbmFnZWQtYnkiOiJwZXJjb25hLXh0cmFkYi1jbHVzdGVyLW9wZXJhdG9yIiwiYXBw"..., + "percona.com/last-config-hash": "eyJyZXBsaWNhcyI6MCwic2VsZWN0b3IiOnsibWF0Y2hMYWJlbHMiOnsiYXBwLmt1YmVybmV0ZXMuaW8vY29tcG9uZW50IjoicHhjIiwiYXBwLmt1YmVybmV0ZXMuaW8vaW5zdGFuY2UiOiJtb25pdG9yaW5nIiwiYXBwLmt1YmVybmV0ZXMuaW8vbWFuYWdlZC1ieSI6InBlcmNvbmEteHRyYWRiLWNsdXN0ZXItb3BlcmF0b3IiLCJhcHAua3Vi"..., + "percona.com/last-config-hash": "eyJyZXBsaWNhcyI6Miwic2VsZWN0b3IiOnsibWF0Y2hMYWJlbHMiOnsiYXBwLmt1YmVybmV0ZXMuaW8vY29tcG9uZW50IjoiaGFwcm94eSIsImFwcC5rdWJlcm5ldGVzLmlvL2luc3RhbmNlIjoibW9uaXRvcmluZyIsImFwcC5rdWJlcm5ldGVzLmlvL21hbmFnZWQtYnkiOiJwZXJjb25hLXh0cmFkYi1jbHVzdGVyLW9wZXJhdG9yIiwiYXBwLmt1YmVybmV0ZXMuaW8vbmFtZSI6InBlcmNvbmEteHRyYWRiLWNsdXN0ZXIiLCJhcHAua3ViZXJuZXRlcy5pby9wYXJ0LW9mIjoicGVyY29uYS14dHJhZGItY2x1c3RlciJ9fSwidGVtcGxhdGUiOnsibWV0YWRhdGEiOnsibGFiZWxzIjp7ImFwcC5rdWJlcm5ldGVzLmlvL2NvbXBvbmVudCI6ImhhcHJveHkiLCJhcHAua3ViZXJuZXRlcy5pby9pbnN0YW5jZSI6Im1vbml0b3JpbmciLCJhcHAua3ViZXJuZXRlcy5pby9tYW5hZ2VkLWJ5IjoicGVyY29uYS14dHJhZGItY2x1c3Rlci1vcGVyYXRvciIsImFwcC5rdWJlcm5ldGVzLmlvL25hbWUiOiJwZXJjb25hLXh0cmFkYi1jbHVzdGVyIiwiYXBwLmt1YmVybmV0ZXMuaW8vcGFydC1vZiI6InBlcmNvbmEteHRyYWRiLWNsdXN0ZXIifSwiYW5ub3RhdGlvbnMiOnsia3ViZWN0bC5rdWJlcm5ldGVzLmlvL2RlZmF1bHQtY29udGFpbmVyIjoiaGFwcm94eSIsImxhc3QtYXBwbGllZC1zZWNyZXQiOiJjZWZkMTY2MDY1NDM2NGMyYWUwNGQ5OGFmYjM5Mzc0OTNhMzg0ODUwZDFiOGRkNWNmZjk5MmJiNDA5YjU0YmRhIiwicGVyY29uYS5jb20vY29uZmlndXJhdGlvbi1oYXNoIjoi"..., + "percona.com/last-config-hash": "eyJyZXBsaWNhcyI6Miwic2VsZWN0b3IiOnsibWF0Y2hMYWJlbHMiOnsiYXBwLmt1YmVybmV0ZXMuaW8vY29tcG9uZW50IjoiaGFwcm94eSIsImFwcC5rdWJlcm5ldGVzLmlvL2luc3RhbmNlIjoibW9uaXRvcmluZyIsImFwcC5rdWJlcm5ldGVzLmlvL21hbmFnZWQtYnkiOiJwZXJjb25hLXh0cmFkYi1jbHVzdGVyLW9wZXJhdG9yIiwiYXBwLmt1YmVybmV0ZXMuaW8vbmFtZSI6InBlcmNvbmEteHRyYWRiLWNsdXN0ZXIiLCJhcHAua3ViZXJuZXRlcy5pby9wYXJ0LW9mIjoicGVyY29uYS14dHJhZGItY2x1c3RlciJ9fSwidGVtcGxhdGUiOnsibWV0YWRhdGEiOnsibGFiZWxzIjp7ImFwcC5rdWJlcm5ldGVzLmlvL2NvbXBvbmVudCI6ImhhcHJveHkiLCJhcHAua3ViZXJuZXRlcy5pby9pbnN0YW5jZSI6Im1vbml0b3JpbmciLCJhcHAua3ViZXJuZXRlcy5pby9tYW5hZ2VkLWJ5IjoicGVyY29uYS14dHJhZGItY2x1c3Rlci1vcGVyYXRvciIsImFwcC5rdWJlcm5ldGVzLmlvL25hbWUiOiJwZXJjb25hLXh0cmFkYi1jbHVzdGVyIiwiYXBwLmt1YmVybmV0ZXMuaW8vcGFydC1vZiI6InBlcmNvbmEteHRyYWRiLWNsdXN0ZXIifSwiYW5ub3RhdGlvbnMiOnsia3ViZWN0bC5rdWJlcm5ldGVzLmlvL2RlZmF1bHQtY29udGFpbmVyIjoiaGFwcm94eSIsInBlcmNvbmEuY29tL2NvbmZpZ3VyYXRpb24taGFzaCI6ImQ0MWQ4Y2Q5OGYwMGIyMDRlOTgwMDk5OGVjZjg0MjdlIiwicGVyY29uYS5jb20vZW52LXNlY3JldC1jb25maWctaGFzaCI6IjViYzQwM2E4ZDQ0MzI0YzFi"..., + "percona.com/last-config-hash": "eyJyZXBsaWNhcyI6Mywic2VsZWN0b3IiOnsibWF0Y2hMYWJlbHMiOnsiYXBwLmt1YmVybmV0ZXMuaW8vY29tcG9uZW50IjoicHhjIiwiYXBwLmt1YmVybmV0ZXMuaW8vaW5zdGFuY2UiOiJtb25pdG9yaW5nIiwiYXBwLmt1YmVybmV0ZXMuaW8vbWFuYWdlZC1ieSI6InBlcmNvbmEteHRyYWRiLWNsdXN0ZXItb3BlcmF0b3IiLCJhcHAua3ViZXJuZXRlcy5pby9uYW1lIjoicGVyY29uYS14dHJhZGItY2x1c3RlciIsImFwcC5rdWJlcm5ldGVzLmlvL3BhcnQtb2YiOiJwZXJjb25hLXh0cmFkYi1jbHVzdGVyIn19LCJ0ZW1wbGF0ZSI6eyJtZXRhZGF0YSI6eyJsYWJlbHMiOnsiYXBwLmt1YmVybmV0ZXMuaW8vY29tcG9uZW50IjoicHhjIiwiYXBwLmt1YmVybmV0ZXMuaW8vaW5zdGFuY2UiOiJtb25pdG9yaW5nIiwiYXBwLmt1YmVybmV0ZXMuaW8vbWFuYWdlZC1ieSI6InBlcmNvbmEteHRyYWRiLWNsdXN0ZXItb3BlcmF0b3IiLCJhcHAua3ViZXJuZXRlcy5pby9uYW1lIjoicGVyY29uYS14dHJhZGItY2x1c3RlciIsImFwcC5rdWJlcm5ldGVzLmlvL3BhcnQtb2YiOiJwZXJjb25hLXh0cmFkYi1jbHVzdGVyIn0sImFubm90YXRpb25zIjp7Imt1YmVjdGwua3ViZXJuZXRlcy5pby9kZWZhdWx0LWNvbnRhaW5lciI6InB4YyIsImxhc3QtYXBwbGllZC1zZWNyZXQiOiJjZWZkMTY2MDY1NDM2NGMyYWUwNGQ5OGFmYjM5Mzc0OTNhMzg0ODUwZDFiOGRkNWNmZjk5MmJiNDA5YjU0YmRhIiwicGVyY29uYS5jb20vY29uZmlndXJhdGlvbi1oYXNoIjoiZDQxZDhjZDk4ZjAw"..., + "percona.com/last-config-hash": "eyJyZXBsaWNhcyI6Mywic2VsZWN0b3IiOnsibWF0Y2hMYWJlbHMiOnsiYXBwLmt1YmVybmV0ZXMuaW8vY29tcG9uZW50IjoicHhjIiwiYXBwLmt1YmVybmV0ZXMuaW8vaW5zdGFuY2UiOiJtb25pdG9yaW5nIiwiYXBwLmt1YmVybmV0ZXMuaW8vbWFuYWdlZC1ieSI6InBlcmNvbmEteHRyYWRiLWNsdXN0ZXItb3BlcmF0b3IiLCJhcHAua3ViZXJuZXRlcy5pby9uYW1lIjoicGVyY29uYS14dHJhZGItY2x1c3RlciIsImFwcC5rdWJlcm5ldGVzLmlvL3BhcnQtb2YiOiJwZXJjb25hLXh0cmFkYi1jbHVzdGVyIn19LCJ0ZW1wbGF0ZSI6eyJtZXRhZGF0YSI6eyJsYWJlbHMiOnsiYXBwLmt1YmVybmV0ZXMuaW8vY29tcG9uZW50IjoicHhjIiwiYXBwLmt1YmVybmV0ZXMuaW8vaW5zdGFuY2UiOiJtb25pdG9yaW5nIiwiYXBwLmt1YmVybmV0ZXMuaW8vbWFuYWdlZC1ieSI6InBlcmNvbmEteHRyYWRiLWNsdXN0ZXItb3BlcmF0b3IiLCJhcHAua3ViZXJuZXRlcy5pby9uYW1lIjoicGVyY29uYS14dHJhZGItY2x1c3RlciIsImFwcC5rdWJlcm5ldGVzLmlvL3BhcnQtb2YiOiJwZXJjb25hLXh0cmFkYi1jbHVzdGVyIn0sImFubm90YXRpb25zIjp7Imt1YmVjdGwua3ViZXJuZXRlcy5pby9kZWZhdWx0LWNvbnRhaW5lciI6InB4YyIsInBlcmNvbmEuY29tL2NvbmZpZ3VyYXRpb24taGFzaCI6ImQ0MWQ4Y2Q5OGYwMGIyMDRlOTgwMDk5OGVjZjg0MjdlIiwicGVyY29uYS5jb20vZW52LXNlY3JldC1jb25maWctaGFzaCI6IjViYzQwM2E4ZDQ0MzI0YzFiYzcwNGYwMmU3NDg3"..., + Annotations: map[string]string{ + AvailableReplicas: 0, + CollisionCount: nil, + CreationTimestamp: v1.Time{}, + CurrentReplicas: 0, + CurrentRevision: "", + Generation: 0, + ManagedFields: nil, + ObservedGeneration: 0, + PersistentVolumeClaimRetentionPolicy: nil, + PodManagementPolicy: "", + ReadyReplicas: 0, + Replicas: 0, + Replicas: &0, + ResourceVersion: "", + RevisionHistoryLimit: nil, + UID: "", + UpdateRevision: "", + UpdatedReplicas: 0, + }, - APIVersion: "v1", - Scheme: "HTTP", - DefaultMode: &420, - DefaultMode: &420, - Protocol: "TCP", - Value: "$(POD_NAMESPACE)-$(POD_NAME)", - PeriodSeconds: 10, - TerminationMessagePath: "/dev/termination-log", - TerminationMessagePolicy: "File", - Phase: "Pending", - VolumeMode: &"Filesystem", - APIVersion: "apps/v1", - APIVersion: "apps/v1", - DNSPolicy: "ClusterFirst", - DeprecatedServiceAccount: "default", - FieldsType: "FieldsV1", - FieldsType: "FieldsV1", - FieldsV1: s`{"f:status":{"f:availableReplicas":{},"f:collisionCount":{},"f:c`..., - FieldsV1: s`{"f:metadata":{"f:annotations":{".":{},"f:percona.com/last-confi`..., - Manager: "kube-controller-manager", - Manager: "percona-xtradb-cluster-operator", - Operation: "Update", - Operation: "Update", - RestartPolicy: "Always", - SchedulerName: "default-scheduler", - SchedulerName: "default-scheduler", - SecurityContext: s"&PodSecurityContext{SELinuxOptions:nil,RunAsUser:nil,RunAsNonRoot:nil,SupplementalGroups:[],FSGroup:nil,RunAsGroup:nil,Sysctls:[]Sysctl{},WindowsOptions:nil,FSGroupChangePolicy:nil,SeccompProfile:nil,AppArmorProfile:nil,SupplementalGroupsPolicy:nil,SELinux"..., - Subresource: "status", - Time: s"2026-10-05 14:34:39 +0000 UTC", - Time: s"2026-10-05 14:36:54 +0000 UTC", - Time: s"2026-10-05 14:43:31 +0000 UTC", - Time: s"2026-10-05 14:45:15 +0000 UTC", - Time: s"2026-10-05 14:46:12 +0000 UTC", - Time: s"2026-10-05 14:47:58 +0000 UTC", - Time: s"2026-10-05 14:33:24 +0000 UTC", - Time: s"2026-10-05 14:42:38 +0000 UTC", - Time: s"2026-10-05 14:45:18 +0000 UTC", - Time: s"2026-10-05 14:45:19 +0000 UTC", - TopologySpreadConstraints: nil, - "percona.com/last-config-hash": "eyJyZXBsaWNhcyI6Miwic2VsZWN0b3IiOnsibWF0Y2hMYWJlbHMiOnsiYXBwLmt1YmVybmV0ZXMuaW8vY29tcG9uZW50IjoiaGFwcm94eSIsImFwcC5rdWJlcm5ldGVzLmlvL2luc3RhbmNlIjoibW9uaXRvcmluZyIsImFwcC5rdWJlcm5ldGVzLmlvL21hbmFnZWQtYnkiOiJwZXJjb25hLXh0cmFkYi1jbHVzdGVyLW9wZXJhdG9yIiwiYXBw"..., - "percona.com/last-config-hash": "eyJyZXBsaWNhcyI6Miwic2VsZWN0b3IiOnsibWF0Y2hMYWJlbHMiOnsiYXBwLmt1YmVybmV0ZXMuaW8vY29tcG9uZW50IjoiaGFwcm94eSIsImFwcC5rdWJlcm5ldGVzLmlvL2luc3RhbmNlIjoibW9uaXRvcmluZyIsImFwcC5rdWJlcm5ldGVzLmlvL21hbmFnZWQtYnkiOiJwZXJjb25hLXh0cmFkYi1jbHVzdGVyLW9wZXJhdG9yIiwiYXBwLmt1YmVybmV0ZXMuaW8vbmFtZSI6InBlcmNvbmEteHRyYWRiLWNsdXN0ZXIiLCJhcHAua3ViZXJuZXRlcy5pby9wYXJ0LW9mIjoicGVyY29uYS14dHJhZGItY2x1c3RlciJ9fSwidGVtcGxhdGUiOnsibWV0YWRhdGEiOnsibGFiZWxzIjp7ImFwcC5rdWJlcm5ldGVzLmlvL2NvbXBvbmVudCI6ImhhcHJveHkiLCJhcHAua3ViZXJuZXRlcy5pby9pbnN0YW5jZSI6Im1vbml0b3JpbmciLCJhcHAua3ViZXJuZXRlcy5pby9tYW5hZ2VkLWJ5IjoicGVyY29uYS14dHJhZGItY2x1c3Rlci1vcGVyYXRvciIsImFwcC5rdWJlcm5ldGVzLmlvL25hbWUiOiJwZXJjb25hLXh0cmFkYi1jbHVzdGVyIiwiYXBwLmt1YmVybmV0ZXMuaW8vcGFydC1vZiI6InBlcmNvbmEteHRyYWRiLWNsdXN0ZXIifSwiYW5ub3RhdGlvbnMiOnsia3ViZWN0bC5rdWJlcm5ldGVzLmlvL2RlZmF1bHQtY29udGFpbmVyIjoiaGFwcm94eSIsInBlcmNvbmEuY29tL2NvbmZpZ3VyYXRpb24taGFzaCI6ImQ0MWQ4Y2Q5OGYwMGIyMDRlOTgwMDk5OGVjZjg0MjdlIiwicGVyY29uYS5jb20vZW52LXNlY3JldC1jb25maWctaGFzaCI6IjViYzQwM2E4ZDQ0MzI0YzFi"..., - "percona.com/last-config-hash": "eyJyZXBsaWNhcyI6Miwic2VsZWN0b3IiOnsibWF0Y2hMYWJlbHMiOnsiYXBwLmt1YmVybmV0ZXMuaW8vY29tcG9uZW50IjoiaGFwcm94eSIsImFwcC5rdWJlcm5ldGVzLmlvL2luc3RhbmNlIjoibW9uaXRvcmluZyIsImFwcC5rdWJlcm5ldGVzLmlvL21hbmFnZWQtYnkiOiJwZXJjb25hLXh0cmFkYi1jbHVzdGVyLW9wZXJhdG9yIiwiYXBwLmt1YmVybmV0ZXMuaW8vbmFtZSI6InBlcmNvbmEteHRyYWRiLWNsdXN0ZXIiLCJhcHAua3ViZXJuZXRlcy5pby9wYXJ0LW9mIjoicGVyY29uYS14dHJhZGItY2x1c3RlciJ9fSwidGVtcGxhdGUiOnsibWV0YWRhdGEiOnsibGFiZWxzIjp7ImFwcC5rdWJlcm5ldGVzLmlvL2NvbXBvbmVudCI6ImhhcHJveHkiLCJhcHAua3ViZXJuZXRlcy5pby9pbnN0YW5jZSI6Im1vbml0b3JpbmciLCJhcHAua3ViZXJuZXRlcy5pby9tYW5hZ2VkLWJ5IjoicGVyY29uYS14dHJhZGItY2x1c3Rlci1vcGVyYXRvciIsImFwcC5rdWJlcm5ldGVzLmlvL25hbWUiOiJwZXJjb25hLXh0cmFkYi1jbHVzdGVyIiwiYXBwLmt1YmVybmV0ZXMuaW8vcGFydC1vZiI6InBlcmNvbmEteHRyYWRiLWNsdXN0ZXIifSwiYW5ub3RhdGlvbnMiOnsia3ViZWN0bC5rdWJlcm5ldGVzLmlvL2RlZmF1bHQtY29udGFpbmVyIjoiaGFwcm94eSIsInBlcmNvbmEuY29tL2NvbmZpZ3VyYXRpb24taGFzaCI6ImQ0MWQ4Y2Q5OGYwMGIyMDRlOTgwMDk5OGVjZjg0MjdlIn19LCJzcGVjIjp7InZvbHVtZXMiOlt7Im5hbWUiOiJoYXByb3h5LWN1c3RvbSIsImNvbmZpZ01h"..., - "percona.com/last-config-hash": "eyJyZXBsaWNhcyI6Mywic2VsZWN0b3IiOnsibWF0Y2hMYWJlbHMiOnsiYXBwLmt1YmVybmV0ZXMuaW8vY29tcG9uZW50IjoicHhjIiwiYXBwLmt1YmVybmV0ZXMuaW8vaW5zdGFuY2UiOiJtb25pdG9yaW5nIiwiYXBwLmt1YmVybmV0ZXMuaW8vbWFuYWdlZC1ieSI6InBlcmNvbmEteHRyYWRiLWNsdXN0ZXItb3BlcmF0b3IiLCJhcHAua3Vi"..., - "percona.com/last-config-hash": "eyJyZXBsaWNhcyI6Mywic2VsZWN0b3IiOnsibWF0Y2hMYWJlbHMiOnsiYXBwLmt1YmVybmV0ZXMuaW8vY29tcG9uZW50IjoicHhjIiwiYXBwLmt1YmVybmV0ZXMuaW8vaW5zdGFuY2UiOiJtb25pdG9yaW5nIiwiYXBwLmt1YmVybmV0ZXMuaW8vbWFuYWdlZC1ieSI6InBlcmNvbmEteHRyYWRiLWNsdXN0ZXItb3BlcmF0b3IiLCJhcHAua3ViZXJuZXRlcy5pby9uYW1lIjoicGVyY29uYS14dHJhZGItY2x1c3RlciIsImFwcC5rdWJlcm5ldGVzLmlvL3BhcnQtb2YiOiJwZXJjb25hLXh0cmFkYi1jbHVzdGVyIn19LCJ0ZW1wbGF0ZSI6eyJtZXRhZGF0YSI6eyJsYWJlbHMiOnsiYXBwLmt1YmVybmV0ZXMuaW8vY29tcG9uZW50IjoicHhjIiwiYXBwLmt1YmVybmV0ZXMuaW8vaW5zdGFuY2UiOiJtb25pdG9yaW5nIiwiYXBwLmt1YmVybmV0ZXMuaW8vbWFuYWdlZC1ieSI6InBlcmNvbmEteHRyYWRiLWNsdXN0ZXItb3BlcmF0b3IiLCJhcHAua3ViZXJuZXRlcy5pby9uYW1lIjoicGVyY29uYS14dHJhZGItY2x1c3RlciIsImFwcC5rdWJlcm5ldGVzLmlvL3BhcnQtb2YiOiJwZXJjb25hLXh0cmFkYi1jbHVzdGVyIn0sImFubm90YXRpb25zIjp7Imt1YmVjdGwua3ViZXJuZXRlcy5pby9kZWZhdWx0LWNvbnRhaW5lciI6InB4YyIsInBlcmNvbmEuY29tL2NvbmZpZ3VyYXRpb24taGFzaCI6ImQ0MWQ4Y2Q5OGYwMGIyMDRlOTgwMDk5OGVjZjg0MjdlIiwicGVyY29uYS5jb20vZW52LXNlY3JldC1jb25maWctaGFzaCI6IjViYzQwM2E4ZDQ0MzI0YzFiYzcwNGYwMmU3NDg3"..., - "percona.com/last-config-hash": "eyJyZXBsaWNhcyI6Mywic2VsZWN0b3IiOnsibWF0Y2hMYWJlbHMiOnsiYXBwLmt1YmVybmV0ZXMuaW8vY29tcG9uZW50IjoicHhjIiwiYXBwLmt1YmVybmV0ZXMuaW8vaW5zdGFuY2UiOiJtb25pdG9yaW5nIiwiYXBwLmt1YmVybmV0ZXMuaW8vbWFuYWdlZC1ieSI6InBlcmNvbmEteHRyYWRiLWNsdXN0ZXItb3BlcmF0b3IiLCJhcHAua3ViZXJuZXRlcy5pby9uYW1lIjoicGVyY29uYS14dHJhZGItY2x1c3RlciIsImFwcC5rdWJlcm5ldGVzLmlvL3BhcnQtb2YiOiJwZXJjb25hLXh0cmFkYi1jbHVzdGVyIn19LCJ0ZW1wbGF0ZSI6eyJtZXRhZGF0YSI6eyJsYWJlbHMiOnsiYXBwLmt1YmVybmV0ZXMuaW8vY29tcG9uZW50IjoicHhjIiwiYXBwLmt1YmVybmV0ZXMuaW8vaW5zdGFuY2UiOiJtb25pdG9yaW5nIiwiYXBwLmt1YmVybmV0ZXMuaW8vbWFuYWdlZC1ieSI6InBlcmNvbmEteHRyYWRiLWNsdXN0ZXItb3BlcmF0b3IiLCJhcHAua3ViZXJuZXRlcy5pby9uYW1lIjoicGVyY29uYS14dHJhZGItY2x1c3RlciIsImFwcC5rdWJlcm5ldGVzLmlvL3BhcnQtb2YiOiJwZXJjb25hLXh0cmFkYi1jbHVzdGVyIn0sImFubm90YXRpb25zIjp7Imt1YmVjdGwua3ViZXJuZXRlcy5pby9kZWZhdWx0LWNvbnRhaW5lciI6InB4YyIsInBlcmNvbmEuY29tL2NvbmZpZ3VyYXRpb24taGFzaCI6ImQ0MWQ4Y2Q5OGYwMGIyMDRlOTgwMDk5OGVjZjg0MjdlIiwicGVyY29uYS5jb20vc3NsLWhhc2giOiJjMzBkMzdiMzg3OGNkMjZmN2I3MjgwMjg2YTIzODgwMiIsInBlcmNvbmEu"..., - { - }, - Annotations: map[string]string{ - AvailableReplicas: 2, - AvailableReplicas: 3, - CollisionCount: &0, - CreationTimestamp: v1.Time{Time: s"2026-10-05 14:33:24 +0000 UTC"}, - CurrentReplicas: 2, - CurrentReplicas: 3, - CurrentRevision: "monitoring-haproxy-6989549656", - CurrentRevision: "monitoring-haproxy-7794989cb4", - CurrentRevision: "monitoring-haproxy-98648cbcd", - CurrentRevision: "monitoring-pxc-5688f6fb48", - CurrentRevision: "monitoring-pxc-65df655f8", - CurrentRevision: "monitoring-pxc-95d6bd6dd", - Generation: 1, - Generation: 2, - Generation: 3, - ManagedFields: []v1.ManagedFieldsEntry{ - ObservedGeneration: 1, - ObservedGeneration: 2, - ObservedGeneration: 3, - PersistentVolumeClaimRetentionPolicy: s"&StatefulSetPersistentVolumeClaimRetentionPolicy{WhenDeleted:Retain,WhenScaled:Retain,}", - PodManagementPolicy: "OrderedReady", - ReadyReplicas: 2, - ReadyReplicas: 3, - Replicas: 2, - Replicas: 3, - Replicas: &2, - Replicas: &3, - ResourceVersion: "1791210879786815015", - ResourceVersion: "1791211014046063019", - ResourceVersion: "1791211411530223015", - ResourceVersion: "1791211515982143019", - ResourceVersion: "1791211572315199015", - ResourceVersion: "1791211678499311019", - RevisionHistoryLimit: &10, - UID: "03546342-765b-4af0-8d69-7b83f42ce05f", - UID: "9009b827-1669-4594-886c-982775c40631", - UpdateRevision: "monitoring-haproxy-6989549656", - UpdateRevision: "monitoring-haproxy-7794989cb4", - UpdateRevision: "monitoring-haproxy-98648cbcd", - UpdateRevision: "monitoring-pxc-5688f6fb48", - UpdateRevision: "monitoring-pxc-65df655f8", - UpdateRevision: "monitoring-pxc-95d6bd6dd", - UpdatedReplicas: 2, - UpdatedReplicas: 3, - }, 2026-10-05T14:29:15.446Z INFO setup Feature gates {"PXCO_FEATURE_GATES": "", "enabled": ""} 2026-10-05T14:29:15.446Z INFO setup Manager starting up {"gitCommit": "caf25420ee3d0d26f55c22c9a7661405b27337d1", "gitBranch": "PR-2546-caf25420", "buildTime": "2026-10-05T11:36:48Z", "goVersion": "go1.26.8", "os": "linux", "arch": "amd64"} 2026-10-05T14:29:15.446Z INFO setup Runs on {"platform": "kubernetes", "version": "v1.34.9-gke.1655001"} 2026-10-05T14:29:15.449Z INFO setup Registering Components. 2026-10-05T14:29:15.824Z INFO controller-runtime.certwatcher Starting certificate poll+watcher {"cert": "/tmp/k8s-webhook-server/serving-certs/tls.crt", "key": "/tmp/k8s-webhook-server/serving-certs/tls.key", "interval": "10s"} 2026-10-05T14:29:15.824Z INFO controller-runtime.certwatcher Updated current TLS certificate {"cert": "/tmp/k8s-webhook-server/serving-certs/tls.crt", "key": "/tmp/k8s-webhook-server/serving-certs/tls.key"} 2026-10-05T14:29:15.824Z INFO controller-runtime.metrics Serving metrics server {"bindAddress": ":8080", "secure": false} 2026-10-05T14:29:15.824Z INFO controller-runtime.metrics Starting metrics server 2026-10-05T14:29:15.824Z INFO controller-runtime.webhook Registering webhook {"path": "/validate-percona-xtradbcluster"} 2026-10-05T14:29:15.824Z INFO controller-runtime.webhook Serving webhook server {"host": "", "port": 9443} 2026-10-05T14:29:15.824Z INFO controller-runtime.webhook Starting webhook server 2026-10-05T14:29:15.824Z INFO setup Starting the Cmd. 2026-10-05T14:29:15.824Z INFO starting server {"name": "health probe", "addr": "[::]:8081"} 2026-10-05T14:29:15.925Z INFO Attempting to acquire leader lease... {"lock": "pxc-operator/08db1feb.percona.com"} 2026-10-05T14:29:15.966Z DEBUG events percona-xtradb-cluster-operator-658fcf4b98-jtpl5_677642bb-5835-420b-8bfc-639713b81131 became leader {"type": "Normal", "object": {"kind":"Lease","namespace":"pxc-operator","name":"08db1feb.percona.com","uid":"2d7f503f-c66f-4a7d-85b9-7b008a76d4dd","apiVersion":"coordination.k8s.io/v1","resourceVersion":"1791210555958383009"}, "reason": "LeaderElection"} 2026-10-05T14:29:15.966Z INFO Successfully acquired lease {"lock": "pxc-operator/08db1feb.percona.com"} 2026-10-05T14:29:15.967Z INFO Starting EventSource {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "source": "kind source: *v1.PerconaXtraDBCluster"} 2026-10-05T14:29:15.967Z INFO Starting EventSource {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "source": "kind source: *v1.Secret"} 2026-10-05T14:29:15.967Z INFO Starting EventSource {"controller": "pxcbackup-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBClusterBackup", "source": "kind source: *v1.PerconaXtraDBClusterBackup"} 2026-10-05T14:29:15.967Z INFO Starting EventSource {"controller": "pxcrestore-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBClusterRestore", "source": "kind source: *v1.PerconaXtraDBClusterRestore"} 2026-10-05T14:29:16.068Z INFO Starting Controller {"controller": "pxcbackup-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBClusterBackup"} 2026-10-05T14:29:16.068Z INFO Starting workers {"controller": "pxcbackup-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBClusterBackup", "worker count": 1} 2026-10-05T14:29:16.168Z INFO Starting Controller {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster"} 2026-10-05T14:29:16.168Z INFO Starting Controller {"controller": "pxcrestore-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBClusterRestore"} 2026-10-05T14:29:16.168Z INFO Starting workers {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "worker count": 1} 2026-10-05T14:29:16.168Z INFO Starting workers {"controller": "pxcrestore-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBClusterRestore", "worker count": 1} 2026-10-05T14:33:22.811Z INFO Set CR version {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"monitoring","namespace":"monitoring-pmm3-26352"}, "namespace": "monitoring-pmm3-26352", "name": "monitoring", "reconcileID": "a48f2fb3-3e1e-4512-91af-9f3c33eaff1f", "version": "1.21.0"} 2026-10-05T14:33:24.257Z DEBUG Creating object {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"monitoring","namespace":"monitoring-pmm3-26352"}, "namespace": "monitoring-pmm3-26352", "name": "monitoring", "reconcileID": "a48f2fb3-3e1e-4512-91af-9f3c33eaff1f", "object": "monitoring-pxc", "kind": "&TypeMeta{Kind:StatefulSet,APIVersion:apps/v1,}"} 2026-10-05T14:33:24.312Z DEBUG Creating object {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"monitoring","namespace":"monitoring-pmm3-26352"}, "namespace": "monitoring-pmm3-26352", "name": "monitoring", "reconcileID": "a48f2fb3-3e1e-4512-91af-9f3c33eaff1f", "object": "monitoring-haproxy", "kind": "&TypeMeta{Kind:StatefulSet,APIVersion:apps/v1,}"} 2026-10-05T14:33:24.450Z DEBUG Creating object {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"monitoring","namespace":"monitoring-pmm3-26352"}, "namespace": "monitoring-pmm3-26352", "name": "monitoring", "reconcileID": "a48f2fb3-3e1e-4512-91af-9f3c33eaff1f", "object": "monitoring-pxc", "kind": "&TypeMeta{Kind:Service,APIVersion:v1,}"} 2026-10-05T14:33:24.516Z DEBUG Creating object {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"monitoring","namespace":"monitoring-pmm3-26352"}, "namespace": "monitoring-pmm3-26352", "name": "monitoring", "reconcileID": "a48f2fb3-3e1e-4512-91af-9f3c33eaff1f", "object": "monitoring-pxc-unready", "kind": "&TypeMeta{Kind:Service,APIVersion:v1,}"} 2026-10-05T14:33:24.573Z DEBUG Creating object {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"monitoring","namespace":"monitoring-pmm3-26352"}, "namespace": "monitoring-pmm3-26352", "name": "monitoring", "reconcileID": "a48f2fb3-3e1e-4512-91af-9f3c33eaff1f", "object": "monitoring-haproxy", "kind": "&TypeMeta{Kind:Service,APIVersion:v1,}"} 2026-10-05T14:33:24.699Z DEBUG Creating object {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"monitoring","namespace":"monitoring-pmm3-26352"}, "namespace": "monitoring-pmm3-26352", "name": "monitoring", "reconcileID": "a48f2fb3-3e1e-4512-91af-9f3c33eaff1f", "object": "monitoring-haproxy-replicas", "kind": "&TypeMeta{Kind:Service,APIVersion:v1,}"} 2026-10-05T14:33:25.922Z DEBUG Creating object {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"monitoring","namespace":"monitoring-pmm3-26352"}, "namespace": "monitoring-pmm3-26352", "name": "monitoring", "reconcileID": "bbb10337-e87f-42dd-a121-0dc4e86e8de3", "object": "monitoring-pxc", "kind": "&TypeMeta{Kind:PodDisruptionBudget,APIVersion:policy/v1,}"} 2026-10-05T14:33:25.972Z DEBUG Creating object {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"monitoring","namespace":"monitoring-pmm3-26352"}, "namespace": "monitoring-pmm3-26352", "name": "monitoring", "reconcileID": "bbb10337-e87f-42dd-a121-0dc4e86e8de3", "object": "monitoring-haproxy", "kind": "&TypeMeta{Kind:PodDisruptionBudget,APIVersion:policy/v1,}"} 2026-10-05T14:34:17.590Z INFO Password expiration policy updated {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"monitoring","namespace":"monitoring-pmm3-26352"}, "namespace": "monitoring-pmm3-26352", "name": "monitoring", "reconcileID": "90c8dc7f-df05-48f8-a062-08d72ed8af65", "user": "operator"} 2026-10-05T14:34:17.625Z INFO Password expiration policy updated {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"monitoring","namespace":"monitoring-pmm3-26352"}, "namespace": "monitoring-pmm3-26352", "name": "monitoring", "reconcileID": "90c8dc7f-df05-48f8-a062-08d72ed8af65", "user": "monitor"} 2026-10-05T14:34:17.679Z INFO User monitor: granted privileges {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"monitoring","namespace":"monitoring-pmm3-26352"}, "namespace": "monitoring-pmm3-26352", "name": "monitoring", "reconcileID": "90c8dc7f-df05-48f8-a062-08d72ed8af65"} 2026-10-05T14:34:17.708Z INFO monitor user privileges granted {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"monitoring","namespace":"monitoring-pmm3-26352"}, "namespace": "monitoring-pmm3-26352", "name": "monitoring", "reconcileID": "90c8dc7f-df05-48f8-a062-08d72ed8af65"} 2026-10-05T14:34:17.742Z INFO monitor user audit admin privilege granted {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"monitoring","namespace":"monitoring-pmm3-26352"}, "namespace": "monitoring-pmm3-26352", "name": "monitoring", "reconcileID": "90c8dc7f-df05-48f8-a062-08d72ed8af65"} 2026-10-05T14:34:17.787Z INFO Password expiration policy updated {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"monitoring","namespace":"monitoring-pmm3-26352"}, "namespace": "monitoring-pmm3-26352", "name": "monitoring", "reconcileID": "90c8dc7f-df05-48f8-a062-08d72ed8af65", "user": "xtrabackup"} 2026-10-05T14:34:17.863Z INFO User xtrabackup: granted privileges {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"monitoring","namespace":"monitoring-pmm3-26352"}, "namespace": "monitoring-pmm3-26352", "name": "monitoring", "reconcileID": "90c8dc7f-df05-48f8-a062-08d72ed8af65"} 2026-10-05T14:34:17.891Z INFO Password expiration policy updated {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"monitoring","namespace":"monitoring-pmm3-26352"}, "namespace": "monitoring-pmm3-26352", "name": "monitoring", "reconcileID": "90c8dc7f-df05-48f8-a062-08d72ed8af65", "user": "replication"} 2026-10-05T14:36:59.393Z INFO Password expiration policy updated {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"monitoring","namespace":"monitoring-pmm3-26352"}, "namespace": "monitoring-pmm3-26352", "name": "monitoring", "reconcileID": "2c06f687-a52d-40b8-9ac4-c8d6ecaf5909", "user": "root"} 2026-10-05T14:36:59.443Z INFO update PXC version (fetched from db) {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"monitoring","namespace":"monitoring-pmm3-26352"}, "namespace": "monitoring-pmm3-26352", "name": "monitoring", "reconcileID": "2c06f687-a52d-40b8-9ac4-c8d6ecaf5909", "new version": "8.4.10-10.1"} 2026-10-05T14:42:38.087Z DEBUG Updating object {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"monitoring","namespace":"monitoring-pmm3-26352"}, "namespace": "monitoring-pmm3-26352", "name": "monitoring", "reconcileID": "93b9b484-6030-49f6-b199-5760bc5633fa", "object": "monitoring-pxc", "kind": "&TypeMeta{Kind:StatefulSet,APIVersion:apps/v1,}", "hashChanged": true, "metaChanged": true} 2026-10-05T14:42:38.211Z DEBUG Updating object {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"monitoring","namespace":"monitoring-pmm3-26352"}, "namespace": "monitoring-pmm3-26352", "name": "monitoring", "reconcileID": "93b9b484-6030-49f6-b199-5760bc5633fa", "object": "monitoring-haproxy", "kind": "&TypeMeta{Kind:StatefulSet,APIVersion:apps/v1,}", "hashChanged": true, "metaChanged": true} 2026-10-05T14:42:38.326Z DEBUG Updating object {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"monitoring","namespace":"monitoring-pmm3-26352"}, "namespace": "monitoring-pmm3-26352", "name": "monitoring", "reconcileID": "93b9b484-6030-49f6-b199-5760bc5633fa", "object": "monitoring-haproxy", "kind": "&TypeMeta{Kind:StatefulSet,APIVersion:apps/v1,}", "hashChanged": true, "metaChanged": true} 2026-10-05T14:43:37.255Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"monitoring","namespace":"monitoring-pmm3-26352"}, "namespace": "monitoring-pmm3-26352", "name": "monitoring", "reconcileID": "f24ba05d-3174-4ab4-97da-63fbae8b1801", "err": "failed to ensure cluster readonly status: connect to pod monitoring-pxc-1: dial tcp: lookup monitoring-pxc-1.monitoring-pxc.monitoring-pmm3-26352 on 34.118.224.10:53: no such host"} 2026-10-05T14:43:38.540Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"monitoring","namespace":"monitoring-pmm3-26352"}, "namespace": "monitoring-pmm3-26352", "name": "monitoring", "reconcileID": "48e803d4-bf71-4016-afd8-829c47ce45c2", "err": "failed to ensure cluster readonly status: connect to pod monitoring-pxc-1: dial tcp: lookup monitoring-pxc-1.monitoring-pxc.monitoring-pmm3-26352 on 34.118.224.10:53: no such host"} 2026-10-05T14:43:44.285Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"monitoring","namespace":"monitoring-pmm3-26352"}, "namespace": "monitoring-pmm3-26352", "name": "monitoring", "reconcileID": "b3a56a45-74d6-4e3f-bbee-a2d680544026", "err": "failed to ensure cluster readonly status: connect to pod monitoring-pxc-1: dial tcp: lookup monitoring-pxc-1.monitoring-pxc.monitoring-pmm3-26352 on 34.118.224.10:53: no such host"} 2026-10-05T14:45:18.826Z INFO Password changed, updating user {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"monitoring","namespace":"monitoring-pmm3-26352"}, "namespace": "monitoring-pmm3-26352", "name": "monitoring", "reconcileID": "f98cd611-1a46-4155-8c8d-db6aa9c6f036", "user": "pmmservertoken"} 2026-10-05T14:45:18.852Z INFO HAProxy pods will be restarted {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"monitoring","namespace":"monitoring-pmm3-26352"}, "namespace": "monitoring-pmm3-26352", "name": "monitoring", "reconcileID": "f98cd611-1a46-4155-8c8d-db6aa9c6f036", "last-applied-secret": "cefd1660654364c2ae04d98afb3937493a384850d1b8dd5cff992bb409b54bda"} 2026-10-05T14:45:18.852Z INFO Internal secrets updated {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"monitoring","namespace":"monitoring-pmm3-26352"}, "namespace": "monitoring-pmm3-26352", "name": "monitoring", "reconcileID": "f98cd611-1a46-4155-8c8d-db6aa9c6f036", "user": "pmmservertoken"} 2026-10-05T14:45:18.852Z INFO PXC pods will be restarted {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"monitoring","namespace":"monitoring-pmm3-26352"}, "namespace": "monitoring-pmm3-26352", "name": "monitoring", "reconcileID": "f98cd611-1a46-4155-8c8d-db6aa9c6f036", "last-applied-secret": "cefd1660654364c2ae04d98afb3937493a384850d1b8dd5cff992bb409b54bda"} 2026-10-05T14:45:18.855Z DEBUG Updating object {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"monitoring","namespace":"monitoring-pmm3-26352"}, "namespace": "monitoring-pmm3-26352", "name": "monitoring", "reconcileID": "f98cd611-1a46-4155-8c8d-db6aa9c6f036", "object": "monitoring-pxc", "kind": "&TypeMeta{Kind:StatefulSet,APIVersion:apps/v1,}", "hashChanged": true, "metaChanged": true} 2026-10-05T14:45:19.232Z DEBUG Updating object {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"monitoring","namespace":"monitoring-pmm3-26352"}, "namespace": "monitoring-pmm3-26352", "name": "monitoring", "reconcileID": "f98cd611-1a46-4155-8c8d-db6aa9c6f036", "object": "monitoring-haproxy", "kind": "&TypeMeta{Kind:StatefulSet,APIVersion:apps/v1,}", "hashChanged": true, "metaChanged": true} 2026-10-05T14:47:25.842Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"monitoring","namespace":"monitoring-pmm3-26352"}, "namespace": "monitoring-pmm3-26352", "name": "monitoring", "reconcileID": "71cf1a46-e1f2-499b-a489-b17ddb929164", "err": "failed to ensure cluster readonly status: connect to pod monitoring-pxc-0: dial tcp: lookup monitoring-pxc-0.monitoring-pxc.monitoring-pmm3-26352 on 34.118.224.10:53: no such host"} 2026-10-05T14:49:40.803Z DEBUG Updating object {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"monitoring","namespace":"monitoring-pmm3-26352"}, "namespace": "monitoring-pmm3-26352", "name": "monitoring", "reconcileID": "45f936f4-241d-499b-886d-2327e4186d07", "object": "monitoring-pxc", "kind": "&TypeMeta{Kind:StatefulSet,APIVersion:apps/v1,}", "hashChanged": true, "metaChanged": true} 2026-10-05T14:49:40.907Z DEBUG Updating object {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"monitoring","namespace":"monitoring-pmm3-26352"}, "namespace": "monitoring-pmm3-26352", "name": "monitoring", "reconcileID": "45f936f4-241d-499b-886d-2327e4186d07", "object": "monitoring-haproxy", "kind": "&TypeMeta{Kind:StatefulSet,APIVersion:apps/v1,}", "hashChanged": true, "metaChanged": true} 2026-10-05T14:49:41.010Z DEBUG Updating object {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"monitoring","namespace":"monitoring-pmm3-26352"}, "namespace": "monitoring-pmm3-26352", "name": "monitoring", "reconcileID": "45f936f4-241d-499b-886d-2327e4186d07", "object": "monitoring-pxc", "kind": "&TypeMeta{Kind:StatefulSet,APIVersion:apps/v1,}", "hashChanged": true, "metaChanged": true} 2026-10-05T14:50:28.348Z ERROR Update status {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"monitoring","namespace":"monitoring-pmm3-26352"}, "namespace": "monitoring-pmm3-26352", "name": "monitoring", "reconcileID": "06f10573-6774-4e91-83bb-7bafe9e07b0a", "error": "get pxc status: parse monitoring-pxc-0 pod logs: get logs from monitoring-pxc-0 pod: failed to get pod logs: get pod logs stream: pods \"monitoring-pxc-0\" not found", "errorVerbose": "pods \"monitoring-pxc-0\" not found\nget pod logs stream\ngithub.com/percona/percona-xtradb-cluster-operator/clientcmd.(*client).PodLogs.func2\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/clientcmd/clientcmd.go:66\nk8s.io/client-go/util/retry.OnError.func1\n\t/go/pkg/mod/k8s.io/client-go@v0.37.0/util/retry/util.go:51\nk8s.io/apimachinery/pkg/util/wait.runConditionWithCrashProtection\n\t/go/pkg/mod/k8s.io/apimachinery@v0.37.0/pkg/util/wait/wait.go:150\nk8s.io/apimachinery/pkg/util/wait.ExponentialBackoff\n\t/go/pkg/mod/k8s.io/apimachinery@v0.37.0/pkg/util/wait/backoff.go:477\nk8s.io/client-go/util/retry.OnError\n\t/go/pkg/mod/k8s.io/client-go@v0.37.0/util/retry/util.go:50\ngithub.com/percona/percona-xtradb-cluster-operator/clientcmd.(*client).PodLogs\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/clientcmd/clientcmd.go:61\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).isPodWaitingForRecovery\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/full_crash_recovery.go:117\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).appStatus\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/status.go:248\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).updateStatus\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/status.go:105\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).Reconcile.func1\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:270\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).Reconcile\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:509\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:221\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:478\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1771\nfailed to get pod logs\ngithub.com/percona/percona-xtradb-cluster-operator/clientcmd.(*client).PodLogs\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/clientcmd/clientcmd.go:80\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).isPodWaitingForRecovery\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/full_crash_recovery.go:117\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).appStatus\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/status.go:248\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).updateStatus\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/status.go:105\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).Reconcile.func1\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:270\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).Reconcile\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:509\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:221\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:478\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1771\nget logs from monitoring-pxc-0 pod\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).isPodWaitingForRecovery\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/full_crash_recovery.go:119\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).appStatus\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/status.go:248\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).updateStatus\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/status.go:105\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).Reconcile.func1\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:270\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).Reconcile\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:509\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:221\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:478\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1771\nparse monitoring-pxc-0 pod logs\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).appStatus\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/status.go:250\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).updateStatus\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/status.go:105\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).Reconcile.func1\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:270\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).Reconcile\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:509\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:221\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:478\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1771\nget pxc status\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).updateStatus\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/status.go:107\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).Reconcile.func1\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:270\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).Reconcile\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:509\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:221\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:478\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1771"} [mysql] 2026/10/05 14:47:12 packets.go:58 unexpected EOF [mysql] 2026/10/05 14:47:24 packets.go:58 unexpected EOF github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).Reconcile github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).Reconcile.func1 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler + local backup_names ++ kubectl_bin get pxc-backup --all-namespaces -o 'jsonpath={range .items[*]}{.metadata.namespace}/{.metadata.name}{"\n"}{end}' + backup_names= + [[ -n '' ]] + local pxc_names ++ kubectl_bin get pxc --all-namespaces -o 'jsonpath={range .items[*]}{.metadata.namespace}/{.metadata.name}{"\n"}{end}' + pxc_names=monitoring-pmm3-26352/monitoring + [[ -n monitoring-pmm3-26352/monitoring ]] + read -r pns name + kubectl_bin get pxc --all-namespaces -o 'jsonpath={range .items[*]}{.metadata.namespace} {.metadata.name}{"\n"}{end}' + kubectl_bin patch pxc -n monitoring-pmm3-26352 monitoring --type=merge -p '{"metadata":{"finalizers":["percona.com/delete-pxc-pvc"]}}' perconaxtradbcluster.pxc.percona.com/monitoring patched + read -r pns name + timeout 300 kubectl delete pxc --all --all-namespaces --ignore-not-found perconaxtradbcluster.pxc.percona.com "monitoring" deleted from monitoring-pmm3-26352 namespace ++ date +%s + local deadline=1791212222 ++ kubectl_bin get pxc --all-namespaces -o name --ignore-not-found + [[ -z '' ]] + timeout 300 kubectl delete pxc-restore --all --all-namespaces --wait=false --ignore-not-found No resources found + kubectl_bin delete ValidatingWebhookConfiguration percona-xtradbcluster-webhook --ignore-not-found ++ mktemp + local LAST_OUT=/tmp/tmp.CpBiL4bwpp ++ mktemp + local LAST_ERR=/tmp/tmp.IjRJzvkCBS + local exit_status=0 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl delete ValidatingWebhookConfiguration percona-xtradbcluster-webhook --ignore-not-found + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.CpBiL4bwpp validatingwebhookconfiguration.admissionregistration.k8s.io "percona-xtradbcluster-webhook" deleted + cat /tmp/tmp.IjRJzvkCBS + rm /tmp/tmp.CpBiL4bwpp /tmp/tmp.IjRJzvkCBS + return 0 + kubectl_bin delete -f https://github.com/jetstack/cert-manager/releases/download/v1.20.2/cert-manager.yaml + : + '[' '!' -z '' ']' + '[' -n pxc-operator ']' + kubectl_bin delete --grace-period=0 --force=true namespace pxc-operator ++ date +%s + kubectl_bin delete --grace-period=0 --force=true namespace monitoring-pmm3-26352 ++ mktemp + local LAST_OUT=/tmp/tmp.6QXl1i7vrz + local deadline=1791212082 ++ mktemp + local LAST_OUT=/tmp/tmp.Y59xRgSnWK ++ mktemp ++ mktemp ++ kubectl_bin get namespace monitoring-pmm3-26352 --ignore-not-found -o name + local LAST_ERR=/tmp/tmp.J1V4WR00hi + local exit_status=0 + local LAST_ERR=/tmp/tmp.C7O4fCBEbw + local exit_status=0 ++ seq 0 2 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl delete --grace-period=0 --force=true namespace monitoring-pmm3-26352 + for i in $(seq 0 2) + set +e + kubectl delete --grace-period=0 --force=true namespace pxc-operator + [[ -z namespace/monitoring-pmm3-26352 ]] ++ date +%s + [[ 1791211965 -ge 1791212082 ]] + sleep 5 ++ kubectl_bin get namespace monitoring-pmm3-26352 --ignore-not-found -o name + [[ -z namespace/monitoring-pmm3-26352 ]] ++ date +%s + [[ 1791211973 -ge 1791212082 ]] + sleep 5 ++ kubectl_bin get namespace monitoring-pmm3-26352 --ignore-not-found -o name + [[ -z namespace/monitoring-pmm3-26352 ]] ++ date +%s + [[ 1791211979 -ge 1791212082 ]] + sleep 5 + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.6QXl1i7vrz namespace "pxc-operator" force deleted + cat /tmp/tmp.C7O4fCBEbw Warning: Immediate deletion does not wait for confirmation that the running resource has been terminated. The resource may continue to run on the cluster indefinitely. + rm /tmp/tmp.6QXl1i7vrz /tmp/tmp.C7O4fCBEbw + return 0 ++ kubectl_bin get namespace monitoring-pmm3-26352 --ignore-not-found -o name + [[ -z namespace/monitoring-pmm3-26352 ]] ++ date +%s + [[ 1791211988 -ge 1791212082 ]] + sleep 5 + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.Y59xRgSnWK namespace "monitoring-pmm3-26352" force deleted + cat /tmp/tmp.J1V4WR00hi Warning: Immediate deletion does not wait for confirmation that the running resource has been terminated. The resource may continue to run on the cluster indefinitely. + rm /tmp/tmp.Y59xRgSnWK /tmp/tmp.J1V4WR00hi + return 0 ++ kubectl_bin get namespace monitoring-pmm3-26352 --ignore-not-found -o name + [[ -z '' ]] ++ date +%s + deadline=1791212116 ++ awk '$2 != ""' ++ kubectl_bin get pv -o 'jsonpath={range .items[*]}{.metadata.name}{"\t"}{.metadata.deletionTimestamp}{"\n"}{end}' + [[ -z '' ]] + rm -rf /tmp/tmp.mnnOTDKNTG + desc 'test passed' + set +o xtrace ----------------------------------------------------------------------------------- test passed -----------------------------------------------------------------------------------