Log: /mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/e2e-tests/logs/monitoring-2-0-8-0.log Warning: version difference between client (1.37) and server (1.34) exceeds the supported minor version skew of +/-1 Warning: version difference between client (1.37) and server (1.34) exceeds the supported minor version skew of +/-1 + cluster=monitoring + create_infra monitoring-2-0-5888 + local ns=monitoring-2-0-5888 + '[' -n pxc-operator ']' + kubectl_bin get pxc --all-namespaces -o 'jsonpath={range .items[*]}{.metadata.namespace} {.metadata.name}{"\n"}{end}' + read -r pns name + [[ -z monitoring-2-0-26814 ]] + [[ -z monitoring ]] + kubectl_bin patch pxc -n monitoring-2-0-26814 monitoring --type=merge -p '{"metadata":{"finalizers":[]}}' ++ mktemp + local LAST_OUT=/tmp/tmp.LIQF4QtS2l ++ mktemp + local LAST_ERR=/tmp/tmp.n1oUSsLbRH + local exit_status=0 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl patch pxc -n monitoring-2-0-26814 monitoring --type=merge -p '{"metadata":{"finalizers":[]}}' + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.LIQF4QtS2l perconaxtradbcluster.pxc.percona.com/monitoring patched (no change) + cat /tmp/tmp.n1oUSsLbRH + rm /tmp/tmp.LIQF4QtS2l /tmp/tmp.n1oUSsLbRH + return 0 + read -r pns name + kubectl_bin get pxc-backup --all-namespaces -o 'jsonpath={range .items[*]}{.metadata.namespace} {.metadata.name}{"\n"}{end}' + read -r pns name + timeout 300 kubectl delete pxc --all --all-namespaces --wait=false --ignore-not-found perconaxtradbcluster.pxc.percona.com "monitoring" deleted from monitoring-2-0-26814 namespace + timeout 300 kubectl delete pxc-backup --all --all-namespaces --wait=false --ignore-not-found No resources found + timeout 300 kubectl delete pxc-restore --all --all-namespaces --wait=false --ignore-not-found No resources found + create_namespace pxc-operator + local namespace=pxc-operator + local skip_clean_namespace= + [[ 1 == 1 ]] + [[ -z '' ]] + destroy_chaos_mesh ++ helm list --all-namespaces --filter chaos-mesh ++ tail -n1 ++ awk '-F ' '{print $2}' ++ sed s/NAMESPACE// + local chaos_mesh_ns= + '[' -n '' ']' ++ kubectl get MutatingWebhookConfiguration ++ grep chaos-mesh ++ awk '{print $1}' + timeout 30 kubectl delete MutatingWebhookConfiguration error: resource(s) were provided, but no name was specified + : ++ kubectl get ValidatingWebhookConfiguration ++ grep chaos-mesh ++ awk '{print $1}' + timeout 30 kubectl delete ValidatingWebhookConfiguration error: resource(s) were provided, but no name was specified + : ++ kubectl get ValidatingWebhookConfiguration ++ awk '{print $1}' ++ grep validate-auth + timeout 30 kubectl delete ValidatingWebhookConfiguration error: resource(s) were provided, but no name was specified + : ++ kubectl api-resources ++ grep chaos-mesh ++ awk '{print $1}' ++ kubectl get crd ++ grep chaos-mesh.org ++ awk '{print $1}' + timeout 30 kubectl delete crd error: resource(s) were provided, but no name was specified + : ++ kubectl get clusterrolebinding ++ grep chaos-mesh ++ awk '{print $1}' + timeout 30 kubectl delete clusterrolebinding error: resource(s) were provided, but no name was specified + : ++ kubectl get clusterrole ++ grep chaos-mesh ++ awk '{print $1}' + timeout 30 kubectl delete clusterrole error: resource(s) were provided, but no name was specified + : + desc 'cleaned up all old namespaces' + set +o xtrace ----------------------------------------------------------------------------------- cleaned up all old namespaces ----------------------------------------------------------------------------------- + '[' -n '' ']' + desc 'cleaned up old namespaces pxc-operator' + set +o xtrace ----------------------------------------------------------------------------------- cleaned up old namespaces pxc-operator ----------------------------------------------------------------------------------- + kubectl_bin delete namespace pxc-operator ++ mktemp + xargs kubectl delete ns + awk '{print$1}' + local LAST_OUT=/tmp/tmp.LSahy667nk + grep -E -v '^kube-|^default|Terminating|pxc-operator|openshift|^gke-|^gmp-|^NAME' + kubectl_bin get ns ++ mktemp ++ mktemp + local LAST_ERR=/tmp/tmp.SvHDTu6vfO + local exit_status=0 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl delete namespace pxc-operator + local LAST_OUT=/tmp/tmp.DtX4sqKNVm ++ mktemp + local LAST_ERR=/tmp/tmp.aFtulxNwfk + local exit_status=0 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl get ns + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.DtX4sqKNVm + cat /tmp/tmp.aFtulxNwfk + rm /tmp/tmp.DtX4sqKNVm /tmp/tmp.aFtulxNwfk + return 0 namespace "monitoring-2-0-26814" deleted + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.LSahy667nk namespace "pxc-operator" deleted + cat /tmp/tmp.SvHDTu6vfO + rm /tmp/tmp.LSahy667nk /tmp/tmp.SvHDTu6vfO + return 0 + wait_for_delete namespace/pxc-operator + local res=namespace/pxc-operator + echo -n 'waiting for namespace/pxc-operator to be deleted' waiting for namespace/pxc-operator to be deleted+ set +o xtrace Error from server (NotFound): namespaces "pxc-operator" not found + desc 'create namespace pxc-operator' + set +o xtrace ----------------------------------------------------------------------------------- create namespace pxc-operator ----------------------------------------------------------------------------------- + kubectl_bin create namespace pxc-operator ++ mktemp + local LAST_OUT=/tmp/tmp.cOlu62GcXU ++ mktemp + local LAST_ERR=/tmp/tmp.ilTmDL3cud + local exit_status=0 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl create namespace pxc-operator + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.cOlu62GcXU namespace/pxc-operator created + cat /tmp/tmp.ilTmDL3cud + rm /tmp/tmp.cOlu62GcXU /tmp/tmp.ilTmDL3cud + return 0 ++ kubectl_bin config current-context +++ mktemp ++ local LAST_OUT=/tmp/tmp.4A2Agyd1B3 +++ mktemp ++ local LAST_ERR=/tmp/tmp.O1p6TB3FS3 ++ local exit_status=0 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl config current-context ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.4A2Agyd1B3 ++ cat /tmp/tmp.O1p6TB3FS3 ++ rm /tmp/tmp.4A2Agyd1B3 /tmp/tmp.O1p6TB3FS3 ++ return 0 + kubectl_bin config set-context gke_cloud-dev-112233_us-central1-a_jen-pxc-2546-caf25420-2-cluster9 --namespace=pxc-operator ++ mktemp + local LAST_OUT=/tmp/tmp.jfgPltERfA ++ mktemp + local LAST_ERR=/tmp/tmp.E3hKstcV61 + local exit_status=0 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl config set-context gke_cloud-dev-112233_us-central1-a_jen-pxc-2546-caf25420-2-cluster9 --namespace=pxc-operator + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.jfgPltERfA Context "gke_cloud-dev-112233_us-central1-a_jen-pxc-2546-caf25420-2-cluster9" modified. + cat /tmp/tmp.E3hKstcV61 + rm /tmp/tmp.jfgPltERfA /tmp/tmp.E3hKstcV61 + return 0 + deploy_operator + desc 'start PXC operator' + set +o xtrace ----------------------------------------------------------------------------------- start PXC operator ----------------------------------------------------------------------------------- + kubectl_bin apply --server-side --force-conflicts -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/deploy/crd.yaml ++ mktemp + local LAST_OUT=/tmp/tmp.84mddpCNPW ++ mktemp + local LAST_ERR=/tmp/tmp.9k74EuzPIJ + local exit_status=0 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl apply --server-side --force-conflicts -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/deploy/crd.yaml + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.84mddpCNPW customresourcedefinition.apiextensions.k8s.io/perconaxtradbclusterbackups.pxc.percona.com serverside-applied customresourcedefinition.apiextensions.k8s.io/perconaxtradbclusterrestores.pxc.percona.com serverside-applied customresourcedefinition.apiextensions.k8s.io/perconaxtradbclusters.pxc.percona.com serverside-applied + cat /tmp/tmp.9k74EuzPIJ + rm /tmp/tmp.84mddpCNPW /tmp/tmp.9k74EuzPIJ + return 0 + kubectl_bin wait --for=condition=Established --timeout=120s crd/perconaxtradbclusters.pxc.percona.com ++ mktemp + local LAST_OUT=/tmp/tmp.KqGZLNSAqG ++ mktemp + local LAST_ERR=/tmp/tmp.GMYF1BX7oN + local exit_status=0 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl wait --for=condition=Established --timeout=120s crd/perconaxtradbclusters.pxc.percona.com + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.KqGZLNSAqG customresourcedefinition.apiextensions.k8s.io/perconaxtradbclusters.pxc.percona.com condition met + cat /tmp/tmp.GMYF1BX7oN + rm /tmp/tmp.KqGZLNSAqG /tmp/tmp.GMYF1BX7oN + return 0 + kubectl_bin wait --for=condition=Established --timeout=120s crd/perconaxtradbclusterbackups.pxc.percona.com ++ mktemp + local LAST_OUT=/tmp/tmp.G74ejQpk06 ++ mktemp + local LAST_ERR=/tmp/tmp.XXoWAnVuiX + local exit_status=0 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl wait --for=condition=Established --timeout=120s crd/perconaxtradbclusterbackups.pxc.percona.com + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.G74ejQpk06 customresourcedefinition.apiextensions.k8s.io/perconaxtradbclusterbackups.pxc.percona.com condition met + cat /tmp/tmp.XXoWAnVuiX + rm /tmp/tmp.G74ejQpk06 /tmp/tmp.XXoWAnVuiX + return 0 + kubectl_bin wait --for=condition=Established --timeout=120s crd/perconaxtradbclusterrestores.pxc.percona.com ++ mktemp + local LAST_OUT=/tmp/tmp.zawBT2ruSP ++ mktemp + local LAST_ERR=/tmp/tmp.9HPrlgFiW0 + local exit_status=0 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl wait --for=condition=Established --timeout=120s crd/perconaxtradbclusterrestores.pxc.percona.com + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.zawBT2ruSP customresourcedefinition.apiextensions.k8s.io/perconaxtradbclusterrestores.pxc.percona.com condition met + cat /tmp/tmp.9HPrlgFiW0 + rm /tmp/tmp.zawBT2ruSP /tmp/tmp.9HPrlgFiW0 + return 0 + '[' -n pxc-operator ']' + apply_rbac cw-rbac + local operator_namespace=pxc-operator + local rbac=cw-rbac + cat /mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/deploy/cw-rbac.yaml + sed -e 's^namespace: .*^namespace: pxc-operator^' + kubectl_bin apply -f - ++ mktemp + local LAST_OUT=/tmp/tmp.AvPj1EBKxf ++ mktemp + local LAST_ERR=/tmp/tmp.h5J49dYBZB + local exit_status=0 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.AvPj1EBKxf clusterrole.rbac.authorization.k8s.io/percona-xtradb-cluster-operator unchanged serviceaccount/percona-xtradb-cluster-operator created clusterrolebinding.rbac.authorization.k8s.io/service-account-percona-xtradb-cluster-operator unchanged + cat /tmp/tmp.h5J49dYBZB + rm /tmp/tmp.AvPj1EBKxf /tmp/tmp.h5J49dYBZB + return 0 + cat /mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/deploy/cw-operator.yaml + sed -e 's^image: .*^image: perconalab/percona-xtradb-cluster-operator:PR-2546-caf25420^' + sed -e 's^failureThreshold: .*^failureThreshold: 10^' + yq eval '(select(.kind == "Deployment").spec.template.spec.containers[] | select(.name == "percona-xtradb-cluster-operator").env[] | select(.name == "DISABLE_TELEMETRY").value) = "true"' - + yq eval '(select(.kind == "Deployment").spec.template.spec.containers[] | select(.name == "percona-xtradb-cluster-operator").env[] | select(.name == "PXCO_FEATURE_GATES").value) = ""' - + yq eval '(select(.kind == "Deployment").spec.template.spec.containers[] | select(.name == "percona-xtradb-cluster-operator").env[] | select(.name == "LOG_LEVEL").value) = "VERBOSE"' - + kubectl_bin apply -f - ++ mktemp + local LAST_OUT=/tmp/tmp.77nYxPemZg ++ mktemp + local LAST_ERR=/tmp/tmp.4QNOJyIeQV + local exit_status=0 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.77nYxPemZg deployment.apps/percona-xtradb-cluster-operator created service/percona-xtradb-cluster-operator created + cat /tmp/tmp.4QNOJyIeQV + rm /tmp/tmp.77nYxPemZg /tmp/tmp.4QNOJyIeQV + return 0 + sleep 10 + kubectl_bin wait --for=condition=Ready pods -l app.kubernetes.io/component=operator,app.kubernetes.io/instance=percona-xtradb-cluster-operator,app.kubernetes.io/name=percona-xtradb-cluster-operator --timeout=30s ++ mktemp + local LAST_OUT=/tmp/tmp.KYKaZIbwVo ++ mktemp + local LAST_ERR=/tmp/tmp.pxlWWqEYy1 + local exit_status=0 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl wait --for=condition=Ready pods -l app.kubernetes.io/component=operator,app.kubernetes.io/instance=percona-xtradb-cluster-operator,app.kubernetes.io/name=percona-xtradb-cluster-operator --timeout=30s + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.KYKaZIbwVo pod/percona-xtradb-cluster-operator-658fcf4b98-jb2nc condition met + cat /tmp/tmp.pxlWWqEYy1 + rm /tmp/tmp.KYKaZIbwVo /tmp/tmp.pxlWWqEYy1 + return 0 ++ get_operator_pod ++ local label_prefix=app.kubernetes.io/ +++ grep -c percona-xtradb-cluster-operator +++ kubectl get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator -n pxc-operator ++ local check_label=1 ++ [[ 1 -eq 0 ]] ++ jq -r '.items[] | select(.metadata.deletionTimestamp == null) | .metadata.name' ++ kubectl_bin get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator --field-selector=status.phase=Running -o json -n pxc-operator +++ mktemp ++ local LAST_OUT=/tmp/tmp.fiuGASj2R8 +++ mktemp ++ local LAST_ERR=/tmp/tmp.kDfBoWUH61 ++ local exit_status=0 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator --field-selector=status.phase=Running -o json -n pxc-operator ++ head -1 ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.fiuGASj2R8 ++ cat /tmp/tmp.kDfBoWUH61 ++ rm /tmp/tmp.fiuGASj2R8 /tmp/tmp.kDfBoWUH61 ++ return 0 + wait_pod percona-xtradb-cluster-operator-658fcf4b98-jb2nc 480 pxc-operator + local pod=percona-xtradb-cluster-operator-658fcf4b98-jb2nc + local max_retry=480 + local ns=pxc-operator ++ echo percona-xtradb-cluster-operator-658fcf4b98-jb2nc ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ grep -E '^(pxc|proxysql)$' + local container= + set +o xtrace pod/percona-xtradb-cluster-operator-658fcf4b98-jb2nc condition met waiting for pod/percona-xtradb-cluster-operator-658fcf4b98-jb2nc to become Ready.Ok + sleep 3 + create_namespace monitoring-2-0-5888 + local namespace=monitoring-2-0-5888 + local skip_clean_namespace= + [[ 1 == 1 ]] + [[ -z '' ]] + destroy_chaos_mesh ++ sed s/NAMESPACE// ++ awk '-F ' '{print $2}' ++ helm list --all-namespaces --filter chaos-mesh ++ tail -n1 + local chaos_mesh_ns= + '[' -n '' ']' ++ kubectl get MutatingWebhookConfiguration ++ awk '{print $1}' ++ grep chaos-mesh + timeout 30 kubectl delete MutatingWebhookConfiguration error: resource(s) were provided, but no name was specified + : ++ kubectl get ValidatingWebhookConfiguration ++ grep chaos-mesh ++ awk '{print $1}' + timeout 30 kubectl delete ValidatingWebhookConfiguration error: resource(s) were provided, but no name was specified + : ++ awk '{print $1}' ++ grep validate-auth ++ kubectl get ValidatingWebhookConfiguration + timeout 30 kubectl delete ValidatingWebhookConfiguration error: resource(s) were provided, but no name was specified + : ++ grep chaos-mesh ++ kubectl api-resources ++ awk '{print $1}' ++ grep chaos-mesh.org ++ awk '{print $1}' ++ kubectl get crd + timeout 30 kubectl delete crd error: resource(s) were provided, but no name was specified + : ++ grep chaos-mesh ++ kubectl get clusterrolebinding ++ awk '{print $1}' + timeout 30 kubectl delete clusterrolebinding error: resource(s) were provided, but no name was specified + : ++ awk '{print $1}' ++ kubectl get clusterrole ++ grep chaos-mesh + timeout 30 kubectl delete clusterrole error: resource(s) were provided, but no name was specified + : + desc 'cleaned up all old namespaces' + set +o xtrace ----------------------------------------------------------------------------------- cleaned up all old namespaces ----------------------------------------------------------------------------------- + kubectl_bin get ns + grep -E -v '^kube-|^default|Terminating|pxc-operator|openshift|^gke-|^gmp-|^NAME' + awk '{print$1}' + '[' -n '' ']' ++ mktemp + desc 'cleaned up old namespaces monitoring-2-0-5888' + set +o xtrace ----------------------------------------------------------------------------------- cleaned up old namespaces monitoring-2-0-5888 ----------------------------------------------------------------------------------- + kubectl_bin delete namespace monitoring-2-0-5888 + local LAST_OUT=/tmp/tmp.sJ68LJy7pw ++ mktemp ++ mktemp + local LAST_ERR=/tmp/tmp.lUUzvIoLnO + local exit_status=0 ++ seq 0 2 + local LAST_OUT=/tmp/tmp.iIhh31fkMY ++ mktemp + for i in $(seq 0 2) + set +e + kubectl get ns + local LAST_ERR=/tmp/tmp.e78uDY4Yra + local exit_status=0 ++ seq 0 2 + xargs kubectl delete ns + for i in $(seq 0 2) + set +e + kubectl delete namespace monitoring-2-0-5888 + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.sJ68LJy7pw + cat /tmp/tmp.lUUzvIoLnO + rm /tmp/tmp.sJ68LJy7pw /tmp/tmp.lUUzvIoLnO + return 0 error: resource(s) were provided, but no name was specified + exit_status=1 + set -e + '[' 1 '!=' 0 ']' + '[' 1 == 1 ']' + sleep 0 + for i in $(seq 0 2) + set +e + kubectl delete namespace monitoring-2-0-5888 + exit_status=1 + set -e + '[' 1 '!=' 0 ']' + '[' 1 == 1 ']' + sleep 0 + for i in $(seq 0 2) + set +e + kubectl delete namespace monitoring-2-0-5888 + exit_status=1 + set -e + '[' 1 '!=' 0 ']' + '[' 1 == 1 ']' + sleep 0 + cat /tmp/tmp.iIhh31fkMY + cat /tmp/tmp.e78uDY4Yra Error from server (NotFound): namespaces "monitoring-2-0-5888" not found + rm /tmp/tmp.iIhh31fkMY /tmp/tmp.e78uDY4Yra + return 1 + : + wait_for_delete namespace/monitoring-2-0-5888 + local res=namespace/monitoring-2-0-5888 + echo -n 'waiting for namespace/monitoring-2-0-5888 to be deleted' waiting for namespace/monitoring-2-0-5888 to be deleted+ set +o xtrace Error from server (NotFound): namespaces "monitoring-2-0-5888" not found + desc 'create namespace monitoring-2-0-5888' + set +o xtrace ----------------------------------------------------------------------------------- create namespace monitoring-2-0-5888 ----------------------------------------------------------------------------------- + kubectl_bin create namespace monitoring-2-0-5888 ++ mktemp + local LAST_OUT=/tmp/tmp.SzfD8dsUX0 ++ mktemp + local LAST_ERR=/tmp/tmp.TM3feWFbCs + local exit_status=0 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl create namespace monitoring-2-0-5888 + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.SzfD8dsUX0 namespace/monitoring-2-0-5888 created + cat /tmp/tmp.TM3feWFbCs + rm /tmp/tmp.SzfD8dsUX0 /tmp/tmp.TM3feWFbCs + return 0 ++ kubectl_bin config current-context +++ mktemp ++ local LAST_OUT=/tmp/tmp.X4UJoQCZgJ +++ mktemp ++ local LAST_ERR=/tmp/tmp.nPVNccVEcr ++ local exit_status=0 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl config current-context ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.X4UJoQCZgJ ++ cat /tmp/tmp.nPVNccVEcr ++ rm /tmp/tmp.X4UJoQCZgJ /tmp/tmp.nPVNccVEcr ++ return 0 + kubectl_bin config set-context gke_cloud-dev-112233_us-central1-a_jen-pxc-2546-caf25420-2-cluster9 --namespace=monitoring-2-0-5888 ++ mktemp + local LAST_OUT=/tmp/tmp.mYI62EaqhU ++ mktemp + local LAST_ERR=/tmp/tmp.TxJT4Sba2r + local exit_status=0 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl config set-context gke_cloud-dev-112233_us-central1-a_jen-pxc-2546-caf25420-2-cluster9 --namespace=monitoring-2-0-5888 + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.mYI62EaqhU Context "gke_cloud-dev-112233_us-central1-a_jen-pxc-2546-caf25420-2-cluster9" modified. + cat /tmp/tmp.TxJT4Sba2r + rm /tmp/tmp.mYI62EaqhU /tmp/tmp.TxJT4Sba2r + return 0 + apply_secrets + desc 'create secrets for cloud storages' + set +o xtrace ----------------------------------------------------------------------------------- create secrets for cloud storages ----------------------------------------------------------------------------------- + '[' -z '' ']' + kubectl_bin apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/e2e-tests/conf/minio-secret.yml -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/e2e-tests/conf/cloud-secret.yml ++ mktemp + local LAST_OUT=/tmp/tmp.TbqVebASeq ++ mktemp + local LAST_ERR=/tmp/tmp.JET5znRPYh + local exit_status=0 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/e2e-tests/conf/minio-secret.yml -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/e2e-tests/conf/cloud-secret.yml + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.TbqVebASeq secret/minio-secret created secret/aws-s3-secret created secret/do-spaces-secret created secret/gcp-cs-secret created secret/azure-secret created + cat /tmp/tmp.JET5znRPYh + rm /tmp/tmp.TbqVebASeq /tmp/tmp.JET5znRPYh + return 0 + deploy_helm monitoring-2-0-5888 + helm repo add hashicorp https://helm.releases.hashicorp.com "hashicorp" already exists with the same configuration, skipping + helm repo add minio https://charts.min.io/ "minio" already exists with the same configuration, skipping + helm repo update Hang tight while we grab the latest from your chart repositories... ...Successfully got an update from the "minio" chart repository ...Successfully got an update from the "chaos-mesh" chart repository ...Successfully got an update from the "hashicorp" chart repository ...Successfully got an update from the "percona" chart repository Update Complete. ⎈Happy Helming!⎈ + desc 'install PMM Server' + set +o xtrace ----------------------------------------------------------------------------------- install PMM Server ----------------------------------------------------------------------------------- + deploy_pmm_server + '[' '!' -z '' ']' + helm repo add percona https://percona.github.io/percona-helm-charts/ "percona" already exists with the same configuration, skipping + helm repo update Hang tight while we grab the latest from your chart repositories... ...Successfully got an update from the "minio" chart repository ...Successfully got an update from the "chaos-mesh" chart repository ...Successfully got an update from the "hashicorp" chart repository ...Successfully got an update from the "percona" chart repository Update Complete. ⎈Happy Helming!⎈ + helm uninstall monitoring Error: uninstall: Release not loaded: monitoring: release: not found + : + retry 10 60 helm install monitoring --set imageRepo=perconalab/pmm-server --set imageTag=dev-latest https://percona-charts.storage.googleapis.com/pmm-server-9.9.9.tgz + local max=10 + local delay=60 + shift 2 + local n=1 + helm install monitoring --set imageRepo=perconalab/pmm-server --set imageTag=dev-latest https://percona-charts.storage.googleapis.com/pmm-server-9.9.9.tgz NAME: monitoring LAST DEPLOYED: Mon Oct 5 13:20:29 2026 NAMESPACE: monitoring-2-0-5888 STATUS: deployed REVISION: 1 TEST SUITE: None NOTES: PMM server can be accessed via HTTPS (port 443) on the following DNS name from within your cluster: endpoint: https://monitoring-service.monitoring-2-0-5888.svc.cluster.local:443 login: admin password: admin + wait_for_pmm_service + local timeout=420 ++ date +%s + local start=1791206430 + grep -q . + kubectl_bin get svc monitoring-service -o 'jsonpath={.status.loadBalancer.ingress[0]}' ++ date +%s + (( 1791206431 - start >= timeout )) + sleep 2 + grep -q . + kubectl_bin get svc monitoring-service -o 'jsonpath={.status.loadBalancer.ingress[0]}' ++ date +%s + (( 1791206435 - start >= timeout )) + sleep 2 + grep -q . + kubectl_bin get svc monitoring-service -o 'jsonpath={.status.loadBalancer.ingress[0]}' ++ date +%s + (( 1791206439 - start >= timeout )) + sleep 2 + kubectl_bin get svc monitoring-service -o 'jsonpath={.status.loadBalancer.ingress[0]}' + grep -q . ++ date +%s + (( 1791206443 - start >= timeout )) + sleep 2 + kubectl_bin get svc monitoring-service -o 'jsonpath={.status.loadBalancer.ingress[0]}' + grep -q . ++ date +%s + (( 1791206446 - start >= timeout )) + sleep 2 + kubectl_bin get svc monitoring-service -o 'jsonpath={.status.loadBalancer.ingress[0]}' + grep -q . ++ date +%s + (( 1791206449 - start >= timeout )) + sleep 2 + kubectl_bin get svc monitoring-service -o 'jsonpath={.status.loadBalancer.ingress[0]}' + grep -q . ++ date +%s + (( 1791206453 - start >= timeout )) + sleep 2 + kubectl_bin get svc monitoring-service -o 'jsonpath={.status.loadBalancer.ingress[0]}' + grep -q . ++ date +%s + (( 1791206456 - start >= timeout )) + sleep 2 + kubectl_bin get svc monitoring-service -o 'jsonpath={.status.loadBalancer.ingress[0]}' + grep -q . ++ date +%s + (( 1791206459 - start >= timeout )) + sleep 2 + kubectl_bin get svc monitoring-service -o 'jsonpath={.status.loadBalancer.ingress[0]}' + grep -q . ++ date +%s + (( 1791206462 - start >= timeout )) + sleep 2 + kubectl_bin get svc monitoring-service -o 'jsonpath={.status.loadBalancer.ingress[0]}' + grep -q . ++ date +%s + (( 1791206465 - start >= timeout )) + sleep 2 + kubectl_bin get svc monitoring-service -o 'jsonpath={.status.loadBalancer.ingress[0]}' + grep -q . ++ date +%s + (( 1791206468 - start >= timeout )) + sleep 2 + kubectl_bin get svc monitoring-service -o 'jsonpath={.status.loadBalancer.ingress[0]}' + grep -q . ++ date +%s + (( 1791206472 - start >= timeout )) + sleep 2 + kubectl_bin get svc monitoring-service -o 'jsonpath={.status.loadBalancer.ingress[0]}' + grep -q . + kubectl_bin wait sts/monitoring '--for=jsonpath={.status.readyReplicas}=1' --timeout=420s ++ mktemp + local LAST_OUT=/tmp/tmp.c0rhIFoRD5 ++ mktemp + local LAST_ERR=/tmp/tmp.CJiV1shdFk + local exit_status=0 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl wait sts/monitoring '--for=jsonpath={.status.readyReplicas}=1' --timeout=420s + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.c0rhIFoRD5 statefulset.apps/monitoring condition met + cat /tmp/tmp.CJiV1shdFk + rm /tmp/tmp.c0rhIFoRD5 /tmp/tmp.CJiV1shdFk + return 0 ++ kubectl_bin exec monitoring-0 -- bash -c 'printenv | grep ADMIN_PASSWORD | cut -d '\''='\'' -f2' +++ mktemp ++ local LAST_OUT=/tmp/tmp.zTV62GqUjE +++ mktemp ++ local LAST_ERR=/tmp/tmp.pFJKPQ90B9 ++ local exit_status=0 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl exec monitoring-0 -- bash -c 'printenv | grep ADMIN_PASSWORD | cut -d '\''='\'' -f2' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.zTV62GqUjE ++ cat /tmp/tmp.pFJKPQ90B9 ++ rm /tmp/tmp.zTV62GqUjE /tmp/tmp.pFJKPQ90B9 ++ return 0 + ADMIN_PASSWORD=admin + sleep 5 + kubectl_bin exec monitoring-0 -- bash -c 'grafana-cli --homepath=/usr/share/grafana --config=/etc/grafana/grafana.ini admin reset-admin-password admin' ++ mktemp + local LAST_OUT=/tmp/tmp.5aCKnnk3vN ++ mktemp + local LAST_ERR=/tmp/tmp.32nC2IpLVY + local exit_status=0 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl exec monitoring-0 -- bash -c 'grafana-cli --homepath=/usr/share/grafana --config=/etc/grafana/grafana.ini admin reset-admin-password admin' + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.5aCKnnk3vN logger=settings t=2026-10-05T13:21:25.314529556Z level=info msg="Starting Grafana" version= commit= branch= compiled=1970-01-01T00:00:00Z logger=settings t=2026-10-05T13:21:25.314656956Z level=info msg="Config loaded from" file=/usr/share/grafana/conf/defaults.ini logger=settings t=2026-10-05T13:21:25.314668156Z level=info msg="Config loaded from" file=/etc/grafana/grafana.ini logger=settings t=2026-10-05T13:21:25.314673756Z level=info msg="Path Home" path=/usr/share/grafana logger=settings t=2026-10-05T13:21:25.314678556Z level=info msg="Path Data" path=/srv/grafana logger=settings t=2026-10-05T13:21:25.314683266Z level=info msg="Path Logs" path=/srv/logs logger=settings t=2026-10-05T13:21:25.314687926Z level=info msg="Path Plugins" path=/srv/grafana/plugins logger=settings t=2026-10-05T13:21:25.314692516Z level=info msg="Path Provisioning" path=/usr/share/grafana/conf/provisioning logger=settings t=2026-10-05T13:21:25.314697076Z level=info msg="App mode production" logger=sqlstore t=2026-10-05T13:21:25.314757126Z level=info msg="Connecting to DB" dbtype=postgres logger=migrator t=2026-10-05T13:21:25.326949525Z level=info msg="Starting DB migrations" logger=migrator t=2026-10-05T13:21:25.330287234Z level=info msg="migrations completed" performed=0 skipped=452 duration=339.919µs logger=secrets t=2026-10-05T13:21:25.331871734Z level=info msg="Envelope encryption state" enabled=true currentprovider=secretKey.v1 logger=plugin.finder t=2026-10-05T13:21:25.361515472Z level=warn msg="Skipping finding plugins as directory does not exist" path=/usr/share/grafana/plugins-bundled logger=plugin.signature.validator t=2026-10-05T13:21:25.477725691Z level=warn msg="Permitting unsigned plugin. This is not recommended" pluginID=pmm-app pluginDir=/srv/grafana/plugins/pmm-app/dist logger=plugin.pmm-app t=2026-10-05T13:21:25.48342254Z level=warn msg="Included dashboard is missing a UID field" logger=plugin.signature.validator t=2026-10-05T13:21:25.4836239Z level=warn msg="Permitting unsigned plugin. This is not recommended" pluginID=grafana-polystat-panel pluginDir=/srv/grafana/plugins/grafana-polystat-panel logger=plugin.signature.validator t=2026-10-05T13:21:25.48364438Z level=warn msg="Permitting unsigned plugin. This is not recommended" pluginID=pmm-pt-summary-panel pluginDir=/srv/grafana/plugins/pmm-app/dist/pmm-pt-summary/panel logger=plugin.signature.validator t=2026-10-05T13:21:25.48369017Z level=warn msg="Permitting unsigned plugin. This is not recommended" pluginID=pmm-qan-app-panel pluginDir=/srv/grafana/plugins/pmm-app/dist/pmm-qan logger=plugin.signature.validator t=2026-10-05T13:21:25.4837515Z level=warn msg="Permitting unsigned plugin. This is not recommended" pluginID=pmm-pt-summary-datasource pluginDir=/srv/grafana/plugins/pmm-app/dist/pmm-pt-summary/datasource logger=plugin.loader t=2026-10-05T13:21:25.48383498Z level=info msg="Plugin registered" pluginID=pmm-app logger=plugin.loader t=2026-10-05T13:21:25.48384888Z level=info msg="Plugin registered" pluginID=camptocamp-prometheus-alertmanager-datasource logger=plugin.loader t=2026-10-05T13:21:25.48385459Z level=info msg="Plugin registered" pluginID=grafana-polystat-panel logger=plugin.loader t=2026-10-05T13:21:25.48385992Z level=info msg="Plugin registered" pluginID=pmm-pt-summary-panel logger=plugin.loader t=2026-10-05T13:21:25.48386585Z level=info msg="Plugin registered" pluginID=grafana-piechart-panel logger=plugin.loader t=2026-10-05T13:21:25.4838712Z level=info msg="Plugin registered" pluginID=grafana-worldmap-panel logger=plugin.loader t=2026-10-05T13:21:25.48387658Z level=info msg="Plugin registered" pluginID=jdbranham-diagram-panel logger=plugin.loader t=2026-10-05T13:21:25.48388168Z level=info msg="Plugin registered" pluginID=natel-discrete-panel logger=plugin.loader t=2026-10-05T13:21:25.48388596Z level=info msg="Plugin registered" pluginID=pmm-qan-app-panel logger=plugin.loader t=2026-10-05T13:21:25.48388964Z level=info msg="Plugin registered" pluginID=grafana-clickhouse-datasource logger=plugin.grafana-clickhouse-datasource t=2026-10-05T13:21:25.493678539Z level=warn msg="Plugin process is running with elevated privileges. This is not recommended" logger=plugin.loader t=2026-10-05T13:21:25.493693999Z level=info msg="Plugin registered" pluginID=petrslavotinek-carpetplot-panel logger=plugin.loader t=2026-10-05T13:21:25.493700449Z level=info msg="Plugin registered" pluginID=pmm-pt-summary-datasource Admin password changed successfully ✔ + cat /tmp/tmp.32nC2IpLVY + rm /tmp/tmp.5aCKnnk3vN /tmp/tmp.32nC2IpLVY + return 0 + desc 'create PXC cluster' + set +o xtrace ----------------------------------------------------------------------------------- create PXC cluster ----------------------------------------------------------------------------------- + spinup_pxc monitoring /mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/e2e-tests/monitoring-2-0/conf/monitoring.yml 3 120 /mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/e2e-tests/monitoring-2-0/conf/secrets.yaml + local cluster=monitoring + local config=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/e2e-tests/monitoring-2-0/conf/monitoring.yml + local size=3 + local sleep=120 + local secretsFile=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/e2e-tests/monitoring-2-0/conf/secrets.yaml + local pxcClientFile=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/e2e-tests/conf/client.yml + local port=3306 + desc 'create first PXC cluster' + set +o xtrace ----------------------------------------------------------------------------------- create first PXC cluster ----------------------------------------------------------------------------------- + kubectl_bin apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/e2e-tests/monitoring-2-0/conf/secrets.yaml ++ mktemp + local LAST_OUT=/tmp/tmp.SUG3Cj72YI ++ mktemp + local LAST_ERR=/tmp/tmp.jlmQf6Rf6k + local exit_status=0 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/e2e-tests/monitoring-2-0/conf/secrets.yaml + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.SUG3Cj72YI secret/my-cluster-secrets created + cat /tmp/tmp.jlmQf6Rf6k + rm /tmp/tmp.SUG3Cj72YI /tmp/tmp.jlmQf6Rf6k + return 0 + apply_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/e2e-tests/conf/client.yml + local config_file=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/e2e-tests/conf/client.yml + local pvc_name= + '[' -z '' ']' + kubectl_bin apply -f - + cat_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/e2e-tests/conf/client.yml '' + local input_file=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/e2e-tests/conf/client.yml + local pvc_name= + /usr/bin/sed -e 's#image:.*-proxysql$#image: perconalab/percona-xtradb-cluster-operator:main-proxysql#' + /usr/bin/sed -e 's#image:.*-backup$#image: perconalab/percona-xtradb-cluster-operator:main-pxc8.0-backup#' + /usr/bin/sed -e 's#image:.*-pmm$#image: perconalab/pmm-client:dev-latest#' + /usr/bin/sed -e 's#image:.*-logcollector$#image: perconalab/fluentbit:main-logcollector#' + /usr/bin/sed -e s~minio-service.#namespace~minio-service.monitoring-2-0-5888~ + /usr/bin/sed -e 's#apply:.*#apply: Never#' + /usr/bin/sed -e 's#claimName:..*-backup-pvc$#claimName: #' ++ mktemp + /usr/bin/sed -e 's#image:.*\/percona-xtradb-cluster:.*$#image: perconalab/percona-xtradb-cluster-operator:main-pxc8.0#' + /usr/bin/sed -e 's#apiVersion: pxc.percona.com/v.*$#apiVersion: pxc.percona.com/v1#' + local LAST_OUT=/tmp/tmp.1DJArQpdXT ++ mktemp + local LAST_ERR=/tmp/tmp.am4N1fEpIN + local exit_status=0 + cat /mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/e2e-tests/conf/client.yml + /usr/bin/sed -e 's#image:.*-init$#image: perconalab/percona-xtradb-cluster-operator:PR-2546-caf25420#' + /usr/bin/sed -e 's#image:.*-pxc\([0-9]*.[0-9]*\)\{0,1\}$#image: perconalab/percona-xtradb-cluster-operator:main-pxc8.0#' ++ seq 0 2 + /usr/bin/sed -e 's#image:.*-haproxy$#image: perconalab/percona-xtradb-cluster-operator:main-haproxy#' + for i in $(seq 0 2) + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.1DJArQpdXT deployment.apps/pxc-client created + cat /tmp/tmp.am4N1fEpIN + rm /tmp/tmp.1DJArQpdXT /tmp/tmp.am4N1fEpIN + return 0 + [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 5\.7 ]] + apply_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/e2e-tests/monitoring-2-0/conf/monitoring.yml + local config_file=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/e2e-tests/monitoring-2-0/conf/monitoring.yml + local pvc_name= + '[' -z '' ']' + kubectl_bin apply -f - + cat_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/e2e-tests/monitoring-2-0/conf/monitoring.yml '' + local input_file=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/e2e-tests/monitoring-2-0/conf/monitoring.yml + local pvc_name= + /usr/bin/sed -e 's#image:.*\/percona-xtradb-cluster:.*$#image: perconalab/percona-xtradb-cluster-operator:main-pxc8.0#' + cat /mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/e2e-tests/monitoring-2-0/conf/monitoring.yml + /usr/bin/sed -e 's#image:.*-init$#image: perconalab/percona-xtradb-cluster-operator:PR-2546-caf25420#' + /usr/bin/sed -e 's#image:.*-pmm$#image: perconalab/pmm-client:dev-latest#' + /usr/bin/sed -e 's#image:.*-backup$#image: perconalab/percona-xtradb-cluster-operator:main-pxc8.0-backup#' + /usr/bin/sed -e 's#apiVersion: pxc.percona.com/v.*$#apiVersion: pxc.percona.com/v1#' + /usr/bin/sed -e 's#image:.*-haproxy$#image: perconalab/percona-xtradb-cluster-operator:main-haproxy#' + /usr/bin/sed -e 's#image:.*-logcollector$#image: perconalab/fluentbit:main-logcollector#' + /usr/bin/sed -e s~minio-service.#namespace~minio-service.monitoring-2-0-5888~ + /usr/bin/sed -e 's#image:.*-proxysql$#image: perconalab/percona-xtradb-cluster-operator:main-proxysql#' ++ mktemp + /usr/bin/sed -e 's#apply:.*#apply: Never#' + /usr/bin/sed -e 's#claimName:..*-backup-pvc$#claimName: #' + /usr/bin/sed -e 's#image:.*-pxc\([0-9]*.[0-9]*\)\{0,1\}$#image: perconalab/percona-xtradb-cluster-operator:main-pxc8.0#' + local LAST_OUT=/tmp/tmp.KJNBbZd8Lf ++ mktemp + local LAST_ERR=/tmp/tmp.jjyugZ0Q9H + local exit_status=0 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.KJNBbZd8Lf perconaxtradbcluster.pxc.percona.com/monitoring created + cat /tmp/tmp.jjyugZ0Q9H + rm /tmp/tmp.KJNBbZd8Lf /tmp/tmp.jjyugZ0Q9H + return 0 + desc 'waiting pods to be running for monitoring' + set +o xtrace ----------------------------------------------------------------------------------- waiting pods to be running for monitoring ----------------------------------------------------------------------------------- ++ get_proxy monitoring ++ local target_cluster=monitoring +++ kubectl_bin get pxc monitoring -o 'jsonpath={.spec.haproxy.enabled}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.RZaDp9ynLY ++++ mktemp +++ local LAST_ERR=/tmp/tmp.QKGx1QfWMh +++ local exit_status=0 ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl get pxc monitoring -o 'jsonpath={.spec.haproxy.enabled}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 ']' +++ break +++ cat /tmp/tmp.RZaDp9ynLY +++ cat /tmp/tmp.QKGx1QfWMh +++ rm /tmp/tmp.RZaDp9ynLY /tmp/tmp.QKGx1QfWMh +++ return 0 ++ [[ true == \t\r\u\e ]] ++ echo monitoring-haproxy ++ return + local proxy=monitoring-haproxy ++ get_proxy_size monitoring ++ local cluster=monitoring ++ local size=0 +++ kubectl_bin get pxc monitoring -o 'jsonpath={.spec.haproxy.enabled}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.rL0pK7BgPA ++++ mktemp +++ local LAST_ERR=/tmp/tmp.o3iUGsdbJF +++ local exit_status=0 ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl get pxc monitoring -o 'jsonpath={.spec.haproxy.enabled}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 ']' +++ break +++ cat /tmp/tmp.rL0pK7BgPA +++ cat /tmp/tmp.o3iUGsdbJF +++ rm /tmp/tmp.rL0pK7BgPA /tmp/tmp.o3iUGsdbJF +++ return 0 ++ [[ true == \t\r\u\e ]] +++ kubectl_bin get pxc monitoring -o 'jsonpath={.spec.haproxy.size}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.zR517o3tcm ++++ mktemp +++ local LAST_ERR=/tmp/tmp.RR79plqvOY +++ local exit_status=0 ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl get pxc monitoring -o 'jsonpath={.spec.haproxy.size}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 ']' +++ break +++ cat /tmp/tmp.zR517o3tcm +++ cat /tmp/tmp.RR79plqvOY +++ rm /tmp/tmp.zR517o3tcm /tmp/tmp.RR79plqvOY +++ return 0 ++ size=2 ++ echo 2 + local proxy_size=2 + wait_for_running monitoring-pxc 3 + local name=monitoring-pxc + local size=3 + local last_pod + local max_retry=480 + [[ -z monitoring-pxc ]] + [[ 3 -le 0 ]] + last_pod=2 + desc 'Waiting for 3 running pods for monitoring-pxc' + set +o xtrace ----------------------------------------------------------------------------------- Waiting for 3 running pods for monitoring-pxc ----------------------------------------------------------------------------------- ++ seq 0 2 + for i in $(seq 0 $last_pod) + wait_pod monitoring-pxc-0 480 + local pod=monitoring-pxc-0 + local max_retry=480 + local ns= ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ echo monitoring-pxc-0 ++ grep -E '^(pxc|proxysql)$' + local container=pxc + set +o xtrace pod/monitoring-pxc-0 condition met waiting for pod/monitoring-pxc-0 to become Ready.Ok + for i in $(seq 0 $last_pod) + wait_pod monitoring-pxc-1 480 + local pod=monitoring-pxc-1 + local max_retry=480 + local ns= ++ echo monitoring-pxc-1 ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ grep -E '^(pxc|proxysql)$' + local container=pxc + set +o xtrace pod/monitoring-pxc-1 condition met waiting for pod/monitoring-pxc-1 to become Ready.Ok + for i in $(seq 0 $last_pod) + wait_pod monitoring-pxc-2 480 + local pod=monitoring-pxc-2 + local max_retry=480 + local ns= ++ echo monitoring-pxc-2 ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ grep -E '^(pxc|proxysql)$' + local container=pxc + set +o xtrace pod/monitoring-pxc-2 condition met waiting for pod/monitoring-pxc-2 to become Ready.Ok + wait_for_running monitoring-haproxy 2 + local name=monitoring-haproxy + local size=2 + local last_pod + local max_retry=480 + [[ -z monitoring-haproxy ]] + [[ 2 -le 0 ]] + last_pod=1 + desc 'Waiting for 2 running pods for monitoring-haproxy' + set +o xtrace ----------------------------------------------------------------------------------- Waiting for 2 running pods for monitoring-haproxy ----------------------------------------------------------------------------------- ++ seq 0 1 + for i in $(seq 0 $last_pod) + wait_pod monitoring-haproxy-0 480 + local pod=monitoring-haproxy-0 + local max_retry=480 + local ns= ++ echo monitoring-haproxy-0 ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ grep -E '^(pxc|proxysql)$' + local container= + set +o xtrace pod/monitoring-haproxy-0 condition met waiting for pod/monitoring-haproxy-0 to become Ready.Ok + for i in $(seq 0 $last_pod) + wait_pod monitoring-haproxy-1 480 + local pod=monitoring-haproxy-1 + local max_retry=480 + local ns= ++ echo monitoring-haproxy-1 ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ grep -E '^(pxc|proxysql)$' + local container= + set +o xtrace pod/monitoring-haproxy-1 condition met waiting for pod/monitoring-haproxy-1 to become Ready.Ok + wait_cluster_consistency monitoring 3 2 + local cluster_name=monitoring + local pxc_size=3 + local proxy_size=2 + desc 'wait cluster consistency' + set +o xtrace ----------------------------------------------------------------------------------- wait cluster consistency ----------------------------------------------------------------------------------- + local max_seconds=1500 + local delay=5 + local retries=300 + local pxc_ready + local proxy_ready + sleep 7 + echo -n 'waiting for pxc/monitoring to be ready' waiting for pxc/monitoring to be ready++ seq 1 300 + for _ in $(seq 1 "${retries}") ++ kubectl_bin get pxc monitoring -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.SaKcmxzEXv +++ mktemp ++ local LAST_ERR=/tmp/tmp.M2uMkEU2uU ++ local exit_status=0 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pxc monitoring -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.SaKcmxzEXv ++ cat /tmp/tmp.M2uMkEU2uU ++ rm /tmp/tmp.SaKcmxzEXv /tmp/tmp.M2uMkEU2uU ++ return 0 + state=ready ++ kubectl_bin get pxc monitoring -o 'jsonpath={.status.pxc.ready}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.8n6Q9Y3NoB +++ mktemp ++ local LAST_ERR=/tmp/tmp.xo7oglLw76 ++ local exit_status=0 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pxc monitoring -o 'jsonpath={.status.pxc.ready}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.8n6Q9Y3NoB ++ cat /tmp/tmp.xo7oglLw76 ++ rm /tmp/tmp.8n6Q9Y3NoB /tmp/tmp.xo7oglLw76 ++ return 0 + pxc_ready=3 + proxy_ready=0 + [[ 2 -gt 0 ]] +++ get_proxy_engine monitoring +++ local cluster_name=monitoring ++++ get_proxy monitoring ++++ local target_cluster=monitoring +++++ kubectl_bin get pxc monitoring -o 'jsonpath={.spec.haproxy.enabled}' ++++++ mktemp +++++ local LAST_OUT=/tmp/tmp.6YoFcDaz83 ++++++ mktemp +++++ local LAST_ERR=/tmp/tmp.tpHpq99ezn +++++ local exit_status=0 ++++++ seq 0 2 +++++ for i in $(seq 0 2) +++++ set +e +++++ kubectl get pxc monitoring -o 'jsonpath={.spec.haproxy.enabled}' +++++ exit_status=0 +++++ set -e +++++ '[' 0 '!=' 0 ']' +++++ break +++++ cat /tmp/tmp.6YoFcDaz83 +++++ cat /tmp/tmp.tpHpq99ezn +++++ rm /tmp/tmp.6YoFcDaz83 /tmp/tmp.tpHpq99ezn +++++ return 0 ++++ [[ true == \t\r\u\e ]] ++++ echo monitoring-haproxy ++++ return +++ local cluster_proxy=monitoring-haproxy +++ echo haproxy ++ kubectl_bin get pxc monitoring -o 'jsonpath={.status.haproxy.ready}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.9W86mSTfte +++ mktemp ++ local LAST_ERR=/tmp/tmp.GB0rlVZjly ++ local exit_status=0 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pxc monitoring -o 'jsonpath={.status.haproxy.ready}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.9W86mSTfte ++ cat /tmp/tmp.GB0rlVZjly ++ rm /tmp/tmp.9W86mSTfte /tmp/tmp.GB0rlVZjly ++ return 0 + proxy_ready=2 + [[ ready == \r\e\a\d\y ]] + [[ 3 == \3 ]] + [[ 2 == \2 ]] + echo + return + sleep 120 ++ kubectl get pxc monitoring -o 'jsonpath={.spec.secretsName}' + local secret_name=my-cluster-secrets ++ getSecretData my-cluster-secrets root ++ local secretName=my-cluster-secrets ++ local dataKey=root ++ base64 --decode ++ kubectl_bin get secrets/my-cluster-secrets '--template={{.data.root}}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.fDzVQgxR24 +++ mktemp ++ local LAST_ERR=/tmp/tmp.BbCGPKKERL ++ local exit_status=0 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get secrets/my-cluster-secrets '--template={{.data.root}}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.fDzVQgxR24 ++ cat /tmp/tmp.BbCGPKKERL ++ rm /tmp/tmp.fDzVQgxR24 /tmp/tmp.BbCGPKKERL ++ return 0 + local root_pass=root_password + desc 'write data' + set +o xtrace ----------------------------------------------------------------------------------- write data ----------------------------------------------------------------------------------- + [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 5\.7 ]] + run_mysql 'CREATE DATABASE IF NOT EXISTS myApp; use myApp; CREATE TABLE IF NOT EXISTS myApp (id int PRIMARY KEY) ;' '-h monitoring-pxc-0.monitoring-pxc -uroot -p'\''root_password'\'' -P3306' + local 'command=CREATE DATABASE IF NOT EXISTS myApp; use myApp; CREATE TABLE IF NOT EXISTS myApp (id int PRIMARY KEY) ;' + local 'uri=-h monitoring-pxc-0.monitoring-pxc -uroot -p'\''root_password'\'' -P3306' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.ch7C9U5Qs4 +++ mktemp ++ local LAST_ERR=/tmp/tmp.fn39pfYCUG ++ local exit_status=0 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.ch7C9U5Qs4 ++ cat /tmp/tmp.fn39pfYCUG ++ rm /tmp/tmp.ch7C9U5Qs4 /tmp/tmp.fn39pfYCUG ++ return 0 + client_pod=pxc-client-6fc75f888d-cqd6g + wait_pod pxc-client-6fc75f888d-cqd6g + local pod=pxc-client-6fc75f888d-cqd6g + local max_retry=480 + local ns= ++ grep -E '^(pxc|proxysql)$' ++ echo pxc-client-6fc75f888d-cqd6g ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' + local container= + set +o xtrace pod/pxc-client-6fc75f888d-cqd6g condition met waiting for pod/pxc-client-6fc75f888d-cqd6g to become ReadyDefaulted container "pxc-client" out of: pxc-client, backup .Ok + set +o xtrace + write_data monitoring 100500 no-proxy -P3306 + local cluster=monitoring + local id=100500 + local use_proxy=no-proxy + local port=-P3306 + local database=myApp + local table=myApp + local root_pass ++ get_password monitoring root ++ local cluster=monitoring ++ local user=root +++ kubectl_bin get pxc monitoring -o 'jsonpath={.spec.secretsName}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.E8V2uzRSPQ ++++ mktemp +++ local LAST_ERR=/tmp/tmp.FpNeUXEKa7 +++ local exit_status=0 ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl get pxc monitoring -o 'jsonpath={.spec.secretsName}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 ']' +++ break +++ cat /tmp/tmp.E8V2uzRSPQ +++ cat /tmp/tmp.FpNeUXEKa7 +++ rm /tmp/tmp.E8V2uzRSPQ /tmp/tmp.FpNeUXEKa7 +++ return 0 ++ getSecretData my-cluster-secrets root ++ local secretName=my-cluster-secrets ++ local dataKey=root ++ base64 --decode ++ kubectl_bin get secrets/my-cluster-secrets '--template={{.data.root}}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.pau3Ff0bbQ +++ mktemp ++ local LAST_ERR=/tmp/tmp.ImeVQKDMh6 ++ local exit_status=0 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get secrets/my-cluster-secrets '--template={{.data.root}}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.pau3Ff0bbQ ++ cat /tmp/tmp.ImeVQKDMh6 ++ rm /tmp/tmp.pau3Ff0bbQ /tmp/tmp.ImeVQKDMh6 ++ return 0 + root_pass=root_password ++ get_proxy monitoring ++ local target_cluster=monitoring +++ kubectl_bin get pxc monitoring -o 'jsonpath={.spec.haproxy.enabled}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.XNsO7BBvDK ++++ mktemp +++ local LAST_ERR=/tmp/tmp.7DnTgsH07x +++ local exit_status=0 ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl get pxc monitoring -o 'jsonpath={.spec.haproxy.enabled}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 ']' +++ break +++ cat /tmp/tmp.XNsO7BBvDK +++ cat /tmp/tmp.7DnTgsH07x +++ rm /tmp/tmp.XNsO7BBvDK /tmp/tmp.7DnTgsH07x +++ return 0 ++ [[ true == \t\r\u\e ]] ++ echo monitoring-haproxy ++ return + proxy=monitoring-haproxy + [[ no-proxy == true ]] + [[ no-proxy == \u\s\e\-\p\r\o\x\y ]] + host='-h monitoring-pxc-0.monitoring-pxc' + run_mysql 'INSERT IGNORE myApp.myApp (id) VALUES (100500)' '-h monitoring-pxc-0.monitoring-pxc -P3306 -uroot -p'\''root_password'\''' + local 'command=INSERT IGNORE myApp.myApp (id) VALUES (100500)' + local 'uri=-h monitoring-pxc-0.monitoring-pxc -P3306 -uroot -p'\''root_password'\''' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.DvSFcYlYJa +++ mktemp ++ local LAST_ERR=/tmp/tmp.MAlpqL93FS ++ local exit_status=0 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.DvSFcYlYJa ++ cat /tmp/tmp.MAlpqL93FS ++ rm /tmp/tmp.DvSFcYlYJa /tmp/tmp.MAlpqL93FS ++ return 0 + client_pod=pxc-client-6fc75f888d-cqd6g + wait_pod pxc-client-6fc75f888d-cqd6g + local pod=pxc-client-6fc75f888d-cqd6g + local max_retry=480 + local ns= ++ echo pxc-client-6fc75f888d-cqd6g ++ grep -E '^(pxc|proxysql)$' ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' + local container= + set +o xtrace pod/pxc-client-6fc75f888d-cqd6g condition met waiting for pod/pxc-client-6fc75f888d-cqd6g to become ReadyDefaulted container "pxc-client" out of: pxc-client, backup .Ok + set +o xtrace + sleep 30 + desc 'compare initial data' + set +o xtrace ----------------------------------------------------------------------------------- compare initial data ----------------------------------------------------------------------------------- + compare_data monitoring select-1 no-proxy -P3306 + local cluster=monitoring + local compare=select-1 + local use_proxy=no-proxy + local port=-P3306 + local database=myApp + local table=myApp + local size + local root_pass ++ get_password monitoring root ++ local cluster=monitoring ++ local user=root +++ kubectl_bin get pxc monitoring -o 'jsonpath={.spec.secretsName}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.be8vfRXe66 ++++ mktemp +++ local LAST_ERR=/tmp/tmp.jVHCjVwINb +++ local exit_status=0 ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl get pxc monitoring -o 'jsonpath={.spec.secretsName}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 ']' +++ break +++ cat /tmp/tmp.be8vfRXe66 +++ cat /tmp/tmp.jVHCjVwINb +++ rm /tmp/tmp.be8vfRXe66 /tmp/tmp.jVHCjVwINb +++ return 0 ++ getSecretData my-cluster-secrets root ++ local secretName=my-cluster-secrets ++ local dataKey=root ++ kubectl_bin get secrets/my-cluster-secrets '--template={{.data.root}}' ++ base64 --decode +++ mktemp ++ local LAST_OUT=/tmp/tmp.uyo9E4wpy3 +++ mktemp ++ local LAST_ERR=/tmp/tmp.vAFIzs3Uxj ++ local exit_status=0 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get secrets/my-cluster-secrets '--template={{.data.root}}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.uyo9E4wpy3 ++ cat /tmp/tmp.vAFIzs3Uxj ++ rm /tmp/tmp.uyo9E4wpy3 /tmp/tmp.vAFIzs3Uxj ++ return 0 + root_pass=root_password + [[ no-proxy == true ]] + [[ no-proxy == \u\s\e\-\p\r\o\x\y ]] ++ get_pxc_size monitoring ++ local cluster=monitoring ++ local size=0 +++ kubectl_bin get pxc monitoring -o 'jsonpath={.spec.pxc.size}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.2fPMd7bsWj ++++ mktemp +++ local LAST_ERR=/tmp/tmp.p7aphJHZ7J +++ local exit_status=0 ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl get pxc monitoring -o 'jsonpath={.spec.pxc.size}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 ']' +++ break +++ cat /tmp/tmp.2fPMd7bsWj +++ cat /tmp/tmp.p7aphJHZ7J +++ rm /tmp/tmp.2fPMd7bsWj /tmp/tmp.p7aphJHZ7J +++ return 0 ++ size=3 ++ echo 3 + size=3 ++ seq 0 2 + for i in $(seq 0 $((size - 1))) + compare_mysql_cmd select-1 'SELECT id FROM myApp.myApp ORDER BY id;' '-h monitoring-pxc-0.monitoring-pxc -P3306 -uroot -p'\''root_password'\''' + local command_id=select-1 + local 'command=SELECT id FROM myApp.myApp ORDER BY id;' + local 'uri=-h monitoring-pxc-0.monitoring-pxc -P3306 -uroot -p'\''root_password'\''' + local postfix= + local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/e2e-tests/monitoring-2-0/compare/select-1.sql + [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 8\.4 ]] + [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 8\.0 ]] + [[ -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/e2e-tests/monitoring-2-0/compare/select-1-80.sql ]] + [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 5\.7 ]] + run_mysql 'SELECT id FROM myApp.myApp ORDER BY id;' '-h monitoring-pxc-0.monitoring-pxc -P3306 -uroot -p'\''root_password'\''' + local 'command=SELECT id FROM myApp.myApp ORDER BY id;' + local 'uri=-h monitoring-pxc-0.monitoring-pxc -P3306 -uroot -p'\''root_password'\''' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.a1i2ODqGqT +++ mktemp ++ local LAST_ERR=/tmp/tmp.AzkUBog3Sr ++ local exit_status=0 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.a1i2ODqGqT ++ cat /tmp/tmp.AzkUBog3Sr ++ rm /tmp/tmp.a1i2ODqGqT /tmp/tmp.AzkUBog3Sr ++ return 0 + client_pod=pxc-client-6fc75f888d-cqd6g + wait_pod pxc-client-6fc75f888d-cqd6g + local pod=pxc-client-6fc75f888d-cqd6g + local max_retry=480 + local ns= ++ echo pxc-client-6fc75f888d-cqd6g ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ grep -E '^(pxc|proxysql)$' + local container= + set +o xtrace pod/pxc-client-6fc75f888d-cqd6g condition met waiting for pod/pxc-client-6fc75f888d-cqd6g to become ReadyDefaulted container "pxc-client" out of: pxc-client, backup .Ok + set +o xtrace + [[ ! -s /tmp/tmp.oE2g1q4ApT/select-1.sql ]] ++ grep 'Unknown MySQL server host' /tmp/tmp.oE2g1q4ApT/select-1.sql + [[ -n '' ]] + [[ 0 -eq 0 ]] + diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/e2e-tests/monitoring-2-0/compare/select-1.sql /tmp/tmp.oE2g1q4ApT/select-1.sql + for i in $(seq 0 $((size - 1))) + compare_mysql_cmd select-1 'SELECT id FROM myApp.myApp ORDER BY id;' '-h monitoring-pxc-1.monitoring-pxc -P3306 -uroot -p'\''root_password'\''' + local command_id=select-1 + local 'command=SELECT id FROM myApp.myApp ORDER BY id;' + local 'uri=-h monitoring-pxc-1.monitoring-pxc -P3306 -uroot -p'\''root_password'\''' + local postfix= + local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/e2e-tests/monitoring-2-0/compare/select-1.sql + [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 8\.4 ]] + [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 8\.0 ]] + [[ -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/e2e-tests/monitoring-2-0/compare/select-1-80.sql ]] + [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 5\.7 ]] + run_mysql 'SELECT id FROM myApp.myApp ORDER BY id;' '-h monitoring-pxc-1.monitoring-pxc -P3306 -uroot -p'\''root_password'\''' + local 'command=SELECT id FROM myApp.myApp ORDER BY id;' + local 'uri=-h monitoring-pxc-1.monitoring-pxc -P3306 -uroot -p'\''root_password'\''' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.CY35Flej5H +++ mktemp ++ local LAST_ERR=/tmp/tmp.4wx0DjDiGf ++ local exit_status=0 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.CY35Flej5H ++ cat /tmp/tmp.4wx0DjDiGf ++ rm /tmp/tmp.CY35Flej5H /tmp/tmp.4wx0DjDiGf ++ return 0 + client_pod=pxc-client-6fc75f888d-cqd6g + wait_pod pxc-client-6fc75f888d-cqd6g + local pod=pxc-client-6fc75f888d-cqd6g + local max_retry=480 + local ns= ++ echo pxc-client-6fc75f888d-cqd6g ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ grep -E '^(pxc|proxysql)$' + local container= + set +o xtrace pod/pxc-client-6fc75f888d-cqd6g condition met waiting for pod/pxc-client-6fc75f888d-cqd6g to become ReadyDefaulted container "pxc-client" out of: pxc-client, backup .Ok + set +o xtrace + [[ ! -s /tmp/tmp.oE2g1q4ApT/select-1.sql ]] ++ grep 'Unknown MySQL server host' /tmp/tmp.oE2g1q4ApT/select-1.sql + [[ -n '' ]] + [[ 0 -eq 0 ]] + diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/e2e-tests/monitoring-2-0/compare/select-1.sql /tmp/tmp.oE2g1q4ApT/select-1.sql + for i in $(seq 0 $((size - 1))) + compare_mysql_cmd select-1 'SELECT id FROM myApp.myApp ORDER BY id;' '-h monitoring-pxc-2.monitoring-pxc -P3306 -uroot -p'\''root_password'\''' + local command_id=select-1 + local 'command=SELECT id FROM myApp.myApp ORDER BY id;' + local 'uri=-h monitoring-pxc-2.monitoring-pxc -P3306 -uroot -p'\''root_password'\''' + local postfix= + local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/e2e-tests/monitoring-2-0/compare/select-1.sql + [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 8\.4 ]] + [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 8\.0 ]] + [[ -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/e2e-tests/monitoring-2-0/compare/select-1-80.sql ]] + [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 5\.7 ]] + run_mysql 'SELECT id FROM myApp.myApp ORDER BY id;' '-h monitoring-pxc-2.monitoring-pxc -P3306 -uroot -p'\''root_password'\''' + local 'command=SELECT id FROM myApp.myApp ORDER BY id;' + local 'uri=-h monitoring-pxc-2.monitoring-pxc -P3306 -uroot -p'\''root_password'\''' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.JLiLITTy8N +++ mktemp ++ local LAST_ERR=/tmp/tmp.ZeNY6WzXWi ++ local exit_status=0 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.JLiLITTy8N ++ cat /tmp/tmp.ZeNY6WzXWi ++ rm /tmp/tmp.JLiLITTy8N /tmp/tmp.ZeNY6WzXWi ++ return 0 + client_pod=pxc-client-6fc75f888d-cqd6g + wait_pod pxc-client-6fc75f888d-cqd6g + local pod=pxc-client-6fc75f888d-cqd6g + local max_retry=480 + local ns= ++ grep -E '^(pxc|proxysql)$' ++ echo pxc-client-6fc75f888d-cqd6g ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' + local container= + set +o xtrace pod/pxc-client-6fc75f888d-cqd6g condition met waiting for pod/pxc-client-6fc75f888d-cqd6g to become ReadyDefaulted container "pxc-client" out of: pxc-client, backup .Ok + set +o xtrace + [[ ! -s /tmp/tmp.oE2g1q4ApT/select-1.sql ]] ++ grep 'Unknown MySQL server host' /tmp/tmp.oE2g1q4ApT/select-1.sql + [[ -n '' ]] + [[ 0 -eq 0 ]] + diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/e2e-tests/monitoring-2-0/compare/select-1.sql /tmp/tmp.oE2g1q4ApT/select-1.sql + is_keyring_plugin_in_use monitoring + local cluster=monitoring + [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 8\.4 ]] + grep -E -o 'early-plugin-load=keyring_\w+.so' + kubectl exec monitoring-pxc-0 -c pxc -- bash -c 'cat /etc/mysql/node.cnf' + return 1 + desc 'add PMM API key to secret' + set +o xtrace ----------------------------------------------------------------------------------- add PMM API key to secret ----------------------------------------------------------------------------------- ++ jq .key +++ get_service_endpoint monitoring-service +++ local service=monitoring-service ++++ kubectl_bin get service/monitoring-service -o 'jsonpath={.status.loadBalancer.ingress[].hostname}' +++++ mktemp ++++ local LAST_OUT=/tmp/tmp.m06I0hlhPG +++++ mktemp ++++ local LAST_ERR=/tmp/tmp.4PVKiK81QP ++++ local exit_status=0 +++++ seq 0 2 ++++ for i in $(seq 0 2) ++++ set +e ++++ kubectl get service/monitoring-service -o 'jsonpath={.status.loadBalancer.ingress[].hostname}' ++++ exit_status=0 ++++ set -e ++++ '[' 0 '!=' 0 ']' ++++ break ++++ cat /tmp/tmp.m06I0hlhPG ++++ cat /tmp/tmp.4PVKiK81QP ++++ rm /tmp/tmp.m06I0hlhPG /tmp/tmp.4PVKiK81QP ++++ return 0 +++ local endpoint= +++ '[' -z '' ']' ++++ kubectl_bin get service/monitoring-service -o 'jsonpath={.status.loadBalancer.ingress[].ip}' +++++ mktemp ++++ local LAST_OUT=/tmp/tmp.jlzZccrvSj +++++ mktemp ++++ local LAST_ERR=/tmp/tmp.BSe5lMLY96 ++++ local exit_status=0 +++++ seq 0 2 ++++ for i in $(seq 0 2) ++++ set +e ++++ kubectl get service/monitoring-service -o 'jsonpath={.status.loadBalancer.ingress[].ip}' ++++ exit_status=0 ++++ set -e ++++ '[' 0 '!=' 0 ']' ++++ break ++++ cat /tmp/tmp.jlzZccrvSj ++++ cat /tmp/tmp.BSe5lMLY96 ++++ rm /tmp/tmp.jlzZccrvSj /tmp/tmp.BSe5lMLY96 ++++ return 0 +++ endpoint=34.71.161.14 +++ '[' -n 34.71.161.14 ']' +++ '[' 34.71.161.14 '!=' null ']' +++ echo 34.71.161.14 +++ head -n 1 +++ sed -e 's/^"//; s/"$//;' +++ return ++ curl --insecure -X POST -H 'Content-Type: application/json' -d '{"name":"operator", "role": "Admin"}' https://admin:admin@34.71.161.14/graph/api/auth/keys % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 155 100 119 100 36 354 107 --:--:-- --:--:-- --:--:-- 461 + API_KEY='"eyJrIjoiSWk3NmlzZUJscXJreHhLenBTRFQ3WTRoU2xsZVBrQW4iLCJuIjoib3BlcmF0b3IiLCJpZCI6MX0="' + kubectl_bin patch secret my-cluster-secrets --type merge --patch '{"stringData": {"pmmserverkey": "eyJrIjoiSWk3NmlzZUJscXJreHhLenBTRFQ3WTRoU2xsZVBrQW4iLCJuIjoib3BlcmF0b3IiLCJpZCI6MX0="}}' ++ mktemp + local LAST_OUT=/tmp/tmp.iS4d1BJfMj ++ mktemp + local LAST_ERR=/tmp/tmp.56pElkWAuZ + local exit_status=0 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl patch secret my-cluster-secrets --type merge --patch '{"stringData": {"pmmserverkey": "eyJrIjoiSWk3NmlzZUJscXJreHhLenBTRFQ3WTRoU2xsZVBrQW4iLCJuIjoib3BlcmF0b3IiLCJpZCI6MX0="}}' + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.iS4d1BJfMj secret/my-cluster-secrets patched + cat /tmp/tmp.56pElkWAuZ + rm /tmp/tmp.iS4d1BJfMj /tmp/tmp.56pElkWAuZ + return 0 + wait_for_generation sts/monitoring-pxc 2 + local resource=sts/monitoring-pxc + local target_generation=2 + echo 'Waiting for sts/monitoring-pxc to reach generation 2...' Waiting for sts/monitoring-pxc to reach generation 2... + true ++ kubectl get sts/monitoring-pxc -o 'jsonpath={.metadata.generation}' + current_generation=1 + '[' 1 -eq 2 ']' + echo 'Resource sts/monitoring-pxc is at generation 1. Waiting...' Resource sts/monitoring-pxc is at generation 1. Waiting... + sleep 5 + true ++ kubectl get sts/monitoring-pxc -o 'jsonpath={.metadata.generation}' + current_generation=2 + '[' 2 -eq 2 ']' + echo 'Resource sts/monitoring-pxc has reached generation 2.' Resource sts/monitoring-pxc has reached generation 2. + break + wait_for_generation sts/monitoring-haproxy 2 + local resource=sts/monitoring-haproxy + local target_generation=2 + echo 'Waiting for sts/monitoring-haproxy to reach generation 2...' Waiting for sts/monitoring-haproxy to reach generation 2... + true ++ kubectl get sts/monitoring-haproxy -o 'jsonpath={.metadata.generation}' + current_generation=2 + '[' 2 -eq 2 ']' + echo 'Resource sts/monitoring-haproxy has reached generation 2.' Resource sts/monitoring-haproxy has reached generation 2. + break + sleep 10 + kubectl wait pod -l app.kubernetes.io/managed-by=percona-xtradb-cluster-operator --for=condition=ready --timeout=600s pod/monitoring-haproxy-0 condition met pod/monitoring-haproxy-1 condition met pod/monitoring-pxc-0 condition met pod/monitoring-pxc-1 condition met pod/monitoring-pxc-2 condition met + wait_cluster_consistency monitoring 3 2 + local cluster_name=monitoring + local pxc_size=3 + local proxy_size=2 + desc 'wait cluster consistency' + set +o xtrace ----------------------------------------------------------------------------------- wait cluster consistency ----------------------------------------------------------------------------------- + local max_seconds=1500 + local delay=5 + local retries=300 + local pxc_ready + local proxy_ready + sleep 7 + echo -n 'waiting for pxc/monitoring to be ready' waiting for pxc/monitoring to be ready++ seq 1 300 + for _ in $(seq 1 "${retries}") ++ kubectl_bin get pxc monitoring -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.ZktynzuhPA +++ mktemp ++ local LAST_ERR=/tmp/tmp.TCy6ZT0pGN ++ local exit_status=0 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pxc monitoring -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.ZktynzuhPA ++ cat /tmp/tmp.TCy6ZT0pGN ++ rm /tmp/tmp.ZktynzuhPA /tmp/tmp.TCy6ZT0pGN ++ return 0 + state=initializing ++ kubectl_bin get pxc monitoring -o 'jsonpath={.status.pxc.ready}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.tlWOWVysMz +++ mktemp ++ local LAST_ERR=/tmp/tmp.GlNh7scQ82 ++ local exit_status=0 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pxc monitoring -o 'jsonpath={.status.pxc.ready}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.tlWOWVysMz ++ cat /tmp/tmp.GlNh7scQ82 ++ rm /tmp/tmp.tlWOWVysMz /tmp/tmp.GlNh7scQ82 ++ return 0 + pxc_ready=1 + proxy_ready=0 + [[ 2 -gt 0 ]] +++ get_proxy_engine monitoring +++ local cluster_name=monitoring ++++ get_proxy monitoring ++++ local target_cluster=monitoring +++++ kubectl_bin get pxc monitoring -o 'jsonpath={.spec.haproxy.enabled}' ++++++ mktemp +++++ local LAST_OUT=/tmp/tmp.PBDk5OWMEG ++++++ mktemp +++++ local LAST_ERR=/tmp/tmp.V3AV0eUTjN +++++ local exit_status=0 ++++++ seq 0 2 +++++ for i in $(seq 0 2) +++++ set +e +++++ kubectl get pxc monitoring -o 'jsonpath={.spec.haproxy.enabled}' +++++ exit_status=0 +++++ set -e +++++ '[' 0 '!=' 0 ']' +++++ break +++++ cat /tmp/tmp.PBDk5OWMEG +++++ cat /tmp/tmp.V3AV0eUTjN +++++ rm /tmp/tmp.PBDk5OWMEG /tmp/tmp.V3AV0eUTjN +++++ return 0 ++++ [[ true == \t\r\u\e ]] ++++ echo monitoring-haproxy ++++ return +++ local cluster_proxy=monitoring-haproxy +++ echo haproxy ++ kubectl_bin get pxc monitoring -o 'jsonpath={.status.haproxy.ready}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.20hDRC0FBs +++ mktemp ++ local LAST_ERR=/tmp/tmp.VGAtWidWRQ ++ local exit_status=0 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pxc monitoring -o 'jsonpath={.status.haproxy.ready}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.20hDRC0FBs ++ cat /tmp/tmp.VGAtWidWRQ ++ rm /tmp/tmp.20hDRC0FBs /tmp/tmp.VGAtWidWRQ ++ return 0 + proxy_ready=2 + [[ initializing == \r\e\a\d\y ]] + echo -n . .+ sleep 5 + for _ in $(seq 1 "${retries}") ++ kubectl_bin get pxc monitoring -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.ZKzUJCSaTB +++ mktemp ++ local LAST_ERR=/tmp/tmp.VLrPJSu9KL ++ local exit_status=0 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pxc monitoring -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.ZKzUJCSaTB ++ cat /tmp/tmp.VLrPJSu9KL ++ rm /tmp/tmp.ZKzUJCSaTB /tmp/tmp.VLrPJSu9KL ++ return 0 + state=initializing ++ kubectl_bin get pxc monitoring -o 'jsonpath={.status.pxc.ready}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.3m3bhaeaqK +++ mktemp ++ local LAST_ERR=/tmp/tmp.ZZOiXIWTYz ++ local exit_status=0 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pxc monitoring -o 'jsonpath={.status.pxc.ready}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.3m3bhaeaqK ++ cat /tmp/tmp.ZZOiXIWTYz ++ rm /tmp/tmp.3m3bhaeaqK /tmp/tmp.ZZOiXIWTYz ++ return 0 + pxc_ready=1 + proxy_ready=0 + [[ 2 -gt 0 ]] +++ get_proxy_engine monitoring +++ local cluster_name=monitoring ++++ get_proxy monitoring ++++ local target_cluster=monitoring +++++ kubectl_bin get pxc monitoring -o 'jsonpath={.spec.haproxy.enabled}' ++++++ mktemp +++++ local LAST_OUT=/tmp/tmp.qs5CCawRhy ++++++ mktemp +++++ local LAST_ERR=/tmp/tmp.6OZBVmZAB6 +++++ local exit_status=0 ++++++ seq 0 2 +++++ for i in $(seq 0 2) +++++ set +e +++++ kubectl get pxc monitoring -o 'jsonpath={.spec.haproxy.enabled}' +++++ exit_status=0 +++++ set -e +++++ '[' 0 '!=' 0 ']' +++++ break +++++ cat /tmp/tmp.qs5CCawRhy +++++ cat /tmp/tmp.6OZBVmZAB6 +++++ rm /tmp/tmp.qs5CCawRhy /tmp/tmp.6OZBVmZAB6 +++++ return 0 ++++ [[ true == \t\r\u\e ]] ++++ echo monitoring-haproxy ++++ return +++ local cluster_proxy=monitoring-haproxy +++ echo haproxy ++ kubectl_bin get pxc monitoring -o 'jsonpath={.status.haproxy.ready}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.jOysiAbfqE +++ mktemp ++ local LAST_ERR=/tmp/tmp.mYoeOnS61A ++ local exit_status=0 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pxc monitoring -o 'jsonpath={.status.haproxy.ready}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.jOysiAbfqE ++ cat /tmp/tmp.mYoeOnS61A ++ rm /tmp/tmp.jOysiAbfqE /tmp/tmp.mYoeOnS61A ++ return 0 + proxy_ready=2 + [[ initializing == \r\e\a\d\y ]] + echo -n . .+ sleep 5 + for _ in $(seq 1 "${retries}") ++ kubectl_bin get pxc monitoring -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.sEtYTvJObe +++ mktemp ++ local LAST_ERR=/tmp/tmp.XfeqBo8S3u ++ local exit_status=0 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pxc monitoring -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.sEtYTvJObe ++ cat /tmp/tmp.XfeqBo8S3u ++ rm /tmp/tmp.sEtYTvJObe /tmp/tmp.XfeqBo8S3u ++ return 0 + state=initializing ++ kubectl_bin get pxc monitoring -o 'jsonpath={.status.pxc.ready}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.AjdnmEBbt0 +++ mktemp ++ local LAST_ERR=/tmp/tmp.PuxTGlhWih ++ local exit_status=0 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pxc monitoring -o 'jsonpath={.status.pxc.ready}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.AjdnmEBbt0 ++ cat /tmp/tmp.PuxTGlhWih ++ rm /tmp/tmp.AjdnmEBbt0 /tmp/tmp.PuxTGlhWih ++ return 0 + pxc_ready=1 + proxy_ready=0 + [[ 2 -gt 0 ]] +++ get_proxy_engine monitoring +++ local cluster_name=monitoring ++++ get_proxy monitoring ++++ local target_cluster=monitoring +++++ kubectl_bin get pxc monitoring -o 'jsonpath={.spec.haproxy.enabled}' ++++++ mktemp +++++ local LAST_OUT=/tmp/tmp.5l7QvVYGW9 ++++++ mktemp +++++ local LAST_ERR=/tmp/tmp.TlcFK9dGZl +++++ local exit_status=0 ++++++ seq 0 2 +++++ for i in $(seq 0 2) +++++ set +e +++++ kubectl get pxc monitoring -o 'jsonpath={.spec.haproxy.enabled}' +++++ exit_status=0 +++++ set -e +++++ '[' 0 '!=' 0 ']' +++++ break +++++ cat /tmp/tmp.5l7QvVYGW9 +++++ cat /tmp/tmp.TlcFK9dGZl +++++ rm /tmp/tmp.5l7QvVYGW9 /tmp/tmp.TlcFK9dGZl +++++ return 0 ++++ [[ true == \t\r\u\e ]] ++++ echo monitoring-haproxy ++++ return +++ local cluster_proxy=monitoring-haproxy +++ echo haproxy ++ kubectl_bin get pxc monitoring -o 'jsonpath={.status.haproxy.ready}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.5Ee7hcQDlX +++ mktemp ++ local LAST_ERR=/tmp/tmp.SaZxopPh70 ++ local exit_status=0 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pxc monitoring -o 'jsonpath={.status.haproxy.ready}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.5Ee7hcQDlX ++ cat /tmp/tmp.SaZxopPh70 ++ rm /tmp/tmp.5Ee7hcQDlX /tmp/tmp.SaZxopPh70 ++ return 0 + proxy_ready=2 + [[ initializing == \r\e\a\d\y ]] + echo -n . .+ sleep 5 + for _ in $(seq 1 "${retries}") ++ kubectl_bin get pxc monitoring -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.MTHF0dxNPO +++ mktemp ++ local LAST_ERR=/tmp/tmp.EudXsqq2Q4 ++ local exit_status=0 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pxc monitoring -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.MTHF0dxNPO ++ cat /tmp/tmp.EudXsqq2Q4 ++ rm /tmp/tmp.MTHF0dxNPO /tmp/tmp.EudXsqq2Q4 ++ return 0 + state=initializing ++ kubectl_bin get pxc monitoring -o 'jsonpath={.status.pxc.ready}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.8V0OecdzXE +++ mktemp ++ local LAST_ERR=/tmp/tmp.X2XrhFFNfa ++ local exit_status=0 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pxc monitoring -o 'jsonpath={.status.pxc.ready}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.8V0OecdzXE ++ cat /tmp/tmp.X2XrhFFNfa ++ rm /tmp/tmp.8V0OecdzXE /tmp/tmp.X2XrhFFNfa ++ return 0 + pxc_ready=1 + proxy_ready=0 + [[ 2 -gt 0 ]] +++ get_proxy_engine monitoring +++ local cluster_name=monitoring ++++ get_proxy monitoring ++++ local target_cluster=monitoring +++++ kubectl_bin get pxc monitoring -o 'jsonpath={.spec.haproxy.enabled}' ++++++ mktemp +++++ local LAST_OUT=/tmp/tmp.gznLWras3M ++++++ mktemp +++++ local LAST_ERR=/tmp/tmp.HTCrAGnycn +++++ local exit_status=0 ++++++ seq 0 2 +++++ for i in $(seq 0 2) +++++ set +e +++++ kubectl get pxc monitoring -o 'jsonpath={.spec.haproxy.enabled}' +++++ exit_status=0 +++++ set -e +++++ '[' 0 '!=' 0 ']' +++++ break +++++ cat /tmp/tmp.gznLWras3M +++++ cat /tmp/tmp.HTCrAGnycn +++++ rm /tmp/tmp.gznLWras3M /tmp/tmp.HTCrAGnycn +++++ return 0 ++++ [[ true == \t\r\u\e ]] ++++ echo monitoring-haproxy ++++ return +++ local cluster_proxy=monitoring-haproxy +++ echo haproxy ++ kubectl_bin get pxc monitoring -o 'jsonpath={.status.haproxy.ready}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.R9aQfmiyKw +++ mktemp ++ local LAST_ERR=/tmp/tmp.i2OMCbdCLV ++ local exit_status=0 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pxc monitoring -o 'jsonpath={.status.haproxy.ready}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.R9aQfmiyKw ++ cat /tmp/tmp.i2OMCbdCLV ++ rm /tmp/tmp.R9aQfmiyKw /tmp/tmp.i2OMCbdCLV ++ return 0 + proxy_ready=2 + [[ initializing == \r\e\a\d\y ]] + echo -n . .+ sleep 5 + for _ in $(seq 1 "${retries}") ++ kubectl_bin get pxc monitoring -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.XkXuULB653 +++ mktemp ++ local LAST_ERR=/tmp/tmp.CjO0Px74IE ++ local exit_status=0 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pxc monitoring -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.XkXuULB653 ++ cat /tmp/tmp.CjO0Px74IE ++ rm /tmp/tmp.XkXuULB653 /tmp/tmp.CjO0Px74IE ++ return 0 + state=initializing ++ kubectl_bin get pxc monitoring -o 'jsonpath={.status.pxc.ready}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.nnyYhorg9e +++ mktemp ++ local LAST_ERR=/tmp/tmp.ZbOtqKACDm ++ local exit_status=0 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pxc monitoring -o 'jsonpath={.status.pxc.ready}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.nnyYhorg9e ++ cat /tmp/tmp.ZbOtqKACDm ++ rm /tmp/tmp.nnyYhorg9e /tmp/tmp.ZbOtqKACDm ++ return 0 + pxc_ready=1 + proxy_ready=0 + [[ 2 -gt 0 ]] +++ get_proxy_engine monitoring +++ local cluster_name=monitoring ++++ get_proxy monitoring ++++ local target_cluster=monitoring +++++ kubectl_bin get pxc monitoring -o 'jsonpath={.spec.haproxy.enabled}' ++++++ mktemp +++++ local LAST_OUT=/tmp/tmp.8IHRwPoF4e ++++++ mktemp +++++ local LAST_ERR=/tmp/tmp.I5fNQmrOiA +++++ local exit_status=0 ++++++ seq 0 2 +++++ for i in $(seq 0 2) +++++ set +e +++++ kubectl get pxc monitoring -o 'jsonpath={.spec.haproxy.enabled}' +++++ exit_status=0 +++++ set -e +++++ '[' 0 '!=' 0 ']' +++++ break +++++ cat /tmp/tmp.8IHRwPoF4e +++++ cat /tmp/tmp.I5fNQmrOiA +++++ rm /tmp/tmp.8IHRwPoF4e /tmp/tmp.I5fNQmrOiA +++++ return 0 ++++ [[ true == \t\r\u\e ]] ++++ echo monitoring-haproxy ++++ return +++ local cluster_proxy=monitoring-haproxy +++ echo haproxy ++ kubectl_bin get pxc monitoring -o 'jsonpath={.status.haproxy.ready}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.wzETSxyvV2 +++ mktemp ++ local LAST_ERR=/tmp/tmp.eiRKTAEulk ++ local exit_status=0 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pxc monitoring -o 'jsonpath={.status.haproxy.ready}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.wzETSxyvV2 ++ cat /tmp/tmp.eiRKTAEulk ++ rm /tmp/tmp.wzETSxyvV2 /tmp/tmp.eiRKTAEulk ++ return 0 + proxy_ready=2 + [[ initializing == \r\e\a\d\y ]] + echo -n . .+ sleep 5 + for _ in $(seq 1 "${retries}") ++ kubectl_bin get pxc monitoring -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.FXlrv8LPle +++ mktemp ++ local LAST_ERR=/tmp/tmp.vP8DKi8Os3 ++ local exit_status=0 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pxc monitoring -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.FXlrv8LPle ++ cat /tmp/tmp.vP8DKi8Os3 ++ rm /tmp/tmp.FXlrv8LPle /tmp/tmp.vP8DKi8Os3 ++ return 0 + state=initializing ++ kubectl_bin get pxc monitoring -o 'jsonpath={.status.pxc.ready}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.9pC87Q8Eeb +++ mktemp ++ local LAST_ERR=/tmp/tmp.zj0KjPsg4r ++ local exit_status=0 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pxc monitoring -o 'jsonpath={.status.pxc.ready}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.9pC87Q8Eeb ++ cat /tmp/tmp.zj0KjPsg4r ++ rm /tmp/tmp.9pC87Q8Eeb /tmp/tmp.zj0KjPsg4r ++ return 0 + pxc_ready=2 + proxy_ready=0 + [[ 2 -gt 0 ]] +++ get_proxy_engine monitoring +++ local cluster_name=monitoring ++++ get_proxy monitoring ++++ local target_cluster=monitoring +++++ kubectl_bin get pxc monitoring -o 'jsonpath={.spec.haproxy.enabled}' ++++++ mktemp +++++ local LAST_OUT=/tmp/tmp.UHiZohrGKD ++++++ mktemp +++++ local LAST_ERR=/tmp/tmp.cHK1Pdo5rO +++++ local exit_status=0 ++++++ seq 0 2 +++++ for i in $(seq 0 2) +++++ set +e +++++ kubectl get pxc monitoring -o 'jsonpath={.spec.haproxy.enabled}' +++++ exit_status=0 +++++ set -e +++++ '[' 0 '!=' 0 ']' +++++ break +++++ cat /tmp/tmp.UHiZohrGKD +++++ cat /tmp/tmp.cHK1Pdo5rO +++++ rm /tmp/tmp.UHiZohrGKD /tmp/tmp.cHK1Pdo5rO +++++ return 0 ++++ [[ true == \t\r\u\e ]] ++++ echo monitoring-haproxy ++++ return +++ local cluster_proxy=monitoring-haproxy +++ echo haproxy ++ kubectl_bin get pxc monitoring -o 'jsonpath={.status.haproxy.ready}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.Fr8ucrTk5v +++ mktemp ++ local LAST_ERR=/tmp/tmp.tPHd9V9AB5 ++ local exit_status=0 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pxc monitoring -o 'jsonpath={.status.haproxy.ready}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.Fr8ucrTk5v ++ cat /tmp/tmp.tPHd9V9AB5 ++ rm /tmp/tmp.Fr8ucrTk5v /tmp/tmp.tPHd9V9AB5 ++ return 0 + proxy_ready=2 + [[ initializing == \r\e\a\d\y ]] + echo -n . .+ sleep 5 + for _ in $(seq 1 "${retries}") ++ kubectl_bin get pxc monitoring -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.ygl7iwgJuw +++ mktemp ++ local LAST_ERR=/tmp/tmp.ob3IGz8VmE ++ local exit_status=0 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pxc monitoring -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.ygl7iwgJuw ++ cat /tmp/tmp.ob3IGz8VmE ++ rm /tmp/tmp.ygl7iwgJuw /tmp/tmp.ob3IGz8VmE ++ return 0 + state=initializing ++ kubectl_bin get pxc monitoring -o 'jsonpath={.status.pxc.ready}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.LTzeJjTQwK +++ mktemp ++ local LAST_ERR=/tmp/tmp.kVYZY9yNbR ++ local exit_status=0 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pxc monitoring -o 'jsonpath={.status.pxc.ready}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.LTzeJjTQwK ++ cat /tmp/tmp.kVYZY9yNbR ++ rm /tmp/tmp.LTzeJjTQwK /tmp/tmp.kVYZY9yNbR ++ return 0 + pxc_ready=2 + proxy_ready=0 + [[ 2 -gt 0 ]] +++ get_proxy_engine monitoring +++ local cluster_name=monitoring ++++ get_proxy monitoring ++++ local target_cluster=monitoring +++++ kubectl_bin get pxc monitoring -o 'jsonpath={.spec.haproxy.enabled}' ++++++ mktemp +++++ local LAST_OUT=/tmp/tmp.LN3TT3lRsq ++++++ mktemp +++++ local LAST_ERR=/tmp/tmp.kbmIGXA0wN +++++ local exit_status=0 ++++++ seq 0 2 +++++ for i in $(seq 0 2) +++++ set +e +++++ kubectl get pxc monitoring -o 'jsonpath={.spec.haproxy.enabled}' +++++ exit_status=0 +++++ set -e +++++ '[' 0 '!=' 0 ']' +++++ break +++++ cat /tmp/tmp.LN3TT3lRsq +++++ cat /tmp/tmp.kbmIGXA0wN +++++ rm /tmp/tmp.LN3TT3lRsq /tmp/tmp.kbmIGXA0wN +++++ return 0 ++++ [[ true == \t\r\u\e ]] ++++ echo monitoring-haproxy ++++ return +++ local cluster_proxy=monitoring-haproxy +++ echo haproxy ++ kubectl_bin get pxc monitoring -o 'jsonpath={.status.haproxy.ready}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.eAn702YR5U +++ mktemp ++ local LAST_ERR=/tmp/tmp.6bjpuMsPwB ++ local exit_status=0 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pxc monitoring -o 'jsonpath={.status.haproxy.ready}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.eAn702YR5U ++ cat /tmp/tmp.6bjpuMsPwB ++ rm /tmp/tmp.eAn702YR5U /tmp/tmp.6bjpuMsPwB ++ return 0 + proxy_ready=2 + [[ initializing == \r\e\a\d\y ]] + echo -n . .+ sleep 5 + for _ in $(seq 1 "${retries}") ++ kubectl_bin get pxc monitoring -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.CdMVV60UBR +++ mktemp ++ local LAST_ERR=/tmp/tmp.XGlzHoLfr8 ++ local exit_status=0 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pxc monitoring -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.CdMVV60UBR ++ cat /tmp/tmp.XGlzHoLfr8 ++ rm /tmp/tmp.CdMVV60UBR /tmp/tmp.XGlzHoLfr8 ++ return 0 + state=initializing ++ kubectl_bin get pxc monitoring -o 'jsonpath={.status.pxc.ready}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.8t62WSvwEU +++ mktemp ++ local LAST_ERR=/tmp/tmp.fOoK4pZmAj ++ local exit_status=0 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pxc monitoring -o 'jsonpath={.status.pxc.ready}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.8t62WSvwEU ++ cat /tmp/tmp.fOoK4pZmAj ++ rm /tmp/tmp.8t62WSvwEU /tmp/tmp.fOoK4pZmAj ++ return 0 + pxc_ready=2 + proxy_ready=0 + [[ 2 -gt 0 ]] +++ get_proxy_engine monitoring +++ local cluster_name=monitoring ++++ get_proxy monitoring ++++ local target_cluster=monitoring +++++ kubectl_bin get pxc monitoring -o 'jsonpath={.spec.haproxy.enabled}' ++++++ mktemp +++++ local LAST_OUT=/tmp/tmp.ocw1YrE0kb ++++++ mktemp +++++ local LAST_ERR=/tmp/tmp.LncDxG0O2y +++++ local exit_status=0 ++++++ seq 0 2 +++++ for i in $(seq 0 2) +++++ set +e +++++ kubectl get pxc monitoring -o 'jsonpath={.spec.haproxy.enabled}' +++++ exit_status=0 +++++ set -e +++++ '[' 0 '!=' 0 ']' +++++ break +++++ cat /tmp/tmp.ocw1YrE0kb +++++ cat /tmp/tmp.LncDxG0O2y +++++ rm /tmp/tmp.ocw1YrE0kb /tmp/tmp.LncDxG0O2y +++++ return 0 ++++ [[ true == \t\r\u\e ]] ++++ echo monitoring-haproxy ++++ return +++ local cluster_proxy=monitoring-haproxy +++ echo haproxy ++ kubectl_bin get pxc monitoring -o 'jsonpath={.status.haproxy.ready}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.21JgyPewtg +++ mktemp ++ local LAST_ERR=/tmp/tmp.MdVXrCxgi7 ++ local exit_status=0 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pxc monitoring -o 'jsonpath={.status.haproxy.ready}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.21JgyPewtg ++ cat /tmp/tmp.MdVXrCxgi7 ++ rm /tmp/tmp.21JgyPewtg /tmp/tmp.MdVXrCxgi7 ++ return 0 + proxy_ready=2 + [[ initializing == \r\e\a\d\y ]] + echo -n . .+ sleep 5 + for _ in $(seq 1 "${retries}") ++ kubectl_bin get pxc monitoring -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.RrVd2QDnYK +++ mktemp ++ local LAST_ERR=/tmp/tmp.VaqjpztZmy ++ local exit_status=0 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pxc monitoring -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.RrVd2QDnYK ++ cat /tmp/tmp.VaqjpztZmy ++ rm /tmp/tmp.RrVd2QDnYK /tmp/tmp.VaqjpztZmy ++ return 0 + state=initializing ++ kubectl_bin get pxc monitoring -o 'jsonpath={.status.pxc.ready}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.i5QLhtDP6M +++ mktemp ++ local LAST_ERR=/tmp/tmp.xSU0lA5SdX ++ local exit_status=0 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pxc monitoring -o 'jsonpath={.status.pxc.ready}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.i5QLhtDP6M ++ cat /tmp/tmp.xSU0lA5SdX ++ rm /tmp/tmp.i5QLhtDP6M /tmp/tmp.xSU0lA5SdX ++ return 0 + pxc_ready=2 + proxy_ready=0 + [[ 2 -gt 0 ]] +++ get_proxy_engine monitoring +++ local cluster_name=monitoring ++++ get_proxy monitoring ++++ local target_cluster=monitoring +++++ kubectl_bin get pxc monitoring -o 'jsonpath={.spec.haproxy.enabled}' ++++++ mktemp +++++ local LAST_OUT=/tmp/tmp.VFihCr6D89 ++++++ mktemp +++++ local LAST_ERR=/tmp/tmp.rRc6gBDQVb +++++ local exit_status=0 ++++++ seq 0 2 +++++ for i in $(seq 0 2) +++++ set +e +++++ kubectl get pxc monitoring -o 'jsonpath={.spec.haproxy.enabled}' +++++ exit_status=0 +++++ set -e +++++ '[' 0 '!=' 0 ']' +++++ break +++++ cat /tmp/tmp.VFihCr6D89 +++++ cat /tmp/tmp.rRc6gBDQVb +++++ rm /tmp/tmp.VFihCr6D89 /tmp/tmp.rRc6gBDQVb +++++ return 0 ++++ [[ true == \t\r\u\e ]] ++++ echo monitoring-haproxy ++++ return +++ local cluster_proxy=monitoring-haproxy +++ echo haproxy ++ kubectl_bin get pxc monitoring -o 'jsonpath={.status.haproxy.ready}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.MT1I5EcuOX +++ mktemp ++ local LAST_ERR=/tmp/tmp.tOb3iXyKF1 ++ local exit_status=0 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pxc monitoring -o 'jsonpath={.status.haproxy.ready}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.MT1I5EcuOX ++ cat /tmp/tmp.tOb3iXyKF1 ++ rm /tmp/tmp.MT1I5EcuOX /tmp/tmp.tOb3iXyKF1 ++ return 0 + proxy_ready=2 + [[ initializing == \r\e\a\d\y ]] + echo -n . .+ sleep 5 + for _ in $(seq 1 "${retries}") ++ kubectl_bin get pxc monitoring -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.QCNcY6srsk +++ mktemp ++ local LAST_ERR=/tmp/tmp.rsd2KpNwCv ++ local exit_status=0 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pxc monitoring -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.QCNcY6srsk ++ cat /tmp/tmp.rsd2KpNwCv ++ rm /tmp/tmp.QCNcY6srsk /tmp/tmp.rsd2KpNwCv ++ return 0 + state=initializing ++ kubectl_bin get pxc monitoring -o 'jsonpath={.status.pxc.ready}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.hspOU50eFj +++ mktemp ++ local LAST_ERR=/tmp/tmp.ArPuZqOyhB ++ local exit_status=0 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pxc monitoring -o 'jsonpath={.status.pxc.ready}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.hspOU50eFj ++ cat /tmp/tmp.ArPuZqOyhB ++ rm /tmp/tmp.hspOU50eFj /tmp/tmp.ArPuZqOyhB ++ return 0 + pxc_ready=2 + proxy_ready=0 + [[ 2 -gt 0 ]] +++ get_proxy_engine monitoring +++ local cluster_name=monitoring ++++ get_proxy monitoring ++++ local target_cluster=monitoring +++++ kubectl_bin get pxc monitoring -o 'jsonpath={.spec.haproxy.enabled}' ++++++ mktemp +++++ local LAST_OUT=/tmp/tmp.AcXXhtPNiJ ++++++ mktemp +++++ local LAST_ERR=/tmp/tmp.BsGbYXfG2v +++++ local exit_status=0 ++++++ seq 0 2 +++++ for i in $(seq 0 2) +++++ set +e +++++ kubectl get pxc monitoring -o 'jsonpath={.spec.haproxy.enabled}' +++++ exit_status=0 +++++ set -e +++++ '[' 0 '!=' 0 ']' +++++ break +++++ cat /tmp/tmp.AcXXhtPNiJ +++++ cat /tmp/tmp.BsGbYXfG2v +++++ rm /tmp/tmp.AcXXhtPNiJ /tmp/tmp.BsGbYXfG2v +++++ return 0 ++++ [[ true == \t\r\u\e ]] ++++ echo monitoring-haproxy ++++ return +++ local cluster_proxy=monitoring-haproxy +++ echo haproxy ++ kubectl_bin get pxc monitoring -o 'jsonpath={.status.haproxy.ready}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.1RMIbDyoPW +++ mktemp ++ local LAST_ERR=/tmp/tmp.9FgBxy2Op4 ++ local exit_status=0 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pxc monitoring -o 'jsonpath={.status.haproxy.ready}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.1RMIbDyoPW ++ cat /tmp/tmp.9FgBxy2Op4 ++ rm /tmp/tmp.1RMIbDyoPW /tmp/tmp.9FgBxy2Op4 ++ return 0 + proxy_ready=2 + [[ initializing == \r\e\a\d\y ]] + echo -n . .+ sleep 5 + for _ in $(seq 1 "${retries}") ++ kubectl_bin get pxc monitoring -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.tuF4EZmsxu +++ mktemp ++ local LAST_ERR=/tmp/tmp.9QxXvBDYTG ++ local exit_status=0 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pxc monitoring -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.tuF4EZmsxu ++ cat /tmp/tmp.9QxXvBDYTG ++ rm /tmp/tmp.tuF4EZmsxu /tmp/tmp.9QxXvBDYTG ++ return 0 + state=ready ++ kubectl_bin get pxc monitoring -o 'jsonpath={.status.pxc.ready}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.Xtw1uaZ4Ys +++ mktemp ++ local LAST_ERR=/tmp/tmp.ACIFgIKcyF ++ local exit_status=0 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pxc monitoring -o 'jsonpath={.status.pxc.ready}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.Xtw1uaZ4Ys ++ cat /tmp/tmp.ACIFgIKcyF ++ rm /tmp/tmp.Xtw1uaZ4Ys /tmp/tmp.ACIFgIKcyF ++ return 0 + pxc_ready=3 + proxy_ready=0 + [[ 2 -gt 0 ]] +++ get_proxy_engine monitoring +++ local cluster_name=monitoring ++++ get_proxy monitoring ++++ local target_cluster=monitoring +++++ kubectl_bin get pxc monitoring -o 'jsonpath={.spec.haproxy.enabled}' ++++++ mktemp +++++ local LAST_OUT=/tmp/tmp.VMl7BHJP4F ++++++ mktemp +++++ local LAST_ERR=/tmp/tmp.GPQC8kDzsQ +++++ local exit_status=0 ++++++ seq 0 2 +++++ for i in $(seq 0 2) +++++ set +e +++++ kubectl get pxc monitoring -o 'jsonpath={.spec.haproxy.enabled}' +++++ exit_status=0 +++++ set -e +++++ '[' 0 '!=' 0 ']' +++++ break +++++ cat /tmp/tmp.VMl7BHJP4F +++++ cat /tmp/tmp.GPQC8kDzsQ +++++ rm /tmp/tmp.VMl7BHJP4F /tmp/tmp.GPQC8kDzsQ +++++ return 0 ++++ [[ true == \t\r\u\e ]] ++++ echo monitoring-haproxy ++++ return +++ local cluster_proxy=monitoring-haproxy +++ echo haproxy ++ kubectl_bin get pxc monitoring -o 'jsonpath={.status.haproxy.ready}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.aymqlG3Stz +++ mktemp ++ local LAST_ERR=/tmp/tmp.VqYurXYl6M ++ local exit_status=0 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pxc monitoring -o 'jsonpath={.status.haproxy.ready}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.aymqlG3Stz ++ cat /tmp/tmp.VqYurXYl6M ++ rm /tmp/tmp.aymqlG3Stz /tmp/tmp.VqYurXYl6M ++ return 0 + proxy_ready=2 + [[ ready == \r\e\a\d\y ]] + [[ 3 == \3 ]] + [[ 2 == \2 ]] + echo + return + compare_kubectl statefulset/monitoring-pxc -no-prefix + local resource=statefulset/monitoring-pxc + local postfix=-no-prefix + local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/e2e-tests/monitoring-2-0/compare/statefulset_monitoring-pxc-no-prefix.yml + local new_result=/tmp/tmp.oE2g1q4ApT/statefulset_monitoring-pxc.yml + desc 'compare statefulset/monitoring-pxc--no-prefix' + set +o xtrace ----------------------------------------------------------------------------------- compare statefulset/monitoring-pxc--no-prefix ----------------------------------------------------------------------------------- + '[' 0 = 1 -a -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/e2e-tests/monitoring-2-0/compare/statefulset_monitoring-pxc-no-prefix-eks.yml ']' + [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 8\.0 ]] + '[' -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/e2e-tests/monitoring-2-0/compare/statefulset_monitoring-pxc-no-prefix-80.yml ']' + [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 8\.4 ]] + version_gt 1.33 ++ bc -l ++ echo '1.34 >= 1.33' + '[' 1 -eq 1 ']' + return 0 + '[' -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/e2e-tests/monitoring-2-0/compare/statefulset_monitoring-pxc-no-prefix-k133.yml ']' + version_gt 1.29 ++ bc -l ++ echo '1.34 >= 1.29' + '[' 1 -eq 1 ']' + return 0 + '[' -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/e2e-tests/monitoring-2-0/compare/statefulset_monitoring-pxc-no-prefix-k129.yml ']' + version_gt 1.27 ++ bc -l ++ echo '1.34 >= 1.27' + '[' 1 -eq 1 ']' + return 0 + '[' -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/e2e-tests/monitoring-2-0/compare/statefulset_monitoring-pxc-no-prefix-k127.yml ']' + expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/e2e-tests/monitoring-2-0/compare/statefulset_monitoring-pxc-no-prefix-k127.yml + '[' '!' -z '' -a -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/e2e-tests/monitoring-2-0/compare/statefulset_monitoring-pxc-no-prefix-k127-oc.yml ']' + version_gt 1.29 ++ bc -l ++ echo '1.34 >= 1.29' + '[' 1 -eq 1 ']' + return 0 + '[' '!' -z '' -a -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/e2e-tests/monitoring-2-0/compare/statefulset_monitoring-pxc-no-prefix-k127-k129-oc.yml ']' + '[' 0 = 1 -a -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/e2e-tests/monitoring-2-0/compare/statefulset_monitoring-pxc-no-prefix-k127-eks.yml ']' + '[' 0 = 1 -a -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/e2e-tests/monitoring-2-0/compare/statefulset_monitoring-pxc-no-prefix-k127-aks.yml ']' + kubectl_bin get -o yaml statefulset/monitoring-pxc ++ mktemp + yq eval ' del(.metadata.managedFields) | del(.. | select(has("creationTimestamp")).creationTimestamp) | del(.. | select(has("namespace")).namespace) | del(.. | select(has("uid")).uid) | del(.metadata.resourceVersion) | del(.spec.template.spec.containers[].env[] | select(.name == "CLUSTER_HASH")) | del(.spec.template.spec.containers[].env[] | select(.name == "S3_BUCKET_PATH")) | del(.spec.template.spec.containers[].env[] | select(.name == "BACKUP_PATH")) | del(.spec.template.spec.containers[].env[] | select(.name == "S3_BUCKET_URL")) | del(.spec.template.spec.containers[].env[] | select(.name == "AZURE_CONTAINER_NAME")) | del(.spec.template.spec.containers[].env[] | select(.name == "XTRABACKUP_ENABLED")) | del(.metadata.selfLink) | del(.metadata.deletionTimestamp) | del(.metadata.labels."batch.kubernetes.io/job-name") | del(.metadata.labels."job-name") | del(.metadata.labels."topology.kubernetes.io/region") | del(.metadata.labels."topology.kubernetes.io/zone") | del(.metadata.annotations."kubectl.kubernetes.io/last-applied-configuration") | del(.metadata.annotations."kubernetes.io/psp") | del(.metadata.annotations."batch.kubernetes.io/job-tracking") | del(.metadata.annotations."networking.gke.io/target-pool") | del(.metadata.annotations."cloud.google.com/neg") | del(.metadata.annotations."k8s.v1.cni.cncf.io*") | del(.metadata.annotations."k8s.ovn.org/pod-networks") | del(.metadata.annotations."kubernetes.digitalocean.com/load-balancer-id") | del(.metadata.annotations."service.beta.kubernetes.io/do-loadbalancer-type") | del(.spec.template.metadata.annotations."last-applied-secret") | del(.spec.template.metadata.labels."batch.kubernetes.io/job-name") | del(.spec.template.metadata.labels."job-name") | del(.. | select(has("batch.kubernetes.io/controller-uid"))."batch.kubernetes.io/controller-uid") | del(.. | select(has("image")).image) | del(.. | select(has("clusterIP")).clusterIP) | del(.. | select(has("clusterIPs")).clusterIPs) | del(.. | select(has("dataSource")).dataSource) | del(.. | select(has("procMount")).procMount) | del(.. | select(has("storageClassName")).storageClassName) | del(.. | select(has("finalizers")).finalizers) | del(.. | select(has("kubernetes.io/pvc-protection"))."kubernetes.io/pvc-protection") | del(.. | select(has("volumeName")).volumeName) | del(.. | select(has("volume.beta.kubernetes.io/storage-provisioner"))."volume.beta.kubernetes.io/storage-provisioner") | del(.. | select(has("volume.kubernetes.io/storage-provisioner"))."volume.kubernetes.io/storage-provisioner") | del(.spec.volumeMode) | del(.spec.nodeName) | del(.. | select(has("volume.kubernetes.io/selected-node"))."volume.kubernetes.io/selected-node") | del(.. | select(has("percona.com/last-config-hash"))."percona.com/last-config-hash") | del(.. | select(has("percona.com/configuration-hash"))."percona.com/configuration-hash") | del(.. | select(has("percona.com/env-secret-config-hash"))."percona.com/env-secret-config-hash") | del(.. | select(has("percona.com/ssl-hash"))."percona.com/ssl-hash") | del(.. | select(has("percona.com/ssl-internal-hash"))."percona.com/ssl-internal-hash") | del(.. | select(has("kubectl.kubernetes.io/default-container"))."kubectl.kubernetes.io/default-container") | del(.spec.volumeClaimTemplates[].spec.volumeMode | select(. == "Filesystem")) | del(.. | select(has("healthCheckNodePort")).healthCheckNodePort) | del(.. | select(has("nodePort")).nodePort) | del(.. | select(has("imagePullSecrets")).imagePullSecrets) | del(.. | select(has("enableServiceLinks")).enableServiceLinks) | del(.status) | del(.spec.volumeClaimTemplates[].apiVersion) | del(.spec.volumeClaimTemplates[].kind) | del(.metadata.ownerReferences[].apiVersion) | del(.. | select(has("controller-uid")).controller-uid) | del(.. | select(has("preemptionPolicy")).preemptionPolicy) | del(.spec.ipFamilies) | del(.spec.ipFamilyPolicy) | del(.spec.updateStrategy.rollingUpdate.+ local LAST_OUT=/tmp/tmp.VEiWcpUiEg maxUnavailable) | (.. | select(. == "policy/v1beta1")) = "policy/v1" | del(.. | select(has("kubernetes.io/hostname"))."kubernetes.io/hostname") | (.. | select(tag == "!!str")) |= sub("monitoring-2-0-5888", "namespace") | (.. | select(tag == "!!str")) |= sub("kube-api-access-.*", "kube-api-access") | del(.. | select(has("annotations")).annotations | select(length==0)) | del(.spec.crVersion) | del(.. | select(.[] == "percona-xtradb-cluster-operator-workload-token*"))' - ++ mktemp + local LAST_ERR=/tmp/tmp.DvPQ4LD5PK + local exit_status=0 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl get -o yaml statefulset/monitoring-pxc + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.VEiWcpUiEg + cat /tmp/tmp.DvPQ4LD5PK + rm /tmp/tmp.VEiWcpUiEg /tmp/tmp.DvPQ4LD5PK + return 0 + [[ 0 -eq 0 ]] + diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/e2e-tests/monitoring-2-0/compare/statefulset_monitoring-pxc-no-prefix-k127.yml /tmp/tmp.oE2g1q4ApT/statefulset_monitoring-pxc.yml + log 'compare_kubectl: statefulset/monitoring-pxc OK' ++ date +%Y-%m-%dT%H:%M:%S%z + echo '[2026-10-05T13:32:05+0000]' compare_kubectl: statefulset/monitoring-pxc OK [2026-10-05T13:32:05+0000] compare_kubectl: statefulset/monitoring-pxc OK + compare_kubectl statefulset/monitoring-haproxy -no-prefix + local resource=statefulset/monitoring-haproxy + local postfix=-no-prefix + local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/e2e-tests/monitoring-2-0/compare/statefulset_monitoring-haproxy-no-prefix.yml + local new_result=/tmp/tmp.oE2g1q4ApT/statefulset_monitoring-haproxy.yml + desc 'compare statefulset/monitoring-haproxy--no-prefix' + set +o xtrace ----------------------------------------------------------------------------------- compare statefulset/monitoring-haproxy--no-prefix ----------------------------------------------------------------------------------- + '[' 0 = 1 -a -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/e2e-tests/monitoring-2-0/compare/statefulset_monitoring-haproxy-no-prefix-eks.yml ']' + [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 8\.0 ]] + '[' -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/e2e-tests/monitoring-2-0/compare/statefulset_monitoring-haproxy-no-prefix-80.yml ']' + [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 8\.4 ]] + version_gt 1.33 ++ bc -l ++ echo '1.34 >= 1.33' + '[' 1 -eq 1 ']' + return 0 + '[' -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/e2e-tests/monitoring-2-0/compare/statefulset_monitoring-haproxy-no-prefix-k133.yml ']' + version_gt 1.29 ++ bc -l ++ echo '1.34 >= 1.29' + '[' 1 -eq 1 ']' + return 0 + '[' -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/e2e-tests/monitoring-2-0/compare/statefulset_monitoring-haproxy-no-prefix-k129.yml ']' + version_gt 1.27 ++ echo '1.34 >= 1.27' ++ bc -l + '[' 1 -eq 1 ']' + return 0 + '[' -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/e2e-tests/monitoring-2-0/compare/statefulset_monitoring-haproxy-no-prefix-k127.yml ']' + expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/e2e-tests/monitoring-2-0/compare/statefulset_monitoring-haproxy-no-prefix-k127.yml + '[' '!' -z '' -a -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/e2e-tests/monitoring-2-0/compare/statefulset_monitoring-haproxy-no-prefix-k127-oc.yml ']' + version_gt 1.29 ++ echo '1.34 >= 1.29' ++ bc -l + '[' 1 -eq 1 ']' + return 0 + '[' '!' -z '' -a -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/e2e-tests/monitoring-2-0/compare/statefulset_monitoring-haproxy-no-prefix-k127-k129-oc.yml ']' + '[' 0 = 1 -a -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/e2e-tests/monitoring-2-0/compare/statefulset_monitoring-haproxy-no-prefix-k127-eks.yml ']' + '[' 0 = 1 -a -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/e2e-tests/monitoring-2-0/compare/statefulset_monitoring-haproxy-no-prefix-k127-aks.yml ']' + kubectl_bin get -o yaml statefulset/monitoring-haproxy + yq eval ' del(.metadata.managedFields) | del(.. | select(has("creationTimestamp")).creationTimestamp) | del(.. | select(has("namespace")).namespace) | del(.. | select(has("uid")).uid) | del(.metadata.resourceVersion) | del(.spec.template.spec.containers[].env[] | select(.name == "CLUSTER_HASH")) | del(.spec.template.spec.containers[].env[] | select(.name == "S3_BUCKET_PATH")) | del(.spec.template.spec.containers[].env[] | select(.name == "BACKUP_PATH")) | del(.spec.template.spec.containers[].env[] | select(.name == "S3_BUCKET_URL")) | del(.spec.template.spec.containers[].env[] | select(.name == "AZURE_CONTAINER_NAME")) | del(.spec.template.spec.containers[].env[] | select(.name == "XTRABACKUP_ENABLED")) | del(.metadata.selfLink) | del(.metadata.deletionTimestamp) | del(.metadata.labels."batch.kubernetes.io/job-name") | del(.metadata.labels."job-name") | del(.metadata.labels."topology.kubernetes.io/region") | del(.metadata.labels."topology.kubernetes.io/zone") | del(.metadata.annotations."kubectl.kubernetes.io/last-applied-configuration") | del(.metadata.annotations."kubernetes.io/psp") | del(.metadata.annotations."batch.kubernetes.io/job-tracking") | del(.metadata.annotations."networking.gke.io/target-pool") | del(.metadata.annotations."cloud.google.com/neg") | del(.metadata.annotations."k8s.v1.cni.cncf.io*") | del(.metadata.annotations."k8s.ovn.org/pod-networks") | del(.metadata.annotations."kubernetes.digitalocean.com/load-balancer-id") | del(.metadata.annotations."service.beta.kubernetes.io/do-loadbalancer-type") | del(.spec.template.metadata.annotations."last-applied-secret") | del(.spec.template.metadata.labels."batch.kubernetes.io/job-name") | del(.spec.template.metadata.labels."job-name") | del(.. | select(has("batch.kubernetes.io/controller-uid"))."batch.kubernetes.io/controller-uid") | del(.. | select(has("image")).image) | del(.. | select(has("clusterIP")).clusterIP) | del(.. | select(has("clusterIPs")).clusterIPs) | del(.. | select(has("dataSource")).dataSource) | del(.. | select(has("procMount")).procMount) | del(.. | select(has("storageClassName")).storageClassName) | del(.. | select(has("finalizers")).finalizers) | del(.. | select(has("kubernetes.io/pvc-protection"))."kubernetes.io/pvc-protection") | del(.. | select(has("volumeName")).volumeName) | del(.. | select(has("volume.beta.kubernetes.io/storage-provisioner"))."volume.beta.kubernetes.io/storage-provisioner") | del(.. | select(has("volume.kubernetes.io/storage-provisioner"))."volume.kubernetes.io/storage-provisioner") | del(.spec.volumeMode) | del(.spec.nodeName) | del(.. | select(has("volume.kubernetes.io/selected-node"))."volume.kubernetes.io/selected-node") | del(.. | select(has("percona.com/last-config-hash"))."percona.com/last-config-hash") | del(.. | select(has("percona.com/configuration-hash"))."percona.com/configuration-hash") | del(.. | select(has("percona.com/env-secret-config-hash"))."percona.com/env-secret-config-hash") | del(.. | select(has("percona.com/ssl-hash"))."percona.com/ssl-hash") | del(.. | select(has("percona.com/ssl-internal-hash"))."percona.com/ssl-internal-hash") | del(.. | select(has("kubectl.kubernetes.io/default-container"))."kubectl.kubernetes.io/default-container") | del(.spec.volumeClaimTemplates[].spec.volumeMode | select(. == "Filesystem")) | del(.. | select(has("healthCheckNodePort")).healthCheckNodePort) | del(.. | select(has("nodePort")).nodePort) | del(.. | select(has("imagePullSecrets")).imagePullSecrets) | del(.. | select(has("enableServiceLinks")).enableServiceLinks) | del(.status) | del(.spec.volumeClaimTemplates[].apiVersion) | del(.spec.volumeClaimTemplates[].kind) | del(.metadata.ownerReferences[].apiVersion) | del(.. | select(has("controller-uid")).controller-uid) | del(.. | select(has("preemptionPolicy")).preemptionPolicy) | del(.spec.ipFamilies) | del(.spec.ipFamilyPolicy) | del(.spec.updateStrategy.rollingUpdate.maxUnavailable) | (.. | select(. == "policy/v1beta1")) = "policy/v1" | del(.. | select(has("kubernetes.io/hostname"))."kubernetes.io/hostname") | (.. | select(tag == "!!str")) |= sub("monitoring-2-0-5888", "namespace") | (.. | select(tag == "!!str")) |= sub("kube-api-access-.*", "kube-api-access") | del(.. | select(has("annotations")).annotations | select(length==0)) | del(.spec.crVersion) | del(.. | select(.[] == "percona-xtradb-cluster-operator-workload-token*"))' - ++ mktemp + local LAST_OUT=/tmp/tmp.aTknZP9Aho ++ mktemp + local LAST_ERR=/tmp/tmp.MJSDfZLBL4 + local exit_status=0 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl get -o yaml statefulset/monitoring-haproxy + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.aTknZP9Aho + cat /tmp/tmp.MJSDfZLBL4 + rm /tmp/tmp.aTknZP9Aho /tmp/tmp.MJSDfZLBL4 + return 0 + [[ 0 -eq 0 ]] + diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/e2e-tests/monitoring-2-0/compare/statefulset_monitoring-haproxy-no-prefix-k127.yml /tmp/tmp.oE2g1q4ApT/statefulset_monitoring-haproxy.yml + log 'compare_kubectl: statefulset/monitoring-haproxy OK' ++ date +%Y-%m-%dT%H:%M:%S%z + echo '[2026-10-05T13:32:07+0000]' compare_kubectl: statefulset/monitoring-haproxy OK [2026-10-05T13:32:07+0000] compare_kubectl: statefulset/monitoring-haproxy OK + desc 'apply my-env-var-secrets to add PMM_PREFIX' + set +o xtrace ----------------------------------------------------------------------------------- apply my-env-var-secrets to add PMM_PREFIX ----------------------------------------------------------------------------------- + kubectl_bin apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/e2e-tests/monitoring-2-0/conf/envsecrets.yaml ++ mktemp + local LAST_OUT=/tmp/tmp.hUsQ0BV12C ++ mktemp + local LAST_ERR=/tmp/tmp.rNgSFTTBjE + local exit_status=0 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/e2e-tests/monitoring-2-0/conf/envsecrets.yaml + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.hUsQ0BV12C secret/my-env-var-secrets created + cat /tmp/tmp.rNgSFTTBjE + rm /tmp/tmp.hUsQ0BV12C /tmp/tmp.rNgSFTTBjE + return 0 + wait_for_generation sts/monitoring-pxc 3 + local resource=sts/monitoring-pxc + local target_generation=3 + echo 'Waiting for sts/monitoring-pxc to reach generation 3...' Waiting for sts/monitoring-pxc to reach generation 3... + true ++ kubectl get sts/monitoring-pxc -o 'jsonpath={.metadata.generation}' + current_generation=2 + '[' 2 -eq 3 ']' + echo 'Resource sts/monitoring-pxc is at generation 2. Waiting...' Resource sts/monitoring-pxc is at generation 2. Waiting... + sleep 5 + true ++ kubectl get sts/monitoring-pxc -o 'jsonpath={.metadata.generation}' + current_generation=3 + '[' 3 -eq 3 ']' + echo 'Resource sts/monitoring-pxc has reached generation 3.' Resource sts/monitoring-pxc has reached generation 3. + break + wait_for_generation sts/monitoring-haproxy 3 + local resource=sts/monitoring-haproxy + local target_generation=3 + echo 'Waiting for sts/monitoring-haproxy to reach generation 3...' Waiting for sts/monitoring-haproxy to reach generation 3... + true ++ kubectl get sts/monitoring-haproxy -o 'jsonpath={.metadata.generation}' + current_generation=3 + '[' 3 -eq 3 ']' + echo 'Resource sts/monitoring-haproxy has reached generation 3.' Resource sts/monitoring-haproxy has reached generation 3. + break + desc 'add new PMM API key to secret' + set +o xtrace ----------------------------------------------------------------------------------- add new PMM API key to secret ----------------------------------------------------------------------------------- ++ jq .key +++ get_service_endpoint monitoring-service +++ local service=monitoring-service ++++ kubectl_bin get service/monitoring-service -o 'jsonpath={.status.loadBalancer.ingress[].hostname}' +++++ mktemp ++++ local LAST_OUT=/tmp/tmp.L9mloyLHXd +++++ mktemp ++++ local LAST_ERR=/tmp/tmp.IbCoU7Rm9w ++++ local exit_status=0 +++++ seq 0 2 ++++ for i in $(seq 0 2) ++++ set +e ++++ kubectl get service/monitoring-service -o 'jsonpath={.status.loadBalancer.ingress[].hostname}' ++++ exit_status=0 ++++ set -e ++++ '[' 0 '!=' 0 ']' ++++ break ++++ cat /tmp/tmp.L9mloyLHXd ++++ cat /tmp/tmp.IbCoU7Rm9w ++++ rm /tmp/tmp.L9mloyLHXd /tmp/tmp.IbCoU7Rm9w ++++ return 0 +++ local endpoint= +++ '[' -z '' ']' ++++ kubectl_bin get service/monitoring-service -o 'jsonpath={.status.loadBalancer.ingress[].ip}' +++++ mktemp ++++ local LAST_OUT=/tmp/tmp.9A4bm9ONvg +++++ mktemp ++++ local LAST_ERR=/tmp/tmp.EOR1MaxQ5v ++++ local exit_status=0 +++++ seq 0 2 ++++ for i in $(seq 0 2) ++++ set +e ++++ kubectl get service/monitoring-service -o 'jsonpath={.status.loadBalancer.ingress[].ip}' ++++ exit_status=0 ++++ set -e ++++ '[' 0 '!=' 0 ']' ++++ break ++++ cat /tmp/tmp.9A4bm9ONvg ++++ cat /tmp/tmp.EOR1MaxQ5v ++++ rm /tmp/tmp.9A4bm9ONvg /tmp/tmp.EOR1MaxQ5v ++++ return 0 +++ endpoint=34.71.161.14 +++ '[' -n 34.71.161.14 ']' +++ '[' 34.71.161.14 '!=' null ']' +++ sed -e 's/^"//; s/"$//;' +++ echo 34.71.161.14 +++ head -n 1 +++ return ++ curl --insecure -X POST -H 'Content-Type: application/json' -d '{"name":"operator-new", "role": "Admin"}' https://admin:admin@34.71.161.14/graph/api/auth/keys % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 40 0 0 100 40 0 123 --:--:-- --:--:-- --:--:-- 123 100 167 100 127 100 40 372 117 --:--:-- --:--:-- --:--:-- 489 + API_KEY_NEW='"eyJrIjoiOUFRZFY0UjBmbG8zZVg3TDZEdTNza1VycXZJaE5MMVUiLCJuIjoib3BlcmF0b3ItbmV3IiwiaWQiOjF9"' + kubectl_bin patch secret my-cluster-secrets --type merge --patch '{"stringData": {"pmmserverkey": "eyJrIjoiOUFRZFY0UjBmbG8zZVg3TDZEdTNza1VycXZJaE5MMVUiLCJuIjoib3BlcmF0b3ItbmV3IiwiaWQiOjF9"}}' ++ mktemp + local LAST_OUT=/tmp/tmp.LCtzYhe0zg ++ mktemp + local LAST_ERR=/tmp/tmp.PwVWu6Cg1c + local exit_status=0 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl patch secret my-cluster-secrets --type merge --patch '{"stringData": {"pmmserverkey": "eyJrIjoiOUFRZFY0UjBmbG8zZVg3TDZEdTNza1VycXZJaE5MMVUiLCJuIjoib3BlcmF0b3ItbmV3IiwiaWQiOjF9"}}' + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.LCtzYhe0zg secret/my-cluster-secrets patched + cat /tmp/tmp.PwVWu6Cg1c + rm /tmp/tmp.LCtzYhe0zg /tmp/tmp.PwVWu6Cg1c + return 0 + desc 'delete old PMM key' + set +o xtrace ----------------------------------------------------------------------------------- delete old PMM key ----------------------------------------------------------------------------------- ++ jq '.[] | select( .name == "operator").id' +++ get_service_endpoint monitoring-service +++ local service=monitoring-service ++++ kubectl_bin get service/monitoring-service -o 'jsonpath={.status.loadBalancer.ingress[].hostname}' +++++ mktemp ++++ local LAST_OUT=/tmp/tmp.JYvlJj5ksa +++++ mktemp ++++ local LAST_ERR=/tmp/tmp.WD5bSq6piL ++++ local exit_status=0 +++++ seq 0 2 ++++ for i in $(seq 0 2) ++++ set +e ++++ kubectl get service/monitoring-service -o 'jsonpath={.status.loadBalancer.ingress[].hostname}' ++++ exit_status=0 ++++ set -e ++++ '[' 0 '!=' 0 ']' ++++ break ++++ cat /tmp/tmp.JYvlJj5ksa ++++ cat /tmp/tmp.WD5bSq6piL ++++ rm /tmp/tmp.JYvlJj5ksa /tmp/tmp.WD5bSq6piL ++++ return 0 +++ local endpoint= +++ '[' -z '' ']' ++++ kubectl_bin get service/monitoring-service -o 'jsonpath={.status.loadBalancer.ingress[].ip}' +++++ mktemp ++++ local LAST_OUT=/tmp/tmp.oYvCnLYAdn +++++ mktemp ++++ local LAST_ERR=/tmp/tmp.KWoRYMnuCn ++++ local exit_status=0 +++++ seq 0 2 ++++ for i in $(seq 0 2) ++++ set +e ++++ kubectl get service/monitoring-service -o 'jsonpath={.status.loadBalancer.ingress[].ip}' ++++ exit_status=0 ++++ set -e ++++ '[' 0 '!=' 0 ']' ++++ break ++++ cat /tmp/tmp.oYvCnLYAdn ++++ cat /tmp/tmp.KWoRYMnuCn ++++ rm /tmp/tmp.oYvCnLYAdn /tmp/tmp.KWoRYMnuCn ++++ return 0 +++ endpoint=34.71.161.14 +++ '[' -n 34.71.161.14 ']' +++ '[' 34.71.161.14 '!=' null ']' +++ echo 34.71.161.14 +++ sed -e 's/^"//; s/"$//;' +++ head -n 1 +++ return ++ curl --insecure -X GET https://admin:admin@34.71.161.14/graph/api/auth/keys % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 596 100 596 0 0 1793 0 --:--:-- --:--:-- --:--:-- 1795 + ID_API_KEY_OLD=6 ++ get_service_endpoint monitoring-service ++ local service=monitoring-service +++ kubectl_bin get service/monitoring-service -o 'jsonpath={.status.loadBalancer.ingress[].hostname}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.KSLUcuyggc ++++ mktemp +++ local LAST_ERR=/tmp/tmp.HTu9hBzUZQ +++ local exit_status=0 ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl get service/monitoring-service -o 'jsonpath={.status.loadBalancer.ingress[].hostname}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 ']' +++ break +++ cat /tmp/tmp.KSLUcuyggc +++ cat /tmp/tmp.HTu9hBzUZQ +++ rm /tmp/tmp.KSLUcuyggc /tmp/tmp.HTu9hBzUZQ +++ return 0 ++ local endpoint= ++ '[' -z '' ']' +++ kubectl_bin get service/monitoring-service -o 'jsonpath={.status.loadBalancer.ingress[].ip}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.n66JRH4Oqp ++++ mktemp +++ local LAST_ERR=/tmp/tmp.zNkppmfMw5 +++ local exit_status=0 ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl get service/monitoring-service -o 'jsonpath={.status.loadBalancer.ingress[].ip}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 ']' +++ break +++ cat /tmp/tmp.n66JRH4Oqp +++ cat /tmp/tmp.zNkppmfMw5 +++ rm /tmp/tmp.n66JRH4Oqp /tmp/tmp.zNkppmfMw5 +++ return 0 ++ endpoint=34.71.161.14 ++ '[' -n 34.71.161.14 ']' ++ '[' 34.71.161.14 '!=' null ']' ++ echo 34.71.161.14 ++ head -n 1 ++ sed -e 's/^"//; s/"$//;' ++ return + curl --insecure -X DELETE https://admin:admin@34.71.161.14/graph/api/auth/keys/6 % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 29 100 29 0 0 88 0 --:--:-- --:--:-- --:--:-- 88 {"message":"API key deleted"}+ wait_for_generation sts/monitoring-pxc 4 + local resource=sts/monitoring-pxc + local target_generation=4 + echo 'Waiting for sts/monitoring-pxc to reach generation 4...' Waiting for sts/monitoring-pxc to reach generation 4... + true ++ kubectl get sts/monitoring-pxc -o 'jsonpath={.metadata.generation}' + current_generation=3 + '[' 3 -eq 4 ']' + echo 'Resource sts/monitoring-pxc is at generation 3. Waiting...' Resource sts/monitoring-pxc is at generation 3. Waiting... + sleep 5 + true ++ kubectl get sts/monitoring-pxc -o 'jsonpath={.metadata.generation}' + current_generation=3 + '[' 3 -eq 4 ']' + echo 'Resource sts/monitoring-pxc is at generation 3. Waiting...' Resource sts/monitoring-pxc is at generation 3. Waiting... + sleep 5 + true ++ kubectl get sts/monitoring-pxc -o 'jsonpath={.metadata.generation}' + current_generation=3 + '[' 3 -eq 4 ']' + echo 'Resource sts/monitoring-pxc is at generation 3. Waiting...' Resource sts/monitoring-pxc is at generation 3. Waiting... + sleep 5 + true ++ kubectl get sts/monitoring-pxc -o 'jsonpath={.metadata.generation}' + current_generation=3 + '[' 3 -eq 4 ']' + echo 'Resource sts/monitoring-pxc is at generation 3. Waiting...' Resource sts/monitoring-pxc is at generation 3. Waiting... + sleep 5 + true ++ kubectl get sts/monitoring-pxc -o 'jsonpath={.metadata.generation}' + current_generation=3 + '[' 3 -eq 4 ']' + echo 'Resource sts/monitoring-pxc is at generation 3. Waiting...' Resource sts/monitoring-pxc is at generation 3. Waiting... + sleep 5 + true ++ kubectl get sts/monitoring-pxc -o 'jsonpath={.metadata.generation}' + current_generation=3 + '[' 3 -eq 4 ']' + echo 'Resource sts/monitoring-pxc is at generation 3. Waiting...' Resource sts/monitoring-pxc is at generation 3. Waiting... + sleep 5 + true ++ kubectl get sts/monitoring-pxc -o 'jsonpath={.metadata.generation}' + current_generation=3 + '[' 3 -eq 4 ']' + echo 'Resource sts/monitoring-pxc is at generation 3. Waiting...' Resource sts/monitoring-pxc is at generation 3. Waiting... + sleep 5 + true ++ kubectl get sts/monitoring-pxc -o 'jsonpath={.metadata.generation}' + current_generation=3 + '[' 3 -eq 4 ']' + echo 'Resource sts/monitoring-pxc is at generation 3. Waiting...' Resource sts/monitoring-pxc is at generation 3. Waiting... + sleep 5 + true ++ kubectl get sts/monitoring-pxc -o 'jsonpath={.metadata.generation}' + current_generation=3 + '[' 3 -eq 4 ']' + echo 'Resource sts/monitoring-pxc is at generation 3. Waiting...' Resource sts/monitoring-pxc is at generation 3. Waiting... + sleep 5 + true ++ kubectl get sts/monitoring-pxc -o 'jsonpath={.metadata.generation}' + current_generation=3 + '[' 3 -eq 4 ']' + echo 'Resource sts/monitoring-pxc is at generation 3. Waiting...' Resource sts/monitoring-pxc is at generation 3. Waiting... + sleep 5 + true ++ kubectl get sts/monitoring-pxc -o 'jsonpath={.metadata.generation}' + current_generation=3 + '[' 3 -eq 4 ']' + echo 'Resource sts/monitoring-pxc is at generation 3. Waiting...' Resource sts/monitoring-pxc is at generation 3. Waiting... + sleep 5 + true ++ kubectl get sts/monitoring-pxc -o 'jsonpath={.metadata.generation}' + current_generation=3 + '[' 3 -eq 4 ']' + echo 'Resource sts/monitoring-pxc is at generation 3. Waiting...' Resource sts/monitoring-pxc is at generation 3. Waiting... + sleep 5 + true ++ kubectl get sts/monitoring-pxc -o 'jsonpath={.metadata.generation}' + current_generation=3 + '[' 3 -eq 4 ']' + echo 'Resource sts/monitoring-pxc is at generation 3. Waiting...' Resource sts/monitoring-pxc is at generation 3. Waiting... + sleep 5 + true ++ kubectl get sts/monitoring-pxc -o 'jsonpath={.metadata.generation}' + current_generation=3 + '[' 3 -eq 4 ']' + echo 'Resource sts/monitoring-pxc is at generation 3. Waiting...' Resource sts/monitoring-pxc is at generation 3. Waiting... + sleep 5 + true ++ kubectl get sts/monitoring-pxc -o 'jsonpath={.metadata.generation}' + current_generation=3 + '[' 3 -eq 4 ']' + echo 'Resource sts/monitoring-pxc is at generation 3. Waiting...' Resource sts/monitoring-pxc is at generation 3. Waiting... + sleep 5 + true ++ kubectl get sts/monitoring-pxc -o 'jsonpath={.metadata.generation}' + current_generation=3 + '[' 3 -eq 4 ']' + echo 'Resource sts/monitoring-pxc is at generation 3. Waiting...' Resource sts/monitoring-pxc is at generation 3. Waiting... + sleep 5 + true ++ kubectl get sts/monitoring-pxc -o 'jsonpath={.metadata.generation}' + current_generation=3 + '[' 3 -eq 4 ']' + echo 'Resource sts/monitoring-pxc is at generation 3. Waiting...' Resource sts/monitoring-pxc is at generation 3. Waiting... + sleep 5 + true ++ kubectl get sts/monitoring-pxc -o 'jsonpath={.metadata.generation}' + current_generation=3 + '[' 3 -eq 4 ']' + echo 'Resource sts/monitoring-pxc is at generation 3. Waiting...' Resource sts/monitoring-pxc is at generation 3. Waiting... + sleep 5 + true ++ kubectl get sts/monitoring-pxc -o 'jsonpath={.metadata.generation}' + current_generation=3 + '[' 3 -eq 4 ']' + echo 'Resource sts/monitoring-pxc is at generation 3. Waiting...' Resource sts/monitoring-pxc is at generation 3. Waiting... + sleep 5 + true ++ kubectl get sts/monitoring-pxc -o 'jsonpath={.metadata.generation}' + current_generation=3 + '[' 3 -eq 4 ']' + echo 'Resource sts/monitoring-pxc is at generation 3. Waiting...' Resource sts/monitoring-pxc is at generation 3. Waiting... + sleep 5 + true ++ kubectl get sts/monitoring-pxc -o 'jsonpath={.metadata.generation}' + current_generation=3 + '[' 3 -eq 4 ']' + echo 'Resource sts/monitoring-pxc is at generation 3. Waiting...' Resource sts/monitoring-pxc is at generation 3. Waiting... + sleep 5 + true ++ kubectl get sts/monitoring-pxc -o 'jsonpath={.metadata.generation}' + current_generation=3 + '[' 3 -eq 4 ']' + echo 'Resource sts/monitoring-pxc is at generation 3. Waiting...' Resource sts/monitoring-pxc is at generation 3. Waiting... + sleep 5 + true ++ kubectl get sts/monitoring-pxc -o 'jsonpath={.metadata.generation}' + current_generation=3 + '[' 3 -eq 4 ']' + echo 'Resource sts/monitoring-pxc is at generation 3. Waiting...' Resource sts/monitoring-pxc is at generation 3. Waiting... + sleep 5 + true ++ kubectl get sts/monitoring-pxc -o 'jsonpath={.metadata.generation}' + current_generation=4 + '[' 4 -eq 4 ']' + echo 'Resource sts/monitoring-pxc has reached generation 4.' Resource sts/monitoring-pxc has reached generation 4. + break + wait_for_generation sts/monitoring-haproxy 4 + local resource=sts/monitoring-haproxy + local target_generation=4 + echo 'Waiting for sts/monitoring-haproxy to reach generation 4...' Waiting for sts/monitoring-haproxy to reach generation 4... + true ++ kubectl get sts/monitoring-haproxy -o 'jsonpath={.metadata.generation}' + current_generation=4 + '[' 4 -eq 4 ']' + echo 'Resource sts/monitoring-haproxy has reached generation 4.' Resource sts/monitoring-haproxy has reached generation 4. + break + sleep 10 + kubectl wait pod -l app.kubernetes.io/managed-by=percona-xtradb-cluster-operator --for=condition=ready --timeout=600s pod/monitoring-haproxy-0 condition met pod/monitoring-haproxy-1 condition met pod/monitoring-pxc-0 condition met pod/monitoring-pxc-1 condition met pod/monitoring-pxc-2 condition met + desc 'check if pmm-client container enabled' + set +o xtrace ----------------------------------------------------------------------------------- check if pmm-client container enabled ----------------------------------------------------------------------------------- + compare_kubectl statefulset/monitoring-pxc + local resource=statefulset/monitoring-pxc + local postfix= + local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/e2e-tests/monitoring-2-0/compare/statefulset_monitoring-pxc.yml + local new_result=/tmp/tmp.oE2g1q4ApT/statefulset_monitoring-pxc.yml + desc 'compare statefulset/monitoring-pxc-' + set +o xtrace ----------------------------------------------------------------------------------- compare statefulset/monitoring-pxc- ----------------------------------------------------------------------------------- + '[' 0 = 1 -a -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/e2e-tests/monitoring-2-0/compare/statefulset_monitoring-pxc-eks.yml ']' + [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 8\.0 ]] + '[' -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/e2e-tests/monitoring-2-0/compare/statefulset_monitoring-pxc-80.yml ']' + [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 8\.4 ]] + version_gt 1.33 ++ echo '1.34 >= 1.33' ++ bc -l + '[' 1 -eq 1 ']' + return 0 + '[' -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/e2e-tests/monitoring-2-0/compare/statefulset_monitoring-pxc-k133.yml ']' + version_gt 1.29 ++ echo '1.34 >= 1.29' ++ bc -l + '[' 1 -eq 1 ']' + return 0 + '[' -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/e2e-tests/monitoring-2-0/compare/statefulset_monitoring-pxc-k129.yml ']' + version_gt 1.27 ++ echo '1.34 >= 1.27' ++ bc -l + '[' 1 -eq 1 ']' + return 0 + '[' -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/e2e-tests/monitoring-2-0/compare/statefulset_monitoring-pxc-k127.yml ']' + expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/e2e-tests/monitoring-2-0/compare/statefulset_monitoring-pxc-k127.yml + '[' '!' -z '' -a -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/e2e-tests/monitoring-2-0/compare/statefulset_monitoring-pxc-k127-oc.yml ']' + version_gt 1.29 ++ echo '1.34 >= 1.29' ++ bc -l + '[' 1 -eq 1 ']' + return 0 + '[' '!' -z '' -a -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/e2e-tests/monitoring-2-0/compare/statefulset_monitoring-pxc-k127-k129-oc.yml ']' + '[' 0 = 1 -a -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/e2e-tests/monitoring-2-0/compare/statefulset_monitoring-pxc-k127-eks.yml ']' + '[' 0 = 1 -a -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/e2e-tests/monitoring-2-0/compare/statefulset_monitoring-pxc-k127-aks.yml ']' + kubectl_bin get -o yaml statefulset/monitoring-pxc + yq eval ' del(.metadata.managedFields) | del(.. | select(has("creationTimestamp")).creationTimestamp) | del(.. | select(has("namespace")).namespace) | del(.. | select(has("uid")).uid) | del(.metadata.resourceVersion) | del(.spec.template.spec.containers[].env[] | select(.name == "CLUSTER_HASH")) | del(.spec.template.spec.containers[].env[] | select(.name == "S3_BUCKET_PATH")) | del(.spec.template.spec.containers[].env[] | select(.name == "BACKUP_PATH")) | del(.spec.template.spec.containers[].env[] | select(.name == "S3_BUCKET_URL")) | del(.spec.template.spec.containers[].env[] | select(.name == "AZURE_CONTAINER_NAME")) | del(.spec.template.spec.containers[].env[] | select(.name == "XTRABACKUP_ENABLED")) | del(.metadata.selfLink) | del(.metadata.deletionTimestamp) | del(.metadata.labels."batch.kubernetes.io/job-name") | del(.metadata.labels."job-name") | del(.metadata.labels."topology.kubernetes.io/region") | del(.metadata.labels."topology.kubernetes.io/zone") | del(.metadata.annotations."kubectl.kubernetes.io/last-applied-configuration") | del(.metadata.annotations."kubernetes.io/psp") | del(.metadata.annotations."batch.kubernetes.io/job-tracking") | del(.metadata.annotations."networking.gke.io/target-pool") | del(.metadata.annotations."cloud.google.com/neg") | del(.metadata.annotations."k8s.v1.cni.cncf.io*") | del(.metadata.annotations."k8s.ovn.org/pod-networks") | del(.metadata.annotations."kubernetes.digitalocean.com/load-balancer-id") | del(.metadata.annotations."service.beta.kubernetes.io/do-loadbalancer-type") | del(.spec.template.metadata.annotations."last-applied-secret") | del(.spec.template.metadata.labels."batch.kubernetes.io/job-name") | del(.spec.template.metadata.labels."job-name") | del(.. | select(has("batch.kubernetes.io/controller-uid"))."batch.kubernetes.io/controller-uid") | del(.. | select(has("image")).image) | del(.. | select(has("clusterIP")).clusterIP) | del(.. | select(has("clusterIPs")).clusterIPs) | del(.. | select(has("dataSource")).dataSource) | del(.. | select(has("procMount")).procMount) | del(.. | select(has("storageClassName")).storageClassName) | del(.. | select(has("finalizers")).finalizers) | del(.. | select(has("kubernetes.io/pvc-protection"))."kubernetes.io/pvc-protection") | del(.. | select(has("volumeName")).volumeName) | del(.. | select(has("volume.beta.kubernetes.io/storage-provisioner"))."volume.beta.kubernetes.io/storage-provisioner") | del(.. | select(has("volume.kubernetes.io/storage-provisioner"))."volume.kubernetes.io/storage-provisioner") | del(.spec.volumeMode) | del(.spec.nodeName) | del(.. | select(has("volume.kubernetes.io/selected-node"))."volume.kubernetes.io/selected-node") | del(.. | select(has("percona.com/last-config-hash"))."percona.com/last-config-hash") | del(.. | select(has("percona.com/configuration-hash"))."percona.com/configuration-hash") | del(.. | select(has("percona.com/env-secret-config-hash"))."percona.com/env-secret-config-hash") | del(.. | select(has("percona.com/ssl-hash"))."percona.com/ssl-hash") | del(.. | select(has("percona.com/ssl-internal-hash"))."percona.com/ssl-internal-hash") | del(.. | select(has("kubectl.kubernetes.io/default-container"))."kubectl.kubernetes.io/default-container") | del(.spec.volumeClaimTemplates[].spec.volumeMode | select(. == "Filesystem")) | del(.. | select(has("healthCheckNodePort")).healthCheckNodePort) | del(.. | select(has("nodePort")).nodePort) | del(.. | select(has("imagePullSecrets")).imagePullSecrets) | del(.. | select(has("enableServiceLinks")).enableServiceLinks) | del(.status) | del(.spec.volumeClaimTemplates[].apiVersion) | del(.spec.volumeClaimTemplates[].kind) | del(.metadata.ownerReferences[].apiVersion) | del(.. | select(has("controller-uid")).controller-uid) | del(.. | select(has("preemptionPolicy")).preemptionPolicy) | del(.spec.ipFamilies) | del(.spec.ipFamilyPolicy) | del(.spec.updateStrategy.rollingUpdate.maxUnavailable) | (.. | select(. == "policy/v1beta1")) = "policy/v1" | del(.. | select(has("kubernetes.io/hostname"))."kubernetes.io/hostname") | (.. | select(tag == "!!str")) |= sub("monitoring-2-0-5888", "namespace") | (.. | select(tag == "!!str")) |= sub("kube-api-access-.*", "kube-api-access") | del(.. | select(has("annotations")).annotations | select(length==0)) | del(.spec.crVersion) | del(.. | select(.[] == "percona-xtradb-cluster-operator-workload-token*"))' - ++ mktemp + local LAST_OUT=/tmp/tmp.9uuZYlf3t5 ++ mktemp + local LAST_ERR=/tmp/tmp.QOoEVHE4Xn + local exit_status=0 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl get -o yaml statefulset/monitoring-pxc + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.9uuZYlf3t5 + cat /tmp/tmp.QOoEVHE4Xn + rm /tmp/tmp.9uuZYlf3t5 /tmp/tmp.QOoEVHE4Xn + return 0 + [[ 0 -eq 0 ]] + diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/e2e-tests/monitoring-2-0/compare/statefulset_monitoring-pxc-k127.yml /tmp/tmp.oE2g1q4ApT/statefulset_monitoring-pxc.yml + log 'compare_kubectl: statefulset/monitoring-pxc OK' ++ date +%Y-%m-%dT%H:%M:%S%z + echo '[2026-10-05T13:35:58+0000]' compare_kubectl: statefulset/monitoring-pxc OK [2026-10-05T13:35:58+0000] compare_kubectl: statefulset/monitoring-pxc OK + compare_kubectl statefulset/monitoring-haproxy + local resource=statefulset/monitoring-haproxy + local postfix= + local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/e2e-tests/monitoring-2-0/compare/statefulset_monitoring-haproxy.yml + local new_result=/tmp/tmp.oE2g1q4ApT/statefulset_monitoring-haproxy.yml + desc 'compare statefulset/monitoring-haproxy-' + set +o xtrace ----------------------------------------------------------------------------------- compare statefulset/monitoring-haproxy- ----------------------------------------------------------------------------------- + '[' 0 = 1 -a -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/e2e-tests/monitoring-2-0/compare/statefulset_monitoring-haproxy-eks.yml ']' + [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 8\.0 ]] + '[' -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/e2e-tests/monitoring-2-0/compare/statefulset_monitoring-haproxy-80.yml ']' + [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 8\.4 ]] + version_gt 1.33 ++ echo '1.34 >= 1.33' ++ bc -l + '[' 1 -eq 1 ']' + return 0 + '[' -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/e2e-tests/monitoring-2-0/compare/statefulset_monitoring-haproxy-k133.yml ']' + version_gt 1.29 ++ echo '1.34 >= 1.29' ++ bc -l + '[' 1 -eq 1 ']' + return 0 + '[' -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/e2e-tests/monitoring-2-0/compare/statefulset_monitoring-haproxy-k129.yml ']' + version_gt 1.27 ++ echo '1.34 >= 1.27' ++ bc -l + '[' 1 -eq 1 ']' + return 0 + '[' -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/e2e-tests/monitoring-2-0/compare/statefulset_monitoring-haproxy-k127.yml ']' + expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/e2e-tests/monitoring-2-0/compare/statefulset_monitoring-haproxy-k127.yml + '[' '!' -z '' -a -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/e2e-tests/monitoring-2-0/compare/statefulset_monitoring-haproxy-k127-oc.yml ']' + version_gt 1.29 ++ echo '1.34 >= 1.29' ++ bc -l + '[' 1 -eq 1 ']' + return 0 + '[' '!' -z '' -a -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/e2e-tests/monitoring-2-0/compare/statefulset_monitoring-haproxy-k127-k129-oc.yml ']' + '[' 0 = 1 -a -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/e2e-tests/monitoring-2-0/compare/statefulset_monitoring-haproxy-k127-eks.yml ']' + '[' 0 = 1 -a -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/e2e-tests/monitoring-2-0/compare/statefulset_monitoring-haproxy-k127-aks.yml ']' + kubectl_bin get -o yaml statefulset/monitoring-haproxy + yq eval ' del(.metadata.managedFields) | del(.. | select(has("creationTimestamp")).creationTimestamp) | del(.. | select(has("namespace")).namespace) | del(.. | select(has("uid")).uid) | del(.metadata.resourceVersion) | del(.spec.template.spec.containers[].env[] | select(.name == "CLUSTER_HASH")) | del(.spec.template.spec.containers[].env[] | select(.name == "S3_BUCKET_PATH")) | del(.spec.template.spec.containers[].env[] | select(.name == "BACKUP_PATH")) | del(.spec.template.spec.containers[].env[] | select(.name == "S3_BUCKET_URL")) | del(.spec.template.spec.containers[].env[] | select(.name == "AZURE_CONTAINER_NAME")) | del(.spec.template.spec.containers[].env[] | select(.name == "XTRABACKUP_ENABLED")) | del(.metadata.selfLink) | del(.metadata.deletionTimestamp) | del(.metadata.labels."batch.kubernetes.io/job-name") | del(.metadata.labels."job-name") | del(.metadata.labels."topology.kubernetes.io/region") | del(.metadata.labels."topology.kubernetes.io/zone") | del(.metadata.annotations."kubectl.kubernetes.io/last-applied-configuration") | del(.metadata.annotations."kubernetes.io/psp") | del(.metadata.annotations."batch.kubernetes.io/job-tracking") | del(.metadata.annotations."networking.gke.io/target-pool") | del(.metadata.annotations."cloud.google.com/neg") | del(.metadata.annotations."k8s.v1.cni.cncf.io*") | del(.metadata.annotations."k8s.ovn.org/pod-networks") | del(.metadata.annotations."kubernetes.digitalocean.com/load-balancer-id") | del(.metadata.annotations."service.beta.kubernetes.io/do-loadbalancer-type") | del(.spec.template.metadata.annotations."last-applied-secret") | del(.spec.template.metadata.labels."batch.kubernetes.io/job-name") | del(.spec.template.metadata.labels."job-name") | del(.. | select(has("batch.kubernetes.io/controller-uid"))."batch.kubernetes.io/controller-uid") | del(.. | select(has("image")).image) | del(.. | select(has("clusterIP")).clusterIP) | del(.. | select(has("clusterIPs")).clusterIPs) | del(.. | select(has("dataSource")).dataSource) | del(.. | select(has("procMount")).procMount) | del(.. | select(has("storageClassName")).storageClassName) | del(.. | select(has("finalizers")).finalizers) | del(.. | select(has("kubernetes.io/pvc-protection"))."kubernetes.io/pvc-protection") | del(.. | select(has("volumeName")).volumeName) | del(.. | select(has("volume.beta.kubernetes.io/storage-provisioner"))."volume.beta.kubernetes.io/storage-provisioner") | del(.. | select(has("volume.kubernetes.io/storage-provisioner"))."volume.kubernetes.io/storage-provisioner") | del(.spec.volumeMode) | del(.spec.nodeName) | del(.. | select(has("volume.kubernetes.io/selected-node"))."volume.kubernetes.io/selected-node") | del(.. | select(has("percona.com/last-config-hash"))."percona.com/last-config-hash") | del(.. | select(has("percona.com/configuration-hash"))."percona.com/configuration-hash") | del(.. | select(has("percona.com/env-secret-config-hash"))."percona.com/env-secret-config-hash") | del(.. | select(has("percona.com/ssl-hash"))."percona.com/ssl-hash") | del(.. | select(has("percona.com/ssl-internal-hash"))."percona.com/ssl-internal-hash") | del(.. | select(has("kubectl.kubernetes.io/default-container"))."kubectl.kubernetes.io/default-container") | del(.spec.volumeClaimTemplates[].spec.volumeMode | select(. == "Filesystem")) | del(.. | select(has("healthCheckNodePort")).healthCheckNodePort) | del(.. | select(has("nodePort")).nodePort) | del(.. | select(has("imagePullSecrets")).imagePullSecrets) | del(.. | select(has("enableServiceLinks")).enableServiceLinks) | del(.status) | del(.spec.volumeClaimTemplates[].apiVersion) | del(.spec.volumeClaimTemplates[].kind) | del(.metadata.ownerReferences[].apiVersion) | del(.. | select(has("controller-uid")).controller-uid) | del(.. | select(has("preemptionPolicy")).preemptionPolicy) | del(.spec.ipFamilies) | del(.spec.ipFamilyPolicy) | del(.spec.updateStrategy.rollingUpdate.maxUnavailable) | (.. | select(. == "policy/v1beta1")) = "policy/v1" | del(.. | select(has("kubernetes.io/hostname"))."kubernetes.io/hostname") | (.. | select(tag == "!!str")) |= sub("monitoring-2-0-5888", "namespace") | (.. | select(tag == "!!str")) |= sub("kube-api-access-.*", "kube-api-access") | del(.. | select(has("annotations")).annotations | select(length==0)) | del(.spec.crVersion) | del(.. | select(.[] == "percona-xtradb-cluster-operator-workload-token*"))' - ++ mktemp + local LAST_OUT=/tmp/tmp.MGjJ1TB0kg ++ mktemp + local LAST_ERR=/tmp/tmp.3DrCs0ubit + local exit_status=0 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl get -o yaml statefulset/monitoring-haproxy + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.MGjJ1TB0kg + cat /tmp/tmp.3DrCs0ubit + rm /tmp/tmp.MGjJ1TB0kg /tmp/tmp.3DrCs0ubit + return 0 + [[ 0 -eq 0 ]] + diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/e2e-tests/monitoring-2-0/compare/statefulset_monitoring-haproxy-k127.yml /tmp/tmp.oE2g1q4ApT/statefulset_monitoring-haproxy.yml + log 'compare_kubectl: statefulset/monitoring-haproxy OK' ++ date +%Y-%m-%dT%H:%M:%S%z + echo '[2026-10-05T13:35:58+0000]' compare_kubectl: statefulset/monitoring-haproxy OK [2026-10-05T13:35:58+0000] compare_kubectl: statefulset/monitoring-haproxy OK + desc 'verify clients agents statuses' + set +o xtrace ----------------------------------------------------------------------------------- verify clients agents statuses ----------------------------------------------------------------------------------- + sleep 300 ++ getSecretData my-cluster-secrets pmmserverkey ++ local secretName=my-cluster-secrets ++ local dataKey=pmmserverkey ++ base64 --decode ++ kubectl_bin get secrets/my-cluster-secrets '--template={{.data.pmmserverkey}}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.eVnU4ITkzc +++ mktemp ++ local LAST_ERR=/tmp/tmp.EBgqWPGlbu ++ local exit_status=0 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get secrets/my-cluster-secrets '--template={{.data.pmmserverkey}}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.eVnU4ITkzc ++ cat /tmp/tmp.EBgqWPGlbu ++ rm /tmp/tmp.eVnU4ITkzc /tmp/tmp.EBgqWPGlbu ++ return 0 + API_KEY=eyJrIjoiOUFRZFY0UjBmbG8zZVg3TDZEdTNza1VycXZJaE5MMVUiLCJuIjoib3BlcmF0b3ItbmV3IiwiaWQiOjF9 +++ get_service_endpoint monitoring-service +++ local service=monitoring-service ++++ kubectl_bin get service/monitoring-service -o 'jsonpath={.status.loadBalancer.ingress[].hostname}' +++++ mktemp ++++ local LAST_OUT=/tmp/tmp.PddNyCN6X3 +++++ mktemp ++++ local LAST_ERR=/tmp/tmp.2gVn298K97 ++++ local exit_status=0 +++++ seq 0 2 ++++ for i in $(seq 0 2) ++++ set +e ++++ kubectl get service/monitoring-service -o 'jsonpath={.status.loadBalancer.ingress[].hostname}' ++++ exit_status=0 ++++ set -e ++++ '[' 0 '!=' 0 ']' ++++ break ++++ cat /tmp/tmp.PddNyCN6X3 ++++ cat /tmp/tmp.2gVn298K97 ++++ rm /tmp/tmp.PddNyCN6X3 /tmp/tmp.2gVn298K97 ++++ return 0 +++ local endpoint= +++ '[' -z '' ']' ++++ kubectl_bin get service/monitoring-service -o 'jsonpath={.status.loadBalancer.ingress[].ip}' +++++ mktemp ++++ local LAST_OUT=/tmp/tmp.WnqJkAFKQz +++++ mktemp ++++ local LAST_ERR=/tmp/tmp.PiZTHXXoM8 ++++ local exit_status=0 +++++ seq 0 2 ++++ for i in $(seq 0 2) ++++ set +e ++++ kubectl get service/monitoring-service -o 'jsonpath={.status.loadBalancer.ingress[].ip}' ++++ exit_status=0 ++++ set -e ++++ '[' 0 '!=' 0 ']' ++++ break ++++ cat /tmp/tmp.WnqJkAFKQz ++++ cat /tmp/tmp.PiZTHXXoM8 ++++ rm /tmp/tmp.WnqJkAFKQz /tmp/tmp.PiZTHXXoM8 ++++ return 0 +++ endpoint=34.71.161.14 +++ '[' -n 34.71.161.14 ']' +++ '[' 34.71.161.14 '!=' null ']' +++ echo 34.71.161.14 +++ head -n 1 +++ sed -e 's/^"//; s/"$//;' +++ return ++ get_mgmnt_service_list eyJrIjoiOUFRZFY0UjBmbG8zZVg3TDZEdTNza1VycXZJaE5MMVUiLCJuIjoib3BlcmF0b3ItbmV3IiwiaWQiOjF9 34.71.161.14 monitoring-2-0-5888 ++ local api_key=eyJrIjoiOUFRZFY0UjBmbG8zZVg3TDZEdTNza1VycXZJaE5MMVUiLCJuIjoib3BlcmF0b3ItbmV3IiwiaWQiOjF9 ++ local endpoint=34.71.161.14 ++ local namespace=monitoring-2-0-5888 ++ curl -s -k -H 'Authorization: Bearer eyJrIjoiOUFRZFY0UjBmbG8zZVg3TDZEdTNza1VycXZJaE5MMVUiLCJuIjoib3BlcmF0b3ItbmV3IiwiaWQiOjF9' -X POST https://34.71.161.14/v1/management/Service/List ++ jq 'walk(if type=="object" then with_entries(select(.key | test("service_id|node_id|agent_id|created_at|updated_at") | not)) else . end)' ++ jq 'walk(if type == "array" then sort_by(.agent_type) else . end)' ++ /usr/bin/sed -i s/monitoring-2-0-5888-//g /tmp/tmp.oE2g1q4ApT/active_pmm_agents.json ++ cat /tmp/tmp.oE2g1q4ApT/active_pmm_agents.json ++ jq '.services | sort_by(.node_name)' ++ echo /tmp/tmp.oE2g1q4ApT/active_pmm_agents_sorted.json + diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/e2e-tests/monitoring-2-0/compare/agents-list.json /tmp/tmp.oE2g1q4ApT/active_pmm_agents_sorted.json + desc 'check mysql metrics' + set +o xtrace ----------------------------------------------------------------------------------- check mysql metrics ----------------------------------------------------------------------------------- + get_metric_values node_boot_time_seconds pxc-prefix-monitoring-2-0-5888-monitoring-pxc-0 admin:admin + local metric=node_boot_time_seconds + local instance=pxc-prefix-monitoring-2-0-5888-monitoring-pxc-0 + local user_pass=admin:admin ++ /usr/bin/date -u +%s -d '-1 minute' + local start=1791207603 ++ /usr/bin/date -u +%s + local end=1791207663 ++ get_service_endpoint monitoring-service ++ local service=monitoring-service +++ kubectl_bin get service/monitoring-service -o 'jsonpath={.status.loadBalancer.ingress[].hostname}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.ChpwbtdHnT ++++ mktemp +++ local LAST_ERR=/tmp/tmp.HEt590ZPTE +++ local exit_status=0 ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl get service/monitoring-service -o 'jsonpath={.status.loadBalancer.ingress[].hostname}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 ']' +++ break +++ cat /tmp/tmp.ChpwbtdHnT +++ cat /tmp/tmp.HEt590ZPTE +++ rm /tmp/tmp.ChpwbtdHnT /tmp/tmp.HEt590ZPTE +++ return 0 ++ local endpoint= ++ '[' -z '' ']' +++ kubectl_bin get service/monitoring-service -o 'jsonpath={.status.loadBalancer.ingress[].ip}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.zvcnlqaWMY ++++ mktemp +++ local LAST_ERR=/tmp/tmp.6tmp3d8b87 +++ local exit_status=0 ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl get service/monitoring-service -o 'jsonpath={.status.loadBalancer.ingress[].ip}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 ']' +++ break +++ cat /tmp/tmp.zvcnlqaWMY +++ cat /tmp/tmp.6tmp3d8b87 +++ rm /tmp/tmp.zvcnlqaWMY /tmp/tmp.6tmp3d8b87 +++ return 0 ++ endpoint=34.71.161.14 ++ '[' -n 34.71.161.14 ']' ++ '[' 34.71.161.14 '!=' null ']' ++ head -n 1 ++ echo 34.71.161.14 ++ sed -e 's/^"//; s/"$//;' ++ return + local endpoint=34.71.161.14 ++ curl -s -k 'https://admin:admin@34.71.161.14/graph/api/datasources/proxy/1/api/v1/query_range?query=min%28node_boot_time_seconds%7Bnode_name%3D%7E%22pxc-prefix-monitoring-2-0-5888-monitoring-pxc-0%22%7d%20or%20node_boot_time_seconds%7Bnode_name%3D%7E%22pxc-prefix-monitoring-2-0-5888-monitoring-pxc-0%22%7D%29&start=1791207603&end=1791207663&step=60' ++ jq '.data.result[0]' + local 'result={ "metric": {}, "values": [ [ 1791207603, "1791200721" ], [ 1791207663, "1791200721" ] ] }' + '[' '{ "metric": {}, "values": [ [ 1791207603, "1791200721" ], [ 1791207663, "1791200721" ] ] }' = null ']' + echo -n '{ "metric": {}, "values": [ [ 1791207603, "1791200721" ], [ 1791207663, "1791200721" ] ] }' + grep '^"[0-9]' + jq '.values[][1]' "1791200721" "1791200721" + get_metric_values mysql_global_status_uptime pxc-prefix-monitoring-2-0-5888-monitoring-pxc-0 admin:admin + local metric=mysql_global_status_uptime + local instance=pxc-prefix-monitoring-2-0-5888-monitoring-pxc-0 + local user_pass=admin:admin ++ /usr/bin/date -u +%s -d '-1 minute' + local start=1791207606 ++ /usr/bin/date -u +%s + local end=1791207666 ++ get_service_endpoint monitoring-service ++ local service=monitoring-service +++ kubectl_bin get service/monitoring-service -o 'jsonpath={.status.loadBalancer.ingress[].hostname}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.3jziPuV880 ++++ mktemp +++ local LAST_ERR=/tmp/tmp.iVZUob024N +++ local exit_status=0 ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl get service/monitoring-service -o 'jsonpath={.status.loadBalancer.ingress[].hostname}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 ']' +++ break +++ cat /tmp/tmp.3jziPuV880 +++ cat /tmp/tmp.iVZUob024N +++ rm /tmp/tmp.3jziPuV880 /tmp/tmp.iVZUob024N +++ return 0 ++ local endpoint= ++ '[' -z '' ']' +++ kubectl_bin get service/monitoring-service -o 'jsonpath={.status.loadBalancer.ingress[].ip}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.UBb8Es1eev ++++ mktemp +++ local LAST_ERR=/tmp/tmp.jM839XT60Q +++ local exit_status=0 ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl get service/monitoring-service -o 'jsonpath={.status.loadBalancer.ingress[].ip}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 ']' +++ break +++ cat /tmp/tmp.UBb8Es1eev +++ cat /tmp/tmp.jM839XT60Q +++ rm /tmp/tmp.UBb8Es1eev /tmp/tmp.jM839XT60Q +++ return 0 ++ endpoint=34.71.161.14 ++ '[' -n 34.71.161.14 ']' ++ '[' 34.71.161.14 '!=' null ']' ++ echo 34.71.161.14 ++ head -n 1 ++ sed -e 's/^"//; s/"$//;' ++ return + local endpoint=34.71.161.14 ++ curl -s -k 'https://admin:admin@34.71.161.14/graph/api/datasources/proxy/1/api/v1/query_range?query=min%28mysql_global_status_uptime%7Bnode_name%3D%7E%22pxc-prefix-monitoring-2-0-5888-monitoring-pxc-0%22%7d%20or%20mysql_global_status_uptime%7Bnode_name%3D%7E%22pxc-prefix-monitoring-2-0-5888-monitoring-pxc-0%22%7D%29&start=1791207606&end=1791207666&step=60' ++ jq '.data.result[0]' + local 'result={ "metric": {}, "values": [ [ 1791207606, "153" ], [ 1791207666, "213" ] ] }' + '[' '{ "metric": {}, "values": [ [ 1791207606, "153" ], [ 1791207666, "213" ] ] }' = null ']' + echo -n '{ "metric": {}, "values": [ [ 1791207606, "153" ], [ 1791207666, "213" ] ] }' + jq '.values[][1]' + grep '^"[0-9]' "153" "213" + desc 'check haproxy metrics' + set +o xtrace ----------------------------------------------------------------------------------- check haproxy metrics ----------------------------------------------------------------------------------- + get_metric_values haproxy_backend_status pxc-prefix-monitoring-2-0-5888-monitoring-haproxy-0 admin:admin + local metric=haproxy_backend_status + local instance=pxc-prefix-monitoring-2-0-5888-monitoring-haproxy-0 + local user_pass=admin:admin ++ /usr/bin/date -u +%s -d '-1 minute' + local start=1791207609 ++ /usr/bin/date -u +%s + local end=1791207669 ++ get_service_endpoint monitoring-service ++ local service=monitoring-service +++ kubectl_bin get service/monitoring-service -o 'jsonpath={.status.loadBalancer.ingress[].hostname}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.QjAsrdZvc2 ++++ mktemp +++ local LAST_ERR=/tmp/tmp.5gsXdw8lUC +++ local exit_status=0 ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl get service/monitoring-service -o 'jsonpath={.status.loadBalancer.ingress[].hostname}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 ']' +++ break +++ cat /tmp/tmp.QjAsrdZvc2 +++ cat /tmp/tmp.5gsXdw8lUC +++ rm /tmp/tmp.QjAsrdZvc2 /tmp/tmp.5gsXdw8lUC +++ return 0 ++ local endpoint= ++ '[' -z '' ']' +++ kubectl_bin get service/monitoring-service -o 'jsonpath={.status.loadBalancer.ingress[].ip}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.bG4vT3V1Mp ++++ mktemp +++ local LAST_ERR=/tmp/tmp.z7COVyi9wZ +++ local exit_status=0 ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl get service/monitoring-service -o 'jsonpath={.status.loadBalancer.ingress[].ip}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 ']' +++ break +++ cat /tmp/tmp.bG4vT3V1Mp +++ cat /tmp/tmp.z7COVyi9wZ +++ rm /tmp/tmp.bG4vT3V1Mp /tmp/tmp.z7COVyi9wZ +++ return 0 ++ endpoint=34.71.161.14 ++ '[' -n 34.71.161.14 ']' ++ '[' 34.71.161.14 '!=' null ']' ++ head -n 1 ++ sed -e 's/^"//; s/"$//;' ++ echo 34.71.161.14 ++ return + local endpoint=34.71.161.14 ++ curl -s -k 'https://admin:admin@34.71.161.14/graph/api/datasources/proxy/1/api/v1/query_range?query=min%28haproxy_backend_status%7Bnode_name%3D%7E%22pxc-prefix-monitoring-2-0-5888-monitoring-haproxy-0%22%7d%20or%20haproxy_backend_status%7Bnode_name%3D%7E%22pxc-prefix-monitoring-2-0-5888-monitoring-haproxy-0%22%7D%29&start=1791207609&end=1791207669&step=60' ++ jq '.data.result[0]' + local 'result={ "metric": {}, "values": [ [ 1791207609, "0" ], [ 1791207669, "0" ] ] }' + '[' '{ "metric": {}, "values": [ [ 1791207609, "0" ], [ 1791207669, "0" ] ] }' = null ']' + echo -n '{ "metric": {}, "values": [ [ 1791207609, "0" ], [ 1791207669, "0" ] ] }' + jq '.values[][1]' + grep '^"[0-9]' "0" "0" + get_metric_values haproxy_backend_active_servers pxc-prefix-monitoring-2-0-5888-monitoring-haproxy-0 admin:admin + local metric=haproxy_backend_active_servers + local instance=pxc-prefix-monitoring-2-0-5888-monitoring-haproxy-0 + local user_pass=admin:admin ++ /usr/bin/date -u +%s -d '-1 minute' + local start=1791207611 ++ /usr/bin/date -u +%s + local end=1791207671 ++ get_service_endpoint monitoring-service ++ local service=monitoring-service +++ kubectl_bin get service/monitoring-service -o 'jsonpath={.status.loadBalancer.ingress[].hostname}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.Y4epsyvxOO ++++ mktemp +++ local LAST_ERR=/tmp/tmp.coQpUL5Srb +++ local exit_status=0 ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl get service/monitoring-service -o 'jsonpath={.status.loadBalancer.ingress[].hostname}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 ']' +++ break +++ cat /tmp/tmp.Y4epsyvxOO +++ cat /tmp/tmp.coQpUL5Srb +++ rm /tmp/tmp.Y4epsyvxOO /tmp/tmp.coQpUL5Srb +++ return 0 ++ local endpoint= ++ '[' -z '' ']' +++ kubectl_bin get service/monitoring-service -o 'jsonpath={.status.loadBalancer.ingress[].ip}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.cVYjQBYsza ++++ mktemp +++ local LAST_ERR=/tmp/tmp.01f1OPPaNH +++ local exit_status=0 ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl get service/monitoring-service -o 'jsonpath={.status.loadBalancer.ingress[].ip}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 ']' +++ break +++ cat /tmp/tmp.cVYjQBYsza +++ cat /tmp/tmp.01f1OPPaNH +++ rm /tmp/tmp.cVYjQBYsza /tmp/tmp.01f1OPPaNH +++ return 0 ++ endpoint=34.71.161.14 ++ '[' -n 34.71.161.14 ']' ++ '[' 34.71.161.14 '!=' null ']' ++ echo 34.71.161.14 ++ head -n 1 ++ sed -e 's/^"//; s/"$//;' ++ return + local endpoint=34.71.161.14 ++ curl -s -k 'https://admin:admin@34.71.161.14/graph/api/datasources/proxy/1/api/v1/query_range?query=min%28haproxy_backend_active_servers%7Bnode_name%3D%7E%22pxc-prefix-monitoring-2-0-5888-monitoring-haproxy-0%22%7d%20or%20haproxy_backend_active_servers%7Bnode_name%3D%7E%22pxc-prefix-monitoring-2-0-5888-monitoring-haproxy-0%22%7D%29&start=1791207611&end=1791207671&step=60' ++ jq '.data.result[0]' + local 'result={ "metric": {}, "values": [ [ 1791207611, "1" ], [ 1791207671, "1" ] ] }' + '[' '{ "metric": {}, "values": [ [ 1791207611, "1" ], [ 1791207671, "1" ] ] }' = null ']' + echo -n '{ "metric": {}, "values": [ [ 1791207611, "1" ], [ 1791207671, "1" ] ] }' + jq '.values[][1]' + grep '^"[0-9]' "1" "1" + desc 'check QAN data' + set +o xtrace ----------------------------------------------------------------------------------- check QAN data ----------------------------------------------------------------------------------- + get_qan20_values monitoring-pxc-0 admin:admin + local instance=monitoring-pxc-0 + local user_pass=admin:admin ++ /usr/bin/date -u +%Y-%m-%dT%H:%M:%S -d '-30 minute' + local start=2026-10-05T13:11:14 ++ /usr/bin/date -u +%Y-%m-%dT%H:%M:%S + local end=2026-10-05T13:41:14 ++ get_service_endpoint monitoring-service ++ local service=monitoring-service +++ kubectl_bin get service/monitoring-service -o 'jsonpath={.status.loadBalancer.ingress[].hostname}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.QPVQDkxV1j ++++ mktemp +++ local LAST_ERR=/tmp/tmp.zvwi4MAoJa +++ local exit_status=0 ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl get service/monitoring-service -o 'jsonpath={.status.loadBalancer.ingress[].hostname}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 ']' +++ break +++ cat /tmp/tmp.QPVQDkxV1j +++ cat /tmp/tmp.zvwi4MAoJa +++ rm /tmp/tmp.QPVQDkxV1j /tmp/tmp.zvwi4MAoJa +++ return 0 ++ local endpoint= ++ '[' -z '' ']' +++ kubectl_bin get service/monitoring-service -o 'jsonpath={.status.loadBalancer.ingress[].ip}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.idiP0TRZDf ++++ mktemp +++ local LAST_ERR=/tmp/tmp.b50xFLfqhx +++ local exit_status=0 ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl get service/monitoring-service -o 'jsonpath={.status.loadBalancer.ingress[].ip}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 ']' +++ break +++ cat /tmp/tmp.idiP0TRZDf +++ cat /tmp/tmp.b50xFLfqhx +++ rm /tmp/tmp.idiP0TRZDf /tmp/tmp.b50xFLfqhx +++ return 0 ++ endpoint=34.71.161.14 ++ '[' -n 34.71.161.14 ']' ++ '[' 34.71.161.14 '!=' null ']' ++ echo 34.71.161.14 ++ head -n 1 ++ sed -e 's/^"//; s/"$//;' ++ return + local endpoint=34.71.161.14 + cat ++ /usr/bin/date -u -d '-12 hour' +%Y-%m-%dT%H:%M:%S%:z ++ /usr/bin/date -u +%Y-%m-%dT%H:%M:%S%:z + curl -s -k -XPOST -d @payload.json https://admin:admin@34.71.161.14/v0/qan/GetReport + jq '.rows[].fingerprint' null + rm -f payload.json + nodeList=($(get_node_id_from_pmm)) ++ get_node_id_from_pmm ++ nodeList=() ++ local -a nodeList +++ kubectl_bin get pods --no-headers -l app.kubernetes.io/component=pxc --output=custom-columns=NAME:.metadata.name ++++ mktemp +++ local LAST_OUT=/tmp/tmp.bgIfjR9Dfq ++++ mktemp +++ local LAST_ERR=/tmp/tmp.zOEDq10M9h +++ local exit_status=0 ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl get pods --no-headers -l app.kubernetes.io/component=pxc --output=custom-columns=NAME:.metadata.name +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 ']' +++ break +++ cat /tmp/tmp.bgIfjR9Dfq +++ cat /tmp/tmp.zOEDq10M9h +++ rm /tmp/tmp.bgIfjR9Dfq /tmp/tmp.zOEDq10M9h +++ return 0 ++ for instance in $(kubectl_bin get pods --no-headers -l app.kubernetes.io/component=pxc --output=custom-columns='NAME:.metadata.name') ++ nodeList+=($(kubectl_bin exec -n "$namespace" $instance -c pmm-client -- pmm-admin status --json | jq -r '.pmm_agent_status.node_id')) +++ kubectl_bin exec -n monitoring-2-0-5888 monitoring-pxc-0 -c pmm-client -- pmm-admin status --json +++ jq -r .pmm_agent_status.node_id ++++ mktemp +++ local LAST_OUT=/tmp/tmp.CQOBZgb9cO ++++ mktemp +++ local LAST_ERR=/tmp/tmp.vf7VT1Kd6x +++ local exit_status=0 ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl exec -n monitoring-2-0-5888 monitoring-pxc-0 -c pmm-client -- pmm-admin status --json +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 ']' +++ break +++ cat /tmp/tmp.CQOBZgb9cO +++ cat /tmp/tmp.vf7VT1Kd6x +++ rm /tmp/tmp.CQOBZgb9cO /tmp/tmp.vf7VT1Kd6x +++ return 0 ++ for instance in $(kubectl_bin get pods --no-headers -l app.kubernetes.io/component=pxc --output=custom-columns='NAME:.metadata.name') ++ nodeList+=($(kubectl_bin exec -n "$namespace" $instance -c pmm-client -- pmm-admin status --json | jq -r '.pmm_agent_status.node_id')) +++ kubectl_bin exec -n monitoring-2-0-5888 monitoring-pxc-1 -c pmm-client -- pmm-admin status --json +++ jq -r .pmm_agent_status.node_id ++++ mktemp +++ local LAST_OUT=/tmp/tmp.ilafcXdvJ3 ++++ mktemp +++ local LAST_ERR=/tmp/tmp.qduE6CjoWl +++ local exit_status=0 ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl exec -n monitoring-2-0-5888 monitoring-pxc-1 -c pmm-client -- pmm-admin status --json +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 ']' +++ break +++ cat /tmp/tmp.ilafcXdvJ3 +++ cat /tmp/tmp.qduE6CjoWl +++ rm /tmp/tmp.ilafcXdvJ3 /tmp/tmp.qduE6CjoWl +++ return 0 ++ for instance in $(kubectl_bin get pods --no-headers -l app.kubernetes.io/component=pxc --output=custom-columns='NAME:.metadata.name') ++ nodeList+=($(kubectl_bin exec -n "$namespace" $instance -c pmm-client -- pmm-admin status --json | jq -r '.pmm_agent_status.node_id')) +++ kubectl_bin exec -n monitoring-2-0-5888 monitoring-pxc-2 -c pmm-client -- pmm-admin status --json +++ jq -r .pmm_agent_status.node_id ++++ mktemp +++ local LAST_OUT=/tmp/tmp.KhX34MgbCb ++++ mktemp +++ local LAST_ERR=/tmp/tmp.pm2Hoc73ID +++ local exit_status=0 ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl exec -n monitoring-2-0-5888 monitoring-pxc-2 -c pmm-client -- pmm-admin status --json +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 ']' +++ break +++ cat /tmp/tmp.KhX34MgbCb +++ cat /tmp/tmp.pm2Hoc73ID +++ rm /tmp/tmp.KhX34MgbCb /tmp/tmp.pm2Hoc73ID +++ return 0 ++ echo /node_id/7c7df884-2e67-4221-949c-f410fa2ff5c2 /node_id/4c10f294-fd3a-4f74-b6a8-c00d5bc405f8 /node_id/65dc596d-67c6-47f5-89f4-43e78bd925ab + nodeList_from_pmm=($(does_node_id_exists "${nodeList[@]}")) ++ does_node_id_exists /node_id/7c7df884-2e67-4221-949c-f410fa2ff5c2 /node_id/4c10f294-fd3a-4f74-b6a8-c00d5bc405f8 /node_id/65dc596d-67c6-47f5-89f4-43e78bd925ab ++ nodeList=('/node_id/7c7df884-2e67-4221-949c-f410fa2ff5c2' '/node_id/4c10f294-fd3a-4f74-b6a8-c00d5bc405f8' '/node_id/65dc596d-67c6-47f5-89f4-43e78bd925ab') ++ local -a nodeList ++ nodeList_from_pmm=() ++ local -a nodeList_from_pmm ++ for node_id in "${nodeList[@]}" ++ nodeList_from_pmm+=($(kubectl_bin exec -n "${namespace}" monitoring-0 -- pmm-admin --server-url=https://admin:admin@$(get_service_ip monitoring-service)/ --server-insecure-tls inventory list nodes --node-type=CONTAINER_NODE | grep $node_id | awk '{print $4}')) +++ grep /node_id/7c7df884-2e67-4221-949c-f410fa2ff5c2 +++ awk '{print $4}' ++++ get_service_ip monitoring-service ++++ local service=monitoring-service ++++ kubectl_bin get service/monitoring-service -o 'jsonpath={.spec.type}' ++++ grep -q NotFound +++++ kubectl_bin get service/monitoring-service -o 'jsonpath={.spec.type}' ++++++ mktemp +++++ local LAST_OUT=/tmp/tmp.oSiqr73U9v ++++++ mktemp +++++ local LAST_ERR=/tmp/tmp.CEeCRCARlB +++++ local exit_status=0 ++++++ seq 0 2 +++++ for i in $(seq 0 2) +++++ set +e +++++ kubectl get service/monitoring-service -o 'jsonpath={.spec.type}' +++++ exit_status=0 +++++ set -e +++++ '[' 0 '!=' 0 ']' +++++ break +++++ cat /tmp/tmp.oSiqr73U9v +++++ cat /tmp/tmp.CEeCRCARlB +++++ rm /tmp/tmp.oSiqr73U9v /tmp/tmp.CEeCRCARlB +++++ return 0 ++++ '[' LoadBalancer = ClusterIP ']' ++++ kubectl_bin get service/monitoring-service -o 'jsonpath={.status.loadBalancer.ingress[]}' ++++ grep -E -q 'hostname|ip' ++++ kubectl_bin get service/monitoring-service -o 'jsonpath={.status.loadBalancer.ingress[].ip}' +++++ mktemp ++++ local LAST_OUT=/tmp/tmp.ND3HJ91VF1 +++++ mktemp ++++ local LAST_ERR=/tmp/tmp.jOQMbOup2D ++++ local exit_status=0 +++++ seq 0 2 ++++ for i in $(seq 0 2) ++++ set +e ++++ kubectl get service/monitoring-service -o 'jsonpath={.status.loadBalancer.ingress[].ip}' ++++ exit_status=0 ++++ set -e ++++ '[' 0 '!=' 0 ']' ++++ break ++++ cat /tmp/tmp.ND3HJ91VF1 ++++ cat /tmp/tmp.jOQMbOup2D ++++ rm /tmp/tmp.ND3HJ91VF1 /tmp/tmp.jOQMbOup2D ++++ return 0 ++++ kubectl_bin get service/monitoring-service -o 'jsonpath={.status.loadBalancer.ingress[].hostname}' +++++ mktemp ++++ local LAST_OUT=/tmp/tmp.kYmLFFKACV +++++ mktemp ++++ local LAST_ERR=/tmp/tmp.1Pb8gXXcBX ++++ local exit_status=0 +++++ seq 0 2 ++++ for i in $(seq 0 2) ++++ set +e ++++ kubectl get service/monitoring-service -o 'jsonpath={.status.loadBalancer.ingress[].hostname}' ++++ exit_status=0 ++++ set -e ++++ '[' 0 '!=' 0 ']' ++++ break ++++ cat /tmp/tmp.kYmLFFKACV ++++ cat /tmp/tmp.1Pb8gXXcBX ++++ rm /tmp/tmp.kYmLFFKACV /tmp/tmp.1Pb8gXXcBX ++++ return 0 +++ kubectl_bin exec -n monitoring-2-0-5888 monitoring-0 -- pmm-admin --server-url=https://admin:admin@34.71.161.14/ --server-insecure-tls inventory list nodes --node-type=CONTAINER_NODE ++++ mktemp +++ local LAST_OUT=/tmp/tmp.zTjD7UzCdx ++++ mktemp +++ local LAST_ERR=/tmp/tmp.X648Wk1yVH +++ local exit_status=0 ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl exec -n monitoring-2-0-5888 monitoring-0 -- pmm-admin --server-url=https://admin:admin@34.71.161.14/ --server-insecure-tls inventory list nodes --node-type=CONTAINER_NODE +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 ']' +++ break +++ cat /tmp/tmp.zTjD7UzCdx +++ cat /tmp/tmp.X648Wk1yVH +++ rm /tmp/tmp.zTjD7UzCdx /tmp/tmp.X648Wk1yVH +++ return 0 ++ for node_id in "${nodeList[@]}" ++ nodeList_from_pmm+=($(kubectl_bin exec -n "${namespace}" monitoring-0 -- pmm-admin --server-url=https://admin:admin@$(get_service_ip monitoring-service)/ --server-insecure-tls inventory list nodes --node-type=CONTAINER_NODE | grep $node_id | awk '{print $4}')) +++ grep /node_id/4c10f294-fd3a-4f74-b6a8-c00d5bc405f8 +++ awk '{print $4}' ++++ get_service_ip monitoring-service ++++ local service=monitoring-service ++++ grep -q NotFound ++++ kubectl_bin get service/monitoring-service -o 'jsonpath={.spec.type}' +++++ kubectl_bin get service/monitoring-service -o 'jsonpath={.spec.type}' ++++++ mktemp +++++ local LAST_OUT=/tmp/tmp.4qQuzaZED0 ++++++ mktemp +++++ local LAST_ERR=/tmp/tmp.DxDmnxIIWh +++++ local exit_status=0 ++++++ seq 0 2 +++++ for i in $(seq 0 2) +++++ set +e +++++ kubectl get service/monitoring-service -o 'jsonpath={.spec.type}' +++++ exit_status=0 +++++ set -e +++++ '[' 0 '!=' 0 ']' +++++ break +++++ cat /tmp/tmp.4qQuzaZED0 +++++ cat /tmp/tmp.DxDmnxIIWh +++++ rm /tmp/tmp.4qQuzaZED0 /tmp/tmp.DxDmnxIIWh +++++ return 0 ++++ '[' LoadBalancer = ClusterIP ']' ++++ kubectl_bin get service/monitoring-service -o 'jsonpath={.status.loadBalancer.ingress[]}' ++++ grep -E -q 'hostname|ip' ++++ kubectl_bin get service/monitoring-service -o 'jsonpath={.status.loadBalancer.ingress[].ip}' +++++ mktemp ++++ local LAST_OUT=/tmp/tmp.cZEQKcUpoP +++++ mktemp ++++ local LAST_ERR=/tmp/tmp.sK7imhdCpz ++++ local exit_status=0 +++++ seq 0 2 ++++ for i in $(seq 0 2) ++++ set +e ++++ kubectl get service/monitoring-service -o 'jsonpath={.status.loadBalancer.ingress[].ip}' ++++ exit_status=0 ++++ set -e ++++ '[' 0 '!=' 0 ']' ++++ break ++++ cat /tmp/tmp.cZEQKcUpoP ++++ cat /tmp/tmp.sK7imhdCpz ++++ rm /tmp/tmp.cZEQKcUpoP /tmp/tmp.sK7imhdCpz ++++ return 0 ++++ kubectl_bin get service/monitoring-service -o 'jsonpath={.status.loadBalancer.ingress[].hostname}' +++++ mktemp ++++ local LAST_OUT=/tmp/tmp.LKAU8Sun5Y +++++ mktemp ++++ local LAST_ERR=/tmp/tmp.AJidYnNaZ6 ++++ local exit_status=0 +++++ seq 0 2 ++++ for i in $(seq 0 2) ++++ set +e ++++ kubectl get service/monitoring-service -o 'jsonpath={.status.loadBalancer.ingress[].hostname}' ++++ exit_status=0 ++++ set -e ++++ '[' 0 '!=' 0 ']' ++++ break ++++ cat /tmp/tmp.LKAU8Sun5Y ++++ cat /tmp/tmp.AJidYnNaZ6 ++++ rm /tmp/tmp.LKAU8Sun5Y /tmp/tmp.AJidYnNaZ6 ++++ return 0 +++ kubectl_bin exec -n monitoring-2-0-5888 monitoring-0 -- pmm-admin --server-url=https://admin:admin@34.71.161.14/ --server-insecure-tls inventory list nodes --node-type=CONTAINER_NODE ++++ mktemp +++ local LAST_OUT=/tmp/tmp.WrGbXehbtw ++++ mktemp +++ local LAST_ERR=/tmp/tmp.1Z1sVv9eiw +++ local exit_status=0 ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl exec -n monitoring-2-0-5888 monitoring-0 -- pmm-admin --server-url=https://admin:admin@34.71.161.14/ --server-insecure-tls inventory list nodes --node-type=CONTAINER_NODE +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 ']' +++ break +++ cat /tmp/tmp.WrGbXehbtw +++ cat /tmp/tmp.1Z1sVv9eiw +++ rm /tmp/tmp.WrGbXehbtw /tmp/tmp.1Z1sVv9eiw +++ return 0 ++ for node_id in "${nodeList[@]}" ++ nodeList_from_pmm+=($(kubectl_bin exec -n "${namespace}" monitoring-0 -- pmm-admin --server-url=https://admin:admin@$(get_service_ip monitoring-service)/ --server-insecure-tls inventory list nodes --node-type=CONTAINER_NODE | grep $node_id | awk '{print $4}')) +++ awk '{print $4}' +++ grep /node_id/65dc596d-67c6-47f5-89f4-43e78bd925ab ++++ get_service_ip monitoring-service ++++ local service=monitoring-service ++++ kubectl_bin get service/monitoring-service -o 'jsonpath={.spec.type}' ++++ grep -q NotFound +++++ kubectl_bin get service/monitoring-service -o 'jsonpath={.spec.type}' ++++++ mktemp +++++ local LAST_OUT=/tmp/tmp.QKnXgLwrwh ++++++ mktemp +++++ local LAST_ERR=/tmp/tmp.xpnLvCk2ot +++++ local exit_status=0 ++++++ seq 0 2 +++++ for i in $(seq 0 2) +++++ set +e +++++ kubectl get service/monitoring-service -o 'jsonpath={.spec.type}' +++++ exit_status=0 +++++ set -e +++++ '[' 0 '!=' 0 ']' +++++ break +++++ cat /tmp/tmp.QKnXgLwrwh +++++ cat /tmp/tmp.xpnLvCk2ot +++++ rm /tmp/tmp.QKnXgLwrwh /tmp/tmp.xpnLvCk2ot +++++ return 0 ++++ '[' LoadBalancer = ClusterIP ']' ++++ kubectl_bin get service/monitoring-service -o 'jsonpath={.status.loadBalancer.ingress[]}' ++++ grep -E -q 'hostname|ip' ++++ kubectl_bin get service/monitoring-service -o 'jsonpath={.status.loadBalancer.ingress[].ip}' +++++ mktemp ++++ local LAST_OUT=/tmp/tmp.3GF67kkbVk +++++ mktemp ++++ local LAST_ERR=/tmp/tmp.q80CWsqxqv ++++ local exit_status=0 +++++ seq 0 2 ++++ for i in $(seq 0 2) ++++ set +e ++++ kubectl get service/monitoring-service -o 'jsonpath={.status.loadBalancer.ingress[].ip}' ++++ exit_status=0 ++++ set -e ++++ '[' 0 '!=' 0 ']' ++++ break ++++ cat /tmp/tmp.3GF67kkbVk ++++ cat /tmp/tmp.q80CWsqxqv ++++ rm /tmp/tmp.3GF67kkbVk /tmp/tmp.q80CWsqxqv ++++ return 0 ++++ kubectl_bin get service/monitoring-service -o 'jsonpath={.status.loadBalancer.ingress[].hostname}' +++++ mktemp ++++ local LAST_OUT=/tmp/tmp.kikOpsCiuq +++++ mktemp ++++ local LAST_ERR=/tmp/tmp.cKfLqmF88O ++++ local exit_status=0 +++++ seq 0 2 ++++ for i in $(seq 0 2) ++++ set +e ++++ kubectl get service/monitoring-service -o 'jsonpath={.status.loadBalancer.ingress[].hostname}' ++++ exit_status=0 ++++ set -e ++++ '[' 0 '!=' 0 ']' ++++ break ++++ cat /tmp/tmp.kikOpsCiuq ++++ cat /tmp/tmp.cKfLqmF88O ++++ rm /tmp/tmp.kikOpsCiuq /tmp/tmp.cKfLqmF88O ++++ return 0 +++ kubectl_bin exec -n monitoring-2-0-5888 monitoring-0 -- pmm-admin --server-url=https://admin:admin@34.71.161.14/ --server-insecure-tls inventory list nodes --node-type=CONTAINER_NODE ++++ mktemp +++ local LAST_OUT=/tmp/tmp.Ndqfgb74AM ++++ mktemp +++ local LAST_ERR=/tmp/tmp.LXIeOU4bFt +++ local exit_status=0 ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl exec -n monitoring-2-0-5888 monitoring-0 -- pmm-admin --server-url=https://admin:admin@34.71.161.14/ --server-insecure-tls inventory list nodes --node-type=CONTAINER_NODE +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 ']' +++ break +++ cat /tmp/tmp.Ndqfgb74AM +++ cat /tmp/tmp.LXIeOU4bFt +++ rm /tmp/tmp.Ndqfgb74AM /tmp/tmp.LXIeOU4bFt +++ return 0 ++ echo /node_id/7c7df884-2e67-4221-949c-f410fa2ff5c2 /node_id/4c10f294-fd3a-4f74-b6a8-c00d5bc405f8 /node_id/65dc596d-67c6-47f5-89f4-43e78bd925ab + for node_id in "${nodeList_from_pmm[@]}" + '[' -z /node_id/7c7df884-2e67-4221-949c-f410fa2ff5c2 ']' + for node_id in "${nodeList_from_pmm[@]}" + '[' -z /node_id/4c10f294-fd3a-4f74-b6a8-c00d5bc405f8 ']' + for node_id in "${nodeList_from_pmm[@]}" + '[' -z /node_id/65dc596d-67c6-47f5-89f4-43e78bd925ab ']' + kubectl_bin patch pxc monitoring --type json '-p=[{"op":"add","path":"/spec/pause","value":true}]' ++ mktemp + local LAST_OUT=/tmp/tmp.ZxedyK40gx ++ mktemp + local LAST_ERR=/tmp/tmp.cDGdvF4oXT + local exit_status=0 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl patch pxc monitoring --type json '-p=[{"op":"add","path":"/spec/pause","value":true}]' + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.ZxedyK40gx perconaxtradbcluster.pxc.percona.com/monitoring patched + cat /tmp/tmp.cDGdvF4oXT + rm /tmp/tmp.ZxedyK40gx /tmp/tmp.cDGdvF4oXT + return 0 + wait_for_delete pod/monitoring-pxc-0 + local res=pod/monitoring-pxc-0 + echo -n 'waiting for pod/monitoring-pxc-0 to be deleted' waiting for pod/monitoring-pxc-0 to be deleted+ set +o xtrace ..................Error from server (NotFound): pods "monitoring-pxc-0" not found + does_node_id_exists_in_pmm=($(does_node_id_exists "${nodeList[@]}")) ++ does_node_id_exists /node_id/7c7df884-2e67-4221-949c-f410fa2ff5c2 /node_id/4c10f294-fd3a-4f74-b6a8-c00d5bc405f8 /node_id/65dc596d-67c6-47f5-89f4-43e78bd925ab ++ nodeList=('/node_id/7c7df884-2e67-4221-949c-f410fa2ff5c2' '/node_id/4c10f294-fd3a-4f74-b6a8-c00d5bc405f8' '/node_id/65dc596d-67c6-47f5-89f4-43e78bd925ab') ++ local -a nodeList ++ nodeList_from_pmm=() ++ local -a nodeList_from_pmm ++ for node_id in "${nodeList[@]}" ++ nodeList_from_pmm+=($(kubectl_bin exec -n "${namespace}" monitoring-0 -- pmm-admin --server-url=https://admin:admin@$(get_service_ip monitoring-service)/ --server-insecure-tls inventory list nodes --node-type=CONTAINER_NODE | grep $node_id | awk '{print $4}')) +++ grep /node_id/7c7df884-2e67-4221-949c-f410fa2ff5c2 +++ awk '{print $4}' ++++ get_service_ip monitoring-service ++++ local service=monitoring-service ++++ grep -q NotFound ++++ kubectl_bin get service/monitoring-service -o 'jsonpath={.spec.type}' +++++ kubectl_bin get service/monitoring-service -o 'jsonpath={.spec.type}' ++++++ mktemp +++++ local LAST_OUT=/tmp/tmp.cRJMQvixu7 ++++++ mktemp +++++ local LAST_ERR=/tmp/tmp.t2YlA8ND9h +++++ local exit_status=0 ++++++ seq 0 2 +++++ for i in $(seq 0 2) +++++ set +e +++++ kubectl get service/monitoring-service -o 'jsonpath={.spec.type}' +++++ exit_status=0 +++++ set -e +++++ '[' 0 '!=' 0 ']' +++++ break +++++ cat /tmp/tmp.cRJMQvixu7 +++++ cat /tmp/tmp.t2YlA8ND9h +++++ rm /tmp/tmp.cRJMQvixu7 /tmp/tmp.t2YlA8ND9h +++++ return 0 ++++ '[' LoadBalancer = ClusterIP ']' ++++ grep -E -q 'hostname|ip' ++++ kubectl_bin get service/monitoring-service -o 'jsonpath={.status.loadBalancer.ingress[]}' ++++ kubectl_bin get service/monitoring-service -o 'jsonpath={.status.loadBalancer.ingress[].ip}' +++++ mktemp ++++ local LAST_OUT=/tmp/tmp.9YiypkItMK +++++ mktemp ++++ local LAST_ERR=/tmp/tmp.MyMDKQTpJz ++++ local exit_status=0 +++++ seq 0 2 ++++ for i in $(seq 0 2) ++++ set +e ++++ kubectl get service/monitoring-service -o 'jsonpath={.status.loadBalancer.ingress[].ip}' ++++ exit_status=0 ++++ set -e ++++ '[' 0 '!=' 0 ']' ++++ break ++++ cat /tmp/tmp.9YiypkItMK ++++ cat /tmp/tmp.MyMDKQTpJz ++++ rm /tmp/tmp.9YiypkItMK /tmp/tmp.MyMDKQTpJz ++++ return 0 ++++ kubectl_bin get service/monitoring-service -o 'jsonpath={.status.loadBalancer.ingress[].hostname}' +++++ mktemp ++++ local LAST_OUT=/tmp/tmp.MSMaoiyt1i +++++ mktemp ++++ local LAST_ERR=/tmp/tmp.AZWwxXJXxG ++++ local exit_status=0 +++++ seq 0 2 ++++ for i in $(seq 0 2) ++++ set +e ++++ kubectl get service/monitoring-service -o 'jsonpath={.status.loadBalancer.ingress[].hostname}' ++++ exit_status=0 ++++ set -e ++++ '[' 0 '!=' 0 ']' ++++ break ++++ cat /tmp/tmp.MSMaoiyt1i ++++ cat /tmp/tmp.AZWwxXJXxG ++++ rm /tmp/tmp.MSMaoiyt1i /tmp/tmp.AZWwxXJXxG ++++ return 0 +++ kubectl_bin exec -n monitoring-2-0-5888 monitoring-0 -- pmm-admin --server-url=https://admin:admin@34.71.161.14/ --server-insecure-tls inventory list nodes --node-type=CONTAINER_NODE ++++ mktemp +++ local LAST_OUT=/tmp/tmp.d5z0S73uEB ++++ mktemp +++ local LAST_ERR=/tmp/tmp.FiRLJNIsuH +++ local exit_status=0 ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl exec -n monitoring-2-0-5888 monitoring-0 -- pmm-admin --server-url=https://admin:admin@34.71.161.14/ --server-insecure-tls inventory list nodes --node-type=CONTAINER_NODE +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 ']' +++ break +++ cat /tmp/tmp.d5z0S73uEB +++ cat /tmp/tmp.FiRLJNIsuH +++ rm /tmp/tmp.d5z0S73uEB /tmp/tmp.FiRLJNIsuH +++ return 0 ++ for node_id in "${nodeList[@]}" ++ nodeList_from_pmm+=($(kubectl_bin exec -n "${namespace}" monitoring-0 -- pmm-admin --server-url=https://admin:admin@$(get_service_ip monitoring-service)/ --server-insecure-tls inventory list nodes --node-type=CONTAINER_NODE | grep $node_id | awk '{print $4}')) +++ grep /node_id/4c10f294-fd3a-4f74-b6a8-c00d5bc405f8 ++++ get_service_ip monitoring-service ++++ local service=monitoring-service ++++ kubectl_bin get service/monitoring-service -o 'jsonpath={.spec.type}' +++ awk '{print $4}' ++++ grep -q NotFound +++++ kubectl_bin get service/monitoring-service -o 'jsonpath={.spec.type}' ++++++ mktemp +++++ local LAST_OUT=/tmp/tmp.BbqNfSijz2 ++++++ mktemp +++++ local LAST_ERR=/tmp/tmp.H0zPv6MIUg +++++ local exit_status=0 ++++++ seq 0 2 +++++ for i in $(seq 0 2) +++++ set +e +++++ kubectl get service/monitoring-service -o 'jsonpath={.spec.type}' +++++ exit_status=0 +++++ set -e +++++ '[' 0 '!=' 0 ']' +++++ break +++++ cat /tmp/tmp.BbqNfSijz2 +++++ cat /tmp/tmp.H0zPv6MIUg +++++ rm /tmp/tmp.BbqNfSijz2 /tmp/tmp.H0zPv6MIUg +++++ return 0 ++++ '[' LoadBalancer = ClusterIP ']' ++++ kubectl_bin get service/monitoring-service -o 'jsonpath={.status.loadBalancer.ingress[]}' ++++ grep -E -q 'hostname|ip' ++++ kubectl_bin get service/monitoring-service -o 'jsonpath={.status.loadBalancer.ingress[].ip}' +++++ mktemp ++++ local LAST_OUT=/tmp/tmp.Q0QRk5mtJC +++++ mktemp ++++ local LAST_ERR=/tmp/tmp.0XTpOSQk1z ++++ local exit_status=0 +++++ seq 0 2 ++++ for i in $(seq 0 2) ++++ set +e ++++ kubectl get service/monitoring-service -o 'jsonpath={.status.loadBalancer.ingress[].ip}' ++++ exit_status=0 ++++ set -e ++++ '[' 0 '!=' 0 ']' ++++ break ++++ cat /tmp/tmp.Q0QRk5mtJC ++++ cat /tmp/tmp.0XTpOSQk1z ++++ rm /tmp/tmp.Q0QRk5mtJC /tmp/tmp.0XTpOSQk1z ++++ return 0 ++++ kubectl_bin get service/monitoring-service -o 'jsonpath={.status.loadBalancer.ingress[].hostname}' +++++ mktemp ++++ local LAST_OUT=/tmp/tmp.13RCdjjYG4 +++++ mktemp ++++ local LAST_ERR=/tmp/tmp.L929GWTVOj ++++ local exit_status=0 +++++ seq 0 2 ++++ for i in $(seq 0 2) ++++ set +e ++++ kubectl get service/monitoring-service -o 'jsonpath={.status.loadBalancer.ingress[].hostname}' ++++ exit_status=0 ++++ set -e ++++ '[' 0 '!=' 0 ']' ++++ break ++++ cat /tmp/tmp.13RCdjjYG4 ++++ cat /tmp/tmp.L929GWTVOj ++++ rm /tmp/tmp.13RCdjjYG4 /tmp/tmp.L929GWTVOj ++++ return 0 +++ kubectl_bin exec -n monitoring-2-0-5888 monitoring-0 -- pmm-admin --server-url=https://admin:admin@34.71.161.14/ --server-insecure-tls inventory list nodes --node-type=CONTAINER_NODE ++++ mktemp +++ local LAST_OUT=/tmp/tmp.LkvWzmml0Q ++++ mktemp +++ local LAST_ERR=/tmp/tmp.96I1M9Qifa +++ local exit_status=0 ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl exec -n monitoring-2-0-5888 monitoring-0 -- pmm-admin --server-url=https://admin:admin@34.71.161.14/ --server-insecure-tls inventory list nodes --node-type=CONTAINER_NODE +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 ']' +++ break +++ cat /tmp/tmp.LkvWzmml0Q +++ cat /tmp/tmp.96I1M9Qifa +++ rm /tmp/tmp.LkvWzmml0Q /tmp/tmp.96I1M9Qifa +++ return 0 ++ for node_id in "${nodeList[@]}" ++ nodeList_from_pmm+=($(kubectl_bin exec -n "${namespace}" monitoring-0 -- pmm-admin --server-url=https://admin:admin@$(get_service_ip monitoring-service)/ --server-insecure-tls inventory list nodes --node-type=CONTAINER_NODE | grep $node_id | awk '{print $4}')) +++ grep /node_id/65dc596d-67c6-47f5-89f4-43e78bd925ab ++++ get_service_ip monitoring-service ++++ local service=monitoring-service ++++ kubectl_bin get service/monitoring-service -o 'jsonpath={.spec.type}' ++++ grep -q NotFound +++ awk '{print $4}' +++++ kubectl_bin get service/monitoring-service -o 'jsonpath={.spec.type}' ++++++ mktemp +++++ local LAST_OUT=/tmp/tmp.uz5EG2WDeM ++++++ mktemp +++++ local LAST_ERR=/tmp/tmp.ozzXuAsUva +++++ local exit_status=0 ++++++ seq 0 2 +++++ for i in $(seq 0 2) +++++ set +e +++++ kubectl get service/monitoring-service -o 'jsonpath={.spec.type}' +++++ exit_status=0 +++++ set -e +++++ '[' 0 '!=' 0 ']' +++++ break +++++ cat /tmp/tmp.uz5EG2WDeM +++++ cat /tmp/tmp.ozzXuAsUva +++++ rm /tmp/tmp.uz5EG2WDeM /tmp/tmp.ozzXuAsUva +++++ return 0 ++++ '[' LoadBalancer = ClusterIP ']' ++++ kubectl_bin get service/monitoring-service -o 'jsonpath={.status.loadBalancer.ingress[]}' ++++ grep -E -q 'hostname|ip' ++++ kubectl_bin get service/monitoring-service -o 'jsonpath={.status.loadBalancer.ingress[].ip}' +++++ mktemp ++++ local LAST_OUT=/tmp/tmp.xrZZxJB5Zn +++++ mktemp ++++ local LAST_ERR=/tmp/tmp.lSDgXo8Ot8 ++++ local exit_status=0 +++++ seq 0 2 ++++ for i in $(seq 0 2) ++++ set +e ++++ kubectl get service/monitoring-service -o 'jsonpath={.status.loadBalancer.ingress[].ip}' ++++ exit_status=0 ++++ set -e ++++ '[' 0 '!=' 0 ']' ++++ break ++++ cat /tmp/tmp.xrZZxJB5Zn ++++ cat /tmp/tmp.lSDgXo8Ot8 ++++ rm /tmp/tmp.xrZZxJB5Zn /tmp/tmp.lSDgXo8Ot8 ++++ return 0 ++++ kubectl_bin get service/monitoring-service -o 'jsonpath={.status.loadBalancer.ingress[].hostname}' +++++ mktemp ++++ local LAST_OUT=/tmp/tmp.yEStQj6ChP +++++ mktemp ++++ local LAST_ERR=/tmp/tmp.fm7EhR5CmP ++++ local exit_status=0 +++++ seq 0 2 ++++ for i in $(seq 0 2) ++++ set +e ++++ kubectl get service/monitoring-service -o 'jsonpath={.status.loadBalancer.ingress[].hostname}' ++++ exit_status=0 ++++ set -e ++++ '[' 0 '!=' 0 ']' ++++ break ++++ cat /tmp/tmp.yEStQj6ChP ++++ cat /tmp/tmp.fm7EhR5CmP ++++ rm /tmp/tmp.yEStQj6ChP /tmp/tmp.fm7EhR5CmP ++++ return 0 +++ kubectl_bin exec -n monitoring-2-0-5888 monitoring-0 -- pmm-admin --server-url=https://admin:admin@34.71.161.14/ --server-insecure-tls inventory list nodes --node-type=CONTAINER_NODE ++++ mktemp +++ local LAST_OUT=/tmp/tmp.YO6RK3rRGT ++++ mktemp +++ local LAST_ERR=/tmp/tmp.uXzbe0t6ri +++ local exit_status=0 ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl exec -n monitoring-2-0-5888 monitoring-0 -- pmm-admin --server-url=https://admin:admin@34.71.161.14/ --server-insecure-tls inventory list nodes --node-type=CONTAINER_NODE +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 ']' +++ break +++ cat /tmp/tmp.YO6RK3rRGT +++ cat /tmp/tmp.uXzbe0t6ri +++ rm /tmp/tmp.YO6RK3rRGT /tmp/tmp.uXzbe0t6ri +++ return 0 ++ echo + [[ -n '' ]] + helm uninstall monitoring release "monitoring" uninstalled + destroy monitoring-2-0-5888 + local namespace=monitoring-2-0-5888 + local ignore_logs=true + [[ 0 == 1 ]] + desc 'destroy cluster/operator and all other resources' + set +o xtrace ----------------------------------------------------------------------------------- destroy cluster/operator and all other resources ----------------------------------------------------------------------------------- + '[' true == false -o 1 == 1 ']' + /usr/bin/sed -r 's/"ts":[0-9.]+//; s^limits-[0-9.]+/^^g' + grep -v 'get backup status: Job.batch' + grep -v level=info + grep -v 'the object has been modified' ++ get_operator_pod ++ local label_prefix=app.kubernetes.io/ + sort -u +++ kubectl get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator -n pxc-operator + tee /tmp/tmp.oE2g1q4ApT/operator.log +++ grep -c percona-xtradb-cluster-operator ++ local check_label=1 ++ [[ 1 -eq 0 ]] ++ kubectl_bin get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator --field-selector=status.phase=Running -o json -n pxc-operator ++ jq -r '.items[] | select(.metadata.deletionTimestamp == null) | .metadata.name' ++ head -1 +++ mktemp ++ local LAST_OUT=/tmp/tmp.mj7NH05pvU +++ mktemp ++ local LAST_ERR=/tmp/tmp.TWOHqKBs8s ++ local exit_status=0 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator --field-selector=status.phase=Running -o json -n pxc-operator ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.mj7NH05pvU ++ cat /tmp/tmp.TWOHqKBs8s ++ rm /tmp/tmp.mj7NH05pvU /tmp/tmp.TWOHqKBs8s ++ return 0 + kubectl_bin logs -n pxc-operator percona-xtradb-cluster-operator-658fcf4b98-jb2nc ++ mktemp + local LAST_OUT=/tmp/tmp.S9r1KdmFKp ++ mktemp + local LAST_ERR=/tmp/tmp.tICZ3BwWCJ + local exit_status=0 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl logs -n pxc-operator percona-xtradb-cluster-operator-658fcf4b98-jb2nc + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.S9r1KdmFKp + cat /tmp/tmp.tICZ3BwWCJ + rm /tmp/tmp.S9r1KdmFKp /tmp/tmp.tICZ3BwWCJ + return 0 +  APIVersion: "", +  Key: "pmmserverkey", +  Scheme: "", +  DefaultMode: nil, +  DefaultMode: nil, +  Protocol: "", +  Value: "$(PMM_PREFIX)$(POD_NAMESPASE)-$(POD_NAME)", +  Value: "api_key", +  PeriodSeconds: 0, +  TerminationMessagePath: "", +  TerminationMessagePolicy: "", +  "last-applied-secret": "424b8237346222ce589b2d9709d9497cd4ee835efb19befb4da0825f0a734865", +  "percona.com/env-secret-config-hash": "5bc403a8d44324c1bc704f02e74876d0", +  Phase: "", +  VolumeMode: nil, +  DNSPolicy: "", +  DeprecatedServiceAccount: "", +  RestartPolicy: "", +  SchedulerName: "", +  SchedulerName: "", +  SecurityContext: nil, +  TopologySpreadConstraints: []v1.TopologySpreadConstraint{}, +  "percona.com/last-config-hash": "eyJyZXBsaWNhcyI6MCwic2VsZWN0b3IiOnsibWF0Y2hMYWJlbHMiOnsiYXBwLmt1YmVybmV0ZXMuaW8vY29tcG9uZW50IjoiaGFwcm94eSIsImFwcC5rdWJlcm5ldGVzLmlvL2luc3RhbmNlIjoibW9uaXRvcmluZyIsImFwcC5rdWJlcm5ldGVzLmlvL21hbmFnZWQtYnkiOiJwZXJjb25hLXh0cmFkYi1jbHVzdGVyLW9wZXJhdG9yIiwiYXBw"..., +  "percona.com/last-config-hash": "eyJyZXBsaWNhcyI6MCwic2VsZWN0b3IiOnsibWF0Y2hMYWJlbHMiOnsiYXBwLmt1YmVybmV0ZXMuaW8vY29tcG9uZW50IjoicHhjIiwiYXBwLmt1YmVybmV0ZXMuaW8vaW5zdGFuY2UiOiJtb25pdG9yaW5nIiwiYXBwLmt1YmVybmV0ZXMuaW8vbWFuYWdlZC1ieSI6InBlcmNvbmEteHRyYWRiLWNsdXN0ZXItb3BlcmF0b3IiLCJhcHAua3Vi"..., +  "percona.com/last-config-hash": "eyJyZXBsaWNhcyI6Miwic2VsZWN0b3IiOnsibWF0Y2hMYWJlbHMiOnsiYXBwLmt1YmVybmV0ZXMuaW8vY29tcG9uZW50IjoiaGFwcm94eSIsImFwcC5rdWJlcm5ldGVzLmlvL2luc3RhbmNlIjoibW9uaXRvcmluZyIsImFwcC5rdWJlcm5ldGVzLmlvL21hbmFnZWQtYnkiOiJwZXJjb25hLXh0cmFkYi1jbHVzdGVyLW9wZXJhdG9yIiwiYXBwLmt1YmVybmV0ZXMuaW8vbmFtZSI6InBlcmNvbmEteHRyYWRiLWNsdXN0ZXIiLCJhcHAua3ViZXJuZXRlcy5pby9wYXJ0LW9mIjoicGVyY29uYS14dHJhZGItY2x1c3RlciJ9fSwidGVtcGxhdGUiOnsibWV0YWRhdGEiOnsibGFiZWxzIjp7ImFwcC5rdWJlcm5ldGVzLmlvL2NvbXBvbmVudCI6ImhhcHJveHkiLCJhcHAua3ViZXJuZXRlcy5pby9pbnN0YW5jZSI6Im1vbml0b3JpbmciLCJhcHAua3ViZXJuZXRlcy5pby9tYW5hZ2VkLWJ5IjoicGVyY29uYS14dHJhZGItY2x1c3Rlci1vcGVyYXRvciIsImFwcC5rdWJlcm5ldGVzLmlvL25hbWUiOiJwZXJjb25hLXh0cmFkYi1jbHVzdGVyIiwiYXBwLmt1YmVybmV0ZXMuaW8vcGFydC1vZiI6InBlcmNvbmEteHRyYWRiLWNsdXN0ZXIifSwiYW5ub3RhdGlvbnMiOnsia3ViZWN0bC5rdWJlcm5ldGVzLmlvL2RlZmF1bHQtY29udGFpbmVyIjoiaGFwcm94eSIsImxhc3QtYXBwbGllZC1zZWNyZXQiOiI0MjRiODIzNzM0NjIyMmNlNTg5YjJkOTcwOWQ5NDk3Y2Q0ZWU4MzVlZmIxOWJlZmI0ZGEwODI1ZjBhNzM0ODY1IiwicGVyY29uYS5jb20vY29uZmlndXJhdGlvbi1oYXNoIjoi"..., +  "percona.com/last-config-hash": "eyJyZXBsaWNhcyI6Miwic2VsZWN0b3IiOnsibWF0Y2hMYWJlbHMiOnsiYXBwLmt1YmVybmV0ZXMuaW8vY29tcG9uZW50IjoiaGFwcm94eSIsImFwcC5rdWJlcm5ldGVzLmlvL2luc3RhbmNlIjoibW9uaXRvcmluZyIsImFwcC5rdWJlcm5ldGVzLmlvL21hbmFnZWQtYnkiOiJwZXJjb25hLXh0cmFkYi1jbHVzdGVyLW9wZXJhdG9yIiwiYXBwLmt1YmVybmV0ZXMuaW8vbmFtZSI6InBlcmNvbmEteHRyYWRiLWNsdXN0ZXIiLCJhcHAua3ViZXJuZXRlcy5pby9wYXJ0LW9mIjoicGVyY29uYS14dHJhZGItY2x1c3RlciJ9fSwidGVtcGxhdGUiOnsibWV0YWRhdGEiOnsibGFiZWxzIjp7ImFwcC5rdWJlcm5ldGVzLmlvL2NvbXBvbmVudCI6ImhhcHJveHkiLCJhcHAua3ViZXJuZXRlcy5pby9pbnN0YW5jZSI6Im1vbml0b3JpbmciLCJhcHAua3ViZXJuZXRlcy5pby9tYW5hZ2VkLWJ5IjoicGVyY29uYS14dHJhZGItY2x1c3Rlci1vcGVyYXRvciIsImFwcC5rdWJlcm5ldGVzLmlvL25hbWUiOiJwZXJjb25hLXh0cmFkYi1jbHVzdGVyIiwiYXBwLmt1YmVybmV0ZXMuaW8vcGFydC1vZiI6InBlcmNvbmEteHRyYWRiLWNsdXN0ZXIifSwiYW5ub3RhdGlvbnMiOnsia3ViZWN0bC5rdWJlcm5ldGVzLmlvL2RlZmF1bHQtY29udGFpbmVyIjoiaGFwcm94eSIsInBlcmNvbmEuY29tL2NvbmZpZ3VyYXRpb24taGFzaCI6ImQ0MWQ4Y2Q5OGYwMGIyMDRlOTgwMDk5OGVjZjg0MjdlIiwicGVyY29uYS5jb20vZW52LXNlY3JldC1jb25maWctaGFzaCI6IjViYzQwM2E4ZDQ0MzI0YzFi"..., +  "percona.com/last-config-hash": "eyJyZXBsaWNhcyI6Miwic2VsZWN0b3IiOnsibWF0Y2hMYWJlbHMiOnsiYXBwLmt1YmVybmV0ZXMuaW8vY29tcG9uZW50IjoiaGFwcm94eSIsImFwcC5rdWJlcm5ldGVzLmlvL2luc3RhbmNlIjoibW9uaXRvcmluZyIsImFwcC5rdWJlcm5ldGVzLmlvL21hbmFnZWQtYnkiOiJwZXJjb25hLXh0cmFkYi1jbHVzdGVyLW9wZXJhdG9yIiwiYXBwLmt1YmVybmV0ZXMuaW8vbmFtZSI6InBlcmNvbmEteHRyYWRiLWNsdXN0ZXIiLCJhcHAua3ViZXJuZXRlcy5pby9wYXJ0LW9mIjoicGVyY29uYS14dHJhZGItY2x1c3RlciJ9fSwidGVtcGxhdGUiOnsibWV0YWRhdGEiOnsibGFiZWxzIjp7ImFwcC5rdWJlcm5ldGVzLmlvL2NvbXBvbmVudCI6ImhhcHJveHkiLCJhcHAua3ViZXJuZXRlcy5pby9pbnN0YW5jZSI6Im1vbml0b3JpbmciLCJhcHAua3ViZXJuZXRlcy5pby9tYW5hZ2VkLWJ5IjoicGVyY29uYS14dHJhZGItY2x1c3Rlci1vcGVyYXRvciIsImFwcC5rdWJlcm5ldGVzLmlvL25hbWUiOiJwZXJjb25hLXh0cmFkYi1jbHVzdGVyIiwiYXBwLmt1YmVybmV0ZXMuaW8vcGFydC1vZiI6InBlcmNvbmEteHRyYWRiLWNsdXN0ZXIifSwiYW5ub3RhdGlvbnMiOnsia3ViZWN0bC5rdWJlcm5ldGVzLmlvL2RlZmF1bHQtY29udGFpbmVyIjoiaGFwcm94eSIsInBlcmNvbmEuY29tL2NvbmZpZ3VyYXRpb24taGFzaCI6ImQ0MWQ4Y2Q5OGYwMGIyMDRlOTgwMDk5OGVjZjg0MjdlIn19LCJzcGVjIjp7InZvbHVtZXMiOlt7Im5hbWUiOiJoYXByb3h5LWN1c3RvbSIsImNvbmZpZ01hcCI6eyJuYW1lIjoibW9uaXRvcmluZy1oYXByb3h5Iiwib3B0aW9uYWwiOnRydWV9fSx7Im5hbWUiOiJoYXByb3h5LWF1dG8iLCJlbXB0eURpciI6e319LHsibmFtZSI6Im15c3FsLXVzZXJzLXNlY3JldC1maWxlIiwic2VjcmV0Ijp7InNlY3JldE5hbWUiOiJpbnRlcm5hbC1tb25pdG9yaW5nIiwib3B0aW9uYWwiOmZhbHNlfX0seyJuYW1lIjoibXktZW52LXZhci1zZWNyZXRzIiwic2VjcmV0Ijp7InNlY3JldE5hbWUiOiJteS1lbnYtdmFyLXNlY3JldHMiLCJvcHRpb25hbCI6dHJ1ZX19LHsibmFtZSI6ImJpbiIsImVtcHR5RGlyIjp7fX1dLCJpbml0Q29udGFpbmVycyI6W3sibmFtZSI6InB4Yy1pbml0IiwiaW1hZ2UiOiJwZXJjb25hbGFiL3BlcmNvbmEteHRyYWRiLWNsdXN0ZXItb3BlcmF0b3I6UFItMjU0Ni1jYWYyNTQyMCIsImNvbW1hbmQiOlsiL3B4Yy1pbml0LWVudHJ5cG9pbnQuc2giXSwicmVzb3VyY2VzIjp7ImxpbWl0cyI6eyJjcHUiOiI1MG0iLCJtZW1vcnkiOiI1ME0ifX0sInZvbHVtZU1vdW50cyI6W3sibmFtZSI6ImJpbiIsIm1vdW50UGF0aCI6Ii92YXIvbGliL215c3FsIn1dLCJpbWFnZVB1bGxQb2xpY3kiOiJBbHdheXMifSx7Im5hbWUiOiJoYXByb3h5LWluaXQiLCJpbWFnZSI6InBlcmNvbmFsYWIvcGVyY29uYS14dHJhZGItY2x1c3Rlci1vcGVyYXRvcjpQUi0yNTQ2LWNhZjI1NDIwIiwiY29tbWFuZCI6WyIvaGFwcm94eS1pbml0LWVudHJ5cG9pbnQuc2giXSwicmVzb3VyY2VzIjp7ImxpbWl0cyI6eyJjcHUiOiI1MG0iLCJtZW1vcnkiOiI1ME0ifX0sInZvbHVtZU1vdW50cyI6W3sibmFtZSI6ImJpbiIsIm1vdW50UGF0aCI6Ii9vcHQvcGVyY29uYSJ9XSwiaW1hZ2VQdWxsUG9saWN5IjoiQWx3YXlzIn1dLCJjb250YWluZXJzIjpbeyJuYW1lIjoicG1tLWNsaWVudCIsImltYWdlIjoicGVyY29uYWxhYi9wbW0tY2xpZW50OmRldi1sYXRlc3QiLCJwb3J0cyI6W3siY29udGFpbmVyUG9ydCI6Nzc3N30seyJjb250YWluZXJQb3J0IjozMDEwMH0seyJjb250YWluZXJQb3J0IjozMDEwMX0seyJjb250YWluZXJQb3J0IjozMDEwMn0seyJjb250YWluZXJQb3J0IjozMDEwM30seyJjb250YWluZXJQb3J0IjozMDEwNH0seyJjb250YWluZXJQb3J0IjozMDEwNX1dLCJlbnZGcm9tIjpbeyJzZWNyZXRSZWYiOnsibmFtZSI6Im15LWVudi12YXItc2VjcmV0cyIsIm9wdGlvbmFsIjp0cnVlfX1dLCJlbnYiOlt7Im5hbWUiOiJQTU1fU0VSVkVSIiwidmFsdWUiOiJtb25pdG9yaW5nLXNlcnZpY2UifSx7Im5hbWUiOiJQTU1fVVNFUiIsInZhbHVlIjoiYXBpX2tleSJ9LHsibmFtZSI6IlBNTV9QQVNTV09SRCIsInZhbHVlRnJvbSI6eyJzZWNyZXRLZXlSZWYiOnsibmFtZSI6ImludGVybmFsLW1vbml0b3JpbmciLCJrZXkiOiJwbW1zZXJ2ZXJrZXkifX19LHsibmFtZSI6IkNMSUVOVF9QT1JUX0xJU1RFTiIsInZhbHVlIjoiNzc3NyJ9LHsibmFtZSI6IkNMSUVOVF9QT1JUX01JTiIsInZhbHVlIjoiMzAxMDAifSx7Im5hbWUiOiJDTElFTlRfUE9SVF9NQVgiLCJ2YWx1ZSI6IjMwMTA1In0seyJuYW1lIjoiUE9EX05BTUUiLCJ2YWx1ZUZyb20iOnsiZmllbGRSZWYiOnsiZmllbGRQYXRoIjoibWV0YWRhdGEubmFtZSJ9fX0seyJuYW1lIjoiUE9EX05BTUVTUEFTRSIsInZhbHVlRnJvbSI6eyJmaWVsZFJlZiI6eyJmaWVsZFBhdGgiOiJtZXRhZGF0YS5uYW1lc3BhY2UifX19LHsibmFtZSI6IlBNTV9BR0VOVF9TRVJWRVJfQUREUkVTUyIsInZhbHVlIjoibW9uaXRvcmluZy1zZXJ2aWNlIn0seyJuYW1lIjoiUE1NX0FHRU5UX1NFUlZFUl9VU0VSTkFNRSIsInZhbHVlIjoiYXBpX2tleSJ9LHsibmFtZSI6IlBNTV9BR0VOVF9TRVJWRVJfUEFTU1dPUkQiLCJ2YWx1ZUZyb20iOnsic2VjcmV0S2V5UmVmIjp7Im5hbWUiOiJpbnRlcm5hbC1tb25pdG9yaW5nIiwia2V5IjoicG1tc2VydmVya2V5In19fSx7Im5hbWUiOiJQTU1fQUdFTlRfTElTVEVOX1BPUlQiLCJ2YWx1ZSI6Ijc3NzcifSx7Im5hbWUiOiJQTU1fQUdFTlRfUE9SVFNfTUlOIiwidmFsdWUiOiIzMDEwMCJ9LHsibmFtZSI6IlBNTV9BR0VOVF9QT1JUU19NQVgiLCJ2YWx1ZSI6IjMwMTA1In0seyJuYW1lIjoiUE1NX0FHRU5UX0NPTkZJR19GSUxFIiwidmFsdWUiOiIvdXNyL2xvY2FsL3BlcmNvbmEvcG1tMi9jb25maWcvcG1tLWFnZW50LnlhbWwifSx7Im5hbWUiOiJQTU1fQUdFTlRfU0VSVkVSX0lOU0VDVVJFX1RMUyIsInZhbHVlIjoiMSJ9LHsibmFtZSI6IlBNTV9BR0VOVF9MSVNURU5fQUREUkVTUyIsInZhbHVlIjoiMC4wLjAuMCJ9LHsibmFtZSI6IlBNTV9BR0VOVF9TRVRVUF9NRVRSSUNTX01PREUiLCJ2"..., +  "percona.com/last-config-hash": "eyJyZXBsaWNhcyI6Mywic2VsZWN0b3IiOnsibWF0Y2hMYWJlbHMiOnsiYXBwLmt1YmVybmV0ZXMuaW8vY29tcG9uZW50IjoicHhjIiwiYXBwLmt1YmVybmV0ZXMuaW8vaW5zdGFuY2UiOiJtb25pdG9yaW5nIiwiYXBwLmt1YmVybmV0ZXMuaW8vbWFuYWdlZC1ieSI6InBlcmNvbmEteHRyYWRiLWNsdXN0ZXItb3BlcmF0b3IiLCJhcHAua3ViZXJuZXRlcy5pby9uYW1lIjoicGVyY29uYS14dHJhZGItY2x1c3RlciIsImFwcC5rdWJlcm5ldGVzLmlvL3BhcnQtb2YiOiJwZXJjb25hLXh0cmFkYi1jbHVzdGVyIn19LCJ0ZW1wbGF0ZSI6eyJtZXRhZGF0YSI6eyJsYWJlbHMiOnsiYXBwLmt1YmVybmV0ZXMuaW8vY29tcG9uZW50IjoicHhjIiwiYXBwLmt1YmVybmV0ZXMuaW8vaW5zdGFuY2UiOiJtb25pdG9yaW5nIiwiYXBwLmt1YmVybmV0ZXMuaW8vbWFuYWdlZC1ieSI6InBlcmNvbmEteHRyYWRiLWNsdXN0ZXItb3BlcmF0b3IiLCJhcHAua3ViZXJuZXRlcy5pby9uYW1lIjoicGVyY29uYS14dHJhZGItY2x1c3RlciIsImFwcC5rdWJlcm5ldGVzLmlvL3BhcnQtb2YiOiJwZXJjb25hLXh0cmFkYi1jbHVzdGVyIn0sImFubm90YXRpb25zIjp7Imt1YmVjdGwua3ViZXJuZXRlcy5pby9kZWZhdWx0LWNvbnRhaW5lciI6InB4YyIsImxhc3QtYXBwbGllZC1zZWNyZXQiOiI0MjRiODIzNzM0NjIyMmNlNTg5YjJkOTcwOWQ5NDk3Y2Q0ZWU4MzVlZmIxOWJlZmI0ZGEwODI1ZjBhNzM0ODY1IiwicGVyY29uYS5jb20vY29uZmlndXJhdGlvbi1oYXNoIjoiZDQxZDhjZDk4ZjAw"..., +  "percona.com/last-config-hash": "eyJyZXBsaWNhcyI6Mywic2VsZWN0b3IiOnsibWF0Y2hMYWJlbHMiOnsiYXBwLmt1YmVybmV0ZXMuaW8vY29tcG9uZW50IjoicHhjIiwiYXBwLmt1YmVybmV0ZXMuaW8vaW5zdGFuY2UiOiJtb25pdG9yaW5nIiwiYXBwLmt1YmVybmV0ZXMuaW8vbWFuYWdlZC1ieSI6InBlcmNvbmEteHRyYWRiLWNsdXN0ZXItb3BlcmF0b3IiLCJhcHAua3ViZXJuZXRlcy5pby9uYW1lIjoicGVyY29uYS14dHJhZGItY2x1c3RlciIsImFwcC5rdWJlcm5ldGVzLmlvL3BhcnQtb2YiOiJwZXJjb25hLXh0cmFkYi1jbHVzdGVyIn19LCJ0ZW1wbGF0ZSI6eyJtZXRhZGF0YSI6eyJsYWJlbHMiOnsiYXBwLmt1YmVybmV0ZXMuaW8vY29tcG9uZW50IjoicHhjIiwiYXBwLmt1YmVybmV0ZXMuaW8vaW5zdGFuY2UiOiJtb25pdG9yaW5nIiwiYXBwLmt1YmVybmV0ZXMuaW8vbWFuYWdlZC1ieSI6InBlcmNvbmEteHRyYWRiLWNsdXN0ZXItb3BlcmF0b3IiLCJhcHAua3ViZXJuZXRlcy5pby9uYW1lIjoicGVyY29uYS14dHJhZGItY2x1c3RlciIsImFwcC5rdWJlcm5ldGVzLmlvL3BhcnQtb2YiOiJwZXJjb25hLXh0cmFkYi1jbHVzdGVyIn0sImFubm90YXRpb25zIjp7Imt1YmVjdGwua3ViZXJuZXRlcy5pby9kZWZhdWx0LWNvbnRhaW5lciI6InB4YyIsInBlcmNvbmEuY29tL2NvbmZpZ3VyYXRpb24taGFzaCI6ImQ0MWQ4Y2Q5OGYwMGIyMDRlOTgwMDk5OGVjZjg0MjdlIiwicGVyY29uYS5jb20vZW52LXNlY3JldC1jb25maWctaGFzaCI6IjViYzQwM2E4ZDQ0MzI0YzFiYzcwNGYwMmU3NDg3"..., +  "percona.com/last-config-hash": "eyJyZXBsaWNhcyI6Mywic2VsZWN0b3IiOnsibWF0Y2hMYWJlbHMiOnsiYXBwLmt1YmVybmV0ZXMuaW8vY29tcG9uZW50IjoicHhjIiwiYXBwLmt1YmVybmV0ZXMuaW8vaW5zdGFuY2UiOiJtb25pdG9yaW5nIiwiYXBwLmt1YmVybmV0ZXMuaW8vbWFuYWdlZC1ieSI6InBlcmNvbmEteHRyYWRiLWNsdXN0ZXItb3BlcmF0b3IiLCJhcHAua3ViZXJuZXRlcy5pby9uYW1lIjoicGVyY29uYS14dHJhZGItY2x1c3RlciIsImFwcC5rdWJlcm5ldGVzLmlvL3BhcnQtb2YiOiJwZXJjb25hLXh0cmFkYi1jbHVzdGVyIn19LCJ0ZW1wbGF0ZSI6eyJtZXRhZGF0YSI6eyJsYWJlbHMiOnsiYXBwLmt1YmVybmV0ZXMuaW8vY29tcG9uZW50IjoicHhjIiwiYXBwLmt1YmVybmV0ZXMuaW8vaW5zdGFuY2UiOiJtb25pdG9yaW5nIiwiYXBwLmt1YmVybmV0ZXMuaW8vbWFuYWdlZC1ieSI6InBlcmNvbmEteHRyYWRiLWNsdXN0ZXItb3BlcmF0b3IiLCJhcHAua3ViZXJuZXRlcy5pby9uYW1lIjoicGVyY29uYS14dHJhZGItY2x1c3RlciIsImFwcC5rdWJlcm5ldGVzLmlvL3BhcnQtb2YiOiJwZXJjb25hLXh0cmFkYi1jbHVzdGVyIn0sImFubm90YXRpb25zIjp7Imt1YmVjdGwua3ViZXJuZXRlcy5pby9kZWZhdWx0LWNvbnRhaW5lciI6InB4YyIsInBlcmNvbmEuY29tL2NvbmZpZ3VyYXRpb24taGFzaCI6ImQ0MWQ4Y2Q5OGYwMGIyMDRlOTgwMDk5OGVjZjg0MjdlIiwicGVyY29uYS5jb20vc3NsLWhhc2giOiI0NTlkNWQ1ZTRkMTA2YWIxMDgxMzFiZmI0NTg4OWRlMSIsInBlcmNvbmEuY29tL3NzbC1pbnRlcm5hbC1oYXNoIjoiOTUxNzAzMDMzZTBkZWRkMDEyNjBiZTBkMjVkNDM1OTEifX0sInNwZWMiOnsidm9sdW1lcyI6W3sibmFtZSI6InRtcCIsImVtcHR5RGlyIjp7fX0seyJuYW1lIjoiY29uZmlnIiwiY29uZmlnTWFwIjp7Im5hbWUiOiJtb25pdG9yaW5nLXB4YyIsIm9wdGlvbmFsIjp0cnVlfX0seyJuYW1lIjoic3NsLWludGVybmFsIiwic2VjcmV0Ijp7InNlY3JldE5hbWUiOiJtb25pdG9yaW5nLXNzbC1pbnRlcm5hbCIsIm9wdGlvbmFsIjp0cnVlfX0seyJuYW1lIjoic3NsIiwic2VjcmV0Ijp7InNlY3JldE5hbWUiOiJzb21lLW5hbWUtc3NsIiwib3B0aW9uYWwiOmZhbHNlfX0seyJuYW1lIjoiYXV0by1jb25maWciLCJjb25maWdNYXAiOnsibmFtZSI6ImF1dG8tbW9uaXRvcmluZy1weGMiLCJvcHRpb25hbCI6dHJ1ZX19LHsibmFtZSI6InZhdWx0LWtleXJpbmctc2VjcmV0Iiwic2VjcmV0Ijp7InNlY3JldE5hbWUiOiJtb25pdG9yaW5nLXZhdWx0Iiwib3B0aW9uYWwiOnRydWV9fSx7Im5hbWUiOiJteXNxbC11c2Vycy1zZWNyZXQtZmlsZSIsInNlY3JldCI6eyJzZWNyZXROYW1lIjoiaW50ZXJuYWwtbW9uaXRvcmluZyIsIm9wdGlvbmFsIjpmYWxzZX19LHsibmFtZSI6Im15c3FsLWluaXQtZmlsZSIsInNlY3JldCI6eyJzZWNyZXROYW1lIjoibW9uaXRvcmluZy1teXNxbC1pbml0Iiwib3B0aW9uYWwiOnRydWV9fSx7Im5hbWUiOiJiaW4iLCJlbXB0eURpciI6e319XSwiaW5pdENvbnRhaW5lcnMiOlt7Im5hbWUiOiJweGMtaW5pdCIsImltYWdlIjoicGVyY29uYWxhYi9wZXJjb25hLXh0cmFkYi1jbHVzdGVyLW9wZXJhdG9yOlBSLTI1NDYtY2FmMjU0MjAiLCJjb21tYW5kIjpbIi9weGMtaW5pdC1lbnRyeXBvaW50LnNoIl0sInJlc291cmNlcyI6eyJsaW1pdHMiOnsiY3B1IjoiNTBtIiwibWVtb3J5IjoiNTBNIn19LCJ2b2x1bWVNb3VudHMiOlt7Im5hbWUiOiJkYXRhZGlyIiwibW91bnRQYXRoIjoiL3Zhci9saWIvbXlzcWwifSx7Im5hbWUiOiJiaW4iLCJtb3VudFBhdGgiOiIvb3B0L3BlcmNvbmEifV0sImltYWdlUHVsbFBvbGljeSI6IkFsd2F5cyJ9XSwiY29udGFpbmVycyI6W3sibmFtZSI6InBtbS1jbGllbnQiLCJpbWFnZSI6InBlcmNvbmFsYWIvcG1tLWNsaWVudDpkZXYtbGF0ZXN0IiwicG9ydHMiOlt7ImNvbnRhaW5lclBvcnQiOjc3Nzd9LHsiY29udGFpbmVyUG9ydCI6MzAxMDB9LHsiY29udGFpbmVyUG9ydCI6MzAxMDF9LHsiY29udGFpbmVyUG9ydCI6MzAxMDJ9LHsiY29udGFpbmVyUG9ydCI6MzAxMDN9LHsiY29udGFpbmVyUG9ydCI6MzAxMDR9LHsiY29udGFpbmVyUG9ydCI6MzAxMDV9XSwiZW52RnJvbSI6W3sic2VjcmV0UmVmIjp7Im5hbWUiOiJteS1lbnYtdmFyLXNlY3JldHMiLCJvcHRpb25hbCI6dHJ1ZX19XSwiZW52IjpbeyJuYW1lIjoiUE1NX1NFUlZFUiIsInZhbHVlIjoibW9uaXRvcmluZy1zZXJ2aWNlIn0seyJuYW1lIjoiUE1NX1VTRVIiLCJ2YWx1ZSI6ImFwaV9rZXkifSx7Im5hbWUiOiJQTU1fUEFTU1dPUkQiLCJ2YWx1ZUZyb20iOnsic2VjcmV0S2V5UmVmIjp7Im5hbWUiOiJpbnRlcm5hbC1tb25pdG9yaW5nIiwia2V5IjoicG1tc2VydmVya2V5In19fSx7Im5hbWUiOiJDTElFTlRfUE9SVF9MSVNURU4iLCJ2YWx1ZSI6Ijc3NzcifSx7Im5hbWUiOiJDTElFTlRfUE9SVF9NSU4iLCJ2YWx1ZSI6IjMwMTAwIn0seyJuYW1lIjoiQ0xJRU5UX1BPUlRfTUFYIiwidmFsdWUiOiIzMDEwNSJ9LHsibmFtZSI6IlBPRF9OQU1FIiwidmFsdWVGcm9tIjp7ImZpZWxkUmVmIjp7ImZpZWxkUGF0aCI6Im1ldGFkYXRhLm5hbWUifX19LHsibmFtZSI6IlBPRF9OQU1FU1BBU0UiLCJ2YWx1ZUZyb20iOnsiZmllbGRSZWYiOnsiZmllbGRQYXRoIjoibWV0YWRhdGEubmFtZXNwYWNlIn19fSx7Im5hbWUiOiJQTU1fQUdFTlRfU0VSVkVSX0FERFJFU1MiLCJ2YWx1ZSI6Im1vbml0b3Jpbmctc2VydmljZSJ9LHsibmFtZSI6IlBNTV9BR0VOVF9TRVJWRVJfVVNFUk5BTUUiLCJ2YWx1ZSI6ImFwaV9rZXkifSx7Im5hbWUiOiJQTU1fQUdFTlRfU0VSVkVSX1BBU1NXT1JEIiwidmFsdWVGcm9tIjp7InNlY3JldEtleVJlZiI6eyJuYW1lIjoiaW50ZXJuYWwtbW9uaXRvcmluZyIsImtleSI6InBtbXNlcnZlcmtleSJ9fX0seyJuYW1lIjoiUE1NX0FHRU5UX0xJU1RFTl9QT1JUIiwidmFsdWUiOiI3Nzc3In0seyJuYW1lIjoiUE1NX0FHRU5UX1BPUlRTX01JTiIsInZhbHVlIjoiMzAxMDAifSx7Im5hbWUiOiJQTU1fQUdFTlRfUE9SVFNfTUFYIiwidmFsdWUiOiIzMDEwNSJ9LHsibmFtZSI6IlBNTV9BR0VOVF9DT05GSUdfRklMRSIsInZhbHVl"..., +  Annotations: map[string]string{ +  AvailableReplicas: 0, +  CollisionCount: nil, +  CreationTimestamp: v1.Time{}, +  CurrentReplicas: 0, +  CurrentRevision: "", +  Generation: 0, +  ManagedFields: nil, +  ObservedGeneration: 0, +  PersistentVolumeClaimRetentionPolicy: nil, +  PodManagementPolicy: "", +  ReadyReplicas: 0, +  Replicas: 0, +  Replicas: &0, +  ResourceVersion: "", +  RevisionHistoryLimit: nil, +  UID: "", +  UpdateRevision: "", +  UpdatedReplicas: 0, +  }, -  APIVersion: "v1", -  Key: "pmmserver", -  Scheme: "HTTP", -  DefaultMode: &420, -  DefaultMode: &420, -  Protocol: "TCP", -  Value: "$(POD_NAMESPASE)-$(POD_NAME)", -  Value: "admin", -  PeriodSeconds: 10, -  TerminationMessagePath: "/dev/termination-log", -  TerminationMessagePolicy: "File", -  Phase: "Pending", -  VolumeMode: &"Filesystem", -  APIVersion: "apps/v1", -  APIVersion: "apps/v1", -  DNSPolicy: "ClusterFirst", -  DeprecatedServiceAccount: "default", -  FieldsType: "FieldsV1", -  FieldsType: "FieldsV1", -  FieldsV1: s`{"f:status":{"f:availableReplicas":{},"f:collisionCount":{},"f:c`..., -  FieldsV1: s`{"f:metadata":{"f:annotations":{".":{},"f:percona.com/last-confi`..., -  Manager: "kube-controller-manager", -  Manager: "percona-xtradb-cluster-operator", -  Operation: "Update", -  Operation: "Update", -  RestartPolicy: "Always", -  SchedulerName: "default-scheduler", -  SchedulerName: "default-scheduler", -  SecurityContext: s"&PodSecurityContext{SELinuxOptions:nil,RunAsUser:nil,RunAsNonRoot:nil,SupplementalGroups:[],FSGroup:nil,RunAsGroup:nil,Sysctls:[]Sysctl{},WindowsOptions:nil,FSGroupChangePolicy:nil,SeccompProfile:nil,AppArmorProfile:nil,SupplementalGroupsPolicy:nil,SELinux"..., -  Subresource: "status", -  Time: s"2026-10-05 13:22:51 +0000 UTC", -  Time: s"2026-10-05 13:24:55 +0000 UTC", -  Time: s"2026-10-05 13:29:53 +0000 UTC", -  Time: s"2026-10-05 13:31:52 +0000 UTC", -  Time: s"2026-10-05 13:33:08 +0000 UTC", -  Time: s"2026-10-05 13:34:56 +0000 UTC", -  Time: s"2026-10-05 13:35:52 +0000 UTC", -  Time: s"2026-10-05 13:37:51 +0000 UTC", -  Time: s"2026-10-05 13:21:36 +0000 UTC", -  Time: s"2026-10-05 13:29:02 +0000 UTC", -  Time: s"2026-10-05 13:32:12 +0000 UTC", -  Time: s"2026-10-05 13:34:58 +0000 UTC", -  TopologySpreadConstraints: nil, -  "percona.com/last-config-hash": "eyJyZXBsaWNhcyI6Miwic2VsZWN0b3IiOnsibWF0Y2hMYWJlbHMiOnsiYXBwLmt1YmVybmV0ZXMuaW8vY29tcG9uZW50IjoiaGFwcm94eSIsImFwcC5rdWJlcm5ldGVzLmlvL2luc3RhbmNlIjoibW9uaXRvcmluZyIsImFwcC5rdWJlcm5ldGVzLmlvL21hbmFnZWQtYnkiOiJwZXJjb25hLXh0cmFkYi1jbHVzdGVyLW9wZXJhdG9yIiwiYXBw"..., -  "percona.com/last-config-hash": "eyJyZXBsaWNhcyI6Miwic2VsZWN0b3IiOnsibWF0Y2hMYWJlbHMiOnsiYXBwLmt1YmVybmV0ZXMuaW8vY29tcG9uZW50IjoiaGFwcm94eSIsImFwcC5rdWJlcm5ldGVzLmlvL2luc3RhbmNlIjoibW9uaXRvcmluZyIsImFwcC5rdWJlcm5ldGVzLmlvL21hbmFnZWQtYnkiOiJwZXJjb25hLXh0cmFkYi1jbHVzdGVyLW9wZXJhdG9yIiwiYXBwLmt1YmVybmV0ZXMuaW8vbmFtZSI6InBlcmNvbmEteHRyYWRiLWNsdXN0ZXIiLCJhcHAua3ViZXJuZXRlcy5pby9wYXJ0LW9mIjoicGVyY29uYS14dHJhZGItY2x1c3RlciJ9fSwidGVtcGxhdGUiOnsibWV0YWRhdGEiOnsibGFiZWxzIjp7ImFwcC5rdWJlcm5ldGVzLmlvL2NvbXBvbmVudCI6ImhhcHJveHkiLCJhcHAua3ViZXJuZXRlcy5pby9pbnN0YW5jZSI6Im1vbml0b3JpbmciLCJhcHAua3ViZXJuZXRlcy5pby9tYW5hZ2VkLWJ5IjoicGVyY29uYS14dHJhZGItY2x1c3Rlci1vcGVyYXRvciIsImFwcC5rdWJlcm5ldGVzLmlvL25hbWUiOiJwZXJjb25hLXh0cmFkYi1jbHVzdGVyIiwiYXBwLmt1YmVybmV0ZXMuaW8vcGFydC1vZiI6InBlcmNvbmEteHRyYWRiLWNsdXN0ZXIifSwiYW5ub3RhdGlvbnMiOnsia3ViZWN0bC5rdWJlcm5ldGVzLmlvL2RlZmF1bHQtY29udGFpbmVyIjoiaGFwcm94eSIsInBlcmNvbmEuY29tL2NvbmZpZ3VyYXRpb24taGFzaCI6ImQ0MWQ4Y2Q5OGYwMGIyMDRlOTgwMDk5OGVjZjg0MjdlIiwicGVyY29uYS5jb20vZW52LXNlY3JldC1jb25maWctaGFzaCI6IjViYzQwM2E4ZDQ0MzI0YzFi"..., -  "percona.com/last-config-hash": "eyJyZXBsaWNhcyI6Miwic2VsZWN0b3IiOnsibWF0Y2hMYWJlbHMiOnsiYXBwLmt1YmVybmV0ZXMuaW8vY29tcG9uZW50IjoiaGFwcm94eSIsImFwcC5rdWJlcm5ldGVzLmlvL2luc3RhbmNlIjoibW9uaXRvcmluZyIsImFwcC5rdWJlcm5ldGVzLmlvL21hbmFnZWQtYnkiOiJwZXJjb25hLXh0cmFkYi1jbHVzdGVyLW9wZXJhdG9yIiwiYXBwLmt1YmVybmV0ZXMuaW8vbmFtZSI6InBlcmNvbmEteHRyYWRiLWNsdXN0ZXIiLCJhcHAua3ViZXJuZXRlcy5pby9wYXJ0LW9mIjoicGVyY29uYS14dHJhZGItY2x1c3RlciJ9fSwidGVtcGxhdGUiOnsibWV0YWRhdGEiOnsibGFiZWxzIjp7ImFwcC5rdWJlcm5ldGVzLmlvL2NvbXBvbmVudCI6ImhhcHJveHkiLCJhcHAua3ViZXJuZXRlcy5pby9pbnN0YW5jZSI6Im1vbml0b3JpbmciLCJhcHAua3ViZXJuZXRlcy5pby9tYW5hZ2VkLWJ5IjoicGVyY29uYS14dHJhZGItY2x1c3Rlci1vcGVyYXRvciIsImFwcC5rdWJlcm5ldGVzLmlvL25hbWUiOiJwZXJjb25hLXh0cmFkYi1jbHVzdGVyIiwiYXBwLmt1YmVybmV0ZXMuaW8vcGFydC1vZiI6InBlcmNvbmEteHRyYWRiLWNsdXN0ZXIifSwiYW5ub3RhdGlvbnMiOnsia3ViZWN0bC5rdWJlcm5ldGVzLmlvL2RlZmF1bHQtY29udGFpbmVyIjoiaGFwcm94eSIsInBlcmNvbmEuY29tL2NvbmZpZ3VyYXRpb24taGFzaCI6ImQ0MWQ4Y2Q5OGYwMGIyMDRlOTgwMDk5OGVjZjg0MjdlIn19LCJzcGVjIjp7InZvbHVtZXMiOlt7Im5hbWUiOiJoYXByb3h5LWN1c3RvbSIsImNvbmZpZ01h"..., -  "percona.com/last-config-hash": "eyJyZXBsaWNhcyI6Miwic2VsZWN0b3IiOnsibWF0Y2hMYWJlbHMiOnsiYXBwLmt1YmVybmV0ZXMuaW8vY29tcG9uZW50IjoiaGFwcm94eSIsImFwcC5rdWJlcm5ldGVzLmlvL2luc3RhbmNlIjoibW9uaXRvcmluZyIsImFwcC5rdWJlcm5ldGVzLmlvL21hbmFnZWQtYnkiOiJwZXJjb25hLXh0cmFkYi1jbHVzdGVyLW9wZXJhdG9yIiwiYXBwLmt1YmVybmV0ZXMuaW8vbmFtZSI6InBlcmNvbmEteHRyYWRiLWNsdXN0ZXIiLCJhcHAua3ViZXJuZXRlcy5pby9wYXJ0LW9mIjoicGVyY29uYS14dHJhZGItY2x1c3RlciJ9fSwidGVtcGxhdGUiOnsibWV0YWRhdGEiOnsibGFiZWxzIjp7ImFwcC5rdWJlcm5ldGVzLmlvL2NvbXBvbmVudCI6ImhhcHJveHkiLCJhcHAua3ViZXJuZXRlcy5pby9pbnN0YW5jZSI6Im1vbml0b3JpbmciLCJhcHAua3ViZXJuZXRlcy5pby9tYW5hZ2VkLWJ5IjoicGVyY29uYS14dHJhZGItY2x1c3Rlci1vcGVyYXRvciIsImFwcC5rdWJlcm5ldGVzLmlvL25hbWUiOiJwZXJjb25hLXh0cmFkYi1jbHVzdGVyIiwiYXBwLmt1YmVybmV0ZXMuaW8vcGFydC1vZiI6InBlcmNvbmEteHRyYWRiLWNsdXN0ZXIifSwiYW5ub3RhdGlvbnMiOnsia3ViZWN0bC5rdWJlcm5ldGVzLmlvL2RlZmF1bHQtY29udGFpbmVyIjoiaGFwcm94eSIsInBlcmNvbmEuY29tL2NvbmZpZ3VyYXRpb24taGFzaCI6ImQ0MWQ4Y2Q5OGYwMGIyMDRlOTgwMDk5OGVjZjg0MjdlIn19LCJzcGVjIjp7InZvbHVtZXMiOlt7Im5hbWUiOiJoYXByb3h5LWN1c3RvbSIsImNvbmZpZ01hcCI6eyJuYW1lIjoibW9uaXRvcmluZy1oYXByb3h5Iiwib3B0aW9uYWwiOnRydWV9fSx7Im5hbWUiOiJoYXByb3h5LWF1dG8iLCJlbXB0eURpciI6e319LHsibmFtZSI6Im15c3FsLXVzZXJzLXNlY3JldC1maWxlIiwic2VjcmV0Ijp7InNlY3JldE5hbWUiOiJpbnRlcm5hbC1tb25pdG9yaW5nIiwib3B0aW9uYWwiOmZhbHNlfX0seyJuYW1lIjoibXktZW52LXZhci1zZWNyZXRzIiwic2VjcmV0Ijp7InNlY3JldE5hbWUiOiJteS1lbnYtdmFyLXNlY3JldHMiLCJvcHRpb25hbCI6dHJ1ZX19LHsibmFtZSI6ImJpbiIsImVtcHR5RGlyIjp7fX1dLCJpbml0Q29udGFpbmVycyI6W3sibmFtZSI6InB4Yy1pbml0IiwiaW1hZ2UiOiJwZXJjb25hbGFiL3BlcmNvbmEteHRyYWRiLWNsdXN0ZXItb3BlcmF0b3I6UFItMjU0Ni1jYWYyNTQyMCIsImNvbW1hbmQiOlsiL3B4Yy1pbml0LWVudHJ5cG9pbnQuc2giXSwicmVzb3VyY2VzIjp7ImxpbWl0cyI6eyJjcHUiOiI1MG0iLCJtZW1vcnkiOiI1ME0ifX0sInZvbHVtZU1vdW50cyI6W3sibmFtZSI6ImJpbiIsIm1vdW50UGF0aCI6Ii92YXIvbGliL215c3FsIn1dLCJpbWFnZVB1bGxQb2xpY3kiOiJBbHdheXMifSx7Im5hbWUiOiJoYXByb3h5LWluaXQiLCJpbWFnZSI6InBlcmNvbmFsYWIvcGVyY29uYS14dHJhZGItY2x1c3Rlci1vcGVyYXRvcjpQUi0yNTQ2LWNhZjI1NDIwIiwiY29tbWFuZCI6WyIvaGFwcm94eS1pbml0LWVudHJ5cG9pbnQuc2giXSwicmVzb3VyY2VzIjp7ImxpbWl0cyI6eyJjcHUiOiI1MG0iLCJtZW1vcnkiOiI1ME0ifX0sInZvbHVtZU1vdW50cyI6W3sibmFtZSI6ImJpbiIsIm1vdW50UGF0aCI6Ii9vcHQvcGVyY29uYSJ9XSwiaW1hZ2VQdWxsUG9saWN5IjoiQWx3YXlzIn1dLCJjb250YWluZXJzIjpbeyJuYW1lIjoicG1tLWNsaWVudCIsImltYWdlIjoicGVyY29uYWxhYi9wbW0tY2xpZW50OmRldi1sYXRlc3QiLCJwb3J0cyI6W3siY29udGFpbmVyUG9ydCI6Nzc3N30seyJjb250YWluZXJQb3J0IjozMDEwMH0seyJjb250YWluZXJQb3J0IjozMDEwMX0seyJjb250YWluZXJQb3J0IjozMDEwMn0seyJjb250YWluZXJQb3J0IjozMDEwM30seyJjb250YWluZXJQb3J0IjozMDEwNH0seyJjb250YWluZXJQb3J0IjozMDEwNX1dLCJlbnZGcm9tIjpbeyJzZWNyZXRSZWYiOnsibmFtZSI6Im15LWVudi12YXItc2VjcmV0cyIsIm9wdGlvbmFsIjp0cnVlfX1dLCJlbnYiOlt7Im5hbWUiOiJQTU1fU0VSVkVSIiwidmFsdWUiOiJtb25pdG9yaW5nLXNlcnZpY2UifSx7Im5hbWUiOiJQTU1fVVNFUiIsInZhbHVlIjoiYWRtaW4ifSx7Im5hbWUiOiJQTU1fUEFTU1dPUkQiLCJ2YWx1ZUZyb20iOnsic2VjcmV0S2V5UmVmIjp7Im5hbWUiOiJpbnRlcm5hbC1tb25pdG9yaW5nIiwia2V5IjoicG1tc2VydmVyIn19fSx7Im5hbWUiOiJDTElFTlRfUE9SVF9MSVNURU4iLCJ2YWx1ZSI6Ijc3NzcifSx7Im5hbWUiOiJDTElFTlRfUE9SVF9NSU4iLCJ2YWx1ZSI6IjMwMTAwIn0seyJuYW1lIjoiQ0xJRU5UX1BPUlRfTUFYIiwidmFsdWUiOiIzMDEwNSJ9LHsibmFtZSI6IlBPRF9OQU1FIiwidmFsdWVGcm9tIjp7ImZpZWxkUmVmIjp7ImZpZWxkUGF0aCI6Im1ldGFkYXRhLm5hbWUifX19LHsibmFtZSI6IlBPRF9OQU1FU1BBU0UiLCJ2YWx1ZUZyb20iOnsiZmllbGRSZWYiOnsiZmllbGRQYXRoIjoibWV0YWRhdGEubmFtZXNwYWNlIn19fSx7Im5hbWUiOiJQTU1fQUdFTlRfU0VSVkVSX0FERFJFU1MiLCJ2YWx1ZSI6Im1vbml0b3Jpbmctc2VydmljZSJ9LHsibmFtZSI6IlBNTV9BR0VOVF9TRVJWRVJfVVNFUk5BTUUiLCJ2YWx1ZSI6ImFkbWluIn0seyJuYW1lIjoiUE1NX0FHRU5UX1NFUlZFUl9QQVNTV09SRCIsInZhbHVlRnJvbSI6eyJzZWNyZXRLZXlSZWYiOnsibmFtZSI6ImludGVybmFsLW1vbml0b3JpbmciLCJrZXkiOiJwbW1zZXJ2ZXIifX19LHsibmFtZSI6IlBNTV9BR0VOVF9MSVNURU5fUE9SVCIsInZhbHVlIjoiNzc3NyJ9LHsibmFtZSI6IlBNTV9BR0VOVF9QT1JUU19NSU4iLCJ2YWx1ZSI6IjMwMTAwIn0seyJuYW1lIjoiUE1NX0FHRU5UX1BPUlRTX01BWCIsInZhbHVlIjoiMzAxMDUifSx7Im5hbWUiOiJQTU1fQUdFTlRfQ09ORklHX0ZJTEUiLCJ2YWx1ZSI6Ii91c3IvbG9jYWwvcGVyY29uYS9wbW0yL2NvbmZpZy9wbW0tYWdlbnQueWFtbCJ9LHsibmFtZSI6IlBNTV9BR0VOVF9TRVJWRVJfSU5TRUNVUkVfVExTIiwidmFsdWUiOiIxIn0seyJuYW1lIjoiUE1NX0FHRU5UX0xJU1RFTl9BRERSRVNTIiwidmFsdWUiOiIwLjAuMC4wIn0seyJuYW1lIjoiUE1NX0FHRU5UX1NFVFVQX01FVFJJQ1NfTU9ERSIsInZhbHVlIjoicHVz"..., -  "percona.com/last-config-hash": "eyJyZXBsaWNhcyI6Mywic2VsZWN0b3IiOnsibWF0Y2hMYWJlbHMiOnsiYXBwLmt1YmVybmV0ZXMuaW8vY29tcG9uZW50IjoicHhjIiwiYXBwLmt1YmVybmV0ZXMuaW8vaW5zdGFuY2UiOiJtb25pdG9yaW5nIiwiYXBwLmt1YmVybmV0ZXMuaW8vbWFuYWdlZC1ieSI6InBlcmNvbmEteHRyYWRiLWNsdXN0ZXItb3BlcmF0b3IiLCJhcHAua3Vi"..., -  "percona.com/last-config-hash": "eyJyZXBsaWNhcyI6Mywic2VsZWN0b3IiOnsibWF0Y2hMYWJlbHMiOnsiYXBwLmt1YmVybmV0ZXMuaW8vY29tcG9uZW50IjoicHhjIiwiYXBwLmt1YmVybmV0ZXMuaW8vaW5zdGFuY2UiOiJtb25pdG9yaW5nIiwiYXBwLmt1YmVybmV0ZXMuaW8vbWFuYWdlZC1ieSI6InBlcmNvbmEteHRyYWRiLWNsdXN0ZXItb3BlcmF0b3IiLCJhcHAua3ViZXJuZXRlcy5pby9uYW1lIjoicGVyY29uYS14dHJhZGItY2x1c3RlciIsImFwcC5rdWJlcm5ldGVzLmlvL3BhcnQtb2YiOiJwZXJjb25hLXh0cmFkYi1jbHVzdGVyIn19LCJ0ZW1wbGF0ZSI6eyJtZXRhZGF0YSI6eyJsYWJlbHMiOnsiYXBwLmt1YmVybmV0ZXMuaW8vY29tcG9uZW50IjoicHhjIiwiYXBwLmt1YmVybmV0ZXMuaW8vaW5zdGFuY2UiOiJtb25pdG9yaW5nIiwiYXBwLmt1YmVybmV0ZXMuaW8vbWFuYWdlZC1ieSI6InBlcmNvbmEteHRyYWRiLWNsdXN0ZXItb3BlcmF0b3IiLCJhcHAua3ViZXJuZXRlcy5pby9uYW1lIjoicGVyY29uYS14dHJhZGItY2x1c3RlciIsImFwcC5rdWJlcm5ldGVzLmlvL3BhcnQtb2YiOiJwZXJjb25hLXh0cmFkYi1jbHVzdGVyIn0sImFubm90YXRpb25zIjp7Imt1YmVjdGwua3ViZXJuZXRlcy5pby9kZWZhdWx0LWNvbnRhaW5lciI6InB4YyIsInBlcmNvbmEuY29tL2NvbmZpZ3VyYXRpb24taGFzaCI6ImQ0MWQ4Y2Q5OGYwMGIyMDRlOTgwMDk5OGVjZjg0MjdlIiwicGVyY29uYS5jb20vZW52LXNlY3JldC1jb25maWctaGFzaCI6IjViYzQwM2E4ZDQ0MzI0YzFiYzcwNGYwMmU3NDg3"..., -  "percona.com/last-config-hash": "eyJyZXBsaWNhcyI6Mywic2VsZWN0b3IiOnsibWF0Y2hMYWJlbHMiOnsiYXBwLmt1YmVybmV0ZXMuaW8vY29tcG9uZW50IjoicHhjIiwiYXBwLmt1YmVybmV0ZXMuaW8vaW5zdGFuY2UiOiJtb25pdG9yaW5nIiwiYXBwLmt1YmVybmV0ZXMuaW8vbWFuYWdlZC1ieSI6InBlcmNvbmEteHRyYWRiLWNsdXN0ZXItb3BlcmF0b3IiLCJhcHAua3ViZXJuZXRlcy5pby9uYW1lIjoicGVyY29uYS14dHJhZGItY2x1c3RlciIsImFwcC5rdWJlcm5ldGVzLmlvL3BhcnQtb2YiOiJwZXJjb25hLXh0cmFkYi1jbHVzdGVyIn19LCJ0ZW1wbGF0ZSI6eyJtZXRhZGF0YSI6eyJsYWJlbHMiOnsiYXBwLmt1YmVybmV0ZXMuaW8vY29tcG9uZW50IjoicHhjIiwiYXBwLmt1YmVybmV0ZXMuaW8vaW5zdGFuY2UiOiJtb25pdG9yaW5nIiwiYXBwLmt1YmVybmV0ZXMuaW8vbWFuYWdlZC1ieSI6InBlcmNvbmEteHRyYWRiLWNsdXN0ZXItb3BlcmF0b3IiLCJhcHAua3ViZXJuZXRlcy5pby9uYW1lIjoicGVyY29uYS14dHJhZGItY2x1c3RlciIsImFwcC5rdWJlcm5ldGVzLmlvL3BhcnQtb2YiOiJwZXJjb25hLXh0cmFkYi1jbHVzdGVyIn0sImFubm90YXRpb25zIjp7Imt1YmVjdGwua3ViZXJuZXRlcy5pby9kZWZhdWx0LWNvbnRhaW5lciI6InB4YyIsInBlcmNvbmEuY29tL2NvbmZpZ3VyYXRpb24taGFzaCI6ImQ0MWQ4Y2Q5OGYwMGIyMDRlOTgwMDk5OGVjZjg0MjdlIiwicGVyY29uYS5jb20vc3NsLWhhc2giOiI0NTlkNWQ1ZTRkMTA2YWIxMDgxMzFiZmI0NTg4OWRlMSIsInBlcmNvbmEu"..., -  "percona.com/last-config-hash": "eyJyZXBsaWNhcyI6Mywic2VsZWN0b3IiOnsibWF0Y2hMYWJlbHMiOnsiYXBwLmt1YmVybmV0ZXMuaW8vY29tcG9uZW50IjoicHhjIiwiYXBwLmt1YmVybmV0ZXMuaW8vaW5zdGFuY2UiOiJtb25pdG9yaW5nIiwiYXBwLmt1YmVybmV0ZXMuaW8vbWFuYWdlZC1ieSI6InBlcmNvbmEteHRyYWRiLWNsdXN0ZXItb3BlcmF0b3IiLCJhcHAua3ViZXJuZXRlcy5pby9uYW1lIjoicGVyY29uYS14dHJhZGItY2x1c3RlciIsImFwcC5rdWJlcm5ldGVzLmlvL3BhcnQtb2YiOiJwZXJjb25hLXh0cmFkYi1jbHVzdGVyIn19LCJ0ZW1wbGF0ZSI6eyJtZXRhZGF0YSI6eyJsYWJlbHMiOnsiYXBwLmt1YmVybmV0ZXMuaW8vY29tcG9uZW50IjoicHhjIiwiYXBwLmt1YmVybmV0ZXMuaW8vaW5zdGFuY2UiOiJtb25pdG9yaW5nIiwiYXBwLmt1YmVybmV0ZXMuaW8vbWFuYWdlZC1ieSI6InBlcmNvbmEteHRyYWRiLWNsdXN0ZXItb3BlcmF0b3IiLCJhcHAua3ViZXJuZXRlcy5pby9uYW1lIjoicGVyY29uYS14dHJhZGItY2x1c3RlciIsImFwcC5rdWJlcm5ldGVzLmlvL3BhcnQtb2YiOiJwZXJjb25hLXh0cmFkYi1jbHVzdGVyIn0sImFubm90YXRpb25zIjp7Imt1YmVjdGwua3ViZXJuZXRlcy5pby9kZWZhdWx0LWNvbnRhaW5lciI6InB4YyIsInBlcmNvbmEuY29tL2NvbmZpZ3VyYXRpb24taGFzaCI6ImQ0MWQ4Y2Q5OGYwMGIyMDRlOTgwMDk5OGVjZjg0MjdlIiwicGVyY29uYS5jb20vc3NsLWhhc2giOiI0NTlkNWQ1ZTRkMTA2YWIxMDgxMzFiZmI0NTg4OWRlMSIsInBlcmNvbmEuY29tL3NzbC1pbnRlcm5hbC1oYXNoIjoiOTUxNzAzMDMzZTBkZWRkMDEyNjBiZTBkMjVkNDM1OTEifX0sInNwZWMiOnsidm9sdW1lcyI6W3sibmFtZSI6InRtcCIsImVtcHR5RGlyIjp7fX0seyJuYW1lIjoiY29uZmlnIiwiY29uZmlnTWFwIjp7Im5hbWUiOiJtb25pdG9yaW5nLXB4YyIsIm9wdGlvbmFsIjp0cnVlfX0seyJuYW1lIjoic3NsLWludGVybmFsIiwic2VjcmV0Ijp7InNlY3JldE5hbWUiOiJtb25pdG9yaW5nLXNzbC1pbnRlcm5hbCIsIm9wdGlvbmFsIjp0cnVlfX0seyJuYW1lIjoic3NsIiwic2VjcmV0Ijp7InNlY3JldE5hbWUiOiJzb21lLW5hbWUtc3NsIiwib3B0aW9uYWwiOmZhbHNlfX0seyJuYW1lIjoiYXV0by1jb25maWciLCJjb25maWdNYXAiOnsibmFtZSI6ImF1dG8tbW9uaXRvcmluZy1weGMiLCJvcHRpb25hbCI6dHJ1ZX19LHsibmFtZSI6InZhdWx0LWtleXJpbmctc2VjcmV0Iiwic2VjcmV0Ijp7InNlY3JldE5hbWUiOiJtb25pdG9yaW5nLXZhdWx0Iiwib3B0aW9uYWwiOnRydWV9fSx7Im5hbWUiOiJteXNxbC11c2Vycy1zZWNyZXQtZmlsZSIsInNlY3JldCI6eyJzZWNyZXROYW1lIjoiaW50ZXJuYWwtbW9uaXRvcmluZyIsIm9wdGlvbmFsIjpmYWxzZX19LHsibmFtZSI6Im15c3FsLWluaXQtZmlsZSIsInNlY3JldCI6eyJzZWNyZXROYW1lIjoibW9uaXRvcmluZy1teXNxbC1pbml0Iiwib3B0aW9uYWwiOnRydWV9fSx7Im5hbWUiOiJiaW4iLCJlbXB0eURpciI6e319XSwiaW5pdENvbnRhaW5lcnMiOlt7Im5hbWUiOiJweGMtaW5pdCIsImltYWdlIjoicGVyY29uYWxhYi9wZXJjb25hLXh0cmFkYi1jbHVzdGVyLW9wZXJhdG9yOlBSLTI1NDYtY2FmMjU0MjAiLCJjb21tYW5kIjpbIi9weGMtaW5pdC1lbnRyeXBvaW50LnNoIl0sInJlc291cmNlcyI6eyJsaW1pdHMiOnsiY3B1IjoiNTBtIiwibWVtb3J5IjoiNTBNIn19LCJ2b2x1bWVNb3VudHMiOlt7Im5hbWUiOiJkYXRhZGlyIiwibW91bnRQYXRoIjoiL3Zhci9saWIvbXlzcWwifSx7Im5hbWUiOiJiaW4iLCJtb3VudFBhdGgiOiIvb3B0L3BlcmNvbmEifV0sImltYWdlUHVsbFBvbGljeSI6IkFsd2F5cyJ9XSwiY29udGFpbmVycyI6W3sibmFtZSI6InBtbS1jbGllbnQiLCJpbWFnZSI6InBlcmNvbmFsYWIvcG1tLWNsaWVudDpkZXYtbGF0ZXN0IiwicG9ydHMiOlt7ImNvbnRhaW5lclBvcnQiOjc3Nzd9LHsiY29udGFpbmVyUG9ydCI6MzAxMDB9LHsiY29udGFpbmVyUG9ydCI6MzAxMDF9LHsiY29udGFpbmVyUG9ydCI6MzAxMDJ9LHsiY29udGFpbmVyUG9ydCI6MzAxMDN9LHsiY29udGFpbmVyUG9ydCI6MzAxMDR9LHsiY29udGFpbmVyUG9ydCI6MzAxMDV9XSwiZW52RnJvbSI6W3sic2VjcmV0UmVmIjp7Im5hbWUiOiJteS1lbnYtdmFyLXNlY3JldHMiLCJvcHRpb25hbCI6dHJ1ZX19XSwiZW52IjpbeyJuYW1lIjoiUE1NX1NFUlZFUiIsInZhbHVlIjoibW9uaXRvcmluZy1zZXJ2aWNlIn0seyJuYW1lIjoiUE1NX1VTRVIiLCJ2YWx1ZSI6ImFkbWluIn0seyJuYW1lIjoiUE1NX1BBU1NXT1JEIiwidmFsdWVGcm9tIjp7InNlY3JldEtleVJlZiI6eyJuYW1lIjoiaW50ZXJuYWwtbW9uaXRvcmluZyIsImtleSI6InBtbXNlcnZlciJ9fX0seyJuYW1lIjoiQ0xJRU5UX1BPUlRfTElTVEVOIiwidmFsdWUiOiI3Nzc3In0seyJuYW1lIjoiQ0xJRU5UX1BPUlRfTUlOIiwidmFsdWUiOiIzMDEwMCJ9LHsibmFtZSI6IkNMSUVOVF9QT1JUX01BWCIsInZhbHVlIjoiMzAxMDUifSx7Im5hbWUiOiJQT0RfTkFNRSIsInZhbHVlRnJvbSI6eyJmaWVsZFJlZiI6eyJmaWVsZFBhdGgiOiJtZXRhZGF0YS5uYW1lIn19fSx7Im5hbWUiOiJQT0RfTkFNRVNQQVNFIiwidmFsdWVGcm9tIjp7ImZpZWxkUmVmIjp7ImZpZWxkUGF0aCI6Im1ldGFkYXRhLm5hbWVzcGFjZSJ9fX0seyJuYW1lIjoiUE1NX0FHRU5UX1NFUlZFUl9BRERSRVNTIiwidmFsdWUiOiJtb25pdG9yaW5nLXNlcnZpY2UifSx7Im5hbWUiOiJQTU1fQUdFTlRfU0VSVkVSX1VTRVJOQU1FIiwidmFsdWUiOiJhZG1pbiJ9LHsibmFtZSI6IlBNTV9BR0VOVF9TRVJWRVJfUEFTU1dPUkQiLCJ2YWx1ZUZyb20iOnsic2VjcmV0S2V5UmVmIjp7Im5hbWUiOiJpbnRlcm5hbC1tb25pdG9yaW5nIiwia2V5IjoicG1tc2VydmVyIn19fSx7Im5hbWUiOiJQTU1fQUdFTlRfTElTVEVOX1BPUlQiLCJ2YWx1ZSI6Ijc3NzcifSx7Im5hbWUiOiJQTU1fQUdFTlRfUE9SVFNfTUlOIiwidmFsdWUiOiIzMDEwMCJ9LHsibmFtZSI6IlBNTV9BR0VOVF9QT1JUU19NQVgiLCJ2YWx1ZSI6IjMwMTA1In0seyJuYW1lIjoiUE1NX0FHRU5UX0NPTkZJR19GSUxFIiwidmFsdWUiOiIvdXNyL2xv"..., -  { -  }, -  Annotations: map[string]string{ -  AvailableReplicas: 2, -  AvailableReplicas: 3, -  CollisionCount: &0, -  CreationTimestamp: v1.Time{Time: s"2026-10-05 13:21:36 +0000 UTC"}, -  CurrentReplicas: 2, -  CurrentReplicas: 3, -  CurrentRevision: "monitoring-haproxy-66f59c86f8", -  CurrentRevision: "monitoring-haproxy-74c5b4bd8", -  CurrentRevision: "monitoring-haproxy-7b88f98b45", -  CurrentRevision: "monitoring-haproxy-7cd8c9677b", -  CurrentRevision: "monitoring-pxc-5d45f885cc", -  CurrentRevision: "monitoring-pxc-7f464ffd85", -  CurrentRevision: "monitoring-pxc-85c5f769b6", -  CurrentRevision: "monitoring-pxc-cd6c44b47", -  Generation: 1, -  Generation: 2, -  Generation: 3, -  Generation: 4, -  ManagedFields: []v1.ManagedFieldsEntry{ -  ObservedGeneration: 1, -  ObservedGeneration: 2, -  ObservedGeneration: 3, -  ObservedGeneration: 4, -  PersistentVolumeClaimRetentionPolicy: s"&StatefulSetPersistentVolumeClaimRetentionPolicy{WhenDeleted:Retain,WhenScaled:Retain,}", -  PodManagementPolicy: "OrderedReady", -  ReadyReplicas: 2, -  ReadyReplicas: 3, -  Replicas: 2, -  Replicas: 3, -  Replicas: &2, -  Replicas: &3, -  ResourceVersion: "1791206571264591001", -  ResourceVersion: "1791206695546111006", -  ResourceVersion: "1791206993281119001", -  ResourceVersion: "1791207112222943006", -  ResourceVersion: "1791207188698367001", -  ResourceVersion: "1791207296633519006", -  ResourceVersion: "1791207352745711001", -  ResourceVersion: "1791207472015583006", -  RevisionHistoryLimit: &10, -  UID: "8937b13d-126e-4942-9ee3-002b61634ec9", -  UID: "8a391924-3056-4ebd-a6cc-5bdd2eac4a95", -  UpdateRevision: "monitoring-haproxy-66f59c86f8", -  UpdateRevision: "monitoring-haproxy-74c5b4bd8", -  UpdateRevision: "monitoring-haproxy-7b88f98b45", -  UpdateRevision: "monitoring-haproxy-7cd8c9677b", -  UpdateRevision: "monitoring-pxc-5d45f885cc", -  UpdateRevision: "monitoring-pxc-7f464ffd85", -  UpdateRevision: "monitoring-pxc-85c5f769b6", -  UpdateRevision: "monitoring-pxc-cd6c44b47", -  UpdatedReplicas: 2, -  UpdatedReplicas: 3, -  }, 2026-10-05T13:19:38.696Z INFO setup Feature gates {"PXCO_FEATURE_GATES": "", "enabled": ""} 2026-10-05T13:19:38.696Z INFO setup Manager starting up {"gitCommit": "caf25420ee3d0d26f55c22c9a7661405b27337d1", "gitBranch": "PR-2546-caf25420", "buildTime": "2026-10-05T11:36:48Z", "goVersion": "go1.26.8", "os": "linux", "arch": "amd64"} 2026-10-05T13:19:38.696Z INFO setup Runs on {"platform": "kubernetes", "version": "v1.34.9-gke.1655001"} 2026-10-05T13:19:38.699Z INFO setup Registering Components. 2026-10-05T13:19:39.622Z INFO controller-runtime.certwatcher Starting certificate poll+watcher {"cert": "/tmp/k8s-webhook-server/serving-certs/tls.crt", "key": "/tmp/k8s-webhook-server/serving-certs/tls.key", "interval": "10s"} 2026-10-05T13:19:39.622Z INFO controller-runtime.certwatcher Updated current TLS certificate {"cert": "/tmp/k8s-webhook-server/serving-certs/tls.crt", "key": "/tmp/k8s-webhook-server/serving-certs/tls.key"} 2026-10-05T13:19:39.622Z INFO controller-runtime.metrics Serving metrics server {"bindAddress": ":8080", "secure": false} 2026-10-05T13:19:39.622Z INFO controller-runtime.metrics Starting metrics server 2026-10-05T13:19:39.622Z INFO controller-runtime.webhook Registering webhook {"path": "/validate-percona-xtradbcluster"} 2026-10-05T13:19:39.622Z INFO controller-runtime.webhook Serving webhook server {"host": "", "port": 9443} 2026-10-05T13:19:39.622Z INFO controller-runtime.webhook Starting webhook server 2026-10-05T13:19:39.622Z INFO setup Starting the Cmd. 2026-10-05T13:19:39.622Z INFO starting server {"name": "health probe", "addr": "[::]:8081"} 2026-10-05T13:19:39.723Z INFO Attempting to acquire leader lease... {"lock": "pxc-operator/08db1feb.percona.com"} 2026-10-05T13:19:39.759Z DEBUG events percona-xtradb-cluster-operator-658fcf4b98-jb2nc_f4c12524-3439-4725-b500-2b742cd66cda became leader {"type": "Normal", "object": {"kind":"Lease","namespace":"pxc-operator","name":"08db1feb.percona.com","uid":"5fd93f62-b5af-402a-8611-7a8ea8fd140b","apiVersion":"coordination.k8s.io/v1","resourceVersion":"1791206379749759009"}, "reason": "LeaderElection"} 2026-10-05T13:19:39.759Z INFO Starting EventSource {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "source": "kind source: *v1.PerconaXtraDBCluster"} 2026-10-05T13:19:39.759Z INFO Starting EventSource {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "source": "kind source: *v1.Secret"} 2026-10-05T13:19:39.759Z INFO Successfully acquired lease {"lock": "pxc-operator/08db1feb.percona.com"} 2026-10-05T13:19:39.760Z INFO Starting EventSource {"controller": "pxcbackup-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBClusterBackup", "source": "kind source: *v1.PerconaXtraDBClusterBackup"} 2026-10-05T13:19:39.760Z INFO Starting EventSource {"controller": "pxcrestore-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBClusterRestore", "source": "kind source: *v1.PerconaXtraDBClusterRestore"} 2026-10-05T13:19:39.860Z INFO Starting Controller {"controller": "pxcbackup-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBClusterBackup"} 2026-10-05T13:19:39.860Z INFO Starting workers {"controller": "pxcbackup-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBClusterBackup", "worker count": 1} 2026-10-05T13:19:39.961Z INFO Starting Controller {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster"} 2026-10-05T13:19:39.961Z INFO Starting Controller {"controller": "pxcrestore-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBClusterRestore"} 2026-10-05T13:19:39.961Z INFO Starting workers {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "worker count": 1} 2026-10-05T13:19:39.961Z INFO Starting workers {"controller": "pxcrestore-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBClusterRestore", "worker count": 1} 2026-10-05T13:21:34.588Z INFO Set CR version {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"monitoring","namespace":"monitoring-2-0-5888"}, "namespace": "monitoring-2-0-5888", "name": "monitoring", "reconcileID": "2f626ed0-642e-4df3-8beb-97cb28465ed6", "version": "1.21.0"} 2026-10-05T13:21:36.632Z DEBUG Creating object {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"monitoring","namespace":"monitoring-2-0-5888"}, "namespace": "monitoring-2-0-5888", "name": "monitoring", "reconcileID": "2f626ed0-642e-4df3-8beb-97cb28465ed6", "object": "monitoring-pxc", "kind": "&TypeMeta{Kind:StatefulSet,APIVersion:apps/v1,}"} 2026-10-05T13:21:36.696Z DEBUG Creating object {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"monitoring","namespace":"monitoring-2-0-5888"}, "namespace": "monitoring-2-0-5888", "name": "monitoring", "reconcileID": "2f626ed0-642e-4df3-8beb-97cb28465ed6", "object": "monitoring-haproxy", "kind": "&TypeMeta{Kind:StatefulSet,APIVersion:apps/v1,}"} 2026-10-05T13:21:36.797Z DEBUG Creating object {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"monitoring","namespace":"monitoring-2-0-5888"}, "namespace": "monitoring-2-0-5888", "name": "monitoring", "reconcileID": "2f626ed0-642e-4df3-8beb-97cb28465ed6", "object": "monitoring-pxc", "kind": "&TypeMeta{Kind:Service,APIVersion:v1,}"} 2026-10-05T13:21:36.847Z DEBUG Creating object {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"monitoring","namespace":"monitoring-2-0-5888"}, "namespace": "monitoring-2-0-5888", "name": "monitoring", "reconcileID": "2f626ed0-642e-4df3-8beb-97cb28465ed6", "object": "monitoring-pxc-unready", "kind": "&TypeMeta{Kind:Service,APIVersion:v1,}"} 2026-10-05T13:21:36.900Z DEBUG Creating object {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"monitoring","namespace":"monitoring-2-0-5888"}, "namespace": "monitoring-2-0-5888", "name": "monitoring", "reconcileID": "2f626ed0-642e-4df3-8beb-97cb28465ed6", "object": "monitoring-haproxy", "kind": "&TypeMeta{Kind:Service,APIVersion:v1,}"} 2026-10-05T13:21:37.028Z DEBUG Creating object {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"monitoring","namespace":"monitoring-2-0-5888"}, "namespace": "monitoring-2-0-5888", "name": "monitoring", "reconcileID": "2f626ed0-642e-4df3-8beb-97cb28465ed6", "object": "monitoring-haproxy-replicas", "kind": "&TypeMeta{Kind:Service,APIVersion:v1,}"} 2026-10-05T13:21:38.453Z DEBUG Creating object {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"monitoring","namespace":"monitoring-2-0-5888"}, "namespace": "monitoring-2-0-5888", "name": "monitoring", "reconcileID": "715ff408-20af-49d1-8d3a-094d3c71e4e3", "object": "monitoring-pxc", "kind": "&TypeMeta{Kind:PodDisruptionBudget,APIVersion:policy/v1,}"} 2026-10-05T13:21:38.486Z DEBUG Creating object {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"monitoring","namespace":"monitoring-2-0-5888"}, "namespace": "monitoring-2-0-5888", "name": "monitoring", "reconcileID": "715ff408-20af-49d1-8d3a-094d3c71e4e3", "object": "monitoring-haproxy", "kind": "&TypeMeta{Kind:PodDisruptionBudget,APIVersion:policy/v1,}"} 2026-10-05T13:22:24.240Z INFO Password expiration policy updated {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"monitoring","namespace":"monitoring-2-0-5888"}, "namespace": "monitoring-2-0-5888", "name": "monitoring", "reconcileID": "3e305069-05b8-4b1d-b272-3ab200e2cc31", "user": "operator"} 2026-10-05T13:22:24.270Z INFO Password expiration policy updated {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"monitoring","namespace":"monitoring-2-0-5888"}, "namespace": "monitoring-2-0-5888", "name": "monitoring", "reconcileID": "3e305069-05b8-4b1d-b272-3ab200e2cc31", "user": "monitor"} 2026-10-05T13:22:24.316Z INFO User monitor: granted privileges {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"monitoring","namespace":"monitoring-2-0-5888"}, "namespace": "monitoring-2-0-5888", "name": "monitoring", "reconcileID": "3e305069-05b8-4b1d-b272-3ab200e2cc31"} 2026-10-05T13:22:24.349Z INFO monitor user privileges granted {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"monitoring","namespace":"monitoring-2-0-5888"}, "namespace": "monitoring-2-0-5888", "name": "monitoring", "reconcileID": "3e305069-05b8-4b1d-b272-3ab200e2cc31"} 2026-10-05T13:22:24.384Z INFO Password expiration policy updated {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"monitoring","namespace":"monitoring-2-0-5888"}, "namespace": "monitoring-2-0-5888", "name": "monitoring", "reconcileID": "3e305069-05b8-4b1d-b272-3ab200e2cc31", "user": "xtrabackup"} 2026-10-05T13:22:24.422Z INFO User xtrabackup: granted privileges {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"monitoring","namespace":"monitoring-2-0-5888"}, "namespace": "monitoring-2-0-5888", "name": "monitoring", "reconcileID": "3e305069-05b8-4b1d-b272-3ab200e2cc31"} 2026-10-05T13:22:24.452Z INFO Password expiration policy updated {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"monitoring","namespace":"monitoring-2-0-5888"}, "namespace": "monitoring-2-0-5888", "name": "monitoring", "reconcileID": "3e305069-05b8-4b1d-b272-3ab200e2cc31", "user": "replication"} 2026-10-05T13:22:26.458Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"monitoring","namespace":"monitoring-2-0-5888"}, "namespace": "monitoring-2-0-5888", "name": "monitoring", "reconcileID": "3e305069-05b8-4b1d-b272-3ab200e2cc31", "err": "get primary pxc pod: failed to get proxy connection: dial tcp 34.118.238.186:3306: connect: connection refused"} 2026-10-05T13:22:33.645Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"monitoring","namespace":"monitoring-2-0-5888"}, "namespace": "monitoring-2-0-5888", "name": "monitoring", "reconcileID": "53c53932-9fbd-4e81-9e5b-8ec910e0a785", "err": "get primary pxc pod: failed to get proxy connection: dial tcp 34.118.238.186:3306: connect: connection refused"} 2026-10-05T13:22:35.817Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"monitoring","namespace":"monitoring-2-0-5888"}, "namespace": "monitoring-2-0-5888", "name": "monitoring", "reconcileID": "10cf9f61-fc5a-4de2-b315-4c77636e79ff", "err": "get primary pxc pod: failed to get proxy connection: dial tcp 34.118.238.186:3306: connect: connection refused"} 2026-10-05T13:24:57.803Z INFO Password expiration policy updated {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"monitoring","namespace":"monitoring-2-0-5888"}, "namespace": "monitoring-2-0-5888", "name": "monitoring", "reconcileID": "32404ba1-ce19-4c6a-999e-43d3f3cef3bb", "user": "root"} 2026-10-05T13:24:57.863Z INFO update PXC version (fetched from db) {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"monitoring","namespace":"monitoring-2-0-5888"}, "namespace": "monitoring-2-0-5888", "name": "monitoring", "reconcileID": "32404ba1-ce19-4c6a-999e-43d3f3cef3bb", "new version": "8.0.43-34.1"} 2026-10-05T13:28:57.400Z INFO Internal secrets updated {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"monitoring","namespace":"monitoring-2-0-5888"}, "namespace": "monitoring-2-0-5888", "name": "monitoring", "reconcileID": "f16a646e-9f65-4953-85bf-c87dc233ddf3", "user": "pmmserverkey"} 2026-10-05T13:29:02.704Z DEBUG Updating object {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"monitoring","namespace":"monitoring-2-0-5888"}, "namespace": "monitoring-2-0-5888", "name": "monitoring", "reconcileID": "1cfba2dd-f6d3-4130-9ace-ef29c3faf5aa", "object": "monitoring-pxc", "kind": "&TypeMeta{Kind:StatefulSet,APIVersion:apps/v1,}", "hashChanged": true, "metaChanged": true} 2026-10-05T13:29:02.855Z DEBUG Updating object {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"monitoring","namespace":"monitoring-2-0-5888"}, "namespace": "monitoring-2-0-5888", "name": "monitoring", "reconcileID": "1cfba2dd-f6d3-4130-9ace-ef29c3faf5aa", "object": "monitoring-haproxy", "kind": "&TypeMeta{Kind:StatefulSet,APIVersion:apps/v1,}", "hashChanged": true, "metaChanged": true} 2026-10-05T13:30:10.207Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"monitoring","namespace":"monitoring-2-0-5888"}, "namespace": "monitoring-2-0-5888", "name": "monitoring", "reconcileID": "9a672c9c-e7f3-4278-92c6-3c30c131a75c", "err": "failed to ensure cluster readonly status: connect to pod monitoring-pxc-1: dial tcp 10.144.41.21:33062: connect: connection refused"} 2026-10-05T13:32:12.151Z DEBUG Updating object {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"monitoring","namespace":"monitoring-2-0-5888"}, "namespace": "monitoring-2-0-5888", "name": "monitoring", "reconcileID": "207afbd7-304f-4a03-b183-2dc04bbd4e65", "object": "monitoring-pxc", "kind": "&TypeMeta{Kind:StatefulSet,APIVersion:apps/v1,}", "hashChanged": true, "metaChanged": true} 2026-10-05T13:32:12.256Z DEBUG Updating object {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"monitoring","namespace":"monitoring-2-0-5888"}, "namespace": "monitoring-2-0-5888", "name": "monitoring", "reconcileID": "207afbd7-304f-4a03-b183-2dc04bbd4e65", "object": "monitoring-haproxy", "kind": "&TypeMeta{Kind:StatefulSet,APIVersion:apps/v1,}", "hashChanged": true, "metaChanged": true} 2026-10-05T13:32:12.505Z DEBUG Updating object {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"monitoring","namespace":"monitoring-2-0-5888"}, "namespace": "monitoring-2-0-5888", "name": "monitoring", "reconcileID": "207afbd7-304f-4a03-b183-2dc04bbd4e65", "object": "monitoring-haproxy", "kind": "&TypeMeta{Kind:StatefulSet,APIVersion:apps/v1,}", "hashChanged": true, "metaChanged": true} 2026-10-05T13:34:58.513Z INFO Password changed, updating user {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"monitoring","namespace":"monitoring-2-0-5888"}, "namespace": "monitoring-2-0-5888", "name": "monitoring", "reconcileID": "a31d235b-5ff7-4ae4-b517-7984c0abdb69", "user": "pmmserverkey"} 2026-10-05T13:34:58.542Z INFO HAProxy pods will be restarted {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"monitoring","namespace":"monitoring-2-0-5888"}, "namespace": "monitoring-2-0-5888", "name": "monitoring", "reconcileID": "a31d235b-5ff7-4ae4-b517-7984c0abdb69", "last-applied-secret": "424b8237346222ce589b2d9709d9497cd4ee835efb19befb4da0825f0a734865"} 2026-10-05T13:34:58.542Z INFO Internal secrets updated {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"monitoring","namespace":"monitoring-2-0-5888"}, "namespace": "monitoring-2-0-5888", "name": "monitoring", "reconcileID": "a31d235b-5ff7-4ae4-b517-7984c0abdb69", "user": "pmmserverkey"} 2026-10-05T13:34:58.542Z INFO PXC pods will be restarted {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"monitoring","namespace":"monitoring-2-0-5888"}, "namespace": "monitoring-2-0-5888", "name": "monitoring", "reconcileID": "a31d235b-5ff7-4ae4-b517-7984c0abdb69", "last-applied-secret": "424b8237346222ce589b2d9709d9497cd4ee835efb19befb4da0825f0a734865"} 2026-10-05T13:34:58.545Z DEBUG Updating object {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"monitoring","namespace":"monitoring-2-0-5888"}, "namespace": "monitoring-2-0-5888", "name": "monitoring", "reconcileID": "a31d235b-5ff7-4ae4-b517-7984c0abdb69", "object": "monitoring-pxc", "kind": "&TypeMeta{Kind:StatefulSet,APIVersion:apps/v1,}", "hashChanged": true, "metaChanged": true} 2026-10-05T13:34:58.634Z DEBUG Updating object {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"monitoring","namespace":"monitoring-2-0-5888"}, "namespace": "monitoring-2-0-5888", "name": "monitoring", "reconcileID": "a31d235b-5ff7-4ae4-b517-7984c0abdb69", "object": "monitoring-haproxy", "kind": "&TypeMeta{Kind:StatefulSet,APIVersion:apps/v1,}", "hashChanged": true, "metaChanged": true} 2026-10-05T13:41:40.912Z DEBUG Updating object {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"monitoring","namespace":"monitoring-2-0-5888"}, "namespace": "monitoring-2-0-5888", "name": "monitoring", "reconcileID": "d6a00860-e27c-4e1d-a44f-8d8c5d109234", "object": "monitoring-pxc", "kind": "&TypeMeta{Kind:StatefulSet,APIVersion:apps/v1,}", "hashChanged": true, "metaChanged": true} 2026-10-05T13:41:40.991Z DEBUG Updating object {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"monitoring","namespace":"monitoring-2-0-5888"}, "namespace": "monitoring-2-0-5888", "name": "monitoring", "reconcileID": "d6a00860-e27c-4e1d-a44f-8d8c5d109234", "object": "monitoring-haproxy", "kind": "&TypeMeta{Kind:StatefulSet,APIVersion:apps/v1,}", "hashChanged": true, "metaChanged": true} 2026-10-05T13:41:41.147Z DEBUG Updating object {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"monitoring","namespace":"monitoring-2-0-5888"}, "namespace": "monitoring-2-0-5888", "name": "monitoring", "reconcileID": "d6a00860-e27c-4e1d-a44f-8d8c5d109234", "object": "monitoring-haproxy", "kind": "&TypeMeta{Kind:StatefulSet,APIVersion:apps/v1,}", "hashChanged": true, "metaChanged": true} [mysql] 2026/10/05 13:34:36 packets.go:58 read tcp 10.144.40.32:36068->34.118.238.186:3306: i/o timeout [mysql] 2026/10/05 13:37:17 packets.go:58 unexpected EOF    FieldPath: "metadata.name",    FieldPath: "metadata.namespace",    LocalObjectReference: {Name: "internal-monitoring"},    Optional: nil,    ... // 2 identical fields    ConfigMapKeyRef: nil,    FieldRef: nil,    FieldRef: &v1.ObjectFieldSelector{    FileKeyRef: nil,    HTTPHeaders: nil,    Host: "",    Path: "/local/Status",    Port: {IntVal: 7777},    Protocol: nil,    ResourceFieldRef: nil,    SecretKeyRef: &v1.SecretKeySelector{    },    ContainerPort: 30100,    ContainerPort: 30101,    ContainerPort: 30102,    ContainerPort: 30103,    ContainerPort: 30104,    ContainerPort: 30105,    ContainerPort: 3306,    ContainerPort: 33060,    ContainerPort: 33062,    ContainerPort: 3307,    ContainerPort: 3309,    ContainerPort: 4444,    ContainerPort: 4567,    ContainerPort: 4568,    ContainerPort: 7777,    ContainerPort: 8404,    DefaultUser: nil,    DefaultUser: nil,    Exec: nil,    GRPC: nil,    HTTPGet: &v1.HTTPGetAction{    HostIP: "",    HostPort: 0,    Items: nil,    Items: nil,    LocalObjectReference: {Name: "auto-monitoring-pxc"},    LocalObjectReference: {Name: "monitoring-haproxy"},    LocalObjectReference: {Name: "monitoring-pxc"},    Name: "",    Name: "ist",    Name: "mysql",    Name: "mysql-admin",    Name: "mysql-replicas",    Name: "mysqlx",    Name: "proxy-protocol",    Name: "sst",    Name: "stats",    Name: "write-set",    Name: "PMM_AGENT_SERVER_USERNAME",    Name: "PMM_AGENT_SETUP_NODE_NAME",    Name: "PMM_USER",    Name: "PMM_AGENT_SERVER_PASSWORD",    Name: "PMM_PASSWORD",    Name: "POD_NAME",    Name: "POD_NAMESPASE",    Optional: &true,    Optional: &false,    Optional: &true,    SecretName: "internal-monitoring",    SecretName: "monitoring-mysql-init",    SecretName: "monitoring-ssl-internal",    SecretName: "monitoring-vault",    SecretName: "my-env-var-secrets",    SecretName: "some-name-ssl",    TCPSocket: nil,    Value: "",    ValueFrom: &v1.EnvVarSource{    ValueFrom: nil,    },    ... // 11 identical elements    ... // 12 identical elements    ... // 16 identical fields    ... // 22 identical elements    ... // 22 identical fields    ... // 23 identical elements    ... // 25 identical elements    ... // 26 identical elements    ... // 3 identical fields    ... // 4 identical elements    ... // 4 identical fields    ... // 9 identical elements    ... // 9 identical fields    AWSElasticBlockStore: nil,    AzureFile: nil,    ConfigMap: &v1.ConfigMapVolumeSource{    FC: nil,    FailureThreshold: 3,    GitRepo: nil,    ISCSI: nil,    InitialDelaySeconds: 300,    InitialDelaySeconds: 15,    InitialDelaySeconds: 300,    NFS: nil,    ProbeHandler: {Exec: &{Command: {"/var/lib/mysql/liveness-check.sh"}}},    ProbeHandler: v1.ProbeHandler{    Quobyte: nil,    Secret: &v1.SecretVolumeSource{    SuccessThreshold: 1,    TerminationGracePeriodSeconds: nil,    TimeoutSeconds: 5,    TimeoutSeconds: 15,    TimeoutSeconds: 5,    VsphereVolume: nil,    {    {Name: "CLIENT_PORT_LISTEN", Value: "7777"},    {Name: "CLIENT_PORT_MAX", Value: "30105"},    {Name: "CLIENT_PORT_MIN", Value: "30100"},    {Name: "DB_TYPE", Value: "haproxy"},    {Name: "DB_TYPE", Value: "mysql"},    {Name: "DB_USER", Value: "monitor"},    {Name: "MONITOR_USER", Value: "monitor"},    {Name: "PMM_AGENT_LISTEN_PORT", Value: "7777"},    {Name: "PMM_AGENT_PORTS_MIN", Value: "30100"},    {Name: "PMM_AGENT_SERVER_ADDRESS", Value: "monitoring-service"},    {Name: "PMM_AGENT_SERVER_USERNAME", Value: "api_key"},    {Name: "PMM_AGENT_SETUP_FORCE", Value: "1"},    {Name: "PMM_AGENT_SETUP_NODE_TYPE", Value: "container"},    {Name: "PMM_SERVER", Value: "monitoring-service"},    },    ... // 16 identical fields    ... // 2 identical fields    ... // 3 identical fields    ... // 4 identical fields    ... // 8 identical fields    Args: nil,    Args: {"haproxy"},    Args: {"mysqld"},    Env: {{Name: "PXC_SERVICE", Value: "monitoring-pxc"}, {Name: "LIVENESS_CHECK_TIMEOUT", Value: "5"}, {Name: "READINESS_CHECK_TIMEOUT", Value: "1"}},    Env: {{Name: "PXC_SERVICE", Value: "monitoring-pxc-unready"}, {Name: "MONITOR_HOST", Value: "%"}, {Name: "MYSQL_ROOT_PASSWORD", ValueFrom: &{SecretKeyRef: &{LocalObjectReference: {Name: "internal-monitoring"}, Key: "root"}}}, {Name: "XTRABACKUP_PASSWORD", ValueFrom: &{SecretKeyRef: &{LocalObjectReference: {Name: "internal-monitoring"}, Key: "xtrabackup"}}}, ...},    Env: []v1.EnvVar{    EnvFrom: {{SecretRef: &{LocalObjectReference: {Name: "my-env-var-secrets"}, Optional: &true}}},    ImagePullPolicy: "Always",    Lifecycle: &{PreStop: &{Exec: &{Command: {"bash", "-c", "pmm-admin unregister --force"}}}},    Lifecycle: nil,    LivenessProbe: &v1.Probe{    Name: "auto-config",    Name: "config",    Name: "haproxy-custom",    Name: "my-env-var-secrets",    Name: "mysql-init-file",    Name: "mysql-users-secret-file",    Name: "ssl",    Name: "ssl-internal",    Name: "vault-keyring-secret",    Ports: []v1.ContainerPort{    ReadinessProbe: &{ProbeHandler: {Exec: &{Command: {"/var/lib/mysql/readiness-check.sh"}}}, InitialDelaySeconds: 15, TimeoutSeconds: 15, PeriodSeconds: 30, ...},    ReadinessProbe: &v1.Probe{    ResizePolicy: nil,    Resources: {Limits: {s"cpu": {i: {...}, Format: "DecimalSI"}, s"memory": {i: {...}, Format: "DecimalSI"}}, Requests: {s"cpu": {i: {...}, s: "308m", Format: "DecimalSI"}, s"memory": {i: {...}, s: "508M", Format: "DecimalSI"}}},    SecurityContext: nil,    StartupProbe: nil,    VolumeDevices: nil,    VolumeMounts: {{Name: "bin", MountPath: "/var/lib/mysql"}},    VolumeMounts: {{Name: "datadir", MountPath: "/var/lib/mysql"}, {Name: "config", MountPath: "/etc/percona-xtradb-cluster.conf.d"}, {Name: "tmp", MountPath: "/tmp"}, {Name: "ssl", MountPath: "/etc/mysql/ssl"}, ...},    VolumeMounts: {{Name: "datadir", MountPath: "/var/lib/mysql"}},    VolumeSource: v1.VolumeSource{    WorkingDir: "",    },    "kubectl.kubernetes.io/default-container": "haproxy",    "kubectl.kubernetes.io/default-container": "pxc",    "percona.com/configuration-hash": "d41d8cd98f00b204e9800998ecf8427e",    "percona.com/env-secret-config-hash": "5bc403a8d44324c1bc704f02e74876d0",    "percona.com/ssl-hash": "459d5d5e4d106ab108131bfb45889de1",    "percona.com/ssl-internal-hash": "951703033e0dedd01260be0d25d43591",    ... // 2 identical entries    ... // 3 identical fields    ... // 6 identical fields    AccessModes: nil,    Capacity: nil,    DataSource: nil,    DataSourceRef: nil,    StorageClassName: nil,    VolumeAttributesClassName: nil,    VolumeName: "",    {    {Name: "bin", VolumeSource: {EmptyDir: &{}}},    {Name: "haproxy-auto", VolumeSource: {EmptyDir: &{}}},    {Name: "tmp", VolumeSource: {EmptyDir: &{}}},    },    ... // 2 identical fields    ... // 6 identical fields    ... // 7 identical fields    ... // 9 identical fields    ActiveDeadlineSeconds: nil,    Affinity: nil,    Affinity: nil,    Annotations: map[string]string{    AutomountServiceAccountToken: nil,    Containers: []v1.Container{    DeletionGracePeriodSeconds: nil,    EphemeralContainers: nil,    Finalizers: nil,    HostAliases: nil,    HostAliases: nil,    HostIPC: false,    Hostname: "",    ImagePullSecrets: nil,    InitContainers: []v1.Container{    Labels: {"app.kubernetes.io/component": "haproxy", "app.kubernetes.io/instance": "monitoring", "app.kubernetes.io/managed-by": "percona-xtradb-cluster-operator", "app.kubernetes.io/name": "percona-xtradb-cluster", ...},    Labels: {"app.kubernetes.io/component": "pxc", "app.kubernetes.io/instance": "monitoring", "app.kubernetes.io/managed-by": "percona-xtradb-cluster-operator", "app.kubernetes.io/name": "percona-xtradb-cluster", ...},    ManagedFields: nil,    NodeName: "",    NodeSelector: nil,    OS: nil,    ObjectMeta: {Name: "datadir", Labels: {"app.kubernetes.io/component": "pxc", "app.kubernetes.io/instance": "monitoring", "app.kubernetes.io/managed-by": "percona-xtradb-cluster-operator", "app.kubernetes.io/name": "percona-xtradb-cluster", ...}},    Overhead: nil,    OwnerReferences: nil,    PreemptionPolicy: nil,    ServiceAccountName: "default",    SetHostnameAsFQDN: nil,    ShareProcessNamespace: nil,    Spec: v1.PersistentVolumeClaimSpec{    Status: v1.PersistentVolumeClaimStatus{    Subdomain: "",    Subdomain: "",    TerminationGracePeriodSeconds: &30,    TerminationGracePeriodSeconds: &600,    Tolerations: nil,    Tolerations: nil,    TypeMeta: {},    Volumes: []v1.Volume{    },    ObjectMeta: v1.ObjectMeta{    ObjectMeta: {Labels: {"app.kubernetes.io/component": "haproxy", "app.kubernetes.io/instance": "monitoring", "app.kubernetes.io/managed-by": "percona-xtradb-cluster-operator", "app.kubernetes.io/name": "percona-xtradb-cluster", ...}, Annotations: {"kubectl.kubernetes.io/default-container": "haproxy", "last-applied-secret": "424b8237346222ce589b2d9709d9497cd4ee835efb19befb4da0825f0a734865", "percona.com/configuration-hash": "d41d8cd98f00b204e9800998ecf8427e", "percona.com/env-secret-config-hash": "5bc403a8d44324c1bc704f02e74876d0"}},    ObjectMeta: {Labels: {"app.kubernetes.io/component": "haproxy", "app.kubernetes.io/instance": "monitoring", "app.kubernetes.io/managed-by": "percona-xtradb-cluster-operator", "app.kubernetes.io/name": "percona-xtradb-cluster", ...}, Annotations: {"kubectl.kubernetes.io/default-container": "haproxy", "percona.com/configuration-hash": "d41d8cd98f00b204e9800998ecf8427e"}},    ObjectMeta: {Labels: {"app.kubernetes.io/component": "pxc", "app.kubernetes.io/instance": "monitoring", "app.kubernetes.io/managed-by": "percona-xtradb-cluster-operator", "app.kubernetes.io/name": "percona-xtradb-cluster", ...}, Annotations: {"kubectl.kubernetes.io/default-container": "pxc", "last-applied-secret": "424b8237346222ce589b2d9709d9497cd4ee835efb19befb4da0825f0a734865", "percona.com/configuration-hash": "d41d8cd98f00b204e9800998ecf8427e", "percona.com/env-secret-config-hash": "5bc403a8d44324c1bc704f02e74876d0", ...}},    ObjectMeta: {Labels: {"app.kubernetes.io/component": "pxc", "app.kubernetes.io/instance": "monitoring", "app.kubernetes.io/managed-by": "percona-xtradb-cluster-operator", "app.kubernetes.io/name": "percona-xtradb-cluster", ...}, Annotations: {"kubectl.kubernetes.io/default-container": "pxc", "percona.com/configuration-hash": "d41d8cd98f00b204e9800998ecf8427e", "percona.com/ssl-hash": "459d5d5e4d106ab108131bfb45889de1", "percona.com/ssl-internal-hash": "951703033e0dedd01260be0d25d43591"}},    Spec: v1.PodSpec{    {    },    ... // 2 identical fields    Conditions: nil,    DeletionGracePeriodSeconds: nil,    DeletionTimestamp: nil,    Finalizers: nil,    Labels: nil,    MinReadySeconds: 0,    Namespace: "monitoring-2-0-5888",    Ordinals: nil,    OwnerReferences: {{APIVersion: "pxc.percona.com/v1", Kind: "PerconaXtraDBCluster", Name: "monitoring", UID: "1cc4a470-5a18-4902-9a48-0591fbb99217", ...}},    Replicas: &2,    Replicas: &3,    Selector: &{MatchLabels: {"app.kubernetes.io/component": "haproxy", "app.kubernetes.io/instance": "monitoring", "app.kubernetes.io/managed-by": "percona-xtradb-cluster-operator", "app.kubernetes.io/name": "percona-xtradb-cluster", ...}},    Selector: &{MatchLabels: {"app.kubernetes.io/component": "pxc", "app.kubernetes.io/instance": "monitoring", "app.kubernetes.io/managed-by": "percona-xtradb-cluster-operator", "app.kubernetes.io/name": "percona-xtradb-cluster", ...}},    SelfLink: "",    ServiceName: "monitoring-haproxy",    ServiceName: "monitoring-pxc",    Template: v1.PodTemplateSpec{    UpdateStrategy: {Type: "RollingUpdate", RollingUpdate: &{Partition: &0}},    VolumeClaimTemplates: nil,    VolumeClaimTemplates: []v1.PersistentVolumeClaim{    },    ObjectMeta: v1.ObjectMeta{    Spec: v1.StatefulSetSpec{    Status: v1.StatefulSetStatus{    TypeMeta: {Kind: "StatefulSet", APIVersion: "apps/v1"},    },   &v1.StatefulSet{   } + local backup_names ++ kubectl_bin get pxc-backup --all-namespaces -o 'jsonpath={range .items[*]}{.metadata.namespace}/{.metadata.name}{"\n"}{end}' + backup_names= + [[ -n '' ]] + local pxc_names ++ kubectl_bin get pxc --all-namespaces -o 'jsonpath={range .items[*]}{.metadata.namespace}/{.metadata.name}{"\n"}{end}' + pxc_names=monitoring-2-0-5888/monitoring + [[ -n monitoring-2-0-5888/monitoring ]] + kubectl_bin get pxc --all-namespaces -o 'jsonpath={range .items[*]}{.metadata.namespace} {.metadata.name}{"\n"}{end}' + read -r pns name + kubectl_bin patch pxc -n monitoring-2-0-5888 monitoring --type=merge -p '{"metadata":{"finalizers":["percona.com/delete-pxc-pvc"]}}' perconaxtradbcluster.pxc.percona.com/monitoring patched + read -r pns name + timeout 300 kubectl delete pxc --all --all-namespaces --ignore-not-found perconaxtradbcluster.pxc.percona.com "monitoring" deleted from monitoring-2-0-5888 namespace ++ date +%s + local deadline=1791208077 ++ kubectl_bin get pxc --all-namespaces -o name --ignore-not-found + [[ -z '' ]] + timeout 300 kubectl delete pxc-restore --all --all-namespaces --wait=false --ignore-not-found No resources found + kubectl_bin delete ValidatingWebhookConfiguration percona-xtradbcluster-webhook --ignore-not-found ++ mktemp + local LAST_OUT=/tmp/tmp.z1yJytSS4r ++ mktemp + local LAST_ERR=/tmp/tmp.mOtPdrpaM1 + local exit_status=0 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl delete ValidatingWebhookConfiguration percona-xtradbcluster-webhook --ignore-not-found + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.z1yJytSS4r validatingwebhookconfiguration.admissionregistration.k8s.io "percona-xtradbcluster-webhook" deleted + cat /tmp/tmp.mOtPdrpaM1 + rm /tmp/tmp.z1yJytSS4r /tmp/tmp.mOtPdrpaM1 + return 0 + kubectl_bin delete -f https://github.com/jetstack/cert-manager/releases/download/v1.20.2/cert-manager.yaml + : + '[' '!' -z '' ']' + '[' -n pxc-operator ']' + kubectl_bin delete --grace-period=0 --force=true namespace monitoring-2-0-5888 ++ mktemp + kubectl_bin delete --grace-period=0 --force=true namespace pxc-operator ++ mktemp ++ date +%s + local LAST_OUT=/tmp/tmp.fKrN01IV1u ++ mktemp + local LAST_OUT=/tmp/tmp.e1WYVvHuWx + local deadline=1791207926 ++ mktemp ++ kubectl_bin get namespace monitoring-2-0-5888 --ignore-not-found -o name + local LAST_ERR=/tmp/tmp.Ec34rIRg9W + local exit_status=0 ++ seq 0 2 + local LAST_ERR=/tmp/tmp.05u1LtWkWG + local exit_status=0 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl delete --grace-period=0 --force=true namespace monitoring-2-0-5888 + for i in $(seq 0 2) + set +e + kubectl delete --grace-period=0 --force=true namespace pxc-operator + [[ -z namespace/monitoring-2-0-5888 ]] ++ date +%s + [[ 1791207808 -ge 1791207926 ]] + sleep 5 ++ kubectl_bin get namespace monitoring-2-0-5888 --ignore-not-found -o name + [[ -z namespace/monitoring-2-0-5888 ]] ++ date +%s + [[ 1791207815 -ge 1791207926 ]] + sleep 5 ++ kubectl_bin get namespace monitoring-2-0-5888 --ignore-not-found -o name + [[ -z namespace/monitoring-2-0-5888 ]] ++ date +%s + [[ 1791207821 -ge 1791207926 ]] + sleep 5 + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.e1WYVvHuWx namespace "pxc-operator" force deleted + cat /tmp/tmp.05u1LtWkWG Warning: Immediate deletion does not wait for confirmation that the running resource has been terminated. The resource may continue to run on the cluster indefinitely. + rm /tmp/tmp.e1WYVvHuWx /tmp/tmp.05u1LtWkWG + return 0 ++ kubectl_bin get namespace monitoring-2-0-5888 --ignore-not-found -o name + [[ -z namespace/monitoring-2-0-5888 ]] ++ date +%s + [[ 1791207827 -ge 1791207926 ]] + sleep 5 ++ kubectl_bin get namespace monitoring-2-0-5888 --ignore-not-found -o name + [[ -z namespace/monitoring-2-0-5888 ]] ++ date +%s + [[ 1791207834 -ge 1791207926 ]] + sleep 5 ++ kubectl_bin get namespace monitoring-2-0-5888 --ignore-not-found -o name + [[ -z namespace/monitoring-2-0-5888 ]] ++ date +%s + [[ 1791207841 -ge 1791207926 ]] + sleep 5 ++ kubectl_bin get namespace monitoring-2-0-5888 --ignore-not-found -o name + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.fKrN01IV1u namespace "monitoring-2-0-5888" force deleted + cat /tmp/tmp.Ec34rIRg9W Warning: Immediate deletion does not wait for confirmation that the running resource has been terminated. The resource may continue to run on the cluster indefinitely. + rm /tmp/tmp.fKrN01IV1u /tmp/tmp.Ec34rIRg9W + return 0 + [[ -z '' ]] ++ date +%s + deadline=1791207968 ++ awk '$2 != ""' ++ kubectl_bin get pv -o 'jsonpath={range .items[*]}{.metadata.name}{"\t"}{.metadata.deletionTimestamp}{"\n"}{end}' + [[ -z '' ]] + rm -rf /tmp/tmp.oE2g1q4ApT + desc 'test passed' + set +o xtrace ----------------------------------------------------------------------------------- test passed -----------------------------------------------------------------------------------