============================= test session starts ============================== platform linux -- Python 3.14.5, pytest-9.1.1, pluggy-1.6.0 -- /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/.venv/bin/python cachedir: .pytest_cache metadata: {'Python': '3.14.5', 'Platform': 'Linux-7.0.8-100.fc43.x86_64-x86_64-with-glibc2.42', 'Packages': {'pytest': '9.1.1', 'pluggy': '1.6.0'}, 'Plugins': {'html': '4.2.0', 'metadata': '3.1.1', 'dependency': '0.6.1', 'json-report': '1.5.0'}, 'CI': 'true', 'BUILD_NUMBER': '22', 'BUILD_ID': '22', 'BUILD_URL': 'https://cloud.cd.percona.com/job/cloud-psmdb-operator/job/PR-2058/22/', 'NODE_NAME': 'hcloud-3hmxjv56rzynxord', 'JOB_NAME': 'cloud-psmdb-operator/PR-2058', 'BUILD_TAG': 'jenkins-cloud-psmdb-operator-PR-2058-22', 'EXECUTOR_NUMBER': '0', 'JENKINS_URL': 'https://cloud.cd.percona.com/', 'WORKSPACE': '/mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058', 'GIT_COMMIT': '78121c56762758785b75e9b7c017e15fc887ae5d', 'GIT_URL': 'https://github.com/percona/percona-server-mongodb-operator.git', 'GIT_BRANCH': 'PR-2058'} rootdir: /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058 configfile: pyproject.toml plugins: html-4.2.0, metadata-3.1.1, dependency-0.6.1, json-report-1.5.0 collecting ... collected 1 item e2e-tests/test_pytest_wrapper.py::test_bash_wrapper[monitoring-pmm3] [01:09:37.548977] WARNING kubectl error: Warning: version difference between client (1.36) and server (1.33) exceeds the supported minor version skew of +/-1 [01:09:39.993142] INFO Environment variables: KUBE_VERSION=1.33 EKS=0 GKE=1 OPENSHIFT= MINIKUBE= API=psmdb.percona.com/v1 REGISTRY_NAME=docker.io GIT_COMMIT=78121c56762758785b75e9b7c017e15fc887ae5d GIT_BRANCH=PR-2058 OPERATOR_VERSION=1.23.0 IMAGE=docker.io/perconalab/percona-server-mongodb-ope rator:PR-2058-78121c567 IMAGE_MONGOD=docker.io/perconalab/percona-server-mong odb-operator:main-mongod8.0 IMAGE_MONGOD_CHAIN=docker.io/perconalab/percona-serve r-mongodb-operator:main-mongod6.0 docker.io/perconalab/percona-server-mongodb-operator: main-mongod7.0 docker.io/perconalab/percona-server-mongodb-operator: main-mongod8.0 IMAGE_SEARCH=perconalab/percona-server-mongodb-operat or:main-mongot IMAGE_BACKUP=docker.io/perconalab/percona-server-mong odb-operator:main-backup IMAGE_PMM_CLIENT=docker.io/percona/pmm-client:2.44.1- 1 IMAGE_PMM_SERVER=docker.io/perconalab/pmm-server:dev- latest IMAGE_PMM3_CLIENT=docker.io/perconalab/pmm-client:3-d ev-latest IMAGE_PMM3_SERVER=docker.io/perconalab/pmm-server:3-d ev-latest IMAGE_LOGCOLLECTOR=docker.io/perconalab/fluentbit:mai n-logcollector IMAGE_CLUSTERSYNC=docker.io/percona/percona-clustersy nc-mongodb:0.9.0 CERT_MANAGER_VER=1.19.3 CHAOS_MESH_VER=2.7.1 MINIO_VER=5.4.0 PMM_SERVER_VER=9.9.9 CLEAN_NAMESPACE=1 DELETE_CRD_ON_START=1 SKIP_DELETE=0 SKIP_BACKUPS_TO_AWS_GCP_AZURE= UPDATE_COMPARE_FILES=0 Warning: version difference between client (1.36) and server (1.33) exceeds the supported minor version skew of +/-1 Debug logging disabled. + create_infra monitoring-pmm3-14502 + local ns=monitoring-pmm3-14502 + echo monitoring-pmm3-14502 + [[ 1 == 1 ]] + delete_crd + desc 'get and delete old CRDs and RBAC' + set +o xtrace ----------------------------------------------------------------------------------- get and delete old CRDs and RBAC ----------------------------------------------------------------------------------- + kubectl_bin delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/deploy/crd.yaml --ignore-not-found --wait=false ++ mktemp + local LAST_OUT=/tmp/tmp.H3hEkX9JW7 ++ mktemp + local LAST_ERR=/tmp/tmp.uBKc02E8M3 + local exit_status=0 + local timeout=4 + args=('delete' '-f' '/mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/deploy/crd.yaml' '--ignore-not-found' '--wait=false') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/deploy/crd.yaml --ignore-not-found --wait=false + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.H3hEkX9JW7 customresourcedefinition.apiextensions.k8s.io "perconaservermongodbbackups.psmdb.percona.com" deleted customresourcedefinition.apiextensions.k8s.io "perconaservermongodbclustersyncs.psmdb.percona.com" deleted customresourcedefinition.apiextensions.k8s.io "perconaservermongodbrestores.psmdb.percona.com" deleted customresourcedefinition.apiextensions.k8s.io "perconaservermongodbs.psmdb.percona.com" deleted + cat /tmp/tmp.uBKc02E8M3 + rm /tmp/tmp.H3hEkX9JW7 /tmp/tmp.uBKc02E8M3 + return 0 ++ yq eval .metadata.name /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/deploy/crd.yaml ++ grep -v '\-\-\-' grep: warning: stray \ before - grep: warning: stray \ before - + for crd_name in $(yq eval '.metadata.name' "${src_dir}/deploy/crd.yaml" | grep -v '\-\-\-') + kubectl get perconaservermongodbbackups.psmdb.percona.com --all-namespaces -o wide + grep -v NAMESPACE + xargs -L 1 sh -xc 'kubectl patch perconaservermongodbbackups.psmdb.percona.com -n $0 $1 --type=merge -p "{\"metadata\":{\"finalizers\":[]}}"' No resources found + kubectl patch perconaservermongodbbackups.psmdb.percona.com -n sh --type=merge -p '{"metadata":{"finalizers":[]}}' error: resource(s) were provided, but no name was specified + : + kubectl_bin wait --for=delete crd perconaservermongodbbackups.psmdb.percona.com ++ mktemp + local LAST_OUT=/tmp/tmp.Bhlw8OxKlG ++ mktemp + local LAST_ERR=/tmp/tmp.IzPXdoY1WG + local exit_status=0 + local timeout=4 + args=('wait' '--for=delete' 'crd' 'perconaservermongodbbackups.psmdb.percona.com') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl wait --for=delete crd perconaservermongodbbackups.psmdb.percona.com + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.Bhlw8OxKlG + cat /tmp/tmp.IzPXdoY1WG + rm /tmp/tmp.Bhlw8OxKlG /tmp/tmp.IzPXdoY1WG + return 0 + for crd_name in $(yq eval '.metadata.name' "${src_dir}/deploy/crd.yaml" | grep -v '\-\-\-') + kubectl get perconaservermongodbclustersyncs.psmdb.percona.com --all-namespaces -o wide + grep -v NAMESPACE + xargs -L 1 sh -xc 'kubectl patch perconaservermongodbclustersyncs.psmdb.percona.com -n $0 $1 --type=merge -p "{\"metadata\":{\"finalizers\":[]}}"' error: the server doesn't have a resource type "perconaservermongodbclustersyncs" + kubectl patch perconaservermongodbclustersyncs.psmdb.percona.com -n sh --type=merge -p '{"metadata":{"finalizers":[]}}' error: the server doesn't have a resource type "perconaservermongodbclustersyncs" + : + kubectl_bin wait --for=delete crd perconaservermongodbclustersyncs.psmdb.percona.com ++ mktemp + local LAST_OUT=/tmp/tmp.p332GrTSvW ++ mktemp + local LAST_ERR=/tmp/tmp.KmL32C7jMj + local exit_status=0 + local timeout=4 + args=('wait' '--for=delete' 'crd' 'perconaservermongodbclustersyncs.psmdb.percona.com') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl wait --for=delete crd perconaservermongodbclustersyncs.psmdb.percona.com + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.p332GrTSvW + cat /tmp/tmp.KmL32C7jMj + rm /tmp/tmp.p332GrTSvW /tmp/tmp.KmL32C7jMj + return 0 + for crd_name in $(yq eval '.metadata.name' "${src_dir}/deploy/crd.yaml" | grep -v '\-\-\-') + kubectl get perconaservermongodbrestores.psmdb.percona.com --all-namespaces -o wide + grep -v NAMESPACE + xargs -L 1 sh -xc 'kubectl patch perconaservermongodbrestores.psmdb.percona.com -n $0 $1 --type=merge -p "{\"metadata\":{\"finalizers\":[]}}"' error: the server doesn't have a resource type "perconaservermongodbrestores" + kubectl patch perconaservermongodbrestores.psmdb.percona.com -n sh --type=merge -p '{"metadata":{"finalizers":[]}}' error: the server doesn't have a resource type "perconaservermongodbrestores" + : + kubectl_bin wait --for=delete crd perconaservermongodbrestores.psmdb.percona.com ++ mktemp + local LAST_OUT=/tmp/tmp.sqoKKneIxN ++ mktemp + local LAST_ERR=/tmp/tmp.WwFw3grXk8 + local exit_status=0 + local timeout=4 + args=('wait' '--for=delete' 'crd' 'perconaservermongodbrestores.psmdb.percona.com') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl wait --for=delete crd perconaservermongodbrestores.psmdb.percona.com + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.sqoKKneIxN + cat /tmp/tmp.WwFw3grXk8 + rm /tmp/tmp.sqoKKneIxN /tmp/tmp.WwFw3grXk8 + return 0 + for crd_name in $(yq eval '.metadata.name' "${src_dir}/deploy/crd.yaml" | grep -v '\-\-\-') + kubectl get perconaservermongodbs.psmdb.percona.com --all-namespaces -o wide + grep -v NAMESPACE + xargs -L 1 sh -xc 'kubectl patch perconaservermongodbs.psmdb.percona.com -n $0 $1 --type=merge -p "{\"metadata\":{\"finalizers\":[]}}"' error: the server doesn't have a resource type "perconaservermongodbs" + kubectl patch perconaservermongodbs.psmdb.percona.com -n sh --type=merge -p '{"metadata":{"finalizers":[]}}' error: the server doesn't have a resource type "perconaservermongodbs" + : + kubectl_bin wait --for=delete crd perconaservermongodbs.psmdb.percona.com ++ mktemp + local LAST_OUT=/tmp/tmp.JzMnO6plZM ++ mktemp + local LAST_ERR=/tmp/tmp.n0drdtQ6E3 + local exit_status=0 + local timeout=4 + args=('wait' '--for=delete' 'crd' 'perconaservermongodbs.psmdb.percona.com') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl wait --for=delete crd perconaservermongodbs.psmdb.percona.com + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.JzMnO6plZM + cat /tmp/tmp.n0drdtQ6E3 + rm /tmp/tmp.JzMnO6plZM /tmp/tmp.n0drdtQ6E3 + return 0 + local rbac_yaml=rbac.yaml + '[' -n psmdb-operator ']' + rbac_yaml=cw-rbac.yaml + kubectl_bin delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/deploy/cw-rbac.yaml --ignore-not-found ++ mktemp + local LAST_OUT=/tmp/tmp.3UFQHZOEtX ++ mktemp + local LAST_ERR=/tmp/tmp.vPqyMQ3qok + local exit_status=0 + local timeout=4 + args=('delete' '-f' '/mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/deploy/cw-rbac.yaml' '--ignore-not-found') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/deploy/cw-rbac.yaml --ignore-not-found + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.3UFQHZOEtX clusterrole.rbac.authorization.k8s.io "percona-server-mongodb-operator" deleted clusterrolebinding.rbac.authorization.k8s.io "service-account-percona-server-mongodb-operator" deleted + cat /tmp/tmp.vPqyMQ3qok + rm /tmp/tmp.3UFQHZOEtX /tmp/tmp.vPqyMQ3qok + return 0 + check_crd_for_deletion PR-2058-78121c567 + local git_tag=PR-2058-78121c567 ++ curl -s https://raw.githubusercontent.com/percona/percona-server-mongodb-operator/PR-2058-78121c567/deploy/crd.yaml ++ /usr/sbin/sed s/---//g ++ yq eval .metadata.name ++ /usr/sbin/sed ':a;N;$!ba;s/\n/ /g' + for crd_name in $(curl -s https://raw.githubusercontent.com/percona/percona-server-mongodb-operator/${git_tag}/deploy/crd.yaml | yq eval '.metadata.name' | $sed 's/---//g' | $sed ':a;N;$!ba;s/\n/ /g') ++ kubectl_bin get crd/null -o 'jsonpath={.status.conditions[-1].type}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.7fDdBjCeQS +++ mktemp ++ local LAST_ERR=/tmp/tmp.TnL9g8ozSx ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'crd/null' '-o' 'jsonpath={.status.conditions[-1].type}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get crd/null -o 'jsonpath={.status.conditions[-1].type}' ++ exit_status=1 ++ set -e ++ '[' 1 '!=' 0 -a -n 1 ']' ++ cat /tmp/tmp.7fDdBjCeQS ++ cat /tmp/tmp.TnL9g8ozSx Error from server (NotFound): customresourcedefinitions.apiextensions.k8s.io "null" not found ++ sleep 0 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get crd/null -o 'jsonpath={.status.conditions[-1].type}' ++ exit_status=1 ++ set -e ++ '[' 1 '!=' 0 -a -n 1 ']' ++ cat /tmp/tmp.7fDdBjCeQS ++ cat /tmp/tmp.TnL9g8ozSx Error from server (NotFound): customresourcedefinitions.apiextensions.k8s.io "null" not found ++ sleep 4 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get crd/null -o 'jsonpath={.status.conditions[-1].type}' ++ exit_status=1 ++ set -e ++ '[' 1 '!=' 0 -a -n 1 ']' ++ cat /tmp/tmp.7fDdBjCeQS ++ cat /tmp/tmp.TnL9g8ozSx Error from server (NotFound): customresourcedefinitions.apiextensions.k8s.io "null" not found ++ sleep 8 ++ cat /tmp/tmp.7fDdBjCeQS ++ cat /tmp/tmp.TnL9g8ozSx Error from server (NotFound): customresourcedefinitions.apiextensions.k8s.io "null" not found ++ rm /tmp/tmp.7fDdBjCeQS /tmp/tmp.TnL9g8ozSx ++ return 1 + [[ '' == Terminating ]] + '[' -n psmdb-operator ']' + create_namespace psmdb-operator + local namespace=psmdb-operator + local skip_clean_namespace= + [[ 1 == 1 ]] + [[ -z '' ]] + destroy_chaos_mesh ++ tail -n1 ++ helm list --all-namespaces --filter chaos-mesh ++ sed s/NAMESPACE// ++ awk '-F ' '{print $2}' + local chaos_mesh_ns= + desc 'destroy chaos-mesh' + set +o xtrace ----------------------------------------------------------------------------------- destroy chaos-mesh ----------------------------------------------------------------------------------- + '[' -n '' ']' ++ kubectl get MutatingWebhookConfiguration ++ grep chaos-mesh ++ awk '{print $1}' + timeout 30 kubectl delete MutatingWebhookConfiguration error: resource(s) were provided, but no name was specified + : ++ kubectl get ValidatingWebhookConfiguration ++ grep chaos-mesh ++ awk '{print $1}' + timeout 30 kubectl delete ValidatingWebhookConfiguration error: resource(s) were provided, but no name was specified + : ++ kubectl get ValidatingWebhookConfiguration ++ grep validate-auth ++ awk '{print $1}' + timeout 30 kubectl delete ValidatingWebhookConfiguration error: resource(s) were provided, but no name was specified + : ++ kubectl api-resources ++ grep chaos-mesh ++ awk '{print $1}' ++ kubectl get crd ++ grep chaos-mesh.org ++ awk '{print $1}' + timeout 30 kubectl delete crd error: resource(s) were provided, but no name was specified + : ++ kubectl get clusterrolebinding ++ grep chaos-mesh ++ awk '{print $1}' + timeout 30 kubectl delete clusterrolebinding error: resource(s) were provided, but no name was specified + : ++ kubectl get clusterrole ++ grep chaos-mesh ++ awk '{print $1}' + timeout 30 kubectl delete clusterrole error: resource(s) were provided, but no name was specified + : + desc 'cleaned up all old namespaces' + set +o xtrace ----------------------------------------------------------------------------------- cleaned up all old namespaces ----------------------------------------------------------------------------------- + kubectl_bin get ns + grep -E -v '^kube-|^default|Terminating|psmdb-operator|openshift|^gke-|^gmp-|^NAME' + awk '{print$1}' + [[ -n '' ]] ++ mktemp + desc 'cleaned up old namespaces psmdb-operator' + xargs kubectl delete ns + set +o xtrace ----------------------------------------------------------------------------------- cleaned up old namespaces psmdb-operator ----------------------------------------------------------------------------------- + kubectl_bin delete namespace psmdb-operator --ignore-not-found ++ mktemp + local LAST_OUT=/tmp/tmp.pS624Ww2aD ++ mktemp + local LAST_OUT=/tmp/tmp.qlEKfHjj3C + local LAST_ERR=/tmp/tmp.3NIc4EsAPk + local exit_status=0 + local timeout=4 + args=('get' 'ns') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] ++ mktemp + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + local LAST_ERR=/tmp/tmp.ksgYHSrFgI + local exit_status=0 + local timeout=4 + args=('delete' 'namespace' 'psmdb-operator' '--ignore-not-found') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] + for i in $(seq 0 2) + set +e + kubectl get ns ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl delete namespace psmdb-operator --ignore-not-found + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.pS624Ww2aD + cat /tmp/tmp.3NIc4EsAPk + rm /tmp/tmp.pS624Ww2aD /tmp/tmp.3NIc4EsAPk + return 0 namespace "demand-backup-snapshot-15011" deleted + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.qlEKfHjj3C namespace "psmdb-operator" deleted + cat /tmp/tmp.ksgYHSrFgI + rm /tmp/tmp.qlEKfHjj3C /tmp/tmp.ksgYHSrFgI + return 0 + kubectl_bin wait --for=delete namespace psmdb-operator ++ mktemp + local LAST_OUT=/tmp/tmp.zReDiR6C4y ++ mktemp + local LAST_ERR=/tmp/tmp.bjpwG2PkSL + local exit_status=0 + local timeout=4 + args=('wait' '--for=delete' 'namespace' 'psmdb-operator') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl wait --for=delete namespace psmdb-operator + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.zReDiR6C4y + cat /tmp/tmp.bjpwG2PkSL + rm /tmp/tmp.zReDiR6C4y /tmp/tmp.bjpwG2PkSL + return 0 + desc 'create namespace psmdb-operator' + set +o xtrace ----------------------------------------------------------------------------------- create namespace psmdb-operator ----------------------------------------------------------------------------------- + kubectl_bin create namespace psmdb-operator ++ mktemp + local LAST_OUT=/tmp/tmp.tu0Z08FrTR ++ mktemp + local LAST_ERR=/tmp/tmp.ZD4fpilQMJ + local exit_status=0 + local timeout=4 + args=('create' 'namespace' 'psmdb-operator') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl create namespace psmdb-operator + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.tu0Z08FrTR namespace/psmdb-operator created + cat /tmp/tmp.ZD4fpilQMJ + rm /tmp/tmp.tu0Z08FrTR /tmp/tmp.ZD4fpilQMJ + return 0 + set_kube_ctx psmdb-operator + local namespace=psmdb-operator ++ kubectl_bin config current-context +++ mktemp ++ local LAST_OUT=/tmp/tmp.NveGWtewYT +++ mktemp ++ local LAST_ERR=/tmp/tmp.ZfM1wAXGO0 ++ local exit_status=0 ++ local timeout=4 ++ args=('config' 'current-context') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl config current-context ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.NveGWtewYT ++ cat /tmp/tmp.ZfM1wAXGO0 ++ rm /tmp/tmp.NveGWtewYT /tmp/tmp.ZfM1wAXGO0 ++ return 0 + kubectl_bin config set-context gke_cloud-dev-112233_us-central1_jen-psmdb-2058-78121c567-22-cluster11 --namespace=psmdb-operator ++ mktemp + local LAST_OUT=/tmp/tmp.lTV2JNoxx2 ++ mktemp + local LAST_ERR=/tmp/tmp.Uq5UMNRmWh + local exit_status=0 + local timeout=4 + args=('config' 'set-context' 'gke_cloud-dev-112233_us-central1_jen-psmdb-2058-78121c567-22-cluster11' '--namespace=psmdb-operator') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl config set-context gke_cloud-dev-112233_us-central1_jen-psmdb-2058-78121c567-22-cluster11 --namespace=psmdb-operator + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.lTV2JNoxx2 Context "gke_cloud-dev-112233_us-central1_jen-psmdb-2058-78121c567-22-cluster11" modified. + cat /tmp/tmp.Uq5UMNRmWh + rm /tmp/tmp.lTV2JNoxx2 /tmp/tmp.Uq5UMNRmWh + return 0 + deploy_operator + desc 'start PSMDB operator: docker.io/perconalab/percona-server-mongodb-operator:PR-2058-78121c567' + set +o xtrace ----------------------------------------------------------------------------------- start PSMDB operator: docker.io/perconalab/percona-server-mongodb-operator:PR-2058-78121c567 ----------------------------------------------------------------------------------- + local cr_file + '[' -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/e2e-tests/monitoring-pmm3/conf/crd.yaml ']' + cr_file=/mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/deploy/crd.yaml + kubectl_bin apply --server-side --force-conflicts -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/deploy/crd.yaml ++ mktemp + local LAST_OUT=/tmp/tmp.cUe1ppFIj9 ++ mktemp + local LAST_ERR=/tmp/tmp.1L3lj9Yx4F + local exit_status=0 + local timeout=4 + args=('apply' '--server-side' '--force-conflicts' '-f' '/mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/deploy/crd.yaml') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl apply --server-side --force-conflicts -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/deploy/crd.yaml + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.cUe1ppFIj9 customresourcedefinition.apiextensions.k8s.io/perconaservermongodbbackups.psmdb.percona.com serverside-applied customresourcedefinition.apiextensions.k8s.io/perconaservermongodbclustersyncs.psmdb.percona.com serverside-applied customresourcedefinition.apiextensions.k8s.io/perconaservermongodbrestores.psmdb.percona.com serverside-applied customresourcedefinition.apiextensions.k8s.io/perconaservermongodbs.psmdb.percona.com serverside-applied + cat /tmp/tmp.1L3lj9Yx4F + rm /tmp/tmp.cUe1ppFIj9 /tmp/tmp.1L3lj9Yx4F + return 0 + '[' -n psmdb-operator ']' + apply_rbac cw-rbac + local operator_namespace=psmdb-operator + local rbac=cw-rbac + cat /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/deploy/cw-rbac.yaml + sed -e 's^namespace: .*^namespace: psmdb-operator^' + kubectl_bin apply -n psmdb-operator -f - ++ mktemp + local LAST_OUT=/tmp/tmp.hrZkJtwpjl ++ mktemp + local LAST_ERR=/tmp/tmp.m6XSIqVcLa + local exit_status=0 + local timeout=4 + args=('apply' '-n' 'psmdb-operator' '-f' '-') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl apply -n psmdb-operator -f - + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.hrZkJtwpjl clusterrole.rbac.authorization.k8s.io/percona-server-mongodb-operator created serviceaccount/percona-server-mongodb-operator created clusterrolebinding.rbac.authorization.k8s.io/service-account-percona-server-mongodb-operator created + cat /tmp/tmp.m6XSIqVcLa + rm /tmp/tmp.hrZkJtwpjl /tmp/tmp.m6XSIqVcLa + return 0 + yq eval $'\n\t\t\t(.spec.template.spec.containers[].image = "docker.io/perconalab/percona-server-mongodb-operator:PR-2058-78121c567") |\n\t\t\t((.. | select(.[] == "DISABLE_TELEMETRY")) |= .value="true") |\n\t\t\t((.. | select(.[] == "LOG_LEVEL")) |= .value="DEBUG")' /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/deploy/cw-operator.yaml + add_arch_to_podspecs - + kubectl_bin apply -n psmdb-operator -f - + arch_scheduling_enabled + [[ -n '' ]] + cat - ++ mktemp + local LAST_OUT=/tmp/tmp.pMtmJnRE0R ++ mktemp + local LAST_ERR=/tmp/tmp.YGI9iyJTZi + local exit_status=0 + local timeout=4 + args=('apply' '-n' 'psmdb-operator' '-f' '-') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl apply -n psmdb-operator -f - + return + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.pMtmJnRE0R deployment.apps/percona-server-mongodb-operator created + cat /tmp/tmp.YGI9iyJTZi + rm /tmp/tmp.pMtmJnRE0R /tmp/tmp.YGI9iyJTZi + return 0 + sleep 20 ++ get_operator_pod ++ kubectl_bin get pods --selector=name=percona-server-mongodb-operator -o 'jsonpath={.items[].metadata.name}' -n psmdb-operator +++ mktemp ++ local LAST_OUT=/tmp/tmp.7iSaoi4tyb +++ mktemp ++ local LAST_ERR=/tmp/tmp.PGUQg6c64t ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'pods' '--selector=name=percona-server-mongodb-operator' '-o' 'jsonpath={.items[].metadata.name}' '-n' 'psmdb-operator') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pods --selector=name=percona-server-mongodb-operator -o 'jsonpath={.items[].metadata.name}' -n psmdb-operator ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.7iSaoi4tyb ++ cat /tmp/tmp.PGUQg6c64t ++ rm /tmp/tmp.7iSaoi4tyb /tmp/tmp.PGUQg6c64t ++ return 0 + wait_operator_pod percona-server-mongodb-operator-66cd7b75cf-c8k42 + local pod=percona-server-mongodb-operator-66cd7b75cf-c8k42 + set +o xtrace waiting for pod/percona-server-mongodb-operator-66cd7b75cf-c8k42 to be ready.OK + echo 'Print operator info from log' Print operator info from log + grep 'Manager starting up' ++ get_operator_pod ++ kubectl_bin get pods --selector=name=percona-server-mongodb-operator -o 'jsonpath={.items[].metadata.name}' -n psmdb-operator +++ mktemp ++ local LAST_OUT=/tmp/tmp.6uKcaVLVCu +++ mktemp ++ local LAST_ERR=/tmp/tmp.KTndgHoDg9 ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'pods' '--selector=name=percona-server-mongodb-operator' '-o' 'jsonpath={.items[].metadata.name}' '-n' 'psmdb-operator') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pods --selector=name=percona-server-mongodb-operator -o 'jsonpath={.items[].metadata.name}' -n psmdb-operator ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.6uKcaVLVCu ++ cat /tmp/tmp.KTndgHoDg9 ++ rm /tmp/tmp.6uKcaVLVCu /tmp/tmp.KTndgHoDg9 ++ return 0 + kubectl_bin logs -n psmdb-operator percona-server-mongodb-operator-66cd7b75cf-c8k42 ++ mktemp + local LAST_OUT=/tmp/tmp.qjoaWXFvMu ++ mktemp + local LAST_ERR=/tmp/tmp.CjLpaAzyob + local exit_status=0 + local timeout=4 + args=('logs' '-n' 'psmdb-operator' 'percona-server-mongodb-operator-66cd7b75cf-c8k42') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl logs -n psmdb-operator percona-server-mongodb-operator-66cd7b75cf-c8k42 + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.qjoaWXFvMu + cat /tmp/tmp.CjLpaAzyob + rm /tmp/tmp.qjoaWXFvMu /tmp/tmp.CjLpaAzyob + return 0 2026-07-17T01:10:55.973Z INFO setup Manager starting up {"gitCommit": "78121c56762758785b75e9b7c017e15fc887ae5d", "gitBranch": "PR-2058-78121c567", "buildTime": "", "goVersion": "go1.26.5", "os": "linux", "arch": "amd64"} + create_namespace monitoring-pmm3-14502 + local namespace=monitoring-pmm3-14502 + local skip_clean_namespace= + [[ 1 == 1 ]] + [[ -z '' ]] + destroy_chaos_mesh ++ helm list --all-namespaces --filter chaos-mesh ++ tail -n1 ++ awk '-F ' '{print $2}' ++ sed s/NAMESPACE// + local chaos_mesh_ns= + desc 'destroy chaos-mesh' + set +o xtrace ----------------------------------------------------------------------------------- destroy chaos-mesh ----------------------------------------------------------------------------------- + '[' -n '' ']' ++ kubectl get MutatingWebhookConfiguration ++ grep chaos-mesh ++ awk '{print $1}' + timeout 30 kubectl delete MutatingWebhookConfiguration error: resource(s) were provided, but no name was specified + : ++ kubectl get ValidatingWebhookConfiguration ++ grep chaos-mesh ++ awk '{print $1}' + timeout 30 kubectl delete ValidatingWebhookConfiguration error: resource(s) were provided, but no name was specified + : ++ kubectl get ValidatingWebhookConfiguration ++ grep validate-auth ++ awk '{print $1}' + timeout 30 kubectl delete ValidatingWebhookConfiguration error: resource(s) were provided, but no name was specified + : ++ grep chaos-mesh ++ kubectl api-resources ++ awk '{print $1}' ++ kubectl get crd ++ grep chaos-mesh.org ++ awk '{print $1}' + timeout 30 kubectl delete crd error: resource(s) were provided, but no name was specified + : ++ kubectl get clusterrolebinding ++ grep chaos-mesh ++ awk '{print $1}' + timeout 30 kubectl delete clusterrolebinding error: resource(s) were provided, but no name was specified + : ++ kubectl get clusterrole ++ grep chaos-mesh ++ awk '{print $1}' + timeout 30 kubectl delete clusterrole error: resource(s) were provided, but no name was specified + : + desc 'cleaned up all old namespaces' + set +o xtrace ----------------------------------------------------------------------------------- cleaned up all old namespaces ----------------------------------------------------------------------------------- + kubectl_bin get ns + grep -E -v '^kube-|^default|Terminating|psmdb-operator|openshift|^gke-|^gmp-|^NAME' + awk '{print$1}' + [[ -n '' ]] + desc 'cleaned up old namespaces monitoring-pmm3-14502' + set +o xtrace ----------------------------------------------------------------------------------- cleaned up old namespaces monitoring-pmm3-14502 ----------------------------------------------------------------------------------- + kubectl_bin delete namespace monitoring-pmm3-14502 --ignore-not-found ++ mktemp + xargs kubectl delete ns ++ mktemp + local LAST_OUT=/tmp/tmp.Q3xXmadPOn + local LAST_OUT=/tmp/tmp.hOt3MeK4XS ++ mktemp ++ mktemp + local LAST_ERR=/tmp/tmp.KtzfLL2E0W + local exit_status=0 + local timeout=4 + args=('get' 'ns') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] + local LAST_ERR=/tmp/tmp.cpqoBuQFGg + local exit_status=0 + local timeout=4 + args=('delete' 'namespace' 'monitoring-pmm3-14502' '--ignore-not-found') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl get ns + for i in $(seq 0 2) + set +e + kubectl delete namespace monitoring-pmm3-14502 --ignore-not-found + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.Q3xXmadPOn + cat /tmp/tmp.KtzfLL2E0W + rm /tmp/tmp.Q3xXmadPOn /tmp/tmp.KtzfLL2E0W + return 0 error: resource(s) were provided, but no name was specified + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.hOt3MeK4XS + cat /tmp/tmp.cpqoBuQFGg + rm /tmp/tmp.hOt3MeK4XS /tmp/tmp.cpqoBuQFGg + return 0 + kubectl_bin wait --for=delete namespace monitoring-pmm3-14502 ++ mktemp + local LAST_OUT=/tmp/tmp.FwRg0t4OSz ++ mktemp + local LAST_ERR=/tmp/tmp.Hbfvss57gS + local exit_status=0 + local timeout=4 + args=('wait' '--for=delete' 'namespace' 'monitoring-pmm3-14502') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl wait --for=delete namespace monitoring-pmm3-14502 + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.FwRg0t4OSz + cat /tmp/tmp.Hbfvss57gS + rm /tmp/tmp.FwRg0t4OSz /tmp/tmp.Hbfvss57gS + return 0 + desc 'create namespace monitoring-pmm3-14502' + set +o xtrace ----------------------------------------------------------------------------------- create namespace monitoring-pmm3-14502 ----------------------------------------------------------------------------------- + kubectl_bin create namespace monitoring-pmm3-14502 ++ mktemp + local LAST_OUT=/tmp/tmp.jE81wAVW9y ++ mktemp + local LAST_ERR=/tmp/tmp.N04XmVH7DE + local exit_status=0 + local timeout=4 + args=('create' 'namespace' 'monitoring-pmm3-14502') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl create namespace monitoring-pmm3-14502 + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.jE81wAVW9y namespace/monitoring-pmm3-14502 created + cat /tmp/tmp.N04XmVH7DE + rm /tmp/tmp.jE81wAVW9y /tmp/tmp.N04XmVH7DE + return 0 + set_kube_ctx monitoring-pmm3-14502 + local namespace=monitoring-pmm3-14502 ++ kubectl_bin config current-context +++ mktemp ++ local LAST_OUT=/tmp/tmp.eF7scnrguX +++ mktemp ++ local LAST_ERR=/tmp/tmp.oBcXPHOleW ++ local exit_status=0 ++ local timeout=4 ++ args=('config' 'current-context') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl config current-context ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.eF7scnrguX ++ cat /tmp/tmp.oBcXPHOleW ++ rm /tmp/tmp.eF7scnrguX /tmp/tmp.oBcXPHOleW ++ return 0 + kubectl_bin config set-context gke_cloud-dev-112233_us-central1_jen-psmdb-2058-78121c567-22-cluster11 --namespace=monitoring-pmm3-14502 ++ mktemp + local LAST_OUT=/tmp/tmp.oGpYHH7znX ++ mktemp + local LAST_ERR=/tmp/tmp.GMuXH6aIZw + local exit_status=0 + local timeout=4 + args=('config' 'set-context' 'gke_cloud-dev-112233_us-central1_jen-psmdb-2058-78121c567-22-cluster11' '--namespace=monitoring-pmm3-14502') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl config set-context gke_cloud-dev-112233_us-central1_jen-psmdb-2058-78121c567-22-cluster11 --namespace=monitoring-pmm3-14502 + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.oGpYHH7znX Context "gke_cloud-dev-112233_us-central1_jen-psmdb-2058-78121c567-22-cluster11" modified. + cat /tmp/tmp.GMuXH6aIZw + rm /tmp/tmp.oGpYHH7znX /tmp/tmp.GMuXH6aIZw + return 0 + deploy_cert_manager + [[ gke == rancher ]] + desc 'deploy cert manager' + set +o xtrace ----------------------------------------------------------------------------------- deploy cert manager ----------------------------------------------------------------------------------- + kubectl_bin -n cert-manager delete clusterissuer --all ++ mktemp + local LAST_OUT=/tmp/tmp.nvAJ0fcSpl ++ mktemp + local LAST_ERR=/tmp/tmp.gfKTBVFRMK + local exit_status=0 + local timeout=4 + args=('-n' 'cert-manager' 'delete' 'clusterissuer' '--all') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl -n cert-manager delete clusterissuer --all + exit_status=1 + set -e + '[' 1 '!=' 0 -a -n 1 ']' + cat /tmp/tmp.nvAJ0fcSpl + cat /tmp/tmp.gfKTBVFRMK error: the server doesn't have a resource type "clusterissuer" + sleep 0 + for i in $(seq 0 2) + set +e + kubectl -n cert-manager delete clusterissuer --all + exit_status=1 + set -e + '[' 1 '!=' 0 -a -n 1 ']' + cat /tmp/tmp.nvAJ0fcSpl + cat /tmp/tmp.gfKTBVFRMK error: the server doesn't have a resource type "clusterissuer" + sleep 4 + for i in $(seq 0 2) + set +e + kubectl -n cert-manager delete clusterissuer --all + exit_status=1 + set -e + '[' 1 '!=' 0 -a -n 1 ']' + cat /tmp/tmp.nvAJ0fcSpl + cat /tmp/tmp.gfKTBVFRMK error: the server doesn't have a resource type "clusterissuer" + sleep 8 + cat /tmp/tmp.nvAJ0fcSpl + cat /tmp/tmp.gfKTBVFRMK error: the server doesn't have a resource type "clusterissuer" + rm /tmp/tmp.nvAJ0fcSpl /tmp/tmp.gfKTBVFRMK + return 1 + : + kubectl_bin -n cert-manager delete certificate --all ++ mktemp + local LAST_OUT=/tmp/tmp.N0XHmS2cDq ++ mktemp + local LAST_ERR=/tmp/tmp.XamJ9uMQB4 + local exit_status=0 + local timeout=4 + args=('-n' 'cert-manager' 'delete' 'certificate' '--all') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl -n cert-manager delete certificate --all + exit_status=1 + set -e + '[' 1 '!=' 0 -a -n 1 ']' + cat /tmp/tmp.N0XHmS2cDq + cat /tmp/tmp.XamJ9uMQB4 error: the server doesn't have a resource type "certificate" + sleep 0 + for i in $(seq 0 2) + set +e + kubectl -n cert-manager delete certificate --all + exit_status=1 + set -e + '[' 1 '!=' 0 -a -n 1 ']' + cat /tmp/tmp.N0XHmS2cDq + cat /tmp/tmp.XamJ9uMQB4 error: the server doesn't have a resource type "certificate" + sleep 4 + for i in $(seq 0 2) + set +e + kubectl -n cert-manager delete certificate --all + exit_status=1 + set -e + '[' 1 '!=' 0 -a -n 1 ']' + cat /tmp/tmp.N0XHmS2cDq + cat /tmp/tmp.XamJ9uMQB4 error: the server doesn't have a resource type "certificate" + sleep 8 + cat /tmp/tmp.N0XHmS2cDq + cat /tmp/tmp.XamJ9uMQB4 error: the server doesn't have a resource type "certificate" + rm /tmp/tmp.N0XHmS2cDq /tmp/tmp.XamJ9uMQB4 + return 1 + : + kubectl_bin create namespace cert-manager ++ mktemp + local LAST_OUT=/tmp/tmp.3YQenTk3L2 ++ mktemp + local LAST_ERR=/tmp/tmp.oBwZoSDWcC + local exit_status=0 + local timeout=4 + args=('create' 'namespace' 'cert-manager') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl create namespace cert-manager + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.3YQenTk3L2 namespace/cert-manager created + cat /tmp/tmp.oBwZoSDWcC + rm /tmp/tmp.3YQenTk3L2 /tmp/tmp.oBwZoSDWcC + return 0 + kubectl_bin label namespace cert-manager certmanager.k8s.io/disable-validation=true ++ mktemp + local LAST_OUT=/tmp/tmp.ColOGg7CiV ++ mktemp + local LAST_ERR=/tmp/tmp.0K2YZYQZq2 + local exit_status=0 + local timeout=4 + args=('label' 'namespace' 'cert-manager' 'certmanager.k8s.io/disable-validation=true') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl label namespace cert-manager certmanager.k8s.io/disable-validation=true + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.ColOGg7CiV namespace/cert-manager labeled + cat /tmp/tmp.0K2YZYQZq2 + rm /tmp/tmp.ColOGg7CiV /tmp/tmp.0K2YZYQZq2 + return 0 + curl -sL https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml + add_arch_to_podspecs - + arch_scheduling_enabled + kubectl_bin apply -f - --validate=false + [[ -n '' ]] + cat - + return namespace/cert-manager configured customresourcedefinition.apiextensions.k8s.io/challenges.acme.cert-manager.io created customresourcedefinition.apiextensions.k8s.io/orders.acme.cert-manager.io created customresourcedefinition.apiextensions.k8s.io/certificaterequests.cert-manager.io created customresourcedefinition.apiextensions.k8s.io/certificates.cert-manager.io created customresourcedefinition.apiextensions.k8s.io/clusterissuers.cert-manager.io created customresourcedefinition.apiextensions.k8s.io/issuers.cert-manager.io created serviceaccount/cert-manager-cainjector created serviceaccount/cert-manager created serviceaccount/cert-manager-webhook created clusterrole.rbac.authorization.k8s.io/cert-manager-cainjector created clusterrole.rbac.authorization.k8s.io/cert-manager-controller-issuers created clusterrole.rbac.authorization.k8s.io/cert-manager-controller-clusterissuers created clusterrole.rbac.authorization.k8s.io/cert-manager-controller-certificates created clusterrole.rbac.authorization.k8s.io/cert-manager-controller-orders created clusterrole.rbac.authorization.k8s.io/cert-manager-controller-challenges created clusterrole.rbac.authorization.k8s.io/cert-manager-controller-ingress-shim created clusterrole.rbac.authorization.k8s.io/cert-manager-cluster-view created clusterrole.rbac.authorization.k8s.io/cert-manager-view created clusterrole.rbac.authorization.k8s.io/cert-manager-edit created clusterrole.rbac.authorization.k8s.io/cert-manager-controller-approve:cert-manager-io created clusterrole.rbac.authorization.k8s.io/cert-manager-controller-certificatesigningrequests created clusterrole.rbac.authorization.k8s.io/cert-manager-webhook:subjectaccessreviews created clusterrolebinding.rbac.authorization.k8s.io/cert-manager-cainjector created clusterrolebinding.rbac.authorization.k8s.io/cert-manager-controller-issuers created clusterrolebinding.rbac.authorization.k8s.io/cert-manager-controller-clusterissuers created clusterrolebinding.rbac.authorization.k8s.io/cert-manager-controller-certificates created clusterrolebinding.rbac.authorization.k8s.io/cert-manager-controller-orders created clusterrolebinding.rbac.authorization.k8s.io/cert-manager-controller-challenges created clusterrolebinding.rbac.authorization.k8s.io/cert-manager-controller-ingress-shim created clusterrolebinding.rbac.authorization.k8s.io/cert-manager-controller-approve:cert-manager-io created clusterrolebinding.rbac.authorization.k8s.io/cert-manager-controller-certificatesigningrequests created clusterrolebinding.rbac.authorization.k8s.io/cert-manager-webhook:subjectaccessreviews created role.rbac.authorization.k8s.io/cert-manager-cainjector:leaderelection created role.rbac.authorization.k8s.io/cert-manager:leaderelection created role.rbac.authorization.k8s.io/cert-manager-tokenrequest created role.rbac.authorization.k8s.io/cert-manager-webhook:dynamic-serving created rolebinding.rbac.authorization.k8s.io/cert-manager-cainjector:leaderelection created rolebinding.rbac.authorization.k8s.io/cert-manager:leaderelection created rolebinding.rbac.authorization.k8s.io/cert-manager-tokenrequest created rolebinding.rbac.authorization.k8s.io/cert-manager-webhook:dynamic-serving created service/cert-manager-cainjector created service/cert-manager created service/cert-manager-webhook created deployment.apps/cert-manager-cainjector created deployment.apps/cert-manager created deployment.apps/cert-manager-webhook created mutatingwebhookconfiguration.admissionregistration.k8s.io/cert-manager-webhook created validatingwebhookconfiguration.admissionregistration.k8s.io/cert-manager-webhook created + kubectl_bin -n cert-manager wait pod -l app.kubernetes.io/instance=cert-manager --for=condition=ready ++ mktemp + local LAST_OUT=/tmp/tmp.x6KKUEXiy5 ++ mktemp + local LAST_ERR=/tmp/tmp.W27dbVW8BT + local exit_status=0 + local timeout=4 + args=('-n' 'cert-manager' 'wait' 'pod' '-l' 'app.kubernetes.io/instance=cert-manager' '--for=condition=ready') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl -n cert-manager wait pod -l app.kubernetes.io/instance=cert-manager --for=condition=ready + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.x6KKUEXiy5 pod/cert-manager-559d798845-rk6xw condition met pod/cert-manager-cainjector-64958d9c7c-bztpw condition met pod/cert-manager-webhook-7fb6f99b56-nh89r condition met + cat /tmp/tmp.W27dbVW8BT E0717 01:12:25.340253 1903816 reflector.go:227] "Failed to watch" err="Get \"https://136.119.161.237/api/v1/namespaces/cert-manager/pods?allowWatchBookmarks=true&fieldSelector=metadata.name%3Dcert-manager-cainjector-64958d9c7c-bztpw&resourceVersion=1784250744964447000&timeoutSeconds=408&watch=true\": context canceled" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" type="*unstructured.Unstructured" + rm /tmp/tmp.x6KKUEXiy5 /tmp/tmp.W27dbVW8BT + return 0 + sleep 120 + desc 'install PMM Server' + set +o xtrace ----------------------------------------------------------------------------------- install PMM Server ----------------------------------------------------------------------------------- + deploy_pmm3_server + helm uninstall monitoring Error: uninstall: Release not loaded: monitoring: release: not found + : + helm repo remove percona "percona" has been removed from your repositories + kubectl delete clusterrole monitoring --ignore-not-found + kubectl delete clusterrolebinding monitoring --ignore-not-found + helm repo add percona https://percona.github.io/percona-helm-charts/ "percona" has been added to your repositories + helm repo update Hang tight while we grab the latest from your chart repositories... ...Successfully got an update from the "minio" chart repository ...Successfully got an update from the "chaos-mesh" chart repository ...Successfully got an update from the "hashicorp" chart repository ...Successfully got an update from the "percona" chart repository ...Successfully got an update from the "stable" chart repository Update Complete. ⎈Happy Helming!⎈ + [[ -n '' ]] + pmm_args=('--set' 'fullnameOverride=monitoring-server' '--set' 'image.tag=3-dev-latest' '--set' 'image.repository=docker.io/perconalab/pmm-server' '--set' 'service.type=LoadBalancer') + local pmm_args + arch_scheduling_enabled + [[ -n '' ]] + retry 10 60 helm install monitoring percona/pmm --set fullnameOverride=monitoring-server --set image.tag=3-dev-latest --set image.repository=docker.io/perconalab/pmm-server --set service.type=LoadBalancer --force + local max=10 + local delay=60 + shift 2 + local n=1 + helm install monitoring percona/pmm --set fullnameOverride=monitoring-server --set image.tag=3-dev-latest --set image.repository=docker.io/perconalab/pmm-server --set service.type=LoadBalancer --force NAME: monitoring LAST DEPLOYED: Fri Jul 17 01:14:42 2026 NAMESPACE: monitoring-pmm3-14502 STATUS: deployed REVISION: 1 TEST SUITE: None NOTES: Percona Monitoring and Management (PMM) An open source database monitoring, observability and management tool Check more info here: https://docs.percona.com/percona-monitoring-and-management/index.html Get the application URL: NOTE: It may take a few minutes for the LoadBalancer IP to be available. You can watch the status of by running 'kubectl get --namespace monitoring-pmm3-14502 svc -w monitoring-service' export SERVICE_IP=$(kubectl get svc --namespace monitoring-pmm3-14502 monitoring-service -o jsonpath="{.status.loadBalancer.ingress[0].ip}") echo https://$SERVICE_IP: Get password for the "admin" user: export ADMIN_PASS=$(kubectl get secret pmm-secret --namespace monitoring-pmm3-14502 -o jsonpath='{.data.PMM_ADMIN_PASSWORD}' | base64 --decode) echo $ADMIN_PASS + sleep 20 + kubectl_bin exec monitoring-server-0 -- bash -c 'ls -l /proc/*/exe 2>/dev/null| grep postgres >/dev/null' ++ mktemp + local LAST_OUT=/tmp/tmp.Fw6NU0RAAW ++ mktemp + local LAST_ERR=/tmp/tmp.pofKKYtvj4 + local exit_status=0 + local timeout=4 + args=('exec' 'monitoring-server-0' '--' 'bash' '-c' 'ls -l /proc/*/exe 2>/dev/null| grep postgres >/dev/null') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl exec monitoring-server-0 -- bash -c 'ls -l /proc/*/exe 2>/dev/null| grep postgres >/dev/null' + exit_status=1 + set -e + '[' 1 '!=' 0 -a -n 1 ']' + cat /tmp/tmp.Fw6NU0RAAW + cat /tmp/tmp.pofKKYtvj4 error: Internal error occurred: unable to upgrade connection: container not found ("pmm") + sleep 0 + for i in $(seq 0 2) + set +e + kubectl exec monitoring-server-0 -- bash -c 'ls -l /proc/*/exe 2>/dev/null| grep postgres >/dev/null' + exit_status=1 + set -e + '[' 1 '!=' 0 -a -n 1 ']' + cat /tmp/tmp.Fw6NU0RAAW + cat /tmp/tmp.pofKKYtvj4 error: Internal error occurred: unable to upgrade connection: container not found ("pmm") + sleep 4 + for i in $(seq 0 2) + set +e + kubectl exec monitoring-server-0 -- bash -c 'ls -l /proc/*/exe 2>/dev/null| grep postgres >/dev/null' + exit_status=1 + set -e + '[' 1 '!=' 0 -a -n 1 ']' + cat /tmp/tmp.Fw6NU0RAAW + cat /tmp/tmp.pofKKYtvj4 error: Internal error occurred: unable to upgrade connection: container not found ("pmm") + sleep 8 + cat /tmp/tmp.Fw6NU0RAAW + cat /tmp/tmp.pofKKYtvj4 error: Internal error occurred: unable to upgrade connection: container not found ("pmm") + rm /tmp/tmp.Fw6NU0RAAW /tmp/tmp.pofKKYtvj4 + return 1 + echo 'Retry 0' Retry 0 + sleep 5 + let retry+=1 + '[' 1 -ge 20 ']' + kubectl_bin exec monitoring-server-0 -- bash -c 'ls -l /proc/*/exe 2>/dev/null| grep postgres >/dev/null' ++ mktemp + local LAST_OUT=/tmp/tmp.ArbKbFfxh0 ++ mktemp + local LAST_ERR=/tmp/tmp.bj9KzuzpN2 + local exit_status=0 + local timeout=4 + args=('exec' 'monitoring-server-0' '--' 'bash' '-c' 'ls -l /proc/*/exe 2>/dev/null| grep postgres >/dev/null') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl exec monitoring-server-0 -- bash -c 'ls -l /proc/*/exe 2>/dev/null| grep postgres >/dev/null' + exit_status=1 + set -e + '[' 1 '!=' 0 -a -n 1 ']' + cat /tmp/tmp.ArbKbFfxh0 + cat /tmp/tmp.bj9KzuzpN2 error: Internal error occurred: unable to upgrade connection: container not found ("pmm") + sleep 0 + for i in $(seq 0 2) + set +e + kubectl exec monitoring-server-0 -- bash -c 'ls -l /proc/*/exe 2>/dev/null| grep postgres >/dev/null' + exit_status=1 + set -e + '[' 1 '!=' 0 -a -n 1 ']' + cat /tmp/tmp.ArbKbFfxh0 + cat /tmp/tmp.bj9KzuzpN2 error: Internal error occurred: unable to upgrade connection: container not found ("pmm") + sleep 4 + for i in $(seq 0 2) + set +e + kubectl exec monitoring-server-0 -- bash -c 'ls -l /proc/*/exe 2>/dev/null| grep postgres >/dev/null' + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.ArbKbFfxh0 + cat /tmp/tmp.bj9KzuzpN2 + rm /tmp/tmp.ArbKbFfxh0 /tmp/tmp.bj9KzuzpN2 + return 0 + cluster=monitoring-pmm3 + desc 'create secrets and start client' + set +o xtrace ----------------------------------------------------------------------------------- create secrets and start client ----------------------------------------------------------------------------------- + kubectl_bin apply -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/e2e-tests/conf/secrets.yml -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/e2e-tests/monitoring-pmm3/conf/secrets.yml ++ mktemp + local LAST_OUT=/tmp/tmp.PnfCg8aLHC ++ mktemp + local LAST_ERR=/tmp/tmp.3nSdXDh7dI + local exit_status=0 + local timeout=4 + args=('apply' '-f' '/mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/e2e-tests/conf/secrets.yml' '-f' '/mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/e2e-tests/monitoring-pmm3/conf/secrets.yml') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl apply -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/e2e-tests/conf/secrets.yml -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/e2e-tests/monitoring-pmm3/conf/secrets.yml + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.PnfCg8aLHC secret/some-users created secret/some-users unchanged + cat /tmp/tmp.3nSdXDh7dI + rm /tmp/tmp.PnfCg8aLHC /tmp/tmp.3nSdXDh7dI + return 0 + desc 'create first PSMDB cluster monitoring-pmm3' + set +o xtrace ----------------------------------------------------------------------------------- create first PSMDB cluster monitoring-pmm3 ----------------------------------------------------------------------------------- + custom_cluster_name=super-custom + yq eval '(.spec | select(.image == null)).image = "docker.io/perconalab/percona-server-mongodb-operator:main-mongod8.0"' /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/e2e-tests/monitoring-pmm3/conf/monitoring-pmm3-rs0.yml + yq eval '(.spec | select(has("pmm"))).pmm.image = "docker.io/perconalab/pmm-client:3-dev-latest"' - + yq eval '(.spec | select(has("pmm"))).pmm.customClusterName = "super-custom"' - + yq eval '(.spec | select(has("logcollector"))).logcollector.image = "docker.io/perconalab/fluentbit:main-logcollector"' - + yq eval '(.spec | select(has("initImage"))).initImage = "docker.io/perconalab/percona-server-mongodb-operator:PR-2058-78121c567"' - + yq eval '(.spec | select(has("backup"))).backup.image = "docker.io/perconalab/percona-server-mongodb-operator:main-backup"' - + yq eval '.spec.upgradeOptions.apply = "Never"' - + kubectl_bin apply -f - ++ mktemp + local LAST_OUT=/tmp/tmp.4L1ZUKxVdC ++ mktemp + local LAST_ERR=/tmp/tmp.5VhYmftg2I + local exit_status=0 + local timeout=4 + args=('apply' '-f' '-') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.4L1ZUKxVdC perconaservermongodb.psmdb.percona.com/monitoring-pmm3 created + cat /tmp/tmp.5VhYmftg2I + rm /tmp/tmp.4L1ZUKxVdC /tmp/tmp.5VhYmftg2I + return 0 + wait_for_running monitoring-pmm3-rs0 3 + local name=monitoring-pmm3-rs0 + let last_pod=2 + local check_cluster_readiness=true + set_debug + [[ 1 == 1 ]] + set -o xtrace + local rs_name=rs0 + local cluster_name=monitoring-pmm3 ++ seq 0 2 + for i in $(seq 0 $last_pod) + [[ 0 -eq 2 ]] + wait_pod monitoring-pmm3-rs0-0 + local pod=monitoring-pmm3-rs0-0 + set +o xtrace waiting for pod/monitoring-pmm3-rs0-0 to be ready............OK + for i in $(seq 0 $last_pod) + [[ 1 -eq 2 ]] + wait_pod monitoring-pmm3-rs0-1 + local pod=monitoring-pmm3-rs0-1 + set +o xtrace waiting for pod/monitoring-pmm3-rs0-1 to be ready...........OK + for i in $(seq 0 $last_pod) + [[ 2 -eq 2 ]] ++ kubectl_bin get psmdb monitoring-pmm3 -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].arbiter.enabled}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.W22P0dKtYo +++ mktemp ++ local LAST_ERR=/tmp/tmp.AjPIM3g6zg ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'psmdb' 'monitoring-pmm3' '-o' 'jsonpath={.spec.replsets[?(@.name=="rs0")].arbiter.enabled}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb monitoring-pmm3 -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].arbiter.enabled}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.W22P0dKtYo ++ cat /tmp/tmp.AjPIM3g6zg ++ rm /tmp/tmp.W22P0dKtYo /tmp/tmp.AjPIM3g6zg ++ return 0 + [[ '' == true ]] + wait_pod monitoring-pmm3-rs0-2 + local pod=monitoring-pmm3-rs0-2 + set +o xtrace waiting for pod/monitoring-pmm3-rs0-2 to be ready.........OK ++ kubectl_bin get psmdb monitoring-pmm3 -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].nonvoting.enabled}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.qvOynyx5Hc +++ mktemp ++ local LAST_ERR=/tmp/tmp.Uh1S73iWIM ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'psmdb' 'monitoring-pmm3' '-o' 'jsonpath={.spec.replsets[?(@.name=="rs0")].nonvoting.enabled}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb monitoring-pmm3 -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].nonvoting.enabled}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.qvOynyx5Hc ++ cat /tmp/tmp.Uh1S73iWIM ++ rm /tmp/tmp.qvOynyx5Hc /tmp/tmp.Uh1S73iWIM ++ return 0 + [[ '' == true ]] ++ kubectl_bin get psmdb monitoring-pmm3 -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].hidden.enabled}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.mgmqypXjZ0 +++ mktemp ++ local LAST_ERR=/tmp/tmp.JInpjmtbvy ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'psmdb' 'monitoring-pmm3' '-o' 'jsonpath={.spec.replsets[?(@.name=="rs0")].hidden.enabled}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb monitoring-pmm3 -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].hidden.enabled}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.mgmqypXjZ0 ++ cat /tmp/tmp.JInpjmtbvy ++ rm /tmp/tmp.mgmqypXjZ0 /tmp/tmp.JInpjmtbvy ++ return 0 + [[ '' == true ]] + sleep 10 + [[ true == true ]] + set +x Waiting for cluster monitoring-pmm3 to be ready........................ + apply_client_with_tls monitoring-pmm3 + local cluster=monitoring-pmm3 + yq '.spec.template.spec.volumes[0].secret.secretName="monitoring-pmm3-ssl"' /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/e2e-tests/conf/client_with_tls.yml + kubectl_bin apply -f - ++ mktemp + local LAST_OUT=/tmp/tmp.tTHIA0bGJx ++ mktemp + local LAST_ERR=/tmp/tmp.y4B9BLR5Ex + local exit_status=0 + local timeout=4 + args=('apply' '-f' '-') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.tTHIA0bGJx deployment.apps/psmdb-client created + cat /tmp/tmp.y4B9BLR5Ex + rm /tmp/tmp.tTHIA0bGJx /tmp/tmp.y4B9BLR5Ex + return 0 + kubectl_bin rollout status deployment/psmdb-client --timeout=360s ++ mktemp + local LAST_OUT=/tmp/tmp.ljaW9o84Qf ++ mktemp + local LAST_ERR=/tmp/tmp.ols8UAxQv0 + local exit_status=0 + local timeout=4 + args=('rollout' 'status' 'deployment/psmdb-client' '--timeout=360s') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl rollout status deployment/psmdb-client --timeout=360s + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.ljaW9o84Qf Waiting for deployment "psmdb-client" rollout to finish: 0 of 1 updated replicas are available... deployment "psmdb-client" successfully rolled out + cat /tmp/tmp.ols8UAxQv0 + rm /tmp/tmp.ljaW9o84Qf /tmp/tmp.ols8UAxQv0 + return 0 + desc 'check if pmm-client container is not enabled' + set +o xtrace ----------------------------------------------------------------------------------- check if pmm-client container is not enabled ----------------------------------------------------------------------------------- + compare_kubectl statefulset/monitoring-pmm3-rs0 -no-pmm + local resource=statefulset/monitoring-pmm3-rs0 + local postfix=-no-pmm + local skip_generation_check= + local expected_result=/mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/e2e-tests/monitoring-pmm3/compare/statefulset_monitoring-pmm3-rs0-no-pmm.yml + local new_result=/tmp/tmp.9ONBZDsnOT/statefulset_monitoring-pmm3-rs0.yml + [[ -n '' ]] + kubectl_bin get -o yaml statefulset/monitoring-pmm3-rs0 + yq eval $'\n\t\t\tdel(.metadata.ownerReferences[].apiVersion) |\n\t\t\tdel(.metadata.managedFields) |\n\t\t\tdel(.. | select(has("creationTimestamp")).creationTimestamp) |\n\t\t\tdel(.. | select(has("namespace")).namespace) |\n\t\t\tdel(.. | select(has("uid")).uid) |\n\t\t\tdel(.metadata.resourceVersion) |\n\t\t\tdel(.spec.template.spec.containers[].env[] | select(.name == "NAMESPACE")) |\n\t\t\tdel(.metadata.selfLink) |\n\t\t\tdel(.metadata.annotations."cloud.google.com/neg") |\n\t\t\tdel(.metadata.annotations."field.cattle.io/publicEndpoints") |\n\t\t\tdel(.metadata.annotations."metallb.io/ip-allocated-from-pool") |\n\t\t\tdel(.metadata.annotations."networking.gke.io/target-pool") |\n\t\t\tdel(.metadata.annotations."kubectl.kubernetes.io/last-applied-configuration") |\n\t\t\tdel(.. | select(has("image")).image) |\n\t\t\tdel(.. | select(has("clusterIP")).clusterIP) |\n\t\t\tdel(.. | select(has("clusterIPs")).clusterIPs) |\n\t\t\tdel(.. | select(has("dataSource")).dataSource) |\n\t\t\tdel(.. | select(has("procMount")).procMount) |\n\t\t\tdel(.. | select(has("storageClassName")).storageClassName) |\n\t\t\tdel(.. | select(has("finalizers")).finalizers) |\n\t\t\tdel(.. | select(has("kubernetes.io/pvc-protection"))."kubernetes.io/pvc-protection") |\n\t\t\tdel(.. | select(has("volumeName")).volumeName) |\n\t\t\tdel(.. | select(has("volume.beta.kubernetes.io/storage-provisioner"))."volume.beta.kubernetes.io/storage-provisioner") |\n\t\t\tdel(.. | select(has("volume.kubernetes.io/storage-provisioner"))."volume.kubernetes.io/storage-provisioner") |\n\t\t\tdel(.spec.volumeMode) |\n\t\t\tdel(.. | select(has("volume.kubernetes.io/selected-node"))."volume.kubernetes.io/selected-node") |\n\t\t\tdel(.. | select(has("percona.com/last-config-hash"))."percona.com/last-config-hash") |\n\t\t\tdel(.. | select(has("percona.com/configuration-hash"))."percona.com/configuration-hash") |\n\t\t\tdel(.. | select(has("percona.com/ssl-hash"))."percona.com/ssl-hash") |\n\t\t\tdel(.. | select(has("percona.com/ssl-internal-hash"))."percona.com/ssl-internal-hash") |\n\t\t\tdel(.spec.volumeClaimTemplates[].spec.volumeMode | select(. == "Filesystem")) |\n\t\t\tdel(.. | select(has("healthCheckNodePort")).healthCheckNodePort) |\n\t\t\tdel(.. | select(has("nodePort")).nodePort) |\n\t\t\tdel(.status) |\n\t\t\t(.. | select(tag == "!!str")) |= sub("monitoring-pmm3-14502", "NAME_SPACE") |\n\t\t\tdel(.spec.volumeClaimTemplates[].apiVersion) |\n\t\t\tdel(.spec.volumeClaimTemplates[].kind) |\n\t\t\tdel(.spec.ipFamilies) |\n\t\t\tdel(.spec.ipFamilyPolicy) |\n\t\t\t(.. | select(. == "extensions/v1beta1")) = "apps/v1" |\n\t\t\t(.. | select(. == "batch/v1beta1")) = "batch/v1" ' - ++ mktemp + local LAST_OUT=/tmp/tmp.kQwF80RI2a ++ mktemp + local LAST_ERR=/tmp/tmp.TCAVVF9U30 + local exit_status=0 + local timeout=4 + args=('get' '-o' 'yaml' 'statefulset/monitoring-pmm3-rs0') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl get -o yaml statefulset/monitoring-pmm3-rs0 + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.kQwF80RI2a + cat /tmp/tmp.TCAVVF9U30 + rm /tmp/tmp.kQwF80RI2a /tmp/tmp.TCAVVF9U30 + return 0 + yq -i eval 'del(.spec.persistentVolumeClaimRetentionPolicy)' /tmp/tmp.9ONBZDsnOT/statefulset_monitoring-pmm3-rs0.yml + arch_scheduling_enabled + [[ -n '' ]] + version_gt 1.22 ++ echo '1.33 >= 1.22' ++ bc -l + '[' 1 -eq 1 ']' + return 0 + yq -i eval 'del(.spec.internalTrafficPolicy)' /tmp/tmp.9ONBZDsnOT/statefulset_monitoring-pmm3-rs0.yml + yq -i eval 'del(.spec.allocateLoadBalancerNodePorts)' /tmp/tmp.9ONBZDsnOT/statefulset_monitoring-pmm3-rs0.yml + [[ /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/e2e-tests/monitoring-pmm3/compare/statefulset_monitoring-pmm3-rs0-no-pmm.yml == */cronjob* ]] + '[' -n '' ']' + [[ 0 -eq 0 ]] + diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/e2e-tests/monitoring-pmm3/compare/statefulset_monitoring-pmm3-rs0-no-pmm.yml /tmp/tmp.9ONBZDsnOT/statefulset_monitoring-pmm3-rs0.yml + log 'compare_kubectl: statefulset/monitoring-pmm3-rs0 OK' + set +o xtrace [2026-07-17T01:18:15+0000] compare_kubectl: statefulset/monitoring-pmm3-rs0 OK + sleep 10 + custom_port=27019 + run_mongos 'db.createUser({user:"myApp",pwd:"myPass",roles:[{db:"myApp",role:"readWrite"}]})' userAdmin:userAdmin123456@monitoring-pmm3-mongos.monitoring-pmm3-14502 '' '' '--tlsCertificateKeyFile /tmp/tls.pem --tlsCAFile /etc/mongodb-ssl/ca.crt --tls' 27019 + local 'command=db.createUser({user:"myApp",pwd:"myPass",roles:[{db:"myApp",role:"readWrite"}]})' + local uri=userAdmin:userAdmin123456@monitoring-pmm3-mongos.monitoring-pmm3-14502 + local driver=mongodb + local suffix=.svc.cluster.local + local 'mongo_flag=--tlsCertificateKeyFile /tmp/tls.pem --tlsCAFile /etc/mongodb-ssl/ca.crt --tls' + local port=27019 + local mongo_bin=mongo ++ echo .svc.cluster.local ++ awk -F: '{print $2}' + suffix_port= + [[ -z '' ]] + suffix=.svc.cluster.local:27019 ++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.N2mHVDQ4gv +++ mktemp ++ local LAST_ERR=/tmp/tmp.Rxvc9NG2si ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'pods' '--selector=name=psmdb-client' '-o' 'jsonpath={.items[].metadata.name}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.N2mHVDQ4gv ++ cat /tmp/tmp.Rxvc9NG2si ++ rm /tmp/tmp.N2mHVDQ4gv /tmp/tmp.Rxvc9NG2si ++ return 0 + local client_container=psmdb-client-7d7dc78cc7-wh9f4 + kubectl_bin exec psmdb-client-7d7dc78cc7-wh9f4 -- bash -c 'export HOME=/tmp; printf '\''db.createUser({user:"myApp",pwd:"myPass",roles:[{db:"myApp",role:"readWrite"}]})\n'\'' | mongo mongodb://userAdmin:userAdmin123456@monitoring-pmm3-mongos.monitoring-pmm3-14502.svc.cluster.local:27019/admin --tlsCertificateKeyFile /tmp/tls.pem --tlsCAFile /etc/mongodb-ssl/ca.crt --tls' ++ mktemp + local LAST_OUT=/tmp/tmp.Y48hcqCewv ++ mktemp + local LAST_ERR=/tmp/tmp.VFJLUNlswr + local exit_status=0 + local timeout=4 + args=('exec' 'psmdb-client-7d7dc78cc7-wh9f4' '--' 'bash' '-c' 'export HOME=/tmp; printf '\''db.createUser({user:"myApp",pwd:"myPass",roles:[{db:"myApp",role:"readWrite"}]})\n'\'' | mongo mongodb://userAdmin:userAdmin123456@monitoring-pmm3-mongos.monitoring-pmm3-14502.svc.cluster.local:27019/admin --tlsCertificateKeyFile /tmp/tls.pem --tlsCAFile /etc/mongodb-ssl/ca.crt --tls') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl exec psmdb-client-7d7dc78cc7-wh9f4 -- bash -c 'export HOME=/tmp; printf '\''db.createUser({user:"myApp",pwd:"myPass",roles:[{db:"myApp",role:"readWrite"}]})\n'\'' | mongo mongodb://userAdmin:userAdmin123456@monitoring-pmm3-mongos.monitoring-pmm3-14502.svc.cluster.local:27019/admin --tlsCertificateKeyFile /tmp/tls.pem --tlsCAFile /etc/mongodb-ssl/ca.crt --tls' + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.Y48hcqCewv Percona Server for MongoDB shell version v4.4.29-28 connecting to: mongodb://monitoring-pmm3-mongos.monitoring-pmm3-14502.svc.cluster.local:27019/admin?compressors=disabled&gssapiServiceName=mongodb {"t":{"$date":"2026-07-17T01:18:27.606Z"},"s":"I", "c":"NETWORK", "id":5490002, "ctx":"thread1","msg":"Started a new thread for the timer service"} Implicit session: session { "id" : UUID("02969b6b-bdf6-481f-8b62-86d7e6c2c440") } Percona Server for MongoDB server version: v8.0.26-11 WARNING: shell and server versions do not match Successfully added user: { "user" : "myApp", "roles" : [ { "db" : "myApp", "role" : "readWrite" } ] } bye + cat /tmp/tmp.VFJLUNlswr + rm /tmp/tmp.Y48hcqCewv /tmp/tmp.VFJLUNlswr + return 0 + run_mongos 'sh.enableSharding("myApp")' clusterAdmin:clusterAdmin123456@monitoring-pmm3-mongos.monitoring-pmm3-14502 '' '' '--tlsCertificateKeyFile /tmp/tls.pem --tlsCAFile /etc/mongodb-ssl/ca.crt --tls' 27019 + local 'command=sh.enableSharding("myApp")' + local uri=clusterAdmin:clusterAdmin123456@monitoring-pmm3-mongos.monitoring-pmm3-14502 + local driver=mongodb + local suffix=.svc.cluster.local + local 'mongo_flag=--tlsCertificateKeyFile /tmp/tls.pem --tlsCAFile /etc/mongodb-ssl/ca.crt --tls' + local port=27019 + local mongo_bin=mongo ++ awk -F: '{print $2}' ++ echo .svc.cluster.local + suffix_port= + [[ -z '' ]] + suffix=.svc.cluster.local:27019 ++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.z83jj3Rlxc +++ mktemp ++ local LAST_ERR=/tmp/tmp.vdwEyJqxCz ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'pods' '--selector=name=psmdb-client' '-o' 'jsonpath={.items[].metadata.name}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.z83jj3Rlxc ++ cat /tmp/tmp.vdwEyJqxCz ++ rm /tmp/tmp.z83jj3Rlxc /tmp/tmp.vdwEyJqxCz ++ return 0 + local client_container=psmdb-client-7d7dc78cc7-wh9f4 + kubectl_bin exec psmdb-client-7d7dc78cc7-wh9f4 -- bash -c 'export HOME=/tmp; printf '\''sh.enableSharding("myApp")\n'\'' | mongo mongodb://clusterAdmin:clusterAdmin123456@monitoring-pmm3-mongos.monitoring-pmm3-14502.svc.cluster.local:27019/admin --tlsCertificateKeyFile /tmp/tls.pem --tlsCAFile /etc/mongodb-ssl/ca.crt --tls' ++ mktemp + local LAST_OUT=/tmp/tmp.iR1i90H2CT ++ mktemp + local LAST_ERR=/tmp/tmp.MHzw1nZaRl + local exit_status=0 + local timeout=4 + args=('exec' 'psmdb-client-7d7dc78cc7-wh9f4' '--' 'bash' '-c' 'export HOME=/tmp; printf '\''sh.enableSharding("myApp")\n'\'' | mongo mongodb://clusterAdmin:clusterAdmin123456@monitoring-pmm3-mongos.monitoring-pmm3-14502.svc.cluster.local:27019/admin --tlsCertificateKeyFile /tmp/tls.pem --tlsCAFile /etc/mongodb-ssl/ca.crt --tls') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl exec psmdb-client-7d7dc78cc7-wh9f4 -- bash -c 'export HOME=/tmp; printf '\''sh.enableSharding("myApp")\n'\'' | mongo mongodb://clusterAdmin:clusterAdmin123456@monitoring-pmm3-mongos.monitoring-pmm3-14502.svc.cluster.local:27019/admin --tlsCertificateKeyFile /tmp/tls.pem --tlsCAFile /etc/mongodb-ssl/ca.crt --tls' + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.iR1i90H2CT Percona Server for MongoDB shell version v4.4.29-28 connecting to: mongodb://monitoring-pmm3-mongos.monitoring-pmm3-14502.svc.cluster.local:27019/admin?compressors=disabled&gssapiServiceName=mongodb {"t":{"$date":"2026-07-17T01:18:30.232Z"},"s":"I", "c":"NETWORK", "id":5490002, "ctx":"thread1","msg":"Started a new thread for the timer service"} Implicit session: session { "id" : UUID("dd1f5ea5-4073-42c1-8746-439411637829") } Percona Server for MongoDB server version: v8.0.26-11 WARNING: shell and server versions do not match { "ok" : 1, "$clusterTime" : { "clusterTime" : Timestamp(1784251110, 8), "signature" : { "hash" : BinData(0,"BZPGneSbiTGUQfNTjGW/LcbCRPA="), "keyId" : NumberLong("7663299740099936280") } }, "operationTime" : Timestamp(1784251110, 5) } bye + cat /tmp/tmp.MHzw1nZaRl + rm /tmp/tmp.iR1i90H2CT /tmp/tmp.MHzw1nZaRl + return 0 + insert_data_mongos 100500 myApp '--tlsCertificateKeyFile /tmp/tls.pem --tlsCAFile /etc/mongodb-ssl/ca.crt --tls' 27019 + local data=100500 + local db_name=myApp + local 'flags=--tlsCertificateKeyFile /tmp/tls.pem --tlsCAFile /etc/mongodb-ssl/ca.crt --tls' + local port=27019 + run_mongos 'use myApp\n db.test.insert({ x: 100500 })' myApp:myPass@monitoring-pmm3-mongos.monitoring-pmm3-14502 '' '' '--tlsCertificateKeyFile /tmp/tls.pem --tlsCAFile /etc/mongodb-ssl/ca.crt --tls' 27019 + local 'command=use myApp\n db.test.insert({ x: 100500 })' + local uri=myApp:myPass@monitoring-pmm3-mongos.monitoring-pmm3-14502 + local driver=mongodb + local suffix=.svc.cluster.local + local 'mongo_flag=--tlsCertificateKeyFile /tmp/tls.pem --tlsCAFile /etc/mongodb-ssl/ca.crt --tls' + local port=27019 + local mongo_bin=mongo ++ echo .svc.cluster.local ++ awk -F: '{print $2}' + suffix_port= + [[ -z '' ]] + suffix=.svc.cluster.local:27019 ++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.CztZHLlPoe +++ mktemp ++ local LAST_ERR=/tmp/tmp.16N1kCEYFG ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'pods' '--selector=name=psmdb-client' '-o' 'jsonpath={.items[].metadata.name}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.CztZHLlPoe ++ cat /tmp/tmp.16N1kCEYFG ++ rm /tmp/tmp.CztZHLlPoe /tmp/tmp.16N1kCEYFG ++ return 0 + local client_container=psmdb-client-7d7dc78cc7-wh9f4 + kubectl_bin exec psmdb-client-7d7dc78cc7-wh9f4 -- bash -c 'export HOME=/tmp; printf '\''use myApp\n db.test.insert({ x: 100500 })\n'\'' | mongo mongodb://myApp:myPass@monitoring-pmm3-mongos.monitoring-pmm3-14502.svc.cluster.local:27019/admin --tlsCertificateKeyFile /tmp/tls.pem --tlsCAFile /etc/mongodb-ssl/ca.crt --tls' ++ mktemp + local LAST_OUT=/tmp/tmp.tDs5NlKqk9 ++ mktemp + local LAST_ERR=/tmp/tmp.7N5J6gO80s + local exit_status=0 + local timeout=4 + args=('exec' 'psmdb-client-7d7dc78cc7-wh9f4' '--' 'bash' '-c' 'export HOME=/tmp; printf '\''use myApp\n db.test.insert({ x: 100500 })\n'\'' | mongo mongodb://myApp:myPass@monitoring-pmm3-mongos.monitoring-pmm3-14502.svc.cluster.local:27019/admin --tlsCertificateKeyFile /tmp/tls.pem --tlsCAFile /etc/mongodb-ssl/ca.crt --tls') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl exec psmdb-client-7d7dc78cc7-wh9f4 -- bash -c 'export HOME=/tmp; printf '\''use myApp\n db.test.insert({ x: 100500 })\n'\'' | mongo mongodb://myApp:myPass@monitoring-pmm3-mongos.monitoring-pmm3-14502.svc.cluster.local:27019/admin --tlsCertificateKeyFile /tmp/tls.pem --tlsCAFile /etc/mongodb-ssl/ca.crt --tls' + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.tDs5NlKqk9 Percona Server for MongoDB shell version v4.4.29-28 connecting to: mongodb://monitoring-pmm3-mongos.monitoring-pmm3-14502.svc.cluster.local:27019/admin?compressors=disabled&gssapiServiceName=mongodb {"t":{"$date":"2026-07-17T01:18:32.993Z"},"s":"I", "c":"NETWORK", "id":5490002, "ctx":"thread1","msg":"Started a new thread for the timer service"} Implicit session: session { "id" : UUID("3bdfa02e-22e9-49bd-9953-5041c5beb9ed") } Percona Server for MongoDB server version: v8.0.26-11 WARNING: shell and server versions do not match switched to db myApp WriteResult({ "nInserted" : 1 }) bye + cat /tmp/tmp.7N5J6gO80s + rm /tmp/tmp.tDs5NlKqk9 /tmp/tmp.7N5J6gO80s + return 0 + insert_data_mongos 100600 myApp '--tlsCertificateKeyFile /tmp/tls.pem --tlsCAFile /etc/mongodb-ssl/ca.crt --tls' 27019 + local data=100600 + local db_name=myApp + local 'flags=--tlsCertificateKeyFile /tmp/tls.pem --tlsCAFile /etc/mongodb-ssl/ca.crt --tls' + local port=27019 + run_mongos 'use myApp\n db.test.insert({ x: 100600 })' myApp:myPass@monitoring-pmm3-mongos.monitoring-pmm3-14502 '' '' '--tlsCertificateKeyFile /tmp/tls.pem --tlsCAFile /etc/mongodb-ssl/ca.crt --tls' 27019 + local 'command=use myApp\n db.test.insert({ x: 100600 })' + local uri=myApp:myPass@monitoring-pmm3-mongos.monitoring-pmm3-14502 + local driver=mongodb + local suffix=.svc.cluster.local + local 'mongo_flag=--tlsCertificateKeyFile /tmp/tls.pem --tlsCAFile /etc/mongodb-ssl/ca.crt --tls' + local port=27019 + local mongo_bin=mongo ++ echo .svc.cluster.local ++ awk -F: '{print $2}' + suffix_port= + [[ -z '' ]] + suffix=.svc.cluster.local:27019 ++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.65jWthzhEs +++ mktemp ++ local LAST_ERR=/tmp/tmp.CoDpiA2443 ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'pods' '--selector=name=psmdb-client' '-o' 'jsonpath={.items[].metadata.name}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.65jWthzhEs ++ cat /tmp/tmp.CoDpiA2443 ++ rm /tmp/tmp.65jWthzhEs /tmp/tmp.CoDpiA2443 ++ return 0 + local client_container=psmdb-client-7d7dc78cc7-wh9f4 + kubectl_bin exec psmdb-client-7d7dc78cc7-wh9f4 -- bash -c 'export HOME=/tmp; printf '\''use myApp\n db.test.insert({ x: 100600 })\n'\'' | mongo mongodb://myApp:myPass@monitoring-pmm3-mongos.monitoring-pmm3-14502.svc.cluster.local:27019/admin --tlsCertificateKeyFile /tmp/tls.pem --tlsCAFile /etc/mongodb-ssl/ca.crt --tls' ++ mktemp + local LAST_OUT=/tmp/tmp.ebO5C3SwhZ ++ mktemp + local LAST_ERR=/tmp/tmp.vvttV3zYUg + local exit_status=0 + local timeout=4 + args=('exec' 'psmdb-client-7d7dc78cc7-wh9f4' '--' 'bash' '-c' 'export HOME=/tmp; printf '\''use myApp\n db.test.insert({ x: 100600 })\n'\'' | mongo mongodb://myApp:myPass@monitoring-pmm3-mongos.monitoring-pmm3-14502.svc.cluster.local:27019/admin --tlsCertificateKeyFile /tmp/tls.pem --tlsCAFile /etc/mongodb-ssl/ca.crt --tls') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl exec psmdb-client-7d7dc78cc7-wh9f4 -- bash -c 'export HOME=/tmp; printf '\''use myApp\n db.test.insert({ x: 100600 })\n'\'' | mongo mongodb://myApp:myPass@monitoring-pmm3-mongos.monitoring-pmm3-14502.svc.cluster.local:27019/admin --tlsCertificateKeyFile /tmp/tls.pem --tlsCAFile /etc/mongodb-ssl/ca.crt --tls' + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.ebO5C3SwhZ Percona Server for MongoDB shell version v4.4.29-28 connecting to: mongodb://monitoring-pmm3-mongos.monitoring-pmm3-14502.svc.cluster.local:27019/admin?compressors=disabled&gssapiServiceName=mongodb {"t":{"$date":"2026-07-17T01:18:35.710Z"},"s":"I", "c":"NETWORK", "id":5490002, "ctx":"thread1","msg":"Started a new thread for the timer service"} Implicit session: session { "id" : UUID("b6788f11-c7ca-4aa3-bd4c-303d9062fac9") } Percona Server for MongoDB server version: v8.0.26-11 WARNING: shell and server versions do not match switched to db myApp WriteResult({ "nInserted" : 1 }) bye + cat /tmp/tmp.vvttV3zYUg + rm /tmp/tmp.ebO5C3SwhZ /tmp/tmp.vvttV3zYUg + return 0 + insert_data_mongos 100700 myApp '--tlsCertificateKeyFile /tmp/tls.pem --tlsCAFile /etc/mongodb-ssl/ca.crt --tls' 27019 + local data=100700 + local db_name=myApp + local 'flags=--tlsCertificateKeyFile /tmp/tls.pem --tlsCAFile /etc/mongodb-ssl/ca.crt --tls' + local port=27019 + run_mongos 'use myApp\n db.test.insert({ x: 100700 })' myApp:myPass@monitoring-pmm3-mongos.monitoring-pmm3-14502 '' '' '--tlsCertificateKeyFile /tmp/tls.pem --tlsCAFile /etc/mongodb-ssl/ca.crt --tls' 27019 + local 'command=use myApp\n db.test.insert({ x: 100700 })' + local uri=myApp:myPass@monitoring-pmm3-mongos.monitoring-pmm3-14502 + local driver=mongodb + local suffix=.svc.cluster.local + local 'mongo_flag=--tlsCertificateKeyFile /tmp/tls.pem --tlsCAFile /etc/mongodb-ssl/ca.crt --tls' + local port=27019 + local mongo_bin=mongo ++ echo .svc.cluster.local ++ awk -F: '{print $2}' + suffix_port= + [[ -z '' ]] + suffix=.svc.cluster.local:27019 ++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.VGWwUixXVk +++ mktemp ++ local LAST_ERR=/tmp/tmp.QaS2bpVuEz ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'pods' '--selector=name=psmdb-client' '-o' 'jsonpath={.items[].metadata.name}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.VGWwUixXVk ++ cat /tmp/tmp.QaS2bpVuEz ++ rm /tmp/tmp.VGWwUixXVk /tmp/tmp.QaS2bpVuEz ++ return 0 + local client_container=psmdb-client-7d7dc78cc7-wh9f4 + kubectl_bin exec psmdb-client-7d7dc78cc7-wh9f4 -- bash -c 'export HOME=/tmp; printf '\''use myApp\n db.test.insert({ x: 100700 })\n'\'' | mongo mongodb://myApp:myPass@monitoring-pmm3-mongos.monitoring-pmm3-14502.svc.cluster.local:27019/admin --tlsCertificateKeyFile /tmp/tls.pem --tlsCAFile /etc/mongodb-ssl/ca.crt --tls' ++ mktemp + local LAST_OUT=/tmp/tmp.7C1PeB4ejT ++ mktemp + local LAST_ERR=/tmp/tmp.GjqN4Uf00q + local exit_status=0 + local timeout=4 + args=('exec' 'psmdb-client-7d7dc78cc7-wh9f4' '--' 'bash' '-c' 'export HOME=/tmp; printf '\''use myApp\n db.test.insert({ x: 100700 })\n'\'' | mongo mongodb://myApp:myPass@monitoring-pmm3-mongos.monitoring-pmm3-14502.svc.cluster.local:27019/admin --tlsCertificateKeyFile /tmp/tls.pem --tlsCAFile /etc/mongodb-ssl/ca.crt --tls') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl exec psmdb-client-7d7dc78cc7-wh9f4 -- bash -c 'export HOME=/tmp; printf '\''use myApp\n db.test.insert({ x: 100700 })\n'\'' | mongo mongodb://myApp:myPass@monitoring-pmm3-mongos.monitoring-pmm3-14502.svc.cluster.local:27019/admin --tlsCertificateKeyFile /tmp/tls.pem --tlsCAFile /etc/mongodb-ssl/ca.crt --tls' + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.7C1PeB4ejT Percona Server for MongoDB shell version v4.4.29-28 connecting to: mongodb://monitoring-pmm3-mongos.monitoring-pmm3-14502.svc.cluster.local:27019/admin?compressors=disabled&gssapiServiceName=mongodb {"t":{"$date":"2026-07-17T01:18:38.371Z"},"s":"I", "c":"NETWORK", "id":5490002, "ctx":"thread1","msg":"Started a new thread for the timer service"} Implicit session: session { "id" : UUID("88051019-d69a-4cee-87ce-979771d71bd6") } Percona Server for MongoDB server version: v8.0.26-11 WARNING: shell and server versions do not match switched to db myApp WriteResult({ "nInserted" : 1 }) bye + cat /tmp/tmp.GjqN4Uf00q + rm /tmp/tmp.7C1PeB4ejT /tmp/tmp.GjqN4Uf00q + return 0 + desc 'add PMM3 token to secret' + set +o xtrace ----------------------------------------------------------------------------------- add PMM3 token to secret ----------------------------------------------------------------------------------- ++ get_pmm_server_token operator ++ local key_name=operator ++ [[ -z operator ]] ++ local ADMIN_PASSWORD +++ kubectl get secret pmm-secret -o 'jsonpath={.data.PMM_ADMIN_PASSWORD}' +++ base64 --decode ++ ADMIN_PASSWORD='J:k%i1RN=xa^Z5a!' ++ [[ -z J:k%i1RN=xa^Z5a! ]] ++ local endpoint encoded_user_credentials +++ get_service_endpoint monitoring-service +++ local service=monitoring-service +++ local endpoint +++ local service_type ++++ kubectl_bin get service/monitoring-service -o 'jsonpath={.spec.type}' +++++ mktemp ++++ local LAST_OUT=/tmp/tmp.aeoLu3mnbv +++++ mktemp ++++ local LAST_ERR=/tmp/tmp.wDaSX6C5jR ++++ local exit_status=0 ++++ local timeout=4 ++++ args=('get' 'service/monitoring-service' '-o' 'jsonpath={.spec.type}') ++++ local args ++++ transformed_files=() ++++ local transformed_files ++++ arch_scheduling_enabled ++++ [[ -n '' ]] ++++ arch_scheduling_enabled ++++ [[ -n '' ]] +++++ seq 0 2 ++++ for i in $(seq 0 2) ++++ set +e ++++ kubectl get service/monitoring-service -o 'jsonpath={.spec.type}' ++++ exit_status=0 ++++ set -e ++++ '[' 0 '!=' 0 -a -n 1 ']' ++++ break ++++ cat /tmp/tmp.aeoLu3mnbv ++++ cat /tmp/tmp.wDaSX6C5jR ++++ rm /tmp/tmp.aeoLu3mnbv /tmp/tmp.wDaSX6C5jR ++++ return 0 +++ service_type=LoadBalancer +++ [[ LoadBalancer == LoadBalancer ]] ++++ get_load_balancer_endpoint monitoring-service ++++ local service=monitoring-service ++++ local endpoint +++++ kubectl_bin get service/monitoring-service -o json +++++ jq -r '[.status.loadBalancer.ingress[]?.hostname, .status.loadBalancer.ingress[]?.ip] | map(select(. != null and . != "")) | first // empty' ++++++ mktemp +++++ local LAST_OUT=/tmp/tmp.6YW9YWnDci ++++++ mktemp +++++ local LAST_ERR=/tmp/tmp.qU1oCTu0Kf +++++ local exit_status=0 +++++ local timeout=4 +++++ args=('get' 'service/monitoring-service' '-o' 'json') +++++ local args +++++ transformed_files=() +++++ local transformed_files +++++ arch_scheduling_enabled +++++ [[ -n '' ]] +++++ arch_scheduling_enabled +++++ [[ -n '' ]] ++++++ seq 0 2 +++++ for i in $(seq 0 2) +++++ set +e +++++ kubectl get service/monitoring-service -o json +++++ exit_status=0 +++++ set -e +++++ '[' 0 '!=' 0 -a -n 1 ']' +++++ break +++++ cat /tmp/tmp.6YW9YWnDci +++++ cat /tmp/tmp.qU1oCTu0Kf +++++ rm /tmp/tmp.6YW9YWnDci /tmp/tmp.qU1oCTu0Kf +++++ return 0 ++++ endpoint=136.112.205.251 ++++ '[' -n 136.112.205.251 -a 136.112.205.251 '!=' null ']' ++++ echo 136.112.205.251 ++++ return +++ endpoint=136.112.205.251 +++ echo 136.112.205.251 +++ return ++ endpoint=136.112.205.251 +++ urlencode_userinfo 'admin:J:k%i1RN=xa^Z5a!' +++ local 'user_pass=admin:J:k%i1RN=xa^Z5a!' +++ local username=admin +++ local 'password=J:k%i1RN=xa^Z5a!' ++++ urlencode admin ++++ uri=admin ++++ echo -n admin ++++ jq -s -R -r @uri ++++ urlencode 'J:k%i1RN=xa^Z5a!' ++++ uri='J:k%i1RN=xa^Z5a!' ++++ echo -n 'J:k%i1RN=xa^Z5a!' ++++ jq -s -R -r @uri +++ printf %s:%s admin J%3Ak%25i1RN%3Dxa%5EZ5a%21 ++ encoded_user_credentials=admin:J%3Ak%25i1RN%3Dxa%5EZ5a%21 ++ local create_response create_status_code create_json_response +++ execute_curl --insecure -s -X POST -H 'Content-Type: application/json' -H 'Accept: application/json' -d '{"name":"operator", "role":"Admin", "isDisabled":false}' https://admin:J%3Ak%25i1RN%3Dxa%5EZ5a%21@136.112.205.251/graph/api/serviceaccounts -w '\n%{http_code}' +++ local name=curl-20355 +++ local image=docker.io/alpine/curl:latest +++ local phase= +++ local exit_status=0 +++ kubectl_bin run curl-20355 -n monitoring-pmm3-14502 --restart=Never --image=docker.io/alpine/curl:latest --command -- curl -sS --insecure -s -X POST -H 'Content-Type: application/json' -H 'Accept: application/json' -d '{"name":"operator", "role":"Admin", "isDisabled":false}' https://admin:J%3Ak%25i1RN%3Dxa%5EZ5a%21@136.112.205.251/graph/api/serviceaccounts -w '\n%{http_code}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.zwoEGufg9M ++++ mktemp +++ local LAST_ERR=/tmp/tmp.pdzZPhc0lX +++ local exit_status=0 +++ local timeout=4 +++ args=('run' 'curl-20355' '-n' 'monitoring-pmm3-14502' '--restart=Never' '--image=docker.io/alpine/curl:latest' '--command' '--' 'curl' '-sS' '--insecure' '-s' '-X' 'POST' '-H' 'Content-Type: application/json' '-H' 'Accept: application/json' '-d' '{"name":"operator", "role":"Admin", "isDisabled":false}' 'https://admin:J%3Ak%25i1RN%3Dxa%5EZ5a%21@136.112.205.251/graph/api/serviceaccounts' '-w' '\n%{http_code}') +++ local args +++ transformed_files=() +++ local transformed_files +++ arch_scheduling_enabled +++ [[ -n '' ]] +++ arch_scheduling_enabled +++ [[ -n '' ]] ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl run curl-20355 -n monitoring-pmm3-14502 --restart=Never --image=docker.io/alpine/curl:latest --command -- curl -sS --insecure -s -X POST -H 'Content-Type: application/json' -H 'Accept: application/json' -d '{"name":"operator", "role":"Admin", "isDisabled":false}' https://admin:J%3Ak%25i1RN%3Dxa%5EZ5a%21@136.112.205.251/graph/api/serviceaccounts -w '\n%{http_code}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 -a -n 1 ']' +++ break +++ cat /tmp/tmp.zwoEGufg9M +++ cat /tmp/tmp.pdzZPhc0lX +++ rm /tmp/tmp.zwoEGufg9M /tmp/tmp.pdzZPhc0lX +++ return 0 ++++ seq 1 60 +++ for _ in $(seq 1 60) ++++ kubectl_bin get pod curl-20355 -n monitoring-pmm3-14502 -o 'jsonpath={.status.phase}' +++ phase=Pending +++ [[ Pending == Succeeded ]] +++ [[ Pending == Failed ]] +++ sleep 1 +++ for _ in $(seq 1 60) ++++ kubectl_bin get pod curl-20355 -n monitoring-pmm3-14502 -o 'jsonpath={.status.phase}' +++ phase=Running +++ [[ Running == Succeeded ]] +++ [[ Running == Failed ]] +++ sleep 1 +++ for _ in $(seq 1 60) ++++ kubectl_bin get pod curl-20355 -n monitoring-pmm3-14502 -o 'jsonpath={.status.phase}' +++ phase=Succeeded +++ [[ Succeeded == Succeeded ]] +++ break +++ kubectl_bin logs curl-20355 -n monitoring-pmm3-14502 ++++ mktemp +++ local LAST_OUT=/tmp/tmp.a47Uf2P6Mb ++++ mktemp +++ local LAST_ERR=/tmp/tmp.q2SACuOk7A +++ local exit_status=0 +++ local timeout=4 +++ args=('logs' 'curl-20355' '-n' 'monitoring-pmm3-14502') +++ local args +++ transformed_files=() +++ local transformed_files +++ arch_scheduling_enabled +++ [[ -n '' ]] +++ arch_scheduling_enabled +++ [[ -n '' ]] ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl logs curl-20355 -n monitoring-pmm3-14502 +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 -a -n 1 ']' +++ break +++ cat /tmp/tmp.a47Uf2P6Mb +++ cat /tmp/tmp.q2SACuOk7A +++ rm /tmp/tmp.a47Uf2P6Mb /tmp/tmp.q2SACuOk7A +++ return 0 +++ kubectl_bin delete pod curl-20355 -n monitoring-pmm3-14502 --ignore-not-found --wait=false +++ [[ Succeeded != Succeeded ]] +++ return 0 ++ create_response=$'{"id":2,"uid":"afsc39pmk0a9sb","name":"operator","login":"sa-1-operator","orgId":1,"isDisabled":false,"role":"Admin","tokens":0,"avatarUrl":""}\n201' +++ echo $'{"id":2,"uid":"afsc39pmk0a9sb","name":"operator","login":"sa-1-operator","orgId":1,"isDisabled":false,"role":"Admin","tokens":0,"avatarUrl":""}\n201' +++ tail -n1 ++ create_status_code=201 +++ echo $'{"id":2,"uid":"afsc39pmk0a9sb","name":"operator","login":"sa-1-operator","orgId":1,"isDisabled":false,"role":"Admin","tokens":0,"avatarUrl":""}\n201' +++ sed '$ d' ++ create_json_response='{"id":2,"uid":"afsc39pmk0a9sb","name":"operator","login":"sa-1-operator","orgId":1,"isDisabled":false,"role":"Admin","tokens":0,"avatarUrl":""}' ++ [[ 201 -ne 201 ]] ++ local service_account_id +++ echo '{"id":2,"uid":"afsc39pmk0a9sb","name":"operator","login":"sa-1-operator","orgId":1,"isDisabled":false,"role":"Admin","tokens":0,"avatarUrl":""}' +++ jq -r .id ++ service_account_id=2 ++ [[ -z 2 ]] ++ [[ 2 == null ]] ++ local token_response token_status_code token_json_response +++ execute_curl --insecure -s -X POST -H 'Content-Type: application/json' -d '{"name":"operator"}' https://admin:J%3Ak%25i1RN%3Dxa%5EZ5a%21@136.112.205.251/graph/api/serviceaccounts/2/tokens -w '\n%{http_code}' +++ local name=curl-6824 +++ local image=docker.io/alpine/curl:latest +++ local phase= +++ local exit_status=0 +++ kubectl_bin run curl-6824 -n monitoring-pmm3-14502 --restart=Never --image=docker.io/alpine/curl:latest --command -- curl -sS --insecure -s -X POST -H 'Content-Type: application/json' -d '{"name":"operator"}' https://admin:J%3Ak%25i1RN%3Dxa%5EZ5a%21@136.112.205.251/graph/api/serviceaccounts/2/tokens -w '\n%{http_code}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.dqOAbVM7i3 ++++ mktemp +++ local LAST_ERR=/tmp/tmp.yUL8tmU7Qn +++ local exit_status=0 +++ local timeout=4 +++ args=('run' 'curl-6824' '-n' 'monitoring-pmm3-14502' '--restart=Never' '--image=docker.io/alpine/curl:latest' '--command' '--' 'curl' '-sS' '--insecure' '-s' '-X' 'POST' '-H' 'Content-Type: application/json' '-d' '{"name":"operator"}' 'https://admin:J%3Ak%25i1RN%3Dxa%5EZ5a%21@136.112.205.251/graph/api/serviceaccounts/2/tokens' '-w' '\n%{http_code}') +++ local args +++ transformed_files=() +++ local transformed_files +++ arch_scheduling_enabled +++ [[ -n '' ]] +++ arch_scheduling_enabled +++ [[ -n '' ]] ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl run curl-6824 -n monitoring-pmm3-14502 --restart=Never --image=docker.io/alpine/curl:latest --command -- curl -sS --insecure -s -X POST -H 'Content-Type: application/json' -d '{"name":"operator"}' https://admin:J%3Ak%25i1RN%3Dxa%5EZ5a%21@136.112.205.251/graph/api/serviceaccounts/2/tokens -w '\n%{http_code}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 -a -n 1 ']' +++ break +++ cat /tmp/tmp.dqOAbVM7i3 +++ cat /tmp/tmp.yUL8tmU7Qn +++ rm /tmp/tmp.dqOAbVM7i3 /tmp/tmp.yUL8tmU7Qn +++ return 0 ++++ seq 1 60 +++ for _ in $(seq 1 60) ++++ kubectl_bin get pod curl-6824 -n monitoring-pmm3-14502 -o 'jsonpath={.status.phase}' +++ phase=Pending +++ [[ Pending == Succeeded ]] +++ [[ Pending == Failed ]] +++ sleep 1 +++ for _ in $(seq 1 60) ++++ kubectl_bin get pod curl-6824 -n monitoring-pmm3-14502 -o 'jsonpath={.status.phase}' +++ phase=Succeeded +++ [[ Succeeded == Succeeded ]] +++ break +++ kubectl_bin logs curl-6824 -n monitoring-pmm3-14502 ++++ mktemp +++ local LAST_OUT=/tmp/tmp.sNSnIIiZkl ++++ mktemp +++ local LAST_ERR=/tmp/tmp.h7ihneo8S8 +++ local exit_status=0 +++ local timeout=4 +++ args=('logs' 'curl-6824' '-n' 'monitoring-pmm3-14502') +++ local args +++ transformed_files=() +++ local transformed_files +++ arch_scheduling_enabled +++ [[ -n '' ]] +++ arch_scheduling_enabled +++ [[ -n '' ]] ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl logs curl-6824 -n monitoring-pmm3-14502 +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 -a -n 1 ']' +++ break +++ cat /tmp/tmp.sNSnIIiZkl +++ cat /tmp/tmp.h7ihneo8S8 +++ rm /tmp/tmp.sNSnIIiZkl /tmp/tmp.h7ihneo8S8 +++ return 0 +++ kubectl_bin delete pod curl-6824 -n monitoring-pmm3-14502 --ignore-not-found --wait=false +++ [[ Succeeded != Succeeded ]] +++ return 0 ++ token_response=$'{"id":1,"name":"operator","key":"glsa_uDL6JtZHlEes6NSH2mybvtSnR0ngIwcl_b4868b35"}\n200' +++ echo $'{"id":1,"name":"operator","key":"glsa_uDL6JtZHlEes6NSH2mybvtSnR0ngIwcl_b4868b35"}\n200' +++ tail -n1 ++ token_status_code=200 +++ echo $'{"id":1,"name":"operator","key":"glsa_uDL6JtZHlEes6NSH2mybvtSnR0ngIwcl_b4868b35"}\n200' +++ sed '$ d' ++ token_json_response='{"id":1,"name":"operator","key":"glsa_uDL6JtZHlEes6NSH2mybvtSnR0ngIwcl_b4868b35"}' ++ [[ 200 -ne 200 ]] ++ echo '{"id":1,"name":"operator","key":"glsa_uDL6JtZHlEes6NSH2mybvtSnR0ngIwcl_b4868b35"}' ++ jq -r .key + TOKEN=glsa_uDL6JtZHlEes6NSH2mybvtSnR0ngIwcl_b4868b35 + kubectl_bin patch secret some-users --type merge --patch '{"stringData": {"PMM_SERVER_TOKEN": "glsa_uDL6JtZHlEes6NSH2mybvtSnR0ngIwcl_b4868b35"}}' ++ mktemp + local LAST_OUT=/tmp/tmp.RCmHp1qrxv ++ mktemp + local LAST_ERR=/tmp/tmp.99hhf4Sn1R + local exit_status=0 + local timeout=4 + args=('patch' 'secret' 'some-users' '--type' 'merge' '--patch' '{"stringData": {"PMM_SERVER_TOKEN": "glsa_uDL6JtZHlEes6NSH2mybvtSnR0ngIwcl_b4868b35"}}') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl patch secret some-users --type merge --patch '{"stringData": {"PMM_SERVER_TOKEN": "glsa_uDL6JtZHlEes6NSH2mybvtSnR0ngIwcl_b4868b35"}}' + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.RCmHp1qrxv secret/some-users patched + cat /tmp/tmp.99hhf4Sn1R + rm /tmp/tmp.RCmHp1qrxv /tmp/tmp.99hhf4Sn1R + return 0 + desc 'check if all 3 Pods started' + set +o xtrace ----------------------------------------------------------------------------------- check if all 3 Pods started ----------------------------------------------------------------------------------- + wait_for_running monitoring-pmm3-rs0 3 + local name=monitoring-pmm3-rs0 + let last_pod=2 + local check_cluster_readiness=true + set_debug + [[ 1 == 1 ]] + set -o xtrace + local rs_name=rs0 + local cluster_name=monitoring-pmm3 ++ seq 0 2 + for i in $(seq 0 $last_pod) + [[ 0 -eq 2 ]] + wait_pod monitoring-pmm3-rs0-0 + local pod=monitoring-pmm3-rs0-0 + set +o xtrace waiting for pod/monitoring-pmm3-rs0-0 to be ready.OK + for i in $(seq 0 $last_pod) + [[ 1 -eq 2 ]] + wait_pod monitoring-pmm3-rs0-1 + local pod=monitoring-pmm3-rs0-1 + set +o xtrace waiting for pod/monitoring-pmm3-rs0-1 to be ready.OK + for i in $(seq 0 $last_pod) + [[ 2 -eq 2 ]] ++ kubectl_bin get psmdb monitoring-pmm3 -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].arbiter.enabled}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.Rtxj9mKgeR +++ mktemp ++ local LAST_ERR=/tmp/tmp.wDxYhC89fb ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'psmdb' 'monitoring-pmm3' '-o' 'jsonpath={.spec.replsets[?(@.name=="rs0")].arbiter.enabled}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb monitoring-pmm3 -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].arbiter.enabled}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.Rtxj9mKgeR ++ cat /tmp/tmp.wDxYhC89fb ++ rm /tmp/tmp.Rtxj9mKgeR /tmp/tmp.wDxYhC89fb ++ return 0 + [[ '' == true ]] + wait_pod monitoring-pmm3-rs0-2 + local pod=monitoring-pmm3-rs0-2 + set +o xtrace waiting for pod/monitoring-pmm3-rs0-2 to be ready.OK ++ kubectl_bin get psmdb monitoring-pmm3 -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].nonvoting.enabled}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.nA67Yl4WnD +++ mktemp ++ local LAST_ERR=/tmp/tmp.exnQ5zZWrF ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'psmdb' 'monitoring-pmm3' '-o' 'jsonpath={.spec.replsets[?(@.name=="rs0")].nonvoting.enabled}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb monitoring-pmm3 -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].nonvoting.enabled}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.nA67Yl4WnD ++ cat /tmp/tmp.exnQ5zZWrF ++ rm /tmp/tmp.nA67Yl4WnD /tmp/tmp.exnQ5zZWrF ++ return 0 + [[ '' == true ]] ++ kubectl_bin get psmdb monitoring-pmm3 -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].hidden.enabled}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.J7qjAQVM5L +++ mktemp ++ local LAST_ERR=/tmp/tmp.O0Q3oUWB57 ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'psmdb' 'monitoring-pmm3' '-o' 'jsonpath={.spec.replsets[?(@.name=="rs0")].hidden.enabled}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb monitoring-pmm3 -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].hidden.enabled}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.J7qjAQVM5L ++ cat /tmp/tmp.O0Q3oUWB57 ++ rm /tmp/tmp.J7qjAQVM5L /tmp/tmp.O0Q3oUWB57 ++ return 0 + [[ '' == true ]] + sleep 10 + [[ true == true ]] + set +x Waiting for cluster monitoring-pmm3 to be ready......................................................................................................................................................... + sleep 90 + desc 'check if pmm-client container enabled' + set +o xtrace ----------------------------------------------------------------------------------- check if pmm-client container enabled ----------------------------------------------------------------------------------- + compare_kubectl statefulset/monitoring-pmm3-rs0 + local resource=statefulset/monitoring-pmm3-rs0 + local postfix= + local skip_generation_check= + local expected_result=/mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/e2e-tests/monitoring-pmm3/compare/statefulset_monitoring-pmm3-rs0.yml + local new_result=/tmp/tmp.9ONBZDsnOT/statefulset_monitoring-pmm3-rs0.yml + [[ -n '' ]] + kubectl_bin get -o yaml statefulset/monitoring-pmm3-rs0 + yq eval $'\n\t\t\tdel(.metadata.ownerReferences[].apiVersion) |\n\t\t\tdel(.metadata.managedFields) |\n\t\t\tdel(.. | select(has("creationTimestamp")).creationTimestamp) |\n\t\t\tdel(.. | select(has("namespace")).namespace) |\n\t\t\tdel(.. | select(has("uid")).uid) |\n\t\t\tdel(.metadata.resourceVersion) |\n\t\t\tdel(.spec.template.spec.containers[].env[] | select(.name == "NAMESPACE")) |\n\t\t\tdel(.metadata.selfLink) |\n\t\t\tdel(.metadata.annotations."cloud.google.com/neg") |\n\t\t\tdel(.metadata.annotations."field.cattle.io/publicEndpoints") |\n\t\t\tdel(.metadata.annotations."metallb.io/ip-allocated-from-pool") |\n\t\t\tdel(.metadata.annotations."networking.gke.io/target-pool") |\n\t\t\tdel(.metadata.annotations."kubectl.kubernetes.io/last-applied-configuration") |\n\t\t\tdel(.. | select(has("image")).image) |\n\t\t\tdel(.. | select(has("clusterIP")).clusterIP) |\n\t\t\tdel(.. | select(has("clusterIPs")).clusterIPs) |\n\t\t\tdel(.. | select(has("dataSource")).dataSource) |\n\t\t\tdel(.. | select(has("procMount")).procMount) |\n\t\t\tdel(.. | select(has("storageClassName")).storageClassName) |\n\t\t\tdel(.. | select(has("finalizers")).finalizers) |\n\t\t\tdel(.. | select(has("kubernetes.io/pvc-protection"))."kubernetes.io/pvc-protection") |\n\t\t\tdel(.. | select(has("volumeName")).volumeName) |\n\t\t\tdel(.. | select(has("volume.beta.kubernetes.io/storage-provisioner"))."volume.beta.kubernetes.io/storage-provisioner") |\n\t\t\tdel(.. | select(has("volume.kubernetes.io/storage-provisioner"))."volume.kubernetes.io/storage-provisioner") |\n\t\t\tdel(.spec.volumeMode) |\n\t\t\tdel(.. | select(has("volume.kubernetes.io/selected-node"))."volume.kubernetes.io/selected-node") |\n\t\t\tdel(.. | select(has("percona.com/last-config-hash"))."percona.com/last-config-hash") |\n\t\t\tdel(.. | select(has("percona.com/configuration-hash"))."percona.com/configuration-hash") |\n\t\t\tdel(.. | select(has("percona.com/ssl-hash"))."percona.com/ssl-hash") |\n\t\t\tdel(.. | select(has("percona.com/ssl-internal-hash"))."percona.com/ssl-internal-hash") |\n\t\t\tdel(.spec.volumeClaimTemplates[].spec.volumeMode | select(. == "Filesystem")) |\n\t\t\tdel(.. | select(has("healthCheckNodePort")).healthCheckNodePort) |\n\t\t\tdel(.. | select(has("nodePort")).nodePort) |\n\t\t\tdel(.status) |\n\t\t\t(.. | select(tag == "!!str")) |= sub("monitoring-pmm3-14502", "NAME_SPACE") |\n\t\t\tdel(.spec.volumeClaimTemplates[].apiVersion) |\n\t\t\tdel(.spec.volumeClaimTemplates[].kind) |\n\t\t\tdel(.spec.ipFamilies) |\n\t\t\tdel(.spec.ipFamilyPolicy) |\n\t\t\t(.. | select(. == "extensions/v1beta1")) = "apps/v1" |\n\t\t\t(.. | select(. == "batch/v1beta1")) = "batch/v1" ' - ++ mktemp + local LAST_OUT=/tmp/tmp.P8IRvZd2Qe ++ mktemp + local LAST_ERR=/tmp/tmp.Y1dGsbyS7s + local exit_status=0 + local timeout=4 + args=('get' '-o' 'yaml' 'statefulset/monitoring-pmm3-rs0') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl get -o yaml statefulset/monitoring-pmm3-rs0 + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.P8IRvZd2Qe + cat /tmp/tmp.Y1dGsbyS7s + rm /tmp/tmp.P8IRvZd2Qe /tmp/tmp.Y1dGsbyS7s + return 0 + yq -i eval 'del(.spec.persistentVolumeClaimRetentionPolicy)' /tmp/tmp.9ONBZDsnOT/statefulset_monitoring-pmm3-rs0.yml + arch_scheduling_enabled + [[ -n '' ]] + version_gt 1.22 ++ echo '1.33 >= 1.22' ++ bc -l + '[' 1 -eq 1 ']' + return 0 + yq -i eval 'del(.spec.internalTrafficPolicy)' /tmp/tmp.9ONBZDsnOT/statefulset_monitoring-pmm3-rs0.yml + yq -i eval 'del(.spec.allocateLoadBalancerNodePorts)' /tmp/tmp.9ONBZDsnOT/statefulset_monitoring-pmm3-rs0.yml + [[ /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/e2e-tests/monitoring-pmm3/compare/statefulset_monitoring-pmm3-rs0.yml == */cronjob* ]] + '[' -n '' ']' + [[ 0 -eq 0 ]] + diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/e2e-tests/monitoring-pmm3/compare/statefulset_monitoring-pmm3-rs0.yml /tmp/tmp.9ONBZDsnOT/statefulset_monitoring-pmm3-rs0.yml + log 'compare_kubectl: statefulset/monitoring-pmm3-rs0 OK' + set +o xtrace [2026-07-17T01:25:53+0000] compare_kubectl: statefulset/monitoring-pmm3-rs0 OK + compare_kubectl service/monitoring-pmm3-rs0 + local resource=service/monitoring-pmm3-rs0 + local postfix= + local skip_generation_check= + local expected_result=/mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/e2e-tests/monitoring-pmm3/compare/service_monitoring-pmm3-rs0.yml + local new_result=/tmp/tmp.9ONBZDsnOT/service_monitoring-pmm3-rs0.yml + [[ -n '' ]] + kubectl_bin get -o yaml service/monitoring-pmm3-rs0 + yq eval $'\n\t\t\tdel(.metadata.ownerReferences[].apiVersion) |\n\t\t\tdel(.metadata.managedFields) |\n\t\t\tdel(.. | select(has("creationTimestamp")).creationTimestamp) |\n\t\t\tdel(.. | select(has("namespace")).namespace) |\n\t\t\tdel(.. | select(has("uid")).uid) |\n\t\t\tdel(.metadata.resourceVersion) |\n\t\t\tdel(.spec.template.spec.containers[].env[] | select(.name == "NAMESPACE")) |\n\t\t\tdel(.metadata.selfLink) |\n\t\t\tdel(.metadata.annotations."cloud.google.com/neg") |\n\t\t\tdel(.metadata.annotations."field.cattle.io/publicEndpoints") |\n\t\t\tdel(.metadata.annotations."metallb.io/ip-allocated-from-pool") |\n\t\t\tdel(.metadata.annotations."networking.gke.io/target-pool") |\n\t\t\tdel(.metadata.annotations."kubectl.kubernetes.io/last-applied-configuration") |\n\t\t\tdel(.. | select(has("image")).image) |\n\t\t\tdel(.. | select(has("clusterIP")).clusterIP) |\n\t\t\tdel(.. | select(has("clusterIPs")).clusterIPs) |\n\t\t\tdel(.. | select(has("dataSource")).dataSource) |\n\t\t\tdel(.. | select(has("procMount")).procMount) |\n\t\t\tdel(.. | select(has("storageClassName")).storageClassName) |\n\t\t\tdel(.. | select(has("finalizers")).finalizers) |\n\t\t\tdel(.. | select(has("kubernetes.io/pvc-protection"))."kubernetes.io/pvc-protection") |\n\t\t\tdel(.. | select(has("volumeName")).volumeName) |\n\t\t\tdel(.. | select(has("volume.beta.kubernetes.io/storage-provisioner"))."volume.beta.kubernetes.io/storage-provisioner") |\n\t\t\tdel(.. | select(has("volume.kubernetes.io/storage-provisioner"))."volume.kubernetes.io/storage-provisioner") |\n\t\t\tdel(.spec.volumeMode) |\n\t\t\tdel(.. | select(has("volume.kubernetes.io/selected-node"))."volume.kubernetes.io/selected-node") |\n\t\t\tdel(.. | select(has("percona.com/last-config-hash"))."percona.com/last-config-hash") |\n\t\t\tdel(.. | select(has("percona.com/configuration-hash"))."percona.com/configuration-hash") |\n\t\t\tdel(.. | select(has("percona.com/ssl-hash"))."percona.com/ssl-hash") |\n\t\t\tdel(.. | select(has("percona.com/ssl-internal-hash"))."percona.com/ssl-internal-hash") |\n\t\t\tdel(.spec.volumeClaimTemplates[].spec.volumeMode | select(. == "Filesystem")) |\n\t\t\tdel(.. | select(has("healthCheckNodePort")).healthCheckNodePort) |\n\t\t\tdel(.. | select(has("nodePort")).nodePort) |\n\t\t\tdel(.status) |\n\t\t\t(.. | select(tag == "!!str")) |= sub("monitoring-pmm3-14502", "NAME_SPACE") |\n\t\t\tdel(.spec.volumeClaimTemplates[].apiVersion) |\n\t\t\tdel(.spec.volumeClaimTemplates[].kind) |\n\t\t\tdel(.spec.ipFamilies) |\n\t\t\tdel(.spec.ipFamilyPolicy) |\n\t\t\t(.. | select(. == "extensions/v1beta1")) = "apps/v1" |\n\t\t\t(.. | select(. == "batch/v1beta1")) = "batch/v1" ' - ++ mktemp + local LAST_OUT=/tmp/tmp.HyElDSVbPF ++ mktemp + local LAST_ERR=/tmp/tmp.CfAyaKmaDh + local exit_status=0 + local timeout=4 + args=('get' '-o' 'yaml' 'service/monitoring-pmm3-rs0') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl get -o yaml service/monitoring-pmm3-rs0 + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.HyElDSVbPF + cat /tmp/tmp.CfAyaKmaDh + rm /tmp/tmp.HyElDSVbPF /tmp/tmp.CfAyaKmaDh + return 0 + yq -i eval 'del(.spec.persistentVolumeClaimRetentionPolicy)' /tmp/tmp.9ONBZDsnOT/service_monitoring-pmm3-rs0.yml + arch_scheduling_enabled + [[ -n '' ]] + version_gt 1.22 ++ echo '1.33 >= 1.22' ++ bc -l + '[' 1 -eq 1 ']' + return 0 + yq -i eval 'del(.spec.internalTrafficPolicy)' /tmp/tmp.9ONBZDsnOT/service_monitoring-pmm3-rs0.yml + yq -i eval 'del(.spec.allocateLoadBalancerNodePorts)' /tmp/tmp.9ONBZDsnOT/service_monitoring-pmm3-rs0.yml + [[ /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/e2e-tests/monitoring-pmm3/compare/service_monitoring-pmm3-rs0.yml == */cronjob* ]] + '[' -n '' ']' + [[ 0 -eq 0 ]] + diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/e2e-tests/monitoring-pmm3/compare/service_monitoring-pmm3-rs0.yml /tmp/tmp.9ONBZDsnOT/service_monitoring-pmm3-rs0.yml + log 'compare_kubectl: service/monitoring-pmm3-rs0 OK' + set +o xtrace [2026-07-17T01:25:54+0000] compare_kubectl: service/monitoring-pmm3-rs0 OK + compare_kubectl service/monitoring-pmm3-mongos + local resource=service/monitoring-pmm3-mongos + local postfix= + local skip_generation_check= + local expected_result=/mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/e2e-tests/monitoring-pmm3/compare/service_monitoring-pmm3-mongos.yml + local new_result=/tmp/tmp.9ONBZDsnOT/service_monitoring-pmm3-mongos.yml + [[ -n '' ]] + kubectl_bin get -o yaml service/monitoring-pmm3-mongos + yq eval $'\n\t\t\tdel(.metadata.ownerReferences[].apiVersion) |\n\t\t\tdel(.metadata.managedFields) |\n\t\t\tdel(.. | select(has("creationTimestamp")).creationTimestamp) |\n\t\t\tdel(.. | select(has("namespace")).namespace) |\n\t\t\tdel(.. | select(has("uid")).uid) |\n\t\t\tdel(.metadata.resourceVersion) |\n\t\t\tdel(.spec.template.spec.containers[].env[] | select(.name == "NAMESPACE")) |\n\t\t\tdel(.metadata.selfLink) |\n\t\t\tdel(.metadata.annotations."cloud.google.com/neg") |\n\t\t\tdel(.metadata.annotations."field.cattle.io/publicEndpoints") |\n\t\t\tdel(.metadata.annotations."metallb.io/ip-allocated-from-pool") |\n\t\t\tdel(.metadata.annotations."networking.gke.io/target-pool") |\n\t\t\tdel(.metadata.annotations."kubectl.kubernetes.io/last-applied-configuration") |\n\t\t\tdel(.. | select(has("image")).image) |\n\t\t\tdel(.. | select(has("clusterIP")).clusterIP) |\n\t\t\tdel(.. | select(has("clusterIPs")).clusterIPs) |\n\t\t\tdel(.. | select(has("dataSource")).dataSource) |\n\t\t\tdel(.. | select(has("procMount")).procMount) |\n\t\t\tdel(.. | select(has("storageClassName")).storageClassName) |\n\t\t\tdel(.. | select(has("finalizers")).finalizers) |\n\t\t\tdel(.. | select(has("kubernetes.io/pvc-protection"))."kubernetes.io/pvc-protection") |\n\t\t\tdel(.. | select(has("volumeName")).volumeName) |\n\t\t\tdel(.. | select(has("volume.beta.kubernetes.io/storage-provisioner"))."volume.beta.kubernetes.io/storage-provisioner") |\n\t\t\tdel(.. | select(has("volume.kubernetes.io/storage-provisioner"))."volume.kubernetes.io/storage-provisioner") |\n\t\t\tdel(.spec.volumeMode) |\n\t\t\tdel(.. | select(has("volume.kubernetes.io/selected-node"))."volume.kubernetes.io/selected-node") |\n\t\t\tdel(.. | select(has("percona.com/last-config-hash"))."percona.com/last-config-hash") |\n\t\t\tdel(.. | select(has("percona.com/configuration-hash"))."percona.com/configuration-hash") |\n\t\t\tdel(.. | select(has("percona.com/ssl-hash"))."percona.com/ssl-hash") |\n\t\t\tdel(.. | select(has("percona.com/ssl-internal-hash"))."percona.com/ssl-internal-hash") |\n\t\t\tdel(.spec.volumeClaimTemplates[].spec.volumeMode | select(. == "Filesystem")) |\n\t\t\tdel(.. | select(has("healthCheckNodePort")).healthCheckNodePort) |\n\t\t\tdel(.. | select(has("nodePort")).nodePort) |\n\t\t\tdel(.status) |\n\t\t\t(.. | select(tag == "!!str")) |= sub("monitoring-pmm3-14502", "NAME_SPACE") |\n\t\t\tdel(.spec.volumeClaimTemplates[].apiVersion) |\n\t\t\tdel(.spec.volumeClaimTemplates[].kind) |\n\t\t\tdel(.spec.ipFamilies) |\n\t\t\tdel(.spec.ipFamilyPolicy) |\n\t\t\t(.. | select(. == "extensions/v1beta1")) = "apps/v1" |\n\t\t\t(.. | select(. == "batch/v1beta1")) = "batch/v1" ' - ++ mktemp + local LAST_OUT=/tmp/tmp.Z1e1KclSZ0 ++ mktemp + local LAST_ERR=/tmp/tmp.99V8dvTsDr + local exit_status=0 + local timeout=4 + args=('get' '-o' 'yaml' 'service/monitoring-pmm3-mongos') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl get -o yaml service/monitoring-pmm3-mongos + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.Z1e1KclSZ0 + cat /tmp/tmp.99V8dvTsDr + rm /tmp/tmp.Z1e1KclSZ0 /tmp/tmp.99V8dvTsDr + return 0 + yq -i eval 'del(.spec.persistentVolumeClaimRetentionPolicy)' /tmp/tmp.9ONBZDsnOT/service_monitoring-pmm3-mongos.yml + arch_scheduling_enabled + [[ -n '' ]] + version_gt 1.22 ++ echo '1.33 >= 1.22' ++ bc -l + '[' 1 -eq 1 ']' + return 0 + yq -i eval 'del(.spec.internalTrafficPolicy)' /tmp/tmp.9ONBZDsnOT/service_monitoring-pmm3-mongos.yml + yq -i eval 'del(.spec.allocateLoadBalancerNodePorts)' /tmp/tmp.9ONBZDsnOT/service_monitoring-pmm3-mongos.yml + [[ /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/e2e-tests/monitoring-pmm3/compare/service_monitoring-pmm3-mongos.yml == */cronjob* ]] + '[' -n '' ']' + [[ 0 -eq 0 ]] + diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/e2e-tests/monitoring-pmm3/compare/service_monitoring-pmm3-mongos.yml /tmp/tmp.9ONBZDsnOT/service_monitoring-pmm3-mongos.yml + log 'compare_kubectl: service/monitoring-pmm3-mongos OK' + set +o xtrace [2026-07-17T01:25:55+0000] compare_kubectl: service/monitoring-pmm3-mongos OK + compare_kubectl statefulset/monitoring-pmm3-cfg + local resource=statefulset/monitoring-pmm3-cfg + local postfix= + local skip_generation_check= + local expected_result=/mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/e2e-tests/monitoring-pmm3/compare/statefulset_monitoring-pmm3-cfg.yml + local new_result=/tmp/tmp.9ONBZDsnOT/statefulset_monitoring-pmm3-cfg.yml + [[ -n '' ]] + kubectl_bin get -o yaml statefulset/monitoring-pmm3-cfg + yq eval $'\n\t\t\tdel(.metadata.ownerReferences[].apiVersion) |\n\t\t\tdel(.metadata.managedFields) |\n\t\t\tdel(.. | select(has("creationTimestamp")).creationTimestamp) |\n\t\t\tdel(.. | select(has("namespace")).namespace) |\n\t\t\tdel(.. | select(has("uid")).uid) |\n\t\t\tdel(.metadata.resourceVersion) |\n\t\t\tdel(.spec.template.spec.containers[].env[] | select(.name == "NAMESPACE")) |\n\t\t\tdel(.metadata.selfLink) |\n\t\t\tdel(.metadata.annotations."cloud.google.com/neg") |\n\t\t\tdel(.metadata.annotations."field.cattle.io/publicEndpoints") |\n\t\t\tdel(.metadata.annotations."metallb.io/ip-allocated-from-pool") |\n\t\t\tdel(.metadata.annotations."networking.gke.io/target-pool") |\n\t\t\tdel(.metadata.annotations."kubectl.kubernetes.io/last-applied-configuration") |\n\t\t\tdel(.. | select(has("image")).image) |\n\t\t\tdel(.. | select(has("clusterIP")).clusterIP) |\n\t\t\tdel(.. | select(has("clusterIPs")).clusterIPs) |\n\t\t\tdel(.. | select(has("dataSource")).dataSource) |\n\t\t\tdel(.. | select(has("procMount")).procMount) |\n\t\t\tdel(.. | select(has("storageClassName")).storageClassName) |\n\t\t\tdel(.. | select(has("finalizers")).finalizers) |\n\t\t\tdel(.. | select(has("kubernetes.io/pvc-protection"))."kubernetes.io/pvc-protection") |\n\t\t\tdel(.. | select(has("volumeName")).volumeName) |\n\t\t\tdel(.. | select(has("volume.beta.kubernetes.io/storage-provisioner"))."volume.beta.kubernetes.io/storage-provisioner") |\n\t\t\tdel(.. | select(has("volume.kubernetes.io/storage-provisioner"))."volume.kubernetes.io/storage-provisioner") |\n\t\t\tdel(.spec.volumeMode) |\n\t\t\tdel(.. | select(has("volume.kubernetes.io/selected-node"))."volume.kubernetes.io/selected-node") |\n\t\t\tdel(.. | select(has("percona.com/last-config-hash"))."percona.com/last-config-hash") |\n\t\t\tdel(.. | select(has("percona.com/configuration-hash"))."percona.com/configuration-hash") |\n\t\t\tdel(.. | select(has("percona.com/ssl-hash"))."percona.com/ssl-hash") |\n\t\t\tdel(.. | select(has("percona.com/ssl-internal-hash"))."percona.com/ssl-internal-hash") |\n\t\t\tdel(.spec.volumeClaimTemplates[].spec.volumeMode | select(. == "Filesystem")) |\n\t\t\tdel(.. | select(has("healthCheckNodePort")).healthCheckNodePort) |\n\t\t\tdel(.. | select(has("nodePort")).nodePort) |\n\t\t\tdel(.status) |\n\t\t\t(.. | select(tag == "!!str")) |= sub("monitoring-pmm3-14502", "NAME_SPACE") |\n\t\t\tdel(.spec.volumeClaimTemplates[].apiVersion) |\n\t\t\tdel(.spec.volumeClaimTemplates[].kind) |\n\t\t\tdel(.spec.ipFamilies) |\n\t\t\tdel(.spec.ipFamilyPolicy) |\n\t\t\t(.. | select(. == "extensions/v1beta1")) = "apps/v1" |\n\t\t\t(.. | select(. == "batch/v1beta1")) = "batch/v1" ' - ++ mktemp + local LAST_OUT=/tmp/tmp.Gv9KwCSQJU ++ mktemp + local LAST_ERR=/tmp/tmp.QDNN5IENkF + local exit_status=0 + local timeout=4 + args=('get' '-o' 'yaml' 'statefulset/monitoring-pmm3-cfg') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl get -o yaml statefulset/monitoring-pmm3-cfg + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.Gv9KwCSQJU + cat /tmp/tmp.QDNN5IENkF + rm /tmp/tmp.Gv9KwCSQJU /tmp/tmp.QDNN5IENkF + return 0 + yq -i eval 'del(.spec.persistentVolumeClaimRetentionPolicy)' /tmp/tmp.9ONBZDsnOT/statefulset_monitoring-pmm3-cfg.yml + arch_scheduling_enabled + [[ -n '' ]] + version_gt 1.22 ++ echo '1.33 >= 1.22' ++ bc -l + '[' 1 -eq 1 ']' + return 0 + yq -i eval 'del(.spec.internalTrafficPolicy)' /tmp/tmp.9ONBZDsnOT/statefulset_monitoring-pmm3-cfg.yml + yq -i eval 'del(.spec.allocateLoadBalancerNodePorts)' /tmp/tmp.9ONBZDsnOT/statefulset_monitoring-pmm3-cfg.yml + [[ /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/e2e-tests/monitoring-pmm3/compare/statefulset_monitoring-pmm3-cfg.yml == */cronjob* ]] + '[' -n '' ']' + [[ 0 -eq 0 ]] + diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/e2e-tests/monitoring-pmm3/compare/statefulset_monitoring-pmm3-cfg.yml /tmp/tmp.9ONBZDsnOT/statefulset_monitoring-pmm3-cfg.yml + log 'compare_kubectl: statefulset/monitoring-pmm3-cfg OK' + set +o xtrace [2026-07-17T01:25:55+0000] compare_kubectl: statefulset/monitoring-pmm3-cfg OK + compare_kubectl statefulset/monitoring-pmm3-mongos + local resource=statefulset/monitoring-pmm3-mongos + local postfix= + local skip_generation_check= + local expected_result=/mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/e2e-tests/monitoring-pmm3/compare/statefulset_monitoring-pmm3-mongos.yml + local new_result=/tmp/tmp.9ONBZDsnOT/statefulset_monitoring-pmm3-mongos.yml + [[ -n '' ]] + kubectl_bin get -o yaml statefulset/monitoring-pmm3-mongos + yq eval $'\n\t\t\tdel(.metadata.ownerReferences[].apiVersion) |\n\t\t\tdel(.metadata.managedFields) |\n\t\t\tdel(.. | select(has("creationTimestamp")).creationTimestamp) |\n\t\t\tdel(.. | select(has("namespace")).namespace) |\n\t\t\tdel(.. | select(has("uid")).uid) |\n\t\t\tdel(.metadata.resourceVersion) |\n\t\t\tdel(.spec.template.spec.containers[].env[] | select(.name == "NAMESPACE")) |\n\t\t\tdel(.metadata.selfLink) |\n\t\t\tdel(.metadata.annotations."cloud.google.com/neg") |\n\t\t\tdel(.metadata.annotations."field.cattle.io/publicEndpoints") |\n\t\t\tdel(.metadata.annotations."metallb.io/ip-allocated-from-pool") |\n\t\t\tdel(.metadata.annotations."networking.gke.io/target-pool") |\n\t\t\tdel(.metadata.annotations."kubectl.kubernetes.io/last-applied-configuration") |\n\t\t\tdel(.. | select(has("image")).image) |\n\t\t\tdel(.. | select(has("clusterIP")).clusterIP) |\n\t\t\tdel(.. | select(has("clusterIPs")).clusterIPs) |\n\t\t\tdel(.. | select(has("dataSource")).dataSource) |\n\t\t\tdel(.. | select(has("procMount")).procMount) |\n\t\t\tdel(.. | select(has("storageClassName")).storageClassName) |\n\t\t\tdel(.. | select(has("finalizers")).finalizers) |\n\t\t\tdel(.. | select(has("kubernetes.io/pvc-protection"))."kubernetes.io/pvc-protection") |\n\t\t\tdel(.. | select(has("volumeName")).volumeName) |\n\t\t\tdel(.. | select(has("volume.beta.kubernetes.io/storage-provisioner"))."volume.beta.kubernetes.io/storage-provisioner") |\n\t\t\tdel(.. | select(has("volume.kubernetes.io/storage-provisioner"))."volume.kubernetes.io/storage-provisioner") |\n\t\t\tdel(.spec.volumeMode) |\n\t\t\tdel(.. | select(has("volume.kubernetes.io/selected-node"))."volume.kubernetes.io/selected-node") |\n\t\t\tdel(.. | select(has("percona.com/last-config-hash"))."percona.com/last-config-hash") |\n\t\t\tdel(.. | select(has("percona.com/configuration-hash"))."percona.com/configuration-hash") |\n\t\t\tdel(.. | select(has("percona.com/ssl-hash"))."percona.com/ssl-hash") |\n\t\t\tdel(.. | select(has("percona.com/ssl-internal-hash"))."percona.com/ssl-internal-hash") |\n\t\t\tdel(.spec.volumeClaimTemplates[].spec.volumeMode | select(. == "Filesystem")) |\n\t\t\tdel(.. | select(has("healthCheckNodePort")).healthCheckNodePort) |\n\t\t\tdel(.. | select(has("nodePort")).nodePort) |\n\t\t\tdel(.status) |\n\t\t\t(.. | select(tag == "!!str")) |= sub("monitoring-pmm3-14502", "NAME_SPACE") |\n\t\t\tdel(.spec.volumeClaimTemplates[].apiVersion) |\n\t\t\tdel(.spec.volumeClaimTemplates[].kind) |\n\t\t\tdel(.spec.ipFamilies) |\n\t\t\tdel(.spec.ipFamilyPolicy) |\n\t\t\t(.. | select(. == "extensions/v1beta1")) = "apps/v1" |\n\t\t\t(.. | select(. == "batch/v1beta1")) = "batch/v1" ' - ++ mktemp + local LAST_OUT=/tmp/tmp.xPgDd85tNZ ++ mktemp + local LAST_ERR=/tmp/tmp.bcVHDlwiBp + local exit_status=0 + local timeout=4 + args=('get' '-o' 'yaml' 'statefulset/monitoring-pmm3-mongos') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl get -o yaml statefulset/monitoring-pmm3-mongos + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.xPgDd85tNZ + cat /tmp/tmp.bcVHDlwiBp + rm /tmp/tmp.xPgDd85tNZ /tmp/tmp.bcVHDlwiBp + return 0 + yq -i eval 'del(.spec.persistentVolumeClaimRetentionPolicy)' /tmp/tmp.9ONBZDsnOT/statefulset_monitoring-pmm3-mongos.yml + arch_scheduling_enabled + [[ -n '' ]] + version_gt 1.22 ++ echo '1.33 >= 1.22' ++ bc -l + '[' 1 -eq 1 ']' + return 0 + yq -i eval 'del(.spec.internalTrafficPolicy)' /tmp/tmp.9ONBZDsnOT/statefulset_monitoring-pmm3-mongos.yml + yq -i eval 'del(.spec.allocateLoadBalancerNodePorts)' /tmp/tmp.9ONBZDsnOT/statefulset_monitoring-pmm3-mongos.yml + [[ /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/e2e-tests/monitoring-pmm3/compare/statefulset_monitoring-pmm3-mongos.yml == */cronjob* ]] + '[' -n '' ']' + [[ 0 -eq 0 ]] + diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/e2e-tests/monitoring-pmm3/compare/statefulset_monitoring-pmm3-mongos.yml /tmp/tmp.9ONBZDsnOT/statefulset_monitoring-pmm3-mongos.yml + log 'compare_kubectl: statefulset/monitoring-pmm3-mongos OK' + set +o xtrace [2026-07-17T01:25:56+0000] compare_kubectl: statefulset/monitoring-pmm3-mongos OK + check_custom_pmm_probes + desc 'verify custom PMM liveness/readiness probes set via CR are applied' + set +o xtrace ----------------------------------------------------------------------------------- verify custom PMM liveness/readiness probes set via CR are applied ----------------------------------------------------------------------------------- + kubectl_bin patch psmdb monitoring-pmm3 --type=json $'-p=[\n\t {"op":"add","path":"/spec/pmm/livenessProbe","value":{"exec":{"command":["/bin/true"]},"initialDelaySeconds":30,"periodSeconds":15}},\n\t {"op":"add","path":"/spec/pmm/readinessProbe","value":{"exec":{"command":["/bin/true"]},"initialDelaySeconds":5,"periodSeconds":10}}\n\t]' ++ mktemp + local LAST_OUT=/tmp/tmp.ljUHSazpAB ++ mktemp + local LAST_ERR=/tmp/tmp.Ntl95yEi5z + local exit_status=0 + local timeout=4 + args=('patch' 'psmdb' 'monitoring-pmm3' '--type=json' '-p=[ {"op":"add","path":"/spec/pmm/livenessProbe","value":{"exec":{"command":["/bin/true"]},"initialDelaySeconds":30,"periodSeconds":15}}, {"op":"add","path":"/spec/pmm/readinessProbe","value":{"exec":{"command":["/bin/true"]},"initialDelaySeconds":5,"periodSeconds":10}} ]') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl patch psmdb monitoring-pmm3 --type=json $'-p=[\n\t {"op":"add","path":"/spec/pmm/livenessProbe","value":{"exec":{"command":["/bin/true"]},"initialDelaySeconds":30,"periodSeconds":15}},\n\t {"op":"add","path":"/spec/pmm/readinessProbe","value":{"exec":{"command":["/bin/true"]},"initialDelaySeconds":5,"periodSeconds":10}}\n\t]' + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.ljUHSazpAB perconaservermongodb.psmdb.percona.com/monitoring-pmm3 patched + cat /tmp/tmp.Ntl95yEi5z + rm /tmp/tmp.ljUHSazpAB /tmp/tmp.Ntl95yEi5z + return 0 + wait_for_running monitoring-pmm3-rs0 3 + local name=monitoring-pmm3-rs0 + let last_pod=2 + local check_cluster_readiness=true + set_debug + [[ 1 == 1 ]] + set -o xtrace + local rs_name=rs0 + local cluster_name=monitoring-pmm3 ++ seq 0 2 + for i in $(seq 0 $last_pod) + [[ 0 -eq 2 ]] + wait_pod monitoring-pmm3-rs0-0 + local pod=monitoring-pmm3-rs0-0 + set +o xtrace waiting for pod/monitoring-pmm3-rs0-0 to be ready.OK + for i in $(seq 0 $last_pod) + [[ 1 -eq 2 ]] + wait_pod monitoring-pmm3-rs0-1 + local pod=monitoring-pmm3-rs0-1 + set +o xtrace waiting for pod/monitoring-pmm3-rs0-1 to be ready.OK + for i in $(seq 0 $last_pod) + [[ 2 -eq 2 ]] ++ kubectl_bin get psmdb monitoring-pmm3 -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].arbiter.enabled}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.391mIP2ozu +++ mktemp ++ local LAST_ERR=/tmp/tmp.0qHHGTMdx5 ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'psmdb' 'monitoring-pmm3' '-o' 'jsonpath={.spec.replsets[?(@.name=="rs0")].arbiter.enabled}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb monitoring-pmm3 -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].arbiter.enabled}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.391mIP2ozu ++ cat /tmp/tmp.0qHHGTMdx5 ++ rm /tmp/tmp.391mIP2ozu /tmp/tmp.0qHHGTMdx5 ++ return 0 + [[ '' == true ]] + wait_pod monitoring-pmm3-rs0-2 + local pod=monitoring-pmm3-rs0-2 + set +o xtrace waiting for pod/monitoring-pmm3-rs0-2 to be ready.OK ++ kubectl_bin get psmdb monitoring-pmm3 -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].nonvoting.enabled}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.NPFZ60rzmN +++ mktemp ++ local LAST_ERR=/tmp/tmp.WsflGCaIot ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'psmdb' 'monitoring-pmm3' '-o' 'jsonpath={.spec.replsets[?(@.name=="rs0")].nonvoting.enabled}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb monitoring-pmm3 -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].nonvoting.enabled}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.NPFZ60rzmN ++ cat /tmp/tmp.WsflGCaIot ++ rm /tmp/tmp.NPFZ60rzmN /tmp/tmp.WsflGCaIot ++ return 0 + [[ '' == true ]] ++ kubectl_bin get psmdb monitoring-pmm3 -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].hidden.enabled}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.WdLXohpgU7 +++ mktemp ++ local LAST_ERR=/tmp/tmp.7qv10d1Thm ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'psmdb' 'monitoring-pmm3' '-o' 'jsonpath={.spec.replsets[?(@.name=="rs0")].hidden.enabled}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb monitoring-pmm3 -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].hidden.enabled}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.WdLXohpgU7 ++ cat /tmp/tmp.7qv10d1Thm ++ rm /tmp/tmp.WdLXohpgU7 /tmp/tmp.7qv10d1Thm ++ return 0 + [[ '' == true ]] + sleep 10 + [[ true == true ]] + set +x Waiting for cluster monitoring-pmm3 to be ready..................................................................................................................................... + local liveness readiness ++ kubectl_bin get statefulset/monitoring-pmm3-rs0 -o 'jsonpath={.spec.template.spec.containers[?(@.name=="pmm-client")].livenessProbe.exec.command[0]}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.rUy4N3YUAc +++ mktemp ++ local LAST_ERR=/tmp/tmp.th5AsZi4lI ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'statefulset/monitoring-pmm3-rs0' '-o' 'jsonpath={.spec.template.spec.containers[?(@.name=="pmm-client")].livenessProbe.exec.command[0]}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get statefulset/monitoring-pmm3-rs0 -o 'jsonpath={.spec.template.spec.containers[?(@.name=="pmm-client")].livenessProbe.exec.command[0]}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.rUy4N3YUAc ++ cat /tmp/tmp.th5AsZi4lI ++ rm /tmp/tmp.rUy4N3YUAc /tmp/tmp.th5AsZi4lI ++ return 0 + liveness=/bin/true ++ kubectl_bin get statefulset/monitoring-pmm3-rs0 -o 'jsonpath={.spec.template.spec.containers[?(@.name=="pmm-client")].readinessProbe.exec.command[0]}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.ZBEGNFyIyV +++ mktemp ++ local LAST_ERR=/tmp/tmp.OowJTl2OKj ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'statefulset/monitoring-pmm3-rs0' '-o' 'jsonpath={.spec.template.spec.containers[?(@.name=="pmm-client")].readinessProbe.exec.command[0]}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get statefulset/monitoring-pmm3-rs0 -o 'jsonpath={.spec.template.spec.containers[?(@.name=="pmm-client")].readinessProbe.exec.command[0]}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.ZBEGNFyIyV ++ cat /tmp/tmp.OowJTl2OKj ++ rm /tmp/tmp.ZBEGNFyIyV /tmp/tmp.OowJTl2OKj ++ return 0 + readiness=/bin/true + [[ /bin/true != /bin/true ]] + [[ /bin/true != /bin/true ]] + desc 'revert custom PMM probes to defaults' + set +o xtrace ----------------------------------------------------------------------------------- revert custom PMM probes to defaults ----------------------------------------------------------------------------------- + kubectl_bin patch psmdb monitoring-pmm3 --type=json $'-p=[\n\t {"op":"remove","path":"/spec/pmm/livenessProbe"},\n\t {"op":"remove","path":"/spec/pmm/readinessProbe"}\n\t]' ++ mktemp + local LAST_OUT=/tmp/tmp.LAqrPO2FzX ++ mktemp + local LAST_ERR=/tmp/tmp.RPDKGadyMj + local exit_status=0 + local timeout=4 + args=('patch' 'psmdb' 'monitoring-pmm3' '--type=json' '-p=[ {"op":"remove","path":"/spec/pmm/livenessProbe"}, {"op":"remove","path":"/spec/pmm/readinessProbe"} ]') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl patch psmdb monitoring-pmm3 --type=json $'-p=[\n\t {"op":"remove","path":"/spec/pmm/livenessProbe"},\n\t {"op":"remove","path":"/spec/pmm/readinessProbe"}\n\t]' + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.LAqrPO2FzX perconaservermongodb.psmdb.percona.com/monitoring-pmm3 patched + cat /tmp/tmp.RPDKGadyMj + rm /tmp/tmp.LAqrPO2FzX /tmp/tmp.RPDKGadyMj + return 0 + wait_for_running monitoring-pmm3-rs0 3 + local name=monitoring-pmm3-rs0 + let last_pod=2 + local check_cluster_readiness=true + set_debug + [[ 1 == 1 ]] + set -o xtrace + local rs_name=rs0 + local cluster_name=monitoring-pmm3 ++ seq 0 2 + for i in $(seq 0 $last_pod) + [[ 0 -eq 2 ]] + wait_pod monitoring-pmm3-rs0-0 + local pod=monitoring-pmm3-rs0-0 + set +o xtrace waiting for pod/monitoring-pmm3-rs0-0 to be ready.OK + for i in $(seq 0 $last_pod) + [[ 1 -eq 2 ]] + wait_pod monitoring-pmm3-rs0-1 + local pod=monitoring-pmm3-rs0-1 + set +o xtrace waiting for pod/monitoring-pmm3-rs0-1 to be ready.OK + for i in $(seq 0 $last_pod) + [[ 2 -eq 2 ]] ++ kubectl_bin get psmdb monitoring-pmm3 -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].arbiter.enabled}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.g0fTPFl8Fr +++ mktemp ++ local LAST_ERR=/tmp/tmp.Trgu9V6Dpi ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'psmdb' 'monitoring-pmm3' '-o' 'jsonpath={.spec.replsets[?(@.name=="rs0")].arbiter.enabled}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb monitoring-pmm3 -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].arbiter.enabled}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.g0fTPFl8Fr ++ cat /tmp/tmp.Trgu9V6Dpi ++ rm /tmp/tmp.g0fTPFl8Fr /tmp/tmp.Trgu9V6Dpi ++ return 0 + [[ '' == true ]] + wait_pod monitoring-pmm3-rs0-2 + local pod=monitoring-pmm3-rs0-2 + set +o xtrace waiting for pod/monitoring-pmm3-rs0-2 to be ready.OK ++ kubectl_bin get psmdb monitoring-pmm3 -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].nonvoting.enabled}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.dpJ0IHWirW +++ mktemp ++ local LAST_ERR=/tmp/tmp.eeVOYwiEbN ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'psmdb' 'monitoring-pmm3' '-o' 'jsonpath={.spec.replsets[?(@.name=="rs0")].nonvoting.enabled}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb monitoring-pmm3 -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].nonvoting.enabled}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.dpJ0IHWirW ++ cat /tmp/tmp.eeVOYwiEbN ++ rm /tmp/tmp.dpJ0IHWirW /tmp/tmp.eeVOYwiEbN ++ return 0 + [[ '' == true ]] ++ kubectl_bin get psmdb monitoring-pmm3 -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].hidden.enabled}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.5QJ17wJgSl +++ mktemp ++ local LAST_ERR=/tmp/tmp.BW9EWWv9vY ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'psmdb' 'monitoring-pmm3' '-o' 'jsonpath={.spec.replsets[?(@.name=="rs0")].hidden.enabled}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb monitoring-pmm3 -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].hidden.enabled}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.5QJ17wJgSl ++ cat /tmp/tmp.BW9EWWv9vY ++ rm /tmp/tmp.5QJ17wJgSl /tmp/tmp.BW9EWWv9vY ++ return 0 + [[ '' == true ]] + sleep 10 + [[ true == true ]] + set +x Waiting for cluster monitoring-pmm3 to be ready.............................................................................................................................................................................. + check_custom_logcollector_probes + desc 'verify custom logcollector probes (logs + logrotate) set via CR are applied' + set +o xtrace ----------------------------------------------------------------------------------- verify custom logcollector probes (logs + logrotate) set via CR are applied ----------------------------------------------------------------------------------- + kubectl_bin patch psmdb monitoring-pmm3 --type=json $'-p=[\n\t {"op":"add","path":"/spec/logcollector/livenessProbe","value":{"exec":{"command":["/bin/true"]},"initialDelaySeconds":30,"periodSeconds":15}},\n\t {"op":"add","path":"/spec/logcollector/readinessProbe","value":{"exec":{"command":["/bin/true"]},"initialDelaySeconds":5,"periodSeconds":10}},\n\t {"op":"add","path":"/spec/logcollector/logrotate","value":{"livenessProbe":{"exec":{"command":["/bin/true"]},"initialDelaySeconds":30},"readinessProbe":{"exec":{"command":["/bin/true"]},"initialDelaySeconds":5}}}\n\t]' ++ mktemp + local LAST_OUT=/tmp/tmp.EwJdAIzOnl ++ mktemp + local LAST_ERR=/tmp/tmp.55Lp56JLLn + local exit_status=0 + local timeout=4 + args=('patch' 'psmdb' 'monitoring-pmm3' '--type=json' '-p=[ {"op":"add","path":"/spec/logcollector/livenessProbe","value":{"exec":{"command":["/bin/true"]},"initialDelaySeconds":30,"periodSeconds":15}}, {"op":"add","path":"/spec/logcollector/readinessProbe","value":{"exec":{"command":["/bin/true"]},"initialDelaySeconds":5,"periodSeconds":10}}, {"op":"add","path":"/spec/logcollector/logrotate","value":{"livenessProbe":{"exec":{"command":["/bin/true"]},"initialDelaySeconds":30},"readinessProbe":{"exec":{"command":["/bin/true"]},"initialDelaySeconds":5}}} ]') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl patch psmdb monitoring-pmm3 --type=json $'-p=[\n\t {"op":"add","path":"/spec/logcollector/livenessProbe","value":{"exec":{"command":["/bin/true"]},"initialDelaySeconds":30,"periodSeconds":15}},\n\t {"op":"add","path":"/spec/logcollector/readinessProbe","value":{"exec":{"command":["/bin/true"]},"initialDelaySeconds":5,"periodSeconds":10}},\n\t {"op":"add","path":"/spec/logcollector/logrotate","value":{"livenessProbe":{"exec":{"command":["/bin/true"]},"initialDelaySeconds":30},"readinessProbe":{"exec":{"command":["/bin/true"]},"initialDelaySeconds":5}}}\n\t]' + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.EwJdAIzOnl perconaservermongodb.psmdb.percona.com/monitoring-pmm3 patched + cat /tmp/tmp.55Lp56JLLn + rm /tmp/tmp.EwJdAIzOnl /tmp/tmp.55Lp56JLLn + return 0 + wait_for_running monitoring-pmm3-rs0 3 + local name=monitoring-pmm3-rs0 + let last_pod=2 + local check_cluster_readiness=true + set_debug + [[ 1 == 1 ]] + set -o xtrace + local rs_name=rs0 + local cluster_name=monitoring-pmm3 ++ seq 0 2 + for i in $(seq 0 $last_pod) + [[ 0 -eq 2 ]] + wait_pod monitoring-pmm3-rs0-0 + local pod=monitoring-pmm3-rs0-0 + set +o xtrace waiting for pod/monitoring-pmm3-rs0-0 to be ready.OK + for i in $(seq 0 $last_pod) + [[ 1 -eq 2 ]] + wait_pod monitoring-pmm3-rs0-1 + local pod=monitoring-pmm3-rs0-1 + set +o xtrace waiting for pod/monitoring-pmm3-rs0-1 to be ready.OK + for i in $(seq 0 $last_pod) + [[ 2 -eq 2 ]] ++ kubectl_bin get psmdb monitoring-pmm3 -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].arbiter.enabled}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.TNKtMnmBc9 +++ mktemp ++ local LAST_ERR=/tmp/tmp.bTyNFJ9JSJ ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'psmdb' 'monitoring-pmm3' '-o' 'jsonpath={.spec.replsets[?(@.name=="rs0")].arbiter.enabled}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb monitoring-pmm3 -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].arbiter.enabled}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.TNKtMnmBc9 ++ cat /tmp/tmp.bTyNFJ9JSJ ++ rm /tmp/tmp.TNKtMnmBc9 /tmp/tmp.bTyNFJ9JSJ ++ return 0 + [[ '' == true ]] + wait_pod monitoring-pmm3-rs0-2 + local pod=monitoring-pmm3-rs0-2 + set +o xtrace waiting for pod/monitoring-pmm3-rs0-2 to be ready.OK ++ kubectl_bin get psmdb monitoring-pmm3 -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].nonvoting.enabled}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.3D4sdBDBPk +++ mktemp ++ local LAST_ERR=/tmp/tmp.R1wx7GbeYD ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'psmdb' 'monitoring-pmm3' '-o' 'jsonpath={.spec.replsets[?(@.name=="rs0")].nonvoting.enabled}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb monitoring-pmm3 -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].nonvoting.enabled}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.3D4sdBDBPk ++ cat /tmp/tmp.R1wx7GbeYD ++ rm /tmp/tmp.3D4sdBDBPk /tmp/tmp.R1wx7GbeYD ++ return 0 + [[ '' == true ]] ++ kubectl_bin get psmdb monitoring-pmm3 -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].hidden.enabled}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.5swMOnHCFA +++ mktemp ++ local LAST_ERR=/tmp/tmp.xpcmTkgCjo ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'psmdb' 'monitoring-pmm3' '-o' 'jsonpath={.spec.replsets[?(@.name=="rs0")].hidden.enabled}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb monitoring-pmm3 -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].hidden.enabled}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.5swMOnHCFA ++ cat /tmp/tmp.xpcmTkgCjo ++ rm /tmp/tmp.5swMOnHCFA /tmp/tmp.xpcmTkgCjo ++ return 0 + [[ '' == true ]] + sleep 10 + [[ true == true ]] + set +x Waiting for cluster monitoring-pmm3 to be ready........................................................................................... + local logs_ready logrotate_live logrotate_ready ++ kubectl_bin get statefulset/monitoring-pmm3-rs0 -o 'jsonpath={.spec.template.spec.containers[?(@.name=="logs")].readinessProbe.exec.command[0]}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.Pc9MsPQmGt +++ mktemp ++ local LAST_ERR=/tmp/tmp.mKybZN0yWq ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'statefulset/monitoring-pmm3-rs0' '-o' 'jsonpath={.spec.template.spec.containers[?(@.name=="logs")].readinessProbe.exec.command[0]}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get statefulset/monitoring-pmm3-rs0 -o 'jsonpath={.spec.template.spec.containers[?(@.name=="logs")].readinessProbe.exec.command[0]}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.Pc9MsPQmGt ++ cat /tmp/tmp.mKybZN0yWq ++ rm /tmp/tmp.Pc9MsPQmGt /tmp/tmp.mKybZN0yWq ++ return 0 + logs_ready=/bin/true ++ kubectl_bin get statefulset/monitoring-pmm3-rs0 -o 'jsonpath={.spec.template.spec.containers[?(@.name=="logrotate")].livenessProbe.exec.command[0]}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.jzYFrfYZDX +++ mktemp ++ local LAST_ERR=/tmp/tmp.gx2l5cR2jQ ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'statefulset/monitoring-pmm3-rs0' '-o' 'jsonpath={.spec.template.spec.containers[?(@.name=="logrotate")].livenessProbe.exec.command[0]}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get statefulset/monitoring-pmm3-rs0 -o 'jsonpath={.spec.template.spec.containers[?(@.name=="logrotate")].livenessProbe.exec.command[0]}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.jzYFrfYZDX ++ cat /tmp/tmp.gx2l5cR2jQ ++ rm /tmp/tmp.jzYFrfYZDX /tmp/tmp.gx2l5cR2jQ ++ return 0 + logrotate_live=/bin/true ++ kubectl_bin get statefulset/monitoring-pmm3-rs0 -o 'jsonpath={.spec.template.spec.containers[?(@.name=="logrotate")].readinessProbe.exec.command[0]}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.dSmZ3Crub2 +++ mktemp ++ local LAST_ERR=/tmp/tmp.IUOERrro85 ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'statefulset/monitoring-pmm3-rs0' '-o' 'jsonpath={.spec.template.spec.containers[?(@.name=="logrotate")].readinessProbe.exec.command[0]}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get statefulset/monitoring-pmm3-rs0 -o 'jsonpath={.spec.template.spec.containers[?(@.name=="logrotate")].readinessProbe.exec.command[0]}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.dSmZ3Crub2 ++ cat /tmp/tmp.IUOERrro85 ++ rm /tmp/tmp.dSmZ3Crub2 /tmp/tmp.IUOERrro85 ++ return 0 + logrotate_ready=/bin/true + [[ /bin/true != /bin/true ]] + [[ /bin/true != /bin/true ]] + [[ /bin/true != /bin/true ]] + desc 'revert custom logcollector probes to defaults' + set +o xtrace ----------------------------------------------------------------------------------- revert custom logcollector probes to defaults ----------------------------------------------------------------------------------- + kubectl_bin patch psmdb monitoring-pmm3 --type=json $'-p=[\n\t {"op":"remove","path":"/spec/logcollector/livenessProbe"},\n\t {"op":"remove","path":"/spec/logcollector/readinessProbe"},\n\t {"op":"remove","path":"/spec/logcollector/logrotate"}\n\t]' ++ mktemp + local LAST_OUT=/tmp/tmp.l9L37oOxlu ++ mktemp + local LAST_ERR=/tmp/tmp.MM4rMruwRx + local exit_status=0 + local timeout=4 + args=('patch' 'psmdb' 'monitoring-pmm3' '--type=json' '-p=[ {"op":"remove","path":"/spec/logcollector/livenessProbe"}, {"op":"remove","path":"/spec/logcollector/readinessProbe"}, {"op":"remove","path":"/spec/logcollector/logrotate"} ]') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl patch psmdb monitoring-pmm3 --type=json $'-p=[\n\t {"op":"remove","path":"/spec/logcollector/livenessProbe"},\n\t {"op":"remove","path":"/spec/logcollector/readinessProbe"},\n\t {"op":"remove","path":"/spec/logcollector/logrotate"}\n\t]' + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.l9L37oOxlu perconaservermongodb.psmdb.percona.com/monitoring-pmm3 patched + cat /tmp/tmp.MM4rMruwRx + rm /tmp/tmp.l9L37oOxlu /tmp/tmp.MM4rMruwRx + return 0 + wait_for_running monitoring-pmm3-rs0 3 + local name=monitoring-pmm3-rs0 + let last_pod=2 + local check_cluster_readiness=true + set_debug + [[ 1 == 1 ]] + set -o xtrace + local rs_name=rs0 + local cluster_name=monitoring-pmm3 ++ seq 0 2 + for i in $(seq 0 $last_pod) + [[ 0 -eq 2 ]] + wait_pod monitoring-pmm3-rs0-0 + local pod=monitoring-pmm3-rs0-0 + set +o xtrace waiting for pod/monitoring-pmm3-rs0-0 to be ready.OK + for i in $(seq 0 $last_pod) + [[ 1 -eq 2 ]] + wait_pod monitoring-pmm3-rs0-1 + local pod=monitoring-pmm3-rs0-1 + set +o xtrace waiting for pod/monitoring-pmm3-rs0-1 to be ready.OK + for i in $(seq 0 $last_pod) + [[ 2 -eq 2 ]] ++ kubectl_bin get psmdb monitoring-pmm3 -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].arbiter.enabled}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.AWPcoetqTP +++ mktemp ++ local LAST_ERR=/tmp/tmp.SmHA5JGYbs ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'psmdb' 'monitoring-pmm3' '-o' 'jsonpath={.spec.replsets[?(@.name=="rs0")].arbiter.enabled}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb monitoring-pmm3 -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].arbiter.enabled}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.AWPcoetqTP ++ cat /tmp/tmp.SmHA5JGYbs ++ rm /tmp/tmp.AWPcoetqTP /tmp/tmp.SmHA5JGYbs ++ return 0 + [[ '' == true ]] + wait_pod monitoring-pmm3-rs0-2 + local pod=monitoring-pmm3-rs0-2 + set +o xtrace waiting for pod/monitoring-pmm3-rs0-2 to be ready.OK ++ kubectl_bin get psmdb monitoring-pmm3 -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].nonvoting.enabled}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.TsN9tl1Ol6 +++ mktemp ++ local LAST_ERR=/tmp/tmp.wQZRGpZBmA ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'psmdb' 'monitoring-pmm3' '-o' 'jsonpath={.spec.replsets[?(@.name=="rs0")].nonvoting.enabled}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb monitoring-pmm3 -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].nonvoting.enabled}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.TsN9tl1Ol6 ++ cat /tmp/tmp.wQZRGpZBmA ++ rm /tmp/tmp.TsN9tl1Ol6 /tmp/tmp.wQZRGpZBmA ++ return 0 + [[ '' == true ]] ++ kubectl_bin get psmdb monitoring-pmm3 -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].hidden.enabled}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.GsfdSteWum +++ mktemp ++ local LAST_ERR=/tmp/tmp.9tB6sBbsAt ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'psmdb' 'monitoring-pmm3' '-o' 'jsonpath={.spec.replsets[?(@.name=="rs0")].hidden.enabled}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb monitoring-pmm3 -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].hidden.enabled}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.GsfdSteWum ++ cat /tmp/tmp.9tB6sBbsAt ++ rm /tmp/tmp.GsfdSteWum /tmp/tmp.9tB6sBbsAt ++ return 0 + [[ '' == true ]] + sleep 10 + [[ true == true ]] + set +x Waiting for cluster monitoring-pmm3 to be ready.......................................................................................................... + desc 'create new PMM token and add it to the secret' + set +o xtrace ----------------------------------------------------------------------------------- create new PMM token and add it to the secret ----------------------------------------------------------------------------------- ++ get_pmm_server_token operator_new ++ local key_name=operator_new ++ [[ -z operator_new ]] ++ local ADMIN_PASSWORD +++ base64 --decode +++ kubectl get secret pmm-secret -o 'jsonpath={.data.PMM_ADMIN_PASSWORD}' ++ ADMIN_PASSWORD='J:k%i1RN=xa^Z5a!' ++ [[ -z J:k%i1RN=xa^Z5a! ]] ++ local endpoint encoded_user_credentials +++ get_service_endpoint monitoring-service +++ local service=monitoring-service +++ local endpoint +++ local service_type ++++ kubectl_bin get service/monitoring-service -o 'jsonpath={.spec.type}' +++++ mktemp ++++ local LAST_OUT=/tmp/tmp.FjTDTOHO00 +++++ mktemp ++++ local LAST_ERR=/tmp/tmp.qFPRAgic2Z ++++ local exit_status=0 ++++ local timeout=4 ++++ args=('get' 'service/monitoring-service' '-o' 'jsonpath={.spec.type}') ++++ local args ++++ transformed_files=() ++++ local transformed_files ++++ arch_scheduling_enabled ++++ [[ -n '' ]] ++++ arch_scheduling_enabled ++++ [[ -n '' ]] +++++ seq 0 2 ++++ for i in $(seq 0 2) ++++ set +e ++++ kubectl get service/monitoring-service -o 'jsonpath={.spec.type}' ++++ exit_status=0 ++++ set -e ++++ '[' 0 '!=' 0 -a -n 1 ']' ++++ break ++++ cat /tmp/tmp.FjTDTOHO00 ++++ cat /tmp/tmp.qFPRAgic2Z ++++ rm /tmp/tmp.FjTDTOHO00 /tmp/tmp.qFPRAgic2Z ++++ return 0 +++ service_type=LoadBalancer +++ [[ LoadBalancer == LoadBalancer ]] ++++ get_load_balancer_endpoint monitoring-service ++++ local service=monitoring-service ++++ local endpoint +++++ kubectl_bin get service/monitoring-service -o json +++++ jq -r '[.status.loadBalancer.ingress[]?.hostname, .status.loadBalancer.ingress[]?.ip] | map(select(. != null and . != "")) | first // empty' ++++++ mktemp +++++ local LAST_OUT=/tmp/tmp.MCS7Cf5DTL ++++++ mktemp +++++ local LAST_ERR=/tmp/tmp.ps9KGNVxAH +++++ local exit_status=0 +++++ local timeout=4 +++++ args=('get' 'service/monitoring-service' '-o' 'json') +++++ local args +++++ transformed_files=() +++++ local transformed_files +++++ arch_scheduling_enabled +++++ [[ -n '' ]] +++++ arch_scheduling_enabled +++++ [[ -n '' ]] ++++++ seq 0 2 +++++ for i in $(seq 0 2) +++++ set +e +++++ kubectl get service/monitoring-service -o json +++++ exit_status=0 +++++ set -e +++++ '[' 0 '!=' 0 -a -n 1 ']' +++++ break +++++ cat /tmp/tmp.MCS7Cf5DTL +++++ cat /tmp/tmp.ps9KGNVxAH +++++ rm /tmp/tmp.MCS7Cf5DTL /tmp/tmp.ps9KGNVxAH +++++ return 0 ++++ endpoint=136.112.205.251 ++++ '[' -n 136.112.205.251 -a 136.112.205.251 '!=' null ']' ++++ echo 136.112.205.251 ++++ return +++ endpoint=136.112.205.251 +++ echo 136.112.205.251 +++ return ++ endpoint=136.112.205.251 +++ urlencode_userinfo 'admin:J:k%i1RN=xa^Z5a!' +++ local 'user_pass=admin:J:k%i1RN=xa^Z5a!' +++ local username=admin +++ local 'password=J:k%i1RN=xa^Z5a!' ++++ urlencode admin ++++ uri=admin ++++ echo -n admin ++++ jq -s -R -r @uri ++++ urlencode 'J:k%i1RN=xa^Z5a!' ++++ uri='J:k%i1RN=xa^Z5a!' ++++ echo -n 'J:k%i1RN=xa^Z5a!' ++++ jq -s -R -r @uri +++ printf %s:%s admin J%3Ak%25i1RN%3Dxa%5EZ5a%21 ++ encoded_user_credentials=admin:J%3Ak%25i1RN%3Dxa%5EZ5a%21 ++ local create_response create_status_code create_json_response +++ execute_curl --insecure -s -X POST -H 'Content-Type: application/json' -H 'Accept: application/json' -d '{"name":"operator_new", "role":"Admin", "isDisabled":false}' https://admin:J%3Ak%25i1RN%3Dxa%5EZ5a%21@136.112.205.251/graph/api/serviceaccounts -w '\n%{http_code}' +++ local name=curl-4195 +++ local image=docker.io/alpine/curl:latest +++ local phase= +++ local exit_status=0 +++ kubectl_bin run curl-4195 -n monitoring-pmm3-14502 --restart=Never --image=docker.io/alpine/curl:latest --command -- curl -sS --insecure -s -X POST -H 'Content-Type: application/json' -H 'Accept: application/json' -d '{"name":"operator_new", "role":"Admin", "isDisabled":false}' https://admin:J%3Ak%25i1RN%3Dxa%5EZ5a%21@136.112.205.251/graph/api/serviceaccounts -w '\n%{http_code}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.y7Ss8KCShf ++++ mktemp +++ local LAST_ERR=/tmp/tmp.AujzRdlPFK +++ local exit_status=0 +++ local timeout=4 +++ args=('run' 'curl-4195' '-n' 'monitoring-pmm3-14502' '--restart=Never' '--image=docker.io/alpine/curl:latest' '--command' '--' 'curl' '-sS' '--insecure' '-s' '-X' 'POST' '-H' 'Content-Type: application/json' '-H' 'Accept: application/json' '-d' '{"name":"operator_new", "role":"Admin", "isDisabled":false}' 'https://admin:J%3Ak%25i1RN%3Dxa%5EZ5a%21@136.112.205.251/graph/api/serviceaccounts' '-w' '\n%{http_code}') +++ local args +++ transformed_files=() +++ local transformed_files +++ arch_scheduling_enabled +++ [[ -n '' ]] +++ arch_scheduling_enabled +++ [[ -n '' ]] ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl run curl-4195 -n monitoring-pmm3-14502 --restart=Never --image=docker.io/alpine/curl:latest --command -- curl -sS --insecure -s -X POST -H 'Content-Type: application/json' -H 'Accept: application/json' -d '{"name":"operator_new", "role":"Admin", "isDisabled":false}' https://admin:J%3Ak%25i1RN%3Dxa%5EZ5a%21@136.112.205.251/graph/api/serviceaccounts -w '\n%{http_code}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 -a -n 1 ']' +++ break +++ cat /tmp/tmp.y7Ss8KCShf +++ cat /tmp/tmp.AujzRdlPFK +++ rm /tmp/tmp.y7Ss8KCShf /tmp/tmp.AujzRdlPFK +++ return 0 ++++ seq 1 60 +++ for _ in $(seq 1 60) ++++ kubectl_bin get pod curl-4195 -n monitoring-pmm3-14502 -o 'jsonpath={.status.phase}' +++ phase=Pending +++ [[ Pending == Succeeded ]] +++ [[ Pending == Failed ]] +++ sleep 1 +++ for _ in $(seq 1 60) ++++ kubectl_bin get pod curl-4195 -n monitoring-pmm3-14502 -o 'jsonpath={.status.phase}' +++ phase=Pending +++ [[ Pending == Succeeded ]] +++ [[ Pending == Failed ]] +++ sleep 1 +++ for _ in $(seq 1 60) ++++ kubectl_bin get pod curl-4195 -n monitoring-pmm3-14502 -o 'jsonpath={.status.phase}' +++ phase=Succeeded +++ [[ Succeeded == Succeeded ]] +++ break +++ kubectl_bin logs curl-4195 -n monitoring-pmm3-14502 ++++ mktemp +++ local LAST_OUT=/tmp/tmp.A1CEKDVup9 ++++ mktemp +++ local LAST_ERR=/tmp/tmp.Ze515VXg2t +++ local exit_status=0 +++ local timeout=4 +++ args=('logs' 'curl-4195' '-n' 'monitoring-pmm3-14502') +++ local args +++ transformed_files=() +++ local transformed_files +++ arch_scheduling_enabled +++ [[ -n '' ]] +++ arch_scheduling_enabled +++ [[ -n '' ]] ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl logs curl-4195 -n monitoring-pmm3-14502 +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 -a -n 1 ']' +++ break +++ cat /tmp/tmp.A1CEKDVup9 +++ cat /tmp/tmp.Ze515VXg2t +++ rm /tmp/tmp.A1CEKDVup9 /tmp/tmp.Ze515VXg2t +++ return 0 +++ kubectl_bin delete pod curl-4195 -n monitoring-pmm3-14502 --ignore-not-found --wait=false +++ [[ Succeeded != Succeeded ]] +++ return 0 ++ create_response=$'{"id":42,"uid":"efsc5kx7kc0zkd","name":"operator_new","login":"sa-1-operator_new","orgId":1,"isDisabled":false,"role":"Admin","tokens":0,"avatarUrl":""}\n201' +++ echo $'{"id":42,"uid":"efsc5kx7kc0zkd","name":"operator_new","login":"sa-1-operator_new","orgId":1,"isDisabled":false,"role":"Admin","tokens":0,"avatarUrl":""}\n201' +++ tail -n1 ++ create_status_code=201 +++ echo $'{"id":42,"uid":"efsc5kx7kc0zkd","name":"operator_new","login":"sa-1-operator_new","orgId":1,"isDisabled":false,"role":"Admin","tokens":0,"avatarUrl":""}\n201' +++ sed '$ d' ++ create_json_response='{"id":42,"uid":"efsc5kx7kc0zkd","name":"operator_new","login":"sa-1-operator_new","orgId":1,"isDisabled":false,"role":"Admin","tokens":0,"avatarUrl":""}' ++ [[ 201 -ne 201 ]] ++ local service_account_id +++ echo '{"id":42,"uid":"efsc5kx7kc0zkd","name":"operator_new","login":"sa-1-operator_new","orgId":1,"isDisabled":false,"role":"Admin","tokens":0,"avatarUrl":""}' +++ jq -r .id ++ service_account_id=42 ++ [[ -z 42 ]] ++ [[ 42 == null ]] ++ local token_response token_status_code token_json_response +++ execute_curl --insecure -s -X POST -H 'Content-Type: application/json' -d '{"name":"operator_new"}' https://admin:J%3Ak%25i1RN%3Dxa%5EZ5a%21@136.112.205.251/graph/api/serviceaccounts/42/tokens -w '\n%{http_code}' +++ local name=curl-6948 +++ local image=docker.io/alpine/curl:latest +++ local phase= +++ local exit_status=0 +++ kubectl_bin run curl-6948 -n monitoring-pmm3-14502 --restart=Never --image=docker.io/alpine/curl:latest --command -- curl -sS --insecure -s -X POST -H 'Content-Type: application/json' -d '{"name":"operator_new"}' https://admin:J%3Ak%25i1RN%3Dxa%5EZ5a%21@136.112.205.251/graph/api/serviceaccounts/42/tokens -w '\n%{http_code}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.QerEiA4JC1 ++++ mktemp +++ local LAST_ERR=/tmp/tmp.KjwEQsnZOU +++ local exit_status=0 +++ local timeout=4 +++ args=('run' 'curl-6948' '-n' 'monitoring-pmm3-14502' '--restart=Never' '--image=docker.io/alpine/curl:latest' '--command' '--' 'curl' '-sS' '--insecure' '-s' '-X' 'POST' '-H' 'Content-Type: application/json' '-d' '{"name":"operator_new"}' 'https://admin:J%3Ak%25i1RN%3Dxa%5EZ5a%21@136.112.205.251/graph/api/serviceaccounts/42/tokens' '-w' '\n%{http_code}') +++ local args +++ transformed_files=() +++ local transformed_files +++ arch_scheduling_enabled +++ [[ -n '' ]] +++ arch_scheduling_enabled +++ [[ -n '' ]] ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl run curl-6948 -n monitoring-pmm3-14502 --restart=Never --image=docker.io/alpine/curl:latest --command -- curl -sS --insecure -s -X POST -H 'Content-Type: application/json' -d '{"name":"operator_new"}' https://admin:J%3Ak%25i1RN%3Dxa%5EZ5a%21@136.112.205.251/graph/api/serviceaccounts/42/tokens -w '\n%{http_code}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 -a -n 1 ']' +++ break +++ cat /tmp/tmp.QerEiA4JC1 +++ cat /tmp/tmp.KjwEQsnZOU +++ rm /tmp/tmp.QerEiA4JC1 /tmp/tmp.KjwEQsnZOU +++ return 0 ++++ seq 1 60 +++ for _ in $(seq 1 60) ++++ kubectl_bin get pod curl-6948 -n monitoring-pmm3-14502 -o 'jsonpath={.status.phase}' +++ phase=Pending +++ [[ Pending == Succeeded ]] +++ [[ Pending == Failed ]] +++ sleep 1 +++ for _ in $(seq 1 60) ++++ kubectl_bin get pod curl-6948 -n monitoring-pmm3-14502 -o 'jsonpath={.status.phase}' +++ phase=Succeeded +++ [[ Succeeded == Succeeded ]] +++ break +++ kubectl_bin logs curl-6948 -n monitoring-pmm3-14502 ++++ mktemp +++ local LAST_OUT=/tmp/tmp.lSF27vdono ++++ mktemp +++ local LAST_ERR=/tmp/tmp.orlDAouqio +++ local exit_status=0 +++ local timeout=4 +++ args=('logs' 'curl-6948' '-n' 'monitoring-pmm3-14502') +++ local args +++ transformed_files=() +++ local transformed_files +++ arch_scheduling_enabled +++ [[ -n '' ]] +++ arch_scheduling_enabled +++ [[ -n '' ]] ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl logs curl-6948 -n monitoring-pmm3-14502 +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 -a -n 1 ']' +++ break +++ cat /tmp/tmp.lSF27vdono +++ cat /tmp/tmp.orlDAouqio +++ rm /tmp/tmp.lSF27vdono /tmp/tmp.orlDAouqio +++ return 0 +++ kubectl_bin delete pod curl-6948 -n monitoring-pmm3-14502 --ignore-not-found --wait=false +++ [[ Succeeded != Succeeded ]] +++ return 0 ++ token_response=$'{"id":41,"name":"operator_new","key":"glsa_422RcuDdDskb7TLtapwWaOqcyrTCF2do_8c6a26b0"}\n200' +++ echo $'{"id":41,"name":"operator_new","key":"glsa_422RcuDdDskb7TLtapwWaOqcyrTCF2do_8c6a26b0"}\n200' +++ tail -n1 ++ token_status_code=200 +++ echo $'{"id":41,"name":"operator_new","key":"glsa_422RcuDdDskb7TLtapwWaOqcyrTCF2do_8c6a26b0"}\n200' +++ sed '$ d' ++ token_json_response='{"id":41,"name":"operator_new","key":"glsa_422RcuDdDskb7TLtapwWaOqcyrTCF2do_8c6a26b0"}' ++ [[ 200 -ne 200 ]] ++ echo '{"id":41,"name":"operator_new","key":"glsa_422RcuDdDskb7TLtapwWaOqcyrTCF2do_8c6a26b0"}' ++ jq -r .key + NEW_TOKEN=glsa_422RcuDdDskb7TLtapwWaOqcyrTCF2do_8c6a26b0 + kubectl_bin patch secret some-users --type merge --patch '{"stringData": {"PMM_SERVER_TOKEN": "glsa_422RcuDdDskb7TLtapwWaOqcyrTCF2do_8c6a26b0"}}' ++ mktemp + local LAST_OUT=/tmp/tmp.xKsqjVDfZA ++ mktemp + local LAST_ERR=/tmp/tmp.uuQrXfy6aW + local exit_status=0 + local timeout=4 + args=('patch' 'secret' 'some-users' '--type' 'merge' '--patch' '{"stringData": {"PMM_SERVER_TOKEN": "glsa_422RcuDdDskb7TLtapwWaOqcyrTCF2do_8c6a26b0"}}') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl patch secret some-users --type merge --patch '{"stringData": {"PMM_SERVER_TOKEN": "glsa_422RcuDdDskb7TLtapwWaOqcyrTCF2do_8c6a26b0"}}' + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.xKsqjVDfZA secret/some-users patched + cat /tmp/tmp.uuQrXfy6aW + rm /tmp/tmp.xKsqjVDfZA /tmp/tmp.uuQrXfy6aW + return 0 + desc 'delete old PMM token' + set +o xtrace ----------------------------------------------------------------------------------- delete old PMM token ----------------------------------------------------------------------------------- + delete_pmm_server_token operator + local key_name=operator + [[ -z operator ]] + local ADMIN_PASSWORD ++ kubectl get secret pmm-secret -o 'jsonpath={.data.PMM_ADMIN_PASSWORD}' ++ base64 --decode + ADMIN_PASSWORD='J:k%i1RN=xa^Z5a!' + [[ -z J:k%i1RN=xa^Z5a! ]] + local endpoint encoded_user_credentials ++ get_service_endpoint monitoring-service ++ local service=monitoring-service ++ local endpoint ++ local service_type +++ kubectl_bin get service/monitoring-service -o 'jsonpath={.spec.type}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.6PfugEUMRW ++++ mktemp +++ local LAST_ERR=/tmp/tmp.75Pm8EE9lw +++ local exit_status=0 +++ local timeout=4 +++ args=('get' 'service/monitoring-service' '-o' 'jsonpath={.spec.type}') +++ local args +++ transformed_files=() +++ local transformed_files +++ arch_scheduling_enabled +++ [[ -n '' ]] +++ arch_scheduling_enabled +++ [[ -n '' ]] ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl get service/monitoring-service -o 'jsonpath={.spec.type}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 -a -n 1 ']' +++ break +++ cat /tmp/tmp.6PfugEUMRW +++ cat /tmp/tmp.75Pm8EE9lw +++ rm /tmp/tmp.6PfugEUMRW /tmp/tmp.75Pm8EE9lw +++ return 0 ++ service_type=LoadBalancer ++ [[ LoadBalancer == LoadBalancer ]] +++ get_load_balancer_endpoint monitoring-service +++ local service=monitoring-service +++ local endpoint ++++ kubectl_bin get service/monitoring-service -o json ++++ jq -r '[.status.loadBalancer.ingress[]?.hostname, .status.loadBalancer.ingress[]?.ip] | map(select(. != null and . != "")) | first // empty' +++++ mktemp ++++ local LAST_OUT=/tmp/tmp.2LUOHbjFNv +++++ mktemp ++++ local LAST_ERR=/tmp/tmp.n10rCmpwyF ++++ local exit_status=0 ++++ local timeout=4 ++++ args=('get' 'service/monitoring-service' '-o' 'json') ++++ local args ++++ transformed_files=() ++++ local transformed_files ++++ arch_scheduling_enabled ++++ [[ -n '' ]] ++++ arch_scheduling_enabled ++++ [[ -n '' ]] +++++ seq 0 2 ++++ for i in $(seq 0 2) ++++ set +e ++++ kubectl get service/monitoring-service -o json ++++ exit_status=0 ++++ set -e ++++ '[' 0 '!=' 0 -a -n 1 ']' ++++ break ++++ cat /tmp/tmp.2LUOHbjFNv ++++ cat /tmp/tmp.n10rCmpwyF ++++ rm /tmp/tmp.2LUOHbjFNv /tmp/tmp.n10rCmpwyF ++++ return 0 +++ endpoint=136.112.205.251 +++ '[' -n 136.112.205.251 -a 136.112.205.251 '!=' null ']' +++ echo 136.112.205.251 +++ return ++ endpoint=136.112.205.251 ++ echo 136.112.205.251 ++ return + endpoint=136.112.205.251 ++ urlencode_userinfo 'admin:J:k%i1RN=xa^Z5a!' ++ local 'user_pass=admin:J:k%i1RN=xa^Z5a!' ++ local username=admin ++ local 'password=J:k%i1RN=xa^Z5a!' +++ urlencode admin +++ uri=admin +++ echo -n admin +++ jq -s -R -r @uri +++ urlencode 'J:k%i1RN=xa^Z5a!' +++ uri='J:k%i1RN=xa^Z5a!' +++ echo -n 'J:k%i1RN=xa^Z5a!' +++ jq -s -R -r @uri ++ printf %s:%s admin J%3Ak%25i1RN%3Dxa%5EZ5a%21 + encoded_user_credentials=admin:J%3Ak%25i1RN%3Dxa%5EZ5a%21 + local service_accounts_response service_accounts_status ++ execute_curl --insecure -s -X GET https://admin:J%3Ak%25i1RN%3Dxa%5EZ5a%21@136.112.205.251/graph/api/serviceaccounts/search -w '\n%{http_code}' ++ local name=curl-1727 ++ local image=docker.io/alpine/curl:latest ++ local phase= ++ local exit_status=0 ++ kubectl_bin run curl-1727 -n monitoring-pmm3-14502 --restart=Never --image=docker.io/alpine/curl:latest --command -- curl -sS --insecure -s -X GET https://admin:J%3Ak%25i1RN%3Dxa%5EZ5a%21@136.112.205.251/graph/api/serviceaccounts/search -w '\n%{http_code}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.C64td4Wry9 +++ mktemp ++ local LAST_ERR=/tmp/tmp.A4wxUxytDr ++ local exit_status=0 ++ local timeout=4 ++ args=('run' 'curl-1727' '-n' 'monitoring-pmm3-14502' '--restart=Never' '--image=docker.io/alpine/curl:latest' '--command' '--' 'curl' '-sS' '--insecure' '-s' '-X' 'GET' 'https://admin:J%3Ak%25i1RN%3Dxa%5EZ5a%21@136.112.205.251/graph/api/serviceaccounts/search' '-w' '\n%{http_code}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl run curl-1727 -n monitoring-pmm3-14502 --restart=Never --image=docker.io/alpine/curl:latest --command -- curl -sS --insecure -s -X GET https://admin:J%3Ak%25i1RN%3Dxa%5EZ5a%21@136.112.205.251/graph/api/serviceaccounts/search -w '\n%{http_code}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.C64td4Wry9 ++ cat /tmp/tmp.A4wxUxytDr ++ rm /tmp/tmp.C64td4Wry9 /tmp/tmp.A4wxUxytDr ++ return 0 +++ seq 1 60 ++ for _ in $(seq 1 60) +++ kubectl_bin get pod curl-1727 -n monitoring-pmm3-14502 -o 'jsonpath={.status.phase}' ++ phase=Pending ++ [[ Pending == Succeeded ]] ++ [[ Pending == Failed ]] ++ sleep 1 ++ for _ in $(seq 1 60) +++ kubectl_bin get pod curl-1727 -n monitoring-pmm3-14502 -o 'jsonpath={.status.phase}' ++ phase=Succeeded ++ [[ Succeeded == Succeeded ]] ++ break ++ kubectl_bin logs curl-1727 -n monitoring-pmm3-14502 +++ mktemp ++ local LAST_OUT=/tmp/tmp.bftaDT4dl2 +++ mktemp ++ local LAST_ERR=/tmp/tmp.RjK8sRENIj ++ local exit_status=0 ++ local timeout=4 ++ args=('logs' 'curl-1727' '-n' 'monitoring-pmm3-14502') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl logs curl-1727 -n monitoring-pmm3-14502 ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.bftaDT4dl2 ++ cat /tmp/tmp.RjK8sRENIj ++ rm /tmp/tmp.bftaDT4dl2 /tmp/tmp.RjK8sRENIj ++ return 0 ++ kubectl_bin delete pod curl-1727 -n monitoring-pmm3-14502 --ignore-not-found --wait=false ++ [[ Succeeded != Succeeded ]] ++ return 0 + service_accounts_response=$'{"totalCount":11,"serviceAccounts":[{"id":2,"uid":"afsc39pmk0a9sb","name":"operator","login":"sa-1-operator","orgId":1,"isDisabled":false,"role":"Admin","tokens":1,"avatarUrl":"/graph/avatar/f741e7abdd8def4ed30a37f602429df3"},{"id":42,"uid":"efsc5kx7kc0zkd","name":"operator_new","login":"sa-1-operator_new","orgId":1,"isDisabled":false,"role":"Admin","tokens":1,"avatarUrl":"/graph/avatar/6d1b2db67a8f3e06db4ae076406fece5"},{"id":37,"uid":"bfsc5bf8bs16oe","name":"pmm-agent-sa-monitoring-pmm3-14502-monitoring-pmm3-cfg-0","login":"sa-1-pmm-agent-sa-monitoring-pmm3-14502-monitoring-pmm3-cfg-0","orgId":1,"isDisabled":false,"role":"Admin","tokens":1,"avatarUrl":"/graph/avatar/ff80b4fbcd0e447728f468eb24e1949f"},{"id":36,"uid":"cfsc59j5wdxq8e","name":"pmm-agent-sa-monitoring-pmm3-14502-monitoring-pmm3-cfg-1","login":"sa-1-pmm-agent-sa-monitoring-pmm3-14502-monitoring-pmm3-cfg-1","orgId":1,"isDisabled":false,"role":"Admin","tokens":1,"avatarUrl":"/graph/avatar/adf7fd6b896b1a9b7906dc6cc9531553"},{"id":38,"uid":"dfsc5dk1vuz28f","name":"pmm-agent-sa-monitoring-pmm3-14502-monitoring-pmm3-cfg-2","login":"sa-1-pmm-agent-sa-monitoring-pmm3-14502-monitoring-pmm3-cfg-2","orgId":1,"isDisabled":false,"role":"Admin","tokens":1,"avatarUrl":"/graph/avatar/3b87d0594e2ab224169ad4c149084c97"},{"id":29,"uid":"dfsc4w6fvv6yoa","name":"pmm-agent-sa-monitoring-pmm3-14502-monitoring-pmm3-mongos-0","login":"sa-1-pmm-agent-sa-monitoring-pmm3-14502-monitoring-pmm3-mongos-0","orgId":1,"isDisabled":false,"role":"Admin","tokens":1,"avatarUrl":"/graph/avatar/c9fe7bfcc8e4476abe1dd933e9158d8d"},{"id":28,"uid":"dfsc4umww4h6of","name":"pmm-agent-sa-monitoring-pmm3-14502-monitoring-pmm3-mongos-1","login":"sa-1-pmm-agent-sa-monitoring-pmm3-14502-monitoring-pmm3-mongos-1","orgId":1,"isDisabled":false,"role":"Admin","tokens":1,"avatarUrl":"/graph/avatar/c458e7f6fa74c8c2e7e29f242ed8bcbc"},{"id":27,"uid":"afsc4t3n20d1ce","name":"pmm-agent-sa-monitoring-pmm3-14502-monitoring-pmm3-mongos-2","login":"sa-1-pmm-agent-sa-monitoring-pmm3-14502-monitoring-pmm3-mongos-2","orgId":1,"isDisabled":false,"role":"Admin","tokens":1,"avatarUrl":"/graph/avatar/72079b564397474018d71c100592cd29"},{"id":40,"uid":"ffsc5hzgtkqgwc","name":"pmm-agent-sa-monitoring-pmm3-14502-monitoring-pmm3-rs0-0","login":"sa-1-pmm-agent-sa-monitoring-pmm3-14502-monitoring-pmm3-rs0-0","orgId":1,"isDisabled":false,"role":"Admin","tokens":1,"avatarUrl":"/graph/avatar/154d998f2404ba71756b5a57b1ed46a2"},{"id":41,"uid":"bfsc5jjse5mo0b","name":"pmm-agent-sa-monitoring-pmm3-14502-monitoring-pmm3-rs0-1","login":"sa-1-pmm-agent-sa-monitoring-pmm3-14502-monitoring-pmm3-rs0-1","orgId":1,"isDisabled":false,"role":"Admin","tokens":1,"avatarUrl":"/graph/avatar/771b93d874394ab177dede7f6c72213e"},{"id":39,"uid":"afsc5f73yfq4gc","name":"pmm-agent-sa-monitoring-pmm3-14502-monitoring-pmm3-rs0-2","login":"sa-1-pmm-agent-sa-monitoring-pmm3-14502-monitoring-pmm3-rs0-2","orgId":1,"isDisabled":false,"role":"Admin","tokens":1,"avatarUrl":"/graph/avatar/7d9808b99e79cff9b8c67db8f8f1622f"}],"page":1,"perPage":1000}\n200' ++ echo $'{"totalCount":11,"serviceAccounts":[{"id":2,"uid":"afsc39pmk0a9sb","name":"operator","login":"sa-1-operator","orgId":1,"isDisabled":false,"role":"Admin","tokens":1,"avatarUrl":"/graph/avatar/f741e7abdd8def4ed30a37f602429df3"},{"id":42,"uid":"efsc5kx7kc0zkd","name":"operator_new","login":"sa-1-operator_new","orgId":1,"isDisabled":false,"role":"Admin","tokens":1,"avatarUrl":"/graph/avatar/6d1b2db67a8f3e06db4ae076406fece5"},{"id":37,"uid":"bfsc5bf8bs16oe","name":"pmm-agent-sa-monitoring-pmm3-14502-monitoring-pmm3-cfg-0","login":"sa-1-pmm-agent-sa-monitoring-pmm3-14502-monitoring-pmm3-cfg-0","orgId":1,"isDisabled":false,"role":"Admin","tokens":1,"avatarUrl":"/graph/avatar/ff80b4fbcd0e447728f468eb24e1949f"},{"id":36,"uid":"cfsc59j5wdxq8e","name":"pmm-agent-sa-monitoring-pmm3-14502-monitoring-pmm3-cfg-1","login":"sa-1-pmm-agent-sa-monitoring-pmm3-14502-monitoring-pmm3-cfg-1","orgId":1,"isDisabled":false,"role":"Admin","tokens":1,"avatarUrl":"/graph/avatar/adf7fd6b896b1a9b7906dc6cc9531553"},{"id":38,"uid":"dfsc5dk1vuz28f","name":"pmm-agent-sa-monitoring-pmm3-14502-monitoring-pmm3-cfg-2","login":"sa-1-pmm-agent-sa-monitoring-pmm3-14502-monitoring-pmm3-cfg-2","orgId":1,"isDisabled":false,"role":"Admin","tokens":1,"avatarUrl":"/graph/avatar/3b87d0594e2ab224169ad4c149084c97"},{"id":29,"uid":"dfsc4w6fvv6yoa","name":"pmm-agent-sa-monitoring-pmm3-14502-monitoring-pmm3-mongos-0","login":"sa-1-pmm-agent-sa-monitoring-pmm3-14502-monitoring-pmm3-mongos-0","orgId":1,"isDisabled":false,"role":"Admin","tokens":1,"avatarUrl":"/graph/avatar/c9fe7bfcc8e4476abe1dd933e9158d8d"},{"id":28,"uid":"dfsc4umww4h6of","name":"pmm-agent-sa-monitoring-pmm3-14502-monitoring-pmm3-mongos-1","login":"sa-1-pmm-agent-sa-monitoring-pmm3-14502-monitoring-pmm3-mongos-1","orgId":1,"isDisabled":false,"role":"Admin","tokens":1,"avatarUrl":"/graph/avatar/c458e7f6fa74c8c2e7e29f242ed8bcbc"},{"id":27,"uid":"afsc4t3n20d1ce","name":"pmm-agent-sa-monitoring-pmm3-14502-monitoring-pmm3-mongos-2","login":"sa-1-pmm-agent-sa-monitoring-pmm3-14502-monitoring-pmm3-mongos-2","orgId":1,"isDisabled":false,"role":"Admin","tokens":1,"avatarUrl":"/graph/avatar/72079b564397474018d71c100592cd29"},{"id":40,"uid":"ffsc5hzgtkqgwc","name":"pmm-agent-sa-monitoring-pmm3-14502-monitoring-pmm3-rs0-0","login":"sa-1-pmm-agent-sa-monitoring-pmm3-14502-monitoring-pmm3-rs0-0","orgId":1,"isDisabled":false,"role":"Admin","tokens":1,"avatarUrl":"/graph/avatar/154d998f2404ba71756b5a57b1ed46a2"},{"id":41,"uid":"bfsc5jjse5mo0b","name":"pmm-agent-sa-monitoring-pmm3-14502-monitoring-pmm3-rs0-1","login":"sa-1-pmm-agent-sa-monitoring-pmm3-14502-monitoring-pmm3-rs0-1","orgId":1,"isDisabled":false,"role":"Admin","tokens":1,"avatarUrl":"/graph/avatar/771b93d874394ab177dede7f6c72213e"},{"id":39,"uid":"afsc5f73yfq4gc","name":"pmm-agent-sa-monitoring-pmm3-14502-monitoring-pmm3-rs0-2","login":"sa-1-pmm-agent-sa-monitoring-pmm3-14502-monitoring-pmm3-rs0-2","orgId":1,"isDisabled":false,"role":"Admin","tokens":1,"avatarUrl":"/graph/avatar/7d9808b99e79cff9b8c67db8f8f1622f"}],"page":1,"perPage":1000}\n200' ++ tail -n1 + service_accounts_status=200 ++ echo $'{"totalCount":11,"serviceAccounts":[{"id":2,"uid":"afsc39pmk0a9sb","name":"operator","login":"sa-1-operator","orgId":1,"isDisabled":false,"role":"Admin","tokens":1,"avatarUrl":"/graph/avatar/f741e7abdd8def4ed30a37f602429df3"},{"id":42,"uid":"efsc5kx7kc0zkd","name":"operator_new","login":"sa-1-operator_new","orgId":1,"isDisabled":false,"role":"Admin","tokens":1,"avatarUrl":"/graph/avatar/6d1b2db67a8f3e06db4ae076406fece5"},{"id":37,"uid":"bfsc5bf8bs16oe","name":"pmm-agent-sa-monitoring-pmm3-14502-monitoring-pmm3-cfg-0","login":"sa-1-pmm-agent-sa-monitoring-pmm3-14502-monitoring-pmm3-cfg-0","orgId":1,"isDisabled":false,"role":"Admin","tokens":1,"avatarUrl":"/graph/avatar/ff80b4fbcd0e447728f468eb24e1949f"},{"id":36,"uid":"cfsc59j5wdxq8e","name":"pmm-agent-sa-monitoring-pmm3-14502-monitoring-pmm3-cfg-1","login":"sa-1-pmm-agent-sa-monitoring-pmm3-14502-monitoring-pmm3-cfg-1","orgId":1,"isDisabled":false,"role":"Admin","tokens":1,"avatarUrl":"/graph/avatar/adf7fd6b896b1a9b7906dc6cc9531553"},{"id":38,"uid":"dfsc5dk1vuz28f","name":"pmm-agent-sa-monitoring-pmm3-14502-monitoring-pmm3-cfg-2","login":"sa-1-pmm-agent-sa-monitoring-pmm3-14502-monitoring-pmm3-cfg-2","orgId":1,"isDisabled":false,"role":"Admin","tokens":1,"avatarUrl":"/graph/avatar/3b87d0594e2ab224169ad4c149084c97"},{"id":29,"uid":"dfsc4w6fvv6yoa","name":"pmm-agent-sa-monitoring-pmm3-14502-monitoring-pmm3-mongos-0","login":"sa-1-pmm-agent-sa-monitoring-pmm3-14502-monitoring-pmm3-mongos-0","orgId":1,"isDisabled":false,"role":"Admin","tokens":1,"avatarUrl":"/graph/avatar/c9fe7bfcc8e4476abe1dd933e9158d8d"},{"id":28,"uid":"dfsc4umww4h6of","name":"pmm-agent-sa-monitoring-pmm3-14502-monitoring-pmm3-mongos-1","login":"sa-1-pmm-agent-sa-monitoring-pmm3-14502-monitoring-pmm3-mongos-1","orgId":1,"isDisabled":false,"role":"Admin","tokens":1,"avatarUrl":"/graph/avatar/c458e7f6fa74c8c2e7e29f242ed8bcbc"},{"id":27,"uid":"afsc4t3n20d1ce","name":"pmm-agent-sa-monitoring-pmm3-14502-monitoring-pmm3-mongos-2","login":"sa-1-pmm-agent-sa-monitoring-pmm3-14502-monitoring-pmm3-mongos-2","orgId":1,"isDisabled":false,"role":"Admin","tokens":1,"avatarUrl":"/graph/avatar/72079b564397474018d71c100592cd29"},{"id":40,"uid":"ffsc5hzgtkqgwc","name":"pmm-agent-sa-monitoring-pmm3-14502-monitoring-pmm3-rs0-0","login":"sa-1-pmm-agent-sa-monitoring-pmm3-14502-monitoring-pmm3-rs0-0","orgId":1,"isDisabled":false,"role":"Admin","tokens":1,"avatarUrl":"/graph/avatar/154d998f2404ba71756b5a57b1ed46a2"},{"id":41,"uid":"bfsc5jjse5mo0b","name":"pmm-agent-sa-monitoring-pmm3-14502-monitoring-pmm3-rs0-1","login":"sa-1-pmm-agent-sa-monitoring-pmm3-14502-monitoring-pmm3-rs0-1","orgId":1,"isDisabled":false,"role":"Admin","tokens":1,"avatarUrl":"/graph/avatar/771b93d874394ab177dede7f6c72213e"},{"id":39,"uid":"afsc5f73yfq4gc","name":"pmm-agent-sa-monitoring-pmm3-14502-monitoring-pmm3-rs0-2","login":"sa-1-pmm-agent-sa-monitoring-pmm3-14502-monitoring-pmm3-rs0-2","orgId":1,"isDisabled":false,"role":"Admin","tokens":1,"avatarUrl":"/graph/avatar/7d9808b99e79cff9b8c67db8f8f1622f"}],"page":1,"perPage":1000}\n200' ++ sed '$ d' + service_accounts_json='{"totalCount":11,"serviceAccounts":[{"id":2,"uid":"afsc39pmk0a9sb","name":"operator","login":"sa-1-operator","orgId":1,"isDisabled":false,"role":"Admin","tokens":1,"avatarUrl":"/graph/avatar/f741e7abdd8def4ed30a37f602429df3"},{"id":42,"uid":"efsc5kx7kc0zkd","name":"operator_new","login":"sa-1-operator_new","orgId":1,"isDisabled":false,"role":"Admin","tokens":1,"avatarUrl":"/graph/avatar/6d1b2db67a8f3e06db4ae076406fece5"},{"id":37,"uid":"bfsc5bf8bs16oe","name":"pmm-agent-sa-monitoring-pmm3-14502-monitoring-pmm3-cfg-0","login":"sa-1-pmm-agent-sa-monitoring-pmm3-14502-monitoring-pmm3-cfg-0","orgId":1,"isDisabled":false,"role":"Admin","tokens":1,"avatarUrl":"/graph/avatar/ff80b4fbcd0e447728f468eb24e1949f"},{"id":36,"uid":"cfsc59j5wdxq8e","name":"pmm-agent-sa-monitoring-pmm3-14502-monitoring-pmm3-cfg-1","login":"sa-1-pmm-agent-sa-monitoring-pmm3-14502-monitoring-pmm3-cfg-1","orgId":1,"isDisabled":false,"role":"Admin","tokens":1,"avatarUrl":"/graph/avatar/adf7fd6b896b1a9b7906dc6cc9531553"},{"id":38,"uid":"dfsc5dk1vuz28f","name":"pmm-agent-sa-monitoring-pmm3-14502-monitoring-pmm3-cfg-2","login":"sa-1-pmm-agent-sa-monitoring-pmm3-14502-monitoring-pmm3-cfg-2","orgId":1,"isDisabled":false,"role":"Admin","tokens":1,"avatarUrl":"/graph/avatar/3b87d0594e2ab224169ad4c149084c97"},{"id":29,"uid":"dfsc4w6fvv6yoa","name":"pmm-agent-sa-monitoring-pmm3-14502-monitoring-pmm3-mongos-0","login":"sa-1-pmm-agent-sa-monitoring-pmm3-14502-monitoring-pmm3-mongos-0","orgId":1,"isDisabled":false,"role":"Admin","tokens":1,"avatarUrl":"/graph/avatar/c9fe7bfcc8e4476abe1dd933e9158d8d"},{"id":28,"uid":"dfsc4umww4h6of","name":"pmm-agent-sa-monitoring-pmm3-14502-monitoring-pmm3-mongos-1","login":"sa-1-pmm-agent-sa-monitoring-pmm3-14502-monitoring-pmm3-mongos-1","orgId":1,"isDisabled":false,"role":"Admin","tokens":1,"avatarUrl":"/graph/avatar/c458e7f6fa74c8c2e7e29f242ed8bcbc"},{"id":27,"uid":"afsc4t3n20d1ce","name":"pmm-agent-sa-monitoring-pmm3-14502-monitoring-pmm3-mongos-2","login":"sa-1-pmm-agent-sa-monitoring-pmm3-14502-monitoring-pmm3-mongos-2","orgId":1,"isDisabled":false,"role":"Admin","tokens":1,"avatarUrl":"/graph/avatar/72079b564397474018d71c100592cd29"},{"id":40,"uid":"ffsc5hzgtkqgwc","name":"pmm-agent-sa-monitoring-pmm3-14502-monitoring-pmm3-rs0-0","login":"sa-1-pmm-agent-sa-monitoring-pmm3-14502-monitoring-pmm3-rs0-0","orgId":1,"isDisabled":false,"role":"Admin","tokens":1,"avatarUrl":"/graph/avatar/154d998f2404ba71756b5a57b1ed46a2"},{"id":41,"uid":"bfsc5jjse5mo0b","name":"pmm-agent-sa-monitoring-pmm3-14502-monitoring-pmm3-rs0-1","login":"sa-1-pmm-agent-sa-monitoring-pmm3-14502-monitoring-pmm3-rs0-1","orgId":1,"isDisabled":false,"role":"Admin","tokens":1,"avatarUrl":"/graph/avatar/771b93d874394ab177dede7f6c72213e"},{"id":39,"uid":"afsc5f73yfq4gc","name":"pmm-agent-sa-monitoring-pmm3-14502-monitoring-pmm3-rs0-2","login":"sa-1-pmm-agent-sa-monitoring-pmm3-14502-monitoring-pmm3-rs0-2","orgId":1,"isDisabled":false,"role":"Admin","tokens":1,"avatarUrl":"/graph/avatar/7d9808b99e79cff9b8c67db8f8f1622f"}],"page":1,"perPage":1000}' + [[ 200 -ne 200 ]] + local service_account_id ++ echo -n '{"totalCount":11,"serviceAccounts":[{"id":2,"uid":"afsc39pmk0a9sb","name":"operator","login":"sa-1-operator","orgId":1,"isDisabled":false,"role":"Admin","tokens":1,"avatarUrl":"/graph/avatar/f741e7abdd8def4ed30a37f602429df3"},{"id":42,"uid":"efsc5kx7kc0zkd","name":"operator_new","login":"sa-1-operator_new","orgId":1,"isDisabled":false,"role":"Admin","tokens":1,"avatarUrl":"/graph/avatar/6d1b2db67a8f3e06db4ae076406fece5"},{"id":37,"uid":"bfsc5bf8bs16oe","name":"pmm-agent-sa-monitoring-pmm3-14502-monitoring-pmm3-cfg-0","login":"sa-1-pmm-agent-sa-monitoring-pmm3-14502-monitoring-pmm3-cfg-0","orgId":1,"isDisabled":false,"role":"Admin","tokens":1,"avatarUrl":"/graph/avatar/ff80b4fbcd0e447728f468eb24e1949f"},{"id":36,"uid":"cfsc59j5wdxq8e","name":"pmm-agent-sa-monitoring-pmm3-14502-monitoring-pmm3-cfg-1","login":"sa-1-pmm-agent-sa-monitoring-pmm3-14502-monitoring-pmm3-cfg-1","orgId":1,"isDisabled":false,"role":"Admin","tokens":1,"avatarUrl":"/graph/avatar/adf7fd6b896b1a9b7906dc6cc9531553"},{"id":38,"uid":"dfsc5dk1vuz28f","name":"pmm-agent-sa-monitoring-pmm3-14502-monitoring-pmm3-cfg-2","login":"sa-1-pmm-agent-sa-monitoring-pmm3-14502-monitoring-pmm3-cfg-2","orgId":1,"isDisabled":false,"role":"Admin","tokens":1,"avatarUrl":"/graph/avatar/3b87d0594e2ab224169ad4c149084c97"},{"id":29,"uid":"dfsc4w6fvv6yoa","name":"pmm-agent-sa-monitoring-pmm3-14502-monitoring-pmm3-mongos-0","login":"sa-1-pmm-agent-sa-monitoring-pmm3-14502-monitoring-pmm3-mongos-0","orgId":1,"isDisabled":false,"role":"Admin","tokens":1,"avatarUrl":"/graph/avatar/c9fe7bfcc8e4476abe1dd933e9158d8d"},{"id":28,"uid":"dfsc4umww4h6of","name":"pmm-agent-sa-monitoring-pmm3-14502-monitoring-pmm3-mongos-1","login":"sa-1-pmm-agent-sa-monitoring-pmm3-14502-monitoring-pmm3-mongos-1","orgId":1,"isDisabled":false,"role":"Admin","tokens":1,"avatarUrl":"/graph/avatar/c458e7f6fa74c8c2e7e29f242ed8bcbc"},{"id":27,"uid":"afsc4t3n20d1ce","name":"pmm-agent-sa-monitoring-pmm3-14502-monitoring-pmm3-mongos-2","login":"sa-1-pmm-agent-sa-monitoring-pmm3-14502-monitoring-pmm3-mongos-2","orgId":1,"isDisabled":false,"role":"Admin","tokens":1,"avatarUrl":"/graph/avatar/72079b564397474018d71c100592cd29"},{"id":40,"uid":"ffsc5hzgtkqgwc","name":"pmm-agent-sa-monitoring-pmm3-14502-monitoring-pmm3-rs0-0","login":"sa-1-pmm-agent-sa-monitoring-pmm3-14502-monitoring-pmm3-rs0-0","orgId":1,"isDisabled":false,"role":"Admin","tokens":1,"avatarUrl":"/graph/avatar/154d998f2404ba71756b5a57b1ed46a2"},{"id":41,"uid":"bfsc5jjse5mo0b","name":"pmm-agent-sa-monitoring-pmm3-14502-monitoring-pmm3-rs0-1","login":"sa-1-pmm-agent-sa-monitoring-pmm3-14502-monitoring-pmm3-rs0-1","orgId":1,"isDisabled":false,"role":"Admin","tokens":1,"avatarUrl":"/graph/avatar/771b93d874394ab177dede7f6c72213e"},{"id":39,"uid":"afsc5f73yfq4gc","name":"pmm-agent-sa-monitoring-pmm3-14502-monitoring-pmm3-rs0-2","login":"sa-1-pmm-agent-sa-monitoring-pmm3-14502-monitoring-pmm3-rs0-2","orgId":1,"isDisabled":false,"role":"Admin","tokens":1,"avatarUrl":"/graph/avatar/7d9808b99e79cff9b8c67db8f8f1622f"}],"page":1,"perPage":1000}' ++ jq -r --arg key_name operator '.serviceAccounts[] | select(.name == $key_name).id' ++ head -n1 + service_account_id=2 + [[ -z 2 ]] + [[ 2 == null ]] + local tokens_response tokens_status tokens_json ++ execute_curl --insecure -s -X GET https://admin:J%3Ak%25i1RN%3Dxa%5EZ5a%21@136.112.205.251/graph/api/serviceaccounts/2/tokens -w '\n%{http_code}' ++ local name=curl-13481 ++ local image=docker.io/alpine/curl:latest ++ local phase= ++ local exit_status=0 ++ kubectl_bin run curl-13481 -n monitoring-pmm3-14502 --restart=Never --image=docker.io/alpine/curl:latest --command -- curl -sS --insecure -s -X GET https://admin:J%3Ak%25i1RN%3Dxa%5EZ5a%21@136.112.205.251/graph/api/serviceaccounts/2/tokens -w '\n%{http_code}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.UcZPSFui4h +++ mktemp ++ local LAST_ERR=/tmp/tmp.OosC6KgpMV ++ local exit_status=0 ++ local timeout=4 ++ args=('run' 'curl-13481' '-n' 'monitoring-pmm3-14502' '--restart=Never' '--image=docker.io/alpine/curl:latest' '--command' '--' 'curl' '-sS' '--insecure' '-s' '-X' 'GET' 'https://admin:J%3Ak%25i1RN%3Dxa%5EZ5a%21@136.112.205.251/graph/api/serviceaccounts/2/tokens' '-w' '\n%{http_code}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl run curl-13481 -n monitoring-pmm3-14502 --restart=Never --image=docker.io/alpine/curl:latest --command -- curl -sS --insecure -s -X GET https://admin:J%3Ak%25i1RN%3Dxa%5EZ5a%21@136.112.205.251/graph/api/serviceaccounts/2/tokens -w '\n%{http_code}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.UcZPSFui4h ++ cat /tmp/tmp.OosC6KgpMV ++ rm /tmp/tmp.UcZPSFui4h /tmp/tmp.OosC6KgpMV ++ return 0 +++ seq 1 60 ++ for _ in $(seq 1 60) +++ kubectl_bin get pod curl-13481 -n monitoring-pmm3-14502 -o 'jsonpath={.status.phase}' ++ phase=Pending ++ [[ Pending == Succeeded ]] ++ [[ Pending == Failed ]] ++ sleep 1 ++ for _ in $(seq 1 60) +++ kubectl_bin get pod curl-13481 -n monitoring-pmm3-14502 -o 'jsonpath={.status.phase}' ++ phase=Succeeded ++ [[ Succeeded == Succeeded ]] ++ break ++ kubectl_bin logs curl-13481 -n monitoring-pmm3-14502 +++ mktemp ++ local LAST_OUT=/tmp/tmp.QfZdNafhFg +++ mktemp ++ local LAST_ERR=/tmp/tmp.Dv8FgyuOdm ++ local exit_status=0 ++ local timeout=4 ++ args=('logs' 'curl-13481' '-n' 'monitoring-pmm3-14502') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl logs curl-13481 -n monitoring-pmm3-14502 ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.QfZdNafhFg ++ cat /tmp/tmp.Dv8FgyuOdm ++ rm /tmp/tmp.QfZdNafhFg /tmp/tmp.Dv8FgyuOdm ++ return 0 ++ kubectl_bin delete pod curl-13481 -n monitoring-pmm3-14502 --ignore-not-found --wait=false ++ [[ Succeeded != Succeeded ]] ++ return 0 + tokens_response=$'[{"id":1,"name":"operator","created":"2026-07-17T01:18:51Z","lastUsedAt":"2026-07-17T01:44:42Z","expiration":null,"secondsUntilExpiration":0,"hasExpired":false,"isRevoked":false}]\n200' ++ echo $'[{"id":1,"name":"operator","created":"2026-07-17T01:18:51Z","lastUsedAt":"2026-07-17T01:44:42Z","expiration":null,"secondsUntilExpiration":0,"hasExpired":false,"isRevoked":false}]\n200' ++ tail -n1 + tokens_status=200 ++ echo $'[{"id":1,"name":"operator","created":"2026-07-17T01:18:51Z","lastUsedAt":"2026-07-17T01:44:42Z","expiration":null,"secondsUntilExpiration":0,"hasExpired":false,"isRevoked":false}]\n200' ++ sed '$ d' + tokens_json='[{"id":1,"name":"operator","created":"2026-07-17T01:18:51Z","lastUsedAt":"2026-07-17T01:44:42Z","expiration":null,"secondsUntilExpiration":0,"hasExpired":false,"isRevoked":false}]' + [[ 200 -ne 200 ]] + local token_id ++ echo -n '[{"id":1,"name":"operator","created":"2026-07-17T01:18:51Z","lastUsedAt":"2026-07-17T01:44:42Z","expiration":null,"secondsUntilExpiration":0,"hasExpired":false,"isRevoked":false}]' ++ jq -r --arg key_name operator '.[] | select(.name == $key_name).id' ++ head -n1 + token_id=1 + [[ -z 1 ]] + [[ 1 == null ]] + local delete_response delete_status ++ execute_curl --insecure -s -X DELETE https://admin:J%3Ak%25i1RN%3Dxa%5EZ5a%21@136.112.205.251/graph/api/serviceaccounts/2/tokens/1 -w '\n%{http_code}' ++ local name=curl-8157 ++ local image=docker.io/alpine/curl:latest ++ local phase= ++ local exit_status=0 ++ kubectl_bin run curl-8157 -n monitoring-pmm3-14502 --restart=Never --image=docker.io/alpine/curl:latest --command -- curl -sS --insecure -s -X DELETE https://admin:J%3Ak%25i1RN%3Dxa%5EZ5a%21@136.112.205.251/graph/api/serviceaccounts/2/tokens/1 -w '\n%{http_code}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.3vDRI3IocI +++ mktemp ++ local LAST_ERR=/tmp/tmp.NIbaw8OlA5 ++ local exit_status=0 ++ local timeout=4 ++ args=('run' 'curl-8157' '-n' 'monitoring-pmm3-14502' '--restart=Never' '--image=docker.io/alpine/curl:latest' '--command' '--' 'curl' '-sS' '--insecure' '-s' '-X' 'DELETE' 'https://admin:J%3Ak%25i1RN%3Dxa%5EZ5a%21@136.112.205.251/graph/api/serviceaccounts/2/tokens/1' '-w' '\n%{http_code}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl run curl-8157 -n monitoring-pmm3-14502 --restart=Never --image=docker.io/alpine/curl:latest --command -- curl -sS --insecure -s -X DELETE https://admin:J%3Ak%25i1RN%3Dxa%5EZ5a%21@136.112.205.251/graph/api/serviceaccounts/2/tokens/1 -w '\n%{http_code}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.3vDRI3IocI ++ cat /tmp/tmp.NIbaw8OlA5 ++ rm /tmp/tmp.3vDRI3IocI /tmp/tmp.NIbaw8OlA5 ++ return 0 +++ seq 1 60 ++ for _ in $(seq 1 60) +++ kubectl_bin get pod curl-8157 -n monitoring-pmm3-14502 -o 'jsonpath={.status.phase}' ++ phase=Running ++ [[ Running == Succeeded ]] ++ [[ Running == Failed ]] ++ sleep 1 ++ for _ in $(seq 1 60) +++ kubectl_bin get pod curl-8157 -n monitoring-pmm3-14502 -o 'jsonpath={.status.phase}' ++ phase=Running ++ [[ Running == Succeeded ]] ++ [[ Running == Failed ]] ++ sleep 1 ++ for _ in $(seq 1 60) +++ kubectl_bin get pod curl-8157 -n monitoring-pmm3-14502 -o 'jsonpath={.status.phase}' ++ phase=Succeeded ++ [[ Succeeded == Succeeded ]] ++ break ++ kubectl_bin logs curl-8157 -n monitoring-pmm3-14502 +++ mktemp ++ local LAST_OUT=/tmp/tmp.YUnEFomp6q +++ mktemp ++ local LAST_ERR=/tmp/tmp.oacF06CKOn ++ local exit_status=0 ++ local timeout=4 ++ args=('logs' 'curl-8157' '-n' 'monitoring-pmm3-14502') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl logs curl-8157 -n monitoring-pmm3-14502 ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.YUnEFomp6q ++ cat /tmp/tmp.oacF06CKOn ++ rm /tmp/tmp.YUnEFomp6q /tmp/tmp.oacF06CKOn ++ return 0 ++ kubectl_bin delete pod curl-8157 -n monitoring-pmm3-14502 --ignore-not-found --wait=false ++ [[ Succeeded != Succeeded ]] ++ return 0 + delete_response=$'{"message":"Service account token deleted"}\n200' ++ echo $'{"message":"Service account token deleted"}\n200' ++ tail -n1 + delete_status=200 + [[ 200 -ne 200 ]] + desc 'check for authentication errors in PMM client logs' + set +o xtrace ----------------------------------------------------------------------------------- check for authentication errors in PMM client logs ----------------------------------------------------------------------------------- ++ kubectl get pods --selector=app.kubernetes.io/replset=rs0 -o 'jsonpath={.items[*].metadata.name}' + pods='monitoring-pmm3-rs0-0 monitoring-pmm3-rs0-1 monitoring-pmm3-rs0-2' + for pod in $pods + for i in {1..3} + kubectl logs monitoring-pmm3-rs0-0 pmm-client + grep -q 'Invalid username or password' + sleep 2 + for i in {1..3} + kubectl logs monitoring-pmm3-rs0-0 pmm-client + grep -q 'Invalid username or password' + sleep 2 + for i in {1..3} + kubectl logs monitoring-pmm3-rs0-0 pmm-client + grep -q 'Invalid username or password' + sleep 2 + for pod in $pods + for i in {1..3} + kubectl logs monitoring-pmm3-rs0-1 pmm-client + grep -q 'Invalid username or password' + sleep 2 + for i in {1..3} + kubectl logs monitoring-pmm3-rs0-1 pmm-client + grep -q 'Invalid username or password' + sleep 2 + for i in {1..3} + kubectl logs monitoring-pmm3-rs0-1 pmm-client + grep -q 'Invalid username or password' + sleep 2 + for pod in $pods + for i in {1..3} + kubectl logs monitoring-pmm3-rs0-2 pmm-client + grep -q 'Invalid username or password' + sleep 2 + for i in {1..3} + kubectl logs monitoring-pmm3-rs0-2 pmm-client + grep -q 'Invalid username or password' + sleep 2 + for i in {1..3} + kubectl logs monitoring-pmm3-rs0-2 pmm-client + grep -q 'Invalid username or password' + sleep 2 + desc 'check mongod metrics' + set +o xtrace ----------------------------------------------------------------------------------- check mongod metrics ----------------------------------------------------------------------------------- + get_metric_values node_boot_time_seconds monitoring-pmm3-14502-monitoring-pmm3-rs0-1 glsa_422RcuDdDskb7TLtapwWaOqcyrTCF2do_8c6a26b0 bearer + local metric=node_boot_time_seconds + local instance=monitoring-pmm3-14502-monitoring-pmm3-rs0-1 + local user_pass=glsa_422RcuDdDskb7TLtapwWaOqcyrTCF2do_8c6a26b0 + local auth_type=bearer + local endpoint ++ get_service_endpoint monitoring-service ++ local service=monitoring-service ++ local endpoint ++ local service_type +++ kubectl_bin get service/monitoring-service -o 'jsonpath={.spec.type}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.aYXBcqD4uz ++++ mktemp +++ local LAST_ERR=/tmp/tmp.jcZFz58WJd +++ local exit_status=0 +++ local timeout=4 +++ args=('get' 'service/monitoring-service' '-o' 'jsonpath={.spec.type}') +++ local args +++ transformed_files=() +++ local transformed_files +++ arch_scheduling_enabled +++ [[ -n '' ]] +++ arch_scheduling_enabled +++ [[ -n '' ]] ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl get service/monitoring-service -o 'jsonpath={.spec.type}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 -a -n 1 ']' +++ break +++ cat /tmp/tmp.aYXBcqD4uz +++ cat /tmp/tmp.jcZFz58WJd +++ rm /tmp/tmp.aYXBcqD4uz /tmp/tmp.jcZFz58WJd +++ return 0 ++ service_type=LoadBalancer ++ [[ LoadBalancer == LoadBalancer ]] +++ get_load_balancer_endpoint monitoring-service +++ local service=monitoring-service +++ local endpoint ++++ kubectl_bin get service/monitoring-service -o json ++++ jq -r '[.status.loadBalancer.ingress[]?.hostname, .status.loadBalancer.ingress[]?.ip] | map(select(. != null and . != "")) | first // empty' +++++ mktemp ++++ local LAST_OUT=/tmp/tmp.yP28QxSroE +++++ mktemp ++++ local LAST_ERR=/tmp/tmp.ZrqiYWXrmA ++++ local exit_status=0 ++++ local timeout=4 ++++ args=('get' 'service/monitoring-service' '-o' 'json') ++++ local args ++++ transformed_files=() ++++ local transformed_files ++++ arch_scheduling_enabled ++++ [[ -n '' ]] ++++ arch_scheduling_enabled ++++ [[ -n '' ]] +++++ seq 0 2 ++++ for i in $(seq 0 2) ++++ set +e ++++ kubectl get service/monitoring-service -o json ++++ exit_status=0 ++++ set -e ++++ '[' 0 '!=' 0 -a -n 1 ']' ++++ break ++++ cat /tmp/tmp.yP28QxSroE ++++ cat /tmp/tmp.ZrqiYWXrmA ++++ rm /tmp/tmp.yP28QxSroE /tmp/tmp.ZrqiYWXrmA ++++ return 0 +++ endpoint=136.112.205.251 +++ '[' -n 136.112.205.251 -a 136.112.205.251 '!=' null ']' +++ echo 136.112.205.251 +++ return ++ endpoint=136.112.205.251 ++ echo 136.112.205.251 ++ return + endpoint=136.112.205.251 + retry 30 2 curl_metric_values node_boot_time_seconds monitoring-pmm3-14502-monitoring-pmm3-rs0-1 136.112.205.251 glsa_422RcuDdDskb7TLtapwWaOqcyrTCF2do_8c6a26b0 bearer + local max=30 + local delay=2 + shift 2 + local n=1 + curl_metric_values node_boot_time_seconds monitoring-pmm3-14502-monitoring-pmm3-rs0-1 136.112.205.251 glsa_422RcuDdDskb7TLtapwWaOqcyrTCF2do_8c6a26b0 bearer + local metric=node_boot_time_seconds + local instance=monitoring-pmm3-14502-monitoring-pmm3-rs0-1 + local endpoint=136.112.205.251 + local credentials=glsa_422RcuDdDskb7TLtapwWaOqcyrTCF2do_8c6a26b0 + local auth_type=bearer + local start + local end + local url + curl_args=() + local curl_args ++ /usr/sbin/date -u +%s -d '-1 minute' + start=1784252688 ++ /usr/sbin/date -u +%s + end=1784252748 + url='https://136.112.205.251/graph/api/datasources/proxy/1/api/v1/query_range?query=min%28node_boot_time_seconds%7Bnode_name%3D%7E%22monitoring-pmm3-14502-monitoring-pmm3-rs0-1%22%7D%20or%20node_boot_time_seconds%7Bnode_name%3D%7E%22monitoring-pmm3-14502-monitoring-pmm3-rs0-1%22%7D%29&start=1784252688&end=1784252748&step=60' + [[ bearer == bearer ]] + curl_args=(-H "Authorization: Bearer ${credentials}" "$url") + jq -r '(.data.result[0].values // [])[] | .[1] // empty' + execute_curl -s -k -H 'Authorization: Bearer glsa_422RcuDdDskb7TLtapwWaOqcyrTCF2do_8c6a26b0' 'https://136.112.205.251/graph/api/datasources/proxy/1/api/v1/query_range?query=min%28node_boot_time_seconds%7Bnode_name%3D%7E%22monitoring-pmm3-14502-monitoring-pmm3-rs0-1%22%7D%20or%20node_boot_time_seconds%7Bnode_name%3D%7E%22monitoring-pmm3-14502-monitoring-pmm3-rs0-1%22%7D%29&start=1784252688&end=1784252748&step=60' + grep '^[0-9]' + local name=curl-14542 + local image=docker.io/alpine/curl:latest + local phase= + local exit_status=0 + kubectl_bin run curl-14542 -n monitoring-pmm3-14502 --restart=Never --image=docker.io/alpine/curl:latest --command -- curl -sS -s -k -H 'Authorization: Bearer glsa_422RcuDdDskb7TLtapwWaOqcyrTCF2do_8c6a26b0' 'https://136.112.205.251/graph/api/datasources/proxy/1/api/v1/query_range?query=min%28node_boot_time_seconds%7Bnode_name%3D%7E%22monitoring-pmm3-14502-monitoring-pmm3-rs0-1%22%7D%20or%20node_boot_time_seconds%7Bnode_name%3D%7E%22monitoring-pmm3-14502-monitoring-pmm3-rs0-1%22%7D%29&start=1784252688&end=1784252748&step=60' ++ mktemp + local LAST_OUT=/tmp/tmp.20H47buSn7 ++ mktemp + local LAST_ERR=/tmp/tmp.T2InFrJHjN + local exit_status=0 + local timeout=4 + args=('run' 'curl-14542' '-n' 'monitoring-pmm3-14502' '--restart=Never' '--image=docker.io/alpine/curl:latest' '--command' '--' 'curl' '-sS' '-s' '-k' '-H' 'Authorization: Bearer glsa_422RcuDdDskb7TLtapwWaOqcyrTCF2do_8c6a26b0' 'https://136.112.205.251/graph/api/datasources/proxy/1/api/v1/query_range?query=min%28node_boot_time_seconds%7Bnode_name%3D%7E%22monitoring-pmm3-14502-monitoring-pmm3-rs0-1%22%7D%20or%20node_boot_time_seconds%7Bnode_name%3D%7E%22monitoring-pmm3-14502-monitoring-pmm3-rs0-1%22%7D%29&start=1784252688&end=1784252748&step=60') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl run curl-14542 -n monitoring-pmm3-14502 --restart=Never --image=docker.io/alpine/curl:latest --command -- curl -sS -s -k -H 'Authorization: Bearer glsa_422RcuDdDskb7TLtapwWaOqcyrTCF2do_8c6a26b0' 'https://136.112.205.251/graph/api/datasources/proxy/1/api/v1/query_range?query=min%28node_boot_time_seconds%7Bnode_name%3D%7E%22monitoring-pmm3-14502-monitoring-pmm3-rs0-1%22%7D%20or%20node_boot_time_seconds%7Bnode_name%3D%7E%22monitoring-pmm3-14502-monitoring-pmm3-rs0-1%22%7D%29&start=1784252688&end=1784252748&step=60' + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.20H47buSn7 + cat /tmp/tmp.T2InFrJHjN + rm /tmp/tmp.20H47buSn7 /tmp/tmp.T2InFrJHjN + return 0 ++ seq 1 60 + for _ in $(seq 1 60) ++ kubectl_bin get pod curl-14542 -n monitoring-pmm3-14502 -o 'jsonpath={.status.phase}' + phase=Pending + [[ Pending == Succeeded ]] + [[ Pending == Failed ]] + sleep 1 + for _ in $(seq 1 60) ++ kubectl_bin get pod curl-14542 -n monitoring-pmm3-14502 -o 'jsonpath={.status.phase}' + phase=Succeeded + [[ Succeeded == Succeeded ]] + break + kubectl_bin logs curl-14542 -n monitoring-pmm3-14502 ++ mktemp + local LAST_OUT=/tmp/tmp.XDlOgpxyDG ++ mktemp + local LAST_ERR=/tmp/tmp.6aZMXkIG7Y + local exit_status=0 + local timeout=4 + args=('logs' 'curl-14542' '-n' 'monitoring-pmm3-14502') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl logs curl-14542 -n monitoring-pmm3-14502 + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.XDlOgpxyDG + cat /tmp/tmp.6aZMXkIG7Y + rm /tmp/tmp.XDlOgpxyDG /tmp/tmp.6aZMXkIG7Y + return 0 + kubectl_bin delete pod curl-14542 -n monitoring-pmm3-14502 --ignore-not-found --wait=false + [[ Succeeded != Succeeded ]] + return 0 1784246222 1784246222 + get_metric_values mongodb_connections monitoring-pmm3-14502-monitoring-pmm3-rs0-1 glsa_422RcuDdDskb7TLtapwWaOqcyrTCF2do_8c6a26b0 bearer + local metric=mongodb_connections + local instance=monitoring-pmm3-14502-monitoring-pmm3-rs0-1 + local user_pass=glsa_422RcuDdDskb7TLtapwWaOqcyrTCF2do_8c6a26b0 + local auth_type=bearer + local endpoint ++ get_service_endpoint monitoring-service ++ local service=monitoring-service ++ local endpoint ++ local service_type +++ kubectl_bin get service/monitoring-service -o 'jsonpath={.spec.type}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.3dT7ycHVLu ++++ mktemp +++ local LAST_ERR=/tmp/tmp.9v3PVJiBdz +++ local exit_status=0 +++ local timeout=4 +++ args=('get' 'service/monitoring-service' '-o' 'jsonpath={.spec.type}') +++ local args +++ transformed_files=() +++ local transformed_files +++ arch_scheduling_enabled +++ [[ -n '' ]] +++ arch_scheduling_enabled +++ [[ -n '' ]] ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl get service/monitoring-service -o 'jsonpath={.spec.type}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 -a -n 1 ']' +++ break +++ cat /tmp/tmp.3dT7ycHVLu +++ cat /tmp/tmp.9v3PVJiBdz +++ rm /tmp/tmp.3dT7ycHVLu /tmp/tmp.9v3PVJiBdz +++ return 0 ++ service_type=LoadBalancer ++ [[ LoadBalancer == LoadBalancer ]] +++ get_load_balancer_endpoint monitoring-service +++ local service=monitoring-service +++ local endpoint ++++ kubectl_bin get service/monitoring-service -o json ++++ jq -r '[.status.loadBalancer.ingress[]?.hostname, .status.loadBalancer.ingress[]?.ip] | map(select(. != null and . != "")) | first // empty' +++++ mktemp ++++ local LAST_OUT=/tmp/tmp.nqanvor0XQ +++++ mktemp ++++ local LAST_ERR=/tmp/tmp.feU5FqBRnn ++++ local exit_status=0 ++++ local timeout=4 ++++ args=('get' 'service/monitoring-service' '-o' 'json') ++++ local args ++++ transformed_files=() ++++ local transformed_files ++++ arch_scheduling_enabled ++++ [[ -n '' ]] ++++ arch_scheduling_enabled ++++ [[ -n '' ]] +++++ seq 0 2 ++++ for i in $(seq 0 2) ++++ set +e ++++ kubectl get service/monitoring-service -o json ++++ exit_status=0 ++++ set -e ++++ '[' 0 '!=' 0 -a -n 1 ']' ++++ break ++++ cat /tmp/tmp.nqanvor0XQ ++++ cat /tmp/tmp.feU5FqBRnn ++++ rm /tmp/tmp.nqanvor0XQ /tmp/tmp.feU5FqBRnn ++++ return 0 +++ endpoint=136.112.205.251 +++ '[' -n 136.112.205.251 -a 136.112.205.251 '!=' null ']' +++ echo 136.112.205.251 +++ return ++ endpoint=136.112.205.251 ++ echo 136.112.205.251 ++ return + endpoint=136.112.205.251 + retry 30 2 curl_metric_values mongodb_connections monitoring-pmm3-14502-monitoring-pmm3-rs0-1 136.112.205.251 glsa_422RcuDdDskb7TLtapwWaOqcyrTCF2do_8c6a26b0 bearer + local max=30 + local delay=2 + shift 2 + local n=1 + curl_metric_values mongodb_connections monitoring-pmm3-14502-monitoring-pmm3-rs0-1 136.112.205.251 glsa_422RcuDdDskb7TLtapwWaOqcyrTCF2do_8c6a26b0 bearer + local metric=mongodb_connections + local instance=monitoring-pmm3-14502-monitoring-pmm3-rs0-1 + local endpoint=136.112.205.251 + local credentials=glsa_422RcuDdDskb7TLtapwWaOqcyrTCF2do_8c6a26b0 + local auth_type=bearer + local start + local end + local url + curl_args=() + local curl_args ++ /usr/sbin/date -u +%s -d '-1 minute' + start=1784252696 ++ /usr/sbin/date -u +%s + end=1784252756 + url='https://136.112.205.251/graph/api/datasources/proxy/1/api/v1/query_range?query=min%28mongodb_connections%7Bnode_name%3D%7E%22monitoring-pmm3-14502-monitoring-pmm3-rs0-1%22%7D%20or%20mongodb_connections%7Bnode_name%3D%7E%22monitoring-pmm3-14502-monitoring-pmm3-rs0-1%22%7D%29&start=1784252696&end=1784252756&step=60' + [[ bearer == bearer ]] + curl_args=(-H "Authorization: Bearer ${credentials}" "$url") + execute_curl -s -k -H 'Authorization: Bearer glsa_422RcuDdDskb7TLtapwWaOqcyrTCF2do_8c6a26b0' 'https://136.112.205.251/graph/api/datasources/proxy/1/api/v1/query_range?query=min%28mongodb_connections%7Bnode_name%3D%7E%22monitoring-pmm3-14502-monitoring-pmm3-rs0-1%22%7D%20or%20mongodb_connections%7Bnode_name%3D%7E%22monitoring-pmm3-14502-monitoring-pmm3-rs0-1%22%7D%29&start=1784252696&end=1784252756&step=60' + jq -r '(.data.result[0].values // [])[] | .[1] // empty' + local name=curl-16587 + local image=docker.io/alpine/curl:latest + local phase= + local exit_status=0 + kubectl_bin run curl-16587 -n monitoring-pmm3-14502 --restart=Never --image=docker.io/alpine/curl:latest --command -- curl -sS -s -k -H 'Authorization: Bearer glsa_422RcuDdDskb7TLtapwWaOqcyrTCF2do_8c6a26b0' 'https://136.112.205.251/graph/api/datasources/proxy/1/api/v1/query_range?query=min%28mongodb_connections%7Bnode_name%3D%7E%22monitoring-pmm3-14502-monitoring-pmm3-rs0-1%22%7D%20or%20mongodb_connections%7Bnode_name%3D%7E%22monitoring-pmm3-14502-monitoring-pmm3-rs0-1%22%7D%29&start=1784252696&end=1784252756&step=60' + grep '^[0-9]' ++ mktemp + local LAST_OUT=/tmp/tmp.LHF0YEQYsd ++ mktemp + local LAST_ERR=/tmp/tmp.Sr4vWdG51q + local exit_status=0 + local timeout=4 + args=('run' 'curl-16587' '-n' 'monitoring-pmm3-14502' '--restart=Never' '--image=docker.io/alpine/curl:latest' '--command' '--' 'curl' '-sS' '-s' '-k' '-H' 'Authorization: Bearer glsa_422RcuDdDskb7TLtapwWaOqcyrTCF2do_8c6a26b0' 'https://136.112.205.251/graph/api/datasources/proxy/1/api/v1/query_range?query=min%28mongodb_connections%7Bnode_name%3D%7E%22monitoring-pmm3-14502-monitoring-pmm3-rs0-1%22%7D%20or%20mongodb_connections%7Bnode_name%3D%7E%22monitoring-pmm3-14502-monitoring-pmm3-rs0-1%22%7D%29&start=1784252696&end=1784252756&step=60') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl run curl-16587 -n monitoring-pmm3-14502 --restart=Never --image=docker.io/alpine/curl:latest --command -- curl -sS -s -k -H 'Authorization: Bearer glsa_422RcuDdDskb7TLtapwWaOqcyrTCF2do_8c6a26b0' 'https://136.112.205.251/graph/api/datasources/proxy/1/api/v1/query_range?query=min%28mongodb_connections%7Bnode_name%3D%7E%22monitoring-pmm3-14502-monitoring-pmm3-rs0-1%22%7D%20or%20mongodb_connections%7Bnode_name%3D%7E%22monitoring-pmm3-14502-monitoring-pmm3-rs0-1%22%7D%29&start=1784252696&end=1784252756&step=60' + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.LHF0YEQYsd + cat /tmp/tmp.Sr4vWdG51q + rm /tmp/tmp.LHF0YEQYsd /tmp/tmp.Sr4vWdG51q + return 0 ++ seq 1 60 + for _ in $(seq 1 60) ++ kubectl_bin get pod curl-16587 -n monitoring-pmm3-14502 -o 'jsonpath={.status.phase}' + phase=Pending + [[ Pending == Succeeded ]] + [[ Pending == Failed ]] + sleep 1 + for _ in $(seq 1 60) ++ kubectl_bin get pod curl-16587 -n monitoring-pmm3-14502 -o 'jsonpath={.status.phase}' + phase=Succeeded + [[ Succeeded == Succeeded ]] + break + kubectl_bin logs curl-16587 -n monitoring-pmm3-14502 ++ mktemp + local LAST_OUT=/tmp/tmp.VnYl7Obaos ++ mktemp + local LAST_ERR=/tmp/tmp.GeFzTmx2WS + local exit_status=0 + local timeout=4 + args=('logs' 'curl-16587' '-n' 'monitoring-pmm3-14502') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl logs curl-16587 -n monitoring-pmm3-14502 + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.VnYl7Obaos + cat /tmp/tmp.GeFzTmx2WS + rm /tmp/tmp.VnYl7Obaos /tmp/tmp.GeFzTmx2WS + return 0 + kubectl_bin delete pod curl-16587 -n monitoring-pmm3-14502 --ignore-not-found --wait=false + [[ Succeeded != Succeeded ]] + return 0 0 0 + desc 'check mongo config metrics' + set +o xtrace ----------------------------------------------------------------------------------- check mongo config metrics ----------------------------------------------------------------------------------- + get_metric_values node_boot_time_seconds monitoring-pmm3-14502-monitoring-pmm3-cfg-1 glsa_422RcuDdDskb7TLtapwWaOqcyrTCF2do_8c6a26b0 bearer + local metric=node_boot_time_seconds + local instance=monitoring-pmm3-14502-monitoring-pmm3-cfg-1 + local user_pass=glsa_422RcuDdDskb7TLtapwWaOqcyrTCF2do_8c6a26b0 + local auth_type=bearer + local endpoint ++ get_service_endpoint monitoring-service ++ local service=monitoring-service ++ local endpoint ++ local service_type +++ kubectl_bin get service/monitoring-service -o 'jsonpath={.spec.type}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.h9YVTa76C8 ++++ mktemp +++ local LAST_ERR=/tmp/tmp.rJ59GB5tbb +++ local exit_status=0 +++ local timeout=4 +++ args=('get' 'service/monitoring-service' '-o' 'jsonpath={.spec.type}') +++ local args +++ transformed_files=() +++ local transformed_files +++ arch_scheduling_enabled +++ [[ -n '' ]] +++ arch_scheduling_enabled +++ [[ -n '' ]] ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl get service/monitoring-service -o 'jsonpath={.spec.type}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 -a -n 1 ']' +++ break +++ cat /tmp/tmp.h9YVTa76C8 +++ cat /tmp/tmp.rJ59GB5tbb +++ rm /tmp/tmp.h9YVTa76C8 /tmp/tmp.rJ59GB5tbb +++ return 0 ++ service_type=LoadBalancer ++ [[ LoadBalancer == LoadBalancer ]] +++ get_load_balancer_endpoint monitoring-service +++ local service=monitoring-service +++ local endpoint ++++ kubectl_bin get service/monitoring-service -o json ++++ jq -r '[.status.loadBalancer.ingress[]?.hostname, .status.loadBalancer.ingress[]?.ip] | map(select(. != null and . != "")) | first // empty' +++++ mktemp ++++ local LAST_OUT=/tmp/tmp.L3wsUCtGAL +++++ mktemp ++++ local LAST_ERR=/tmp/tmp.CiaTAWDNgQ ++++ local exit_status=0 ++++ local timeout=4 ++++ args=('get' 'service/monitoring-service' '-o' 'json') ++++ local args ++++ transformed_files=() ++++ local transformed_files ++++ arch_scheduling_enabled ++++ [[ -n '' ]] ++++ arch_scheduling_enabled ++++ [[ -n '' ]] +++++ seq 0 2 ++++ for i in $(seq 0 2) ++++ set +e ++++ kubectl get service/monitoring-service -o json ++++ exit_status=0 ++++ set -e ++++ '[' 0 '!=' 0 -a -n 1 ']' ++++ break ++++ cat /tmp/tmp.L3wsUCtGAL ++++ cat /tmp/tmp.CiaTAWDNgQ ++++ rm /tmp/tmp.L3wsUCtGAL /tmp/tmp.CiaTAWDNgQ ++++ return 0 +++ endpoint=136.112.205.251 +++ '[' -n 136.112.205.251 -a 136.112.205.251 '!=' null ']' +++ echo 136.112.205.251 +++ return ++ endpoint=136.112.205.251 ++ echo 136.112.205.251 ++ return + endpoint=136.112.205.251 + retry 30 2 curl_metric_values node_boot_time_seconds monitoring-pmm3-14502-monitoring-pmm3-cfg-1 136.112.205.251 glsa_422RcuDdDskb7TLtapwWaOqcyrTCF2do_8c6a26b0 bearer + local max=30 + local delay=2 + shift 2 + local n=1 + curl_metric_values node_boot_time_seconds monitoring-pmm3-14502-monitoring-pmm3-cfg-1 136.112.205.251 glsa_422RcuDdDskb7TLtapwWaOqcyrTCF2do_8c6a26b0 bearer + local metric=node_boot_time_seconds + local instance=monitoring-pmm3-14502-monitoring-pmm3-cfg-1 + local endpoint=136.112.205.251 + local credentials=glsa_422RcuDdDskb7TLtapwWaOqcyrTCF2do_8c6a26b0 + local auth_type=bearer + local start + local end + local url + curl_args=() + local curl_args ++ /usr/sbin/date -u +%s -d '-1 minute' + start=1784252704 ++ /usr/sbin/date -u +%s + end=1784252764 + url='https://136.112.205.251/graph/api/datasources/proxy/1/api/v1/query_range?query=min%28node_boot_time_seconds%7Bnode_name%3D%7E%22monitoring-pmm3-14502-monitoring-pmm3-cfg-1%22%7D%20or%20node_boot_time_seconds%7Bnode_name%3D%7E%22monitoring-pmm3-14502-monitoring-pmm3-cfg-1%22%7D%29&start=1784252704&end=1784252764&step=60' + [[ bearer == bearer ]] + curl_args=(-H "Authorization: Bearer ${credentials}" "$url") + execute_curl -s -k -H 'Authorization: Bearer glsa_422RcuDdDskb7TLtapwWaOqcyrTCF2do_8c6a26b0' 'https://136.112.205.251/graph/api/datasources/proxy/1/api/v1/query_range?query=min%28node_boot_time_seconds%7Bnode_name%3D%7E%22monitoring-pmm3-14502-monitoring-pmm3-cfg-1%22%7D%20or%20node_boot_time_seconds%7Bnode_name%3D%7E%22monitoring-pmm3-14502-monitoring-pmm3-cfg-1%22%7D%29&start=1784252704&end=1784252764&step=60' + local name=curl-8494 + local image=docker.io/alpine/curl:latest + local phase= + local exit_status=0 + kubectl_bin run curl-8494 -n monitoring-pmm3-14502 --restart=Never --image=docker.io/alpine/curl:latest --command -- curl -sS -s -k -H 'Authorization: Bearer glsa_422RcuDdDskb7TLtapwWaOqcyrTCF2do_8c6a26b0' 'https://136.112.205.251/graph/api/datasources/proxy/1/api/v1/query_range?query=min%28node_boot_time_seconds%7Bnode_name%3D%7E%22monitoring-pmm3-14502-monitoring-pmm3-cfg-1%22%7D%20or%20node_boot_time_seconds%7Bnode_name%3D%7E%22monitoring-pmm3-14502-monitoring-pmm3-cfg-1%22%7D%29&start=1784252704&end=1784252764&step=60' + jq -r '(.data.result[0].values // [])[] | .[1] // empty' + grep '^[0-9]' ++ mktemp + local LAST_OUT=/tmp/tmp.2dgUNOfTXU ++ mktemp + local LAST_ERR=/tmp/tmp.h5LCOpAYUP + local exit_status=0 + local timeout=4 + args=('run' 'curl-8494' '-n' 'monitoring-pmm3-14502' '--restart=Never' '--image=docker.io/alpine/curl:latest' '--command' '--' 'curl' '-sS' '-s' '-k' '-H' 'Authorization: Bearer glsa_422RcuDdDskb7TLtapwWaOqcyrTCF2do_8c6a26b0' 'https://136.112.205.251/graph/api/datasources/proxy/1/api/v1/query_range?query=min%28node_boot_time_seconds%7Bnode_name%3D%7E%22monitoring-pmm3-14502-monitoring-pmm3-cfg-1%22%7D%20or%20node_boot_time_seconds%7Bnode_name%3D%7E%22monitoring-pmm3-14502-monitoring-pmm3-cfg-1%22%7D%29&start=1784252704&end=1784252764&step=60') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl run curl-8494 -n monitoring-pmm3-14502 --restart=Never --image=docker.io/alpine/curl:latest --command -- curl -sS -s -k -H 'Authorization: Bearer glsa_422RcuDdDskb7TLtapwWaOqcyrTCF2do_8c6a26b0' 'https://136.112.205.251/graph/api/datasources/proxy/1/api/v1/query_range?query=min%28node_boot_time_seconds%7Bnode_name%3D%7E%22monitoring-pmm3-14502-monitoring-pmm3-cfg-1%22%7D%20or%20node_boot_time_seconds%7Bnode_name%3D%7E%22monitoring-pmm3-14502-monitoring-pmm3-cfg-1%22%7D%29&start=1784252704&end=1784252764&step=60' + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.2dgUNOfTXU + cat /tmp/tmp.h5LCOpAYUP + rm /tmp/tmp.2dgUNOfTXU /tmp/tmp.h5LCOpAYUP + return 0 ++ seq 1 60 + for _ in $(seq 1 60) ++ kubectl_bin get pod curl-8494 -n monitoring-pmm3-14502 -o 'jsonpath={.status.phase}' + phase=Pending + [[ Pending == Succeeded ]] + [[ Pending == Failed ]] + sleep 1 + for _ in $(seq 1 60) ++ kubectl_bin get pod curl-8494 -n monitoring-pmm3-14502 -o 'jsonpath={.status.phase}' + phase=Succeeded + [[ Succeeded == Succeeded ]] + break + kubectl_bin logs curl-8494 -n monitoring-pmm3-14502 ++ mktemp + local LAST_OUT=/tmp/tmp.elqRywbh8y ++ mktemp + local LAST_ERR=/tmp/tmp.ake6ZwY3bv + local exit_status=0 + local timeout=4 + args=('logs' 'curl-8494' '-n' 'monitoring-pmm3-14502') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl logs curl-8494 -n monitoring-pmm3-14502 + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.elqRywbh8y + cat /tmp/tmp.ake6ZwY3bv + rm /tmp/tmp.elqRywbh8y /tmp/tmp.ake6ZwY3bv + return 0 + kubectl_bin delete pod curl-8494 -n monitoring-pmm3-14502 --ignore-not-found --wait=false + [[ Succeeded != Succeeded ]] + return 0 1784251933 1784251933 + get_metric_values mongodb_connections monitoring-pmm3-14502-monitoring-pmm3-cfg-1 glsa_422RcuDdDskb7TLtapwWaOqcyrTCF2do_8c6a26b0 bearer + local metric=mongodb_connections + local instance=monitoring-pmm3-14502-monitoring-pmm3-cfg-1 + local user_pass=glsa_422RcuDdDskb7TLtapwWaOqcyrTCF2do_8c6a26b0 + local auth_type=bearer + local endpoint ++ get_service_endpoint monitoring-service ++ local service=monitoring-service ++ local endpoint ++ local service_type +++ kubectl_bin get service/monitoring-service -o 'jsonpath={.spec.type}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.cGqQrjUrTa ++++ mktemp +++ local LAST_ERR=/tmp/tmp.XjwBS72zXt +++ local exit_status=0 +++ local timeout=4 +++ args=('get' 'service/monitoring-service' '-o' 'jsonpath={.spec.type}') +++ local args +++ transformed_files=() +++ local transformed_files +++ arch_scheduling_enabled +++ [[ -n '' ]] +++ arch_scheduling_enabled +++ [[ -n '' ]] ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl get service/monitoring-service -o 'jsonpath={.spec.type}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 -a -n 1 ']' +++ break +++ cat /tmp/tmp.cGqQrjUrTa +++ cat /tmp/tmp.XjwBS72zXt +++ rm /tmp/tmp.cGqQrjUrTa /tmp/tmp.XjwBS72zXt +++ return 0 ++ service_type=LoadBalancer ++ [[ LoadBalancer == LoadBalancer ]] +++ get_load_balancer_endpoint monitoring-service +++ local service=monitoring-service +++ local endpoint ++++ kubectl_bin get service/monitoring-service -o json +++++ mktemp ++++ jq -r '[.status.loadBalancer.ingress[]?.hostname, .status.loadBalancer.ingress[]?.ip] | map(select(. != null and . != "")) | first // empty' ++++ local LAST_OUT=/tmp/tmp.ffIFtpDyQ0 +++++ mktemp ++++ local LAST_ERR=/tmp/tmp.tvAforWOYE ++++ local exit_status=0 ++++ local timeout=4 ++++ args=('get' 'service/monitoring-service' '-o' 'json') ++++ local args ++++ transformed_files=() ++++ local transformed_files ++++ arch_scheduling_enabled ++++ [[ -n '' ]] ++++ arch_scheduling_enabled ++++ [[ -n '' ]] +++++ seq 0 2 ++++ for i in $(seq 0 2) ++++ set +e ++++ kubectl get service/monitoring-service -o json ++++ exit_status=0 ++++ set -e ++++ '[' 0 '!=' 0 -a -n 1 ']' ++++ break ++++ cat /tmp/tmp.ffIFtpDyQ0 ++++ cat /tmp/tmp.tvAforWOYE ++++ rm /tmp/tmp.ffIFtpDyQ0 /tmp/tmp.tvAforWOYE ++++ return 0 +++ endpoint=136.112.205.251 +++ '[' -n 136.112.205.251 -a 136.112.205.251 '!=' null ']' +++ echo 136.112.205.251 +++ return ++ endpoint=136.112.205.251 ++ echo 136.112.205.251 ++ return + endpoint=136.112.205.251 + retry 30 2 curl_metric_values mongodb_connections monitoring-pmm3-14502-monitoring-pmm3-cfg-1 136.112.205.251 glsa_422RcuDdDskb7TLtapwWaOqcyrTCF2do_8c6a26b0 bearer + local max=30 + local delay=2 + shift 2 + local n=1 + curl_metric_values mongodb_connections monitoring-pmm3-14502-monitoring-pmm3-cfg-1 136.112.205.251 glsa_422RcuDdDskb7TLtapwWaOqcyrTCF2do_8c6a26b0 bearer + local metric=mongodb_connections + local instance=monitoring-pmm3-14502-monitoring-pmm3-cfg-1 + local endpoint=136.112.205.251 + local credentials=glsa_422RcuDdDskb7TLtapwWaOqcyrTCF2do_8c6a26b0 + local auth_type=bearer + local start + local end + local url + curl_args=() + local curl_args ++ /usr/sbin/date -u +%s -d '-1 minute' + start=1784252713 ++ /usr/sbin/date -u +%s + end=1784252773 + url='https://136.112.205.251/graph/api/datasources/proxy/1/api/v1/query_range?query=min%28mongodb_connections%7Bnode_name%3D%7E%22monitoring-pmm3-14502-monitoring-pmm3-cfg-1%22%7D%20or%20mongodb_connections%7Bnode_name%3D%7E%22monitoring-pmm3-14502-monitoring-pmm3-cfg-1%22%7D%29&start=1784252713&end=1784252773&step=60' + [[ bearer == bearer ]] + curl_args=(-H "Authorization: Bearer ${credentials}" "$url") + execute_curl -s -k -H 'Authorization: Bearer glsa_422RcuDdDskb7TLtapwWaOqcyrTCF2do_8c6a26b0' 'https://136.112.205.251/graph/api/datasources/proxy/1/api/v1/query_range?query=min%28mongodb_connections%7Bnode_name%3D%7E%22monitoring-pmm3-14502-monitoring-pmm3-cfg-1%22%7D%20or%20mongodb_connections%7Bnode_name%3D%7E%22monitoring-pmm3-14502-monitoring-pmm3-cfg-1%22%7D%29&start=1784252713&end=1784252773&step=60' + jq -r '(.data.result[0].values // [])[] | .[1] // empty' + local name=curl-27086 + local image=docker.io/alpine/curl:latest + local phase= + local exit_status=0 + kubectl_bin run curl-27086 -n monitoring-pmm3-14502 --restart=Never --image=docker.io/alpine/curl:latest --command -- curl -sS -s -k -H 'Authorization: Bearer glsa_422RcuDdDskb7TLtapwWaOqcyrTCF2do_8c6a26b0' 'https://136.112.205.251/graph/api/datasources/proxy/1/api/v1/query_range?query=min%28mongodb_connections%7Bnode_name%3D%7E%22monitoring-pmm3-14502-monitoring-pmm3-cfg-1%22%7D%20or%20mongodb_connections%7Bnode_name%3D%7E%22monitoring-pmm3-14502-monitoring-pmm3-cfg-1%22%7D%29&start=1784252713&end=1784252773&step=60' + grep '^[0-9]' ++ mktemp + local LAST_OUT=/tmp/tmp.GmHoNmsEMA ++ mktemp + local LAST_ERR=/tmp/tmp.HZ6G1fMx7I + local exit_status=0 + local timeout=4 + args=('run' 'curl-27086' '-n' 'monitoring-pmm3-14502' '--restart=Never' '--image=docker.io/alpine/curl:latest' '--command' '--' 'curl' '-sS' '-s' '-k' '-H' 'Authorization: Bearer glsa_422RcuDdDskb7TLtapwWaOqcyrTCF2do_8c6a26b0' 'https://136.112.205.251/graph/api/datasources/proxy/1/api/v1/query_range?query=min%28mongodb_connections%7Bnode_name%3D%7E%22monitoring-pmm3-14502-monitoring-pmm3-cfg-1%22%7D%20or%20mongodb_connections%7Bnode_name%3D%7E%22monitoring-pmm3-14502-monitoring-pmm3-cfg-1%22%7D%29&start=1784252713&end=1784252773&step=60') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl run curl-27086 -n monitoring-pmm3-14502 --restart=Never --image=docker.io/alpine/curl:latest --command -- curl -sS -s -k -H 'Authorization: Bearer glsa_422RcuDdDskb7TLtapwWaOqcyrTCF2do_8c6a26b0' 'https://136.112.205.251/graph/api/datasources/proxy/1/api/v1/query_range?query=min%28mongodb_connections%7Bnode_name%3D%7E%22monitoring-pmm3-14502-monitoring-pmm3-cfg-1%22%7D%20or%20mongodb_connections%7Bnode_name%3D%7E%22monitoring-pmm3-14502-monitoring-pmm3-cfg-1%22%7D%29&start=1784252713&end=1784252773&step=60' + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.GmHoNmsEMA + cat /tmp/tmp.HZ6G1fMx7I + rm /tmp/tmp.GmHoNmsEMA /tmp/tmp.HZ6G1fMx7I + return 0 ++ seq 1 60 + for _ in $(seq 1 60) ++ kubectl_bin get pod curl-27086 -n monitoring-pmm3-14502 -o 'jsonpath={.status.phase}' + phase=Pending + [[ Pending == Succeeded ]] + [[ Pending == Failed ]] + sleep 1 + for _ in $(seq 1 60) ++ kubectl_bin get pod curl-27086 -n monitoring-pmm3-14502 -o 'jsonpath={.status.phase}' + phase=Pending + [[ Pending == Succeeded ]] + [[ Pending == Failed ]] + sleep 1 + for _ in $(seq 1 60) ++ kubectl_bin get pod curl-27086 -n monitoring-pmm3-14502 -o 'jsonpath={.status.phase}' + phase=Succeeded + [[ Succeeded == Succeeded ]] + break + kubectl_bin logs curl-27086 -n monitoring-pmm3-14502 ++ mktemp + local LAST_OUT=/tmp/tmp.ohSu0u9X6F ++ mktemp + local LAST_ERR=/tmp/tmp.BJBqH2ycQV + local exit_status=0 + local timeout=4 + args=('logs' 'curl-27086' '-n' 'monitoring-pmm3-14502') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl logs curl-27086 -n monitoring-pmm3-14502 + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.ohSu0u9X6F + cat /tmp/tmp.BJBqH2ycQV + rm /tmp/tmp.ohSu0u9X6F /tmp/tmp.BJBqH2ycQV + return 0 + kubectl_bin delete pod curl-27086 -n monitoring-pmm3-14502 --ignore-not-found --wait=false + [[ Succeeded != Succeeded ]] + return 0 0 0 + desc 'check mongos metrics' + set +o xtrace ----------------------------------------------------------------------------------- check mongos metrics ----------------------------------------------------------------------------------- ++ kubectl get pod -l app.kubernetes.io/component=mongos -o 'jsonpath={.items[0].metadata.name}' + MONGOS_POD_NAME=monitoring-pmm3-mongos-0 + get_metric_values node_boot_time_seconds monitoring-pmm3-14502-monitoring-pmm3-mongos-0 glsa_422RcuDdDskb7TLtapwWaOqcyrTCF2do_8c6a26b0 bearer + local metric=node_boot_time_seconds + local instance=monitoring-pmm3-14502-monitoring-pmm3-mongos-0 + local user_pass=glsa_422RcuDdDskb7TLtapwWaOqcyrTCF2do_8c6a26b0 + local auth_type=bearer + local endpoint ++ get_service_endpoint monitoring-service ++ local service=monitoring-service ++ local endpoint ++ local service_type +++ kubectl_bin get service/monitoring-service -o 'jsonpath={.spec.type}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.PgQod1XNI5 ++++ mktemp +++ local LAST_ERR=/tmp/tmp.sHkG6w1gJT +++ local exit_status=0 +++ local timeout=4 +++ args=('get' 'service/monitoring-service' '-o' 'jsonpath={.spec.type}') +++ local args +++ transformed_files=() +++ local transformed_files +++ arch_scheduling_enabled +++ [[ -n '' ]] +++ arch_scheduling_enabled +++ [[ -n '' ]] ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl get service/monitoring-service -o 'jsonpath={.spec.type}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 -a -n 1 ']' +++ break +++ cat /tmp/tmp.PgQod1XNI5 +++ cat /tmp/tmp.sHkG6w1gJT +++ rm /tmp/tmp.PgQod1XNI5 /tmp/tmp.sHkG6w1gJT +++ return 0 ++ service_type=LoadBalancer ++ [[ LoadBalancer == LoadBalancer ]] +++ get_load_balancer_endpoint monitoring-service +++ local service=monitoring-service +++ local endpoint ++++ kubectl_bin get service/monitoring-service -o json ++++ jq -r '[.status.loadBalancer.ingress[]?.hostname, .status.loadBalancer.ingress[]?.ip] | map(select(. != null and . != "")) | first // empty' +++++ mktemp ++++ local LAST_OUT=/tmp/tmp.UJ4Nt4sJe1 +++++ mktemp ++++ local LAST_ERR=/tmp/tmp.8W2X1asrlo ++++ local exit_status=0 ++++ local timeout=4 ++++ args=('get' 'service/monitoring-service' '-o' 'json') ++++ local args ++++ transformed_files=() ++++ local transformed_files ++++ arch_scheduling_enabled ++++ [[ -n '' ]] ++++ arch_scheduling_enabled ++++ [[ -n '' ]] +++++ seq 0 2 ++++ for i in $(seq 0 2) ++++ set +e ++++ kubectl get service/monitoring-service -o json ++++ exit_status=0 ++++ set -e ++++ '[' 0 '!=' 0 -a -n 1 ']' ++++ break ++++ cat /tmp/tmp.UJ4Nt4sJe1 ++++ cat /tmp/tmp.8W2X1asrlo ++++ rm /tmp/tmp.UJ4Nt4sJe1 /tmp/tmp.8W2X1asrlo ++++ return 0 +++ endpoint=136.112.205.251 +++ '[' -n 136.112.205.251 -a 136.112.205.251 '!=' null ']' +++ echo 136.112.205.251 +++ return ++ endpoint=136.112.205.251 ++ echo 136.112.205.251 ++ return + endpoint=136.112.205.251 + retry 30 2 curl_metric_values node_boot_time_seconds monitoring-pmm3-14502-monitoring-pmm3-mongos-0 136.112.205.251 glsa_422RcuDdDskb7TLtapwWaOqcyrTCF2do_8c6a26b0 bearer + local max=30 + local delay=2 + shift 2 + local n=1 + curl_metric_values node_boot_time_seconds monitoring-pmm3-14502-monitoring-pmm3-mongos-0 136.112.205.251 glsa_422RcuDdDskb7TLtapwWaOqcyrTCF2do_8c6a26b0 bearer + local metric=node_boot_time_seconds + local instance=monitoring-pmm3-14502-monitoring-pmm3-mongos-0 + local endpoint=136.112.205.251 + local credentials=glsa_422RcuDdDskb7TLtapwWaOqcyrTCF2do_8c6a26b0 + local auth_type=bearer + local start + local end + local url + curl_args=() + local curl_args ++ /usr/sbin/date -u +%s -d '-1 minute' + start=1784252724 ++ /usr/sbin/date -u +%s + end=1784252784 + url='https://136.112.205.251/graph/api/datasources/proxy/1/api/v1/query_range?query=min%28node_boot_time_seconds%7Bnode_name%3D%7E%22monitoring-pmm3-14502-monitoring-pmm3-mongos-0%22%7D%20or%20node_boot_time_seconds%7Bnode_name%3D%7E%22monitoring-pmm3-14502-monitoring-pmm3-mongos-0%22%7D%29&start=1784252724&end=1784252784&step=60' + [[ bearer == bearer ]] + curl_args=(-H "Authorization: Bearer ${credentials}" "$url") + execute_curl -s -k -H 'Authorization: Bearer glsa_422RcuDdDskb7TLtapwWaOqcyrTCF2do_8c6a26b0' 'https://136.112.205.251/graph/api/datasources/proxy/1/api/v1/query_range?query=min%28node_boot_time_seconds%7Bnode_name%3D%7E%22monitoring-pmm3-14502-monitoring-pmm3-mongos-0%22%7D%20or%20node_boot_time_seconds%7Bnode_name%3D%7E%22monitoring-pmm3-14502-monitoring-pmm3-mongos-0%22%7D%29&start=1784252724&end=1784252784&step=60' + jq -r '(.data.result[0].values // [])[] | .[1] // empty' + local name=curl-3965 + local image=docker.io/alpine/curl:latest + local phase= + local exit_status=0 + kubectl_bin run curl-3965 -n monitoring-pmm3-14502 --restart=Never --image=docker.io/alpine/curl:latest --command -- curl -sS -s -k -H 'Authorization: Bearer glsa_422RcuDdDskb7TLtapwWaOqcyrTCF2do_8c6a26b0' 'https://136.112.205.251/graph/api/datasources/proxy/1/api/v1/query_range?query=min%28node_boot_time_seconds%7Bnode_name%3D%7E%22monitoring-pmm3-14502-monitoring-pmm3-mongos-0%22%7D%20or%20node_boot_time_seconds%7Bnode_name%3D%7E%22monitoring-pmm3-14502-monitoring-pmm3-mongos-0%22%7D%29&start=1784252724&end=1784252784&step=60' + grep '^[0-9]' ++ mktemp + local LAST_OUT=/tmp/tmp.IzaCHr39fS ++ mktemp + local LAST_ERR=/tmp/tmp.BhDVh39txC + local exit_status=0 + local timeout=4 + args=('run' 'curl-3965' '-n' 'monitoring-pmm3-14502' '--restart=Never' '--image=docker.io/alpine/curl:latest' '--command' '--' 'curl' '-sS' '-s' '-k' '-H' 'Authorization: Bearer glsa_422RcuDdDskb7TLtapwWaOqcyrTCF2do_8c6a26b0' 'https://136.112.205.251/graph/api/datasources/proxy/1/api/v1/query_range?query=min%28node_boot_time_seconds%7Bnode_name%3D%7E%22monitoring-pmm3-14502-monitoring-pmm3-mongos-0%22%7D%20or%20node_boot_time_seconds%7Bnode_name%3D%7E%22monitoring-pmm3-14502-monitoring-pmm3-mongos-0%22%7D%29&start=1784252724&end=1784252784&step=60') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl run curl-3965 -n monitoring-pmm3-14502 --restart=Never --image=docker.io/alpine/curl:latest --command -- curl -sS -s -k -H 'Authorization: Bearer glsa_422RcuDdDskb7TLtapwWaOqcyrTCF2do_8c6a26b0' 'https://136.112.205.251/graph/api/datasources/proxy/1/api/v1/query_range?query=min%28node_boot_time_seconds%7Bnode_name%3D%7E%22monitoring-pmm3-14502-monitoring-pmm3-mongos-0%22%7D%20or%20node_boot_time_seconds%7Bnode_name%3D%7E%22monitoring-pmm3-14502-monitoring-pmm3-mongos-0%22%7D%29&start=1784252724&end=1784252784&step=60' + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.IzaCHr39fS + cat /tmp/tmp.BhDVh39txC + rm /tmp/tmp.IzaCHr39fS /tmp/tmp.BhDVh39txC + return 0 ++ seq 1 60 + for _ in $(seq 1 60) ++ kubectl_bin get pod curl-3965 -n monitoring-pmm3-14502 -o 'jsonpath={.status.phase}' + phase=Pending + [[ Pending == Succeeded ]] + [[ Pending == Failed ]] + sleep 1 + for _ in $(seq 1 60) ++ kubectl_bin get pod curl-3965 -n monitoring-pmm3-14502 -o 'jsonpath={.status.phase}' + phase=Pending + [[ Pending == Succeeded ]] + [[ Pending == Failed ]] + sleep 1 + for _ in $(seq 1 60) ++ kubectl_bin get pod curl-3965 -n monitoring-pmm3-14502 -o 'jsonpath={.status.phase}' + phase=Succeeded + [[ Succeeded == Succeeded ]] + break + kubectl_bin logs curl-3965 -n monitoring-pmm3-14502 ++ mktemp + local LAST_OUT=/tmp/tmp.9FqluyV2p5 ++ mktemp + local LAST_ERR=/tmp/tmp.ICgyvCtYmR + local exit_status=0 + local timeout=4 + args=('logs' 'curl-3965' '-n' 'monitoring-pmm3-14502') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl logs curl-3965 -n monitoring-pmm3-14502 + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.9FqluyV2p5 + cat /tmp/tmp.ICgyvCtYmR + rm /tmp/tmp.9FqluyV2p5 /tmp/tmp.ICgyvCtYmR + return 0 + kubectl_bin delete pod curl-3965 -n monitoring-pmm3-14502 --ignore-not-found --wait=false + [[ Succeeded != Succeeded ]] + return 0 1784246223 1784246223 + sleep 90 + desc 'check QAN data' + set +o xtrace ----------------------------------------------------------------------------------- check QAN data ----------------------------------------------------------------------------------- + get_qan_values mongodb dev-mongod glsa_422RcuDdDskb7TLtapwWaOqcyrTCF2do_8c6a26b0 bearer + local service_type=mongodb + local environment=dev-mongod + local user_pass=glsa_422RcuDdDskb7TLtapwWaOqcyrTCF2do_8c6a26b0 + local auth_type=bearer + local start + local end + local endpoint ++ /usr/sbin/date -u -d '-12 hour' +%Y-%m-%dT%H:%M:%S%:z + start=2026-07-16T13:48:01+00:00 ++ /usr/sbin/date -u +%Y-%m-%dT%H:%M:%S%:z + end=2026-07-17T01:48:01+00:00 ++ get_service_endpoint monitoring-service ++ local service=monitoring-service ++ local endpoint ++ local service_type +++ kubectl_bin get service/monitoring-service -o 'jsonpath={.spec.type}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.vsOl48TVBC ++++ mktemp +++ local LAST_ERR=/tmp/tmp.SlMQMfng3j +++ local exit_status=0 +++ local timeout=4 +++ args=('get' 'service/monitoring-service' '-o' 'jsonpath={.spec.type}') +++ local args +++ transformed_files=() +++ local transformed_files +++ arch_scheduling_enabled +++ [[ -n '' ]] +++ arch_scheduling_enabled +++ [[ -n '' ]] ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl get service/monitoring-service -o 'jsonpath={.spec.type}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 -a -n 1 ']' +++ break +++ cat /tmp/tmp.vsOl48TVBC +++ cat /tmp/tmp.SlMQMfng3j +++ rm /tmp/tmp.vsOl48TVBC /tmp/tmp.SlMQMfng3j +++ return 0 ++ service_type=LoadBalancer ++ [[ LoadBalancer == LoadBalancer ]] +++ get_load_balancer_endpoint monitoring-service +++ local service=monitoring-service +++ local endpoint ++++ kubectl_bin get service/monitoring-service -o json ++++ jq -r '[.status.loadBalancer.ingress[]?.hostname, .status.loadBalancer.ingress[]?.ip] | map(select(. != null and . != "")) | first // empty' +++++ mktemp ++++ local LAST_OUT=/tmp/tmp.xiXJmdfQ3c +++++ mktemp ++++ local LAST_ERR=/tmp/tmp.v5YdfYDgEZ ++++ local exit_status=0 ++++ local timeout=4 ++++ args=('get' 'service/monitoring-service' '-o' 'json') ++++ local args ++++ transformed_files=() ++++ local transformed_files ++++ arch_scheduling_enabled ++++ [[ -n '' ]] ++++ arch_scheduling_enabled ++++ [[ -n '' ]] +++++ seq 0 2 ++++ for i in $(seq 0 2) ++++ set +e ++++ kubectl get service/monitoring-service -o json ++++ exit_status=0 ++++ set -e ++++ '[' 0 '!=' 0 -a -n 1 ']' ++++ break ++++ cat /tmp/tmp.xiXJmdfQ3c ++++ cat /tmp/tmp.v5YdfYDgEZ ++++ rm /tmp/tmp.xiXJmdfQ3c /tmp/tmp.v5YdfYDgEZ ++++ return 0 +++ endpoint=136.112.205.251 +++ '[' -n 136.112.205.251 -a 136.112.205.251 '!=' null ']' +++ echo 136.112.205.251 +++ return ++ endpoint=136.112.205.251 ++ echo 136.112.205.251 ++ return + endpoint=136.112.205.251 + cat + retry 30 2 curl_qan_values payload.json 136.112.205.251 glsa_422RcuDdDskb7TLtapwWaOqcyrTCF2do_8c6a26b0 bearer + local max=30 + local delay=2 + shift 2 + local n=1 + curl_qan_values payload.json 136.112.205.251 glsa_422RcuDdDskb7TLtapwWaOqcyrTCF2do_8c6a26b0 bearer + local payload_file=payload.json + local endpoint=136.112.205.251 + local credentials=glsa_422RcuDdDskb7TLtapwWaOqcyrTCF2do_8c6a26b0 + local auth_type=bearer + local url + curl_args=() + local curl_args + [[ bearer == bearer ]] + url=https://136.112.205.251/v1/qan/metrics:getReport + curl_args=(-H "Authorization: Bearer ${credentials}" "$url") + jq -e 'any(.rows[]?; .sparkline != null)' ++ cat payload.json + execute_curl -s -k -XPOST -d $'{\n "period_start_from": "2026-07-16T13:48:01+00:00",\n "period_start_to": "2026-07-17T01:48:01+00:00",\n "group_by": "queryid",\n "labels": [\n {\n "key": "service_type",\n "value": [\n "mongodb"\n ]\n },\n {\n "key": "environment",\n "value": [\n "dev-mongod"\n ]\n }\n ],\n "columns": [\n "load",\n "num_queries",\n "query_time"\n ],\n "order_by": "-load",\n "offset": 0,\n "limit": 10,\n "main_metric": "load",\n "search": ""\n}' -H 'Authorization: Bearer glsa_422RcuDdDskb7TLtapwWaOqcyrTCF2do_8c6a26b0' https://136.112.205.251/v1/qan/metrics:getReport + local name=curl-16505 + local image=docker.io/alpine/curl:latest + local phase= + local exit_status=0 + kubectl_bin run curl-16505 -n monitoring-pmm3-14502 --restart=Never --image=docker.io/alpine/curl:latest --command -- curl -sS -s -k -XPOST -d $'{\n "period_start_from": "2026-07-16T13:48:01+00:00",\n "period_start_to": "2026-07-17T01:48:01+00:00",\n "group_by": "queryid",\n "labels": [\n {\n "key": "service_type",\n "value": [\n "mongodb"\n ]\n },\n {\n "key": "environment",\n "value": [\n "dev-mongod"\n ]\n }\n ],\n "columns": [\n "load",\n "num_queries",\n "query_time"\n ],\n "order_by": "-load",\n "offset": 0,\n "limit": 10,\n "main_metric": "load",\n "search": ""\n}' -H 'Authorization: Bearer glsa_422RcuDdDskb7TLtapwWaOqcyrTCF2do_8c6a26b0' https://136.112.205.251/v1/qan/metrics:getReport ++ mktemp + local LAST_OUT=/tmp/tmp.AsuPdj7ycI ++ mktemp + local LAST_ERR=/tmp/tmp.5v4MivWWQ8 + local exit_status=0 + local timeout=4 + args=('run' 'curl-16505' '-n' 'monitoring-pmm3-14502' '--restart=Never' '--image=docker.io/alpine/curl:latest' '--command' '--' 'curl' '-sS' '-s' '-k' '-XPOST' '-d' '{ "period_start_from": "2026-07-16T13:48:01+00:00", "period_start_to": "2026-07-17T01:48:01+00:00", "group_by": "queryid", "labels": [ { "key": "service_type", "value": [ "mongodb" ] }, { "key": "environment", "value": [ "dev-mongod" ] } ], "columns": [ "load", "num_queries", "query_time" ], "order_by": "-load", "offset": 0, "limit": 10, "main_metric": "load", "search": "" }' '-H' 'Authorization: Bearer glsa_422RcuDdDskb7TLtapwWaOqcyrTCF2do_8c6a26b0' 'https://136.112.205.251/v1/qan/metrics:getReport') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl run curl-16505 -n monitoring-pmm3-14502 --restart=Never --image=docker.io/alpine/curl:latest --command -- curl -sS -s -k -XPOST -d $'{\n "period_start_from": "2026-07-16T13:48:01+00:00",\n "period_start_to": "2026-07-17T01:48:01+00:00",\n "group_by": "queryid",\n "labels": [\n {\n "key": "service_type",\n "value": [\n "mongodb"\n ]\n },\n {\n "key": "environment",\n "value": [\n "dev-mongod"\n ]\n }\n ],\n "columns": [\n "load",\n "num_queries",\n "query_time"\n ],\n "order_by": "-load",\n "offset": 0,\n "limit": 10,\n "main_metric": "load",\n "search": ""\n}' -H 'Authorization: Bearer glsa_422RcuDdDskb7TLtapwWaOqcyrTCF2do_8c6a26b0' https://136.112.205.251/v1/qan/metrics:getReport + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.AsuPdj7ycI + cat /tmp/tmp.5v4MivWWQ8 + rm /tmp/tmp.AsuPdj7ycI /tmp/tmp.5v4MivWWQ8 + return 0 ++ seq 1 60 + for _ in $(seq 1 60) ++ kubectl_bin get pod curl-16505 -n monitoring-pmm3-14502 -o 'jsonpath={.status.phase}' + phase=Pending + [[ Pending == Succeeded ]] + [[ Pending == Failed ]] + sleep 1 + for _ in $(seq 1 60) ++ kubectl_bin get pod curl-16505 -n monitoring-pmm3-14502 -o 'jsonpath={.status.phase}' + phase=Succeeded + [[ Succeeded == Succeeded ]] + break + kubectl_bin logs curl-16505 -n monitoring-pmm3-14502 ++ mktemp + local LAST_OUT=/tmp/tmp.rHoOdSpujc ++ mktemp + local LAST_ERR=/tmp/tmp.MOdNEQDhbU + local exit_status=0 + local timeout=4 + args=('logs' 'curl-16505' '-n' 'monitoring-pmm3-14502') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl logs curl-16505 -n monitoring-pmm3-14502 + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.rHoOdSpujc + cat /tmp/tmp.MOdNEQDhbU + rm /tmp/tmp.rHoOdSpujc /tmp/tmp.MOdNEQDhbU + return 0 + kubectl_bin delete pod curl-16505 -n monitoring-pmm3-14502 --ignore-not-found --wait=false + [[ Succeeded != Succeeded ]] + return 0 true + rm -f payload.json + get_qan_values mongodb dev-mongos glsa_422RcuDdDskb7TLtapwWaOqcyrTCF2do_8c6a26b0 bearer + local service_type=mongodb + local environment=dev-mongos + local user_pass=glsa_422RcuDdDskb7TLtapwWaOqcyrTCF2do_8c6a26b0 + local auth_type=bearer + local start + local end + local endpoint ++ /usr/sbin/date -u -d '-12 hour' +%Y-%m-%dT%H:%M:%S%:z + start=2026-07-16T13:48:09+00:00 ++ /usr/sbin/date -u +%Y-%m-%dT%H:%M:%S%:z + end=2026-07-17T01:48:09+00:00 ++ get_service_endpoint monitoring-service ++ local service=monitoring-service ++ local endpoint ++ local service_type +++ kubectl_bin get service/monitoring-service -o 'jsonpath={.spec.type}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.1EEruzclNF ++++ mktemp +++ local LAST_ERR=/tmp/tmp.u8SAtMn0NQ +++ local exit_status=0 +++ local timeout=4 +++ args=('get' 'service/monitoring-service' '-o' 'jsonpath={.spec.type}') +++ local args +++ transformed_files=() +++ local transformed_files +++ arch_scheduling_enabled +++ [[ -n '' ]] +++ arch_scheduling_enabled +++ [[ -n '' ]] ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl get service/monitoring-service -o 'jsonpath={.spec.type}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 -a -n 1 ']' +++ break +++ cat /tmp/tmp.1EEruzclNF +++ cat /tmp/tmp.u8SAtMn0NQ +++ rm /tmp/tmp.1EEruzclNF /tmp/tmp.u8SAtMn0NQ +++ return 0 ++ service_type=LoadBalancer ++ [[ LoadBalancer == LoadBalancer ]] +++ get_load_balancer_endpoint monitoring-service +++ local service=monitoring-service +++ local endpoint ++++ kubectl_bin get service/monitoring-service -o json ++++ jq -r '[.status.loadBalancer.ingress[]?.hostname, .status.loadBalancer.ingress[]?.ip] | map(select(. != null and . != "")) | first // empty' +++++ mktemp ++++ local LAST_OUT=/tmp/tmp.w1WaOqmzsk +++++ mktemp ++++ local LAST_ERR=/tmp/tmp.P345r0GlO0 ++++ local exit_status=0 ++++ local timeout=4 ++++ args=('get' 'service/monitoring-service' '-o' 'json') ++++ local args ++++ transformed_files=() ++++ local transformed_files ++++ arch_scheduling_enabled ++++ [[ -n '' ]] ++++ arch_scheduling_enabled ++++ [[ -n '' ]] +++++ seq 0 2 ++++ for i in $(seq 0 2) ++++ set +e ++++ kubectl get service/monitoring-service -o json ++++ exit_status=0 ++++ set -e ++++ '[' 0 '!=' 0 -a -n 1 ']' ++++ break ++++ cat /tmp/tmp.w1WaOqmzsk ++++ cat /tmp/tmp.P345r0GlO0 ++++ rm /tmp/tmp.w1WaOqmzsk /tmp/tmp.P345r0GlO0 ++++ return 0 +++ endpoint=136.112.205.251 +++ '[' -n 136.112.205.251 -a 136.112.205.251 '!=' null ']' +++ echo 136.112.205.251 +++ return ++ endpoint=136.112.205.251 ++ echo 136.112.205.251 ++ return + endpoint=136.112.205.251 + cat + retry 30 2 curl_qan_values payload.json 136.112.205.251 glsa_422RcuDdDskb7TLtapwWaOqcyrTCF2do_8c6a26b0 bearer + local max=30 + local delay=2 + shift 2 + local n=1 + curl_qan_values payload.json 136.112.205.251 glsa_422RcuDdDskb7TLtapwWaOqcyrTCF2do_8c6a26b0 bearer + local payload_file=payload.json + local endpoint=136.112.205.251 + local credentials=glsa_422RcuDdDskb7TLtapwWaOqcyrTCF2do_8c6a26b0 + local auth_type=bearer + local url + curl_args=() + local curl_args + [[ bearer == bearer ]] + url=https://136.112.205.251/v1/qan/metrics:getReport + curl_args=(-H "Authorization: Bearer ${credentials}" "$url") + jq -e 'any(.rows[]?; .sparkline != null)' ++ cat payload.json + execute_curl -s -k -XPOST -d $'{\n "period_start_from": "2026-07-16T13:48:09+00:00",\n "period_start_to": "2026-07-17T01:48:09+00:00",\n "group_by": "queryid",\n "labels": [\n {\n "key": "service_type",\n "value": [\n "mongodb"\n ]\n },\n {\n "key": "environment",\n "value": [\n "dev-mongos"\n ]\n }\n ],\n "columns": [\n "load",\n "num_queries",\n "query_time"\n ],\n "order_by": "-load",\n "offset": 0,\n "limit": 10,\n "main_metric": "load",\n "search": ""\n}' -H 'Authorization: Bearer glsa_422RcuDdDskb7TLtapwWaOqcyrTCF2do_8c6a26b0' https://136.112.205.251/v1/qan/metrics:getReport + local name=curl-12896 + local image=docker.io/alpine/curl:latest + local phase= + local exit_status=0 + kubectl_bin run curl-12896 -n monitoring-pmm3-14502 --restart=Never --image=docker.io/alpine/curl:latest --command -- curl -sS -s -k -XPOST -d $'{\n "period_start_from": "2026-07-16T13:48:09+00:00",\n "period_start_to": "2026-07-17T01:48:09+00:00",\n "group_by": "queryid",\n "labels": [\n {\n "key": "service_type",\n "value": [\n "mongodb"\n ]\n },\n {\n "key": "environment",\n "value": [\n "dev-mongos"\n ]\n }\n ],\n "columns": [\n "load",\n "num_queries",\n "query_time"\n ],\n "order_by": "-load",\n "offset": 0,\n "limit": 10,\n "main_metric": "load",\n "search": ""\n}' -H 'Authorization: Bearer glsa_422RcuDdDskb7TLtapwWaOqcyrTCF2do_8c6a26b0' https://136.112.205.251/v1/qan/metrics:getReport ++ mktemp + local LAST_OUT=/tmp/tmp.FKvHGLT7Mb ++ mktemp + local LAST_ERR=/tmp/tmp.7clGemdLhs + local exit_status=0 + local timeout=4 + args=('run' 'curl-12896' '-n' 'monitoring-pmm3-14502' '--restart=Never' '--image=docker.io/alpine/curl:latest' '--command' '--' 'curl' '-sS' '-s' '-k' '-XPOST' '-d' '{ "period_start_from": "2026-07-16T13:48:09+00:00", "period_start_to": "2026-07-17T01:48:09+00:00", "group_by": "queryid", "labels": [ { "key": "service_type", "value": [ "mongodb" ] }, { "key": "environment", "value": [ "dev-mongos" ] } ], "columns": [ "load", "num_queries", "query_time" ], "order_by": "-load", "offset": 0, "limit": 10, "main_metric": "load", "search": "" }' '-H' 'Authorization: Bearer glsa_422RcuDdDskb7TLtapwWaOqcyrTCF2do_8c6a26b0' 'https://136.112.205.251/v1/qan/metrics:getReport') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl run curl-12896 -n monitoring-pmm3-14502 --restart=Never --image=docker.io/alpine/curl:latest --command -- curl -sS -s -k -XPOST -d $'{\n "period_start_from": "2026-07-16T13:48:09+00:00",\n "period_start_to": "2026-07-17T01:48:09+00:00",\n "group_by": "queryid",\n "labels": [\n {\n "key": "service_type",\n "value": [\n "mongodb"\n ]\n },\n {\n "key": "environment",\n "value": [\n "dev-mongos"\n ]\n }\n ],\n "columns": [\n "load",\n "num_queries",\n "query_time"\n ],\n "order_by": "-load",\n "offset": 0,\n "limit": 10,\n "main_metric": "load",\n "search": ""\n}' -H 'Authorization: Bearer glsa_422RcuDdDskb7TLtapwWaOqcyrTCF2do_8c6a26b0' https://136.112.205.251/v1/qan/metrics:getReport + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.FKvHGLT7Mb + cat /tmp/tmp.7clGemdLhs + rm /tmp/tmp.FKvHGLT7Mb /tmp/tmp.7clGemdLhs + return 0 ++ seq 1 60 + for _ in $(seq 1 60) ++ kubectl_bin get pod curl-12896 -n monitoring-pmm3-14502 -o 'jsonpath={.status.phase}' + phase=Pending + [[ Pending == Succeeded ]] + [[ Pending == Failed ]] + sleep 1 + for _ in $(seq 1 60) ++ kubectl_bin get pod curl-12896 -n monitoring-pmm3-14502 -o 'jsonpath={.status.phase}' + phase=Succeeded + [[ Succeeded == Succeeded ]] + break + kubectl_bin logs curl-12896 -n monitoring-pmm3-14502 ++ mktemp + local LAST_OUT=/tmp/tmp.vf7evEedyW ++ mktemp + local LAST_ERR=/tmp/tmp.XZRqgSFdDW + local exit_status=0 + local timeout=4 + args=('logs' 'curl-12896' '-n' 'monitoring-pmm3-14502') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl logs curl-12896 -n monitoring-pmm3-14502 + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.vf7evEedyW + cat /tmp/tmp.XZRqgSFdDW + rm /tmp/tmp.vf7evEedyW /tmp/tmp.XZRqgSFdDW + return 0 + kubectl_bin delete pod curl-12896 -n monitoring-pmm3-14502 --ignore-not-found --wait=false + [[ Succeeded != Succeeded ]] + return 0 true + rm -f payload.json + desc 'verify that the custom cluster name is configured' + set +o xtrace ----------------------------------------------------------------------------------- verify that the custom cluster name is configured ----------------------------------------------------------------------------------- + verify_custom_cluster_name super-custom glsa_422RcuDdDskb7TLtapwWaOqcyrTCF2do_8c6a26b0 monitoring-pmm3-14502-monitoring-pmm3-mongos-0 monitoring-pmm3-14502-monitoring-pmm3-cfg-0 monitoring-pmm3-14502-monitoring-pmm3-rs0-0 + local expected_cluster=super-custom + local token=glsa_422RcuDdDskb7TLtapwWaOqcyrTCF2do_8c6a26b0 + shift 2 + service_names=('monitoring-pmm3-14502-monitoring-pmm3-mongos-0' 'monitoring-pmm3-14502-monitoring-pmm3-cfg-0' 'monitoring-pmm3-14502-monitoring-pmm3-rs0-0') + local service_names + local endpoint ++ get_service_endpoint monitoring-service ++ local service=monitoring-service ++ local endpoint ++ local service_type +++ kubectl_bin get service/monitoring-service -o 'jsonpath={.spec.type}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.qoKkh4kwgU ++++ mktemp +++ local LAST_ERR=/tmp/tmp.sG5TCujYss +++ local exit_status=0 +++ local timeout=4 +++ args=('get' 'service/monitoring-service' '-o' 'jsonpath={.spec.type}') +++ local args +++ transformed_files=() +++ local transformed_files +++ arch_scheduling_enabled +++ [[ -n '' ]] +++ arch_scheduling_enabled +++ [[ -n '' ]] ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl get service/monitoring-service -o 'jsonpath={.spec.type}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 -a -n 1 ']' +++ break +++ cat /tmp/tmp.qoKkh4kwgU +++ cat /tmp/tmp.sG5TCujYss +++ rm /tmp/tmp.qoKkh4kwgU /tmp/tmp.sG5TCujYss +++ return 0 ++ service_type=LoadBalancer ++ [[ LoadBalancer == LoadBalancer ]] +++ get_load_balancer_endpoint monitoring-service +++ local service=monitoring-service +++ local endpoint ++++ kubectl_bin get service/monitoring-service -o json ++++ jq -r '[.status.loadBalancer.ingress[]?.hostname, .status.loadBalancer.ingress[]?.ip] | map(select(. != null and . != "")) | first // empty' +++++ mktemp ++++ local LAST_OUT=/tmp/tmp.roc6TUKIoX +++++ mktemp ++++ local LAST_ERR=/tmp/tmp.zfy0YnenVS ++++ local exit_status=0 ++++ local timeout=4 ++++ args=('get' 'service/monitoring-service' '-o' 'json') ++++ local args ++++ transformed_files=() ++++ local transformed_files ++++ arch_scheduling_enabled ++++ [[ -n '' ]] ++++ arch_scheduling_enabled ++++ [[ -n '' ]] +++++ seq 0 2 ++++ for i in $(seq 0 2) ++++ set +e ++++ kubectl get service/monitoring-service -o json ++++ exit_status=0 ++++ set -e ++++ '[' 0 '!=' 0 -a -n 1 ']' ++++ break ++++ cat /tmp/tmp.roc6TUKIoX ++++ cat /tmp/tmp.zfy0YnenVS ++++ rm /tmp/tmp.roc6TUKIoX /tmp/tmp.zfy0YnenVS ++++ return 0 +++ endpoint=136.112.205.251 +++ '[' -n 136.112.205.251 -a 136.112.205.251 '!=' null ']' +++ echo 136.112.205.251 +++ return ++ endpoint=136.112.205.251 ++ echo 136.112.205.251 ++ return + endpoint=136.112.205.251 + local response ++ execute_curl -s -k -H 'Authorization: Bearer glsa_422RcuDdDskb7TLtapwWaOqcyrTCF2do_8c6a26b0' 'https://136.112.205.251/v1/inventory/services?service_type=SERVICE_TYPE_MONGODB_SERVICE' ++ local name=curl-19980 ++ local image=docker.io/alpine/curl:latest ++ local phase= ++ local exit_status=0 ++ kubectl_bin run curl-19980 -n monitoring-pmm3-14502 --restart=Never --image=docker.io/alpine/curl:latest --command -- curl -sS -s -k -H 'Authorization: Bearer glsa_422RcuDdDskb7TLtapwWaOqcyrTCF2do_8c6a26b0' 'https://136.112.205.251/v1/inventory/services?service_type=SERVICE_TYPE_MONGODB_SERVICE' +++ mktemp ++ local LAST_OUT=/tmp/tmp.NETTgrTqDi +++ mktemp ++ local LAST_ERR=/tmp/tmp.3pXiQMuDty ++ local exit_status=0 ++ local timeout=4 ++ args=('run' 'curl-19980' '-n' 'monitoring-pmm3-14502' '--restart=Never' '--image=docker.io/alpine/curl:latest' '--command' '--' 'curl' '-sS' '-s' '-k' '-H' 'Authorization: Bearer glsa_422RcuDdDskb7TLtapwWaOqcyrTCF2do_8c6a26b0' 'https://136.112.205.251/v1/inventory/services?service_type=SERVICE_TYPE_MONGODB_SERVICE') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl run curl-19980 -n monitoring-pmm3-14502 --restart=Never --image=docker.io/alpine/curl:latest --command -- curl -sS -s -k -H 'Authorization: Bearer glsa_422RcuDdDskb7TLtapwWaOqcyrTCF2do_8c6a26b0' 'https://136.112.205.251/v1/inventory/services?service_type=SERVICE_TYPE_MONGODB_SERVICE' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.NETTgrTqDi ++ cat /tmp/tmp.3pXiQMuDty ++ rm /tmp/tmp.NETTgrTqDi /tmp/tmp.3pXiQMuDty ++ return 0 +++ seq 1 60 ++ for _ in $(seq 1 60) +++ kubectl_bin get pod curl-19980 -n monitoring-pmm3-14502 -o 'jsonpath={.status.phase}' ++ phase=Pending ++ [[ Pending == Succeeded ]] ++ [[ Pending == Failed ]] ++ sleep 1 ++ for _ in $(seq 1 60) +++ kubectl_bin get pod curl-19980 -n monitoring-pmm3-14502 -o 'jsonpath={.status.phase}' ++ phase=Succeeded ++ [[ Succeeded == Succeeded ]] ++ break ++ kubectl_bin logs curl-19980 -n monitoring-pmm3-14502 +++ mktemp ++ local LAST_OUT=/tmp/tmp.V6xIpGtdoM +++ mktemp ++ local LAST_ERR=/tmp/tmp.yDc1PaQQt0 ++ local exit_status=0 ++ local timeout=4 ++ args=('logs' 'curl-19980' '-n' 'monitoring-pmm3-14502') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl logs curl-19980 -n monitoring-pmm3-14502 ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.V6xIpGtdoM ++ cat /tmp/tmp.yDc1PaQQt0 ++ rm /tmp/tmp.V6xIpGtdoM /tmp/tmp.yDc1PaQQt0 ++ return 0 ++ kubectl_bin delete pod curl-19980 -n monitoring-pmm3-14502 --ignore-not-found --wait=false ++ [[ Succeeded != Succeeded ]] ++ return 0 + response=$'{\n "mysql": [],\n "mongodb": [\n {\n "service_id": "0b223f5a-5c05-40a3-b3cf-67a7bf9710f8",\n "service_name": "monitoring-pmm3-14502-monitoring-pmm3-mongos-0",\n "node_id": "568ad465-8ec4-4f7c-a1df-adaeb7b46d26",\n "address": "localhost",\n "port": 27019,\n "socket": "",\n "environment": "dev-mongos",\n "cluster": "super-custom",\n "replication_set": "",\n "custom_labels": {},\n "version": ""\n },\n {\n "service_id": "0c977ddb-0f55-4f95-a3e8-59dec3e4c8d0",\n "service_name": "monitoring-pmm3-14502-monitoring-pmm3-cfg-0",\n "node_id": "9f635edf-62b7-42c4-829c-71f022e4324a",\n "address": "localhost",\n "port": 27019,\n "socket": "",\n "environment": "dev-mongod",\n "cluster": "super-custom",\n "replication_set": "",\n "custom_labels": {},\n "version": ""\n },\n {\n "service_id": "125658bd-2b56-452f-bc9a-c8eaabf3dd33",\n "service_name": "monitoring-pmm3-14502-monitoring-pmm3-rs0-0",\n "node_id": "62ab6a63-009a-4bd8-80e5-865935478e6c",\n "address": "localhost",\n "port": 27019,\n "socket": "",\n "environment": "dev-mongod",\n "cluster": "super-custom",\n "replication_set": "",\n "custom_labels": {},\n "version": ""\n },\n {\n "service_id": "3c27e158-ad06-4fe1-819f-acc80f8f32a2",\n "service_name": "monitoring-pmm3-14502-monitoring-pmm3-rs0-2",\n "node_id": "dc90c940-2f86-44f6-8fa5-3c58a2fff7c1",\n "address": "localhost",\n "port": 27019,\n "socket": "",\n "environment": "dev-mongod",\n "cluster": "super-custom",\n "replication_set": "",\n "custom_labels": {},\n "version": ""\n },\n {\n "service_id": "46394641-bdb9-4dea-9e93-383402bf94f0",\n "service_name": "monitoring-pmm3-14502-monitoring-pmm3-mongos-1",\n "node_id": "4d9d6d17-3ba2-4719-9a0f-da52c142fe57",\n "address": "localhost",\n "port": 27019,\n "socket": "",\n "environment": "dev-mongos",\n "cluster": "super-custom",\n "replication_set": "",\n "custom_labels": {},\n "version": ""\n },\n {\n "service_id": "7c494ba1-0ac7-4c9f-8eff-697b8cf61271",\n "service_name": "monitoring-pmm3-14502-monitoring-pmm3-cfg-1",\n "node_id": "9cb595c2-e403-447d-a53a-e340c7d2d8cb",\n "address": "localhost",\n "port": 27019,\n "socket": "",\n "environment": "dev-mongod",\n "cluster": "super-custom",\n "replication_set": "",\n "custom_labels": {},\n "version": ""\n },\n {\n "service_id": "8617596a-b2a2-4cbf-8f96-85e00118ce7a",\n "service_name": "monitoring-pmm3-14502-monitoring-pmm3-mongos-2",\n "node_id": "e43bf5f4-2d9d-43c3-9e29-8c198e451f6b",\n "address": "localhost",\n "port": 27019,\n "socket": "",\n "environment": "dev-mongos",\n "cluster": "super-custom",\n "replication_set": "",\n "custom_labels": {},\n "version": ""\n },\n {\n "service_id": "a03af921-b2a2-4253-97d7-29b9773f6b0f",\n "service_name": "monitoring-pmm3-14502-monitoring-pmm3-rs0-1",\n "node_id": "38970820-7985-4a2b-ab41-91454f079afd",\n "address": "localhost",\n "port": 27019,\n "socket": "",\n "environment": "dev-mongod",\n "cluster": "super-custom",\n "replication_set": "",\n "custom_labels": {},\n "version": ""\n },\n {\n "service_id": "d45a8270-b4f1-4af9-8a19-6d100eb36a8e",\n "service_name": "monitoring-pmm3-14502-monitoring-pmm3-cfg-2",\n "node_id": "6e5e579d-0991-4d93-89f2-2390d520c3e4",\n "address": "localhost",\n "port": 27019,\n "socket": "",\n "environment": "dev-mongod",\n "cluster": "super-custom",\n "replication_set": "",\n "custom_labels": {},\n "version": ""\n }\n ],\n "postgresql": [],\n "proxysql": [],\n "haproxy": [],\n "external": [],\n "valkey": []\n}' + local verified=0 + for service_name in "${service_names[@]}" + local actual_cluster ++ jq -r --arg name monitoring-pmm3-14502-monitoring-pmm3-mongos-0 $'\n\t\t\t.mongodb[] | select(.service_name == $name) | .cluster\n\t\t' ++ echo $'{\n "mysql": [],\n "mongodb": [\n {\n "service_id": "0b223f5a-5c05-40a3-b3cf-67a7bf9710f8",\n "service_name": "monitoring-pmm3-14502-monitoring-pmm3-mongos-0",\n "node_id": "568ad465-8ec4-4f7c-a1df-adaeb7b46d26",\n "address": "localhost",\n "port": 27019,\n "socket": "",\n "environment": "dev-mongos",\n "cluster": "super-custom",\n "replication_set": "",\n "custom_labels": {},\n "version": ""\n },\n {\n "service_id": "0c977ddb-0f55-4f95-a3e8-59dec3e4c8d0",\n "service_name": "monitoring-pmm3-14502-monitoring-pmm3-cfg-0",\n "node_id": "9f635edf-62b7-42c4-829c-71f022e4324a",\n "address": "localhost",\n "port": 27019,\n "socket": "",\n "environment": "dev-mongod",\n "cluster": "super-custom",\n "replication_set": "",\n "custom_labels": {},\n "version": ""\n },\n {\n "service_id": "125658bd-2b56-452f-bc9a-c8eaabf3dd33",\n "service_name": "monitoring-pmm3-14502-monitoring-pmm3-rs0-0",\n "node_id": "62ab6a63-009a-4bd8-80e5-865935478e6c",\n "address": "localhost",\n "port": 27019,\n "socket": "",\n "environment": "dev-mongod",\n "cluster": "super-custom",\n "replication_set": "",\n "custom_labels": {},\n "version": ""\n },\n {\n "service_id": "3c27e158-ad06-4fe1-819f-acc80f8f32a2",\n "service_name": "monitoring-pmm3-14502-monitoring-pmm3-rs0-2",\n "node_id": "dc90c940-2f86-44f6-8fa5-3c58a2fff7c1",\n "address": "localhost",\n "port": 27019,\n "socket": "",\n "environment": "dev-mongod",\n "cluster": "super-custom",\n "replication_set": "",\n "custom_labels": {},\n "version": ""\n },\n {\n "service_id": "46394641-bdb9-4dea-9e93-383402bf94f0",\n "service_name": "monitoring-pmm3-14502-monitoring-pmm3-mongos-1",\n "node_id": "4d9d6d17-3ba2-4719-9a0f-da52c142fe57",\n "address": "localhost",\n "port": 27019,\n "socket": "",\n "environment": "dev-mongos",\n "cluster": "super-custom",\n "replication_set": "",\n "custom_labels": {},\n "version": ""\n },\n {\n "service_id": "7c494ba1-0ac7-4c9f-8eff-697b8cf61271",\n "service_name": "monitoring-pmm3-14502-monitoring-pmm3-cfg-1",\n "node_id": "9cb595c2-e403-447d-a53a-e340c7d2d8cb",\n "address": "localhost",\n "port": 27019,\n "socket": "",\n "environment": "dev-mongod",\n "cluster": "super-custom",\n "replication_set": "",\n "custom_labels": {},\n "version": ""\n },\n {\n "service_id": "8617596a-b2a2-4cbf-8f96-85e00118ce7a",\n "service_name": "monitoring-pmm3-14502-monitoring-pmm3-mongos-2",\n "node_id": "e43bf5f4-2d9d-43c3-9e29-8c198e451f6b",\n "address": "localhost",\n "port": 27019,\n "socket": "",\n "environment": "dev-mongos",\n "cluster": "super-custom",\n "replication_set": "",\n "custom_labels": {},\n "version": ""\n },\n {\n "service_id": "a03af921-b2a2-4253-97d7-29b9773f6b0f",\n "service_name": "monitoring-pmm3-14502-monitoring-pmm3-rs0-1",\n "node_id": "38970820-7985-4a2b-ab41-91454f079afd",\n "address": "localhost",\n "port": 27019,\n "socket": "",\n "environment": "dev-mongod",\n "cluster": "super-custom",\n "replication_set": "",\n "custom_labels": {},\n "version": ""\n },\n {\n "service_id": "d45a8270-b4f1-4af9-8a19-6d100eb36a8e",\n "service_name": "monitoring-pmm3-14502-monitoring-pmm3-cfg-2",\n "node_id": "6e5e579d-0991-4d93-89f2-2390d520c3e4",\n "address": "localhost",\n "port": 27019,\n "socket": "",\n "environment": "dev-mongod",\n "cluster": "super-custom",\n "replication_set": "",\n "custom_labels": {},\n "version": ""\n }\n ],\n "postgresql": [],\n "proxysql": [],\n "haproxy": [],\n "external": [],\n "valkey": []\n}' + actual_cluster=super-custom + [[ -z super-custom ]] + [[ super-custom == null ]] + [[ super-custom != super\-custom ]] + for service_name in "${service_names[@]}" + local actual_cluster ++ echo $'{\n "mysql": [],\n "mongodb": [\n {\n "service_id": "0b223f5a-5c05-40a3-b3cf-67a7bf9710f8",\n "service_name": "monitoring-pmm3-14502-monitoring-pmm3-mongos-0",\n "node_id": "568ad465-8ec4-4f7c-a1df-adaeb7b46d26",\n "address": "localhost",\n "port": 27019,\n "socket": "",\n "environment": "dev-mongos",\n "cluster": "super-custom",\n "replication_set": "",\n "custom_labels": {},\n "version": ""\n },\n {\n "service_id": "0c977ddb-0f55-4f95-a3e8-59dec3e4c8d0",\n "service_name": "monitoring-pmm3-14502-monitoring-pmm3-cfg-0",\n "node_id": "9f635edf-62b7-42c4-829c-71f022e4324a",\n "address": "localhost",\n "port": 27019,\n "socket": "",\n "environment": "dev-mongod",\n "cluster": "super-custom",\n "replication_set": "",\n "custom_labels": {},\n "version": ""\n },\n {\n "service_id": "125658bd-2b56-452f-bc9a-c8eaabf3dd33",\n "service_name": "monitoring-pmm3-14502-monitoring-pmm3-rs0-0",\n "node_id": "62ab6a63-009a-4bd8-80e5-865935478e6c",\n "address": "localhost",\n "port": 27019,\n "socket": "",\n "environment": "dev-mongod",\n "cluster": "super-custom",\n "replication_set": "",\n "custom_labels": {},\n "version": ""\n },\n {\n "service_id": "3c27e158-ad06-4fe1-819f-acc80f8f32a2",\n "service_name": "monitoring-pmm3-14502-monitoring-pmm3-rs0-2",\n "node_id": "dc90c940-2f86-44f6-8fa5-3c58a2fff7c1",\n "address": "localhost",\n "port": 27019,\n "socket": "",\n "environment": "dev-mongod",\n "cluster": "super-custom",\n "replication_set": "",\n "custom_labels": {},\n "version": ""\n },\n {\n "service_id": "46394641-bdb9-4dea-9e93-383402bf94f0",\n "service_name": "monitoring-pmm3-14502-monitoring-pmm3-mongos-1",\n "node_id": "4d9d6d17-3ba2-4719-9a0f-da52c142fe57",\n "address": "localhost",\n "port": 27019,\n "socket": "",\n "environment": "dev-mongos",\n "cluster": "super-custom",\n "replication_set": "",\n "custom_labels": {},\n "version": ""\n },\n {\n "service_id": "7c494ba1-0ac7-4c9f-8eff-697b8cf61271",\n "service_name": "monitoring-pmm3-14502-monitoring-pmm3-cfg-1",\n "node_id": "9cb595c2-e403-447d-a53a-e340c7d2d8cb",\n "address": "localhost",\n "port": 27019,\n "socket": "",\n "environment": "dev-mongod",\n "cluster": "super-custom",\n "replication_set": "",\n "custom_labels": {},\n "version": ""\n },\n {\n "service_id": "8617596a-b2a2-4cbf-8f96-85e00118ce7a",\n "service_name": "monitoring-pmm3-14502-monitoring-pmm3-mongos-2",\n "node_id": "e43bf5f4-2d9d-43c3-9e29-8c198e451f6b",\n "address": "localhost",\n "port": 27019,\n "socket": "",\n "environment": "dev-mongos",\n "cluster": "super-custom",\n "replication_set": "",\n "custom_labels": {},\n "version": ""\n },\n {\n "service_id": "a03af921-b2a2-4253-97d7-29b9773f6b0f",\n "service_name": "monitoring-pmm3-14502-monitoring-pmm3-rs0-1",\n "node_id": "38970820-7985-4a2b-ab41-91454f079afd",\n "address": "localhost",\n "port": 27019,\n "socket": "",\n "environment": "dev-mongod",\n "cluster": "super-custom",\n "replication_set": "",\n "custom_labels": {},\n "version": ""\n },\n {\n "service_id": "d45a8270-b4f1-4af9-8a19-6d100eb36a8e",\n "service_name": "monitoring-pmm3-14502-monitoring-pmm3-cfg-2",\n "node_id": "6e5e579d-0991-4d93-89f2-2390d520c3e4",\n "address": "localhost",\n "port": 27019,\n "socket": "",\n "environment": "dev-mongod",\n "cluster": "super-custom",\n "replication_set": "",\n "custom_labels": {},\n "version": ""\n }\n ],\n "postgresql": [],\n "proxysql": [],\n "haproxy": [],\n "external": [],\n "valkey": []\n}' ++ jq -r --arg name monitoring-pmm3-14502-monitoring-pmm3-cfg-0 $'\n\t\t\t.mongodb[] | select(.service_name == $name) | .cluster\n\t\t' + actual_cluster=super-custom + [[ -z super-custom ]] + [[ super-custom == null ]] + [[ super-custom != super\-custom ]] + for service_name in "${service_names[@]}" + local actual_cluster ++ echo $'{\n "mysql": [],\n "mongodb": [\n {\n "service_id": "0b223f5a-5c05-40a3-b3cf-67a7bf9710f8",\n "service_name": "monitoring-pmm3-14502-monitoring-pmm3-mongos-0",\n "node_id": "568ad465-8ec4-4f7c-a1df-adaeb7b46d26",\n "address": "localhost",\n "port": 27019,\n "socket": "",\n "environment": "dev-mongos",\n "cluster": "super-custom",\n "replication_set": "",\n "custom_labels": {},\n "version": ""\n },\n {\n "service_id": "0c977ddb-0f55-4f95-a3e8-59dec3e4c8d0",\n "service_name": "monitoring-pmm3-14502-monitoring-pmm3-cfg-0",\n "node_id": "9f635edf-62b7-42c4-829c-71f022e4324a",\n "address": "localhost",\n "port": 27019,\n "socket": "",\n "environment": "dev-mongod",\n "cluster": "super-custom",\n "replication_set": "",\n "custom_labels": {},\n "version": ""\n },\n {\n "service_id": "125658bd-2b56-452f-bc9a-c8eaabf3dd33",\n "service_name": "monitoring-pmm3-14502-monitoring-pmm3-rs0-0",\n "node_id": "62ab6a63-009a-4bd8-80e5-865935478e6c",\n "address": "localhost",\n "port": 27019,\n "socket": "",\n "environment": "dev-mongod",\n "cluster": "super-custom",\n "replication_set": "",\n "custom_labels": {},\n "version": ""\n },\n {\n "service_id": "3c27e158-ad06-4fe1-819f-acc80f8f32a2",\n "service_name": "monitoring-pmm3-14502-monitoring-pmm3-rs0-2",\n "node_id": "dc90c940-2f86-44f6-8fa5-3c58a2fff7c1",\n "address": "localhost",\n "port": 27019,\n "socket": "",\n "environment": "dev-mongod",\n "cluster": "super-custom",\n "replication_set": "",\n "custom_labels": {},\n "version": ""\n },\n {\n "service_id": "46394641-bdb9-4dea-9e93-383402bf94f0",\n "service_name": "monitoring-pmm3-14502-monitoring-pmm3-mongos-1",\n "node_id": "4d9d6d17-3ba2-4719-9a0f-da52c142fe57",\n "address": "localhost",\n "port": 27019,\n "socket": "",\n "environment": "dev-mongos",\n "cluster": "super-custom",\n "replication_set": "",\n "custom_labels": {},\n "version": ""\n },\n {\n "service_id": "7c494ba1-0ac7-4c9f-8eff-697b8cf61271",\n "service_name": "monitoring-pmm3-14502-monitoring-pmm3-cfg-1",\n "node_id": "9cb595c2-e403-447d-a53a-e340c7d2d8cb",\n "address": "localhost",\n "port": 27019,\n "socket": "",\n "environment": "dev-mongod",\n "cluster": "super-custom",\n "replication_set": "",\n "custom_labels": {},\n "version": ""\n },\n {\n "service_id": "8617596a-b2a2-4cbf-8f96-85e00118ce7a",\n "service_name": "monitoring-pmm3-14502-monitoring-pmm3-mongos-2",\n "node_id": "e43bf5f4-2d9d-43c3-9e29-8c198e451f6b",\n "address": "localhost",\n "port": 27019,\n "socket": "",\n "environment": "dev-mongos",\n "cluster": "super-custom",\n "replication_set": "",\n "custom_labels": {},\n "version": ""\n },\n {\n "service_id": "a03af921-b2a2-4253-97d7-29b9773f6b0f",\n "service_name": "monitoring-pmm3-14502-monitoring-pmm3-rs0-1",\n "node_id": "38970820-7985-4a2b-ab41-91454f079afd",\n "address": "localhost",\n "port": 27019,\n "socket": "",\n "environment": "dev-mongod",\n "cluster": "super-custom",\n "replication_set": "",\n "custom_labels": {},\n "version": ""\n },\n {\n "service_id": "d45a8270-b4f1-4af9-8a19-6d100eb36a8e",\n "service_name": "monitoring-pmm3-14502-monitoring-pmm3-cfg-2",\n "node_id": "6e5e579d-0991-4d93-89f2-2390d520c3e4",\n "address": "localhost",\n "port": 27019,\n "socket": "",\n "environment": "dev-mongod",\n "cluster": "super-custom",\n "replication_set": "",\n "custom_labels": {},\n "version": ""\n }\n ],\n "postgresql": [],\n "proxysql": [],\n "haproxy": [],\n "external": [],\n "valkey": []\n}' ++ jq -r --arg name monitoring-pmm3-14502-monitoring-pmm3-rs0-0 $'\n\t\t\t.mongodb[] | select(.service_name == $name) | .cluster\n\t\t' + actual_cluster=super-custom + [[ -z super-custom ]] + [[ super-custom == null ]] + [[ super-custom != super\-custom ]] + return 0 + nodeList=($(get_node_id_from_pmm)) ++ get_node_id_from_pmm ++ nodeList=() ++ local -a nodeList +++ kubectl_bin get pods --no-headers -l app.kubernetes.io/name=percona-server-mongodb --output=custom-columns=NAME:.metadata.name ++++ mktemp +++ local LAST_OUT=/tmp/tmp.QPFpcPS6zh ++++ mktemp +++ local LAST_ERR=/tmp/tmp.qsJmkVFr9o +++ local exit_status=0 +++ local timeout=4 +++ args=('get' 'pods' '--no-headers' '-l' 'app.kubernetes.io/name=percona-server-mongodb' '--output=custom-columns=NAME:.metadata.name') +++ local args +++ transformed_files=() +++ local transformed_files +++ arch_scheduling_enabled +++ [[ -n '' ]] +++ arch_scheduling_enabled +++ [[ -n '' ]] ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl get pods --no-headers -l app.kubernetes.io/name=percona-server-mongodb --output=custom-columns=NAME:.metadata.name +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 -a -n 1 ']' +++ break +++ cat /tmp/tmp.QPFpcPS6zh +++ cat /tmp/tmp.qsJmkVFr9o +++ rm /tmp/tmp.QPFpcPS6zh /tmp/tmp.qsJmkVFr9o +++ return 0 ++ for instance in $(kubectl_bin get pods --no-headers -l app.kubernetes.io/name=percona-server-mongodb --output=custom-columns='NAME:.metadata.name') ++ nodeList+=($(kubectl_bin exec -n "$namespace" $instance -c pmm-client -- pmm-admin status --json | jq -r '.pmm_agent_status.node_id')) +++ kubectl_bin exec -n monitoring-pmm3-14502 monitoring-pmm3-cfg-0 -c pmm-client -- pmm-admin status --json +++ jq -r .pmm_agent_status.node_id ++++ mktemp +++ local LAST_OUT=/tmp/tmp.QXBHF9NZT6 ++++ mktemp +++ local LAST_ERR=/tmp/tmp.MjrxiVIYai +++ local exit_status=0 +++ local timeout=4 +++ args=('exec' '-n' 'monitoring-pmm3-14502' 'monitoring-pmm3-cfg-0' '-c' 'pmm-client' '--' 'pmm-admin' 'status' '--json') +++ local args +++ transformed_files=() +++ local transformed_files +++ arch_scheduling_enabled +++ [[ -n '' ]] +++ arch_scheduling_enabled +++ [[ -n '' ]] ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl exec -n monitoring-pmm3-14502 monitoring-pmm3-cfg-0 -c pmm-client -- pmm-admin status --json +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 -a -n 1 ']' +++ break +++ cat /tmp/tmp.QXBHF9NZT6 +++ cat /tmp/tmp.MjrxiVIYai +++ rm /tmp/tmp.QXBHF9NZT6 /tmp/tmp.MjrxiVIYai +++ return 0 ++ for instance in $(kubectl_bin get pods --no-headers -l app.kubernetes.io/name=percona-server-mongodb --output=custom-columns='NAME:.metadata.name') ++ nodeList+=($(kubectl_bin exec -n "$namespace" $instance -c pmm-client -- pmm-admin status --json | jq -r '.pmm_agent_status.node_id')) +++ kubectl_bin exec -n monitoring-pmm3-14502 monitoring-pmm3-cfg-1 -c pmm-client -- pmm-admin status --json +++ jq -r .pmm_agent_status.node_id ++++ mktemp +++ local LAST_OUT=/tmp/tmp.6ub7WhWjpl ++++ mktemp +++ local LAST_ERR=/tmp/tmp.d1JkZpp4Ci +++ local exit_status=0 +++ local timeout=4 +++ args=('exec' '-n' 'monitoring-pmm3-14502' 'monitoring-pmm3-cfg-1' '-c' 'pmm-client' '--' 'pmm-admin' 'status' '--json') +++ local args +++ transformed_files=() +++ local transformed_files +++ arch_scheduling_enabled +++ [[ -n '' ]] +++ arch_scheduling_enabled +++ [[ -n '' ]] ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl exec -n monitoring-pmm3-14502 monitoring-pmm3-cfg-1 -c pmm-client -- pmm-admin status --json +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 -a -n 1 ']' +++ break +++ cat /tmp/tmp.6ub7WhWjpl +++ cat /tmp/tmp.d1JkZpp4Ci +++ rm /tmp/tmp.6ub7WhWjpl /tmp/tmp.d1JkZpp4Ci +++ return 0 ++ for instance in $(kubectl_bin get pods --no-headers -l app.kubernetes.io/name=percona-server-mongodb --output=custom-columns='NAME:.metadata.name') ++ nodeList+=($(kubectl_bin exec -n "$namespace" $instance -c pmm-client -- pmm-admin status --json | jq -r '.pmm_agent_status.node_id')) +++ kubectl_bin exec -n monitoring-pmm3-14502 monitoring-pmm3-cfg-2 -c pmm-client -- pmm-admin status --json +++ jq -r .pmm_agent_status.node_id ++++ mktemp +++ local LAST_OUT=/tmp/tmp.v7cEx2hp9X ++++ mktemp +++ local LAST_ERR=/tmp/tmp.VTKMqxKTr4 +++ local exit_status=0 +++ local timeout=4 +++ args=('exec' '-n' 'monitoring-pmm3-14502' 'monitoring-pmm3-cfg-2' '-c' 'pmm-client' '--' 'pmm-admin' 'status' '--json') +++ local args +++ transformed_files=() +++ local transformed_files +++ arch_scheduling_enabled +++ [[ -n '' ]] +++ arch_scheduling_enabled +++ [[ -n '' ]] ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl exec -n monitoring-pmm3-14502 monitoring-pmm3-cfg-2 -c pmm-client -- pmm-admin status --json +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 -a -n 1 ']' +++ break +++ cat /tmp/tmp.v7cEx2hp9X +++ cat /tmp/tmp.VTKMqxKTr4 +++ rm /tmp/tmp.v7cEx2hp9X /tmp/tmp.VTKMqxKTr4 +++ return 0 ++ for instance in $(kubectl_bin get pods --no-headers -l app.kubernetes.io/name=percona-server-mongodb --output=custom-columns='NAME:.metadata.name') ++ nodeList+=($(kubectl_bin exec -n "$namespace" $instance -c pmm-client -- pmm-admin status --json | jq -r '.pmm_agent_status.node_id')) +++ kubectl_bin exec -n monitoring-pmm3-14502 monitoring-pmm3-mongos-0 -c pmm-client -- pmm-admin status --json +++ jq -r .pmm_agent_status.node_id ++++ mktemp +++ local LAST_OUT=/tmp/tmp.ROsLR9LDRj ++++ mktemp +++ local LAST_ERR=/tmp/tmp.slToJYdYOB +++ local exit_status=0 +++ local timeout=4 +++ args=('exec' '-n' 'monitoring-pmm3-14502' 'monitoring-pmm3-mongos-0' '-c' 'pmm-client' '--' 'pmm-admin' 'status' '--json') +++ local args +++ transformed_files=() +++ local transformed_files +++ arch_scheduling_enabled +++ [[ -n '' ]] +++ arch_scheduling_enabled +++ [[ -n '' ]] ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl exec -n monitoring-pmm3-14502 monitoring-pmm3-mongos-0 -c pmm-client -- pmm-admin status --json +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 -a -n 1 ']' +++ break +++ cat /tmp/tmp.ROsLR9LDRj +++ cat /tmp/tmp.slToJYdYOB +++ rm /tmp/tmp.ROsLR9LDRj /tmp/tmp.slToJYdYOB +++ return 0 ++ for instance in $(kubectl_bin get pods --no-headers -l app.kubernetes.io/name=percona-server-mongodb --output=custom-columns='NAME:.metadata.name') ++ nodeList+=($(kubectl_bin exec -n "$namespace" $instance -c pmm-client -- pmm-admin status --json | jq -r '.pmm_agent_status.node_id')) +++ kubectl_bin exec -n monitoring-pmm3-14502 monitoring-pmm3-mongos-1 -c pmm-client -- pmm-admin status --json +++ jq -r .pmm_agent_status.node_id ++++ mktemp +++ local LAST_OUT=/tmp/tmp.SlqHdOlVsG ++++ mktemp +++ local LAST_ERR=/tmp/tmp.zphdrwh0HG +++ local exit_status=0 +++ local timeout=4 +++ args=('exec' '-n' 'monitoring-pmm3-14502' 'monitoring-pmm3-mongos-1' '-c' 'pmm-client' '--' 'pmm-admin' 'status' '--json') +++ local args +++ transformed_files=() +++ local transformed_files +++ arch_scheduling_enabled +++ [[ -n '' ]] +++ arch_scheduling_enabled +++ [[ -n '' ]] ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl exec -n monitoring-pmm3-14502 monitoring-pmm3-mongos-1 -c pmm-client -- pmm-admin status --json +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 -a -n 1 ']' +++ break +++ cat /tmp/tmp.SlqHdOlVsG +++ cat /tmp/tmp.zphdrwh0HG +++ rm /tmp/tmp.SlqHdOlVsG /tmp/tmp.zphdrwh0HG +++ return 0 ++ for instance in $(kubectl_bin get pods --no-headers -l app.kubernetes.io/name=percona-server-mongodb --output=custom-columns='NAME:.metadata.name') ++ nodeList+=($(kubectl_bin exec -n "$namespace" $instance -c pmm-client -- pmm-admin status --json | jq -r '.pmm_agent_status.node_id')) +++ kubectl_bin exec -n monitoring-pmm3-14502 monitoring-pmm3-mongos-2 -c pmm-client -- pmm-admin status --json +++ jq -r .pmm_agent_status.node_id ++++ mktemp +++ local LAST_OUT=/tmp/tmp.BtDFaFdYmZ ++++ mktemp +++ local LAST_ERR=/tmp/tmp.qE80RXF8PR +++ local exit_status=0 +++ local timeout=4 +++ args=('exec' '-n' 'monitoring-pmm3-14502' 'monitoring-pmm3-mongos-2' '-c' 'pmm-client' '--' 'pmm-admin' 'status' '--json') +++ local args +++ transformed_files=() +++ local transformed_files +++ arch_scheduling_enabled +++ [[ -n '' ]] +++ arch_scheduling_enabled +++ [[ -n '' ]] ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl exec -n monitoring-pmm3-14502 monitoring-pmm3-mongos-2 -c pmm-client -- pmm-admin status --json +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 -a -n 1 ']' +++ break +++ cat /tmp/tmp.BtDFaFdYmZ +++ cat /tmp/tmp.qE80RXF8PR +++ rm /tmp/tmp.BtDFaFdYmZ /tmp/tmp.qE80RXF8PR +++ return 0 ++ for instance in $(kubectl_bin get pods --no-headers -l app.kubernetes.io/name=percona-server-mongodb --output=custom-columns='NAME:.metadata.name') ++ nodeList+=($(kubectl_bin exec -n "$namespace" $instance -c pmm-client -- pmm-admin status --json | jq -r '.pmm_agent_status.node_id')) +++ kubectl_bin exec -n monitoring-pmm3-14502 monitoring-pmm3-rs0-0 -c pmm-client -- pmm-admin status --json +++ jq -r .pmm_agent_status.node_id ++++ mktemp +++ local LAST_OUT=/tmp/tmp.PAqe5NRNU3 ++++ mktemp +++ local LAST_ERR=/tmp/tmp.cUcpOCn498 +++ local exit_status=0 +++ local timeout=4 +++ args=('exec' '-n' 'monitoring-pmm3-14502' 'monitoring-pmm3-rs0-0' '-c' 'pmm-client' '--' 'pmm-admin' 'status' '--json') +++ local args +++ transformed_files=() +++ local transformed_files +++ arch_scheduling_enabled +++ [[ -n '' ]] +++ arch_scheduling_enabled +++ [[ -n '' ]] ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl exec -n monitoring-pmm3-14502 monitoring-pmm3-rs0-0 -c pmm-client -- pmm-admin status --json +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 -a -n 1 ']' +++ break +++ cat /tmp/tmp.PAqe5NRNU3 +++ cat /tmp/tmp.cUcpOCn498 +++ rm /tmp/tmp.PAqe5NRNU3 /tmp/tmp.cUcpOCn498 +++ return 0 ++ for instance in $(kubectl_bin get pods --no-headers -l app.kubernetes.io/name=percona-server-mongodb --output=custom-columns='NAME:.metadata.name') ++ nodeList+=($(kubectl_bin exec -n "$namespace" $instance -c pmm-client -- pmm-admin status --json | jq -r '.pmm_agent_status.node_id')) +++ kubectl_bin exec -n monitoring-pmm3-14502 monitoring-pmm3-rs0-1 -c pmm-client -- pmm-admin status --json +++ jq -r .pmm_agent_status.node_id ++++ mktemp +++ local LAST_OUT=/tmp/tmp.562tQDlVTw ++++ mktemp +++ local LAST_ERR=/tmp/tmp.FbZQ8GvFoa +++ local exit_status=0 +++ local timeout=4 +++ args=('exec' '-n' 'monitoring-pmm3-14502' 'monitoring-pmm3-rs0-1' '-c' 'pmm-client' '--' 'pmm-admin' 'status' '--json') +++ local args +++ transformed_files=() +++ local transformed_files +++ arch_scheduling_enabled +++ [[ -n '' ]] +++ arch_scheduling_enabled +++ [[ -n '' ]] ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl exec -n monitoring-pmm3-14502 monitoring-pmm3-rs0-1 -c pmm-client -- pmm-admin status --json +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 -a -n 1 ']' +++ break +++ cat /tmp/tmp.562tQDlVTw +++ cat /tmp/tmp.FbZQ8GvFoa +++ rm /tmp/tmp.562tQDlVTw /tmp/tmp.FbZQ8GvFoa +++ return 0 ++ for instance in $(kubectl_bin get pods --no-headers -l app.kubernetes.io/name=percona-server-mongodb --output=custom-columns='NAME:.metadata.name') ++ nodeList+=($(kubectl_bin exec -n "$namespace" $instance -c pmm-client -- pmm-admin status --json | jq -r '.pmm_agent_status.node_id')) +++ kubectl_bin exec -n monitoring-pmm3-14502 monitoring-pmm3-rs0-2 -c pmm-client -- pmm-admin status --json +++ jq -r .pmm_agent_status.node_id ++++ mktemp +++ local LAST_OUT=/tmp/tmp.9MOMaqbNCZ ++++ mktemp +++ local LAST_ERR=/tmp/tmp.khOoloNeEL +++ local exit_status=0 +++ local timeout=4 +++ args=('exec' '-n' 'monitoring-pmm3-14502' 'monitoring-pmm3-rs0-2' '-c' 'pmm-client' '--' 'pmm-admin' 'status' '--json') +++ local args +++ transformed_files=() +++ local transformed_files +++ arch_scheduling_enabled +++ [[ -n '' ]] +++ arch_scheduling_enabled +++ [[ -n '' ]] ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl exec -n monitoring-pmm3-14502 monitoring-pmm3-rs0-2 -c pmm-client -- pmm-admin status --json +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 -a -n 1 ']' +++ break +++ cat /tmp/tmp.9MOMaqbNCZ +++ cat /tmp/tmp.khOoloNeEL +++ rm /tmp/tmp.9MOMaqbNCZ /tmp/tmp.khOoloNeEL +++ return 0 ++ echo 9f635edf-62b7-42c4-829c-71f022e4324a 9cb595c2-e403-447d-a53a-e340c7d2d8cb 6e5e579d-0991-4d93-89f2-2390d520c3e4 568ad465-8ec4-4f7c-a1df-adaeb7b46d26 4d9d6d17-3ba2-4719-9a0f-da52c142fe57 e43bf5f4-2d9d-43c3-9e29-8c198e451f6b 62ab6a63-009a-4bd8-80e5-865935478e6c 38970820-7985-4a2b-ab41-91454f079afd dc90c940-2f86-44f6-8fa5-3c58a2fff7c1 + nodeList_from_pmm=($(does_node_id_exists "${nodeList[@]}")) ++ does_node_id_exists 9f635edf-62b7-42c4-829c-71f022e4324a 9cb595c2-e403-447d-a53a-e340c7d2d8cb 6e5e579d-0991-4d93-89f2-2390d520c3e4 568ad465-8ec4-4f7c-a1df-adaeb7b46d26 4d9d6d17-3ba2-4719-9a0f-da52c142fe57 e43bf5f4-2d9d-43c3-9e29-8c198e451f6b 62ab6a63-009a-4bd8-80e5-865935478e6c 38970820-7985-4a2b-ab41-91454f079afd dc90c940-2f86-44f6-8fa5-3c58a2fff7c1 ++ nodeList=('9f635edf-62b7-42c4-829c-71f022e4324a' '9cb595c2-e403-447d-a53a-e340c7d2d8cb' '6e5e579d-0991-4d93-89f2-2390d520c3e4' '568ad465-8ec4-4f7c-a1df-adaeb7b46d26' '4d9d6d17-3ba2-4719-9a0f-da52c142fe57' 'e43bf5f4-2d9d-43c3-9e29-8c198e451f6b' '62ab6a63-009a-4bd8-80e5-865935478e6c' '38970820-7985-4a2b-ab41-91454f079afd' 'dc90c940-2f86-44f6-8fa5-3c58a2fff7c1') ++ local -a nodeList ++ nodeList_from_pmm=() ++ local -a nodeList_from_pmm ++ local admin_pass endpoint encoded_admin_userinfo +++ kubectl_bin get secret pmm-secret -o 'jsonpath={.data.PMM_ADMIN_PASSWORD}' +++ base64 --decode ++++ mktemp +++ local LAST_OUT=/tmp/tmp.khADM0ewNJ ++++ mktemp +++ local LAST_ERR=/tmp/tmp.4CFY6NENix +++ local exit_status=0 +++ local timeout=4 +++ args=('get' 'secret' 'pmm-secret' '-o' 'jsonpath={.data.PMM_ADMIN_PASSWORD}') +++ local args +++ transformed_files=() +++ local transformed_files +++ arch_scheduling_enabled +++ [[ -n '' ]] +++ arch_scheduling_enabled +++ [[ -n '' ]] ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl get secret pmm-secret -o 'jsonpath={.data.PMM_ADMIN_PASSWORD}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 -a -n 1 ']' +++ break +++ cat /tmp/tmp.khADM0ewNJ +++ cat /tmp/tmp.4CFY6NENix +++ rm /tmp/tmp.khADM0ewNJ /tmp/tmp.4CFY6NENix +++ return 0 ++ admin_pass='J:k%i1RN=xa^Z5a!' +++ get_service_endpoint monitoring-service +++ local service=monitoring-service +++ local endpoint +++ local service_type ++++ kubectl_bin get service/monitoring-service -o 'jsonpath={.spec.type}' +++++ mktemp ++++ local LAST_OUT=/tmp/tmp.gTxA8EGCvz +++++ mktemp ++++ local LAST_ERR=/tmp/tmp.TvKH7kpn9D ++++ local exit_status=0 ++++ local timeout=4 ++++ args=('get' 'service/monitoring-service' '-o' 'jsonpath={.spec.type}') ++++ local args ++++ transformed_files=() ++++ local transformed_files ++++ arch_scheduling_enabled ++++ [[ -n '' ]] ++++ arch_scheduling_enabled ++++ [[ -n '' ]] +++++ seq 0 2 ++++ for i in $(seq 0 2) ++++ set +e ++++ kubectl get service/monitoring-service -o 'jsonpath={.spec.type}' ++++ exit_status=0 ++++ set -e ++++ '[' 0 '!=' 0 -a -n 1 ']' ++++ break ++++ cat /tmp/tmp.gTxA8EGCvz ++++ cat /tmp/tmp.TvKH7kpn9D ++++ rm /tmp/tmp.gTxA8EGCvz /tmp/tmp.TvKH7kpn9D ++++ return 0 +++ service_type=LoadBalancer +++ [[ LoadBalancer == LoadBalancer ]] ++++ get_load_balancer_endpoint monitoring-service ++++ local service=monitoring-service ++++ local endpoint +++++ kubectl_bin get service/monitoring-service -o json +++++ jq -r '[.status.loadBalancer.ingress[]?.hostname, .status.loadBalancer.ingress[]?.ip] | map(select(. != null and . != "")) | first // empty' ++++++ mktemp +++++ local LAST_OUT=/tmp/tmp.nWhj5GD83R ++++++ mktemp +++++ local LAST_ERR=/tmp/tmp.5KoncUUTBm +++++ local exit_status=0 +++++ local timeout=4 +++++ args=('get' 'service/monitoring-service' '-o' 'json') +++++ local args +++++ transformed_files=() +++++ local transformed_files +++++ arch_scheduling_enabled +++++ [[ -n '' ]] +++++ arch_scheduling_enabled +++++ [[ -n '' ]] ++++++ seq 0 2 +++++ for i in $(seq 0 2) +++++ set +e +++++ kubectl get service/monitoring-service -o json +++++ exit_status=0 +++++ set -e +++++ '[' 0 '!=' 0 -a -n 1 ']' +++++ break +++++ cat /tmp/tmp.nWhj5GD83R +++++ cat /tmp/tmp.5KoncUUTBm +++++ rm /tmp/tmp.nWhj5GD83R /tmp/tmp.5KoncUUTBm +++++ return 0 ++++ endpoint=136.112.205.251 ++++ '[' -n 136.112.205.251 -a 136.112.205.251 '!=' null ']' ++++ echo 136.112.205.251 ++++ return +++ endpoint=136.112.205.251 +++ echo 136.112.205.251 +++ return ++ endpoint=136.112.205.251 +++ urlencode_userinfo 'admin:J:k%i1RN=xa^Z5a!' +++ local 'user_pass=admin:J:k%i1RN=xa^Z5a!' +++ local username=admin +++ local 'password=J:k%i1RN=xa^Z5a!' ++++ urlencode admin ++++ uri=admin ++++ echo -n admin ++++ jq -s -R -r @uri ++++ urlencode 'J:k%i1RN=xa^Z5a!' ++++ uri='J:k%i1RN=xa^Z5a!' ++++ echo -n 'J:k%i1RN=xa^Z5a!' ++++ jq -s -R -r @uri +++ printf %s:%s admin J%3Ak%25i1RN%3Dxa%5EZ5a%21 ++ encoded_admin_userinfo=admin:J%3Ak%25i1RN%3Dxa%5EZ5a%21 ++ for node_id in "${nodeList[@]}" ++ nodeList_from_pmm+=($(kubectl_bin exec -n "${namespace}" monitoring-server-0 -- pmm-admin --server-url=https://${encoded_admin_userinfo}@${endpoint}/ --server-insecure-tls inventory list nodes --node-type=container | grep $node_id | awk '{print $4}')) +++ kubectl_bin exec -n monitoring-pmm3-14502 monitoring-server-0 -- pmm-admin --server-url=https://admin:J%3Ak%25i1RN%3Dxa%5EZ5a%21@136.112.205.251/ --server-insecure-tls inventory list nodes --node-type=container +++ grep 9f635edf-62b7-42c4-829c-71f022e4324a +++ awk '{print $4}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.TqiXfPt5ci ++++ mktemp +++ local LAST_ERR=/tmp/tmp.xMhNMUIjwL +++ local exit_status=0 +++ local timeout=4 +++ args=('exec' '-n' 'monitoring-pmm3-14502' 'monitoring-server-0' '--' 'pmm-admin' '--server-url=https://admin:J%3Ak%25i1RN%3Dxa%5EZ5a%21@136.112.205.251/' '--server-insecure-tls' 'inventory' 'list' 'nodes' '--node-type=container') +++ local args +++ transformed_files=() +++ local transformed_files +++ arch_scheduling_enabled +++ [[ -n '' ]] +++ arch_scheduling_enabled +++ [[ -n '' ]] ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl exec -n monitoring-pmm3-14502 monitoring-server-0 -- pmm-admin --server-url=https://admin:J%3Ak%25i1RN%3Dxa%5EZ5a%21@136.112.205.251/ --server-insecure-tls inventory list nodes --node-type=container +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 -a -n 1 ']' +++ break +++ cat /tmp/tmp.TqiXfPt5ci +++ cat /tmp/tmp.xMhNMUIjwL +++ rm /tmp/tmp.TqiXfPt5ci /tmp/tmp.xMhNMUIjwL +++ return 0 ++ for node_id in "${nodeList[@]}" ++ nodeList_from_pmm+=($(kubectl_bin exec -n "${namespace}" monitoring-server-0 -- pmm-admin --server-url=https://${encoded_admin_userinfo}@${endpoint}/ --server-insecure-tls inventory list nodes --node-type=container | grep $node_id | awk '{print $4}')) +++ kubectl_bin exec -n monitoring-pmm3-14502 monitoring-server-0 -- pmm-admin --server-url=https://admin:J%3Ak%25i1RN%3Dxa%5EZ5a%21@136.112.205.251/ --server-insecure-tls inventory list nodes --node-type=container +++ grep 9cb595c2-e403-447d-a53a-e340c7d2d8cb +++ awk '{print $4}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.JscGlnroxn ++++ mktemp +++ local LAST_ERR=/tmp/tmp.bLkKPRzjwQ +++ local exit_status=0 +++ local timeout=4 +++ args=('exec' '-n' 'monitoring-pmm3-14502' 'monitoring-server-0' '--' 'pmm-admin' '--server-url=https://admin:J%3Ak%25i1RN%3Dxa%5EZ5a%21@136.112.205.251/' '--server-insecure-tls' 'inventory' 'list' 'nodes' '--node-type=container') +++ local args +++ transformed_files=() +++ local transformed_files +++ arch_scheduling_enabled +++ [[ -n '' ]] +++ arch_scheduling_enabled +++ [[ -n '' ]] ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl exec -n monitoring-pmm3-14502 monitoring-server-0 -- pmm-admin --server-url=https://admin:J%3Ak%25i1RN%3Dxa%5EZ5a%21@136.112.205.251/ --server-insecure-tls inventory list nodes --node-type=container +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 -a -n 1 ']' +++ break +++ cat /tmp/tmp.JscGlnroxn +++ cat /tmp/tmp.bLkKPRzjwQ +++ rm /tmp/tmp.JscGlnroxn /tmp/tmp.bLkKPRzjwQ +++ return 0 ++ for node_id in "${nodeList[@]}" ++ nodeList_from_pmm+=($(kubectl_bin exec -n "${namespace}" monitoring-server-0 -- pmm-admin --server-url=https://${encoded_admin_userinfo}@${endpoint}/ --server-insecure-tls inventory list nodes --node-type=container | grep $node_id | awk '{print $4}')) +++ kubectl_bin exec -n monitoring-pmm3-14502 monitoring-server-0 -- pmm-admin --server-url=https://admin:J%3Ak%25i1RN%3Dxa%5EZ5a%21@136.112.205.251/ --server-insecure-tls inventory list nodes --node-type=container +++ grep 6e5e579d-0991-4d93-89f2-2390d520c3e4 +++ awk '{print $4}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.Ji217Iu27j ++++ mktemp +++ local LAST_ERR=/tmp/tmp.Ki5iisxJ72 +++ local exit_status=0 +++ local timeout=4 +++ args=('exec' '-n' 'monitoring-pmm3-14502' 'monitoring-server-0' '--' 'pmm-admin' '--server-url=https://admin:J%3Ak%25i1RN%3Dxa%5EZ5a%21@136.112.205.251/' '--server-insecure-tls' 'inventory' 'list' 'nodes' '--node-type=container') +++ local args +++ transformed_files=() +++ local transformed_files +++ arch_scheduling_enabled +++ [[ -n '' ]] +++ arch_scheduling_enabled +++ [[ -n '' ]] ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl exec -n monitoring-pmm3-14502 monitoring-server-0 -- pmm-admin --server-url=https://admin:J%3Ak%25i1RN%3Dxa%5EZ5a%21@136.112.205.251/ --server-insecure-tls inventory list nodes --node-type=container +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 -a -n 1 ']' +++ break +++ cat /tmp/tmp.Ji217Iu27j +++ cat /tmp/tmp.Ki5iisxJ72 +++ rm /tmp/tmp.Ji217Iu27j /tmp/tmp.Ki5iisxJ72 +++ return 0 ++ for node_id in "${nodeList[@]}" ++ nodeList_from_pmm+=($(kubectl_bin exec -n "${namespace}" monitoring-server-0 -- pmm-admin --server-url=https://${encoded_admin_userinfo}@${endpoint}/ --server-insecure-tls inventory list nodes --node-type=container | grep $node_id | awk '{print $4}')) +++ kubectl_bin exec -n monitoring-pmm3-14502 monitoring-server-0 -- pmm-admin --server-url=https://admin:J%3Ak%25i1RN%3Dxa%5EZ5a%21@136.112.205.251/ --server-insecure-tls inventory list nodes --node-type=container +++ grep 568ad465-8ec4-4f7c-a1df-adaeb7b46d26 +++ awk '{print $4}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.ef6vGrA4zV ++++ mktemp +++ local LAST_ERR=/tmp/tmp.OTdxEx8Gj8 +++ local exit_status=0 +++ local timeout=4 +++ args=('exec' '-n' 'monitoring-pmm3-14502' 'monitoring-server-0' '--' 'pmm-admin' '--server-url=https://admin:J%3Ak%25i1RN%3Dxa%5EZ5a%21@136.112.205.251/' '--server-insecure-tls' 'inventory' 'list' 'nodes' '--node-type=container') +++ local args +++ transformed_files=() +++ local transformed_files +++ arch_scheduling_enabled +++ [[ -n '' ]] +++ arch_scheduling_enabled +++ [[ -n '' ]] ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl exec -n monitoring-pmm3-14502 monitoring-server-0 -- pmm-admin --server-url=https://admin:J%3Ak%25i1RN%3Dxa%5EZ5a%21@136.112.205.251/ --server-insecure-tls inventory list nodes --node-type=container +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 -a -n 1 ']' +++ break +++ cat /tmp/tmp.ef6vGrA4zV +++ cat /tmp/tmp.OTdxEx8Gj8 +++ rm /tmp/tmp.ef6vGrA4zV /tmp/tmp.OTdxEx8Gj8 +++ return 0 ++ for node_id in "${nodeList[@]}" ++ nodeList_from_pmm+=($(kubectl_bin exec -n "${namespace}" monitoring-server-0 -- pmm-admin --server-url=https://${encoded_admin_userinfo}@${endpoint}/ --server-insecure-tls inventory list nodes --node-type=container | grep $node_id | awk '{print $4}')) +++ kubectl_bin exec -n monitoring-pmm3-14502 monitoring-server-0 -- pmm-admin --server-url=https://admin:J%3Ak%25i1RN%3Dxa%5EZ5a%21@136.112.205.251/ --server-insecure-tls inventory list nodes --node-type=container +++ grep 4d9d6d17-3ba2-4719-9a0f-da52c142fe57 +++ awk '{print $4}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.oOe0daW8RQ ++++ mktemp +++ local LAST_ERR=/tmp/tmp.giOlItC9Ez +++ local exit_status=0 +++ local timeout=4 +++ args=('exec' '-n' 'monitoring-pmm3-14502' 'monitoring-server-0' '--' 'pmm-admin' '--server-url=https://admin:J%3Ak%25i1RN%3Dxa%5EZ5a%21@136.112.205.251/' '--server-insecure-tls' 'inventory' 'list' 'nodes' '--node-type=container') +++ local args +++ transformed_files=() +++ local transformed_files +++ arch_scheduling_enabled +++ [[ -n '' ]] +++ arch_scheduling_enabled +++ [[ -n '' ]] ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl exec -n monitoring-pmm3-14502 monitoring-server-0 -- pmm-admin --server-url=https://admin:J%3Ak%25i1RN%3Dxa%5EZ5a%21@136.112.205.251/ --server-insecure-tls inventory list nodes --node-type=container +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 -a -n 1 ']' +++ break +++ cat /tmp/tmp.oOe0daW8RQ +++ cat /tmp/tmp.giOlItC9Ez +++ rm /tmp/tmp.oOe0daW8RQ /tmp/tmp.giOlItC9Ez +++ return 0 ++ for node_id in "${nodeList[@]}" ++ nodeList_from_pmm+=($(kubectl_bin exec -n "${namespace}" monitoring-server-0 -- pmm-admin --server-url=https://${encoded_admin_userinfo}@${endpoint}/ --server-insecure-tls inventory list nodes --node-type=container | grep $node_id | awk '{print $4}')) +++ kubectl_bin exec -n monitoring-pmm3-14502 monitoring-server-0 -- pmm-admin --server-url=https://admin:J%3Ak%25i1RN%3Dxa%5EZ5a%21@136.112.205.251/ --server-insecure-tls inventory list nodes --node-type=container +++ grep e43bf5f4-2d9d-43c3-9e29-8c198e451f6b +++ awk '{print $4}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.ZDTX4aZtTv ++++ mktemp +++ local LAST_ERR=/tmp/tmp.nfBL27bnXB +++ local exit_status=0 +++ local timeout=4 +++ args=('exec' '-n' 'monitoring-pmm3-14502' 'monitoring-server-0' '--' 'pmm-admin' '--server-url=https://admin:J%3Ak%25i1RN%3Dxa%5EZ5a%21@136.112.205.251/' '--server-insecure-tls' 'inventory' 'list' 'nodes' '--node-type=container') +++ local args +++ transformed_files=() +++ local transformed_files +++ arch_scheduling_enabled +++ [[ -n '' ]] +++ arch_scheduling_enabled +++ [[ -n '' ]] ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl exec -n monitoring-pmm3-14502 monitoring-server-0 -- pmm-admin --server-url=https://admin:J%3Ak%25i1RN%3Dxa%5EZ5a%21@136.112.205.251/ --server-insecure-tls inventory list nodes --node-type=container +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 -a -n 1 ']' +++ break +++ cat /tmp/tmp.ZDTX4aZtTv +++ cat /tmp/tmp.nfBL27bnXB +++ rm /tmp/tmp.ZDTX4aZtTv /tmp/tmp.nfBL27bnXB +++ return 0 ++ for node_id in "${nodeList[@]}" ++ nodeList_from_pmm+=($(kubectl_bin exec -n "${namespace}" monitoring-server-0 -- pmm-admin --server-url=https://${encoded_admin_userinfo}@${endpoint}/ --server-insecure-tls inventory list nodes --node-type=container | grep $node_id | awk '{print $4}')) +++ kubectl_bin exec -n monitoring-pmm3-14502 monitoring-server-0 -- pmm-admin --server-url=https://admin:J%3Ak%25i1RN%3Dxa%5EZ5a%21@136.112.205.251/ --server-insecure-tls inventory list nodes --node-type=container +++ grep 62ab6a63-009a-4bd8-80e5-865935478e6c +++ awk '{print $4}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.q6mS5SChHi ++++ mktemp +++ local LAST_ERR=/tmp/tmp.sCNGwLWNRA +++ local exit_status=0 +++ local timeout=4 +++ args=('exec' '-n' 'monitoring-pmm3-14502' 'monitoring-server-0' '--' 'pmm-admin' '--server-url=https://admin:J%3Ak%25i1RN%3Dxa%5EZ5a%21@136.112.205.251/' '--server-insecure-tls' 'inventory' 'list' 'nodes' '--node-type=container') +++ local args +++ transformed_files=() +++ local transformed_files +++ arch_scheduling_enabled +++ [[ -n '' ]] +++ arch_scheduling_enabled +++ [[ -n '' ]] ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl exec -n monitoring-pmm3-14502 monitoring-server-0 -- pmm-admin --server-url=https://admin:J%3Ak%25i1RN%3Dxa%5EZ5a%21@136.112.205.251/ --server-insecure-tls inventory list nodes --node-type=container +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 -a -n 1 ']' +++ break +++ cat /tmp/tmp.q6mS5SChHi +++ cat /tmp/tmp.sCNGwLWNRA +++ rm /tmp/tmp.q6mS5SChHi /tmp/tmp.sCNGwLWNRA +++ return 0 ++ for node_id in "${nodeList[@]}" ++ nodeList_from_pmm+=($(kubectl_bin exec -n "${namespace}" monitoring-server-0 -- pmm-admin --server-url=https://${encoded_admin_userinfo}@${endpoint}/ --server-insecure-tls inventory list nodes --node-type=container | grep $node_id | awk '{print $4}')) +++ kubectl_bin exec -n monitoring-pmm3-14502 monitoring-server-0 -- pmm-admin --server-url=https://admin:J%3Ak%25i1RN%3Dxa%5EZ5a%21@136.112.205.251/ --server-insecure-tls inventory list nodes --node-type=container +++ grep 38970820-7985-4a2b-ab41-91454f079afd +++ awk '{print $4}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.joXhV6fdjU ++++ mktemp +++ local LAST_ERR=/tmp/tmp.dhbcN2pFkC +++ local exit_status=0 +++ local timeout=4 +++ args=('exec' '-n' 'monitoring-pmm3-14502' 'monitoring-server-0' '--' 'pmm-admin' '--server-url=https://admin:J%3Ak%25i1RN%3Dxa%5EZ5a%21@136.112.205.251/' '--server-insecure-tls' 'inventory' 'list' 'nodes' '--node-type=container') +++ local args +++ transformed_files=() +++ local transformed_files +++ arch_scheduling_enabled +++ [[ -n '' ]] +++ arch_scheduling_enabled +++ [[ -n '' ]] ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl exec -n monitoring-pmm3-14502 monitoring-server-0 -- pmm-admin --server-url=https://admin:J%3Ak%25i1RN%3Dxa%5EZ5a%21@136.112.205.251/ --server-insecure-tls inventory list nodes --node-type=container +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 -a -n 1 ']' +++ break +++ cat /tmp/tmp.joXhV6fdjU +++ cat /tmp/tmp.dhbcN2pFkC +++ rm /tmp/tmp.joXhV6fdjU /tmp/tmp.dhbcN2pFkC +++ return 0 ++ for node_id in "${nodeList[@]}" ++ nodeList_from_pmm+=($(kubectl_bin exec -n "${namespace}" monitoring-server-0 -- pmm-admin --server-url=https://${encoded_admin_userinfo}@${endpoint}/ --server-insecure-tls inventory list nodes --node-type=container | grep $node_id | awk '{print $4}')) +++ kubectl_bin exec -n monitoring-pmm3-14502 monitoring-server-0 -- pmm-admin --server-url=https://admin:J%3Ak%25i1RN%3Dxa%5EZ5a%21@136.112.205.251/ --server-insecure-tls inventory list nodes --node-type=container +++ grep dc90c940-2f86-44f6-8fa5-3c58a2fff7c1 +++ awk '{print $4}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.0D556aa9ft ++++ mktemp +++ local LAST_ERR=/tmp/tmp.EF9BfNyYL7 +++ local exit_status=0 +++ local timeout=4 +++ args=('exec' '-n' 'monitoring-pmm3-14502' 'monitoring-server-0' '--' 'pmm-admin' '--server-url=https://admin:J%3Ak%25i1RN%3Dxa%5EZ5a%21@136.112.205.251/' '--server-insecure-tls' 'inventory' 'list' 'nodes' '--node-type=container') +++ local args +++ transformed_files=() +++ local transformed_files +++ arch_scheduling_enabled +++ [[ -n '' ]] +++ arch_scheduling_enabled +++ [[ -n '' ]] ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl exec -n monitoring-pmm3-14502 monitoring-server-0 -- pmm-admin --server-url=https://admin:J%3Ak%25i1RN%3Dxa%5EZ5a%21@136.112.205.251/ --server-insecure-tls inventory list nodes --node-type=container +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 -a -n 1 ']' +++ break +++ cat /tmp/tmp.0D556aa9ft +++ cat /tmp/tmp.EF9BfNyYL7 +++ rm /tmp/tmp.0D556aa9ft /tmp/tmp.EF9BfNyYL7 +++ return 0 ++ echo 9f635edf-62b7-42c4-829c-71f022e4324a 9cb595c2-e403-447d-a53a-e340c7d2d8cb 6e5e579d-0991-4d93-89f2-2390d520c3e4 568ad465-8ec4-4f7c-a1df-adaeb7b46d26 4d9d6d17-3ba2-4719-9a0f-da52c142fe57 e43bf5f4-2d9d-43c3-9e29-8c198e451f6b 62ab6a63-009a-4bd8-80e5-865935478e6c 38970820-7985-4a2b-ab41-91454f079afd dc90c940-2f86-44f6-8fa5-3c58a2fff7c1 + for node_id in "${nodeList_from_pmm[@]}" + '[' -z 9f635edf-62b7-42c4-829c-71f022e4324a ']' + for node_id in "${nodeList_from_pmm[@]}" + '[' -z 9cb595c2-e403-447d-a53a-e340c7d2d8cb ']' + for node_id in "${nodeList_from_pmm[@]}" + '[' -z 6e5e579d-0991-4d93-89f2-2390d520c3e4 ']' + for node_id in "${nodeList_from_pmm[@]}" + '[' -z 568ad465-8ec4-4f7c-a1df-adaeb7b46d26 ']' + for node_id in "${nodeList_from_pmm[@]}" + '[' -z 4d9d6d17-3ba2-4719-9a0f-da52c142fe57 ']' + for node_id in "${nodeList_from_pmm[@]}" + '[' -z e43bf5f4-2d9d-43c3-9e29-8c198e451f6b ']' + for node_id in "${nodeList_from_pmm[@]}" + '[' -z 62ab6a63-009a-4bd8-80e5-865935478e6c ']' + for node_id in "${nodeList_from_pmm[@]}" + '[' -z 38970820-7985-4a2b-ab41-91454f079afd ']' + for node_id in "${nodeList_from_pmm[@]}" + '[' -z dc90c940-2f86-44f6-8fa5-3c58a2fff7c1 ']' + desc 'patch cluster to switch QAN source to mongolog' + set +o xtrace ----------------------------------------------------------------------------------- patch cluster to switch QAN source to mongolog ----------------------------------------------------------------------------------- + kubectl_bin patch psmdb monitoring-pmm3 --type=json $'-p=[\n {"op":"add","path":"/spec/pmm/querySource","value":"mongolog"},\n {"op":"replace","path":"/spec/pmm/mongodParams","value":"--enable-all-collectors --environment=dev-mongod-mongolog"}\n]' ++ mktemp + local LAST_OUT=/tmp/tmp.hocICzUe4h ++ mktemp + local LAST_ERR=/tmp/tmp.dcpSbqBKrY + local exit_status=0 + local timeout=4 + args=('patch' 'psmdb' 'monitoring-pmm3' '--type=json' '-p=[ {"op":"add","path":"/spec/pmm/querySource","value":"mongolog"}, {"op":"replace","path":"/spec/pmm/mongodParams","value":"--enable-all-collectors --environment=dev-mongod-mongolog"} ]') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl patch psmdb monitoring-pmm3 --type=json $'-p=[\n {"op":"add","path":"/spec/pmm/querySource","value":"mongolog"},\n {"op":"replace","path":"/spec/pmm/mongodParams","value":"--enable-all-collectors --environment=dev-mongod-mongolog"}\n]' + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.hocICzUe4h perconaservermongodb.psmdb.percona.com/monitoring-pmm3 patched + cat /tmp/tmp.dcpSbqBKrY + rm /tmp/tmp.hocICzUe4h /tmp/tmp.dcpSbqBKrY + return 0 + wait_for_running monitoring-pmm3-rs0 3 + local name=monitoring-pmm3-rs0 + let last_pod=2 + local check_cluster_readiness=true + set_debug + [[ 1 == 1 ]] + set -o xtrace + local rs_name=rs0 + local cluster_name=monitoring-pmm3 ++ seq 0 2 + for i in $(seq 0 $last_pod) + [[ 0 -eq 2 ]] + wait_pod monitoring-pmm3-rs0-0 + local pod=monitoring-pmm3-rs0-0 + set +o xtrace waiting for pod/monitoring-pmm3-rs0-0 to be ready.OK + for i in $(seq 0 $last_pod) + [[ 1 -eq 2 ]] + wait_pod monitoring-pmm3-rs0-1 + local pod=monitoring-pmm3-rs0-1 + set +o xtrace waiting for pod/monitoring-pmm3-rs0-1 to be ready.OK + for i in $(seq 0 $last_pod) + [[ 2 -eq 2 ]] ++ kubectl_bin get psmdb monitoring-pmm3 -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].arbiter.enabled}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.QanNaYKBLB +++ mktemp ++ local LAST_ERR=/tmp/tmp.G5bA2IIze8 ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'psmdb' 'monitoring-pmm3' '-o' 'jsonpath={.spec.replsets[?(@.name=="rs0")].arbiter.enabled}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb monitoring-pmm3 -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].arbiter.enabled}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.QanNaYKBLB ++ cat /tmp/tmp.G5bA2IIze8 ++ rm /tmp/tmp.QanNaYKBLB /tmp/tmp.G5bA2IIze8 ++ return 0 + [[ '' == true ]] + wait_pod monitoring-pmm3-rs0-2 + local pod=monitoring-pmm3-rs0-2 + set +o xtrace waiting for pod/monitoring-pmm3-rs0-2 to be ready.OK ++ kubectl_bin get psmdb monitoring-pmm3 -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].nonvoting.enabled}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.qd3TgkRwlu +++ mktemp ++ local LAST_ERR=/tmp/tmp.nxVzmbRGRx ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'psmdb' 'monitoring-pmm3' '-o' 'jsonpath={.spec.replsets[?(@.name=="rs0")].nonvoting.enabled}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb monitoring-pmm3 -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].nonvoting.enabled}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.qd3TgkRwlu ++ cat /tmp/tmp.nxVzmbRGRx ++ rm /tmp/tmp.qd3TgkRwlu /tmp/tmp.nxVzmbRGRx ++ return 0 + [[ '' == true ]] ++ kubectl_bin get psmdb monitoring-pmm3 -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].hidden.enabled}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.GsRHSgyWcu +++ mktemp ++ local LAST_ERR=/tmp/tmp.ulEQ2pHstm ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'psmdb' 'monitoring-pmm3' '-o' 'jsonpath={.spec.replsets[?(@.name=="rs0")].hidden.enabled}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb monitoring-pmm3 -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].hidden.enabled}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.GsRHSgyWcu ++ cat /tmp/tmp.ulEQ2pHstm ++ rm /tmp/tmp.GsRHSgyWcu /tmp/tmp.ulEQ2pHstm ++ return 0 + [[ '' == true ]] + sleep 10 + [[ true == true ]] + set +x Waiting for cluster monitoring-pmm3 to be ready................................................................................................................................................................................... + sleep 90 + desc 'check mongolog QAN data' + set +o xtrace ----------------------------------------------------------------------------------- check mongolog QAN data ----------------------------------------------------------------------------------- + get_qan_values mongodb dev-mongod-mongolog glsa_422RcuDdDskb7TLtapwWaOqcyrTCF2do_8c6a26b0 bearer + local service_type=mongodb + local environment=dev-mongod-mongolog + local user_pass=glsa_422RcuDdDskb7TLtapwWaOqcyrTCF2do_8c6a26b0 + local auth_type=bearer + local start + local end + local endpoint ++ /usr/sbin/date -u -d '-12 hour' +%Y-%m-%dT%H:%M:%S%:z + start=2026-07-16T13:56:52+00:00 ++ /usr/sbin/date -u +%Y-%m-%dT%H:%M:%S%:z + end=2026-07-17T01:56:52+00:00 ++ get_service_endpoint monitoring-service ++ local service=monitoring-service ++ local endpoint ++ local service_type +++ kubectl_bin get service/monitoring-service -o 'jsonpath={.spec.type}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.hGbISCpHtZ ++++ mktemp +++ local LAST_ERR=/tmp/tmp.zdHzYj01iX +++ local exit_status=0 +++ local timeout=4 +++ args=('get' 'service/monitoring-service' '-o' 'jsonpath={.spec.type}') +++ local args +++ transformed_files=() +++ local transformed_files +++ arch_scheduling_enabled +++ [[ -n '' ]] +++ arch_scheduling_enabled +++ [[ -n '' ]] ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl get service/monitoring-service -o 'jsonpath={.spec.type}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 -a -n 1 ']' +++ break +++ cat /tmp/tmp.hGbISCpHtZ +++ cat /tmp/tmp.zdHzYj01iX +++ rm /tmp/tmp.hGbISCpHtZ /tmp/tmp.zdHzYj01iX +++ return 0 ++ service_type=LoadBalancer ++ [[ LoadBalancer == LoadBalancer ]] +++ get_load_balancer_endpoint monitoring-service +++ local service=monitoring-service +++ local endpoint ++++ kubectl_bin get service/monitoring-service -o json ++++ jq -r '[.status.loadBalancer.ingress[]?.hostname, .status.loadBalancer.ingress[]?.ip] | map(select(. != null and . != "")) | first // empty' +++++ mktemp ++++ local LAST_OUT=/tmp/tmp.5JTUtMW1fB +++++ mktemp ++++ local LAST_ERR=/tmp/tmp.PfZyy7170C ++++ local exit_status=0 ++++ local timeout=4 ++++ args=('get' 'service/monitoring-service' '-o' 'json') ++++ local args ++++ transformed_files=() ++++ local transformed_files ++++ arch_scheduling_enabled ++++ [[ -n '' ]] ++++ arch_scheduling_enabled ++++ [[ -n '' ]] +++++ seq 0 2 ++++ for i in $(seq 0 2) ++++ set +e ++++ kubectl get service/monitoring-service -o json ++++ exit_status=0 ++++ set -e ++++ '[' 0 '!=' 0 -a -n 1 ']' ++++ break ++++ cat /tmp/tmp.5JTUtMW1fB ++++ cat /tmp/tmp.PfZyy7170C ++++ rm /tmp/tmp.5JTUtMW1fB /tmp/tmp.PfZyy7170C ++++ return 0 +++ endpoint=136.112.205.251 +++ '[' -n 136.112.205.251 -a 136.112.205.251 '!=' null ']' +++ echo 136.112.205.251 +++ return ++ endpoint=136.112.205.251 ++ echo 136.112.205.251 ++ return + endpoint=136.112.205.251 + cat + retry 30 2 curl_qan_values payload.json 136.112.205.251 glsa_422RcuDdDskb7TLtapwWaOqcyrTCF2do_8c6a26b0 bearer + local max=30 + local delay=2 + shift 2 + local n=1 + curl_qan_values payload.json 136.112.205.251 glsa_422RcuDdDskb7TLtapwWaOqcyrTCF2do_8c6a26b0 bearer + local payload_file=payload.json + local endpoint=136.112.205.251 + local credentials=glsa_422RcuDdDskb7TLtapwWaOqcyrTCF2do_8c6a26b0 + local auth_type=bearer + local url + curl_args=() + local curl_args + [[ bearer == bearer ]] + url=https://136.112.205.251/v1/qan/metrics:getReport + curl_args=(-H "Authorization: Bearer ${credentials}" "$url") + jq -e 'any(.rows[]?; .sparkline != null)' ++ cat payload.json + execute_curl -s -k -XPOST -d $'{\n "period_start_from": "2026-07-16T13:56:52+00:00",\n "period_start_to": "2026-07-17T01:56:52+00:00",\n "group_by": "queryid",\n "labels": [\n {\n "key": "service_type",\n "value": [\n "mongodb"\n ]\n },\n {\n "key": "environment",\n "value": [\n "dev-mongod-mongolog"\n ]\n }\n ],\n "columns": [\n "load",\n "num_queries",\n "query_time"\n ],\n "order_by": "-load",\n "offset": 0,\n "limit": 10,\n "main_metric": "load",\n "search": ""\n}' -H 'Authorization: Bearer glsa_422RcuDdDskb7TLtapwWaOqcyrTCF2do_8c6a26b0' https://136.112.205.251/v1/qan/metrics:getReport + local name=curl-3624 + local image=docker.io/alpine/curl:latest + local phase= + local exit_status=0 + kubectl_bin run curl-3624 -n monitoring-pmm3-14502 --restart=Never --image=docker.io/alpine/curl:latest --command -- curl -sS -s -k -XPOST -d $'{\n "period_start_from": "2026-07-16T13:56:52+00:00",\n "period_start_to": "2026-07-17T01:56:52+00:00",\n "group_by": "queryid",\n "labels": [\n {\n "key": "service_type",\n "value": [\n "mongodb"\n ]\n },\n {\n "key": "environment",\n "value": [\n "dev-mongod-mongolog"\n ]\n }\n ],\n "columns": [\n "load",\n "num_queries",\n "query_time"\n ],\n "order_by": "-load",\n "offset": 0,\n "limit": 10,\n "main_metric": "load",\n "search": ""\n}' -H 'Authorization: Bearer glsa_422RcuDdDskb7TLtapwWaOqcyrTCF2do_8c6a26b0' https://136.112.205.251/v1/qan/metrics:getReport ++ mktemp + local LAST_OUT=/tmp/tmp.ANs31TMaW9 ++ mktemp + local LAST_ERR=/tmp/tmp.9uQaWyJ0Ss + local exit_status=0 + local timeout=4 + args=('run' 'curl-3624' '-n' 'monitoring-pmm3-14502' '--restart=Never' '--image=docker.io/alpine/curl:latest' '--command' '--' 'curl' '-sS' '-s' '-k' '-XPOST' '-d' '{ "period_start_from": "2026-07-16T13:56:52+00:00", "period_start_to": "2026-07-17T01:56:52+00:00", "group_by": "queryid", "labels": [ { "key": "service_type", "value": [ "mongodb" ] }, { "key": "environment", "value": [ "dev-mongod-mongolog" ] } ], "columns": [ "load", "num_queries", "query_time" ], "order_by": "-load", "offset": 0, "limit": 10, "main_metric": "load", "search": "" }' '-H' 'Authorization: Bearer glsa_422RcuDdDskb7TLtapwWaOqcyrTCF2do_8c6a26b0' 'https://136.112.205.251/v1/qan/metrics:getReport') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl run curl-3624 -n monitoring-pmm3-14502 --restart=Never --image=docker.io/alpine/curl:latest --command -- curl -sS -s -k -XPOST -d $'{\n "period_start_from": "2026-07-16T13:56:52+00:00",\n "period_start_to": "2026-07-17T01:56:52+00:00",\n "group_by": "queryid",\n "labels": [\n {\n "key": "service_type",\n "value": [\n "mongodb"\n ]\n },\n {\n "key": "environment",\n "value": [\n "dev-mongod-mongolog"\n ]\n }\n ],\n "columns": [\n "load",\n "num_queries",\n "query_time"\n ],\n "order_by": "-load",\n "offset": 0,\n "limit": 10,\n "main_metric": "load",\n "search": ""\n}' -H 'Authorization: Bearer glsa_422RcuDdDskb7TLtapwWaOqcyrTCF2do_8c6a26b0' https://136.112.205.251/v1/qan/metrics:getReport + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.ANs31TMaW9 + cat /tmp/tmp.9uQaWyJ0Ss + rm /tmp/tmp.ANs31TMaW9 /tmp/tmp.9uQaWyJ0Ss + return 0 ++ seq 1 60 + for _ in $(seq 1 60) ++ kubectl_bin get pod curl-3624 -n monitoring-pmm3-14502 -o 'jsonpath={.status.phase}' + phase=Pending + [[ Pending == Succeeded ]] + [[ Pending == Failed ]] + sleep 1 + for _ in $(seq 1 60) ++ kubectl_bin get pod curl-3624 -n monitoring-pmm3-14502 -o 'jsonpath={.status.phase}' + phase=Succeeded + [[ Succeeded == Succeeded ]] + break + kubectl_bin logs curl-3624 -n monitoring-pmm3-14502 ++ mktemp + local LAST_OUT=/tmp/tmp.3JkoSiUBVM ++ mktemp + local LAST_ERR=/tmp/tmp.7p67RxzByc + local exit_status=0 + local timeout=4 + args=('logs' 'curl-3624' '-n' 'monitoring-pmm3-14502') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl logs curl-3624 -n monitoring-pmm3-14502 + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.3JkoSiUBVM + cat /tmp/tmp.7p67RxzByc + rm /tmp/tmp.3JkoSiUBVM /tmp/tmp.7p67RxzByc + return 0 + kubectl_bin delete pod curl-3624 -n monitoring-pmm3-14502 --ignore-not-found --wait=false + [[ Succeeded != Succeeded ]] + return 0 true + rm -f payload.json + kubectl_bin patch psmdb monitoring-pmm3 --type json '-p=[{"op":"add","path":"/spec/pause","value":true}]' ++ mktemp + local LAST_OUT=/tmp/tmp.wXeE9ak2Hb ++ mktemp + local LAST_ERR=/tmp/tmp.a1avkmve24 + local exit_status=0 + local timeout=4 + args=('patch' 'psmdb' 'monitoring-pmm3' '--type' 'json' '-p=[{"op":"add","path":"/spec/pause","value":true}]') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl patch psmdb monitoring-pmm3 --type json '-p=[{"op":"add","path":"/spec/pause","value":true}]' + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.wXeE9ak2Hb perconaservermongodb.psmdb.percona.com/monitoring-pmm3 patched + cat /tmp/tmp.a1avkmve24 + rm /tmp/tmp.wXeE9ak2Hb /tmp/tmp.a1avkmve24 + return 0 + wait_for_delete pod/monitoring-pmm3-mongos-0 + local res=pod/monitoring-pmm3-mongos-0 + local wait_time=60 + set +o xtrace waiting for pod/monitoring-pmm3-mongos-0 to be deleted.......................Error from server (NotFound): pods "monitoring-pmm3-mongos-0" not found Error from server (NotFound): pods "monitoring-pmm3-mongos-0" not found Error from server (NotFound): pods "monitoring-pmm3-mongos-0" not found Error from server (NotFound): pods "monitoring-pmm3-mongos-0" not found + wait_for_delete pod/monitoring-pmm3-rs0-0 + local res=pod/monitoring-pmm3-rs0-0 + local wait_time=60 + set +o xtrace waiting for pod/monitoring-pmm3-rs0-0 to be deleted...Error from server (NotFound): pods "monitoring-pmm3-rs0-0" not found Error from server (NotFound): pods "monitoring-pmm3-rs0-0" not found Error from server (NotFound): pods "monitoring-pmm3-rs0-0" not found Error from server (NotFound): pods "monitoring-pmm3-rs0-0" not found + wait_for_delete pod/monitoring-pmm3-cfg-0 + local res=pod/monitoring-pmm3-cfg-0 + local wait_time=60 + set +o xtrace waiting for pod/monitoring-pmm3-cfg-0 to be deleted.......Error from server (NotFound): pods "monitoring-pmm3-cfg-0" not found Error from server (NotFound): pods "monitoring-pmm3-cfg-0" not found Error from server (NotFound): pods "monitoring-pmm3-cfg-0" not found Error from server (NotFound): pods "monitoring-pmm3-cfg-0" not found + desc 'check if services are not deleted' + set +o xtrace ----------------------------------------------------------------------------------- check if services are not deleted ----------------------------------------------------------------------------------- + kubectl_bin get svc monitoring-pmm3-rs0 ++ mktemp + local LAST_OUT=/tmp/tmp.x4fGpIQoJp ++ mktemp + local LAST_ERR=/tmp/tmp.r6xnM1HAb6 + local exit_status=0 + local timeout=4 + args=('get' 'svc' 'monitoring-pmm3-rs0') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl get svc monitoring-pmm3-rs0 + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.x4fGpIQoJp NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE monitoring-pmm3-rs0 ClusterIP None 27019/TCP 43m + cat /tmp/tmp.r6xnM1HAb6 + rm /tmp/tmp.x4fGpIQoJp /tmp/tmp.r6xnM1HAb6 + return 0 + kubectl_bin get svc monitoring-pmm3-cfg ++ mktemp + local LAST_OUT=/tmp/tmp.lUbx19iTfg ++ mktemp + local LAST_ERR=/tmp/tmp.cDVkzfGsHC + local exit_status=0 + local timeout=4 + args=('get' 'svc' 'monitoring-pmm3-cfg') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl get svc monitoring-pmm3-cfg + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.lUbx19iTfg NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE monitoring-pmm3-cfg ClusterIP None 27019/TCP 43m + cat /tmp/tmp.cDVkzfGsHC + rm /tmp/tmp.lUbx19iTfg /tmp/tmp.cDVkzfGsHC + return 0 + kubectl_bin get svc monitoring-pmm3-mongos ++ mktemp + local LAST_OUT=/tmp/tmp.3ia0H330v0 ++ mktemp + local LAST_ERR=/tmp/tmp.zwZgpSvJmr + local exit_status=0 + local timeout=4 + args=('get' 'svc' 'monitoring-pmm3-mongos') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl get svc monitoring-pmm3-mongos + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.3ia0H330v0 NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE monitoring-pmm3-mongos ClusterIP 34.118.230.23 27019/TCP 43m + cat /tmp/tmp.zwZgpSvJmr + rm /tmp/tmp.3ia0H330v0 /tmp/tmp.zwZgpSvJmr + return 0 + does_node_id_exists_in_pmm=($(does_node_id_exists "${nodeList[@]}")) ++ does_node_id_exists 9f635edf-62b7-42c4-829c-71f022e4324a 9cb595c2-e403-447d-a53a-e340c7d2d8cb 6e5e579d-0991-4d93-89f2-2390d520c3e4 568ad465-8ec4-4f7c-a1df-adaeb7b46d26 4d9d6d17-3ba2-4719-9a0f-da52c142fe57 e43bf5f4-2d9d-43c3-9e29-8c198e451f6b 62ab6a63-009a-4bd8-80e5-865935478e6c 38970820-7985-4a2b-ab41-91454f079afd dc90c940-2f86-44f6-8fa5-3c58a2fff7c1 ++ nodeList=('9f635edf-62b7-42c4-829c-71f022e4324a' '9cb595c2-e403-447d-a53a-e340c7d2d8cb' '6e5e579d-0991-4d93-89f2-2390d520c3e4' '568ad465-8ec4-4f7c-a1df-adaeb7b46d26' '4d9d6d17-3ba2-4719-9a0f-da52c142fe57' 'e43bf5f4-2d9d-43c3-9e29-8c198e451f6b' '62ab6a63-009a-4bd8-80e5-865935478e6c' '38970820-7985-4a2b-ab41-91454f079afd' 'dc90c940-2f86-44f6-8fa5-3c58a2fff7c1') ++ local -a nodeList ++ nodeList_from_pmm=() ++ local -a nodeList_from_pmm ++ local admin_pass endpoint encoded_admin_userinfo +++ kubectl_bin get secret pmm-secret -o 'jsonpath={.data.PMM_ADMIN_PASSWORD}' +++ base64 --decode ++++ mktemp +++ local LAST_OUT=/tmp/tmp.EpCTPVJgc9 ++++ mktemp +++ local LAST_ERR=/tmp/tmp.z729ev7RMt +++ local exit_status=0 +++ local timeout=4 +++ args=('get' 'secret' 'pmm-secret' '-o' 'jsonpath={.data.PMM_ADMIN_PASSWORD}') +++ local args +++ transformed_files=() +++ local transformed_files +++ arch_scheduling_enabled +++ [[ -n '' ]] +++ arch_scheduling_enabled +++ [[ -n '' ]] ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl get secret pmm-secret -o 'jsonpath={.data.PMM_ADMIN_PASSWORD}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 -a -n 1 ']' +++ break +++ cat /tmp/tmp.EpCTPVJgc9 +++ cat /tmp/tmp.z729ev7RMt +++ rm /tmp/tmp.EpCTPVJgc9 /tmp/tmp.z729ev7RMt +++ return 0 ++ admin_pass='J:k%i1RN=xa^Z5a!' +++ get_service_endpoint monitoring-service +++ local service=monitoring-service +++ local endpoint +++ local service_type ++++ kubectl_bin get service/monitoring-service -o 'jsonpath={.spec.type}' +++++ mktemp ++++ local LAST_OUT=/tmp/tmp.zTVF3JRs4n +++++ mktemp ++++ local LAST_ERR=/tmp/tmp.HK6WbPOEu2 ++++ local exit_status=0 ++++ local timeout=4 ++++ args=('get' 'service/monitoring-service' '-o' 'jsonpath={.spec.type}') ++++ local args ++++ transformed_files=() ++++ local transformed_files ++++ arch_scheduling_enabled ++++ [[ -n '' ]] ++++ arch_scheduling_enabled ++++ [[ -n '' ]] +++++ seq 0 2 ++++ for i in $(seq 0 2) ++++ set +e ++++ kubectl get service/monitoring-service -o 'jsonpath={.spec.type}' ++++ exit_status=0 ++++ set -e ++++ '[' 0 '!=' 0 -a -n 1 ']' ++++ break ++++ cat /tmp/tmp.zTVF3JRs4n ++++ cat /tmp/tmp.HK6WbPOEu2 ++++ rm /tmp/tmp.zTVF3JRs4n /tmp/tmp.HK6WbPOEu2 ++++ return 0 +++ service_type=LoadBalancer +++ [[ LoadBalancer == LoadBalancer ]] ++++ get_load_balancer_endpoint monitoring-service ++++ local service=monitoring-service ++++ local endpoint +++++ kubectl_bin get service/monitoring-service -o json +++++ jq -r '[.status.loadBalancer.ingress[]?.hostname, .status.loadBalancer.ingress[]?.ip] | map(select(. != null and . != "")) | first // empty' ++++++ mktemp +++++ local LAST_OUT=/tmp/tmp.3sgZRjoDwu ++++++ mktemp +++++ local LAST_ERR=/tmp/tmp.wsjRRoKRna +++++ local exit_status=0 +++++ local timeout=4 +++++ args=('get' 'service/monitoring-service' '-o' 'json') +++++ local args +++++ transformed_files=() +++++ local transformed_files +++++ arch_scheduling_enabled +++++ [[ -n '' ]] +++++ arch_scheduling_enabled +++++ [[ -n '' ]] ++++++ seq 0 2 +++++ for i in $(seq 0 2) +++++ set +e +++++ kubectl get service/monitoring-service -o json +++++ exit_status=0 +++++ set -e +++++ '[' 0 '!=' 0 -a -n 1 ']' +++++ break +++++ cat /tmp/tmp.3sgZRjoDwu +++++ cat /tmp/tmp.wsjRRoKRna +++++ rm /tmp/tmp.3sgZRjoDwu /tmp/tmp.wsjRRoKRna +++++ return 0 ++++ endpoint=136.112.205.251 ++++ '[' -n 136.112.205.251 -a 136.112.205.251 '!=' null ']' ++++ echo 136.112.205.251 ++++ return +++ endpoint=136.112.205.251 +++ echo 136.112.205.251 +++ return ++ endpoint=136.112.205.251 +++ urlencode_userinfo 'admin:J:k%i1RN=xa^Z5a!' +++ local 'user_pass=admin:J:k%i1RN=xa^Z5a!' +++ local username=admin +++ local 'password=J:k%i1RN=xa^Z5a!' ++++ urlencode admin ++++ uri=admin ++++ echo -n admin ++++ jq -s -R -r @uri ++++ urlencode 'J:k%i1RN=xa^Z5a!' ++++ uri='J:k%i1RN=xa^Z5a!' ++++ echo -n 'J:k%i1RN=xa^Z5a!' ++++ jq -s -R -r @uri +++ printf %s:%s admin J%3Ak%25i1RN%3Dxa%5EZ5a%21 ++ encoded_admin_userinfo=admin:J%3Ak%25i1RN%3Dxa%5EZ5a%21 ++ for node_id in "${nodeList[@]}" ++ nodeList_from_pmm+=($(kubectl_bin exec -n "${namespace}" monitoring-server-0 -- pmm-admin --server-url=https://${encoded_admin_userinfo}@${endpoint}/ --server-insecure-tls inventory list nodes --node-type=container | grep $node_id | awk '{print $4}')) +++ kubectl_bin exec -n monitoring-pmm3-14502 monitoring-server-0 -- pmm-admin --server-url=https://admin:J%3Ak%25i1RN%3Dxa%5EZ5a%21@136.112.205.251/ --server-insecure-tls inventory list nodes --node-type=container +++ grep 9f635edf-62b7-42c4-829c-71f022e4324a +++ awk '{print $4}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.SuZLsATGii ++++ mktemp +++ local LAST_ERR=/tmp/tmp.O9t035SHNr +++ local exit_status=0 +++ local timeout=4 +++ args=('exec' '-n' 'monitoring-pmm3-14502' 'monitoring-server-0' '--' 'pmm-admin' '--server-url=https://admin:J%3Ak%25i1RN%3Dxa%5EZ5a%21@136.112.205.251/' '--server-insecure-tls' 'inventory' 'list' 'nodes' '--node-type=container') +++ local args +++ transformed_files=() +++ local transformed_files +++ arch_scheduling_enabled +++ [[ -n '' ]] +++ arch_scheduling_enabled +++ [[ -n '' ]] ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl exec -n monitoring-pmm3-14502 monitoring-server-0 -- pmm-admin --server-url=https://admin:J%3Ak%25i1RN%3Dxa%5EZ5a%21@136.112.205.251/ --server-insecure-tls inventory list nodes --node-type=container +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 -a -n 1 ']' +++ break +++ cat /tmp/tmp.SuZLsATGii +++ cat /tmp/tmp.O9t035SHNr +++ rm /tmp/tmp.SuZLsATGii /tmp/tmp.O9t035SHNr +++ return 0 ++ for node_id in "${nodeList[@]}" ++ nodeList_from_pmm+=($(kubectl_bin exec -n "${namespace}" monitoring-server-0 -- pmm-admin --server-url=https://${encoded_admin_userinfo}@${endpoint}/ --server-insecure-tls inventory list nodes --node-type=container | grep $node_id | awk '{print $4}')) +++ kubectl_bin exec -n monitoring-pmm3-14502 monitoring-server-0 -- pmm-admin --server-url=https://admin:J%3Ak%25i1RN%3Dxa%5EZ5a%21@136.112.205.251/ --server-insecure-tls inventory list nodes --node-type=container +++ grep 9cb595c2-e403-447d-a53a-e340c7d2d8cb +++ awk '{print $4}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.PS6KcBWFe0 ++++ mktemp +++ local LAST_ERR=/tmp/tmp.81qcqaELWL +++ local exit_status=0 +++ local timeout=4 +++ args=('exec' '-n' 'monitoring-pmm3-14502' 'monitoring-server-0' '--' 'pmm-admin' '--server-url=https://admin:J%3Ak%25i1RN%3Dxa%5EZ5a%21@136.112.205.251/' '--server-insecure-tls' 'inventory' 'list' 'nodes' '--node-type=container') +++ local args +++ transformed_files=() +++ local transformed_files +++ arch_scheduling_enabled +++ [[ -n '' ]] +++ arch_scheduling_enabled +++ [[ -n '' ]] ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl exec -n monitoring-pmm3-14502 monitoring-server-0 -- pmm-admin --server-url=https://admin:J%3Ak%25i1RN%3Dxa%5EZ5a%21@136.112.205.251/ --server-insecure-tls inventory list nodes --node-type=container +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 -a -n 1 ']' +++ break +++ cat /tmp/tmp.PS6KcBWFe0 +++ cat /tmp/tmp.81qcqaELWL +++ rm /tmp/tmp.PS6KcBWFe0 /tmp/tmp.81qcqaELWL +++ return 0 ++ for node_id in "${nodeList[@]}" ++ nodeList_from_pmm+=($(kubectl_bin exec -n "${namespace}" monitoring-server-0 -- pmm-admin --server-url=https://${encoded_admin_userinfo}@${endpoint}/ --server-insecure-tls inventory list nodes --node-type=container | grep $node_id | awk '{print $4}')) +++ kubectl_bin exec -n monitoring-pmm3-14502 monitoring-server-0 -- pmm-admin --server-url=https://admin:J%3Ak%25i1RN%3Dxa%5EZ5a%21@136.112.205.251/ --server-insecure-tls inventory list nodes --node-type=container +++ grep 6e5e579d-0991-4d93-89f2-2390d520c3e4 ++++ mktemp +++ awk '{print $4}' +++ local LAST_OUT=/tmp/tmp.IAbw2ZamjK ++++ mktemp +++ local LAST_ERR=/tmp/tmp.rdVj1PylMK +++ local exit_status=0 +++ local timeout=4 +++ args=('exec' '-n' 'monitoring-pmm3-14502' 'monitoring-server-0' '--' 'pmm-admin' '--server-url=https://admin:J%3Ak%25i1RN%3Dxa%5EZ5a%21@136.112.205.251/' '--server-insecure-tls' 'inventory' 'list' 'nodes' '--node-type=container') +++ local args +++ transformed_files=() +++ local transformed_files +++ arch_scheduling_enabled +++ [[ -n '' ]] +++ arch_scheduling_enabled +++ [[ -n '' ]] ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl exec -n monitoring-pmm3-14502 monitoring-server-0 -- pmm-admin --server-url=https://admin:J%3Ak%25i1RN%3Dxa%5EZ5a%21@136.112.205.251/ --server-insecure-tls inventory list nodes --node-type=container +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 -a -n 1 ']' +++ break +++ cat /tmp/tmp.IAbw2ZamjK +++ cat /tmp/tmp.rdVj1PylMK +++ rm /tmp/tmp.IAbw2ZamjK /tmp/tmp.rdVj1PylMK +++ return 0 ++ for node_id in "${nodeList[@]}" ++ nodeList_from_pmm+=($(kubectl_bin exec -n "${namespace}" monitoring-server-0 -- pmm-admin --server-url=https://${encoded_admin_userinfo}@${endpoint}/ --server-insecure-tls inventory list nodes --node-type=container | grep $node_id | awk '{print $4}')) +++ kubectl_bin exec -n monitoring-pmm3-14502 monitoring-server-0 -- pmm-admin --server-url=https://admin:J%3Ak%25i1RN%3Dxa%5EZ5a%21@136.112.205.251/ --server-insecure-tls inventory list nodes --node-type=container +++ grep 568ad465-8ec4-4f7c-a1df-adaeb7b46d26 +++ awk '{print $4}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.2kcmc6BWUz ++++ mktemp +++ local LAST_ERR=/tmp/tmp.MUq3IgUXLl +++ local exit_status=0 +++ local timeout=4 +++ args=('exec' '-n' 'monitoring-pmm3-14502' 'monitoring-server-0' '--' 'pmm-admin' '--server-url=https://admin:J%3Ak%25i1RN%3Dxa%5EZ5a%21@136.112.205.251/' '--server-insecure-tls' 'inventory' 'list' 'nodes' '--node-type=container') +++ local args +++ transformed_files=() +++ local transformed_files +++ arch_scheduling_enabled +++ [[ -n '' ]] +++ arch_scheduling_enabled +++ [[ -n '' ]] ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl exec -n monitoring-pmm3-14502 monitoring-server-0 -- pmm-admin --server-url=https://admin:J%3Ak%25i1RN%3Dxa%5EZ5a%21@136.112.205.251/ --server-insecure-tls inventory list nodes --node-type=container +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 -a -n 1 ']' +++ break +++ cat /tmp/tmp.2kcmc6BWUz +++ cat /tmp/tmp.MUq3IgUXLl +++ rm /tmp/tmp.2kcmc6BWUz /tmp/tmp.MUq3IgUXLl +++ return 0 ++ for node_id in "${nodeList[@]}" ++ nodeList_from_pmm+=($(kubectl_bin exec -n "${namespace}" monitoring-server-0 -- pmm-admin --server-url=https://${encoded_admin_userinfo}@${endpoint}/ --server-insecure-tls inventory list nodes --node-type=container | grep $node_id | awk '{print $4}')) +++ kubectl_bin exec -n monitoring-pmm3-14502 monitoring-server-0 -- pmm-admin --server-url=https://admin:J%3Ak%25i1RN%3Dxa%5EZ5a%21@136.112.205.251/ --server-insecure-tls inventory list nodes --node-type=container +++ grep 4d9d6d17-3ba2-4719-9a0f-da52c142fe57 +++ awk '{print $4}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.q7jHd4cOXF ++++ mktemp +++ local LAST_ERR=/tmp/tmp.AT6UmSG3Gn +++ local exit_status=0 +++ local timeout=4 +++ args=('exec' '-n' 'monitoring-pmm3-14502' 'monitoring-server-0' '--' 'pmm-admin' '--server-url=https://admin:J%3Ak%25i1RN%3Dxa%5EZ5a%21@136.112.205.251/' '--server-insecure-tls' 'inventory' 'list' 'nodes' '--node-type=container') +++ local args +++ transformed_files=() +++ local transformed_files +++ arch_scheduling_enabled +++ [[ -n '' ]] +++ arch_scheduling_enabled +++ [[ -n '' ]] ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl exec -n monitoring-pmm3-14502 monitoring-server-0 -- pmm-admin --server-url=https://admin:J%3Ak%25i1RN%3Dxa%5EZ5a%21@136.112.205.251/ --server-insecure-tls inventory list nodes --node-type=container +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 -a -n 1 ']' +++ break +++ cat /tmp/tmp.q7jHd4cOXF +++ cat /tmp/tmp.AT6UmSG3Gn +++ rm /tmp/tmp.q7jHd4cOXF /tmp/tmp.AT6UmSG3Gn +++ return 0 ++ for node_id in "${nodeList[@]}" ++ nodeList_from_pmm+=($(kubectl_bin exec -n "${namespace}" monitoring-server-0 -- pmm-admin --server-url=https://${encoded_admin_userinfo}@${endpoint}/ --server-insecure-tls inventory list nodes --node-type=container | grep $node_id | awk '{print $4}')) +++ kubectl_bin exec -n monitoring-pmm3-14502 monitoring-server-0 -- pmm-admin --server-url=https://admin:J%3Ak%25i1RN%3Dxa%5EZ5a%21@136.112.205.251/ --server-insecure-tls inventory list nodes --node-type=container +++ grep e43bf5f4-2d9d-43c3-9e29-8c198e451f6b +++ awk '{print $4}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.XKHoz9vQkk ++++ mktemp +++ local LAST_ERR=/tmp/tmp.EtRY8ZYaea +++ local exit_status=0 +++ local timeout=4 +++ args=('exec' '-n' 'monitoring-pmm3-14502' 'monitoring-server-0' '--' 'pmm-admin' '--server-url=https://admin:J%3Ak%25i1RN%3Dxa%5EZ5a%21@136.112.205.251/' '--server-insecure-tls' 'inventory' 'list' 'nodes' '--node-type=container') +++ local args +++ transformed_files=() +++ local transformed_files +++ arch_scheduling_enabled +++ [[ -n '' ]] +++ arch_scheduling_enabled +++ [[ -n '' ]] ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl exec -n monitoring-pmm3-14502 monitoring-server-0 -- pmm-admin --server-url=https://admin:J%3Ak%25i1RN%3Dxa%5EZ5a%21@136.112.205.251/ --server-insecure-tls inventory list nodes --node-type=container +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 -a -n 1 ']' +++ break +++ cat /tmp/tmp.XKHoz9vQkk +++ cat /tmp/tmp.EtRY8ZYaea +++ rm /tmp/tmp.XKHoz9vQkk /tmp/tmp.EtRY8ZYaea +++ return 0 ++ for node_id in "${nodeList[@]}" ++ nodeList_from_pmm+=($(kubectl_bin exec -n "${namespace}" monitoring-server-0 -- pmm-admin --server-url=https://${encoded_admin_userinfo}@${endpoint}/ --server-insecure-tls inventory list nodes --node-type=container | grep $node_id | awk '{print $4}')) +++ kubectl_bin exec -n monitoring-pmm3-14502 monitoring-server-0 -- pmm-admin --server-url=https://admin:J%3Ak%25i1RN%3Dxa%5EZ5a%21@136.112.205.251/ --server-insecure-tls inventory list nodes --node-type=container +++ grep 62ab6a63-009a-4bd8-80e5-865935478e6c +++ awk '{print $4}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.Vbke1FVqbf ++++ mktemp +++ local LAST_ERR=/tmp/tmp.MY8DNRFBHh +++ local exit_status=0 +++ local timeout=4 +++ args=('exec' '-n' 'monitoring-pmm3-14502' 'monitoring-server-0' '--' 'pmm-admin' '--server-url=https://admin:J%3Ak%25i1RN%3Dxa%5EZ5a%21@136.112.205.251/' '--server-insecure-tls' 'inventory' 'list' 'nodes' '--node-type=container') +++ local args +++ transformed_files=() +++ local transformed_files +++ arch_scheduling_enabled +++ [[ -n '' ]] +++ arch_scheduling_enabled +++ [[ -n '' ]] ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl exec -n monitoring-pmm3-14502 monitoring-server-0 -- pmm-admin --server-url=https://admin:J%3Ak%25i1RN%3Dxa%5EZ5a%21@136.112.205.251/ --server-insecure-tls inventory list nodes --node-type=container +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 -a -n 1 ']' +++ break +++ cat /tmp/tmp.Vbke1FVqbf +++ cat /tmp/tmp.MY8DNRFBHh +++ rm /tmp/tmp.Vbke1FVqbf /tmp/tmp.MY8DNRFBHh +++ return 0 ++ for node_id in "${nodeList[@]}" ++ nodeList_from_pmm+=($(kubectl_bin exec -n "${namespace}" monitoring-server-0 -- pmm-admin --server-url=https://${encoded_admin_userinfo}@${endpoint}/ --server-insecure-tls inventory list nodes --node-type=container | grep $node_id | awk '{print $4}')) +++ kubectl_bin exec -n monitoring-pmm3-14502 monitoring-server-0 -- pmm-admin --server-url=https://admin:J%3Ak%25i1RN%3Dxa%5EZ5a%21@136.112.205.251/ --server-insecure-tls inventory list nodes --node-type=container +++ grep 38970820-7985-4a2b-ab41-91454f079afd +++ awk '{print $4}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.qY3gkfCpV7 ++++ mktemp +++ local LAST_ERR=/tmp/tmp.k3jGBWxKzg +++ local exit_status=0 +++ local timeout=4 +++ args=('exec' '-n' 'monitoring-pmm3-14502' 'monitoring-server-0' '--' 'pmm-admin' '--server-url=https://admin:J%3Ak%25i1RN%3Dxa%5EZ5a%21@136.112.205.251/' '--server-insecure-tls' 'inventory' 'list' 'nodes' '--node-type=container') +++ local args +++ transformed_files=() +++ local transformed_files +++ arch_scheduling_enabled +++ [[ -n '' ]] +++ arch_scheduling_enabled +++ [[ -n '' ]] ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl exec -n monitoring-pmm3-14502 monitoring-server-0 -- pmm-admin --server-url=https://admin:J%3Ak%25i1RN%3Dxa%5EZ5a%21@136.112.205.251/ --server-insecure-tls inventory list nodes --node-type=container +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 -a -n 1 ']' +++ break +++ cat /tmp/tmp.qY3gkfCpV7 +++ cat /tmp/tmp.k3jGBWxKzg +++ rm /tmp/tmp.qY3gkfCpV7 /tmp/tmp.k3jGBWxKzg +++ return 0 ++ for node_id in "${nodeList[@]}" ++ nodeList_from_pmm+=($(kubectl_bin exec -n "${namespace}" monitoring-server-0 -- pmm-admin --server-url=https://${encoded_admin_userinfo}@${endpoint}/ --server-insecure-tls inventory list nodes --node-type=container | grep $node_id | awk '{print $4}')) +++ kubectl_bin exec -n monitoring-pmm3-14502 monitoring-server-0 -- pmm-admin --server-url=https://admin:J%3Ak%25i1RN%3Dxa%5EZ5a%21@136.112.205.251/ --server-insecure-tls inventory list nodes --node-type=container +++ grep dc90c940-2f86-44f6-8fa5-3c58a2fff7c1 +++ awk '{print $4}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.WR55zTW6Ll ++++ mktemp +++ local LAST_ERR=/tmp/tmp.FToChgz433 +++ local exit_status=0 +++ local timeout=4 +++ args=('exec' '-n' 'monitoring-pmm3-14502' 'monitoring-server-0' '--' 'pmm-admin' '--server-url=https://admin:J%3Ak%25i1RN%3Dxa%5EZ5a%21@136.112.205.251/' '--server-insecure-tls' 'inventory' 'list' 'nodes' '--node-type=container') +++ local args +++ transformed_files=() +++ local transformed_files +++ arch_scheduling_enabled +++ [[ -n '' ]] +++ arch_scheduling_enabled +++ [[ -n '' ]] ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl exec -n monitoring-pmm3-14502 monitoring-server-0 -- pmm-admin --server-url=https://admin:J%3Ak%25i1RN%3Dxa%5EZ5a%21@136.112.205.251/ --server-insecure-tls inventory list nodes --node-type=container +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 -a -n 1 ']' +++ break +++ cat /tmp/tmp.WR55zTW6Ll +++ cat /tmp/tmp.FToChgz433 +++ rm /tmp/tmp.WR55zTW6Ll /tmp/tmp.FToChgz433 +++ return 0 ++ echo + [[ -n '' ]] ++ kubectl_bin logs monitoring-pmm3-rs0-0 pmm-client +++ mktemp ++ grep -c 'cannot auto discover databases and collections' ++ local LAST_OUT=/tmp/tmp.aYkRGiX8tw +++ mktemp ++ local LAST_ERR=/tmp/tmp.Ze1PYf2D9A ++ local exit_status=0 ++ local timeout=4 ++ args=('logs' 'monitoring-pmm3-rs0-0' 'pmm-client') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl logs monitoring-pmm3-rs0-0 pmm-client ++ exit_status=1 ++ set -e ++ '[' 1 '!=' 0 -a -n 1 ']' ++ cat /tmp/tmp.aYkRGiX8tw ++ cat /tmp/tmp.Ze1PYf2D9A error: error from server (NotFound): pods "monitoring-pmm3-rs0-0" not found in namespace "monitoring-pmm3-14502" ++ sleep 0 ++ for i in $(seq 0 2) ++ set +e ++ kubectl logs monitoring-pmm3-rs0-0 pmm-client ++ exit_status=1 ++ set -e ++ '[' 1 '!=' 0 -a -n 1 ']' ++ cat /tmp/tmp.aYkRGiX8tw ++ cat /tmp/tmp.Ze1PYf2D9A error: error from server (NotFound): pods "monitoring-pmm3-rs0-0" not found in namespace "monitoring-pmm3-14502" ++ sleep 4 ++ for i in $(seq 0 2) ++ set +e ++ kubectl logs monitoring-pmm3-rs0-0 pmm-client ++ exit_status=1 ++ set -e ++ '[' 1 '!=' 0 -a -n 1 ']' ++ cat /tmp/tmp.aYkRGiX8tw ++ cat /tmp/tmp.Ze1PYf2D9A error: error from server (NotFound): pods "monitoring-pmm3-rs0-0" not found in namespace "monitoring-pmm3-14502" ++ sleep 8 ++ cat /tmp/tmp.aYkRGiX8tw ++ cat /tmp/tmp.Ze1PYf2D9A error: error from server (NotFound): pods "monitoring-pmm3-rs0-0" not found in namespace "monitoring-pmm3-14502" ++ rm /tmp/tmp.aYkRGiX8tw /tmp/tmp.Ze1PYf2D9A ++ return 1 + [[ 0 != 0 ]] + helm uninstall monitoring release "monitoring" uninstalled + destroy monitoring-pmm3-14502 + local namespace=monitoring-pmm3-14502 + local ignore_logs=true + [[ 0 == 1 ]] + desc 'destroy cluster/operator and all other resources' + set +o xtrace ----------------------------------------------------------------------------------- destroy cluster/operator and all other resources ----------------------------------------------------------------------------------- + '[' true == false ']' + delete_backups + desc 'Delete psmdb-backup' + set +o xtrace ----------------------------------------------------------------------------------- Delete psmdb-backup ----------------------------------------------------------------------------------- ++ kubectl_bin get psmdb-backup --no-headers ++ wc -l +++ mktemp ++ local LAST_OUT=/tmp/tmp.uVp9qpr9bX +++ mktemp ++ local LAST_ERR=/tmp/tmp.yGfL5vyd7r ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'psmdb-backup' '--no-headers') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb-backup --no-headers ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.uVp9qpr9bX ++ cat /tmp/tmp.yGfL5vyd7r No resources found in monitoring-pmm3-14502 namespace. ++ rm /tmp/tmp.uVp9qpr9bX /tmp/tmp.yGfL5vyd7r ++ return 0 + '[' 0 '!=' 0 ']' + delete_crd + desc 'get and delete old CRDs and RBAC' + set +o xtrace ----------------------------------------------------------------------------------- get and delete old CRDs and RBAC ----------------------------------------------------------------------------------- + kubectl_bin delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/deploy/crd.yaml --ignore-not-found --wait=false ++ mktemp + local LAST_OUT=/tmp/tmp.uaCfopJowq ++ mktemp + local LAST_ERR=/tmp/tmp.4bEzJVF21E + local exit_status=0 + local timeout=4 + args=('delete' '-f' '/mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/deploy/crd.yaml' '--ignore-not-found' '--wait=false') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/deploy/crd.yaml --ignore-not-found --wait=false + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.uaCfopJowq customresourcedefinition.apiextensions.k8s.io "perconaservermongodbbackups.psmdb.percona.com" deleted customresourcedefinition.apiextensions.k8s.io "perconaservermongodbclustersyncs.psmdb.percona.com" deleted customresourcedefinition.apiextensions.k8s.io "perconaservermongodbrestores.psmdb.percona.com" deleted customresourcedefinition.apiextensions.k8s.io "perconaservermongodbs.psmdb.percona.com" deleted + cat /tmp/tmp.4bEzJVF21E + rm /tmp/tmp.uaCfopJowq /tmp/tmp.4bEzJVF21E + return 0 ++ yq eval .metadata.name /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/deploy/crd.yaml ++ grep -v '\-\-\-' grep: warning: stray \ before - grep: warning: stray \ before - + for crd_name in $(yq eval '.metadata.name' "${src_dir}/deploy/crd.yaml" | grep -v '\-\-\-') + kubectl get perconaservermongodbbackups.psmdb.percona.com --all-namespaces -o wide + grep -v NAMESPACE + xargs -L 1 sh -xc 'kubectl patch perconaservermongodbbackups.psmdb.percona.com -n $0 $1 --type=merge -p "{\"metadata\":{\"finalizers\":[]}}"' error: the server doesn't have a resource type "perconaservermongodbbackups" + kubectl patch perconaservermongodbbackups.psmdb.percona.com -n sh --type=merge -p '{"metadata":{"finalizers":[]}}' error: the server doesn't have a resource type "perconaservermongodbbackups" + : + kubectl_bin wait --for=delete crd perconaservermongodbbackups.psmdb.percona.com ++ mktemp + local LAST_OUT=/tmp/tmp.V5Ezq2DrAt ++ mktemp + local LAST_ERR=/tmp/tmp.tZl9AkSF0s + local exit_status=0 + local timeout=4 + args=('wait' '--for=delete' 'crd' 'perconaservermongodbbackups.psmdb.percona.com') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl wait --for=delete crd perconaservermongodbbackups.psmdb.percona.com + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.V5Ezq2DrAt + cat /tmp/tmp.tZl9AkSF0s + rm /tmp/tmp.V5Ezq2DrAt /tmp/tmp.tZl9AkSF0s + return 0 + for crd_name in $(yq eval '.metadata.name' "${src_dir}/deploy/crd.yaml" | grep -v '\-\-\-') + kubectl get perconaservermongodbclustersyncs.psmdb.percona.com --all-namespaces -o wide + grep -v NAMESPACE + xargs -L 1 sh -xc 'kubectl patch perconaservermongodbclustersyncs.psmdb.percona.com -n $0 $1 --type=merge -p "{\"metadata\":{\"finalizers\":[]}}"' error: the server doesn't have a resource type "perconaservermongodbclustersyncs" + kubectl patch perconaservermongodbclustersyncs.psmdb.percona.com -n sh --type=merge -p '{"metadata":{"finalizers":[]}}' error: the server doesn't have a resource type "perconaservermongodbclustersyncs" + : + kubectl_bin wait --for=delete crd perconaservermongodbclustersyncs.psmdb.percona.com ++ mktemp + local LAST_OUT=/tmp/tmp.QYJMqQ9vNi ++ mktemp + local LAST_ERR=/tmp/tmp.zP8DjZ7gxw + local exit_status=0 + local timeout=4 + args=('wait' '--for=delete' 'crd' 'perconaservermongodbclustersyncs.psmdb.percona.com') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl wait --for=delete crd perconaservermongodbclustersyncs.psmdb.percona.com + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.QYJMqQ9vNi + cat /tmp/tmp.zP8DjZ7gxw + rm /tmp/tmp.QYJMqQ9vNi /tmp/tmp.zP8DjZ7gxw + return 0 + for crd_name in $(yq eval '.metadata.name' "${src_dir}/deploy/crd.yaml" | grep -v '\-\-\-') + grep -v NAMESPACE + xargs -L 1 sh -xc 'kubectl patch perconaservermongodbrestores.psmdb.percona.com -n $0 $1 --type=merge -p "{\"metadata\":{\"finalizers\":[]}}"' + kubectl get perconaservermongodbrestores.psmdb.percona.com --all-namespaces -o wide error: the server doesn't have a resource type "perconaservermongodbrestores" + kubectl patch perconaservermongodbrestores.psmdb.percona.com -n sh --type=merge -p '{"metadata":{"finalizers":[]}}' error: the server doesn't have a resource type "perconaservermongodbrestores" + : + kubectl_bin wait --for=delete crd perconaservermongodbrestores.psmdb.percona.com ++ mktemp + local LAST_OUT=/tmp/tmp.xmW7djNcoy ++ mktemp + local LAST_ERR=/tmp/tmp.F1fT9PuSh4 + local exit_status=0 + local timeout=4 + args=('wait' '--for=delete' 'crd' 'perconaservermongodbrestores.psmdb.percona.com') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl wait --for=delete crd perconaservermongodbrestores.psmdb.percona.com + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.xmW7djNcoy + cat /tmp/tmp.F1fT9PuSh4 + rm /tmp/tmp.xmW7djNcoy /tmp/tmp.F1fT9PuSh4 + return 0 + for crd_name in $(yq eval '.metadata.name' "${src_dir}/deploy/crd.yaml" | grep -v '\-\-\-') + kubectl get perconaservermongodbs.psmdb.percona.com --all-namespaces -o wide + grep -v NAMESPACE + xargs -L 1 sh -xc 'kubectl patch perconaservermongodbs.psmdb.percona.com -n $0 $1 --type=merge -p "{\"metadata\":{\"finalizers\":[]}}"' error: the server doesn't have a resource type "perconaservermongodbs" + kubectl patch perconaservermongodbs.psmdb.percona.com -n sh --type=merge -p '{"metadata":{"finalizers":[]}}' error: the server doesn't have a resource type "perconaservermongodbs" + : + kubectl_bin wait --for=delete crd perconaservermongodbs.psmdb.percona.com ++ mktemp + local LAST_OUT=/tmp/tmp.AuqwgM0DCI ++ mktemp + local LAST_ERR=/tmp/tmp.T3AaKfraVr + local exit_status=0 + local timeout=4 + args=('wait' '--for=delete' 'crd' 'perconaservermongodbs.psmdb.percona.com') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl wait --for=delete crd perconaservermongodbs.psmdb.percona.com + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.AuqwgM0DCI + cat /tmp/tmp.T3AaKfraVr + rm /tmp/tmp.AuqwgM0DCI /tmp/tmp.T3AaKfraVr + return 0 + local rbac_yaml=rbac.yaml + '[' -n psmdb-operator ']' + rbac_yaml=cw-rbac.yaml + kubectl_bin delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/deploy/cw-rbac.yaml --ignore-not-found ++ mktemp + local LAST_OUT=/tmp/tmp.q67TmZWcqu ++ mktemp + local LAST_ERR=/tmp/tmp.xxY4M5HdlL + local exit_status=0 + local timeout=4 + args=('delete' '-f' '/mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/deploy/cw-rbac.yaml' '--ignore-not-found') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/deploy/cw-rbac.yaml --ignore-not-found + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.q67TmZWcqu clusterrole.rbac.authorization.k8s.io "percona-server-mongodb-operator" deleted clusterrolebinding.rbac.authorization.k8s.io "service-account-percona-server-mongodb-operator" deleted + cat /tmp/tmp.xxY4M5HdlL + rm /tmp/tmp.q67TmZWcqu /tmp/tmp.xxY4M5HdlL + return 0 + destroy_cert_manager + [[ gke == rancher ]] + kubectl_bin delete -f https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml ++ mktemp + local LAST_OUT=/tmp/tmp.ZQAlLGtnDy ++ mktemp + local LAST_ERR=/tmp/tmp.9FBHI8g3zI + local exit_status=0 + local timeout=4 + args=('delete' '-f' 'https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl delete -f https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.ZQAlLGtnDy namespace "cert-manager" deleted customresourcedefinition.apiextensions.k8s.io "challenges.acme.cert-manager.io" deleted customresourcedefinition.apiextensions.k8s.io "orders.acme.cert-manager.io" deleted customresourcedefinition.apiextensions.k8s.io "certificaterequests.cert-manager.io" deleted customresourcedefinition.apiextensions.k8s.io "certificates.cert-manager.io" deleted customresourcedefinition.apiextensions.k8s.io "clusterissuers.cert-manager.io" deleted customresourcedefinition.apiextensions.k8s.io "issuers.cert-manager.io" deleted serviceaccount "cert-manager-cainjector" deleted from cert-manager namespace serviceaccount "cert-manager" deleted from cert-manager namespace serviceaccount "cert-manager-webhook" deleted from cert-manager namespace clusterrole.rbac.authorization.k8s.io "cert-manager-cainjector" deleted clusterrole.rbac.authorization.k8s.io "cert-manager-controller-issuers" deleted clusterrole.rbac.authorization.k8s.io "cert-manager-controller-clusterissuers" deleted clusterrole.rbac.authorization.k8s.io "cert-manager-controller-certificates" deleted clusterrole.rbac.authorization.k8s.io "cert-manager-controller-orders" deleted clusterrole.rbac.authorization.k8s.io "cert-manager-controller-challenges" deleted clusterrole.rbac.authorization.k8s.io "cert-manager-controller-ingress-shim" deleted clusterrole.rbac.authorization.k8s.io "cert-manager-cluster-view" deleted clusterrole.rbac.authorization.k8s.io "cert-manager-view" deleted clusterrole.rbac.authorization.k8s.io "cert-manager-edit" deleted clusterrole.rbac.authorization.k8s.io "cert-manager-controller-approve:cert-manager-io" deleted clusterrole.rbac.authorization.k8s.io "cert-manager-controller-certificatesigningrequests" deleted clusterrole.rbac.authorization.k8s.io "cert-manager-webhook:subjectaccessreviews" deleted clusterrolebinding.rbac.authorization.k8s.io "cert-manager-cainjector" deleted clusterrolebinding.rbac.authorization.k8s.io "cert-manager-controller-issuers" deleted clusterrolebinding.rbac.authorization.k8s.io "cert-manager-controller-clusterissuers" deleted clusterrolebinding.rbac.authorization.k8s.io "cert-manager-controller-certificates" deleted clusterrolebinding.rbac.authorization.k8s.io "cert-manager-controller-orders" deleted clusterrolebinding.rbac.authorization.k8s.io "cert-manager-controller-challenges" deleted clusterrolebinding.rbac.authorization.k8s.io "cert-manager-controller-ingress-shim" deleted clusterrolebinding.rbac.authorization.k8s.io "cert-manager-controller-approve:cert-manager-io" deleted clusterrolebinding.rbac.authorization.k8s.io "cert-manager-controller-certificatesigningrequests" deleted clusterrolebinding.rbac.authorization.k8s.io "cert-manager-webhook:subjectaccessreviews" deleted role.rbac.authorization.k8s.io "cert-manager-cainjector:leaderelection" deleted from kube-system namespace role.rbac.authorization.k8s.io "cert-manager:leaderelection" deleted from kube-system namespace role.rbac.authorization.k8s.io "cert-manager-tokenrequest" deleted from cert-manager namespace role.rbac.authorization.k8s.io "cert-manager-webhook:dynamic-serving" deleted from cert-manager namespace rolebinding.rbac.authorization.k8s.io "cert-manager-cainjector:leaderelection" deleted from kube-system namespace rolebinding.rbac.authorization.k8s.io "cert-manager:leaderelection" deleted from kube-system namespace rolebinding.rbac.authorization.k8s.io "cert-manager-tokenrequest" deleted from cert-manager namespace rolebinding.rbac.authorization.k8s.io "cert-manager-webhook:dynamic-serving" deleted from cert-manager namespace service "cert-manager-cainjector" deleted from cert-manager namespace service "cert-manager" deleted from cert-manager namespace service "cert-manager-webhook" deleted from cert-manager namespace deployment.apps "cert-manager-cainjector" deleted from cert-manager namespace deployment.apps "cert-manager" deleted from cert-manager namespace deployment.apps "cert-manager-webhook" deleted from cert-manager namespace mutatingwebhookconfiguration.admissionregistration.k8s.io "cert-manager-webhook" deleted validatingwebhookconfiguration.admissionregistration.k8s.io "cert-manager-webhook" deleted + cat /tmp/tmp.9FBHI8g3zI + rm /tmp/tmp.ZQAlLGtnDy /tmp/tmp.9FBHI8g3zI + return 0 + [[ -n '' ]] + '[' -n psmdb-operator ']' + kubectl_bin delete --grace-period=0 --force=true namespace monitoring-pmm3-14502 + rm -rf /tmp/tmp.9ONBZDsnOT + kubectl_bin delete --grace-period=0 --force=true namespace psmdb-operator ++ mktemp ++ mktemp + desc 'test passed' + set +o xtrace ----------------------------------------------------------------------------------- test passed ----------------------------------------------------------------------------------- + local LAST_OUT=/tmp/tmp.JSVWDBbosz + local LAST_OUT=/tmp/tmp.TotdkURkh9 PASSED++ mktemp ++ mktemp + local LAST_ERR=/tmp/tmp.bFTkFlWrzj + local exit_status=0 + local timeout=4 + args=('delete' '--grace-period=0' '--force=true' 'namespace' 'psmdb-operator') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] + local LAST_ERR=/tmp/tmp.YOrxB4UqdT + local exit_status=0 + local timeout=4 + args=('delete' '--grace-period=0' '--force=true' 'namespace' 'monitoring-pmm3-14502') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl delete --grace-period=0 --force=true namespace psmdb-operator + for i in $(seq 0 2) + set +e + kubectl delete --grace-period=0 --force=true namespace monitoring-pmm3-14502 - generated xml file: /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/e2e-tests/reports/monitoring-pmm3.xml - - Generated html report: file:///mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/e2e-tests/reports/monitoring-pmm3.html - ======================== 1 passed in 3040.77s (0:50:40) ======================== + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.JSVWDBbosz namespace "psmdb-operator" force deleted + cat /tmp/tmp.bFTkFlWrzj Warning: Immediate deletion does not wait for confirmation that the running resource has been terminated. The resource may continue to run on the cluster indefinitely. + rm /tmp/tmp.JSVWDBbosz /tmp/tmp.bFTkFlWrzj + return 0 + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.TotdkURkh9 namespace "monitoring-pmm3-14502" force deleted + cat /tmp/tmp.YOrxB4UqdT Warning: Immediate deletion does not wait for confirmation that the running resource has been terminated. The resource may continue to run on the cluster indefinitely. + rm /tmp/tmp.TotdkURkh9 /tmp/tmp.YOrxB4UqdT + return 0