Log: /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2412/e2e-tests/monitoring-pmm3/../logs/monitoring-pmm3.log
+ create_infra monitoring-pmm3-11763
+ local ns=monitoring-pmm3-11763
+ [[ 1 == 1 ]]
+ delete_crd
+ desc 'get and delete old CRDs and RBAC'
+ set +o xtrace


-----------------------------------------------------------------------------------
get and delete old CRDs and RBAC
-----------------------------------------------------------------------------------

+ kubectl_bin delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2412/deploy/crd.yaml --ignore-not-found --wait=false
++ mktemp
+ local LAST_OUT=/tmp/tmp.2IuiniV9IC
++ mktemp
+ local LAST_ERR=/tmp/tmp.otbzS3biT9
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2412/deploy/crd.yaml --ignore-not-found --wait=false
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.2IuiniV9IC
customresourcedefinition.apiextensions.k8s.io "perconaservermongodbbackups.psmdb.percona.com" deleted
customresourcedefinition.apiextensions.k8s.io "perconaservermongodbclustersyncs.psmdb.percona.com" deleted
customresourcedefinition.apiextensions.k8s.io "perconaservermongodbrestores.psmdb.percona.com" deleted
customresourcedefinition.apiextensions.k8s.io "perconaservermongodbs.psmdb.percona.com" deleted
+ cat /tmp/tmp.otbzS3biT9
+ rm /tmp/tmp.2IuiniV9IC /tmp/tmp.otbzS3biT9
+ return 0
++ yq eval .metadata.name /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2412/deploy/crd.yaml
++ grep -v '\-\-\-'
grep: warning: stray \ before -
grep: warning: stray \ before -
+ for crd_name in $(yq eval '.metadata.name' "${src_dir}/deploy/crd.yaml" | grep -v '\-\-\-')
+ kubectl get perconaservermongodbbackups.psmdb.percona.com --all-namespaces -o wide
+ xargs -L 1 sh -xc 'kubectl patch perconaservermongodbbackups.psmdb.percona.com -n $0 $1 --type=merge -p "{\"metadata\":{\"finalizers\":[]}}"'
+ grep -v NAMESPACE
error: the server doesn't have a resource type "perconaservermongodbbackups"
+ kubectl patch perconaservermongodbbackups.psmdb.percona.com -n sh --type=merge -p '{"metadata":{"finalizers":[]}}'
error: the server doesn't have a resource type "perconaservermongodbbackups"
+ :
+ kubectl_bin wait --for=delete crd perconaservermongodbbackups.psmdb.percona.com
++ mktemp
+ local LAST_OUT=/tmp/tmp.vwdHmArO2P
++ mktemp
+ local LAST_ERR=/tmp/tmp.Ki9BftbbxX
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl wait --for=delete crd perconaservermongodbbackups.psmdb.percona.com
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.vwdHmArO2P
+ cat /tmp/tmp.Ki9BftbbxX
+ rm /tmp/tmp.vwdHmArO2P /tmp/tmp.Ki9BftbbxX
+ return 0
+ for crd_name in $(yq eval '.metadata.name' "${src_dir}/deploy/crd.yaml" | grep -v '\-\-\-')
+ kubectl get perconaservermongodbclustersyncs.psmdb.percona.com --all-namespaces -o wide
+ grep -v NAMESPACE
+ xargs -L 1 sh -xc 'kubectl patch perconaservermongodbclustersyncs.psmdb.percona.com -n $0 $1 --type=merge -p "{\"metadata\":{\"finalizers\":[]}}"'
error: the server doesn't have a resource type "perconaservermongodbclustersyncs"
+ kubectl patch perconaservermongodbclustersyncs.psmdb.percona.com -n sh --type=merge -p '{"metadata":{"finalizers":[]}}'
error: the server doesn't have a resource type "perconaservermongodbclustersyncs"
+ :
+ kubectl_bin wait --for=delete crd perconaservermongodbclustersyncs.psmdb.percona.com
++ mktemp
+ local LAST_OUT=/tmp/tmp.AbTP7y2GBw
++ mktemp
+ local LAST_ERR=/tmp/tmp.fGFi07AFH9
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl wait --for=delete crd perconaservermongodbclustersyncs.psmdb.percona.com
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.AbTP7y2GBw
+ cat /tmp/tmp.fGFi07AFH9
+ rm /tmp/tmp.AbTP7y2GBw /tmp/tmp.fGFi07AFH9
+ return 0
+ for crd_name in $(yq eval '.metadata.name' "${src_dir}/deploy/crd.yaml" | grep -v '\-\-\-')
+ kubectl get perconaservermongodbrestores.psmdb.percona.com --all-namespaces -o wide
+ grep -v NAMESPACE
+ xargs -L 1 sh -xc 'kubectl patch perconaservermongodbrestores.psmdb.percona.com -n $0 $1 --type=merge -p "{\"metadata\":{\"finalizers\":[]}}"'
error: the server doesn't have a resource type "perconaservermongodbrestores"
+ kubectl patch perconaservermongodbrestores.psmdb.percona.com -n sh --type=merge -p '{"metadata":{"finalizers":[]}}'
error: the server doesn't have a resource type "perconaservermongodbrestores"
+ :
+ kubectl_bin wait --for=delete crd perconaservermongodbrestores.psmdb.percona.com
++ mktemp
+ local LAST_OUT=/tmp/tmp.j7IRw3rd1m
++ mktemp
+ local LAST_ERR=/tmp/tmp.HwrgcePmPI
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl wait --for=delete crd perconaservermongodbrestores.psmdb.percona.com
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.j7IRw3rd1m
+ cat /tmp/tmp.HwrgcePmPI
+ rm /tmp/tmp.j7IRw3rd1m /tmp/tmp.HwrgcePmPI
+ return 0
+ for crd_name in $(yq eval '.metadata.name' "${src_dir}/deploy/crd.yaml" | grep -v '\-\-\-')
+ kubectl get perconaservermongodbs.psmdb.percona.com --all-namespaces -o wide
+ grep -v NAMESPACE
+ xargs -L 1 sh -xc 'kubectl patch perconaservermongodbs.psmdb.percona.com -n $0 $1 --type=merge -p "{\"metadata\":{\"finalizers\":[]}}"'
error: the server doesn't have a resource type "perconaservermongodbs"
+ kubectl patch perconaservermongodbs.psmdb.percona.com -n sh --type=merge -p '{"metadata":{"finalizers":[]}}'
error: the server doesn't have a resource type "perconaservermongodbs"
+ :
+ kubectl_bin wait --for=delete crd perconaservermongodbs.psmdb.percona.com
++ mktemp
+ local LAST_OUT=/tmp/tmp.qv75R3kI7T
++ mktemp
+ local LAST_ERR=/tmp/tmp.f6gV27URJr
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl wait --for=delete crd perconaservermongodbs.psmdb.percona.com
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.qv75R3kI7T
+ cat /tmp/tmp.f6gV27URJr
+ rm /tmp/tmp.qv75R3kI7T /tmp/tmp.f6gV27URJr
+ return 0
+ local rbac_yaml=rbac.yaml
+ '[' -n psmdb-operator ']'
+ rbac_yaml=cw-rbac.yaml
+ kubectl_bin delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2412/deploy/cw-rbac.yaml --ignore-not-found
++ mktemp
+ local LAST_OUT=/tmp/tmp.lX74r0NYtN
++ mktemp
+ local LAST_ERR=/tmp/tmp.XPHZ8X2Bp6
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2412/deploy/cw-rbac.yaml --ignore-not-found
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.lX74r0NYtN
clusterrole.rbac.authorization.k8s.io "percona-server-mongodb-operator" deleted
clusterrolebinding.rbac.authorization.k8s.io "service-account-percona-server-mongodb-operator" deleted
+ cat /tmp/tmp.XPHZ8X2Bp6
+ rm /tmp/tmp.lX74r0NYtN /tmp/tmp.XPHZ8X2Bp6
+ return 0
+ check_crd_for_deletion PR-2412-25599b5d4
+ local git_tag=PR-2412-25599b5d4
++ curl -s https://raw.githubusercontent.com/percona/percona-server-mongodb-operator/PR-2412-25599b5d4/deploy/crd.yaml
++ yq eval .metadata.name
++ /usr/sbin/sed s/---//g
++ /usr/sbin/sed ':a;N;$!ba;s/\n/ /g'
+ for crd_name in $(curl -s https://raw.githubusercontent.com/percona/percona-server-mongodb-operator/${git_tag}/deploy/crd.yaml | yq eval '.metadata.name' | $sed 's/---//g' | $sed ':a;N;$!ba;s/\n/ /g')
++ kubectl_bin get crd/null -o 'jsonpath={.status.conditions[-1].type}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.bKzP8NOQut
+++ mktemp
++ local LAST_ERR=/tmp/tmp.Kp7PsJtCM4
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get crd/null -o 'jsonpath={.status.conditions[-1].type}'
++ exit_status=1
++ set -e
++ '[' 1 '!=' 0 -a -n 1 ']'
++ cat /tmp/tmp.bKzP8NOQut
++ cat /tmp/tmp.Kp7PsJtCM4
Error from server (NotFound): customresourcedefinitions.apiextensions.k8s.io "null" not found
++ sleep 0
++ for i in $(seq 0 2)
++ set +e
++ kubectl get crd/null -o 'jsonpath={.status.conditions[-1].type}'
++ exit_status=1
++ set -e
++ '[' 1 '!=' 0 -a -n 1 ']'
++ cat /tmp/tmp.bKzP8NOQut
++ cat /tmp/tmp.Kp7PsJtCM4
Error from server (NotFound): customresourcedefinitions.apiextensions.k8s.io "null" not found
++ sleep 4
++ for i in $(seq 0 2)
++ set +e
++ kubectl get crd/null -o 'jsonpath={.status.conditions[-1].type}'
++ exit_status=1
++ set -e
++ '[' 1 '!=' 0 -a -n 1 ']'
++ cat /tmp/tmp.bKzP8NOQut
++ cat /tmp/tmp.Kp7PsJtCM4
Error from server (NotFound): customresourcedefinitions.apiextensions.k8s.io "null" not found
++ sleep 8
++ cat /tmp/tmp.bKzP8NOQut
++ cat /tmp/tmp.Kp7PsJtCM4
Error from server (NotFound): customresourcedefinitions.apiextensions.k8s.io "null" not found
++ rm /tmp/tmp.bKzP8NOQut /tmp/tmp.Kp7PsJtCM4
++ return 1
+ [[ '' == Terminating ]]
+ '[' -n psmdb-operator ']'
+ create_namespace psmdb-operator
+ local namespace=psmdb-operator
+ local skip_clean_namespace=
+ [[ 1 == 1 ]]
+ [[ -z '' ]]
+ destroy_chaos_mesh
++ helm list --all-namespaces --filter chaos-mesh
++ tail -n1
++ awk '-F ' '{print $2}'
++ sed s/NAMESPACE//
+ local chaos_mesh_ns=
+ desc 'destroy chaos-mesh'
+ set +o xtrace


-----------------------------------------------------------------------------------
destroy chaos-mesh
-----------------------------------------------------------------------------------

+ '[' -n '' ']'
++ kubectl get MutatingWebhookConfiguration
++ grep chaos-mesh
++ awk '{print $1}'
+ timeout 30 kubectl delete MutatingWebhookConfiguration
error: resource(s) were provided, but no name was specified
+ :
++ kubectl get ValidatingWebhookConfiguration
++ grep chaos-mesh
++ awk '{print $1}'
+ timeout 30 kubectl delete ValidatingWebhookConfiguration
error: resource(s) were provided, but no name was specified
+ :
++ kubectl get ValidatingWebhookConfiguration
++ grep validate-auth
++ awk '{print $1}'
+ timeout 30 kubectl delete ValidatingWebhookConfiguration
error: resource(s) were provided, but no name was specified
+ :
++ grep chaos-mesh
++ kubectl api-resources
++ awk '{print $1}'
++ kubectl get crd
++ grep chaos-mesh.org
++ awk '{print $1}'
+ timeout 30 kubectl delete crd
error: resource(s) were provided, but no name was specified
+ :
++ kubectl get clusterrolebinding
++ grep chaos-mesh
++ awk '{print $1}'
+ timeout 30 kubectl delete clusterrolebinding
error: resource(s) were provided, but no name was specified
+ :
++ kubectl get clusterrole
++ grep chaos-mesh
++ awk '{print $1}'
+ timeout 30 kubectl delete clusterrole
error: resource(s) were provided, but no name was specified
+ :
+ desc 'cleaned up all old namespaces'
+ set +o xtrace


-----------------------------------------------------------------------------------
cleaned up all old namespaces
-----------------------------------------------------------------------------------

+ kubectl_bin get ns
+ grep -E -v '^kube-|^default|Terminating|psmdb-operator|openshift|^gke-|^gmp-|^NAME'
+ [[ -n '' ]]
+ desc 'cleaned up old namespaces psmdb-operator'
+ set +o xtrace


-----------------------------------------------------------------------------------
cleaned up old namespaces psmdb-operator
-----------------------------------------------------------------------------------

+ xargs kubectl delete ns
+ kubectl_bin delete namespace psmdb-operator --ignore-not-found
+ awk '{print$1}'
++ mktemp
++ mktemp
+ local LAST_OUT=/tmp/tmp.Jm0IBbEf9C
+ local LAST_OUT=/tmp/tmp.pz7sScHEO0
++ mktemp
++ mktemp
+ local LAST_ERR=/tmp/tmp.ixzKABdZhu
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ local LAST_ERR=/tmp/tmp.z4QaHyLWJc
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl get ns
+ for i in $(seq 0 2)
+ set +e
+ kubectl delete namespace psmdb-operator --ignore-not-found
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.Jm0IBbEf9C
+ cat /tmp/tmp.ixzKABdZhu
+ rm /tmp/tmp.Jm0IBbEf9C /tmp/tmp.ixzKABdZhu
+ return 0
namespace "cert-manager" deleted
namespace "monitoring-pmm3-17926" deleted
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.pz7sScHEO0
namespace "psmdb-operator" deleted
+ cat /tmp/tmp.z4QaHyLWJc
+ rm /tmp/tmp.pz7sScHEO0 /tmp/tmp.z4QaHyLWJc
+ return 0
+ kubectl_bin wait --for=delete namespace psmdb-operator
++ mktemp
+ local LAST_OUT=/tmp/tmp.GFpGEQZAWI
++ mktemp
+ local LAST_ERR=/tmp/tmp.kQ57Uhj0CR
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl wait --for=delete namespace psmdb-operator
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.GFpGEQZAWI
+ cat /tmp/tmp.kQ57Uhj0CR
+ rm /tmp/tmp.GFpGEQZAWI /tmp/tmp.kQ57Uhj0CR
+ return 0
+ desc 'create namespace psmdb-operator'
+ set +o xtrace


-----------------------------------------------------------------------------------
create namespace psmdb-operator
-----------------------------------------------------------------------------------

+ kubectl_bin create namespace psmdb-operator
++ mktemp
+ local LAST_OUT=/tmp/tmp.WIrWaZMtvK
++ mktemp
+ local LAST_ERR=/tmp/tmp.3ZcUt1CFZE
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl create namespace psmdb-operator
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.WIrWaZMtvK
namespace/psmdb-operator created
+ cat /tmp/tmp.3ZcUt1CFZE
+ rm /tmp/tmp.WIrWaZMtvK /tmp/tmp.3ZcUt1CFZE
+ return 0
+ set_kube_ctx psmdb-operator
+ local namespace=psmdb-operator
++ kubectl_bin config current-context
+++ mktemp
++ local LAST_OUT=/tmp/tmp.StRrD9GwjD
+++ mktemp
++ local LAST_ERR=/tmp/tmp.zeTdNDh4F8
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl config current-context
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.StRrD9GwjD
++ cat /tmp/tmp.zeTdNDh4F8
++ rm /tmp/tmp.StRrD9GwjD /tmp/tmp.zeTdNDh4F8
++ return 0
+ kubectl_bin config set-context gke_cloud-dev-112233_us-central1-a_jen-psmdb-2412-25599b5d4-9-cluster1 --namespace=psmdb-operator
++ mktemp
+ local LAST_OUT=/tmp/tmp.Gu935oGU9D
++ mktemp
+ local LAST_ERR=/tmp/tmp.dsFzQqoVHi
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl config set-context gke_cloud-dev-112233_us-central1-a_jen-psmdb-2412-25599b5d4-9-cluster1 --namespace=psmdb-operator
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.Gu935oGU9D
Context "gke_cloud-dev-112233_us-central1-a_jen-psmdb-2412-25599b5d4-9-cluster1" modified.
+ cat /tmp/tmp.dsFzQqoVHi
+ rm /tmp/tmp.Gu935oGU9D /tmp/tmp.dsFzQqoVHi
+ return 0
+ deploy_operator
+ desc 'start PSMDB operator: docker.io/perconalab/percona-server-mongodb-operator:PR-2412-25599b5d4'
+ set +o xtrace


-----------------------------------------------------------------------------------
start PSMDB operator: docker.io/perconalab/percona-server-mongodb-operator:PR-2412-25599b5d4
-----------------------------------------------------------------------------------

+ local cr_file
+ '[' -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2412/e2e-tests/monitoring-pmm3/conf/crd.yaml ']'
+ cr_file=/mnt/jenkins/workspace/cloud-psmdb-operator_PR-2412/deploy/crd.yaml
+ kubectl_bin apply --server-side --force-conflicts -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2412/deploy/crd.yaml
++ mktemp
+ local LAST_OUT=/tmp/tmp.q0KqfC5EQ1
++ mktemp
+ local LAST_ERR=/tmp/tmp.vNYDeD3DGR
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl apply --server-side --force-conflicts -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2412/deploy/crd.yaml
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.q0KqfC5EQ1
customresourcedefinition.apiextensions.k8s.io/perconaservermongodbbackups.psmdb.percona.com serverside-applied
customresourcedefinition.apiextensions.k8s.io/perconaservermongodbclustersyncs.psmdb.percona.com serverside-applied
customresourcedefinition.apiextensions.k8s.io/perconaservermongodbrestores.psmdb.percona.com serverside-applied
customresourcedefinition.apiextensions.k8s.io/perconaservermongodbs.psmdb.percona.com serverside-applied
+ cat /tmp/tmp.vNYDeD3DGR
+ rm /tmp/tmp.q0KqfC5EQ1 /tmp/tmp.vNYDeD3DGR
+ return 0
+ '[' -n psmdb-operator ']'
+ apply_rbac cw-rbac
+ local operator_namespace=psmdb-operator
+ local rbac=cw-rbac
+ cat /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2412/deploy/cw-rbac.yaml
+ sed -e 's^namespace: .*^namespace: psmdb-operator^'
+ kubectl_bin apply -n psmdb-operator -f -
++ mktemp
+ local LAST_OUT=/tmp/tmp.01bQ1xAZMK
++ mktemp
+ local LAST_ERR=/tmp/tmp.xc5HzuLek8
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl apply -n psmdb-operator -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.01bQ1xAZMK
clusterrole.rbac.authorization.k8s.io/percona-server-mongodb-operator created
serviceaccount/percona-server-mongodb-operator created
clusterrolebinding.rbac.authorization.k8s.io/service-account-percona-server-mongodb-operator created
+ cat /tmp/tmp.xc5HzuLek8
+ rm /tmp/tmp.01bQ1xAZMK /tmp/tmp.xc5HzuLek8
+ return 0
+ yq eval $'\n\t\t\t(.spec.template.spec.containers[].image = "docker.io/perconalab/percona-server-mongodb-operator:PR-2412-25599b5d4") |\n\t\t\t((.. | select(.[] == "DISABLE_TELEMETRY")) |= .value="true") |\n\t\t\t((.. | select(.[] == "LOG_LEVEL")) |= .value="DEBUG")' /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2412/deploy/cw-operator.yaml
+ kubectl_bin apply -n psmdb-operator -f -
++ mktemp
+ local LAST_OUT=/tmp/tmp.dwWXdpCf3p
++ mktemp
+ local LAST_ERR=/tmp/tmp.2hsDLXxaGJ
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl apply -n psmdb-operator -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.dwWXdpCf3p
deployment.apps/percona-server-mongodb-operator created
+ cat /tmp/tmp.2hsDLXxaGJ
+ rm /tmp/tmp.dwWXdpCf3p /tmp/tmp.2hsDLXxaGJ
+ return 0
+ sleep 20
++ get_operator_pod
++ kubectl_bin get pods --selector=name=percona-server-mongodb-operator -o 'jsonpath={.items[].metadata.name}' -n psmdb-operator
+++ mktemp
++ local LAST_OUT=/tmp/tmp.VzUm8YwOAY
+++ mktemp
++ local LAST_ERR=/tmp/tmp.aMJbXcWMon
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=percona-server-mongodb-operator -o 'jsonpath={.items[].metadata.name}' -n psmdb-operator
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.VzUm8YwOAY
++ cat /tmp/tmp.aMJbXcWMon
++ rm /tmp/tmp.VzUm8YwOAY /tmp/tmp.aMJbXcWMon
++ return 0
+ wait_operator_pod percona-server-mongodb-operator-5d88bccdc4-lglbz
+ local pod=percona-server-mongodb-operator-5d88bccdc4-lglbz
+ set +o xtrace
waiting for pod/percona-server-mongodb-operator-5d88bccdc4-lglbz to be ready.OK
+ echo 'Print operator info from log'
Print operator info from log
+ grep 'Manager starting up'
++ get_operator_pod
++ kubectl_bin get pods --selector=name=percona-server-mongodb-operator -o 'jsonpath={.items[].metadata.name}' -n psmdb-operator
+++ mktemp
++ local LAST_OUT=/tmp/tmp.LQlrT6wG3T
+++ mktemp
++ local LAST_ERR=/tmp/tmp.y4RYyn4Gkd
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=percona-server-mongodb-operator -o 'jsonpath={.items[].metadata.name}' -n psmdb-operator
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.LQlrT6wG3T
++ cat /tmp/tmp.y4RYyn4Gkd
++ rm /tmp/tmp.LQlrT6wG3T /tmp/tmp.y4RYyn4Gkd
++ return 0
+ kubectl_bin logs -n psmdb-operator percona-server-mongodb-operator-5d88bccdc4-lglbz
++ mktemp
+ local LAST_OUT=/tmp/tmp.htkYok0Z9c
++ mktemp
+ local LAST_ERR=/tmp/tmp.bFIWsKvsB1
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl logs -n psmdb-operator percona-server-mongodb-operator-5d88bccdc4-lglbz
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.htkYok0Z9c
+ cat /tmp/tmp.bFIWsKvsB1
+ rm /tmp/tmp.htkYok0Z9c /tmp/tmp.bFIWsKvsB1
+ return 0
2026-07-07T19:35:56.317Z	INFO	setup	Manager starting up	{"gitCommit": "25599b5d4f8569dcabc6e4bd2154994cf21a6242", "gitBranch": "PR-2412-25599b5d4", "buildTime": "", "goVersion": "go1.26.4", "os": "linux", "arch": "amd64"}
+ create_namespace monitoring-pmm3-11763
+ local namespace=monitoring-pmm3-11763
+ local skip_clean_namespace=
+ [[ 1 == 1 ]]
+ [[ -z '' ]]
+ destroy_chaos_mesh
++ helm list --all-namespaces --filter chaos-mesh
++ awk '-F ' '{print $2}'
++ tail -n1
++ sed s/NAMESPACE//
+ local chaos_mesh_ns=
+ desc 'destroy chaos-mesh'
+ set +o xtrace


-----------------------------------------------------------------------------------
destroy chaos-mesh
-----------------------------------------------------------------------------------

+ '[' -n '' ']'
++ kubectl get MutatingWebhookConfiguration
++ grep chaos-mesh
++ awk '{print $1}'
+ timeout 30 kubectl delete MutatingWebhookConfiguration
error: resource(s) were provided, but no name was specified
+ :
++ kubectl get ValidatingWebhookConfiguration
++ grep chaos-mesh
++ awk '{print $1}'
+ timeout 30 kubectl delete ValidatingWebhookConfiguration
error: resource(s) were provided, but no name was specified
+ :
++ kubectl get ValidatingWebhookConfiguration
++ grep validate-auth
++ awk '{print $1}'
+ timeout 30 kubectl delete ValidatingWebhookConfiguration
error: resource(s) were provided, but no name was specified
+ :
++ kubectl api-resources
++ grep chaos-mesh
++ awk '{print $1}'
++ kubectl get crd
++ grep chaos-mesh.org
++ awk '{print $1}'
+ timeout 30 kubectl delete crd
error: resource(s) were provided, but no name was specified
+ :
++ kubectl get clusterrolebinding
++ grep chaos-mesh
++ awk '{print $1}'
+ timeout 30 kubectl delete clusterrolebinding
error: resource(s) were provided, but no name was specified
+ :
++ kubectl get clusterrole
++ grep chaos-mesh
++ awk '{print $1}'
+ timeout 30 kubectl delete clusterrole
error: resource(s) were provided, but no name was specified
+ :
+ desc 'cleaned up all old namespaces'
+ set +o xtrace


-----------------------------------------------------------------------------------
cleaned up all old namespaces
-----------------------------------------------------------------------------------

+ kubectl_bin get ns
+ grep -E -v '^kube-|^default|Terminating|psmdb-operator|openshift|^gke-|^gmp-|^NAME'
++ mktemp
+ awk '{print$1}'
+ [[ -n '' ]]
+ desc 'cleaned up old namespaces monitoring-pmm3-11763'
+ xargs kubectl delete ns
+ set +o xtrace


-----------------------------------------------------------------------------------
cleaned up old namespaces monitoring-pmm3-11763
-----------------------------------------------------------------------------------

+ kubectl_bin delete namespace monitoring-pmm3-11763 --ignore-not-found
++ mktemp
+ local LAST_OUT=/tmp/tmp.n5IfqUJobT
++ mktemp
+ local LAST_OUT=/tmp/tmp.UktlLQTRd7
++ mktemp
+ local LAST_ERR=/tmp/tmp.5M5xgrh4Z2
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ local LAST_ERR=/tmp/tmp.WTKVdaeP5F
+ local exit_status=0
+ local timeout=4
+ for i in $(seq 0 2)
+ set +e
+ kubectl get ns
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl delete namespace monitoring-pmm3-11763 --ignore-not-found
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.n5IfqUJobT
+ cat /tmp/tmp.5M5xgrh4Z2
+ rm /tmp/tmp.n5IfqUJobT /tmp/tmp.5M5xgrh4Z2
+ return 0
error: resource(s) were provided, but no name was specified
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.UktlLQTRd7
+ cat /tmp/tmp.WTKVdaeP5F
+ rm /tmp/tmp.UktlLQTRd7 /tmp/tmp.WTKVdaeP5F
+ return 0
+ kubectl_bin wait --for=delete namespace monitoring-pmm3-11763
++ mktemp
+ local LAST_OUT=/tmp/tmp.gIcXx8UYBJ
++ mktemp
+ local LAST_ERR=/tmp/tmp.c1ddgyhxjB
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl wait --for=delete namespace monitoring-pmm3-11763
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.gIcXx8UYBJ
+ cat /tmp/tmp.c1ddgyhxjB
+ rm /tmp/tmp.gIcXx8UYBJ /tmp/tmp.c1ddgyhxjB
+ return 0
+ desc 'create namespace monitoring-pmm3-11763'
+ set +o xtrace


-----------------------------------------------------------------------------------
create namespace monitoring-pmm3-11763
-----------------------------------------------------------------------------------

+ kubectl_bin create namespace monitoring-pmm3-11763
++ mktemp
+ local LAST_OUT=/tmp/tmp.RolD4QBFMU
++ mktemp
+ local LAST_ERR=/tmp/tmp.u9YAluj2NY
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl create namespace monitoring-pmm3-11763
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.RolD4QBFMU
namespace/monitoring-pmm3-11763 created
+ cat /tmp/tmp.u9YAluj2NY
+ rm /tmp/tmp.RolD4QBFMU /tmp/tmp.u9YAluj2NY
+ return 0
+ set_kube_ctx monitoring-pmm3-11763
+ local namespace=monitoring-pmm3-11763
++ kubectl_bin config current-context
+++ mktemp
++ local LAST_OUT=/tmp/tmp.ziLJGseoTx
+++ mktemp
++ local LAST_ERR=/tmp/tmp.SaapifcKVK
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl config current-context
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.ziLJGseoTx
++ cat /tmp/tmp.SaapifcKVK
++ rm /tmp/tmp.ziLJGseoTx /tmp/tmp.SaapifcKVK
++ return 0
+ kubectl_bin config set-context gke_cloud-dev-112233_us-central1-a_jen-psmdb-2412-25599b5d4-9-cluster1 --namespace=monitoring-pmm3-11763
++ mktemp
+ local LAST_OUT=/tmp/tmp.BWRdRiIson
++ mktemp
+ local LAST_ERR=/tmp/tmp.L1rkEtQtK6
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl config set-context gke_cloud-dev-112233_us-central1-a_jen-psmdb-2412-25599b5d4-9-cluster1 --namespace=monitoring-pmm3-11763
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.BWRdRiIson
Context "gke_cloud-dev-112233_us-central1-a_jen-psmdb-2412-25599b5d4-9-cluster1" modified.
+ cat /tmp/tmp.L1rkEtQtK6
+ rm /tmp/tmp.BWRdRiIson /tmp/tmp.L1rkEtQtK6
+ return 0
+ deploy_cert_manager
+ [[ gke == rancher ]]
+ desc 'deploy cert manager'
+ set +o xtrace


-----------------------------------------------------------------------------------
deploy cert manager
-----------------------------------------------------------------------------------

+ kubectl_bin -n cert-manager delete clusterissuer --all
++ mktemp
+ local LAST_OUT=/tmp/tmp.HLj4v135fB
++ mktemp
+ local LAST_ERR=/tmp/tmp.6w5Olsaxx5
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl -n cert-manager delete clusterissuer --all
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.HLj4v135fB
No resources found
+ cat /tmp/tmp.6w5Olsaxx5
+ rm /tmp/tmp.HLj4v135fB /tmp/tmp.6w5Olsaxx5
+ return 0
+ kubectl_bin -n cert-manager delete certificate --all
++ mktemp
+ local LAST_OUT=/tmp/tmp.TaowWdIMhB
++ mktemp
+ local LAST_ERR=/tmp/tmp.a8VNFU7Pb7
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl -n cert-manager delete certificate --all
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.TaowWdIMhB
No resources found
+ cat /tmp/tmp.a8VNFU7Pb7
+ rm /tmp/tmp.TaowWdIMhB /tmp/tmp.a8VNFU7Pb7
+ return 0
+ kubectl_bin create namespace cert-manager
++ mktemp
+ local LAST_OUT=/tmp/tmp.8A9v53RynN
++ mktemp
+ local LAST_ERR=/tmp/tmp.cdMGFB3Z7k
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl create namespace cert-manager
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.8A9v53RynN
namespace/cert-manager created
+ cat /tmp/tmp.cdMGFB3Z7k
+ rm /tmp/tmp.8A9v53RynN /tmp/tmp.cdMGFB3Z7k
+ return 0
+ kubectl_bin label namespace cert-manager certmanager.k8s.io/disable-validation=true
++ mktemp
+ local LAST_OUT=/tmp/tmp.FFe0l9kJzc
++ mktemp
+ local LAST_ERR=/tmp/tmp.2UP8ilaGRw
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl label namespace cert-manager certmanager.k8s.io/disable-validation=true
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.FFe0l9kJzc
namespace/cert-manager labeled
+ cat /tmp/tmp.2UP8ilaGRw
+ rm /tmp/tmp.FFe0l9kJzc /tmp/tmp.2UP8ilaGRw
+ return 0
+ kubectl_bin apply -f https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml --validate=false
++ mktemp
+ local LAST_OUT=/tmp/tmp.Zu6ZFsNUuw
++ mktemp
+ local LAST_ERR=/tmp/tmp.mX24kOWS03
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl apply -f https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml --validate=false
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.Zu6ZFsNUuw
namespace/cert-manager configured
customresourcedefinition.apiextensions.k8s.io/challenges.acme.cert-manager.io unchanged
customresourcedefinition.apiextensions.k8s.io/orders.acme.cert-manager.io unchanged
customresourcedefinition.apiextensions.k8s.io/certificaterequests.cert-manager.io unchanged
customresourcedefinition.apiextensions.k8s.io/certificates.cert-manager.io unchanged
customresourcedefinition.apiextensions.k8s.io/clusterissuers.cert-manager.io unchanged
customresourcedefinition.apiextensions.k8s.io/issuers.cert-manager.io unchanged
serviceaccount/cert-manager-cainjector created
serviceaccount/cert-manager created
serviceaccount/cert-manager-webhook created
clusterrole.rbac.authorization.k8s.io/cert-manager-cainjector unchanged
clusterrole.rbac.authorization.k8s.io/cert-manager-controller-issuers unchanged
clusterrole.rbac.authorization.k8s.io/cert-manager-controller-clusterissuers unchanged
clusterrole.rbac.authorization.k8s.io/cert-manager-controller-certificates unchanged
clusterrole.rbac.authorization.k8s.io/cert-manager-controller-orders unchanged
clusterrole.rbac.authorization.k8s.io/cert-manager-controller-challenges unchanged
clusterrole.rbac.authorization.k8s.io/cert-manager-controller-ingress-shim unchanged
clusterrole.rbac.authorization.k8s.io/cert-manager-cluster-view unchanged
clusterrole.rbac.authorization.k8s.io/cert-manager-view unchanged
clusterrole.rbac.authorization.k8s.io/cert-manager-edit unchanged
clusterrole.rbac.authorization.k8s.io/cert-manager-controller-approve:cert-manager-io unchanged
clusterrole.rbac.authorization.k8s.io/cert-manager-controller-certificatesigningrequests unchanged
clusterrole.rbac.authorization.k8s.io/cert-manager-webhook:subjectaccessreviews unchanged
clusterrolebinding.rbac.authorization.k8s.io/cert-manager-cainjector unchanged
clusterrolebinding.rbac.authorization.k8s.io/cert-manager-controller-issuers unchanged
clusterrolebinding.rbac.authorization.k8s.io/cert-manager-controller-clusterissuers unchanged
clusterrolebinding.rbac.authorization.k8s.io/cert-manager-controller-certificates unchanged
clusterrolebinding.rbac.authorization.k8s.io/cert-manager-controller-orders unchanged
clusterrolebinding.rbac.authorization.k8s.io/cert-manager-controller-challenges unchanged
clusterrolebinding.rbac.authorization.k8s.io/cert-manager-controller-ingress-shim unchanged
clusterrolebinding.rbac.authorization.k8s.io/cert-manager-controller-approve:cert-manager-io unchanged
clusterrolebinding.rbac.authorization.k8s.io/cert-manager-controller-certificatesigningrequests unchanged
clusterrolebinding.rbac.authorization.k8s.io/cert-manager-webhook:subjectaccessreviews unchanged
role.rbac.authorization.k8s.io/cert-manager-cainjector:leaderelection unchanged
role.rbac.authorization.k8s.io/cert-manager:leaderelection unchanged
role.rbac.authorization.k8s.io/cert-manager-tokenrequest created
role.rbac.authorization.k8s.io/cert-manager-webhook:dynamic-serving created
rolebinding.rbac.authorization.k8s.io/cert-manager-cainjector:leaderelection unchanged
rolebinding.rbac.authorization.k8s.io/cert-manager:leaderelection unchanged
rolebinding.rbac.authorization.k8s.io/cert-manager-tokenrequest created
rolebinding.rbac.authorization.k8s.io/cert-manager-webhook:dynamic-serving created
service/cert-manager-cainjector created
service/cert-manager created
service/cert-manager-webhook created
deployment.apps/cert-manager-cainjector created
deployment.apps/cert-manager created
deployment.apps/cert-manager-webhook created
mutatingwebhookconfiguration.admissionregistration.k8s.io/cert-manager-webhook configured
validatingwebhookconfiguration.admissionregistration.k8s.io/cert-manager-webhook configured
+ cat /tmp/tmp.mX24kOWS03
Warning: resource namespaces/cert-manager is missing the kubectl.kubernetes.io/last-applied-configuration annotation which is required by kubectl apply. kubectl apply should only be used on resources created declaratively by either kubectl create --save-config or kubectl apply. The missing annotation will be patched automatically.
+ rm /tmp/tmp.Zu6ZFsNUuw /tmp/tmp.mX24kOWS03
+ return 0
+ kubectl_bin -n cert-manager wait pod -l app.kubernetes.io/instance=cert-manager --for=condition=ready
++ mktemp
+ local LAST_OUT=/tmp/tmp.VH96IyfPyi
++ mktemp
+ local LAST_ERR=/tmp/tmp.PKzhTlYHFP
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl -n cert-manager wait pod -l app.kubernetes.io/instance=cert-manager --for=condition=ready
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.VH96IyfPyi
pod/cert-manager-559d798845-np2sl condition met
pod/cert-manager-cainjector-64958d9c7c-f665c condition met
pod/cert-manager-webhook-7fb6f99b56-k446k condition met
+ cat /tmp/tmp.PKzhTlYHFP
E0707 19:36:54.163204  535818 reflector.go:227] "Failed to watch" err="Get \"https://34.27.139.243/api/v1/namespaces/cert-manager/pods?allowWatchBookmarks=true&fieldSelector=metadata.name%3Dcert-manager-cainjector-64958d9c7c-f665c&resourceVersion=1783453013780031009&timeoutSeconds=514&watch=true\": context canceled" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" type="*unstructured.Unstructured"
+ rm /tmp/tmp.VH96IyfPyi /tmp/tmp.PKzhTlYHFP
+ return 0
+ sleep 120
+ desc 'install PMM Server'
+ set +o xtrace


-----------------------------------------------------------------------------------
install PMM Server
-----------------------------------------------------------------------------------

+ deploy_pmm3_server
+ helm uninstall monitoring
Error: uninstall: Release not loaded: monitoring: release: not found
+ :
+ helm repo remove percona
"percona" has been removed from your repositories
+ kubectl delete clusterrole monitoring --ignore-not-found
+ kubectl delete clusterrolebinding monitoring --ignore-not-found
+ helm repo add percona https://percona.github.io/percona-helm-charts/
"percona" has been added to your repositories
+ helm repo update
Hang tight while we grab the latest from your chart repositories...
...Successfully got an update from the "minio" chart repository
...Successfully got an update from the "percona" chart repository
...Successfully got an update from the "chaos-mesh" chart repository
...Successfully got an update from the "hashicorp" chart repository
...Successfully got an update from the "stable" chart repository
Update Complete. ⎈Happy Helming!⎈
+ [[ -n '' ]]
+ retry 10 60 helm install monitoring percona/pmm --set fullnameOverride=monitoring-server --set image.tag=3-dev-latest --set image.repository=docker.io/perconalab/pmm-server --set service.type=LoadBalancer --force
+ local max=10
+ local delay=60
+ shift 2
+ local n=1
+ helm install monitoring percona/pmm --set fullnameOverride=monitoring-server --set image.tag=3-dev-latest --set image.repository=docker.io/perconalab/pmm-server --set service.type=LoadBalancer --force
NAME: monitoring
LAST DEPLOYED: Tue Jul  7 19:39:07 2026
NAMESPACE: monitoring-pmm3-11763
STATUS: deployed
REVISION: 1
TEST SUITE: None
NOTES:
Percona Monitoring and Management (PMM)

An open source database monitoring, observability and management tool
Check more info here: https://docs.percona.com/percona-monitoring-and-management/index.html

Get the application URL:
     NOTE: It may take a few minutes for the LoadBalancer IP to be available.
           You can watch the status of by running 'kubectl get --namespace monitoring-pmm3-11763 svc -w monitoring-service'
  export SERVICE_IP=$(kubectl get svc --namespace monitoring-pmm3-11763 monitoring-service -o jsonpath="{.status.loadBalancer.ingress[0].ip}")
  echo https://$SERVICE_IP:

Get password for the "admin" user:
  export ADMIN_PASS=$(kubectl get secret pmm-secret --namespace monitoring-pmm3-11763 -o jsonpath='{.data.PMM_ADMIN_PASSWORD}' | base64 --decode)
  echo $ADMIN_PASS
+ sleep 20
+ kubectl_bin exec monitoring-server-0 -- bash -c 'ls -l /proc/*/exe 2>/dev/null| grep postgres >/dev/null'
++ mktemp
+ local LAST_OUT=/tmp/tmp.nACj9ozvGy
++ mktemp
+ local LAST_ERR=/tmp/tmp.F0cXsEsevS
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec monitoring-server-0 -- bash -c 'ls -l /proc/*/exe 2>/dev/null| grep postgres >/dev/null'
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.nACj9ozvGy
+ cat /tmp/tmp.F0cXsEsevS
+ rm /tmp/tmp.nACj9ozvGy /tmp/tmp.F0cXsEsevS
+ return 0
+ cluster=monitoring-pmm3
+ desc 'create secrets and start client'
+ set +o xtrace


-----------------------------------------------------------------------------------
create secrets and start client
-----------------------------------------------------------------------------------

+ kubectl_bin apply -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2412/e2e-tests/conf/secrets.yml -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2412/e2e-tests/monitoring-pmm3/conf/secrets.yml
++ mktemp
+ local LAST_OUT=/tmp/tmp.ult3VgZNKN
++ mktemp
+ local LAST_ERR=/tmp/tmp.zfyFT6mIEN
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl apply -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2412/e2e-tests/conf/secrets.yml -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2412/e2e-tests/monitoring-pmm3/conf/secrets.yml
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.ult3VgZNKN
secret/some-users created
secret/some-users unchanged
+ cat /tmp/tmp.zfyFT6mIEN
+ rm /tmp/tmp.ult3VgZNKN /tmp/tmp.zfyFT6mIEN
+ return 0
+ yq '.spec.template.spec.volumes[0].secret.secretName="monitoring-pmm3-ssl"' /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2412/e2e-tests/conf/client_with_tls.yml
+ kubectl_bin apply -f -
++ mktemp
+ local LAST_OUT=/tmp/tmp.NoVg7o2Vh8
++ mktemp
+ local LAST_ERR=/tmp/tmp.XW9L2sxC0a
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl apply -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.NoVg7o2Vh8
deployment.apps/psmdb-client created
+ cat /tmp/tmp.XW9L2sxC0a
+ rm /tmp/tmp.NoVg7o2Vh8 /tmp/tmp.XW9L2sxC0a
+ return 0
+ sleep 90
+ desc 'create first PSMDB cluster monitoring-pmm3'
+ set +o xtrace


-----------------------------------------------------------------------------------
create first PSMDB cluster monitoring-pmm3
-----------------------------------------------------------------------------------

+ custom_cluster_name=super-custom
+ yq eval '(.spec | select(.image == null)).image = "docker.io/perconalab/percona-server-mongodb-operator:main-mongod8.0"' /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2412/e2e-tests/monitoring-pmm3/conf/monitoring-pmm3-rs0.yml
+ yq eval '(.spec | select(has("pmm"))).pmm.customClusterName = "super-custom"' -
+ yq eval '(.spec | select(has("pmm"))).pmm.image = "docker.io/perconalab/pmm-client:3-dev-latest"' -
+ yq eval '(.spec | select(has("logcollector"))).logcollector.image = "docker.io/perconalab/fluentbit:main-logcollector"' -
+ yq eval '(.spec | select(has("initImage"))).initImage = "docker.io/perconalab/percona-server-mongodb-operator:PR-2412-25599b5d4"' -
+ yq eval '(.spec | select(has("backup"))).backup.image = "docker.io/perconalab/percona-server-mongodb-operator:main-backup"' -
+ yq eval '.spec.upgradeOptions.apply = "Never"' -
+ kubectl_bin apply -f -
++ mktemp
+ local LAST_OUT=/tmp/tmp.POYlwpgGwG
++ mktemp
+ local LAST_ERR=/tmp/tmp.SIVDsMGqXX
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl apply -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.POYlwpgGwG
perconaservermongodb.psmdb.percona.com/monitoring-pmm3 created
+ cat /tmp/tmp.SIVDsMGqXX
+ rm /tmp/tmp.POYlwpgGwG /tmp/tmp.SIVDsMGqXX
+ return 0
+ wait_for_running monitoring-pmm3-rs0 3
+ local name=monitoring-pmm3-rs0
+ let last_pod=2
+ local check_cluster_readiness=true
+ set_debug
+ [[ 1 == 1 ]]
+ set -o xtrace
+ local rs_name=rs0
+ local cluster_name=monitoring-pmm3
++ seq 0 2
+ for i in $(seq 0 $last_pod)
+ [[ 0 -eq 2 ]]
+ wait_pod monitoring-pmm3-rs0-0
+ local pod=monitoring-pmm3-rs0-0
+ set +o xtrace
waiting for pod/monitoring-pmm3-rs0-0 to be ready.......OK
+ for i in $(seq 0 $last_pod)
+ [[ 1 -eq 2 ]]
+ wait_pod monitoring-pmm3-rs0-1
+ local pod=monitoring-pmm3-rs0-1
+ set +o xtrace
waiting for pod/monitoring-pmm3-rs0-1 to be ready.......OK
+ for i in $(seq 0 $last_pod)
+ [[ 2 -eq 2 ]]
++ kubectl_bin get psmdb monitoring-pmm3 -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].arbiter.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.JpopPZyYO2
+++ mktemp
++ local LAST_ERR=/tmp/tmp.zFKANJ4F4q
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb monitoring-pmm3 -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].arbiter.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.JpopPZyYO2
++ cat /tmp/tmp.zFKANJ4F4q
++ rm /tmp/tmp.JpopPZyYO2 /tmp/tmp.zFKANJ4F4q
++ return 0
+ [[ '' == true ]]
+ wait_pod monitoring-pmm3-rs0-2
+ local pod=monitoring-pmm3-rs0-2
+ set +o xtrace
waiting for pod/monitoring-pmm3-rs0-2 to be ready......OK
++ kubectl_bin get psmdb monitoring-pmm3 -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].nonvoting.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.6AbdfrtN1q
+++ mktemp
++ local LAST_ERR=/tmp/tmp.YHcLdBwFU7
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb monitoring-pmm3 -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].nonvoting.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.6AbdfrtN1q
++ cat /tmp/tmp.YHcLdBwFU7
++ rm /tmp/tmp.6AbdfrtN1q /tmp/tmp.YHcLdBwFU7
++ return 0
+ [[ '' == true ]]
++ kubectl_bin get psmdb monitoring-pmm3 -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].hidden.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.hiv1JqU26o
+++ mktemp
++ local LAST_ERR=/tmp/tmp.su7wIYvEtQ
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb monitoring-pmm3 -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].hidden.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.hiv1JqU26o
++ cat /tmp/tmp.su7wIYvEtQ
++ rm /tmp/tmp.hiv1JqU26o /tmp/tmp.su7wIYvEtQ
++ return 0
+ [[ '' == true ]]
+ sleep 10
+ [[ true == true ]]
+ set +x
Waiting for cluster monitoring-pmm3 to be ready......................
+ desc 'check if pmm-client container is not enabled'
+ set +o xtrace


-----------------------------------------------------------------------------------
check if pmm-client container is not enabled
-----------------------------------------------------------------------------------

+ compare_kubectl statefulset/monitoring-pmm3-rs0 -no-pmm
+ local resource=statefulset/monitoring-pmm3-rs0
+ local postfix=-no-pmm
+ local skip_generation_check=
+ local expected_result=/mnt/jenkins/workspace/cloud-psmdb-operator_PR-2412/e2e-tests/monitoring-pmm3/compare/statefulset_monitoring-pmm3-rs0-no-pmm.yml
+ local new_result=/tmp/tmp.5urvO7mwus/statefulset_monitoring-pmm3-rs0.yml
+ [[ -n '' ]]
+ kubectl_bin get -o yaml statefulset/monitoring-pmm3-rs0
+ yq eval $'\n\t\t\tdel(.metadata.ownerReferences[].apiVersion) |\n\t\t\tdel(.metadata.managedFields) |\n\t\t\tdel(.. | select(has("creationTimestamp")).creationTimestamp) |\n\t\t\tdel(.. | select(has("namespace")).namespace) |\n\t\t\tdel(.. | select(has("uid")).uid) |\n\t\t\tdel(.metadata.resourceVersion) |\n\t\t\tdel(.spec.template.spec.containers[].env[] | select(.name == "NAMESPACE")) |\n\t\t\tdel(.metadata.selfLink) |\n\t\t\tdel(.metadata.annotations."cloud.google.com/neg") |\n\t\t\tdel(.metadata.annotations."field.cattle.io/publicEndpoints") |\n\t\t\tdel(.metadata.annotations."metallb.io/ip-allocated-from-pool") |\n\t\t\tdel(.metadata.annotations."kubectl.kubernetes.io/last-applied-configuration") |\n\t\t\tdel(.. | select(has("image")).image) |\n\t\t\tdel(.. | select(has("clusterIP")).clusterIP) |\n\t\t\tdel(.. | select(has("clusterIPs")).clusterIPs) |\n\t\t\tdel(.. | select(has("dataSource")).dataSource) |\n\t\t\tdel(.. | select(has("procMount")).procMount) |\n\t\t\tdel(.. | select(has("storageClassName")).storageClassName) |\n\t\t\tdel(.. | select(has("finalizers")).finalizers) |\n\t\t\tdel(.. | select(has("kubernetes.io/pvc-protection"))."kubernetes.io/pvc-protection") |\n\t\t\tdel(.. | select(has("volumeName")).volumeName) |\n\t\t\tdel(.. | select(has("volume.beta.kubernetes.io/storage-provisioner"))."volume.beta.kubernetes.io/storage-provisioner") |\n\t\t\tdel(.. | select(has("volume.kubernetes.io/storage-provisioner"))."volume.kubernetes.io/storage-provisioner") |\n\t\t\tdel(.spec.volumeMode) |\n\t\t\tdel(.. | select(has("volume.kubernetes.io/selected-node"))."volume.kubernetes.io/selected-node") |\n\t\t\tdel(.. | select(has("percona.com/last-config-hash"))."percona.com/last-config-hash") |\n\t\t\tdel(.. | select(has("percona.com/configuration-hash"))."percona.com/configuration-hash") |\n\t\t\tdel(.. | select(has("percona.com/ssl-hash"))."percona.com/ssl-hash") |\n\t\t\tdel(.. | select(has("percona.com/ssl-internal-hash"))."percona.com/ssl-internal-hash") |\n\t\t\tdel(.spec.volumeClaimTemplates[].spec.volumeMode | select(. == "Filesystem")) |\n\t\t\tdel(.. | select(has("healthCheckNodePort")).healthCheckNodePort) |\n\t\t\tdel(.. | select(has("nodePort")).nodePort) |\n\t\t\tdel(.status) |\n\t\t\t(.. | select(tag == "!!str")) |= sub("monitoring-pmm3-11763", "NAME_SPACE") |\n\t\t\tdel(.spec.volumeClaimTemplates[].apiVersion) |\n\t\t\tdel(.spec.volumeClaimTemplates[].kind) |\n\t\t\tdel(.spec.ipFamilies) |\n\t\t\tdel(.spec.ipFamilyPolicy) |\n\t\t\t(.. | select(. == "extensions/v1beta1")) = "apps/v1" |\n\t\t\t(.. | select(. == "batch/v1beta1")) = "batch/v1" ' -
++ mktemp
+ local LAST_OUT=/tmp/tmp.09aw1jQEvL
++ mktemp
+ local LAST_ERR=/tmp/tmp.6TH5ZitC0F
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl get -o yaml statefulset/monitoring-pmm3-rs0
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.09aw1jQEvL
+ cat /tmp/tmp.6TH5ZitC0F
+ rm /tmp/tmp.09aw1jQEvL /tmp/tmp.6TH5ZitC0F
+ return 0
+ yq -i eval 'del(.spec.persistentVolumeClaimRetentionPolicy)' /tmp/tmp.5urvO7mwus/statefulset_monitoring-pmm3-rs0.yml
+ version_gt 1.22
++ echo '1.33 >= 1.22'
++ bc -l
+ '[' 1 -eq 1 ']'
+ return 0
+ yq -i eval 'del(.spec.internalTrafficPolicy)' /tmp/tmp.5urvO7mwus/statefulset_monitoring-pmm3-rs0.yml
+ yq -i eval 'del(.spec.allocateLoadBalancerNodePorts)' /tmp/tmp.5urvO7mwus/statefulset_monitoring-pmm3-rs0.yml
+ [[ /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2412/e2e-tests/monitoring-pmm3/compare/statefulset_monitoring-pmm3-rs0-no-pmm.yml == */cronjob* ]]
+ '[' -n '' ']'
+ [[ 0 -eq 0 ]]
+ diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2412/e2e-tests/monitoring-pmm3/compare/statefulset_monitoring-pmm3-rs0-no-pmm.yml /tmp/tmp.5urvO7mwus/statefulset_monitoring-pmm3-rs0.yml
+ log 'compare_kubectl: statefulset/monitoring-pmm3-rs0 OK'
+ set +o xtrace
[2026-07-07T19:43:27+0000] compare_kubectl: statefulset/monitoring-pmm3-rs0 OK
+ sleep 10
+ custom_port=27019
+ run_mongos 'db.createUser({user:"myApp",pwd:"myPass",roles:[{db:"myApp",role:"readWrite"}]})' userAdmin:userAdmin123456@monitoring-pmm3-mongos.monitoring-pmm3-11763 '' '' '--tlsCertificateKeyFile /tmp/tls.pem --tlsCAFile /etc/mongodb-ssl/ca.crt --tls' 27019
+ local 'command=db.createUser({user:"myApp",pwd:"myPass",roles:[{db:"myApp",role:"readWrite"}]})'
+ local uri=userAdmin:userAdmin123456@monitoring-pmm3-mongos.monitoring-pmm3-11763
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ local 'mongo_flag=--tlsCertificateKeyFile /tmp/tls.pem --tlsCAFile /etc/mongodb-ssl/ca.crt --tls'
+ local port=27019
+ local mongo_bin=mongo
++ echo .svc.cluster.local
++ awk -F: '{print $2}'
+ suffix_port=
+ [[ -z '' ]]
+ suffix=.svc.cluster.local:27019
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.Nfkgp1wfdT
+++ mktemp
++ local LAST_ERR=/tmp/tmp.EAkiUmpJe7
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.Nfkgp1wfdT
++ cat /tmp/tmp.EAkiUmpJe7
++ rm /tmp/tmp.Nfkgp1wfdT /tmp/tmp.EAkiUmpJe7
++ return 0
+ local client_container=psmdb-client-7c8f77fb65-rvfv7
+ kubectl_bin exec psmdb-client-7c8f77fb65-rvfv7 -- bash -c 'printf '\''db.createUser({user:"myApp",pwd:"myPass",roles:[{db:"myApp",role:"readWrite"}]})\n'\'' | mongo mongodb://userAdmin:userAdmin123456@monitoring-pmm3-mongos.monitoring-pmm3-11763.svc.cluster.local:27019/admin --tlsCertificateKeyFile /tmp/tls.pem --tlsCAFile /etc/mongodb-ssl/ca.crt --tls'
++ mktemp
+ local LAST_OUT=/tmp/tmp.nx35XM9S18
++ mktemp
+ local LAST_ERR=/tmp/tmp.X3TegHWMiH
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-7c8f77fb65-rvfv7 -- bash -c 'printf '\''db.createUser({user:"myApp",pwd:"myPass",roles:[{db:"myApp",role:"readWrite"}]})\n'\'' | mongo mongodb://userAdmin:userAdmin123456@monitoring-pmm3-mongos.monitoring-pmm3-11763.svc.cluster.local:27019/admin --tlsCertificateKeyFile /tmp/tls.pem --tlsCAFile /etc/mongodb-ssl/ca.crt --tls'
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.nx35XM9S18
Percona Server for MongoDB shell version v4.4.29-28
connecting to: mongodb://monitoring-pmm3-mongos.monitoring-pmm3-11763.svc.cluster.local:27019/admin?compressors=disabled&gssapiServiceName=mongodb
{"t":{"$date":"2026-07-07T19:43:40.425Z"},"s":"I",  "c":"NETWORK",  "id":5490002, "ctx":"thread1","msg":"Started a new thread for the timer service"}
Implicit session: session { "id" : UUID("4e2c0071-6692-4740-ba27-afbde1367988") }
Percona Server for MongoDB server version: v8.0.26-11
WARNING: shell and server versions do not match
Successfully added user: {
	"user" : "myApp",
	"roles" : [
		{
			"db" : "myApp",
			"role" : "readWrite"
		}
	]
}
bye
+ cat /tmp/tmp.X3TegHWMiH
+ rm /tmp/tmp.nx35XM9S18 /tmp/tmp.X3TegHWMiH
+ return 0
+ run_mongos 'sh.enableSharding("myApp")' clusterAdmin:clusterAdmin123456@monitoring-pmm3-mongos.monitoring-pmm3-11763 '' '' '--tlsCertificateKeyFile /tmp/tls.pem --tlsCAFile /etc/mongodb-ssl/ca.crt --tls' 27019
+ local 'command=sh.enableSharding("myApp")'
+ local uri=clusterAdmin:clusterAdmin123456@monitoring-pmm3-mongos.monitoring-pmm3-11763
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ local 'mongo_flag=--tlsCertificateKeyFile /tmp/tls.pem --tlsCAFile /etc/mongodb-ssl/ca.crt --tls'
+ local port=27019
+ local mongo_bin=mongo
++ echo .svc.cluster.local
++ awk -F: '{print $2}'
+ suffix_port=
+ [[ -z '' ]]
+ suffix=.svc.cluster.local:27019
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.fCfygbesgP
+++ mktemp
++ local LAST_ERR=/tmp/tmp.Qzj4qhsfsf
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.fCfygbesgP
++ cat /tmp/tmp.Qzj4qhsfsf
++ rm /tmp/tmp.fCfygbesgP /tmp/tmp.Qzj4qhsfsf
++ return 0
+ local client_container=psmdb-client-7c8f77fb65-rvfv7
+ kubectl_bin exec psmdb-client-7c8f77fb65-rvfv7 -- bash -c 'printf '\''sh.enableSharding("myApp")\n'\'' | mongo mongodb://clusterAdmin:clusterAdmin123456@monitoring-pmm3-mongos.monitoring-pmm3-11763.svc.cluster.local:27019/admin --tlsCertificateKeyFile /tmp/tls.pem --tlsCAFile /etc/mongodb-ssl/ca.crt --tls'
++ mktemp
+ local LAST_OUT=/tmp/tmp.sB9U4T7UPO
++ mktemp
+ local LAST_ERR=/tmp/tmp.Go7zjPrKcf
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-7c8f77fb65-rvfv7 -- bash -c 'printf '\''sh.enableSharding("myApp")\n'\'' | mongo mongodb://clusterAdmin:clusterAdmin123456@monitoring-pmm3-mongos.monitoring-pmm3-11763.svc.cluster.local:27019/admin --tlsCertificateKeyFile /tmp/tls.pem --tlsCAFile /etc/mongodb-ssl/ca.crt --tls'
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.sB9U4T7UPO
Percona Server for MongoDB shell version v4.4.29-28
connecting to: mongodb://monitoring-pmm3-mongos.monitoring-pmm3-11763.svc.cluster.local:27019/admin?compressors=disabled&gssapiServiceName=mongodb
{"t":{"$date":"2026-07-07T19:43:43.382Z"},"s":"I",  "c":"NETWORK",  "id":5490002, "ctx":"thread1","msg":"Started a new thread for the timer service"}
Implicit session: session { "id" : UUID("ba6145ae-60ae-486b-91ca-d439acbc17dc") }
Percona Server for MongoDB server version: v8.0.26-11
WARNING: shell and server versions do not match
{
	"ok" : 1,
	"$clusterTime" : {
		"clusterTime" : Timestamp(1783453423, 9),
		"signature" : {
			"hash" : BinData(0,"Ufy2/YeZu9b/NLL1zZuP4fJ3bLU="),
			"keyId" : NumberLong("7659873769241968664")
		}
	},
	"operationTime" : Timestamp(1783453423, 6)
}
bye
+ cat /tmp/tmp.Go7zjPrKcf
+ rm /tmp/tmp.sB9U4T7UPO /tmp/tmp.Go7zjPrKcf
+ return 0
+ insert_data_mongos 100500 myApp '--tlsCertificateKeyFile /tmp/tls.pem --tlsCAFile /etc/mongodb-ssl/ca.crt --tls' 27019
+ local data=100500
+ local db_name=myApp
+ local 'flags=--tlsCertificateKeyFile /tmp/tls.pem --tlsCAFile /etc/mongodb-ssl/ca.crt --tls'
+ local port=27019
+ run_mongos 'use myApp\n db.test.insert({ x: 100500 })' myApp:myPass@monitoring-pmm3-mongos.monitoring-pmm3-11763 '' '' '--tlsCertificateKeyFile /tmp/tls.pem --tlsCAFile /etc/mongodb-ssl/ca.crt --tls' 27019
+ local 'command=use myApp\n db.test.insert({ x: 100500 })'
+ local uri=myApp:myPass@monitoring-pmm3-mongos.monitoring-pmm3-11763
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ local 'mongo_flag=--tlsCertificateKeyFile /tmp/tls.pem --tlsCAFile /etc/mongodb-ssl/ca.crt --tls'
+ local port=27019
+ local mongo_bin=mongo
++ echo .svc.cluster.local
++ awk -F: '{print $2}'
+ suffix_port=
+ [[ -z '' ]]
+ suffix=.svc.cluster.local:27019
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.P9H0ZiN4Hh
+++ mktemp
++ local LAST_ERR=/tmp/tmp.oJjoZH1Jvw
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.P9H0ZiN4Hh
++ cat /tmp/tmp.oJjoZH1Jvw
++ rm /tmp/tmp.P9H0ZiN4Hh /tmp/tmp.oJjoZH1Jvw
++ return 0
+ local client_container=psmdb-client-7c8f77fb65-rvfv7
+ kubectl_bin exec psmdb-client-7c8f77fb65-rvfv7 -- bash -c 'printf '\''use myApp\n db.test.insert({ x: 100500 })\n'\'' | mongo mongodb://myApp:myPass@monitoring-pmm3-mongos.monitoring-pmm3-11763.svc.cluster.local:27019/admin --tlsCertificateKeyFile /tmp/tls.pem --tlsCAFile /etc/mongodb-ssl/ca.crt --tls'
++ mktemp
+ local LAST_OUT=/tmp/tmp.uyX8KousUi
++ mktemp
+ local LAST_ERR=/tmp/tmp.Qdl5FZZezn
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-7c8f77fb65-rvfv7 -- bash -c 'printf '\''use myApp\n db.test.insert({ x: 100500 })\n'\'' | mongo mongodb://myApp:myPass@monitoring-pmm3-mongos.monitoring-pmm3-11763.svc.cluster.local:27019/admin --tlsCertificateKeyFile /tmp/tls.pem --tlsCAFile /etc/mongodb-ssl/ca.crt --tls'
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.uyX8KousUi
Percona Server for MongoDB shell version v4.4.29-28
connecting to: mongodb://monitoring-pmm3-mongos.monitoring-pmm3-11763.svc.cluster.local:27019/admin?compressors=disabled&gssapiServiceName=mongodb
{"t":{"$date":"2026-07-07T19:43:46.185Z"},"s":"I",  "c":"NETWORK",  "id":5490002, "ctx":"thread1","msg":"Started a new thread for the timer service"}
Implicit session: session { "id" : UUID("03deba74-920f-4ef2-be62-c11fb5d36058") }
Percona Server for MongoDB server version: v8.0.26-11
WARNING: shell and server versions do not match
switched to db myApp
WriteResult({ "nInserted" : 1 })
bye
+ cat /tmp/tmp.Qdl5FZZezn
+ rm /tmp/tmp.uyX8KousUi /tmp/tmp.Qdl5FZZezn
+ return 0
+ insert_data_mongos 100600 myApp '--tlsCertificateKeyFile /tmp/tls.pem --tlsCAFile /etc/mongodb-ssl/ca.crt --tls' 27019
+ local data=100600
+ local db_name=myApp
+ local 'flags=--tlsCertificateKeyFile /tmp/tls.pem --tlsCAFile /etc/mongodb-ssl/ca.crt --tls'
+ local port=27019
+ run_mongos 'use myApp\n db.test.insert({ x: 100600 })' myApp:myPass@monitoring-pmm3-mongos.monitoring-pmm3-11763 '' '' '--tlsCertificateKeyFile /tmp/tls.pem --tlsCAFile /etc/mongodb-ssl/ca.crt --tls' 27019
+ local 'command=use myApp\n db.test.insert({ x: 100600 })'
+ local uri=myApp:myPass@monitoring-pmm3-mongos.monitoring-pmm3-11763
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ local 'mongo_flag=--tlsCertificateKeyFile /tmp/tls.pem --tlsCAFile /etc/mongodb-ssl/ca.crt --tls'
+ local port=27019
+ local mongo_bin=mongo
++ echo .svc.cluster.local
++ awk -F: '{print $2}'
+ suffix_port=
+ [[ -z '' ]]
+ suffix=.svc.cluster.local:27019
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.IcrY0QONXv
+++ mktemp
++ local LAST_ERR=/tmp/tmp.QRfmP2kPV8
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.IcrY0QONXv
++ cat /tmp/tmp.QRfmP2kPV8
++ rm /tmp/tmp.IcrY0QONXv /tmp/tmp.QRfmP2kPV8
++ return 0
+ local client_container=psmdb-client-7c8f77fb65-rvfv7
+ kubectl_bin exec psmdb-client-7c8f77fb65-rvfv7 -- bash -c 'printf '\''use myApp\n db.test.insert({ x: 100600 })\n'\'' | mongo mongodb://myApp:myPass@monitoring-pmm3-mongos.monitoring-pmm3-11763.svc.cluster.local:27019/admin --tlsCertificateKeyFile /tmp/tls.pem --tlsCAFile /etc/mongodb-ssl/ca.crt --tls'
++ mktemp
+ local LAST_OUT=/tmp/tmp.Q30r4FtoyO
++ mktemp
+ local LAST_ERR=/tmp/tmp.w4vjfrETnq
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-7c8f77fb65-rvfv7 -- bash -c 'printf '\''use myApp\n db.test.insert({ x: 100600 })\n'\'' | mongo mongodb://myApp:myPass@monitoring-pmm3-mongos.monitoring-pmm3-11763.svc.cluster.local:27019/admin --tlsCertificateKeyFile /tmp/tls.pem --tlsCAFile /etc/mongodb-ssl/ca.crt --tls'
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.Q30r4FtoyO
Percona Server for MongoDB shell version v4.4.29-28
connecting to: mongodb://monitoring-pmm3-mongos.monitoring-pmm3-11763.svc.cluster.local:27019/admin?compressors=disabled&gssapiServiceName=mongodb
{"t":{"$date":"2026-07-07T19:43:49.149Z"},"s":"I",  "c":"NETWORK",  "id":5490002, "ctx":"thread1","msg":"Started a new thread for the timer service"}
Implicit session: session { "id" : UUID("5b2980f5-6044-430a-a4c0-20cbc17f86a0") }
Percona Server for MongoDB server version: v8.0.26-11
WARNING: shell and server versions do not match
switched to db myApp
WriteResult({ "nInserted" : 1 })
bye
+ cat /tmp/tmp.w4vjfrETnq
+ rm /tmp/tmp.Q30r4FtoyO /tmp/tmp.w4vjfrETnq
+ return 0
+ insert_data_mongos 100700 myApp '--tlsCertificateKeyFile /tmp/tls.pem --tlsCAFile /etc/mongodb-ssl/ca.crt --tls' 27019
+ local data=100700
+ local db_name=myApp
+ local 'flags=--tlsCertificateKeyFile /tmp/tls.pem --tlsCAFile /etc/mongodb-ssl/ca.crt --tls'
+ local port=27019
+ run_mongos 'use myApp\n db.test.insert({ x: 100700 })' myApp:myPass@monitoring-pmm3-mongos.monitoring-pmm3-11763 '' '' '--tlsCertificateKeyFile /tmp/tls.pem --tlsCAFile /etc/mongodb-ssl/ca.crt --tls' 27019
+ local 'command=use myApp\n db.test.insert({ x: 100700 })'
+ local uri=myApp:myPass@monitoring-pmm3-mongos.monitoring-pmm3-11763
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ local 'mongo_flag=--tlsCertificateKeyFile /tmp/tls.pem --tlsCAFile /etc/mongodb-ssl/ca.crt --tls'
+ local port=27019
+ local mongo_bin=mongo
++ echo .svc.cluster.local
++ awk -F: '{print $2}'
+ suffix_port=
+ [[ -z '' ]]
+ suffix=.svc.cluster.local:27019
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.u28rcacQrZ
+++ mktemp
++ local LAST_ERR=/tmp/tmp.jdhYOw2Q1b
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.u28rcacQrZ
++ cat /tmp/tmp.jdhYOw2Q1b
++ rm /tmp/tmp.u28rcacQrZ /tmp/tmp.jdhYOw2Q1b
++ return 0
+ local client_container=psmdb-client-7c8f77fb65-rvfv7
+ kubectl_bin exec psmdb-client-7c8f77fb65-rvfv7 -- bash -c 'printf '\''use myApp\n db.test.insert({ x: 100700 })\n'\'' | mongo mongodb://myApp:myPass@monitoring-pmm3-mongos.monitoring-pmm3-11763.svc.cluster.local:27019/admin --tlsCertificateKeyFile /tmp/tls.pem --tlsCAFile /etc/mongodb-ssl/ca.crt --tls'
++ mktemp
+ local LAST_OUT=/tmp/tmp.BSziFJBuE3
++ mktemp
+ local LAST_ERR=/tmp/tmp.f4GxVtjnpK
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-7c8f77fb65-rvfv7 -- bash -c 'printf '\''use myApp\n db.test.insert({ x: 100700 })\n'\'' | mongo mongodb://myApp:myPass@monitoring-pmm3-mongos.monitoring-pmm3-11763.svc.cluster.local:27019/admin --tlsCertificateKeyFile /tmp/tls.pem --tlsCAFile /etc/mongodb-ssl/ca.crt --tls'
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.BSziFJBuE3
Percona Server for MongoDB shell version v4.4.29-28
connecting to: mongodb://monitoring-pmm3-mongos.monitoring-pmm3-11763.svc.cluster.local:27019/admin?compressors=disabled&gssapiServiceName=mongodb
{"t":{"$date":"2026-07-07T19:43:51.410Z"},"s":"I",  "c":"NETWORK",  "id":5490002, "ctx":"thread1","msg":"Started a new thread for the timer service"}
Implicit session: session { "id" : UUID("92f135e4-4c24-4be1-aa03-1e77c9939860") }
Percona Server for MongoDB server version: v8.0.26-11
WARNING: shell and server versions do not match
switched to db myApp
WriteResult({ "nInserted" : 1 })
bye
+ cat /tmp/tmp.f4GxVtjnpK
+ rm /tmp/tmp.BSziFJBuE3 /tmp/tmp.f4GxVtjnpK
+ return 0
+ desc 'add PMM3 token to secret'
+ set +o xtrace


-----------------------------------------------------------------------------------
add PMM3 token to secret
-----------------------------------------------------------------------------------

++ get_pmm_server_token operator
++ local key_name=operator
++ [[ -z operator ]]
++ local ADMIN_PASSWORD
+++ kubectl get secret pmm-secret -o 'jsonpath={.data.PMM_ADMIN_PASSWORD}'
+++ base64 --decode
++ ADMIN_PASSWORD='Os\2'\''.99QS0gs;]f'
++ [[ -z Os\2'.99QS0gs;]f ]]
++ local create_response create_status_code create_json_response
++++ get_service_endpoint monitoring-service
++++ local service=monitoring-service
+++++ kubectl_bin get service/monitoring-service -o json
++++++ mktemp
+++++ jq '.status.loadBalancer.ingress[].hostname'
+++++ sed -e 's/^"//; s/"$//;'
+++++ local LAST_OUT=/tmp/tmp.GV8G2t4lKX
++++++ mktemp
+++++ local LAST_ERR=/tmp/tmp.NgyvySRM0B
+++++ local exit_status=0
+++++ local timeout=4
++++++ seq 0 2
+++++ for i in $(seq 0 2)
+++++ set +e
+++++ kubectl get service/monitoring-service -o json
+++++ exit_status=0
+++++ set -e
+++++ '[' 0 '!=' 0 -a -n 1 ']'
+++++ break
+++++ cat /tmp/tmp.GV8G2t4lKX
+++++ cat /tmp/tmp.NgyvySRM0B
+++++ rm /tmp/tmp.GV8G2t4lKX /tmp/tmp.NgyvySRM0B
+++++ return 0
++++ local hostname=null
++++ '[' -n null -a null '!=' null ']'
+++++ kubectl_bin get service/monitoring-service -o json
+++++ jq '.status.loadBalancer.ingress[].ip'
++++++ mktemp
+++++ sed -e 's/^"//; s/"$//;'
+++++ local LAST_OUT=/tmp/tmp.t91Go4DAA1
++++++ mktemp
+++++ local LAST_ERR=/tmp/tmp.Un9SeopaIA
+++++ local exit_status=0
+++++ local timeout=4
++++++ seq 0 2
+++++ for i in $(seq 0 2)
+++++ set +e
+++++ kubectl get service/monitoring-service -o json
+++++ exit_status=0
+++++ set -e
+++++ '[' 0 '!=' 0 -a -n 1 ']'
+++++ break
+++++ cat /tmp/tmp.t91Go4DAA1
+++++ cat /tmp/tmp.Un9SeopaIA
+++++ rm /tmp/tmp.t91Go4DAA1 /tmp/tmp.Un9SeopaIA
+++++ return 0
++++ local ip=35.254.76.43
++++ '[' -n 35.254.76.43 -a 35.254.76.43 '!=' null ']'
++++ echo 35.254.76.43
++++ return
+++ execute_curl --insecure -s -X POST -H 'Content-Type: application/json' -H 'Accept: application/json' -d '{"name":"operator", "role":"Admin", "isDisabled":false}' --user 'admin:Os\2'\''.99QS0gs;]f' https://35.254.76.43/graph/api/serviceaccounts -w '\n%{http_code}'
+++ local name=curl-9756
+++ local image=docker.io/alpine/curl:latest
+++ kubectl_bin run curl-9756 -n monitoring-pmm3-11763 --restart=Never --image=docker.io/alpine/curl:latest --command -- sleep 3600
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.XObOk4CU6H
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.Ej8eAWDzqr
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in $(seq 0 2)
+++ set +e
+++ kubectl run curl-9756 -n monitoring-pmm3-11763 --restart=Never --image=docker.io/alpine/curl:latest --command -- sleep 3600
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.XObOk4CU6H
+++ cat /tmp/tmp.Ej8eAWDzqr
+++ rm /tmp/tmp.XObOk4CU6H /tmp/tmp.Ej8eAWDzqr
+++ return 0
+++ kubectl_bin wait pod/curl-9756 -n monitoring-pmm3-11763 --for=condition=Ready --timeout=60s
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.2Va6yD6ENg
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.2MCgvISzJR
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in $(seq 0 2)
+++ set +e
+++ kubectl wait pod/curl-9756 -n monitoring-pmm3-11763 --for=condition=Ready --timeout=60s
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.2Va6yD6ENg
+++ cat /tmp/tmp.2MCgvISzJR
+++ rm /tmp/tmp.2Va6yD6ENg /tmp/tmp.2MCgvISzJR
+++ return 0
+++ local exit_status=0
+++ kubectl_bin exec curl-9756 -n monitoring-pmm3-11763 -- curl --insecure -s -X POST -H 'Content-Type: application/json' -H 'Accept: application/json' -d '{"name":"operator", "role":"Admin", "isDisabled":false}' --user 'admin:Os\2'\''.99QS0gs;]f' https://35.254.76.43/graph/api/serviceaccounts -w '\n%{http_code}'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.DFVW4bEdSt
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.mzv1HOQO6s
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in $(seq 0 2)
+++ set +e
+++ kubectl exec curl-9756 -n monitoring-pmm3-11763 -- curl --insecure -s -X POST -H 'Content-Type: application/json' -H 'Accept: application/json' -d '{"name":"operator", "role":"Admin", "isDisabled":false}' --user 'admin:Os\2'\''.99QS0gs;]f' https://35.254.76.43/graph/api/serviceaccounts -w '\n%{http_code}'
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.DFVW4bEdSt
+++ cat /tmp/tmp.mzv1HOQO6s
+++ rm /tmp/tmp.DFVW4bEdSt /tmp/tmp.mzv1HOQO6s
+++ return 0
+++ kubectl_bin delete pod curl-9756 -n monitoring-pmm3-11763 --ignore-not-found --wait=false
+++ return 0
++ create_response=$'{"id":2,"uid":"bfrf5azyzzugwa","name":"operator","login":"sa-1-operator","orgId":1,"isDisabled":false,"role":"Admin","tokens":0,"avatarUrl":""}\n201'
+++ echo $'{"id":2,"uid":"bfrf5azyzzugwa","name":"operator","login":"sa-1-operator","orgId":1,"isDisabled":false,"role":"Admin","tokens":0,"avatarUrl":""}\n201'
+++ tail -n1
++ create_status_code=201
+++ echo $'{"id":2,"uid":"bfrf5azyzzugwa","name":"operator","login":"sa-1-operator","orgId":1,"isDisabled":false,"role":"Admin","tokens":0,"avatarUrl":""}\n201'
+++ sed '$ d'
++ create_json_response='{"id":2,"uid":"bfrf5azyzzugwa","name":"operator","login":"sa-1-operator","orgId":1,"isDisabled":false,"role":"Admin","tokens":0,"avatarUrl":""}'
++ [[ 201 -ne 201 ]]
++ local service_account_id
+++ echo '{"id":2,"uid":"bfrf5azyzzugwa","name":"operator","login":"sa-1-operator","orgId":1,"isDisabled":false,"role":"Admin","tokens":0,"avatarUrl":""}'
+++ jq -r .id
++ service_account_id=2
++ [[ -z 2 ]]
++ [[ 2 == null ]]
++ local token_response token_status_code token_json_response
++++ get_service_endpoint monitoring-service
++++ local service=monitoring-service
+++++ kubectl_bin get service/monitoring-service -o json
+++++ jq '.status.loadBalancer.ingress[].hostname'
++++++ mktemp
+++++ sed -e 's/^"//; s/"$//;'
+++++ local LAST_OUT=/tmp/tmp.sRAbcuJKXo
++++++ mktemp
+++++ local LAST_ERR=/tmp/tmp.x9KxM3NkGI
+++++ local exit_status=0
+++++ local timeout=4
++++++ seq 0 2
+++++ for i in $(seq 0 2)
+++++ set +e
+++++ kubectl get service/monitoring-service -o json
+++++ exit_status=0
+++++ set -e
+++++ '[' 0 '!=' 0 -a -n 1 ']'
+++++ break
+++++ cat /tmp/tmp.sRAbcuJKXo
+++++ cat /tmp/tmp.x9KxM3NkGI
+++++ rm /tmp/tmp.sRAbcuJKXo /tmp/tmp.x9KxM3NkGI
+++++ return 0
++++ local hostname=null
++++ '[' -n null -a null '!=' null ']'
+++++ kubectl_bin get service/monitoring-service -o json
+++++ jq '.status.loadBalancer.ingress[].ip'
++++++ mktemp
+++++ sed -e 's/^"//; s/"$//;'
+++++ local LAST_OUT=/tmp/tmp.520flGezLP
++++++ mktemp
+++++ local LAST_ERR=/tmp/tmp.RI1C7GlPeq
+++++ local exit_status=0
+++++ local timeout=4
++++++ seq 0 2
+++++ for i in $(seq 0 2)
+++++ set +e
+++++ kubectl get service/monitoring-service -o json
+++++ exit_status=0
+++++ set -e
+++++ '[' 0 '!=' 0 -a -n 1 ']'
+++++ break
+++++ cat /tmp/tmp.520flGezLP
+++++ cat /tmp/tmp.RI1C7GlPeq
+++++ rm /tmp/tmp.520flGezLP /tmp/tmp.RI1C7GlPeq
+++++ return 0
++++ local ip=35.254.76.43
++++ '[' -n 35.254.76.43 -a 35.254.76.43 '!=' null ']'
++++ echo 35.254.76.43
++++ return
+++ execute_curl --insecure -s -X POST -H 'Content-Type: application/json' -d '{"name":"operator"}' --user 'admin:Os\2'\''.99QS0gs;]f' https://35.254.76.43/graph/api/serviceaccounts/2/tokens -w '\n%{http_code}'
+++ local name=curl-410
+++ local image=docker.io/alpine/curl:latest
+++ kubectl_bin run curl-410 -n monitoring-pmm3-11763 --restart=Never --image=docker.io/alpine/curl:latest --command -- sleep 3600
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.WOIxkXCzwo
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.HTWzKNsQAv
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in $(seq 0 2)
+++ set +e
+++ kubectl run curl-410 -n monitoring-pmm3-11763 --restart=Never --image=docker.io/alpine/curl:latest --command -- sleep 3600
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.WOIxkXCzwo
+++ cat /tmp/tmp.HTWzKNsQAv
+++ rm /tmp/tmp.WOIxkXCzwo /tmp/tmp.HTWzKNsQAv
+++ return 0
+++ kubectl_bin wait pod/curl-410 -n monitoring-pmm3-11763 --for=condition=Ready --timeout=60s
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.RB75Lzog5l
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.0ld3cKe9q2
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in $(seq 0 2)
+++ set +e
+++ kubectl wait pod/curl-410 -n monitoring-pmm3-11763 --for=condition=Ready --timeout=60s
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.RB75Lzog5l
+++ cat /tmp/tmp.0ld3cKe9q2
+++ rm /tmp/tmp.RB75Lzog5l /tmp/tmp.0ld3cKe9q2
+++ return 0
+++ local exit_status=0
+++ kubectl_bin exec curl-410 -n monitoring-pmm3-11763 -- curl --insecure -s -X POST -H 'Content-Type: application/json' -d '{"name":"operator"}' --user 'admin:Os\2'\''.99QS0gs;]f' https://35.254.76.43/graph/api/serviceaccounts/2/tokens -w '\n%{http_code}'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.IiQJNuApgr
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.Cu30bAvwWU
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in $(seq 0 2)
+++ set +e
+++ kubectl exec curl-410 -n monitoring-pmm3-11763 -- curl --insecure -s -X POST -H 'Content-Type: application/json' -d '{"name":"operator"}' --user 'admin:Os\2'\''.99QS0gs;]f' https://35.254.76.43/graph/api/serviceaccounts/2/tokens -w '\n%{http_code}'
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.IiQJNuApgr
+++ cat /tmp/tmp.Cu30bAvwWU
+++ rm /tmp/tmp.IiQJNuApgr /tmp/tmp.Cu30bAvwWU
+++ return 0
+++ kubectl_bin delete pod curl-410 -n monitoring-pmm3-11763 --ignore-not-found --wait=false
+++ return 0
++ token_response=$'{"id":1,"name":"operator","key":"glsa_UHnQmMoBhzBlyasS1sLaprD33Abk6a5t_4aa3a1e6"}\n200'
+++ echo $'{"id":1,"name":"operator","key":"glsa_UHnQmMoBhzBlyasS1sLaprD33Abk6a5t_4aa3a1e6"}\n200'
+++ tail -n1
++ token_status_code=200
+++ echo $'{"id":1,"name":"operator","key":"glsa_UHnQmMoBhzBlyasS1sLaprD33Abk6a5t_4aa3a1e6"}\n200'
+++ sed '$ d'
++ token_json_response='{"id":1,"name":"operator","key":"glsa_UHnQmMoBhzBlyasS1sLaprD33Abk6a5t_4aa3a1e6"}'
++ [[ 200 -ne 200 ]]
++ echo '{"id":1,"name":"operator","key":"glsa_UHnQmMoBhzBlyasS1sLaprD33Abk6a5t_4aa3a1e6"}'
++ jq -r .key
+ TOKEN=glsa_UHnQmMoBhzBlyasS1sLaprD33Abk6a5t_4aa3a1e6
+ kubectl_bin patch secret some-users --type merge --patch '{"stringData": {"PMM_SERVER_TOKEN": "glsa_UHnQmMoBhzBlyasS1sLaprD33Abk6a5t_4aa3a1e6"}}'
++ mktemp
+ local LAST_OUT=/tmp/tmp.P317MItt8y
++ mktemp
+ local LAST_ERR=/tmp/tmp.4wkGY5oTen
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl patch secret some-users --type merge --patch '{"stringData": {"PMM_SERVER_TOKEN": "glsa_UHnQmMoBhzBlyasS1sLaprD33Abk6a5t_4aa3a1e6"}}'
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.P317MItt8y
secret/some-users patched
+ cat /tmp/tmp.4wkGY5oTen
+ rm /tmp/tmp.P317MItt8y /tmp/tmp.4wkGY5oTen
+ return 0
+ desc 'check if all 3 Pods started'
+ set +o xtrace


-----------------------------------------------------------------------------------
check if all 3 Pods started
-----------------------------------------------------------------------------------

+ wait_for_running monitoring-pmm3-rs0 3
+ local name=monitoring-pmm3-rs0
+ let last_pod=2
+ local check_cluster_readiness=true
+ set_debug
+ [[ 1 == 1 ]]
+ set -o xtrace
+ local rs_name=rs0
+ local cluster_name=monitoring-pmm3
++ seq 0 2
+ for i in $(seq 0 $last_pod)
+ [[ 0 -eq 2 ]]
+ wait_pod monitoring-pmm3-rs0-0
+ local pod=monitoring-pmm3-rs0-0
+ set +o xtrace
waiting for pod/monitoring-pmm3-rs0-0 to be ready.OK
+ for i in $(seq 0 $last_pod)
+ [[ 1 -eq 2 ]]
+ wait_pod monitoring-pmm3-rs0-1
+ local pod=monitoring-pmm3-rs0-1
+ set +o xtrace
waiting for pod/monitoring-pmm3-rs0-1 to be ready.OK
+ for i in $(seq 0 $last_pod)
+ [[ 2 -eq 2 ]]
++ kubectl_bin get psmdb monitoring-pmm3 -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].arbiter.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.IAa9IaR6pI
+++ mktemp
++ local LAST_ERR=/tmp/tmp.bg7YJfXxIu
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb monitoring-pmm3 -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].arbiter.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.IAa9IaR6pI
++ cat /tmp/tmp.bg7YJfXxIu
++ rm /tmp/tmp.IAa9IaR6pI /tmp/tmp.bg7YJfXxIu
++ return 0
+ [[ '' == true ]]
+ wait_pod monitoring-pmm3-rs0-2
+ local pod=monitoring-pmm3-rs0-2
+ set +o xtrace
waiting for pod/monitoring-pmm3-rs0-2 to be ready.OK
++ kubectl_bin get psmdb monitoring-pmm3 -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].nonvoting.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.r3ucU0jZrr
+++ mktemp
++ local LAST_ERR=/tmp/tmp.LlqrilPSv9
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb monitoring-pmm3 -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].nonvoting.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.r3ucU0jZrr
++ cat /tmp/tmp.LlqrilPSv9
++ rm /tmp/tmp.r3ucU0jZrr /tmp/tmp.LlqrilPSv9
++ return 0
+ [[ '' == true ]]
++ kubectl_bin get psmdb monitoring-pmm3 -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].hidden.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.4yugmeirJv
+++ mktemp
++ local LAST_ERR=/tmp/tmp.knEYTcvizX
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb monitoring-pmm3 -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].hidden.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.4yugmeirJv
++ cat /tmp/tmp.knEYTcvizX
++ rm /tmp/tmp.4yugmeirJv /tmp/tmp.knEYTcvizX
++ return 0
+ [[ '' == true ]]
+ sleep 10
+ [[ true == true ]]
+ set +x
Waiting for cluster monitoring-pmm3 to be ready.............................................................................................................................
+ sleep 90
+ desc 'check if pmm-client container enabled'
+ set +o xtrace


-----------------------------------------------------------------------------------
check if pmm-client container enabled
-----------------------------------------------------------------------------------

+ compare_kubectl statefulset/monitoring-pmm3-rs0
+ local resource=statefulset/monitoring-pmm3-rs0
+ local postfix=
+ local skip_generation_check=
+ local expected_result=/mnt/jenkins/workspace/cloud-psmdb-operator_PR-2412/e2e-tests/monitoring-pmm3/compare/statefulset_monitoring-pmm3-rs0.yml
+ local new_result=/tmp/tmp.5urvO7mwus/statefulset_monitoring-pmm3-rs0.yml
+ [[ -n '' ]]
+ kubectl_bin get -o yaml statefulset/monitoring-pmm3-rs0
+ yq eval $'\n\t\t\tdel(.metadata.ownerReferences[].apiVersion) |\n\t\t\tdel(.metadata.managedFields) |\n\t\t\tdel(.. | select(has("creationTimestamp")).creationTimestamp) |\n\t\t\tdel(.. | select(has("namespace")).namespace) |\n\t\t\tdel(.. | select(has("uid")).uid) |\n\t\t\tdel(.metadata.resourceVersion) |\n\t\t\tdel(.spec.template.spec.containers[].env[] | select(.name == "NAMESPACE")) |\n\t\t\tdel(.metadata.selfLink) |\n\t\t\tdel(.metadata.annotations."cloud.google.com/neg") |\n\t\t\tdel(.metadata.annotations."field.cattle.io/publicEndpoints") |\n\t\t\tdel(.metadata.annotations."metallb.io/ip-allocated-from-pool") |\n\t\t\tdel(.metadata.annotations."kubectl.kubernetes.io/last-applied-configuration") |\n\t\t\tdel(.. | select(has("image")).image) |\n\t\t\tdel(.. | select(has("clusterIP")).clusterIP) |\n\t\t\tdel(.. | select(has("clusterIPs")).clusterIPs) |\n\t\t\tdel(.. | select(has("dataSource")).dataSource) |\n\t\t\tdel(.. | select(has("procMount")).procMount) |\n\t\t\tdel(.. | select(has("storageClassName")).storageClassName) |\n\t\t\tdel(.. | select(has("finalizers")).finalizers) |\n\t\t\tdel(.. | select(has("kubernetes.io/pvc-protection"))."kubernetes.io/pvc-protection") |\n\t\t\tdel(.. | select(has("volumeName")).volumeName) |\n\t\t\tdel(.. | select(has("volume.beta.kubernetes.io/storage-provisioner"))."volume.beta.kubernetes.io/storage-provisioner") |\n\t\t\tdel(.. | select(has("volume.kubernetes.io/storage-provisioner"))."volume.kubernetes.io/storage-provisioner") |\n\t\t\tdel(.spec.volumeMode) |\n\t\t\tdel(.. | select(has("volume.kubernetes.io/selected-node"))."volume.kubernetes.io/selected-node") |\n\t\t\tdel(.. | select(has("percona.com/last-config-hash"))."percona.com/last-config-hash") |\n\t\t\tdel(.. | select(has("percona.com/configuration-hash"))."percona.com/configuration-hash") |\n\t\t\tdel(.. | select(has("percona.com/ssl-hash"))."percona.com/ssl-hash") |\n\t\t\tdel(.. | select(has("percona.com/ssl-internal-hash"))."percona.com/ssl-internal-hash") |\n\t\t\tdel(.spec.volumeClaimTemplates[].spec.volumeMode | select(. == "Filesystem")) |\n\t\t\tdel(.. | select(has("healthCheckNodePort")).healthCheckNodePort) |\n\t\t\tdel(.. | select(has("nodePort")).nodePort) |\n\t\t\tdel(.status) |\n\t\t\t(.. | select(tag == "!!str")) |= sub("monitoring-pmm3-11763", "NAME_SPACE") |\n\t\t\tdel(.spec.volumeClaimTemplates[].apiVersion) |\n\t\t\tdel(.spec.volumeClaimTemplates[].kind) |\n\t\t\tdel(.spec.ipFamilies) |\n\t\t\tdel(.spec.ipFamilyPolicy) |\n\t\t\t(.. | select(. == "extensions/v1beta1")) = "apps/v1" |\n\t\t\t(.. | select(. == "batch/v1beta1")) = "batch/v1" ' -
++ mktemp
+ local LAST_OUT=/tmp/tmp.wzOly8qFcI
++ mktemp
+ local LAST_ERR=/tmp/tmp.HBUwirsnd1
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl get -o yaml statefulset/monitoring-pmm3-rs0
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.wzOly8qFcI
+ cat /tmp/tmp.HBUwirsnd1
+ rm /tmp/tmp.wzOly8qFcI /tmp/tmp.HBUwirsnd1
+ return 0
+ yq -i eval 'del(.spec.persistentVolumeClaimRetentionPolicy)' /tmp/tmp.5urvO7mwus/statefulset_monitoring-pmm3-rs0.yml
+ version_gt 1.22
++ echo '1.33 >= 1.22'
++ bc -l
+ '[' 1 -eq 1 ']'
+ return 0
+ yq -i eval 'del(.spec.internalTrafficPolicy)' /tmp/tmp.5urvO7mwus/statefulset_monitoring-pmm3-rs0.yml
+ yq -i eval 'del(.spec.allocateLoadBalancerNodePorts)' /tmp/tmp.5urvO7mwus/statefulset_monitoring-pmm3-rs0.yml
+ [[ /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2412/e2e-tests/monitoring-pmm3/compare/statefulset_monitoring-pmm3-rs0.yml == */cronjob* ]]
+ '[' -n '' ']'
+ [[ 0 -eq 0 ]]
+ diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2412/e2e-tests/monitoring-pmm3/compare/statefulset_monitoring-pmm3-rs0.yml /tmp/tmp.5urvO7mwus/statefulset_monitoring-pmm3-rs0.yml
+ log 'compare_kubectl: statefulset/monitoring-pmm3-rs0 OK'
+ set +o xtrace
[2026-07-07T19:50:32+0000] compare_kubectl: statefulset/monitoring-pmm3-rs0 OK
+ compare_kubectl service/monitoring-pmm3-rs0
+ local resource=service/monitoring-pmm3-rs0
+ local postfix=
+ local skip_generation_check=
+ local expected_result=/mnt/jenkins/workspace/cloud-psmdb-operator_PR-2412/e2e-tests/monitoring-pmm3/compare/service_monitoring-pmm3-rs0.yml
+ local new_result=/tmp/tmp.5urvO7mwus/service_monitoring-pmm3-rs0.yml
+ [[ -n '' ]]
+ kubectl_bin get -o yaml service/monitoring-pmm3-rs0
+ yq eval $'\n\t\t\tdel(.metadata.ownerReferences[].apiVersion) |\n\t\t\tdel(.metadata.managedFields) |\n\t\t\tdel(.. | select(has("creationTimestamp")).creationTimestamp) |\n\t\t\tdel(.. | select(has("namespace")).namespace) |\n\t\t\tdel(.. | select(has("uid")).uid) |\n\t\t\tdel(.metadata.resourceVersion) |\n\t\t\tdel(.spec.template.spec.containers[].env[] | select(.name == "NAMESPACE")) |\n\t\t\tdel(.metadata.selfLink) |\n\t\t\tdel(.metadata.annotations."cloud.google.com/neg") |\n\t\t\tdel(.metadata.annotations."field.cattle.io/publicEndpoints") |\n\t\t\tdel(.metadata.annotations."metallb.io/ip-allocated-from-pool") |\n\t\t\tdel(.metadata.annotations."kubectl.kubernetes.io/last-applied-configuration") |\n\t\t\tdel(.. | select(has("image")).image) |\n\t\t\tdel(.. | select(has("clusterIP")).clusterIP) |\n\t\t\tdel(.. | select(has("clusterIPs")).clusterIPs) |\n\t\t\tdel(.. | select(has("dataSource")).dataSource) |\n\t\t\tdel(.. | select(has("procMount")).procMount) |\n\t\t\tdel(.. | select(has("storageClassName")).storageClassName) |\n\t\t\tdel(.. | select(has("finalizers")).finalizers) |\n\t\t\tdel(.. | select(has("kubernetes.io/pvc-protection"))."kubernetes.io/pvc-protection") |\n\t\t\tdel(.. | select(has("volumeName")).volumeName) |\n\t\t\tdel(.. | select(has("volume.beta.kubernetes.io/storage-provisioner"))."volume.beta.kubernetes.io/storage-provisioner") |\n\t\t\tdel(.. | select(has("volume.kubernetes.io/storage-provisioner"))."volume.kubernetes.io/storage-provisioner") |\n\t\t\tdel(.spec.volumeMode) |\n\t\t\tdel(.. | select(has("volume.kubernetes.io/selected-node"))."volume.kubernetes.io/selected-node") |\n\t\t\tdel(.. | select(has("percona.com/last-config-hash"))."percona.com/last-config-hash") |\n\t\t\tdel(.. | select(has("percona.com/configuration-hash"))."percona.com/configuration-hash") |\n\t\t\tdel(.. | select(has("percona.com/ssl-hash"))."percona.com/ssl-hash") |\n\t\t\tdel(.. | select(has("percona.com/ssl-internal-hash"))."percona.com/ssl-internal-hash") |\n\t\t\tdel(.spec.volumeClaimTemplates[].spec.volumeMode | select(. == "Filesystem")) |\n\t\t\tdel(.. | select(has("healthCheckNodePort")).healthCheckNodePort) |\n\t\t\tdel(.. | select(has("nodePort")).nodePort) |\n\t\t\tdel(.status) |\n\t\t\t(.. | select(tag == "!!str")) |= sub("monitoring-pmm3-11763", "NAME_SPACE") |\n\t\t\tdel(.spec.volumeClaimTemplates[].apiVersion) |\n\t\t\tdel(.spec.volumeClaimTemplates[].kind) |\n\t\t\tdel(.spec.ipFamilies) |\n\t\t\tdel(.spec.ipFamilyPolicy) |\n\t\t\t(.. | select(. == "extensions/v1beta1")) = "apps/v1" |\n\t\t\t(.. | select(. == "batch/v1beta1")) = "batch/v1" ' -
++ mktemp
+ local LAST_OUT=/tmp/tmp.jzGc1gGctd
++ mktemp
+ local LAST_ERR=/tmp/tmp.Kjin0govNp
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl get -o yaml service/monitoring-pmm3-rs0
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.jzGc1gGctd
+ cat /tmp/tmp.Kjin0govNp
+ rm /tmp/tmp.jzGc1gGctd /tmp/tmp.Kjin0govNp
+ return 0
+ yq -i eval 'del(.spec.persistentVolumeClaimRetentionPolicy)' /tmp/tmp.5urvO7mwus/service_monitoring-pmm3-rs0.yml
+ version_gt 1.22
++ echo '1.33 >= 1.22'
++ bc -l
+ '[' 1 -eq 1 ']'
+ return 0
+ yq -i eval 'del(.spec.internalTrafficPolicy)' /tmp/tmp.5urvO7mwus/service_monitoring-pmm3-rs0.yml
+ yq -i eval 'del(.spec.allocateLoadBalancerNodePorts)' /tmp/tmp.5urvO7mwus/service_monitoring-pmm3-rs0.yml
+ [[ /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2412/e2e-tests/monitoring-pmm3/compare/service_monitoring-pmm3-rs0.yml == */cronjob* ]]
+ '[' -n '' ']'
+ [[ 0 -eq 0 ]]
+ diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2412/e2e-tests/monitoring-pmm3/compare/service_monitoring-pmm3-rs0.yml /tmp/tmp.5urvO7mwus/service_monitoring-pmm3-rs0.yml
+ log 'compare_kubectl: service/monitoring-pmm3-rs0 OK'
+ set +o xtrace
[2026-07-07T19:50:33+0000] compare_kubectl: service/monitoring-pmm3-rs0 OK
+ compare_kubectl service/monitoring-pmm3-mongos
+ local resource=service/monitoring-pmm3-mongos
+ local postfix=
+ local skip_generation_check=
+ local expected_result=/mnt/jenkins/workspace/cloud-psmdb-operator_PR-2412/e2e-tests/monitoring-pmm3/compare/service_monitoring-pmm3-mongos.yml
+ local new_result=/tmp/tmp.5urvO7mwus/service_monitoring-pmm3-mongos.yml
+ [[ -n '' ]]
+ kubectl_bin get -o yaml service/monitoring-pmm3-mongos
++ mktemp
+ yq eval $'\n\t\t\tdel(.metadata.ownerReferences[].apiVersion) |\n\t\t\tdel(.metadata.managedFields) |\n\t\t\tdel(.. | select(has("creationTimestamp")).creationTimestamp) |\n\t\t\tdel(.. | select(has("namespace")).namespace) |\n\t\t\tdel(.. | select(has("uid")).uid) |\n\t\t\tdel(.metadata.resourceVersion) |\n\t\t\tdel(.spec.template.spec.containers[].env[] | select(.name == "NAMESPACE")) |\n\t\t\tdel(.metadata.selfLink) |\n\t\t\tdel(.metadata.annotations."cloud.google.com/neg") |\n\t\t\tdel(.metadata.annotations."field.cattle.io/publicEndpoints") |\n\t\t\tdel(.metadata.annotations."metallb.io/ip-allocated-from-pool") |\n\t\t\tdel(.metadata.annotations."kubectl.kubernetes.io/last-applied-configuration") |\n\t\t\tdel(.. | select(has("image")).image) |\n\t\t\tdel(.. | select(has("clusterIP")).clusterIP) |\n\t\t\tdel(.. | select(has("clusterIPs")).clusterIPs) |\n\t\t\tdel(.. | select(has("dataSource")).dataSource) |\n\t\t\tdel(.. | select(has("procMount")).procMount) |\n\t\t\tdel(.. | select(has("storageClassName")).storageClassName) |\n\t\t\tdel(.. | select(has("finalizers")).finalizers) |\n\t\t\tdel(.. | select(has("kubernetes.io/pvc-protection"))."kubernetes.io/pvc-protection") |\n\t\t\tdel(.. | select(has("volumeName")).volumeName) |\n\t\t\tdel(.. | select(has("volume.beta.kubernetes.io/storage-provisioner"))."volume.beta.kubernetes.io/storage-provisioner") |\n\t\t\tdel(.. | select(has("volume.kubernetes.io/storage-provisioner"))."volume.kubernetes.io/storage-provisioner") |\n\t\t\tdel(.spec.volumeMode) |\n\t\t\tdel(.. | select(has("volume.kubernetes.io/selected-node"))."volume.kubernetes.io/selected-node") |\n\t\t\tdel(.. | select(has("percona.com/last-config-hash"))."percona.com/last-config-hash") |\n\t\t\tdel(.. | select(has("percona.com/configuration-hash"))."percona.com/configuration-hash") |\n\t\t\tdel(.. | select(has("percona.com/ssl-hash"))."percona.com/ssl-hash") |\n\t\t\tdel(.. | select(has("percona.com/ssl-internal-hash"))."percona.com/ssl-internal-hash") |\n\t\t\tdel(.spec.volumeClaimTemplates[].spec.volumeMode | select(. == "Filesystem")) |\n\t\t\tdel(.. | select(has("healthCheckNodePort")).healthCheckNodePort) |\n\t\t\tdel(.. | select(has("nodePort")).nodePort) |\n\t\t\tdel(.status) |\n\t\t\t(.. | select(tag == "!!str")) |= sub("monitoring-pmm3-11763", "NAME_SPACE") |\n\t\t\tdel(.spec.volumeClaimTemplates[].apiVersion) |\n\t\t\tdel(.spec.volumeClaimTemplates[].kind) |\n\t\t\tdel(.spec.ipFamilies) |\n\t\t\tdel(.spec.ipFamilyPolicy) |\n\t\t\t(.. | select(. == "extensions/v1beta1")) = "apps/v1" |\n\t\t\t(.. | select(. == "batch/v1beta1")) = "batch/v1" ' -
+ local LAST_OUT=/tmp/tmp.fCgRfhblyU
++ mktemp
+ local LAST_ERR=/tmp/tmp.O9NPmhm8Gc
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl get -o yaml service/monitoring-pmm3-mongos
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.fCgRfhblyU
+ cat /tmp/tmp.O9NPmhm8Gc
+ rm /tmp/tmp.fCgRfhblyU /tmp/tmp.O9NPmhm8Gc
+ return 0
+ yq -i eval 'del(.spec.persistentVolumeClaimRetentionPolicy)' /tmp/tmp.5urvO7mwus/service_monitoring-pmm3-mongos.yml
+ version_gt 1.22
++ echo '1.33 >= 1.22'
++ bc -l
+ '[' 1 -eq 1 ']'
+ return 0
+ yq -i eval 'del(.spec.internalTrafficPolicy)' /tmp/tmp.5urvO7mwus/service_monitoring-pmm3-mongos.yml
+ yq -i eval 'del(.spec.allocateLoadBalancerNodePorts)' /tmp/tmp.5urvO7mwus/service_monitoring-pmm3-mongos.yml
+ [[ /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2412/e2e-tests/monitoring-pmm3/compare/service_monitoring-pmm3-mongos.yml == */cronjob* ]]
+ '[' -n '' ']'
+ [[ 0 -eq 0 ]]
+ diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2412/e2e-tests/monitoring-pmm3/compare/service_monitoring-pmm3-mongos.yml /tmp/tmp.5urvO7mwus/service_monitoring-pmm3-mongos.yml
+ log 'compare_kubectl: service/monitoring-pmm3-mongos OK'
+ set +o xtrace
[2026-07-07T19:50:34+0000] compare_kubectl: service/monitoring-pmm3-mongos OK
+ compare_kubectl statefulset/monitoring-pmm3-cfg
+ local resource=statefulset/monitoring-pmm3-cfg
+ local postfix=
+ local skip_generation_check=
+ local expected_result=/mnt/jenkins/workspace/cloud-psmdb-operator_PR-2412/e2e-tests/monitoring-pmm3/compare/statefulset_monitoring-pmm3-cfg.yml
+ local new_result=/tmp/tmp.5urvO7mwus/statefulset_monitoring-pmm3-cfg.yml
+ [[ -n '' ]]
+ kubectl_bin get -o yaml statefulset/monitoring-pmm3-cfg
+ yq eval $'\n\t\t\tdel(.metadata.ownerReferences[].apiVersion) |\n\t\t\tdel(.metadata.managedFields) |\n\t\t\tdel(.. | select(has("creationTimestamp")).creationTimestamp) |\n\t\t\tdel(.. | select(has("namespace")).namespace) |\n\t\t\tdel(.. | select(has("uid")).uid) |\n\t\t\tdel(.metadata.resourceVersion) |\n\t\t\tdel(.spec.template.spec.containers[].env[] | select(.name == "NAMESPACE")) |\n\t\t\tdel(.metadata.selfLink) |\n\t\t\tdel(.metadata.annotations."cloud.google.com/neg") |\n\t\t\tdel(.metadata.annotations."field.cattle.io/publicEndpoints") |\n\t\t\tdel(.metadata.annotations."metallb.io/ip-allocated-from-pool") |\n\t\t\tdel(.metadata.annotations."kubectl.kubernetes.io/last-applied-configuration") |\n\t\t\tdel(.. | select(has("image")).image) |\n\t\t\tdel(.. | select(has("clusterIP")).clusterIP) |\n\t\t\tdel(.. | select(has("clusterIPs")).clusterIPs) |\n\t\t\tdel(.. | select(has("dataSource")).dataSource) |\n\t\t\tdel(.. | select(has("procMount")).procMount) |\n\t\t\tdel(.. | select(has("storageClassName")).storageClassName) |\n\t\t\tdel(.. | select(has("finalizers")).finalizers) |\n\t\t\tdel(.. | select(has("kubernetes.io/pvc-protection"))."kubernetes.io/pvc-protection") |\n\t\t\tdel(.. | select(has("volumeName")).volumeName) |\n\t\t\tdel(.. | select(has("volume.beta.kubernetes.io/storage-provisioner"))."volume.beta.kubernetes.io/storage-provisioner") |\n\t\t\tdel(.. | select(has("volume.kubernetes.io/storage-provisioner"))."volume.kubernetes.io/storage-provisioner") |\n\t\t\tdel(.spec.volumeMode) |\n\t\t\tdel(.. | select(has("volume.kubernetes.io/selected-node"))."volume.kubernetes.io/selected-node") |\n\t\t\tdel(.. | select(has("percona.com/last-config-hash"))."percona.com/last-config-hash") |\n\t\t\tdel(.. | select(has("percona.com/configuration-hash"))."percona.com/configuration-hash") |\n\t\t\tdel(.. | select(has("percona.com/ssl-hash"))."percona.com/ssl-hash") |\n\t\t\tdel(.. | select(has("percona.com/ssl-internal-hash"))."percona.com/ssl-internal-hash") |\n\t\t\tdel(.spec.volumeClaimTemplates[].spec.volumeMode | select(. == "Filesystem")) |\n\t\t\tdel(.. | select(has("healthCheckNodePort")).healthCheckNodePort) |\n\t\t\tdel(.. | select(has("nodePort")).nodePort) |\n\t\t\tdel(.status) |\n\t\t\t(.. | select(tag == "!!str")) |= sub("monitoring-pmm3-11763", "NAME_SPACE") |\n\t\t\tdel(.spec.volumeClaimTemplates[].apiVersion) |\n\t\t\tdel(.spec.volumeClaimTemplates[].kind) |\n\t\t\tdel(.spec.ipFamilies) |\n\t\t\tdel(.spec.ipFamilyPolicy) |\n\t\t\t(.. | select(. == "extensions/v1beta1")) = "apps/v1" |\n\t\t\t(.. | select(. == "batch/v1beta1")) = "batch/v1" ' -
++ mktemp
+ local LAST_OUT=/tmp/tmp.H9ahgaUHHJ
++ mktemp
+ local LAST_ERR=/tmp/tmp.hcyLpjvCBD
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl get -o yaml statefulset/monitoring-pmm3-cfg
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.H9ahgaUHHJ
+ cat /tmp/tmp.hcyLpjvCBD
+ rm /tmp/tmp.H9ahgaUHHJ /tmp/tmp.hcyLpjvCBD
+ return 0
+ yq -i eval 'del(.spec.persistentVolumeClaimRetentionPolicy)' /tmp/tmp.5urvO7mwus/statefulset_monitoring-pmm3-cfg.yml
+ version_gt 1.22
++ echo '1.33 >= 1.22'
++ bc -l
+ '[' 1 -eq 1 ']'
+ return 0
+ yq -i eval 'del(.spec.internalTrafficPolicy)' /tmp/tmp.5urvO7mwus/statefulset_monitoring-pmm3-cfg.yml
+ yq -i eval 'del(.spec.allocateLoadBalancerNodePorts)' /tmp/tmp.5urvO7mwus/statefulset_monitoring-pmm3-cfg.yml
+ [[ /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2412/e2e-tests/monitoring-pmm3/compare/statefulset_monitoring-pmm3-cfg.yml == */cronjob* ]]
+ '[' -n '' ']'
+ [[ 0 -eq 0 ]]
+ diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2412/e2e-tests/monitoring-pmm3/compare/statefulset_monitoring-pmm3-cfg.yml /tmp/tmp.5urvO7mwus/statefulset_monitoring-pmm3-cfg.yml
+ log 'compare_kubectl: statefulset/monitoring-pmm3-cfg OK'
+ set +o xtrace
[2026-07-07T19:50:35+0000] compare_kubectl: statefulset/monitoring-pmm3-cfg OK
+ compare_kubectl statefulset/monitoring-pmm3-mongos
+ local resource=statefulset/monitoring-pmm3-mongos
+ local postfix=
+ local skip_generation_check=
+ local expected_result=/mnt/jenkins/workspace/cloud-psmdb-operator_PR-2412/e2e-tests/monitoring-pmm3/compare/statefulset_monitoring-pmm3-mongos.yml
+ local new_result=/tmp/tmp.5urvO7mwus/statefulset_monitoring-pmm3-mongos.yml
+ [[ -n '' ]]
+ kubectl_bin get -o yaml statefulset/monitoring-pmm3-mongos
+ yq eval $'\n\t\t\tdel(.metadata.ownerReferences[].apiVersion) |\n\t\t\tdel(.metadata.managedFields) |\n\t\t\tdel(.. | select(has("creationTimestamp")).creationTimestamp) |\n\t\t\tdel(.. | select(has("namespace")).namespace) |\n\t\t\tdel(.. | select(has("uid")).uid) |\n\t\t\tdel(.metadata.resourceVersion) |\n\t\t\tdel(.spec.template.spec.containers[].env[] | select(.name == "NAMESPACE")) |\n\t\t\tdel(.metadata.selfLink) |\n\t\t\tdel(.metadata.annotations."cloud.google.com/neg") |\n\t\t\tdel(.metadata.annotations."field.cattle.io/publicEndpoints") |\n\t\t\tdel(.metadata.annotations."metallb.io/ip-allocated-from-pool") |\n\t\t\tdel(.metadata.annotations."kubectl.kubernetes.io/last-applied-configuration") |\n\t\t\tdel(.. | select(has("image")).image) |\n\t\t\tdel(.. | select(has("clusterIP")).clusterIP) |\n\t\t\tdel(.. | select(has("clusterIPs")).clusterIPs) |\n\t\t\tdel(.. | select(has("dataSource")).dataSource) |\n\t\t\tdel(.. | select(has("procMount")).procMount) |\n\t\t\tdel(.. | select(has("storageClassName")).storageClassName) |\n\t\t\tdel(.. | select(has("finalizers")).finalizers) |\n\t\t\tdel(.. | select(has("kubernetes.io/pvc-protection"))."kubernetes.io/pvc-protection") |\n\t\t\tdel(.. | select(has("volumeName")).volumeName) |\n\t\t\tdel(.. | select(has("volume.beta.kubernetes.io/storage-provisioner"))."volume.beta.kubernetes.io/storage-provisioner") |\n\t\t\tdel(.. | select(has("volume.kubernetes.io/storage-provisioner"))."volume.kubernetes.io/storage-provisioner") |\n\t\t\tdel(.spec.volumeMode) |\n\t\t\tdel(.. | select(has("volume.kubernetes.io/selected-node"))."volume.kubernetes.io/selected-node") |\n\t\t\tdel(.. | select(has("percona.com/last-config-hash"))."percona.com/last-config-hash") |\n\t\t\tdel(.. | select(has("percona.com/configuration-hash"))."percona.com/configuration-hash") |\n\t\t\tdel(.. | select(has("percona.com/ssl-hash"))."percona.com/ssl-hash") |\n\t\t\tdel(.. | select(has("percona.com/ssl-internal-hash"))."percona.com/ssl-internal-hash") |\n\t\t\tdel(.spec.volumeClaimTemplates[].spec.volumeMode | select(. == "Filesystem")) |\n\t\t\tdel(.. | select(has("healthCheckNodePort")).healthCheckNodePort) |\n\t\t\tdel(.. | select(has("nodePort")).nodePort) |\n\t\t\tdel(.status) |\n\t\t\t(.. | select(tag == "!!str")) |= sub("monitoring-pmm3-11763", "NAME_SPACE") |\n\t\t\tdel(.spec.volumeClaimTemplates[].apiVersion) |\n\t\t\tdel(.spec.volumeClaimTemplates[].kind) |\n\t\t\tdel(.spec.ipFamilies) |\n\t\t\tdel(.spec.ipFamilyPolicy) |\n\t\t\t(.. | select(. == "extensions/v1beta1")) = "apps/v1" |\n\t\t\t(.. | select(. == "batch/v1beta1")) = "batch/v1" ' -
++ mktemp
+ local LAST_OUT=/tmp/tmp.5wvsRiPiFn
++ mktemp
+ local LAST_ERR=/tmp/tmp.FIS104qVfF
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl get -o yaml statefulset/monitoring-pmm3-mongos
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.5wvsRiPiFn
+ cat /tmp/tmp.FIS104qVfF
+ rm /tmp/tmp.5wvsRiPiFn /tmp/tmp.FIS104qVfF
+ return 0
+ yq -i eval 'del(.spec.persistentVolumeClaimRetentionPolicy)' /tmp/tmp.5urvO7mwus/statefulset_monitoring-pmm3-mongos.yml
+ version_gt 1.22
++ echo '1.33 >= 1.22'
++ bc -l
+ '[' 1 -eq 1 ']'
+ return 0
+ yq -i eval 'del(.spec.internalTrafficPolicy)' /tmp/tmp.5urvO7mwus/statefulset_monitoring-pmm3-mongos.yml
+ yq -i eval 'del(.spec.allocateLoadBalancerNodePorts)' /tmp/tmp.5urvO7mwus/statefulset_monitoring-pmm3-mongos.yml
+ [[ /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2412/e2e-tests/monitoring-pmm3/compare/statefulset_monitoring-pmm3-mongos.yml == */cronjob* ]]
+ '[' -n '' ']'
+ [[ 0 -eq 0 ]]
+ diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2412/e2e-tests/monitoring-pmm3/compare/statefulset_monitoring-pmm3-mongos.yml /tmp/tmp.5urvO7mwus/statefulset_monitoring-pmm3-mongos.yml
+ log 'compare_kubectl: statefulset/monitoring-pmm3-mongos OK'
+ set +o xtrace
[2026-07-07T19:50:36+0000] compare_kubectl: statefulset/monitoring-pmm3-mongos OK
+ check_custom_pmm_probes
+ desc 'verify custom PMM liveness/readiness probes set via CR are applied'
+ set +o xtrace


-----------------------------------------------------------------------------------
verify custom PMM liveness/readiness probes set via CR are applied
-----------------------------------------------------------------------------------

+ kubectl_bin patch psmdb monitoring-pmm3 --type=json $'-p=[\n\t  {"op":"add","path":"/spec/pmm/livenessProbe","value":{"exec":{"command":["/bin/true"]},"initialDelaySeconds":30,"periodSeconds":15}},\n\t  {"op":"add","path":"/spec/pmm/readinessProbe","value":{"exec":{"command":["/bin/true"]},"initialDelaySeconds":5,"periodSeconds":10}}\n\t]'
++ mktemp
+ local LAST_OUT=/tmp/tmp.7bQqYYq8t5
++ mktemp
+ local LAST_ERR=/tmp/tmp.iPE87jlm9H
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl patch psmdb monitoring-pmm3 --type=json $'-p=[\n\t  {"op":"add","path":"/spec/pmm/livenessProbe","value":{"exec":{"command":["/bin/true"]},"initialDelaySeconds":30,"periodSeconds":15}},\n\t  {"op":"add","path":"/spec/pmm/readinessProbe","value":{"exec":{"command":["/bin/true"]},"initialDelaySeconds":5,"periodSeconds":10}}\n\t]'
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.7bQqYYq8t5
perconaservermongodb.psmdb.percona.com/monitoring-pmm3 patched
+ cat /tmp/tmp.iPE87jlm9H
+ rm /tmp/tmp.7bQqYYq8t5 /tmp/tmp.iPE87jlm9H
+ return 0
+ wait_for_running monitoring-pmm3-rs0 3
+ local name=monitoring-pmm3-rs0
+ let last_pod=2
+ local check_cluster_readiness=true
+ set_debug
+ [[ 1 == 1 ]]
+ set -o xtrace
+ local rs_name=rs0
+ local cluster_name=monitoring-pmm3
++ seq 0 2
+ for i in $(seq 0 $last_pod)
+ [[ 0 -eq 2 ]]
+ wait_pod monitoring-pmm3-rs0-0
+ local pod=monitoring-pmm3-rs0-0
+ set +o xtrace
waiting for pod/monitoring-pmm3-rs0-0 to be ready.OK
+ for i in $(seq 0 $last_pod)
+ [[ 1 -eq 2 ]]
+ wait_pod monitoring-pmm3-rs0-1
+ local pod=monitoring-pmm3-rs0-1
+ set +o xtrace
waiting for pod/monitoring-pmm3-rs0-1 to be ready.OK
+ for i in $(seq 0 $last_pod)
+ [[ 2 -eq 2 ]]
++ kubectl_bin get psmdb monitoring-pmm3 -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].arbiter.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.ZKOKHl0Fyk
+++ mktemp
++ local LAST_ERR=/tmp/tmp.VzJejtlx5K
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb monitoring-pmm3 -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].arbiter.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.ZKOKHl0Fyk
++ cat /tmp/tmp.VzJejtlx5K
++ rm /tmp/tmp.ZKOKHl0Fyk /tmp/tmp.VzJejtlx5K
++ return 0
+ [[ '' == true ]]
+ wait_pod monitoring-pmm3-rs0-2
+ local pod=monitoring-pmm3-rs0-2
+ set +o xtrace
waiting for pod/monitoring-pmm3-rs0-2 to be ready.OK
++ kubectl_bin get psmdb monitoring-pmm3 -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].nonvoting.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.U76wfNuqIm
+++ mktemp
++ local LAST_ERR=/tmp/tmp.UvOkVFG60m
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb monitoring-pmm3 -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].nonvoting.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.U76wfNuqIm
++ cat /tmp/tmp.UvOkVFG60m
++ rm /tmp/tmp.U76wfNuqIm /tmp/tmp.UvOkVFG60m
++ return 0
+ [[ '' == true ]]
++ kubectl_bin get psmdb monitoring-pmm3 -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].hidden.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.kipKW8D4jm
+++ mktemp
++ local LAST_ERR=/tmp/tmp.EkkOC8MrIl
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb monitoring-pmm3 -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].hidden.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.kipKW8D4jm
++ cat /tmp/tmp.EkkOC8MrIl
++ rm /tmp/tmp.kipKW8D4jm /tmp/tmp.EkkOC8MrIl
++ return 0
+ [[ '' == true ]]
+ sleep 10
+ [[ true == true ]]
+ set +x
Waiting for cluster monitoring-pmm3 to be ready..............................................................................................................................
+ local liveness readiness
++ kubectl_bin get statefulset/monitoring-pmm3-rs0 -o 'jsonpath={.spec.template.spec.containers[?(@.name=="pmm-client")].livenessProbe.exec.command[0]}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.WCAFYwy01K
+++ mktemp
++ local LAST_ERR=/tmp/tmp.AYmM9bpluJ
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get statefulset/monitoring-pmm3-rs0 -o 'jsonpath={.spec.template.spec.containers[?(@.name=="pmm-client")].livenessProbe.exec.command[0]}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.WCAFYwy01K
++ cat /tmp/tmp.AYmM9bpluJ
++ rm /tmp/tmp.WCAFYwy01K /tmp/tmp.AYmM9bpluJ
++ return 0
+ liveness=/bin/true
++ kubectl_bin get statefulset/monitoring-pmm3-rs0 -o 'jsonpath={.spec.template.spec.containers[?(@.name=="pmm-client")].readinessProbe.exec.command[0]}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.Vv2E1e0MZk
+++ mktemp
++ local LAST_ERR=/tmp/tmp.EfYcNMDzWz
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get statefulset/monitoring-pmm3-rs0 -o 'jsonpath={.spec.template.spec.containers[?(@.name=="pmm-client")].readinessProbe.exec.command[0]}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.Vv2E1e0MZk
++ cat /tmp/tmp.EfYcNMDzWz
++ rm /tmp/tmp.Vv2E1e0MZk /tmp/tmp.EfYcNMDzWz
++ return 0
+ readiness=/bin/true
+ [[ /bin/true != /bin/true ]]
+ [[ /bin/true != /bin/true ]]
+ desc 'revert custom PMM probes to defaults'
+ set +o xtrace


-----------------------------------------------------------------------------------
revert custom PMM probes to defaults
-----------------------------------------------------------------------------------

+ kubectl_bin patch psmdb monitoring-pmm3 --type=json $'-p=[\n\t  {"op":"remove","path":"/spec/pmm/livenessProbe"},\n\t  {"op":"remove","path":"/spec/pmm/readinessProbe"}\n\t]'
++ mktemp
+ local LAST_OUT=/tmp/tmp.ORRKEHPSjA
++ mktemp
+ local LAST_ERR=/tmp/tmp.9xBxe7OFRu
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl patch psmdb monitoring-pmm3 --type=json $'-p=[\n\t  {"op":"remove","path":"/spec/pmm/livenessProbe"},\n\t  {"op":"remove","path":"/spec/pmm/readinessProbe"}\n\t]'
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.ORRKEHPSjA
perconaservermongodb.psmdb.percona.com/monitoring-pmm3 patched
+ cat /tmp/tmp.9xBxe7OFRu
+ rm /tmp/tmp.ORRKEHPSjA /tmp/tmp.9xBxe7OFRu
+ return 0
+ wait_for_running monitoring-pmm3-rs0 3
+ local name=monitoring-pmm3-rs0
+ let last_pod=2
+ local check_cluster_readiness=true
+ set_debug
+ [[ 1 == 1 ]]
+ set -o xtrace
+ local rs_name=rs0
+ local cluster_name=monitoring-pmm3
++ seq 0 2
+ for i in $(seq 0 $last_pod)
+ [[ 0 -eq 2 ]]
+ wait_pod monitoring-pmm3-rs0-0
+ local pod=monitoring-pmm3-rs0-0
+ set +o xtrace
waiting for pod/monitoring-pmm3-rs0-0 to be ready.OK
+ for i in $(seq 0 $last_pod)
+ [[ 1 -eq 2 ]]
+ wait_pod monitoring-pmm3-rs0-1
+ local pod=monitoring-pmm3-rs0-1
+ set +o xtrace
waiting for pod/monitoring-pmm3-rs0-1 to be ready.OK
+ for i in $(seq 0 $last_pod)
+ [[ 2 -eq 2 ]]
++ kubectl_bin get psmdb monitoring-pmm3 -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].arbiter.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.kexq8feut7
+++ mktemp
++ local LAST_ERR=/tmp/tmp.ttFJy0EWKr
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb monitoring-pmm3 -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].arbiter.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.kexq8feut7
++ cat /tmp/tmp.ttFJy0EWKr
++ rm /tmp/tmp.kexq8feut7 /tmp/tmp.ttFJy0EWKr
++ return 0
+ [[ '' == true ]]
+ wait_pod monitoring-pmm3-rs0-2
+ local pod=monitoring-pmm3-rs0-2
+ set +o xtrace
waiting for pod/monitoring-pmm3-rs0-2 to be ready.OK
++ kubectl_bin get psmdb monitoring-pmm3 -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].nonvoting.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.eMTfLeck6w
+++ mktemp
++ local LAST_ERR=/tmp/tmp.vwYzTWlysQ
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb monitoring-pmm3 -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].nonvoting.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.eMTfLeck6w
++ cat /tmp/tmp.vwYzTWlysQ
++ rm /tmp/tmp.eMTfLeck6w /tmp/tmp.vwYzTWlysQ
++ return 0
+ [[ '' == true ]]
++ kubectl_bin get psmdb monitoring-pmm3 -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].hidden.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.KcdPdQvMUs
+++ mktemp
++ local LAST_ERR=/tmp/tmp.QfxagpE7PL
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb monitoring-pmm3 -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].hidden.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.KcdPdQvMUs
++ cat /tmp/tmp.QfxagpE7PL
++ rm /tmp/tmp.KcdPdQvMUs /tmp/tmp.QfxagpE7PL
++ return 0
+ [[ '' == true ]]
+ sleep 10
+ [[ true == true ]]
+ set +x
Waiting for cluster monitoring-pmm3 to be ready........................................................................................................................................
+ desc 'create new PMM token and add it to the secret'
+ set +o xtrace


-----------------------------------------------------------------------------------
create new PMM token and add it to the secret
-----------------------------------------------------------------------------------

++ get_pmm_server_token operator_new
++ local key_name=operator_new
++ [[ -z operator_new ]]
++ local ADMIN_PASSWORD
+++ kubectl get secret pmm-secret -o 'jsonpath={.data.PMM_ADMIN_PASSWORD}'
+++ base64 --decode
++ ADMIN_PASSWORD='Os\2'\''.99QS0gs;]f'
++ [[ -z Os\2'.99QS0gs;]f ]]
++ local create_response create_status_code create_json_response
++++ get_service_endpoint monitoring-service
++++ local service=monitoring-service
+++++ kubectl_bin get service/monitoring-service -o json
+++++ jq '.status.loadBalancer.ingress[].hostname'
++++++ mktemp
+++++ sed -e 's/^"//; s/"$//;'
+++++ local LAST_OUT=/tmp/tmp.Gsw0BGXxFj
++++++ mktemp
+++++ local LAST_ERR=/tmp/tmp.XMX1ttUgZG
+++++ local exit_status=0
+++++ local timeout=4
++++++ seq 0 2
+++++ for i in $(seq 0 2)
+++++ set +e
+++++ kubectl get service/monitoring-service -o json
+++++ exit_status=0
+++++ set -e
+++++ '[' 0 '!=' 0 -a -n 1 ']'
+++++ break
+++++ cat /tmp/tmp.Gsw0BGXxFj
+++++ cat /tmp/tmp.XMX1ttUgZG
+++++ rm /tmp/tmp.Gsw0BGXxFj /tmp/tmp.XMX1ttUgZG
+++++ return 0
++++ local hostname=null
++++ '[' -n null -a null '!=' null ']'
+++++ kubectl_bin get service/monitoring-service -o json
+++++ jq '.status.loadBalancer.ingress[].ip'
+++++ sed -e 's/^"//; s/"$//;'
++++++ mktemp
+++++ local LAST_OUT=/tmp/tmp.M9NAwqb6iI
++++++ mktemp
+++++ local LAST_ERR=/tmp/tmp.ngwSJKkyOX
+++++ local exit_status=0
+++++ local timeout=4
++++++ seq 0 2
+++++ for i in $(seq 0 2)
+++++ set +e
+++++ kubectl get service/monitoring-service -o json
+++++ exit_status=0
+++++ set -e
+++++ '[' 0 '!=' 0 -a -n 1 ']'
+++++ break
+++++ cat /tmp/tmp.M9NAwqb6iI
+++++ cat /tmp/tmp.ngwSJKkyOX
+++++ rm /tmp/tmp.M9NAwqb6iI /tmp/tmp.ngwSJKkyOX
+++++ return 0
++++ local ip=35.254.76.43
++++ '[' -n 35.254.76.43 -a 35.254.76.43 '!=' null ']'
++++ echo 35.254.76.43
++++ return
+++ execute_curl --insecure -s -X POST -H 'Content-Type: application/json' -H 'Accept: application/json' -d '{"name":"operator_new", "role":"Admin", "isDisabled":false}' --user 'admin:Os\2'\''.99QS0gs;]f' https://35.254.76.43/graph/api/serviceaccounts -w '\n%{http_code}'
+++ local name=curl-14946
+++ local image=docker.io/alpine/curl:latest
+++ kubectl_bin run curl-14946 -n monitoring-pmm3-11763 --restart=Never --image=docker.io/alpine/curl:latest --command -- sleep 3600
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.JPs7SQ6uva
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.2iF7fhBvll
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in $(seq 0 2)
+++ set +e
+++ kubectl run curl-14946 -n monitoring-pmm3-11763 --restart=Never --image=docker.io/alpine/curl:latest --command -- sleep 3600
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.JPs7SQ6uva
+++ cat /tmp/tmp.2iF7fhBvll
+++ rm /tmp/tmp.JPs7SQ6uva /tmp/tmp.2iF7fhBvll
+++ return 0
+++ kubectl_bin wait pod/curl-14946 -n monitoring-pmm3-11763 --for=condition=Ready --timeout=60s
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.2xHpxPxH41
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.gGDxHSBbcZ
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in $(seq 0 2)
+++ set +e
+++ kubectl wait pod/curl-14946 -n monitoring-pmm3-11763 --for=condition=Ready --timeout=60s
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.2xHpxPxH41
+++ cat /tmp/tmp.gGDxHSBbcZ
+++ rm /tmp/tmp.2xHpxPxH41 /tmp/tmp.gGDxHSBbcZ
+++ return 0
+++ local exit_status=0
+++ kubectl_bin exec curl-14946 -n monitoring-pmm3-11763 -- curl --insecure -s -X POST -H 'Content-Type: application/json' -H 'Accept: application/json' -d '{"name":"operator_new", "role":"Admin", "isDisabled":false}' --user 'admin:Os\2'\''.99QS0gs;]f' https://35.254.76.43/graph/api/serviceaccounts -w '\n%{http_code}'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.DxntVSIx0H
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.UwyS21ldTV
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in $(seq 0 2)
+++ set +e
+++ kubectl exec curl-14946 -n monitoring-pmm3-11763 -- curl --insecure -s -X POST -H 'Content-Type: application/json' -H 'Accept: application/json' -d '{"name":"operator_new", "role":"Admin", "isDisabled":false}' --user 'admin:Os\2'\''.99QS0gs;]f' https://35.254.76.43/graph/api/serviceaccounts -w '\n%{http_code}'
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.DxntVSIx0H
+++ cat /tmp/tmp.UwyS21ldTV
+++ rm /tmp/tmp.DxntVSIx0H /tmp/tmp.UwyS21ldTV
+++ return 0
+++ kubectl_bin delete pod curl-14946 -n monitoring-pmm3-11763 --ignore-not-found --wait=false
+++ return 0
++ create_response=$'{"id":3,"uid":"dfrf6s83ih7uoe","name":"operator_new","login":"sa-1-operator_new","orgId":1,"isDisabled":false,"role":"Admin","tokens":0,"avatarUrl":""}\n201'
+++ echo $'{"id":3,"uid":"dfrf6s83ih7uoe","name":"operator_new","login":"sa-1-operator_new","orgId":1,"isDisabled":false,"role":"Admin","tokens":0,"avatarUrl":""}\n201'
+++ tail -n1
++ create_status_code=201
+++ echo $'{"id":3,"uid":"dfrf6s83ih7uoe","name":"operator_new","login":"sa-1-operator_new","orgId":1,"isDisabled":false,"role":"Admin","tokens":0,"avatarUrl":""}\n201'
+++ sed '$ d'
++ create_json_response='{"id":3,"uid":"dfrf6s83ih7uoe","name":"operator_new","login":"sa-1-operator_new","orgId":1,"isDisabled":false,"role":"Admin","tokens":0,"avatarUrl":""}'
++ [[ 201 -ne 201 ]]
++ local service_account_id
+++ echo '{"id":3,"uid":"dfrf6s83ih7uoe","name":"operator_new","login":"sa-1-operator_new","orgId":1,"isDisabled":false,"role":"Admin","tokens":0,"avatarUrl":""}'
+++ jq -r .id
++ service_account_id=3
++ [[ -z 3 ]]
++ [[ 3 == null ]]
++ local token_response token_status_code token_json_response
++++ get_service_endpoint monitoring-service
++++ local service=monitoring-service
+++++ kubectl_bin get service/monitoring-service -o json
+++++ jq '.status.loadBalancer.ingress[].hostname'
+++++ sed -e 's/^"//; s/"$//;'
++++++ mktemp
+++++ local LAST_OUT=/tmp/tmp.J2h9o1QKP7
++++++ mktemp
+++++ local LAST_ERR=/tmp/tmp.TTee3Io0yJ
+++++ local exit_status=0
+++++ local timeout=4
++++++ seq 0 2
+++++ for i in $(seq 0 2)
+++++ set +e
+++++ kubectl get service/monitoring-service -o json
+++++ exit_status=0
+++++ set -e
+++++ '[' 0 '!=' 0 -a -n 1 ']'
+++++ break
+++++ cat /tmp/tmp.J2h9o1QKP7
+++++ cat /tmp/tmp.TTee3Io0yJ
+++++ rm /tmp/tmp.J2h9o1QKP7 /tmp/tmp.TTee3Io0yJ
+++++ return 0
++++ local hostname=null
++++ '[' -n null -a null '!=' null ']'
+++++ kubectl_bin get service/monitoring-service -o json
+++++ jq '.status.loadBalancer.ingress[].ip'
+++++ sed -e 's/^"//; s/"$//;'
++++++ mktemp
+++++ local LAST_OUT=/tmp/tmp.kHl1xXqjSp
++++++ mktemp
+++++ local LAST_ERR=/tmp/tmp.g68H261P4Q
+++++ local exit_status=0
+++++ local timeout=4
++++++ seq 0 2
+++++ for i in $(seq 0 2)
+++++ set +e
+++++ kubectl get service/monitoring-service -o json
+++++ exit_status=0
+++++ set -e
+++++ '[' 0 '!=' 0 -a -n 1 ']'
+++++ break
+++++ cat /tmp/tmp.kHl1xXqjSp
+++++ cat /tmp/tmp.g68H261P4Q
+++++ rm /tmp/tmp.kHl1xXqjSp /tmp/tmp.g68H261P4Q
+++++ return 0
++++ local ip=35.254.76.43
++++ '[' -n 35.254.76.43 -a 35.254.76.43 '!=' null ']'
++++ echo 35.254.76.43
++++ return
+++ execute_curl --insecure -s -X POST -H 'Content-Type: application/json' -d '{"name":"operator_new"}' --user 'admin:Os\2'\''.99QS0gs;]f' https://35.254.76.43/graph/api/serviceaccounts/3/tokens -w '\n%{http_code}'
+++ local name=curl-4298
+++ local image=docker.io/alpine/curl:latest
+++ kubectl_bin run curl-4298 -n monitoring-pmm3-11763 --restart=Never --image=docker.io/alpine/curl:latest --command -- sleep 3600
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.U8JtP72bBf
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.zi5LEoPosd
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in $(seq 0 2)
+++ set +e
+++ kubectl run curl-4298 -n monitoring-pmm3-11763 --restart=Never --image=docker.io/alpine/curl:latest --command -- sleep 3600
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.U8JtP72bBf
+++ cat /tmp/tmp.zi5LEoPosd
+++ rm /tmp/tmp.U8JtP72bBf /tmp/tmp.zi5LEoPosd
+++ return 0
+++ kubectl_bin wait pod/curl-4298 -n monitoring-pmm3-11763 --for=condition=Ready --timeout=60s
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.8xTVlsE8ki
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.Or3TfPLyT9
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in $(seq 0 2)
+++ set +e
+++ kubectl wait pod/curl-4298 -n monitoring-pmm3-11763 --for=condition=Ready --timeout=60s
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.8xTVlsE8ki
+++ cat /tmp/tmp.Or3TfPLyT9
+++ rm /tmp/tmp.8xTVlsE8ki /tmp/tmp.Or3TfPLyT9
+++ return 0
+++ local exit_status=0
+++ kubectl_bin exec curl-4298 -n monitoring-pmm3-11763 -- curl --insecure -s -X POST -H 'Content-Type: application/json' -d '{"name":"operator_new"}' --user 'admin:Os\2'\''.99QS0gs;]f' https://35.254.76.43/graph/api/serviceaccounts/3/tokens -w '\n%{http_code}'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.u1ZcWvzcAK
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.C8v4YJcAar
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in $(seq 0 2)
+++ set +e
+++ kubectl exec curl-4298 -n monitoring-pmm3-11763 -- curl --insecure -s -X POST -H 'Content-Type: application/json' -d '{"name":"operator_new"}' --user 'admin:Os\2'\''.99QS0gs;]f' https://35.254.76.43/graph/api/serviceaccounts/3/tokens -w '\n%{http_code}'
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.u1ZcWvzcAK
+++ cat /tmp/tmp.C8v4YJcAar
+++ rm /tmp/tmp.u1ZcWvzcAK /tmp/tmp.C8v4YJcAar
+++ return 0
+++ kubectl_bin delete pod curl-4298 -n monitoring-pmm3-11763 --ignore-not-found --wait=false
+++ return 0
++ token_response=$'{"id":2,"name":"operator_new","key":"glsa_ENtP2ImukR8cr3g1MllgZoNhKS1S8MWd_73f00929"}\n200'
+++ echo $'{"id":2,"name":"operator_new","key":"glsa_ENtP2ImukR8cr3g1MllgZoNhKS1S8MWd_73f00929"}\n200'
+++ tail -n1
++ token_status_code=200
+++ sed '$ d'
+++ echo $'{"id":2,"name":"operator_new","key":"glsa_ENtP2ImukR8cr3g1MllgZoNhKS1S8MWd_73f00929"}\n200'
++ token_json_response='{"id":2,"name":"operator_new","key":"glsa_ENtP2ImukR8cr3g1MllgZoNhKS1S8MWd_73f00929"}'
++ [[ 200 -ne 200 ]]
++ echo '{"id":2,"name":"operator_new","key":"glsa_ENtP2ImukR8cr3g1MllgZoNhKS1S8MWd_73f00929"}'
++ jq -r .key
+ NEW_TOKEN=glsa_ENtP2ImukR8cr3g1MllgZoNhKS1S8MWd_73f00929
+ kubectl_bin patch secret some-users --type merge --patch '{"stringData": {"PMM_SERVER_TOKEN": "glsa_ENtP2ImukR8cr3g1MllgZoNhKS1S8MWd_73f00929"}}'
++ mktemp
+ local LAST_OUT=/tmp/tmp.0eVkIZMPca
++ mktemp
+ local LAST_ERR=/tmp/tmp.9FO1I12V28
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl patch secret some-users --type merge --patch '{"stringData": {"PMM_SERVER_TOKEN": "glsa_ENtP2ImukR8cr3g1MllgZoNhKS1S8MWd_73f00929"}}'
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.0eVkIZMPca
secret/some-users patched
+ cat /tmp/tmp.9FO1I12V28
+ rm /tmp/tmp.0eVkIZMPca /tmp/tmp.9FO1I12V28
+ return 0
+ desc 'delete old PMM token'
+ set +o xtrace


-----------------------------------------------------------------------------------
delete old PMM token
-----------------------------------------------------------------------------------

+ delete_pmm_server_token operator
+ local key_name=operator
+ [[ -z operator ]]
+ local ADMIN_PASSWORD
++ kubectl get secret pmm-secret -o 'jsonpath={.data.PMM_ADMIN_PASSWORD}'
++ base64 --decode
+ ADMIN_PASSWORD='Os\2'\''.99QS0gs;]f'
+ [[ -z Os\2'.99QS0gs;]f ]]
+ local 'user_credentials=admin:Os\2'\''.99QS0gs;]f'
+ local service_accounts_response service_accounts_status
+++ get_service_endpoint monitoring-service
+++ local service=monitoring-service
++++ kubectl_bin get service/monitoring-service -o json
++++ jq '.status.loadBalancer.ingress[].hostname'
++++ sed -e 's/^"//; s/"$//;'
+++++ mktemp
++++ local LAST_OUT=/tmp/tmp.HV5IwLlfud
+++++ mktemp
++++ local LAST_ERR=/tmp/tmp.LqQ2JsyGEU
++++ local exit_status=0
++++ local timeout=4
+++++ seq 0 2
++++ for i in $(seq 0 2)
++++ set +e
++++ kubectl get service/monitoring-service -o json
++++ exit_status=0
++++ set -e
++++ '[' 0 '!=' 0 -a -n 1 ']'
++++ break
++++ cat /tmp/tmp.HV5IwLlfud
++++ cat /tmp/tmp.LqQ2JsyGEU
++++ rm /tmp/tmp.HV5IwLlfud /tmp/tmp.LqQ2JsyGEU
++++ return 0
+++ local hostname=null
+++ '[' -n null -a null '!=' null ']'
++++ kubectl_bin get service/monitoring-service -o json
++++ jq '.status.loadBalancer.ingress[].ip'
++++ sed -e 's/^"//; s/"$//;'
+++++ mktemp
++++ local LAST_OUT=/tmp/tmp.aBTmWWCsXJ
+++++ mktemp
++++ local LAST_ERR=/tmp/tmp.pIINBhQOr8
++++ local exit_status=0
++++ local timeout=4
+++++ seq 0 2
++++ for i in $(seq 0 2)
++++ set +e
++++ kubectl get service/monitoring-service -o json
++++ exit_status=0
++++ set -e
++++ '[' 0 '!=' 0 -a -n 1 ']'
++++ break
++++ cat /tmp/tmp.aBTmWWCsXJ
++++ cat /tmp/tmp.pIINBhQOr8
++++ rm /tmp/tmp.aBTmWWCsXJ /tmp/tmp.pIINBhQOr8
++++ return 0
+++ local ip=35.254.76.43
+++ '[' -n 35.254.76.43 -a 35.254.76.43 '!=' null ']'
+++ echo 35.254.76.43
+++ return
++ execute_curl --insecure -s -X GET --user 'admin:Os\2'\''.99QS0gs;]f' https://35.254.76.43/graph/api/serviceaccounts/search -w '\n%{http_code}'
++ local name=curl-25349
++ local image=docker.io/alpine/curl:latest
++ kubectl_bin run curl-25349 -n monitoring-pmm3-11763 --restart=Never --image=docker.io/alpine/curl:latest --command -- sleep 3600
+++ mktemp
++ local LAST_OUT=/tmp/tmp.eKFHou9CFp
+++ mktemp
++ local LAST_ERR=/tmp/tmp.YOrA0HHV8p
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl run curl-25349 -n monitoring-pmm3-11763 --restart=Never --image=docker.io/alpine/curl:latest --command -- sleep 3600
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.eKFHou9CFp
++ cat /tmp/tmp.YOrA0HHV8p
++ rm /tmp/tmp.eKFHou9CFp /tmp/tmp.YOrA0HHV8p
++ return 0
++ kubectl_bin wait pod/curl-25349 -n monitoring-pmm3-11763 --for=condition=Ready --timeout=60s
+++ mktemp
++ local LAST_OUT=/tmp/tmp.exrA4lkwUA
+++ mktemp
++ local LAST_ERR=/tmp/tmp.LzQiLDMlag
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl wait pod/curl-25349 -n monitoring-pmm3-11763 --for=condition=Ready --timeout=60s
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.exrA4lkwUA
++ cat /tmp/tmp.LzQiLDMlag
++ rm /tmp/tmp.exrA4lkwUA /tmp/tmp.LzQiLDMlag
++ return 0
++ local exit_status=0
++ kubectl_bin exec curl-25349 -n monitoring-pmm3-11763 -- curl --insecure -s -X GET --user 'admin:Os\2'\''.99QS0gs;]f' https://35.254.76.43/graph/api/serviceaccounts/search -w '\n%{http_code}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.D8t1L9aYyu
+++ mktemp
++ local LAST_ERR=/tmp/tmp.IZ6G4R3H4b
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl exec curl-25349 -n monitoring-pmm3-11763 -- curl --insecure -s -X GET --user 'admin:Os\2'\''.99QS0gs;]f' https://35.254.76.43/graph/api/serviceaccounts/search -w '\n%{http_code}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.D8t1L9aYyu
++ cat /tmp/tmp.IZ6G4R3H4b
++ rm /tmp/tmp.D8t1L9aYyu /tmp/tmp.IZ6G4R3H4b
++ return 0
++ kubectl_bin delete pod curl-25349 -n monitoring-pmm3-11763 --ignore-not-found --wait=false
++ return 0
+ service_accounts_response=$'{"totalCount":2,"serviceAccounts":[{"id":2,"uid":"bfrf5azyzzugwa","name":"operator","login":"sa-1-operator","orgId":1,"isDisabled":false,"role":"Admin","tokens":1,"avatarUrl":"/graph/avatar/f741e7abdd8def4ed30a37f602429df3"},{"id":3,"uid":"dfrf6s83ih7uoe","name":"operator_new","login":"sa-1-operator_new","orgId":1,"isDisabled":false,"role":"Admin","tokens":1,"avatarUrl":"/graph/avatar/6d1b2db67a8f3e06db4ae076406fece5"}],"page":1,"perPage":1000}\n200'
++ echo $'{"totalCount":2,"serviceAccounts":[{"id":2,"uid":"bfrf5azyzzugwa","name":"operator","login":"sa-1-operator","orgId":1,"isDisabled":false,"role":"Admin","tokens":1,"avatarUrl":"/graph/avatar/f741e7abdd8def4ed30a37f602429df3"},{"id":3,"uid":"dfrf6s83ih7uoe","name":"operator_new","login":"sa-1-operator_new","orgId":1,"isDisabled":false,"role":"Admin","tokens":1,"avatarUrl":"/graph/avatar/6d1b2db67a8f3e06db4ae076406fece5"}],"page":1,"perPage":1000}\n200'
++ tail -n1
+ service_accounts_status=200
++ echo $'{"totalCount":2,"serviceAccounts":[{"id":2,"uid":"bfrf5azyzzugwa","name":"operator","login":"sa-1-operator","orgId":1,"isDisabled":false,"role":"Admin","tokens":1,"avatarUrl":"/graph/avatar/f741e7abdd8def4ed30a37f602429df3"},{"id":3,"uid":"dfrf6s83ih7uoe","name":"operator_new","login":"sa-1-operator_new","orgId":1,"isDisabled":false,"role":"Admin","tokens":1,"avatarUrl":"/graph/avatar/6d1b2db67a8f3e06db4ae076406fece5"}],"page":1,"perPage":1000}\n200'
++ sed '$ d'
+ service_accounts_json='{"totalCount":2,"serviceAccounts":[{"id":2,"uid":"bfrf5azyzzugwa","name":"operator","login":"sa-1-operator","orgId":1,"isDisabled":false,"role":"Admin","tokens":1,"avatarUrl":"/graph/avatar/f741e7abdd8def4ed30a37f602429df3"},{"id":3,"uid":"dfrf6s83ih7uoe","name":"operator_new","login":"sa-1-operator_new","orgId":1,"isDisabled":false,"role":"Admin","tokens":1,"avatarUrl":"/graph/avatar/6d1b2db67a8f3e06db4ae076406fece5"}],"page":1,"perPage":1000}'
+ [[ 200 -ne 200 ]]
+ local service_account_id
++ echo '{"totalCount":2,"serviceAccounts":[{"id":2,"uid":"bfrf5azyzzugwa","name":"operator","login":"sa-1-operator","orgId":1,"isDisabled":false,"role":"Admin","tokens":1,"avatarUrl":"/graph/avatar/f741e7abdd8def4ed30a37f602429df3"},{"id":3,"uid":"dfrf6s83ih7uoe","name":"operator_new","login":"sa-1-operator_new","orgId":1,"isDisabled":false,"role":"Admin","tokens":1,"avatarUrl":"/graph/avatar/6d1b2db67a8f3e06db4ae076406fece5"}],"page":1,"perPage":1000}'
++ jq -r '.serviceAccounts[] | select(.name == "operator").id'
+ service_account_id=2
+ [[ -z 2 ]]
+ [[ 2 == null ]]
+ local tokens_response tokens_status tokens_json
+++ get_service_endpoint monitoring-service
+++ local service=monitoring-service
++++ kubectl_bin get service/monitoring-service -o json
++++ jq '.status.loadBalancer.ingress[].hostname'
++++ sed -e 's/^"//; s/"$//;'
+++++ mktemp
++++ local LAST_OUT=/tmp/tmp.ix7LT2JRhl
+++++ mktemp
++++ local LAST_ERR=/tmp/tmp.CTwSI5pt0W
++++ local exit_status=0
++++ local timeout=4
+++++ seq 0 2
++++ for i in $(seq 0 2)
++++ set +e
++++ kubectl get service/monitoring-service -o json
++++ exit_status=0
++++ set -e
++++ '[' 0 '!=' 0 -a -n 1 ']'
++++ break
++++ cat /tmp/tmp.ix7LT2JRhl
++++ cat /tmp/tmp.CTwSI5pt0W
++++ rm /tmp/tmp.ix7LT2JRhl /tmp/tmp.CTwSI5pt0W
++++ return 0
+++ local hostname=null
+++ '[' -n null -a null '!=' null ']'
++++ jq '.status.loadBalancer.ingress[].ip'
++++ kubectl_bin get service/monitoring-service -o json
++++ sed -e 's/^"//; s/"$//;'
+++++ mktemp
++++ local LAST_OUT=/tmp/tmp.vZ2gVCswoO
+++++ mktemp
++++ local LAST_ERR=/tmp/tmp.B0h952mvsE
++++ local exit_status=0
++++ local timeout=4
+++++ seq 0 2
++++ for i in $(seq 0 2)
++++ set +e
++++ kubectl get service/monitoring-service -o json
++++ exit_status=0
++++ set -e
++++ '[' 0 '!=' 0 -a -n 1 ']'
++++ break
++++ cat /tmp/tmp.vZ2gVCswoO
++++ cat /tmp/tmp.B0h952mvsE
++++ rm /tmp/tmp.vZ2gVCswoO /tmp/tmp.B0h952mvsE
++++ return 0
+++ local ip=35.254.76.43
+++ '[' -n 35.254.76.43 -a 35.254.76.43 '!=' null ']'
+++ echo 35.254.76.43
+++ return
++ execute_curl --insecure -s -X GET --user 'admin:Os\2'\''.99QS0gs;]f' https://35.254.76.43/graph/api/serviceaccounts/2/tokens -w '\n%{http_code}'
++ local name=curl-31554
++ local image=docker.io/alpine/curl:latest
++ kubectl_bin run curl-31554 -n monitoring-pmm3-11763 --restart=Never --image=docker.io/alpine/curl:latest --command -- sleep 3600
+++ mktemp
++ local LAST_OUT=/tmp/tmp.QUAMvOBVvE
+++ mktemp
++ local LAST_ERR=/tmp/tmp.Gaj34e9f0d
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl run curl-31554 -n monitoring-pmm3-11763 --restart=Never --image=docker.io/alpine/curl:latest --command -- sleep 3600
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.QUAMvOBVvE
++ cat /tmp/tmp.Gaj34e9f0d
++ rm /tmp/tmp.QUAMvOBVvE /tmp/tmp.Gaj34e9f0d
++ return 0
++ kubectl_bin wait pod/curl-31554 -n monitoring-pmm3-11763 --for=condition=Ready --timeout=60s
+++ mktemp
++ local LAST_OUT=/tmp/tmp.dxqyCL7px3
+++ mktemp
++ local LAST_ERR=/tmp/tmp.G7nz6N4ob8
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl wait pod/curl-31554 -n monitoring-pmm3-11763 --for=condition=Ready --timeout=60s
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.dxqyCL7px3
++ cat /tmp/tmp.G7nz6N4ob8
++ rm /tmp/tmp.dxqyCL7px3 /tmp/tmp.G7nz6N4ob8
++ return 0
++ local exit_status=0
++ kubectl_bin exec curl-31554 -n monitoring-pmm3-11763 -- curl --insecure -s -X GET --user 'admin:Os\2'\''.99QS0gs;]f' https://35.254.76.43/graph/api/serviceaccounts/2/tokens -w '\n%{http_code}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.UewHWdHw8B
+++ mktemp
++ local LAST_ERR=/tmp/tmp.LiwBfqk4Rt
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl exec curl-31554 -n monitoring-pmm3-11763 -- curl --insecure -s -X GET --user 'admin:Os\2'\''.99QS0gs;]f' https://35.254.76.43/graph/api/serviceaccounts/2/tokens -w '\n%{http_code}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.UewHWdHw8B
++ cat /tmp/tmp.LiwBfqk4Rt
++ rm /tmp/tmp.UewHWdHw8B /tmp/tmp.LiwBfqk4Rt
++ return 0
++ kubectl_bin delete pod curl-31554 -n monitoring-pmm3-11763 --ignore-not-found --wait=false
++ return 0
+ tokens_response=$'[{"id":1,"name":"operator","created":"2026-07-07T19:44:06Z","lastUsedAt":"2026-07-07T19:59:47Z","expiration":null,"secondsUntilExpiration":0,"hasExpired":false,"isRevoked":false}]\n200'
++ echo $'[{"id":1,"name":"operator","created":"2026-07-07T19:44:06Z","lastUsedAt":"2026-07-07T19:59:47Z","expiration":null,"secondsUntilExpiration":0,"hasExpired":false,"isRevoked":false}]\n200'
++ tail -n1
+ tokens_status=200
++ echo $'[{"id":1,"name":"operator","created":"2026-07-07T19:44:06Z","lastUsedAt":"2026-07-07T19:59:47Z","expiration":null,"secondsUntilExpiration":0,"hasExpired":false,"isRevoked":false}]\n200'
++ sed '$ d'
+ tokens_json='[{"id":1,"name":"operator","created":"2026-07-07T19:44:06Z","lastUsedAt":"2026-07-07T19:59:47Z","expiration":null,"secondsUntilExpiration":0,"hasExpired":false,"isRevoked":false}]'
+ [[ 200 -ne 200 ]]
+ local token_id
++ echo '[{"id":1,"name":"operator","created":"2026-07-07T19:44:06Z","lastUsedAt":"2026-07-07T19:59:47Z","expiration":null,"secondsUntilExpiration":0,"hasExpired":false,"isRevoked":false}]'
++ jq -r '.[] | select(.name == "operator").id'
+ token_id=1
+ [[ -z 1 ]]
+ [[ 1 == null ]]
+ local delete_response delete_status
+++ get_service_endpoint monitoring-service
+++ local service=monitoring-service
++++ kubectl_bin get service/monitoring-service -o json
++++ jq '.status.loadBalancer.ingress[].hostname'
++++ sed -e 's/^"//; s/"$//;'
+++++ mktemp
++++ local LAST_OUT=/tmp/tmp.0QV29DSyBO
+++++ mktemp
++++ local LAST_ERR=/tmp/tmp.keosDRl7rq
++++ local exit_status=0
++++ local timeout=4
+++++ seq 0 2
++++ for i in $(seq 0 2)
++++ set +e
++++ kubectl get service/monitoring-service -o json
++++ exit_status=0
++++ set -e
++++ '[' 0 '!=' 0 -a -n 1 ']'
++++ break
++++ cat /tmp/tmp.0QV29DSyBO
++++ cat /tmp/tmp.keosDRl7rq
++++ rm /tmp/tmp.0QV29DSyBO /tmp/tmp.keosDRl7rq
++++ return 0
+++ local hostname=null
+++ '[' -n null -a null '!=' null ']'
++++ kubectl_bin get service/monitoring-service -o json
++++ jq '.status.loadBalancer.ingress[].ip'
+++++ mktemp
++++ sed -e 's/^"//; s/"$//;'
++++ local LAST_OUT=/tmp/tmp.odhmNfVumt
+++++ mktemp
++++ local LAST_ERR=/tmp/tmp.qXgyDSWnMK
++++ local exit_status=0
++++ local timeout=4
+++++ seq 0 2
++++ for i in $(seq 0 2)
++++ set +e
++++ kubectl get service/monitoring-service -o json
++++ exit_status=0
++++ set -e
++++ '[' 0 '!=' 0 -a -n 1 ']'
++++ break
++++ cat /tmp/tmp.odhmNfVumt
++++ cat /tmp/tmp.qXgyDSWnMK
++++ rm /tmp/tmp.odhmNfVumt /tmp/tmp.qXgyDSWnMK
++++ return 0
+++ local ip=35.254.76.43
+++ '[' -n 35.254.76.43 -a 35.254.76.43 '!=' null ']'
+++ echo 35.254.76.43
+++ return
++ execute_curl --insecure -s -X DELETE --user 'admin:Os\2'\''.99QS0gs;]f' https://35.254.76.43/graph/api/serviceaccounts/2/tokens/1 -w '\n%{http_code}'
++ local name=curl-117
++ local image=docker.io/alpine/curl:latest
++ kubectl_bin run curl-117 -n monitoring-pmm3-11763 --restart=Never --image=docker.io/alpine/curl:latest --command -- sleep 3600
+++ mktemp
++ local LAST_OUT=/tmp/tmp.lmIHVy1f31
+++ mktemp
++ local LAST_ERR=/tmp/tmp.smC9R1k4ab
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl run curl-117 -n monitoring-pmm3-11763 --restart=Never --image=docker.io/alpine/curl:latest --command -- sleep 3600
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.lmIHVy1f31
++ cat /tmp/tmp.smC9R1k4ab
++ rm /tmp/tmp.lmIHVy1f31 /tmp/tmp.smC9R1k4ab
++ return 0
++ kubectl_bin wait pod/curl-117 -n monitoring-pmm3-11763 --for=condition=Ready --timeout=60s
+++ mktemp
++ local LAST_OUT=/tmp/tmp.hEEAyT1KJ1
+++ mktemp
++ local LAST_ERR=/tmp/tmp.yMiwSgN7Nk
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl wait pod/curl-117 -n monitoring-pmm3-11763 --for=condition=Ready --timeout=60s
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.hEEAyT1KJ1
++ cat /tmp/tmp.yMiwSgN7Nk
E0707 20:00:58.974992  673778 reflector.go:227] "Failed to watch" err="Get \"https://34.27.139.243/api/v1/namespaces/monitoring-pmm3-11763/pods?allowWatchBookmarks=true&fieldSelector=metadata.name%3Dcurl-117&resourceVersion=1783454458787951010&timeoutSeconds=495&watch=true\": context canceled" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" type="*unstructured.Unstructured"
++ rm /tmp/tmp.hEEAyT1KJ1 /tmp/tmp.yMiwSgN7Nk
++ return 0
++ local exit_status=0
++ kubectl_bin exec curl-117 -n monitoring-pmm3-11763 -- curl --insecure -s -X DELETE --user 'admin:Os\2'\''.99QS0gs;]f' https://35.254.76.43/graph/api/serviceaccounts/2/tokens/1 -w '\n%{http_code}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.pJBFoN4NzI
+++ mktemp
++ local LAST_ERR=/tmp/tmp.auxeOW9X5w
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl exec curl-117 -n monitoring-pmm3-11763 -- curl --insecure -s -X DELETE --user 'admin:Os\2'\''.99QS0gs;]f' https://35.254.76.43/graph/api/serviceaccounts/2/tokens/1 -w '\n%{http_code}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.pJBFoN4NzI
++ cat /tmp/tmp.auxeOW9X5w
++ rm /tmp/tmp.pJBFoN4NzI /tmp/tmp.auxeOW9X5w
++ return 0
++ kubectl_bin delete pod curl-117 -n monitoring-pmm3-11763 --ignore-not-found --wait=false
++ return 0
+ delete_response=$'{"message":"Service account token deleted"}\n200'
++ echo $'{"message":"Service account token deleted"}\n200'
++ tail -n1
+ delete_status=200
+ [[ 200 -ne 200 ]]
+ desc 'check for authentication errors in PMM client logs'
+ set +o xtrace


-----------------------------------------------------------------------------------
check for authentication errors in PMM client logs
-----------------------------------------------------------------------------------

++ kubectl get pods --selector=app.kubernetes.io/replset=rs0 -o 'jsonpath={.items[*].metadata.name}'
+ pods='monitoring-pmm3-rs0-0 monitoring-pmm3-rs0-1 monitoring-pmm3-rs0-2'
+ for pod in $pods
+ for i in {1..3}
+ kubectl logs monitoring-pmm3-rs0-0 pmm-client
+ grep -q 'Invalid username or password'
+ sleep 2
+ for i in {1..3}
+ kubectl logs monitoring-pmm3-rs0-0 pmm-client
+ grep -q 'Invalid username or password'
+ sleep 2
+ for i in {1..3}
+ kubectl logs monitoring-pmm3-rs0-0 pmm-client
+ grep -q 'Invalid username or password'
+ sleep 2
+ for pod in $pods
+ for i in {1..3}
+ kubectl logs monitoring-pmm3-rs0-1 pmm-client
+ grep -q 'Invalid username or password'
+ sleep 2
+ for i in {1..3}
+ kubectl logs monitoring-pmm3-rs0-1 pmm-client
+ grep -q 'Invalid username or password'
+ sleep 2
+ for i in {1..3}
+ kubectl logs monitoring-pmm3-rs0-1 pmm-client
+ grep -q 'Invalid username or password'
+ sleep 2
+ for pod in $pods
+ for i in {1..3}
+ kubectl logs monitoring-pmm3-rs0-2 pmm-client
+ grep -q 'Invalid username or password'
+ sleep 2
+ for i in {1..3}
+ kubectl logs monitoring-pmm3-rs0-2 pmm-client
+ grep -q 'Invalid username or password'
+ sleep 2
+ for i in {1..3}
+ kubectl logs monitoring-pmm3-rs0-2 pmm-client
+ grep -q 'Invalid username or password'
+ sleep 2
+ desc 'check mongod metrics'
+ set +o xtrace


-----------------------------------------------------------------------------------
check mongod metrics
-----------------------------------------------------------------------------------

+ get_metric_values node_boot_time_seconds monitoring-pmm3-11763-monitoring-pmm3-rs0-1 glsa_ENtP2ImukR8cr3g1MllgZoNhKS1S8MWd_73f00929
+ local metric=node_boot_time_seconds
+ local instance=monitoring-pmm3-11763-monitoring-pmm3-rs0-1
+ local token=glsa_ENtP2ImukR8cr3g1MllgZoNhKS1S8MWd_73f00929
++ /usr/sbin/date -u +%s -d '-1 minute'
+ local start=1783454433
++ /usr/sbin/date -u +%s
+ local end=1783454493
++ get_service_endpoint monitoring-service
++ local service=monitoring-service
+++ kubectl_bin get service/monitoring-service -o json
+++ jq '.status.loadBalancer.ingress[].hostname'
+++ sed -e 's/^"//; s/"$//;'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.JMzESFLDri
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.NhrAXrW93W
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in $(seq 0 2)
+++ set +e
+++ kubectl get service/monitoring-service -o json
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.JMzESFLDri
+++ cat /tmp/tmp.NhrAXrW93W
+++ rm /tmp/tmp.JMzESFLDri /tmp/tmp.NhrAXrW93W
+++ return 0
++ local hostname=null
++ '[' -n null -a null '!=' null ']'
+++ kubectl_bin get service/monitoring-service -o json
+++ jq '.status.loadBalancer.ingress[].ip'
+++ sed -e 's/^"//; s/"$//;'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.HmgLfu6ayO
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.YqqEzxNALI
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in $(seq 0 2)
+++ set +e
+++ kubectl get service/monitoring-service -o json
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.HmgLfu6ayO
+++ cat /tmp/tmp.YqqEzxNALI
+++ rm /tmp/tmp.HmgLfu6ayO /tmp/tmp.YqqEzxNALI
+++ return 0
++ local ip=35.254.76.43
++ '[' -n 35.254.76.43 -a 35.254.76.43 '!=' null ']'
++ echo 35.254.76.43
++ return
+ local endpoint=35.254.76.43
+ '[' -z node_boot_time_seconds ']'
+ '[' -z glsa_ENtP2ImukR8cr3g1MllgZoNhKS1S8MWd_73f00929 ']'
+ local wait_count=30
+ local retry=0
++ execute_curl -s -k -H 'Authorization: Bearer glsa_ENtP2ImukR8cr3g1MllgZoNhKS1S8MWd_73f00929' 'https://35.254.76.43/graph/api/datasources/proxy/1/api/v1/query_range?query=min%28node_boot_time_seconds%7Bnode_name%3D%7E%22monitoring-pmm3-11763-monitoring-pmm3-rs0-1%22%7d%20or%20node_boot_time_seconds%7Bnode_name%3D%7E%22monitoring-pmm3-11763-monitoring-pmm3-rs0-1%22%7D%29&start=1783454433&end=1783454493&step=60'
++ local name=curl-19395
++ local image=docker.io/alpine/curl:latest
++ kubectl_bin run curl-19395 -n monitoring-pmm3-11763 --restart=Never --image=docker.io/alpine/curl:latest --command -- sleep 3600
++ jq '.data.result[0].values[][1]'
+++ mktemp
++ grep '^"[0-9]'
++ local LAST_OUT=/tmp/tmp.3wRHKry8ew
+++ mktemp
++ local LAST_ERR=/tmp/tmp.zC7Ynj3wkF
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl run curl-19395 -n monitoring-pmm3-11763 --restart=Never --image=docker.io/alpine/curl:latest --command -- sleep 3600
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.3wRHKry8ew
++ cat /tmp/tmp.zC7Ynj3wkF
++ rm /tmp/tmp.3wRHKry8ew /tmp/tmp.zC7Ynj3wkF
++ return 0
++ kubectl_bin wait pod/curl-19395 -n monitoring-pmm3-11763 --for=condition=Ready --timeout=60s
+++ mktemp
++ local LAST_OUT=/tmp/tmp.lEhfz4KJ7A
+++ mktemp
++ local LAST_ERR=/tmp/tmp.cAIqhxyeVd
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl wait pod/curl-19395 -n monitoring-pmm3-11763 --for=condition=Ready --timeout=60s
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.lEhfz4KJ7A
++ cat /tmp/tmp.cAIqhxyeVd
E0707 20:01:37.252094  675290 reflector.go:227] "Failed to watch" err="Get \"https://34.27.139.243/api/v1/namespaces/monitoring-pmm3-11763/pods?allowWatchBookmarks=true&fieldSelector=metadata.name%3Dcurl-19395&resourceVersion=1783454496945295019&timeoutSeconds=489&watch=true\": context canceled" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" type="*unstructured.Unstructured"
++ rm /tmp/tmp.lEhfz4KJ7A /tmp/tmp.cAIqhxyeVd
++ return 0
++ local exit_status=0
++ kubectl_bin exec curl-19395 -n monitoring-pmm3-11763 -- curl -s -k -H 'Authorization: Bearer glsa_ENtP2ImukR8cr3g1MllgZoNhKS1S8MWd_73f00929' 'https://35.254.76.43/graph/api/datasources/proxy/1/api/v1/query_range?query=min%28node_boot_time_seconds%7Bnode_name%3D%7E%22monitoring-pmm3-11763-monitoring-pmm3-rs0-1%22%7d%20or%20node_boot_time_seconds%7Bnode_name%3D%7E%22monitoring-pmm3-11763-monitoring-pmm3-rs0-1%22%7D%29&start=1783454433&end=1783454493&step=60'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.XxwIkJFleC
+++ mktemp
++ local LAST_ERR=/tmp/tmp.DE9KUDghb5
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl exec curl-19395 -n monitoring-pmm3-11763 -- curl -s -k -H 'Authorization: Bearer glsa_ENtP2ImukR8cr3g1MllgZoNhKS1S8MWd_73f00929' 'https://35.254.76.43/graph/api/datasources/proxy/1/api/v1/query_range?query=min%28node_boot_time_seconds%7Bnode_name%3D%7E%22monitoring-pmm3-11763-monitoring-pmm3-rs0-1%22%7d%20or%20node_boot_time_seconds%7Bnode_name%3D%7E%22monitoring-pmm3-11763-monitoring-pmm3-rs0-1%22%7D%29&start=1783454433&end=1783454493&step=60'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.XxwIkJFleC
++ cat /tmp/tmp.DE9KUDghb5
++ rm /tmp/tmp.XxwIkJFleC /tmp/tmp.DE9KUDghb5
++ return 0
++ kubectl_bin delete pod curl-19395 -n monitoring-pmm3-11763 --ignore-not-found --wait=false
++ return 0
+ [[ -n "1783451415"
"1783451415" ]]
+ get_metric_values mongodb_connections monitoring-pmm3-11763-monitoring-pmm3-rs0-1 glsa_ENtP2ImukR8cr3g1MllgZoNhKS1S8MWd_73f00929
+ local metric=mongodb_connections
+ local instance=monitoring-pmm3-11763-monitoring-pmm3-rs0-1
+ local token=glsa_ENtP2ImukR8cr3g1MllgZoNhKS1S8MWd_73f00929
++ /usr/sbin/date -u +%s -d '-1 minute'
+ local start=1783454439
++ /usr/sbin/date -u +%s
+ local end=1783454499
++ get_service_endpoint monitoring-service
++ local service=monitoring-service
+++ kubectl_bin get service/monitoring-service -o json
+++ jq '.status.loadBalancer.ingress[].hostname'
+++ sed -e 's/^"//; s/"$//;'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.Rhyznw7lPV
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.TXrdXLNwUX
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in $(seq 0 2)
+++ set +e
+++ kubectl get service/monitoring-service -o json
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.Rhyznw7lPV
+++ cat /tmp/tmp.TXrdXLNwUX
+++ rm /tmp/tmp.Rhyznw7lPV /tmp/tmp.TXrdXLNwUX
+++ return 0
++ local hostname=null
++ '[' -n null -a null '!=' null ']'
+++ kubectl_bin get service/monitoring-service -o json
+++ jq '.status.loadBalancer.ingress[].ip'
++++ mktemp
+++ sed -e 's/^"//; s/"$//;'
+++ local LAST_OUT=/tmp/tmp.eLG2RWPbCY
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.HQMHFD1ir5
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in $(seq 0 2)
+++ set +e
+++ kubectl get service/monitoring-service -o json
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.eLG2RWPbCY
+++ cat /tmp/tmp.HQMHFD1ir5
+++ rm /tmp/tmp.eLG2RWPbCY /tmp/tmp.HQMHFD1ir5
+++ return 0
++ local ip=35.254.76.43
++ '[' -n 35.254.76.43 -a 35.254.76.43 '!=' null ']'
++ echo 35.254.76.43
++ return
+ local endpoint=35.254.76.43
+ '[' -z mongodb_connections ']'
+ '[' -z glsa_ENtP2ImukR8cr3g1MllgZoNhKS1S8MWd_73f00929 ']'
+ local wait_count=30
+ local retry=0
++ execute_curl -s -k -H 'Authorization: Bearer glsa_ENtP2ImukR8cr3g1MllgZoNhKS1S8MWd_73f00929' 'https://35.254.76.43/graph/api/datasources/proxy/1/api/v1/query_range?query=min%28mongodb_connections%7Bnode_name%3D%7E%22monitoring-pmm3-11763-monitoring-pmm3-rs0-1%22%7d%20or%20mongodb_connections%7Bnode_name%3D%7E%22monitoring-pmm3-11763-monitoring-pmm3-rs0-1%22%7D%29&start=1783454439&end=1783454499&step=60'
++ jq '.data.result[0].values[][1]'
++ local name=curl-3972
++ local image=docker.io/alpine/curl:latest
++ kubectl_bin run curl-3972 -n monitoring-pmm3-11763 --restart=Never --image=docker.io/alpine/curl:latest --command -- sleep 3600
++ grep '^"[0-9]'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.FLfbrAwH9q
+++ mktemp
++ local LAST_ERR=/tmp/tmp.Vx9i7RuvXd
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl run curl-3972 -n monitoring-pmm3-11763 --restart=Never --image=docker.io/alpine/curl:latest --command -- sleep 3600
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.FLfbrAwH9q
++ cat /tmp/tmp.Vx9i7RuvXd
++ rm /tmp/tmp.FLfbrAwH9q /tmp/tmp.Vx9i7RuvXd
++ return 0
++ kubectl_bin wait pod/curl-3972 -n monitoring-pmm3-11763 --for=condition=Ready --timeout=60s
+++ mktemp
++ local LAST_OUT=/tmp/tmp.9XrehEomin
+++ mktemp
++ local LAST_ERR=/tmp/tmp.3Jdc3QYp2m
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl wait pod/curl-3972 -n monitoring-pmm3-11763 --for=condition=Ready --timeout=60s
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.9XrehEomin
++ cat /tmp/tmp.3Jdc3QYp2m
++ rm /tmp/tmp.9XrehEomin /tmp/tmp.3Jdc3QYp2m
++ return 0
++ local exit_status=0
++ kubectl_bin exec curl-3972 -n monitoring-pmm3-11763 -- curl -s -k -H 'Authorization: Bearer glsa_ENtP2ImukR8cr3g1MllgZoNhKS1S8MWd_73f00929' 'https://35.254.76.43/graph/api/datasources/proxy/1/api/v1/query_range?query=min%28mongodb_connections%7Bnode_name%3D%7E%22monitoring-pmm3-11763-monitoring-pmm3-rs0-1%22%7d%20or%20mongodb_connections%7Bnode_name%3D%7E%22monitoring-pmm3-11763-monitoring-pmm3-rs0-1%22%7D%29&start=1783454439&end=1783454499&step=60'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.kEWbdpotEK
+++ mktemp
++ local LAST_ERR=/tmp/tmp.fFRX7VGAc0
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl exec curl-3972 -n monitoring-pmm3-11763 -- curl -s -k -H 'Authorization: Bearer glsa_ENtP2ImukR8cr3g1MllgZoNhKS1S8MWd_73f00929' 'https://35.254.76.43/graph/api/datasources/proxy/1/api/v1/query_range?query=min%28mongodb_connections%7Bnode_name%3D%7E%22monitoring-pmm3-11763-monitoring-pmm3-rs0-1%22%7d%20or%20mongodb_connections%7Bnode_name%3D%7E%22monitoring-pmm3-11763-monitoring-pmm3-rs0-1%22%7D%29&start=1783454439&end=1783454499&step=60'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.kEWbdpotEK
++ cat /tmp/tmp.fFRX7VGAc0
++ rm /tmp/tmp.kEWbdpotEK /tmp/tmp.fFRX7VGAc0
++ return 0
++ kubectl_bin delete pod curl-3972 -n monitoring-pmm3-11763 --ignore-not-found --wait=false
++ return 0
+ [[ -n "0"
"0" ]]
+ desc 'check mongo config metrics'
+ set +o xtrace


-----------------------------------------------------------------------------------
check mongo config metrics
-----------------------------------------------------------------------------------

+ get_metric_values node_boot_time_seconds monitoring-pmm3-11763-monitoring-pmm3-cfg-1 glsa_ENtP2ImukR8cr3g1MllgZoNhKS1S8MWd_73f00929
+ local metric=node_boot_time_seconds
+ local instance=monitoring-pmm3-11763-monitoring-pmm3-cfg-1
+ local token=glsa_ENtP2ImukR8cr3g1MllgZoNhKS1S8MWd_73f00929
++ /usr/sbin/date -u +%s -d '-1 minute'
+ local start=1783454444
++ /usr/sbin/date -u +%s
+ local end=1783454504
++ get_service_endpoint monitoring-service
++ local service=monitoring-service
+++ kubectl_bin get service/monitoring-service -o json
+++ jq '.status.loadBalancer.ingress[].hostname'
+++ sed -e 's/^"//; s/"$//;'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.OHWcgR056l
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.cr9QdJIUhN
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in $(seq 0 2)
+++ set +e
+++ kubectl get service/monitoring-service -o json
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.OHWcgR056l
+++ cat /tmp/tmp.cr9QdJIUhN
+++ rm /tmp/tmp.OHWcgR056l /tmp/tmp.cr9QdJIUhN
+++ return 0
++ local hostname=null
++ '[' -n null -a null '!=' null ']'
+++ kubectl_bin get service/monitoring-service -o json
+++ jq '.status.loadBalancer.ingress[].ip'
+++ sed -e 's/^"//; s/"$//;'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.P8h5n3LX3i
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.dIRR9MlukE
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in $(seq 0 2)
+++ set +e
+++ kubectl get service/monitoring-service -o json
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.P8h5n3LX3i
+++ cat /tmp/tmp.dIRR9MlukE
+++ rm /tmp/tmp.P8h5n3LX3i /tmp/tmp.dIRR9MlukE
+++ return 0
++ local ip=35.254.76.43
++ '[' -n 35.254.76.43 -a 35.254.76.43 '!=' null ']'
++ echo 35.254.76.43
++ return
+ local endpoint=35.254.76.43
+ '[' -z node_boot_time_seconds ']'
+ '[' -z glsa_ENtP2ImukR8cr3g1MllgZoNhKS1S8MWd_73f00929 ']'
+ local wait_count=30
+ local retry=0
++ execute_curl -s -k -H 'Authorization: Bearer glsa_ENtP2ImukR8cr3g1MllgZoNhKS1S8MWd_73f00929' 'https://35.254.76.43/graph/api/datasources/proxy/1/api/v1/query_range?query=min%28node_boot_time_seconds%7Bnode_name%3D%7E%22monitoring-pmm3-11763-monitoring-pmm3-cfg-1%22%7d%20or%20node_boot_time_seconds%7Bnode_name%3D%7E%22monitoring-pmm3-11763-monitoring-pmm3-cfg-1%22%7D%29&start=1783454444&end=1783454504&step=60'
++ jq '.data.result[0].values[][1]'
++ local name=curl-10639
++ local image=docker.io/alpine/curl:latest
++ kubectl_bin run curl-10639 -n monitoring-pmm3-11763 --restart=Never --image=docker.io/alpine/curl:latest --command -- sleep 3600
++ grep '^"[0-9]'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.PbhcWSoV0c
+++ mktemp
++ local LAST_ERR=/tmp/tmp.QN59yi9i89
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl run curl-10639 -n monitoring-pmm3-11763 --restart=Never --image=docker.io/alpine/curl:latest --command -- sleep 3600
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.PbhcWSoV0c
++ cat /tmp/tmp.QN59yi9i89
++ rm /tmp/tmp.PbhcWSoV0c /tmp/tmp.QN59yi9i89
++ return 0
++ kubectl_bin wait pod/curl-10639 -n monitoring-pmm3-11763 --for=condition=Ready --timeout=60s
+++ mktemp
++ local LAST_OUT=/tmp/tmp.rN5v2ZSe1j
+++ mktemp
++ local LAST_ERR=/tmp/tmp.xBdtAihqN8
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl wait pod/curl-10639 -n monitoring-pmm3-11763 --for=condition=Ready --timeout=60s
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.rN5v2ZSe1j
++ cat /tmp/tmp.xBdtAihqN8
++ rm /tmp/tmp.rN5v2ZSe1j /tmp/tmp.xBdtAihqN8
++ return 0
++ local exit_status=0
++ kubectl_bin exec curl-10639 -n monitoring-pmm3-11763 -- curl -s -k -H 'Authorization: Bearer glsa_ENtP2ImukR8cr3g1MllgZoNhKS1S8MWd_73f00929' 'https://35.254.76.43/graph/api/datasources/proxy/1/api/v1/query_range?query=min%28node_boot_time_seconds%7Bnode_name%3D%7E%22monitoring-pmm3-11763-monitoring-pmm3-cfg-1%22%7d%20or%20node_boot_time_seconds%7Bnode_name%3D%7E%22monitoring-pmm3-11763-monitoring-pmm3-cfg-1%22%7D%29&start=1783454444&end=1783454504&step=60'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.TM4WN8Etbv
+++ mktemp
++ local LAST_ERR=/tmp/tmp.KzzFHyhxOb
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl exec curl-10639 -n monitoring-pmm3-11763 -- curl -s -k -H 'Authorization: Bearer glsa_ENtP2ImukR8cr3g1MllgZoNhKS1S8MWd_73f00929' 'https://35.254.76.43/graph/api/datasources/proxy/1/api/v1/query_range?query=min%28node_boot_time_seconds%7Bnode_name%3D%7E%22monitoring-pmm3-11763-monitoring-pmm3-cfg-1%22%7d%20or%20node_boot_time_seconds%7Bnode_name%3D%7E%22monitoring-pmm3-11763-monitoring-pmm3-cfg-1%22%7D%29&start=1783454444&end=1783454504&step=60'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.TM4WN8Etbv
++ cat /tmp/tmp.KzzFHyhxOb
++ rm /tmp/tmp.TM4WN8Etbv /tmp/tmp.KzzFHyhxOb
++ return 0
++ kubectl_bin delete pod curl-10639 -n monitoring-pmm3-11763 --ignore-not-found --wait=false
++ return 0
+ [[ -n "1783444860"
"1783444860" ]]
+ get_metric_values mongodb_connections monitoring-pmm3-11763-monitoring-pmm3-cfg-1 glsa_ENtP2ImukR8cr3g1MllgZoNhKS1S8MWd_73f00929
+ local metric=mongodb_connections
+ local instance=monitoring-pmm3-11763-monitoring-pmm3-cfg-1
+ local token=glsa_ENtP2ImukR8cr3g1MllgZoNhKS1S8MWd_73f00929
++ /usr/sbin/date -u +%s -d '-1 minute'
+ local start=1783454450
++ /usr/sbin/date -u +%s
+ local end=1783454510
++ get_service_endpoint monitoring-service
++ local service=monitoring-service
+++ kubectl_bin get service/monitoring-service -o json
+++ sed -e 's/^"//; s/"$//;'
+++ jq '.status.loadBalancer.ingress[].hostname'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.J2zmevaXAU
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.GgHQzZ5vDy
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in $(seq 0 2)
+++ set +e
+++ kubectl get service/monitoring-service -o json
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.J2zmevaXAU
+++ cat /tmp/tmp.GgHQzZ5vDy
+++ rm /tmp/tmp.J2zmevaXAU /tmp/tmp.GgHQzZ5vDy
+++ return 0
++ local hostname=null
++ '[' -n null -a null '!=' null ']'
+++ kubectl_bin get service/monitoring-service -o json
+++ jq '.status.loadBalancer.ingress[].ip'
++++ mktemp
+++ sed -e 's/^"//; s/"$//;'
+++ local LAST_OUT=/tmp/tmp.SKxkYmwv8M
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.rtXIBxZdSe
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in $(seq 0 2)
+++ set +e
+++ kubectl get service/monitoring-service -o json
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.SKxkYmwv8M
+++ cat /tmp/tmp.rtXIBxZdSe
+++ rm /tmp/tmp.SKxkYmwv8M /tmp/tmp.rtXIBxZdSe
+++ return 0
++ local ip=35.254.76.43
++ '[' -n 35.254.76.43 -a 35.254.76.43 '!=' null ']'
++ echo 35.254.76.43
++ return
+ local endpoint=35.254.76.43
+ '[' -z mongodb_connections ']'
+ '[' -z glsa_ENtP2ImukR8cr3g1MllgZoNhKS1S8MWd_73f00929 ']'
+ local wait_count=30
+ local retry=0
++ execute_curl -s -k -H 'Authorization: Bearer glsa_ENtP2ImukR8cr3g1MllgZoNhKS1S8MWd_73f00929' 'https://35.254.76.43/graph/api/datasources/proxy/1/api/v1/query_range?query=min%28mongodb_connections%7Bnode_name%3D%7E%22monitoring-pmm3-11763-monitoring-pmm3-cfg-1%22%7d%20or%20mongodb_connections%7Bnode_name%3D%7E%22monitoring-pmm3-11763-monitoring-pmm3-cfg-1%22%7D%29&start=1783454450&end=1783454510&step=60'
++ jq '.data.result[0].values[][1]'
++ local name=curl-3153
++ local image=docker.io/alpine/curl:latest
++ grep '^"[0-9]'
++ kubectl_bin run curl-3153 -n monitoring-pmm3-11763 --restart=Never --image=docker.io/alpine/curl:latest --command -- sleep 3600
+++ mktemp
++ local LAST_OUT=/tmp/tmp.sm7jtOM74q
+++ mktemp
++ local LAST_ERR=/tmp/tmp.oA9bKXQeOn
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl run curl-3153 -n monitoring-pmm3-11763 --restart=Never --image=docker.io/alpine/curl:latest --command -- sleep 3600
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.sm7jtOM74q
++ cat /tmp/tmp.oA9bKXQeOn
++ rm /tmp/tmp.sm7jtOM74q /tmp/tmp.oA9bKXQeOn
++ return 0
++ kubectl_bin wait pod/curl-3153 -n monitoring-pmm3-11763 --for=condition=Ready --timeout=60s
+++ mktemp
++ local LAST_OUT=/tmp/tmp.X3BTmAyw1I
+++ mktemp
++ local LAST_ERR=/tmp/tmp.QqRlxbQwse
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl wait pod/curl-3153 -n monitoring-pmm3-11763 --for=condition=Ready --timeout=60s
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.X3BTmAyw1I
++ cat /tmp/tmp.QqRlxbQwse
++ rm /tmp/tmp.X3BTmAyw1I /tmp/tmp.QqRlxbQwse
++ return 0
++ local exit_status=0
++ kubectl_bin exec curl-3153 -n monitoring-pmm3-11763 -- curl -s -k -H 'Authorization: Bearer glsa_ENtP2ImukR8cr3g1MllgZoNhKS1S8MWd_73f00929' 'https://35.254.76.43/graph/api/datasources/proxy/1/api/v1/query_range?query=min%28mongodb_connections%7Bnode_name%3D%7E%22monitoring-pmm3-11763-monitoring-pmm3-cfg-1%22%7d%20or%20mongodb_connections%7Bnode_name%3D%7E%22monitoring-pmm3-11763-monitoring-pmm3-cfg-1%22%7D%29&start=1783454450&end=1783454510&step=60'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.PPlwdj4jha
+++ mktemp
++ local LAST_ERR=/tmp/tmp.GxH4dpvocc
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl exec curl-3153 -n monitoring-pmm3-11763 -- curl -s -k -H 'Authorization: Bearer glsa_ENtP2ImukR8cr3g1MllgZoNhKS1S8MWd_73f00929' 'https://35.254.76.43/graph/api/datasources/proxy/1/api/v1/query_range?query=min%28mongodb_connections%7Bnode_name%3D%7E%22monitoring-pmm3-11763-monitoring-pmm3-cfg-1%22%7d%20or%20mongodb_connections%7Bnode_name%3D%7E%22monitoring-pmm3-11763-monitoring-pmm3-cfg-1%22%7D%29&start=1783454450&end=1783454510&step=60'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.PPlwdj4jha
++ cat /tmp/tmp.GxH4dpvocc
++ rm /tmp/tmp.PPlwdj4jha /tmp/tmp.GxH4dpvocc
++ return 0
++ kubectl_bin delete pod curl-3153 -n monitoring-pmm3-11763 --ignore-not-found --wait=false
++ return 0
+ [[ -n "0"
"0" ]]
+ desc 'check mongos metrics'
+ set +o xtrace


-----------------------------------------------------------------------------------
check mongos metrics
-----------------------------------------------------------------------------------

++ kubectl get pod -l app.kubernetes.io/component=mongos -o 'jsonpath={.items[0].metadata.name}'
+ MONGOS_POD_NAME=monitoring-pmm3-mongos-0
+ get_metric_values node_boot_time_seconds monitoring-pmm3-11763-monitoring-pmm3-mongos-0 glsa_ENtP2ImukR8cr3g1MllgZoNhKS1S8MWd_73f00929
+ local metric=node_boot_time_seconds
+ local instance=monitoring-pmm3-11763-monitoring-pmm3-mongos-0
+ local token=glsa_ENtP2ImukR8cr3g1MllgZoNhKS1S8MWd_73f00929
++ /usr/sbin/date -u +%s -d '-1 minute'
+ local start=1783454457
++ /usr/sbin/date -u +%s
+ local end=1783454517
++ get_service_endpoint monitoring-service
++ local service=monitoring-service
+++ kubectl_bin get service/monitoring-service -o json
+++ jq '.status.loadBalancer.ingress[].hostname'
+++ sed -e 's/^"//; s/"$//;'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.TleiCsnhAu
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.3Ouj8vHRwC
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in $(seq 0 2)
+++ set +e
+++ kubectl get service/monitoring-service -o json
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.TleiCsnhAu
+++ cat /tmp/tmp.3Ouj8vHRwC
+++ rm /tmp/tmp.TleiCsnhAu /tmp/tmp.3Ouj8vHRwC
+++ return 0
++ local hostname=null
++ '[' -n null -a null '!=' null ']'
+++ kubectl_bin get service/monitoring-service -o json
+++ jq '.status.loadBalancer.ingress[].ip'
++++ mktemp
+++ sed -e 's/^"//; s/"$//;'
+++ local LAST_OUT=/tmp/tmp.CA0MA89dp1
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.O9NOvDwslI
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in $(seq 0 2)
+++ set +e
+++ kubectl get service/monitoring-service -o json
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.CA0MA89dp1
+++ cat /tmp/tmp.O9NOvDwslI
+++ rm /tmp/tmp.CA0MA89dp1 /tmp/tmp.O9NOvDwslI
+++ return 0
++ local ip=35.254.76.43
++ '[' -n 35.254.76.43 -a 35.254.76.43 '!=' null ']'
++ echo 35.254.76.43
++ return
+ local endpoint=35.254.76.43
+ '[' -z node_boot_time_seconds ']'
+ '[' -z glsa_ENtP2ImukR8cr3g1MllgZoNhKS1S8MWd_73f00929 ']'
+ local wait_count=30
+ local retry=0
++ jq '.data.result[0].values[][1]'
++ execute_curl -s -k -H 'Authorization: Bearer glsa_ENtP2ImukR8cr3g1MllgZoNhKS1S8MWd_73f00929' 'https://35.254.76.43/graph/api/datasources/proxy/1/api/v1/query_range?query=min%28node_boot_time_seconds%7Bnode_name%3D%7E%22monitoring-pmm3-11763-monitoring-pmm3-mongos-0%22%7d%20or%20node_boot_time_seconds%7Bnode_name%3D%7E%22monitoring-pmm3-11763-monitoring-pmm3-mongos-0%22%7D%29&start=1783454457&end=1783454517&step=60'
++ local name=curl-12647
++ local image=docker.io/alpine/curl:latest
++ kubectl_bin run curl-12647 -n monitoring-pmm3-11763 --restart=Never --image=docker.io/alpine/curl:latest --command -- sleep 3600
++ grep '^"[0-9]'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.mQf3S3LVGS
+++ mktemp
++ local LAST_ERR=/tmp/tmp.d4Kn0mUYKC
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl run curl-12647 -n monitoring-pmm3-11763 --restart=Never --image=docker.io/alpine/curl:latest --command -- sleep 3600
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.mQf3S3LVGS
++ cat /tmp/tmp.d4Kn0mUYKC
++ rm /tmp/tmp.mQf3S3LVGS /tmp/tmp.d4Kn0mUYKC
++ return 0
++ kubectl_bin wait pod/curl-12647 -n monitoring-pmm3-11763 --for=condition=Ready --timeout=60s
+++ mktemp
++ local LAST_OUT=/tmp/tmp.mS6vcnkq6k
+++ mktemp
++ local LAST_ERR=/tmp/tmp.KBjzJFjzoJ
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl wait pod/curl-12647 -n monitoring-pmm3-11763 --for=condition=Ready --timeout=60s
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.mS6vcnkq6k
++ cat /tmp/tmp.KBjzJFjzoJ
++ rm /tmp/tmp.mS6vcnkq6k /tmp/tmp.KBjzJFjzoJ
++ return 0
++ local exit_status=0
++ kubectl_bin exec curl-12647 -n monitoring-pmm3-11763 -- curl -s -k -H 'Authorization: Bearer glsa_ENtP2ImukR8cr3g1MllgZoNhKS1S8MWd_73f00929' 'https://35.254.76.43/graph/api/datasources/proxy/1/api/v1/query_range?query=min%28node_boot_time_seconds%7Bnode_name%3D%7E%22monitoring-pmm3-11763-monitoring-pmm3-mongos-0%22%7d%20or%20node_boot_time_seconds%7Bnode_name%3D%7E%22monitoring-pmm3-11763-monitoring-pmm3-mongos-0%22%7D%29&start=1783454457&end=1783454517&step=60'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.wVGcHBthhh
+++ mktemp
++ local LAST_ERR=/tmp/tmp.oiQd6tOtHY
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl exec curl-12647 -n monitoring-pmm3-11763 -- curl -s -k -H 'Authorization: Bearer glsa_ENtP2ImukR8cr3g1MllgZoNhKS1S8MWd_73f00929' 'https://35.254.76.43/graph/api/datasources/proxy/1/api/v1/query_range?query=min%28node_boot_time_seconds%7Bnode_name%3D%7E%22monitoring-pmm3-11763-monitoring-pmm3-mongos-0%22%7d%20or%20node_boot_time_seconds%7Bnode_name%3D%7E%22monitoring-pmm3-11763-monitoring-pmm3-mongos-0%22%7D%29&start=1783454457&end=1783454517&step=60'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.wVGcHBthhh
++ cat /tmp/tmp.oiQd6tOtHY
++ rm /tmp/tmp.wVGcHBthhh /tmp/tmp.oiQd6tOtHY
++ return 0
++ kubectl_bin delete pod curl-12647 -n monitoring-pmm3-11763 --ignore-not-found --wait=false
++ return 0
+ [[ -n "1783451415"
"1783451415" ]]
+ sleep 90
+ desc 'check QAN data'
+ set +o xtrace


-----------------------------------------------------------------------------------
check QAN data
-----------------------------------------------------------------------------------

+ get_qan_values mongodb dev-mongod glsa_ENtP2ImukR8cr3g1MllgZoNhKS1S8MWd_73f00929
+ local service_type=mongodb
+ local environment=dev-mongod
+ local token=glsa_ENtP2ImukR8cr3g1MllgZoNhKS1S8MWd_73f00929
+ local start
+ local end
+ local endpoint
++ /usr/sbin/date -u -d '-12 hour' +%Y-%m-%dT%H:%M:%S%:z
+ start=2026-07-07T08:03:33+00:00
++ /usr/sbin/date -u +%Y-%m-%dT%H:%M:%S%:z
+ end=2026-07-07T20:03:33+00:00
++ get_service_endpoint monitoring-service
++ local service=monitoring-service
+++ kubectl_bin get service/monitoring-service -o json
+++ jq '.status.loadBalancer.ingress[].hostname'
+++ sed -e 's/^"//; s/"$//;'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.pizjU1qH1Z
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.SsCrfd5pv4
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in $(seq 0 2)
+++ set +e
+++ kubectl get service/monitoring-service -o json
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.pizjU1qH1Z
+++ cat /tmp/tmp.SsCrfd5pv4
+++ rm /tmp/tmp.pizjU1qH1Z /tmp/tmp.SsCrfd5pv4
+++ return 0
++ local hostname=null
++ '[' -n null -a null '!=' null ']'
+++ kubectl_bin get service/monitoring-service -o json
+++ jq '.status.loadBalancer.ingress[].ip'
+++ sed -e 's/^"//; s/"$//;'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.cYXRUMdpO1
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.poXkFTqLD9
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in $(seq 0 2)
+++ set +e
+++ kubectl get service/monitoring-service -o json
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.cYXRUMdpO1
+++ cat /tmp/tmp.poXkFTqLD9
+++ rm /tmp/tmp.cYXRUMdpO1 /tmp/tmp.poXkFTqLD9
+++ return 0
++ local ip=35.254.76.43
++ '[' -n 35.254.76.43 -a 35.254.76.43 '!=' null ']'
++ echo 35.254.76.43
++ return
+ endpoint=35.254.76.43
+ cat
+ local response
+ local wait_count=30
+ local retry=0
++ jq -r '(.rows // [])[]? | .fingerprint // empty'
+++ cat payload.json
++ execute_curl -s -k -H 'Authorization: Bearer glsa_ENtP2ImukR8cr3g1MllgZoNhKS1S8MWd_73f00929' -XPOST -d $'{\n  "period_start_from": "2026-07-07T08:03:33+00:00",\n  "period_start_to": "2026-07-07T20:03:33+00:00",\n  "group_by": "queryid",\n  "labels": [\n    {\n      "key": "service_type",\n      "value": [\n        "mongodb"\n      ]\n    },\n    {\n      "key": "environment",\n      "value": [\n        "dev-mongod"\n      ]\n    }\n  ],\n  "columns": [\n    "load",\n    "num_queries",\n    "query_time"\n  ],\n  "order_by": "-load",\n  "offset": 0,\n  "limit": 10,\n  "main_metric": "load",\n  "search": ""\n}' https://35.254.76.43/v1/qan/metrics:getReport
++ local name=curl-16605
++ local image=docker.io/alpine/curl:latest
++ kubectl_bin run curl-16605 -n monitoring-pmm3-11763 --restart=Never --image=docker.io/alpine/curl:latest --command -- sleep 3600
+++ mktemp
++ local LAST_OUT=/tmp/tmp.gC5d0DXTcF
+++ mktemp
++ local LAST_ERR=/tmp/tmp.lrvCULxFHL
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl run curl-16605 -n monitoring-pmm3-11763 --restart=Never --image=docker.io/alpine/curl:latest --command -- sleep 3600
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.gC5d0DXTcF
++ cat /tmp/tmp.lrvCULxFHL
++ rm /tmp/tmp.gC5d0DXTcF /tmp/tmp.lrvCULxFHL
++ return 0
++ kubectl_bin wait pod/curl-16605 -n monitoring-pmm3-11763 --for=condition=Ready --timeout=60s
+++ mktemp
++ local LAST_OUT=/tmp/tmp.DosPq9XWGu
+++ mktemp
++ local LAST_ERR=/tmp/tmp.XcceYE5QDc
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl wait pod/curl-16605 -n monitoring-pmm3-11763 --for=condition=Ready --timeout=60s
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.DosPq9XWGu
++ cat /tmp/tmp.XcceYE5QDc
++ rm /tmp/tmp.DosPq9XWGu /tmp/tmp.XcceYE5QDc
++ return 0
++ local exit_status=0
++ kubectl_bin exec curl-16605 -n monitoring-pmm3-11763 -- curl -s -k -H 'Authorization: Bearer glsa_ENtP2ImukR8cr3g1MllgZoNhKS1S8MWd_73f00929' -XPOST -d $'{\n  "period_start_from": "2026-07-07T08:03:33+00:00",\n  "period_start_to": "2026-07-07T20:03:33+00:00",\n  "group_by": "queryid",\n  "labels": [\n    {\n      "key": "service_type",\n      "value": [\n        "mongodb"\n      ]\n    },\n    {\n      "key": "environment",\n      "value": [\n        "dev-mongod"\n      ]\n    }\n  ],\n  "columns": [\n    "load",\n    "num_queries",\n    "query_time"\n  ],\n  "order_by": "-load",\n  "offset": 0,\n  "limit": 10,\n  "main_metric": "load",\n  "search": ""\n}' https://35.254.76.43/v1/qan/metrics:getReport
+++ mktemp
++ local LAST_OUT=/tmp/tmp.OL0TrmQ2El
+++ mktemp
++ local LAST_ERR=/tmp/tmp.8oL8phqLl3
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl exec curl-16605 -n monitoring-pmm3-11763 -- curl -s -k -H 'Authorization: Bearer glsa_ENtP2ImukR8cr3g1MllgZoNhKS1S8MWd_73f00929' -XPOST -d $'{\n  "period_start_from": "2026-07-07T08:03:33+00:00",\n  "period_start_to": "2026-07-07T20:03:33+00:00",\n  "group_by": "queryid",\n  "labels": [\n    {\n      "key": "service_type",\n      "value": [\n        "mongodb"\n      ]\n    },\n    {\n      "key": "environment",\n      "value": [\n        "dev-mongod"\n      ]\n    }\n  ],\n  "columns": [\n    "load",\n    "num_queries",\n    "query_time"\n  ],\n  "order_by": "-load",\n  "offset": 0,\n  "limit": 10,\n  "main_metric": "load",\n  "search": ""\n}' https://35.254.76.43/v1/qan/metrics:getReport
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.OL0TrmQ2El
++ cat /tmp/tmp.8oL8phqLl3
++ rm /tmp/tmp.OL0TrmQ2El /tmp/tmp.8oL8phqLl3
++ return 0
++ kubectl_bin delete pod curl-16605 -n monitoring-pmm3-11763 --ignore-not-found --wait=false
++ return 0
+ [[ -n TOTAL
db.cache.databases.update({"_id":"?"}, {"$set":{"_id":"?","primary":"?","version":{"lastMod":"?","timestamp":"?","uuid":"?"}}}, {"upsert":true})
db.system.sessions.update({"_id":{"id":"?","uid":"?"}}, [{"$set":{"lastUse":"?"}},{"$set":{"user":{"name":"?"}}}], {"upsert":true})
db.version.find({}).limit(?)
db.runCommand({"$audit":{"$impersonatedRoles":[{"db":"?","role":"?"},{"db":"?","role":"?"},{"db":"?","role":"?"},{"db":"?","role":"?"}],"$impersonatedUser":{"db":"?","user":"?"}},"$client":{"application":{"name":"?"},"driver":{"name":"?","version":"?"},"mongos":{"client":"?","host":"?","version":"?"},"os":{"architecture":"?","type":"?"},"platform":"?"},"$configTime":"?","$topologyTime":"?","dbStats":"?","mayBypassWriteBlocking":"?","readConcern":{"level":"?","provenance":"?"},"scale":"?"})
db.system.sessions.deleteOne({"_id":{"id":"?","uid":"?"}})
db.system.sessions.update({"_id":{"id":"?","uid":"?"}}, [{"$set":{"lastUse":"?"}}], {"upsert":true})
db.system.version.find({"_id":"?"}).limit(?)
db.oplog.rs.find({}).sort({"$natural":-1}).limit(?)
db.oplog.rs.find({}).sort({"$natural":1}).limit(?)
db.runCommand({"$audit":{"$impersonatedRoles":[{"db":"?","role":"?"},{"db":"?","role":"?"},{"db":"?","role":"?"},{"db":"?","role":"?"}],"$impersonatedUser":{"db":"?","user":"?"}},"$client":{"application":{"name":"?"},"driver":{"name":"?","version":"?"},"mongos":{"client":"?","host":"?","version":"?"},"os":{"architecture":"?","type":"?"},"platform":"?"},"$configTime":"?","$topologyTime":"?","dbStats":"?","mayBypassWriteBlocking":"?","readConcern":{"level":"?","provenance":"?"},"scale":"?"}) ]]
+ rm -f payload.json
+ get_qan_values mongodb dev-mongos glsa_ENtP2ImukR8cr3g1MllgZoNhKS1S8MWd_73f00929
+ local service_type=mongodb
+ local environment=dev-mongos
+ local token=glsa_ENtP2ImukR8cr3g1MllgZoNhKS1S8MWd_73f00929
+ local start
+ local end
+ local endpoint
++ /usr/sbin/date -u -d '-12 hour' +%Y-%m-%dT%H:%M:%S%:z
+ start=2026-07-07T08:03:41+00:00
++ /usr/sbin/date -u +%Y-%m-%dT%H:%M:%S%:z
+ end=2026-07-07T20:03:41+00:00
++ get_service_endpoint monitoring-service
++ local service=monitoring-service
+++ kubectl_bin get service/monitoring-service -o json
+++ jq '.status.loadBalancer.ingress[].hostname'
+++ sed -e 's/^"//; s/"$//;'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.Istrnf1mvE
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.nPvBVdZF88
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in $(seq 0 2)
+++ set +e
+++ kubectl get service/monitoring-service -o json
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.Istrnf1mvE
+++ cat /tmp/tmp.nPvBVdZF88
+++ rm /tmp/tmp.Istrnf1mvE /tmp/tmp.nPvBVdZF88
+++ return 0
++ local hostname=null
++ '[' -n null -a null '!=' null ']'
+++ kubectl_bin get service/monitoring-service -o json
+++ jq '.status.loadBalancer.ingress[].ip'
+++ sed -e 's/^"//; s/"$//;'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.CrPzy5diTl
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.Iju1HOrKDh
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in $(seq 0 2)
+++ set +e
+++ kubectl get service/monitoring-service -o json
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.CrPzy5diTl
+++ cat /tmp/tmp.Iju1HOrKDh
+++ rm /tmp/tmp.CrPzy5diTl /tmp/tmp.Iju1HOrKDh
+++ return 0
++ local ip=35.254.76.43
++ '[' -n 35.254.76.43 -a 35.254.76.43 '!=' null ']'
++ echo 35.254.76.43
++ return
+ endpoint=35.254.76.43
+ cat
+ local response
+ local wait_count=30
+ local retry=0
++ jq -r '(.rows // [])[]? | .fingerprint // empty'
+++ cat payload.json
++ execute_curl -s -k -H 'Authorization: Bearer glsa_ENtP2ImukR8cr3g1MllgZoNhKS1S8MWd_73f00929' -XPOST -d $'{\n  "period_start_from": "2026-07-07T08:03:41+00:00",\n  "period_start_to": "2026-07-07T20:03:41+00:00",\n  "group_by": "queryid",\n  "labels": [\n    {\n      "key": "service_type",\n      "value": [\n        "mongodb"\n      ]\n    },\n    {\n      "key": "environment",\n      "value": [\n        "dev-mongos"\n      ]\n    }\n  ],\n  "columns": [\n    "load",\n    "num_queries",\n    "query_time"\n  ],\n  "order_by": "-load",\n  "offset": 0,\n  "limit": 10,\n  "main_metric": "load",\n  "search": ""\n}' https://35.254.76.43/v1/qan/metrics:getReport
++ local name=curl-1840
++ local image=docker.io/alpine/curl:latest
++ kubectl_bin run curl-1840 -n monitoring-pmm3-11763 --restart=Never --image=docker.io/alpine/curl:latest --command -- sleep 3600
+++ mktemp
++ local LAST_OUT=/tmp/tmp.cbS71rvkj9
+++ mktemp
++ local LAST_ERR=/tmp/tmp.6ZuiE8nCst
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl run curl-1840 -n monitoring-pmm3-11763 --restart=Never --image=docker.io/alpine/curl:latest --command -- sleep 3600
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.cbS71rvkj9
++ cat /tmp/tmp.6ZuiE8nCst
++ rm /tmp/tmp.cbS71rvkj9 /tmp/tmp.6ZuiE8nCst
++ return 0
++ kubectl_bin wait pod/curl-1840 -n monitoring-pmm3-11763 --for=condition=Ready --timeout=60s
+++ mktemp
++ local LAST_OUT=/tmp/tmp.4kUf1QbHsD
+++ mktemp
++ local LAST_ERR=/tmp/tmp.4V8ZZmST0K
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl wait pod/curl-1840 -n monitoring-pmm3-11763 --for=condition=Ready --timeout=60s
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.4kUf1QbHsD
++ cat /tmp/tmp.4V8ZZmST0K
++ rm /tmp/tmp.4kUf1QbHsD /tmp/tmp.4V8ZZmST0K
++ return 0
++ local exit_status=0
++ kubectl_bin exec curl-1840 -n monitoring-pmm3-11763 -- curl -s -k -H 'Authorization: Bearer glsa_ENtP2ImukR8cr3g1MllgZoNhKS1S8MWd_73f00929' -XPOST -d $'{\n  "period_start_from": "2026-07-07T08:03:41+00:00",\n  "period_start_to": "2026-07-07T20:03:41+00:00",\n  "group_by": "queryid",\n  "labels": [\n    {\n      "key": "service_type",\n      "value": [\n        "mongodb"\n      ]\n    },\n    {\n      "key": "environment",\n      "value": [\n        "dev-mongos"\n      ]\n    }\n  ],\n  "columns": [\n    "load",\n    "num_queries",\n    "query_time"\n  ],\n  "order_by": "-load",\n  "offset": 0,\n  "limit": 10,\n  "main_metric": "load",\n  "search": ""\n}' https://35.254.76.43/v1/qan/metrics:getReport
+++ mktemp
++ local LAST_OUT=/tmp/tmp.L15sW7pzco
+++ mktemp
++ local LAST_ERR=/tmp/tmp.CalrqQlwe3
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl exec curl-1840 -n monitoring-pmm3-11763 -- curl -s -k -H 'Authorization: Bearer glsa_ENtP2ImukR8cr3g1MllgZoNhKS1S8MWd_73f00929' -XPOST -d $'{\n  "period_start_from": "2026-07-07T08:03:41+00:00",\n  "period_start_to": "2026-07-07T20:03:41+00:00",\n  "group_by": "queryid",\n  "labels": [\n    {\n      "key": "service_type",\n      "value": [\n        "mongodb"\n      ]\n    },\n    {\n      "key": "environment",\n      "value": [\n        "dev-mongos"\n      ]\n    }\n  ],\n  "columns": [\n    "load",\n    "num_queries",\n    "query_time"\n  ],\n  "order_by": "-load",\n  "offset": 0,\n  "limit": 10,\n  "main_metric": "load",\n  "search": ""\n}' https://35.254.76.43/v1/qan/metrics:getReport
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.L15sW7pzco
++ cat /tmp/tmp.CalrqQlwe3
++ rm /tmp/tmp.L15sW7pzco /tmp/tmp.CalrqQlwe3
++ return 0
++ kubectl_bin delete pod curl-1840 -n monitoring-pmm3-11763 --ignore-not-found --wait=false
++ return 0
+ [[ -n TOTAL
db.runCommand({"$audit":{"$impersonatedRoles":[{"db":"?","role":"?"},{"db":"?","role":"?"},{"db":"?","role":"?"},{"db":"?","role":"?"}],"$impersonatedUser":{"db":"?","user":"?"}},"$client":{"application":{"name":"?"},"driver":{"name":"?","version":"?"},"mongos":{"client":"?","host":"?","version":"?"},"os":{"architecture":"?","type":"?"},"platform":"?"},"$configTime":"?","$topologyTime":"?","dbStats":"?","mayBypassWriteBlocking":"?","readConcern":{"level":"?","provenance":"?"},"scale":"?"}) ]]
+ rm -f payload.json
+ desc 'verify that the custom cluster name is configured'
+ set +o xtrace


-----------------------------------------------------------------------------------
verify that the custom cluster name is configured
-----------------------------------------------------------------------------------

+ verify_custom_cluster_name super-custom glsa_ENtP2ImukR8cr3g1MllgZoNhKS1S8MWd_73f00929 monitoring-pmm3-11763-monitoring-pmm3-mongos-0 monitoring-pmm3-11763-monitoring-pmm3-cfg-0 monitoring-pmm3-11763-monitoring-pmm3-rs0-0
+ local expected_cluster=super-custom
+ local token=glsa_ENtP2ImukR8cr3g1MllgZoNhKS1S8MWd_73f00929
+ shift 2
+ service_names=('monitoring-pmm3-11763-monitoring-pmm3-mongos-0' 'monitoring-pmm3-11763-monitoring-pmm3-cfg-0' 'monitoring-pmm3-11763-monitoring-pmm3-rs0-0')
+ local service_names
+ local endpoint
++ get_service_endpoint monitoring-service
++ local service=monitoring-service
+++ kubectl_bin get service/monitoring-service -o json
+++ jq '.status.loadBalancer.ingress[].hostname'
+++ sed -e 's/^"//; s/"$//;'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.htE0soQcPb
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.Y8gof27UzD
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in $(seq 0 2)
+++ set +e
+++ kubectl get service/monitoring-service -o json
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.htE0soQcPb
+++ cat /tmp/tmp.Y8gof27UzD
+++ rm /tmp/tmp.htE0soQcPb /tmp/tmp.Y8gof27UzD
+++ return 0
++ local hostname=null
++ '[' -n null -a null '!=' null ']'
+++ kubectl_bin get service/monitoring-service -o json
+++ jq '.status.loadBalancer.ingress[].ip'
+++ sed -e 's/^"//; s/"$//;'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.I3G0WW3VaR
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.K7yrbDjKuL
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in $(seq 0 2)
+++ set +e
+++ kubectl get service/monitoring-service -o json
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.I3G0WW3VaR
+++ cat /tmp/tmp.K7yrbDjKuL
+++ rm /tmp/tmp.I3G0WW3VaR /tmp/tmp.K7yrbDjKuL
+++ return 0
++ local ip=35.254.76.43
++ '[' -n 35.254.76.43 -a 35.254.76.43 '!=' null ']'
++ echo 35.254.76.43
++ return
+ endpoint=35.254.76.43
+ local response
++ execute_curl -s -k -H 'Authorization: Bearer glsa_ENtP2ImukR8cr3g1MllgZoNhKS1S8MWd_73f00929' 'https://35.254.76.43/v1/inventory/services?service_type=SERVICE_TYPE_MONGODB_SERVICE'
++ local name=curl-21856
++ local image=docker.io/alpine/curl:latest
++ kubectl_bin run curl-21856 -n monitoring-pmm3-11763 --restart=Never --image=docker.io/alpine/curl:latest --command -- sleep 3600
+++ mktemp
++ local LAST_OUT=/tmp/tmp.lRpErKMEHo
+++ mktemp
++ local LAST_ERR=/tmp/tmp.DQ0j2zYkXr
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl run curl-21856 -n monitoring-pmm3-11763 --restart=Never --image=docker.io/alpine/curl:latest --command -- sleep 3600
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.lRpErKMEHo
++ cat /tmp/tmp.DQ0j2zYkXr
++ rm /tmp/tmp.lRpErKMEHo /tmp/tmp.DQ0j2zYkXr
++ return 0
++ kubectl_bin wait pod/curl-21856 -n monitoring-pmm3-11763 --for=condition=Ready --timeout=60s
+++ mktemp
++ local LAST_OUT=/tmp/tmp.DBLADG0izF
+++ mktemp
++ local LAST_ERR=/tmp/tmp.CasGRPT6xG
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl wait pod/curl-21856 -n monitoring-pmm3-11763 --for=condition=Ready --timeout=60s
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.DBLADG0izF
++ cat /tmp/tmp.CasGRPT6xG
++ rm /tmp/tmp.DBLADG0izF /tmp/tmp.CasGRPT6xG
++ return 0
++ local exit_status=0
++ kubectl_bin exec curl-21856 -n monitoring-pmm3-11763 -- curl -s -k -H 'Authorization: Bearer glsa_ENtP2ImukR8cr3g1MllgZoNhKS1S8MWd_73f00929' 'https://35.254.76.43/v1/inventory/services?service_type=SERVICE_TYPE_MONGODB_SERVICE'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.R5aiHemhsc
+++ mktemp
++ local LAST_ERR=/tmp/tmp.iD36Vl7BUA
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl exec curl-21856 -n monitoring-pmm3-11763 -- curl -s -k -H 'Authorization: Bearer glsa_ENtP2ImukR8cr3g1MllgZoNhKS1S8MWd_73f00929' 'https://35.254.76.43/v1/inventory/services?service_type=SERVICE_TYPE_MONGODB_SERVICE'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.R5aiHemhsc
++ cat /tmp/tmp.iD36Vl7BUA
++ rm /tmp/tmp.R5aiHemhsc /tmp/tmp.iD36Vl7BUA
++ return 0
++ kubectl_bin delete pod curl-21856 -n monitoring-pmm3-11763 --ignore-not-found --wait=false
++ return 0
+ response=$'{\n  "mysql": [],\n  "mongodb": [\n    {\n      "service_id": "1cb05588-6161-421a-9588-bd3903571099",\n      "service_name": "monitoring-pmm3-11763-monitoring-pmm3-mongos-2",\n      "node_id": "1c9723e3-915b-48aa-8802-110a65a38734",\n      "address": "localhost",\n      "port": 27019,\n      "socket": "",\n      "environment": "dev-mongos",\n      "cluster": "super-custom",\n      "replication_set": "",\n      "custom_labels": {},\n      "version": ""\n    },\n    {\n      "service_id": "1e944cc7-0fb6-4979-af79-c4e0b6278889",\n      "service_name": "monitoring-pmm3-11763-monitoring-pmm3-cfg-2",\n      "node_id": "7c8bf6ae-311d-46b8-bdd1-8dd64f649283",\n      "address": "localhost",\n      "port": 27019,\n      "socket": "",\n      "environment": "dev-mongod",\n      "cluster": "super-custom",\n      "replication_set": "",\n      "custom_labels": {},\n      "version": ""\n    },\n    {\n      "service_id": "4b6b273c-d335-4409-a866-a4b1cfc58c6b",\n      "service_name": "monitoring-pmm3-11763-monitoring-pmm3-rs0-1",\n      "node_id": "13549544-fc7f-45c7-89f3-c3cf312dbf57",\n      "address": "localhost",\n      "port": 27019,\n      "socket": "",\n      "environment": "dev-mongod",\n      "cluster": "super-custom",\n      "replication_set": "",\n      "custom_labels": {},\n      "version": ""\n    },\n    {\n      "service_id": "52b0d4c8-1ce3-44dd-95ed-9bef3e84fd81",\n      "service_name": "monitoring-pmm3-11763-monitoring-pmm3-rs0-0",\n      "node_id": "d67f78f3-1137-46d0-af49-c33ae7ebe927",\n      "address": "localhost",\n      "port": 27019,\n      "socket": "",\n      "environment": "dev-mongod",\n      "cluster": "super-custom",\n      "replication_set": "",\n      "custom_labels": {},\n      "version": ""\n    },\n    {\n      "service_id": "8319ce49-5405-4a73-a663-3cc7d0dce024",\n      "service_name": "monitoring-pmm3-11763-monitoring-pmm3-rs0-2",\n      "node_id": "f523c611-01f0-4636-999b-16de7ab7a399",\n      "address": "localhost",\n      "port": 27019,\n      "socket": "",\n      "environment": "dev-mongod",\n      "cluster": "super-custom",\n      "replication_set": "",\n      "custom_labels": {},\n      "version": ""\n    },\n    {\n      "service_id": "b4a57260-9efe-47fc-9519-8184779c37c0",\n      "service_name": "monitoring-pmm3-11763-monitoring-pmm3-mongos-1",\n      "node_id": "fa4bfc6f-11ec-42d2-9cc8-d216645f789d",\n      "address": "localhost",\n      "port": 27019,\n      "socket": "",\n      "environment": "dev-mongos",\n      "cluster": "super-custom",\n      "replication_set": "",\n      "custom_labels": {},\n      "version": ""\n    },\n    {\n      "service_id": "bff3eb5c-f2fd-4058-bf2a-37338c743f80",\n      "service_name": "monitoring-pmm3-11763-monitoring-pmm3-cfg-0",\n      "node_id": "cccaab90-d90a-47fe-b618-5e42a0b21edc",\n      "address": "localhost",\n      "port": 27019,\n      "socket": "",\n      "environment": "dev-mongod",\n      "cluster": "super-custom",\n      "replication_set": "",\n      "custom_labels": {},\n      "version": ""\n    },\n    {\n      "service_id": "ca22cb1c-5fda-4ecb-9ff4-7ea082c52aa8",\n      "service_name": "monitoring-pmm3-11763-monitoring-pmm3-cfg-1",\n      "node_id": "b71c4a17-6c43-4666-9275-d206aa45f397",\n      "address": "localhost",\n      "port": 27019,\n      "socket": "",\n      "environment": "dev-mongod",\n      "cluster": "super-custom",\n      "replication_set": "",\n      "custom_labels": {},\n      "version": ""\n    },\n    {\n      "service_id": "d6371ea3-7e68-4c7f-a786-c3cba3508ed8",\n      "service_name": "monitoring-pmm3-11763-monitoring-pmm3-mongos-0",\n      "node_id": "e97e0000-593c-4f24-bd3c-ccf938a451bd",\n      "address": "localhost",\n      "port": 27019,\n      "socket": "",\n      "environment": "dev-mongos",\n      "cluster": "super-custom",\n      "replication_set": "",\n      "custom_labels": {},\n      "version": ""\n    }\n  ],\n  "postgresql": [],\n  "proxysql": [],\n  "haproxy": [],\n  "external": [],\n  "valkey": []\n}'
+ local verified=0
+ for service_name in "${service_names[@]}"
+ local actual_cluster
++ jq -r --arg name monitoring-pmm3-11763-monitoring-pmm3-mongos-0 $'\n\t\t\t.mongodb[] | select(.service_name == $name) | .cluster\n\t\t'
++ echo $'{\n  "mysql": [],\n  "mongodb": [\n    {\n      "service_id": "1cb05588-6161-421a-9588-bd3903571099",\n      "service_name": "monitoring-pmm3-11763-monitoring-pmm3-mongos-2",\n      "node_id": "1c9723e3-915b-48aa-8802-110a65a38734",\n      "address": "localhost",\n      "port": 27019,\n      "socket": "",\n      "environment": "dev-mongos",\n      "cluster": "super-custom",\n      "replication_set": "",\n      "custom_labels": {},\n      "version": ""\n    },\n    {\n      "service_id": "1e944cc7-0fb6-4979-af79-c4e0b6278889",\n      "service_name": "monitoring-pmm3-11763-monitoring-pmm3-cfg-2",\n      "node_id": "7c8bf6ae-311d-46b8-bdd1-8dd64f649283",\n      "address": "localhost",\n      "port": 27019,\n      "socket": "",\n      "environment": "dev-mongod",\n      "cluster": "super-custom",\n      "replication_set": "",\n      "custom_labels": {},\n      "version": ""\n    },\n    {\n      "service_id": "4b6b273c-d335-4409-a866-a4b1cfc58c6b",\n      "service_name": "monitoring-pmm3-11763-monitoring-pmm3-rs0-1",\n      "node_id": "13549544-fc7f-45c7-89f3-c3cf312dbf57",\n      "address": "localhost",\n      "port": 27019,\n      "socket": "",\n      "environment": "dev-mongod",\n      "cluster": "super-custom",\n      "replication_set": "",\n      "custom_labels": {},\n      "version": ""\n    },\n    {\n      "service_id": "52b0d4c8-1ce3-44dd-95ed-9bef3e84fd81",\n      "service_name": "monitoring-pmm3-11763-monitoring-pmm3-rs0-0",\n      "node_id": "d67f78f3-1137-46d0-af49-c33ae7ebe927",\n      "address": "localhost",\n      "port": 27019,\n      "socket": "",\n      "environment": "dev-mongod",\n      "cluster": "super-custom",\n      "replication_set": "",\n      "custom_labels": {},\n      "version": ""\n    },\n    {\n      "service_id": "8319ce49-5405-4a73-a663-3cc7d0dce024",\n      "service_name": "monitoring-pmm3-11763-monitoring-pmm3-rs0-2",\n      "node_id": "f523c611-01f0-4636-999b-16de7ab7a399",\n      "address": "localhost",\n      "port": 27019,\n      "socket": "",\n      "environment": "dev-mongod",\n      "cluster": "super-custom",\n      "replication_set": "",\n      "custom_labels": {},\n      "version": ""\n    },\n    {\n      "service_id": "b4a57260-9efe-47fc-9519-8184779c37c0",\n      "service_name": "monitoring-pmm3-11763-monitoring-pmm3-mongos-1",\n      "node_id": "fa4bfc6f-11ec-42d2-9cc8-d216645f789d",\n      "address": "localhost",\n      "port": 27019,\n      "socket": "",\n      "environment": "dev-mongos",\n      "cluster": "super-custom",\n      "replication_set": "",\n      "custom_labels": {},\n      "version": ""\n    },\n    {\n      "service_id": "bff3eb5c-f2fd-4058-bf2a-37338c743f80",\n      "service_name": "monitoring-pmm3-11763-monitoring-pmm3-cfg-0",\n      "node_id": "cccaab90-d90a-47fe-b618-5e42a0b21edc",\n      "address": "localhost",\n      "port": 27019,\n      "socket": "",\n      "environment": "dev-mongod",\n      "cluster": "super-custom",\n      "replication_set": "",\n      "custom_labels": {},\n      "version": ""\n    },\n    {\n      "service_id": "ca22cb1c-5fda-4ecb-9ff4-7ea082c52aa8",\n      "service_name": "monitoring-pmm3-11763-monitoring-pmm3-cfg-1",\n      "node_id": "b71c4a17-6c43-4666-9275-d206aa45f397",\n      "address": "localhost",\n      "port": 27019,\n      "socket": "",\n      "environment": "dev-mongod",\n      "cluster": "super-custom",\n      "replication_set": "",\n      "custom_labels": {},\n      "version": ""\n    },\n    {\n      "service_id": "d6371ea3-7e68-4c7f-a786-c3cba3508ed8",\n      "service_name": "monitoring-pmm3-11763-monitoring-pmm3-mongos-0",\n      "node_id": "e97e0000-593c-4f24-bd3c-ccf938a451bd",\n      "address": "localhost",\n      "port": 27019,\n      "socket": "",\n      "environment": "dev-mongos",\n      "cluster": "super-custom",\n      "replication_set": "",\n      "custom_labels": {},\n      "version": ""\n    }\n  ],\n  "postgresql": [],\n  "proxysql": [],\n  "haproxy": [],\n  "external": [],\n  "valkey": []\n}'
+ actual_cluster=super-custom
+ [[ -z super-custom ]]
+ [[ super-custom == null ]]
+ [[ super-custom != super\-custom ]]
+ for service_name in "${service_names[@]}"
+ local actual_cluster
++ jq -r --arg name monitoring-pmm3-11763-monitoring-pmm3-cfg-0 $'\n\t\t\t.mongodb[] | select(.service_name == $name) | .cluster\n\t\t'
++ echo $'{\n  "mysql": [],\n  "mongodb": [\n    {\n      "service_id": "1cb05588-6161-421a-9588-bd3903571099",\n      "service_name": "monitoring-pmm3-11763-monitoring-pmm3-mongos-2",\n      "node_id": "1c9723e3-915b-48aa-8802-110a65a38734",\n      "address": "localhost",\n      "port": 27019,\n      "socket": "",\n      "environment": "dev-mongos",\n      "cluster": "super-custom",\n      "replication_set": "",\n      "custom_labels": {},\n      "version": ""\n    },\n    {\n      "service_id": "1e944cc7-0fb6-4979-af79-c4e0b6278889",\n      "service_name": "monitoring-pmm3-11763-monitoring-pmm3-cfg-2",\n      "node_id": "7c8bf6ae-311d-46b8-bdd1-8dd64f649283",\n      "address": "localhost",\n      "port": 27019,\n      "socket": "",\n      "environment": "dev-mongod",\n      "cluster": "super-custom",\n      "replication_set": "",\n      "custom_labels": {},\n      "version": ""\n    },\n    {\n      "service_id": "4b6b273c-d335-4409-a866-a4b1cfc58c6b",\n      "service_name": "monitoring-pmm3-11763-monitoring-pmm3-rs0-1",\n      "node_id": "13549544-fc7f-45c7-89f3-c3cf312dbf57",\n      "address": "localhost",\n      "port": 27019,\n      "socket": "",\n      "environment": "dev-mongod",\n      "cluster": "super-custom",\n      "replication_set": "",\n      "custom_labels": {},\n      "version": ""\n    },\n    {\n      "service_id": "52b0d4c8-1ce3-44dd-95ed-9bef3e84fd81",\n      "service_name": "monitoring-pmm3-11763-monitoring-pmm3-rs0-0",\n      "node_id": "d67f78f3-1137-46d0-af49-c33ae7ebe927",\n      "address": "localhost",\n      "port": 27019,\n      "socket": "",\n      "environment": "dev-mongod",\n      "cluster": "super-custom",\n      "replication_set": "",\n      "custom_labels": {},\n      "version": ""\n    },\n    {\n      "service_id": "8319ce49-5405-4a73-a663-3cc7d0dce024",\n      "service_name": "monitoring-pmm3-11763-monitoring-pmm3-rs0-2",\n      "node_id": "f523c611-01f0-4636-999b-16de7ab7a399",\n      "address": "localhost",\n      "port": 27019,\n      "socket": "",\n      "environment": "dev-mongod",\n      "cluster": "super-custom",\n      "replication_set": "",\n      "custom_labels": {},\n      "version": ""\n    },\n    {\n      "service_id": "b4a57260-9efe-47fc-9519-8184779c37c0",\n      "service_name": "monitoring-pmm3-11763-monitoring-pmm3-mongos-1",\n      "node_id": "fa4bfc6f-11ec-42d2-9cc8-d216645f789d",\n      "address": "localhost",\n      "port": 27019,\n      "socket": "",\n      "environment": "dev-mongos",\n      "cluster": "super-custom",\n      "replication_set": "",\n      "custom_labels": {},\n      "version": ""\n    },\n    {\n      "service_id": "bff3eb5c-f2fd-4058-bf2a-37338c743f80",\n      "service_name": "monitoring-pmm3-11763-monitoring-pmm3-cfg-0",\n      "node_id": "cccaab90-d90a-47fe-b618-5e42a0b21edc",\n      "address": "localhost",\n      "port": 27019,\n      "socket": "",\n      "environment": "dev-mongod",\n      "cluster": "super-custom",\n      "replication_set": "",\n      "custom_labels": {},\n      "version": ""\n    },\n    {\n      "service_id": "ca22cb1c-5fda-4ecb-9ff4-7ea082c52aa8",\n      "service_name": "monitoring-pmm3-11763-monitoring-pmm3-cfg-1",\n      "node_id": "b71c4a17-6c43-4666-9275-d206aa45f397",\n      "address": "localhost",\n      "port": 27019,\n      "socket": "",\n      "environment": "dev-mongod",\n      "cluster": "super-custom",\n      "replication_set": "",\n      "custom_labels": {},\n      "version": ""\n    },\n    {\n      "service_id": "d6371ea3-7e68-4c7f-a786-c3cba3508ed8",\n      "service_name": "monitoring-pmm3-11763-monitoring-pmm3-mongos-0",\n      "node_id": "e97e0000-593c-4f24-bd3c-ccf938a451bd",\n      "address": "localhost",\n      "port": 27019,\n      "socket": "",\n      "environment": "dev-mongos",\n      "cluster": "super-custom",\n      "replication_set": "",\n      "custom_labels": {},\n      "version": ""\n    }\n  ],\n  "postgresql": [],\n  "proxysql": [],\n  "haproxy": [],\n  "external": [],\n  "valkey": []\n}'
+ actual_cluster=super-custom
+ [[ -z super-custom ]]
+ [[ super-custom == null ]]
+ [[ super-custom != super\-custom ]]
+ for service_name in "${service_names[@]}"
+ local actual_cluster
++ echo $'{\n  "mysql": [],\n  "mongodb": [\n    {\n      "service_id": "1cb05588-6161-421a-9588-bd3903571099",\n      "service_name": "monitoring-pmm3-11763-monitoring-pmm3-mongos-2",\n      "node_id": "1c9723e3-915b-48aa-8802-110a65a38734",\n      "address": "localhost",\n      "port": 27019,\n      "socket": "",\n      "environment": "dev-mongos",\n      "cluster": "super-custom",\n      "replication_set": "",\n      "custom_labels": {},\n      "version": ""\n    },\n    {\n      "service_id": "1e944cc7-0fb6-4979-af79-c4e0b6278889",\n      "service_name": "monitoring-pmm3-11763-monitoring-pmm3-cfg-2",\n      "node_id": "7c8bf6ae-311d-46b8-bdd1-8dd64f649283",\n      "address": "localhost",\n      "port": 27019,\n      "socket": "",\n      "environment": "dev-mongod",\n      "cluster": "super-custom",\n      "replication_set": "",\n      "custom_labels": {},\n      "version": ""\n    },\n    {\n      "service_id": "4b6b273c-d335-4409-a866-a4b1cfc58c6b",\n      "service_name": "monitoring-pmm3-11763-monitoring-pmm3-rs0-1",\n      "node_id": "13549544-fc7f-45c7-89f3-c3cf312dbf57",\n      "address": "localhost",\n      "port": 27019,\n      "socket": "",\n      "environment": "dev-mongod",\n      "cluster": "super-custom",\n      "replication_set": "",\n      "custom_labels": {},\n      "version": ""\n    },\n    {\n      "service_id": "52b0d4c8-1ce3-44dd-95ed-9bef3e84fd81",\n      "service_name": "monitoring-pmm3-11763-monitoring-pmm3-rs0-0",\n      "node_id": "d67f78f3-1137-46d0-af49-c33ae7ebe927",\n      "address": "localhost",\n      "port": 27019,\n      "socket": "",\n      "environment": "dev-mongod",\n      "cluster": "super-custom",\n      "replication_set": "",\n      "custom_labels": {},\n      "version": ""\n    },\n    {\n      "service_id": "8319ce49-5405-4a73-a663-3cc7d0dce024",\n      "service_name": "monitoring-pmm3-11763-monitoring-pmm3-rs0-2",\n      "node_id": "f523c611-01f0-4636-999b-16de7ab7a399",\n      "address": "localhost",\n      "port": 27019,\n      "socket": "",\n      "environment": "dev-mongod",\n      "cluster": "super-custom",\n      "replication_set": "",\n      "custom_labels": {},\n      "version": ""\n    },\n    {\n      "service_id": "b4a57260-9efe-47fc-9519-8184779c37c0",\n      "service_name": "monitoring-pmm3-11763-monitoring-pmm3-mongos-1",\n      "node_id": "fa4bfc6f-11ec-42d2-9cc8-d216645f789d",\n      "address": "localhost",\n      "port": 27019,\n      "socket": "",\n      "environment": "dev-mongos",\n      "cluster": "super-custom",\n      "replication_set": "",\n      "custom_labels": {},\n      "version": ""\n    },\n    {\n      "service_id": "bff3eb5c-f2fd-4058-bf2a-37338c743f80",\n      "service_name": "monitoring-pmm3-11763-monitoring-pmm3-cfg-0",\n      "node_id": "cccaab90-d90a-47fe-b618-5e42a0b21edc",\n      "address": "localhost",\n      "port": 27019,\n      "socket": "",\n      "environment": "dev-mongod",\n      "cluster": "super-custom",\n      "replication_set": "",\n      "custom_labels": {},\n      "version": ""\n    },\n    {\n      "service_id": "ca22cb1c-5fda-4ecb-9ff4-7ea082c52aa8",\n      "service_name": "monitoring-pmm3-11763-monitoring-pmm3-cfg-1",\n      "node_id": "b71c4a17-6c43-4666-9275-d206aa45f397",\n      "address": "localhost",\n      "port": 27019,\n      "socket": "",\n      "environment": "dev-mongod",\n      "cluster": "super-custom",\n      "replication_set": "",\n      "custom_labels": {},\n      "version": ""\n    },\n    {\n      "service_id": "d6371ea3-7e68-4c7f-a786-c3cba3508ed8",\n      "service_name": "monitoring-pmm3-11763-monitoring-pmm3-mongos-0",\n      "node_id": "e97e0000-593c-4f24-bd3c-ccf938a451bd",\n      "address": "localhost",\n      "port": 27019,\n      "socket": "",\n      "environment": "dev-mongos",\n      "cluster": "super-custom",\n      "replication_set": "",\n      "custom_labels": {},\n      "version": ""\n    }\n  ],\n  "postgresql": [],\n  "proxysql": [],\n  "haproxy": [],\n  "external": [],\n  "valkey": []\n}'
++ jq -r --arg name monitoring-pmm3-11763-monitoring-pmm3-rs0-0 $'\n\t\t\t.mongodb[] | select(.service_name == $name) | .cluster\n\t\t'
+ actual_cluster=super-custom
+ [[ -z super-custom ]]
+ [[ super-custom == null ]]
+ [[ super-custom != super\-custom ]]
+ return 0
+ nodeList=($(get_node_id_from_pmm))
++ get_node_id_from_pmm
++ nodeList=()
++ local -a nodeList
+++ kubectl_bin get pods --no-headers -l app.kubernetes.io/name=percona-server-mongodb --output=custom-columns=NAME:.metadata.name
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.9BnFenYd0E
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.kAzs3TSCMv
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in $(seq 0 2)
+++ set +e
+++ kubectl get pods --no-headers -l app.kubernetes.io/name=percona-server-mongodb --output=custom-columns=NAME:.metadata.name
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.9BnFenYd0E
+++ cat /tmp/tmp.kAzs3TSCMv
+++ rm /tmp/tmp.9BnFenYd0E /tmp/tmp.kAzs3TSCMv
+++ return 0
++ for instance in $(kubectl_bin get pods --no-headers -l app.kubernetes.io/name=percona-server-mongodb --output=custom-columns='NAME:.metadata.name')
++ nodeList+=($(kubectl_bin exec -n "$namespace" $instance -c pmm-client -- pmm-admin status --json | jq -r '.pmm_agent_status.node_id'))
+++ kubectl_bin exec -n monitoring-pmm3-11763 monitoring-pmm3-cfg-0 -c pmm-client -- pmm-admin status --json
+++ jq -r .pmm_agent_status.node_id
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.xLE1OdXNio
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.NopF8QuXT8
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in $(seq 0 2)
+++ set +e
+++ kubectl exec -n monitoring-pmm3-11763 monitoring-pmm3-cfg-0 -c pmm-client -- pmm-admin status --json
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.xLE1OdXNio
+++ cat /tmp/tmp.NopF8QuXT8
+++ rm /tmp/tmp.xLE1OdXNio /tmp/tmp.NopF8QuXT8
+++ return 0
++ for instance in $(kubectl_bin get pods --no-headers -l app.kubernetes.io/name=percona-server-mongodb --output=custom-columns='NAME:.metadata.name')
++ nodeList+=($(kubectl_bin exec -n "$namespace" $instance -c pmm-client -- pmm-admin status --json | jq -r '.pmm_agent_status.node_id'))
+++ kubectl_bin exec -n monitoring-pmm3-11763 monitoring-pmm3-cfg-1 -c pmm-client -- pmm-admin status --json
+++ jq -r .pmm_agent_status.node_id
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.yM3ZnWa4m5
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.5oWbFHmgYH
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in $(seq 0 2)
+++ set +e
+++ kubectl exec -n monitoring-pmm3-11763 monitoring-pmm3-cfg-1 -c pmm-client -- pmm-admin status --json
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.yM3ZnWa4m5
+++ cat /tmp/tmp.5oWbFHmgYH
+++ rm /tmp/tmp.yM3ZnWa4m5 /tmp/tmp.5oWbFHmgYH
+++ return 0
++ for instance in $(kubectl_bin get pods --no-headers -l app.kubernetes.io/name=percona-server-mongodb --output=custom-columns='NAME:.metadata.name')
++ nodeList+=($(kubectl_bin exec -n "$namespace" $instance -c pmm-client -- pmm-admin status --json | jq -r '.pmm_agent_status.node_id'))
+++ kubectl_bin exec -n monitoring-pmm3-11763 monitoring-pmm3-cfg-2 -c pmm-client -- pmm-admin status --json
+++ jq -r .pmm_agent_status.node_id
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.EEaKwMEnsC
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.ZcQxQ896YN
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in $(seq 0 2)
+++ set +e
+++ kubectl exec -n monitoring-pmm3-11763 monitoring-pmm3-cfg-2 -c pmm-client -- pmm-admin status --json
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.EEaKwMEnsC
+++ cat /tmp/tmp.ZcQxQ896YN
+++ rm /tmp/tmp.EEaKwMEnsC /tmp/tmp.ZcQxQ896YN
+++ return 0
++ for instance in $(kubectl_bin get pods --no-headers -l app.kubernetes.io/name=percona-server-mongodb --output=custom-columns='NAME:.metadata.name')
++ nodeList+=($(kubectl_bin exec -n "$namespace" $instance -c pmm-client -- pmm-admin status --json | jq -r '.pmm_agent_status.node_id'))
+++ kubectl_bin exec -n monitoring-pmm3-11763 monitoring-pmm3-mongos-0 -c pmm-client -- pmm-admin status --json
+++ jq -r .pmm_agent_status.node_id
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.s0oTVBZLua
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.rPnvLWSM8i
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in $(seq 0 2)
+++ set +e
+++ kubectl exec -n monitoring-pmm3-11763 monitoring-pmm3-mongos-0 -c pmm-client -- pmm-admin status --json
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.s0oTVBZLua
+++ cat /tmp/tmp.rPnvLWSM8i
+++ rm /tmp/tmp.s0oTVBZLua /tmp/tmp.rPnvLWSM8i
+++ return 0
++ for instance in $(kubectl_bin get pods --no-headers -l app.kubernetes.io/name=percona-server-mongodb --output=custom-columns='NAME:.metadata.name')
++ nodeList+=($(kubectl_bin exec -n "$namespace" $instance -c pmm-client -- pmm-admin status --json | jq -r '.pmm_agent_status.node_id'))
+++ kubectl_bin exec -n monitoring-pmm3-11763 monitoring-pmm3-mongos-1 -c pmm-client -- pmm-admin status --json
+++ jq -r .pmm_agent_status.node_id
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.MStV4h0aHn
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.GtLLefZImb
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in $(seq 0 2)
+++ set +e
+++ kubectl exec -n monitoring-pmm3-11763 monitoring-pmm3-mongos-1 -c pmm-client -- pmm-admin status --json
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.MStV4h0aHn
+++ cat /tmp/tmp.GtLLefZImb
+++ rm /tmp/tmp.MStV4h0aHn /tmp/tmp.GtLLefZImb
+++ return 0
++ for instance in $(kubectl_bin get pods --no-headers -l app.kubernetes.io/name=percona-server-mongodb --output=custom-columns='NAME:.metadata.name')
++ nodeList+=($(kubectl_bin exec -n "$namespace" $instance -c pmm-client -- pmm-admin status --json | jq -r '.pmm_agent_status.node_id'))
+++ kubectl_bin exec -n monitoring-pmm3-11763 monitoring-pmm3-mongos-2 -c pmm-client -- pmm-admin status --json
+++ jq -r .pmm_agent_status.node_id
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.IMt4fisHo5
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.D3fZ0dg0ZN
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in $(seq 0 2)
+++ set +e
+++ kubectl exec -n monitoring-pmm3-11763 monitoring-pmm3-mongos-2 -c pmm-client -- pmm-admin status --json
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.IMt4fisHo5
+++ cat /tmp/tmp.D3fZ0dg0ZN
+++ rm /tmp/tmp.IMt4fisHo5 /tmp/tmp.D3fZ0dg0ZN
+++ return 0
++ for instance in $(kubectl_bin get pods --no-headers -l app.kubernetes.io/name=percona-server-mongodb --output=custom-columns='NAME:.metadata.name')
++ nodeList+=($(kubectl_bin exec -n "$namespace" $instance -c pmm-client -- pmm-admin status --json | jq -r '.pmm_agent_status.node_id'))
+++ kubectl_bin exec -n monitoring-pmm3-11763 monitoring-pmm3-rs0-0 -c pmm-client -- pmm-admin status --json
+++ jq -r .pmm_agent_status.node_id
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.oQxL4gTF8m
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.VPeKZZk1tp
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in $(seq 0 2)
+++ set +e
+++ kubectl exec -n monitoring-pmm3-11763 monitoring-pmm3-rs0-0 -c pmm-client -- pmm-admin status --json
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.oQxL4gTF8m
+++ cat /tmp/tmp.VPeKZZk1tp
+++ rm /tmp/tmp.oQxL4gTF8m /tmp/tmp.VPeKZZk1tp
+++ return 0
++ for instance in $(kubectl_bin get pods --no-headers -l app.kubernetes.io/name=percona-server-mongodb --output=custom-columns='NAME:.metadata.name')
++ nodeList+=($(kubectl_bin exec -n "$namespace" $instance -c pmm-client -- pmm-admin status --json | jq -r '.pmm_agent_status.node_id'))
+++ kubectl_bin exec -n monitoring-pmm3-11763 monitoring-pmm3-rs0-1 -c pmm-client -- pmm-admin status --json
+++ jq -r .pmm_agent_status.node_id
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.5B3MEmQSp9
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.4jBRjykncr
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in $(seq 0 2)
+++ set +e
+++ kubectl exec -n monitoring-pmm3-11763 monitoring-pmm3-rs0-1 -c pmm-client -- pmm-admin status --json
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.5B3MEmQSp9
+++ cat /tmp/tmp.4jBRjykncr
+++ rm /tmp/tmp.5B3MEmQSp9 /tmp/tmp.4jBRjykncr
+++ return 0
++ for instance in $(kubectl_bin get pods --no-headers -l app.kubernetes.io/name=percona-server-mongodb --output=custom-columns='NAME:.metadata.name')
++ nodeList+=($(kubectl_bin exec -n "$namespace" $instance -c pmm-client -- pmm-admin status --json | jq -r '.pmm_agent_status.node_id'))
+++ kubectl_bin exec -n monitoring-pmm3-11763 monitoring-pmm3-rs0-2 -c pmm-client -- pmm-admin status --json
+++ jq -r .pmm_agent_status.node_id
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.ateKD0ft56
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.mZzjKAIuy3
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in $(seq 0 2)
+++ set +e
+++ kubectl exec -n monitoring-pmm3-11763 monitoring-pmm3-rs0-2 -c pmm-client -- pmm-admin status --json
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.ateKD0ft56
+++ cat /tmp/tmp.mZzjKAIuy3
+++ rm /tmp/tmp.ateKD0ft56 /tmp/tmp.mZzjKAIuy3
+++ return 0
++ echo cccaab90-d90a-47fe-b618-5e42a0b21edc b71c4a17-6c43-4666-9275-d206aa45f397 7c8bf6ae-311d-46b8-bdd1-8dd64f649283 e97e0000-593c-4f24-bd3c-ccf938a451bd fa4bfc6f-11ec-42d2-9cc8-d216645f789d 1c9723e3-915b-48aa-8802-110a65a38734 d67f78f3-1137-46d0-af49-c33ae7ebe927 13549544-fc7f-45c7-89f3-c3cf312dbf57 f523c611-01f0-4636-999b-16de7ab7a399
+ nodeList_from_pmm=($(does_node_id_exists "${nodeList[@]}"))
++ does_node_id_exists cccaab90-d90a-47fe-b618-5e42a0b21edc b71c4a17-6c43-4666-9275-d206aa45f397 7c8bf6ae-311d-46b8-bdd1-8dd64f649283 e97e0000-593c-4f24-bd3c-ccf938a451bd fa4bfc6f-11ec-42d2-9cc8-d216645f789d 1c9723e3-915b-48aa-8802-110a65a38734 d67f78f3-1137-46d0-af49-c33ae7ebe927 13549544-fc7f-45c7-89f3-c3cf312dbf57 f523c611-01f0-4636-999b-16de7ab7a399
++ nodeList=('cccaab90-d90a-47fe-b618-5e42a0b21edc' 'b71c4a17-6c43-4666-9275-d206aa45f397' '7c8bf6ae-311d-46b8-bdd1-8dd64f649283' 'e97e0000-593c-4f24-bd3c-ccf938a451bd' 'fa4bfc6f-11ec-42d2-9cc8-d216645f789d' '1c9723e3-915b-48aa-8802-110a65a38734' 'd67f78f3-1137-46d0-af49-c33ae7ebe927' '13549544-fc7f-45c7-89f3-c3cf312dbf57' 'f523c611-01f0-4636-999b-16de7ab7a399')
++ local -a nodeList
++ nodeList_from_pmm=()
++ local -a nodeList_from_pmm
+++ kubectl_bin get secret pmm-secret -o 'jsonpath={.data.PMM_ADMIN_PASSWORD}'
+++ base64 --decode
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.dq2u4uIwYW
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.i87YF8mDhr
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in $(seq 0 2)
+++ set +e
+++ kubectl get secret pmm-secret -o 'jsonpath={.data.PMM_ADMIN_PASSWORD}'
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.dq2u4uIwYW
+++ cat /tmp/tmp.i87YF8mDhr
+++ rm /tmp/tmp.dq2u4uIwYW /tmp/tmp.i87YF8mDhr
+++ return 0
++ local 'admin_pass=Os\2'\''.99QS0gs;]f'
+++ get_service_endpoint monitoring-service
+++ local service=monitoring-service
++++ kubectl_bin get service/monitoring-service -o json
++++ jq '.status.loadBalancer.ingress[].hostname'
++++ sed -e 's/^"//; s/"$//;'
+++++ mktemp
++++ local LAST_OUT=/tmp/tmp.AZGNwtHsL9
+++++ mktemp
++++ local LAST_ERR=/tmp/tmp.fjX9ZUoOFJ
++++ local exit_status=0
++++ local timeout=4
+++++ seq 0 2
++++ for i in $(seq 0 2)
++++ set +e
++++ kubectl get service/monitoring-service -o json
++++ exit_status=0
++++ set -e
++++ '[' 0 '!=' 0 -a -n 1 ']'
++++ break
++++ cat /tmp/tmp.AZGNwtHsL9
++++ cat /tmp/tmp.fjX9ZUoOFJ
++++ rm /tmp/tmp.AZGNwtHsL9 /tmp/tmp.fjX9ZUoOFJ
++++ return 0
+++ local hostname=null
+++ '[' -n null -a null '!=' null ']'
++++ kubectl_bin get service/monitoring-service -o json
++++ jq '.status.loadBalancer.ingress[].ip'
+++++ mktemp
++++ sed -e 's/^"//; s/"$//;'
++++ local LAST_OUT=/tmp/tmp.SWIaNvAQFX
+++++ mktemp
++++ local LAST_ERR=/tmp/tmp.iCzQE0gIvn
++++ local exit_status=0
++++ local timeout=4
+++++ seq 0 2
++++ for i in $(seq 0 2)
++++ set +e
++++ kubectl get service/monitoring-service -o json
++++ exit_status=0
++++ set -e
++++ '[' 0 '!=' 0 -a -n 1 ']'
++++ break
++++ cat /tmp/tmp.SWIaNvAQFX
++++ cat /tmp/tmp.iCzQE0gIvn
++++ rm /tmp/tmp.SWIaNvAQFX /tmp/tmp.iCzQE0gIvn
++++ return 0
+++ local ip=35.254.76.43
+++ '[' -n 35.254.76.43 -a 35.254.76.43 '!=' null ']'
+++ echo 35.254.76.43
+++ return
++ local endpoint=35.254.76.43
++ for node_id in "${nodeList[@]}"
++ nodeList_from_pmm+=($(execute_curl --insecure -s -X GET --user "admin:${admin_pass}" "https://${endpoint}/v1/inventory/nodes?node_type=NODE_TYPE_CONTAINER_NODE" | jq -r --arg node_id "$node_id" '.nodes[]? | select(.node_id == $node_id) | .node_id'))
+++ execute_curl --insecure -s -X GET --user 'admin:Os\2'\''.99QS0gs;]f' 'https://35.254.76.43/v1/inventory/nodes?node_type=NODE_TYPE_CONTAINER_NODE'
+++ jq -r --arg node_id cccaab90-d90a-47fe-b618-5e42a0b21edc '.nodes[]? | select(.node_id == $node_id) | .node_id'
+++ local name=curl-23926
+++ local image=docker.io/alpine/curl:latest
+++ kubectl_bin run curl-23926 -n monitoring-pmm3-11763 --restart=Never --image=docker.io/alpine/curl:latest --command -- sleep 3600
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.DjIOpA0Qlk
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.O8VMPbpDDx
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in $(seq 0 2)
+++ set +e
+++ kubectl run curl-23926 -n monitoring-pmm3-11763 --restart=Never --image=docker.io/alpine/curl:latest --command -- sleep 3600
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.DjIOpA0Qlk
+++ cat /tmp/tmp.O8VMPbpDDx
+++ rm /tmp/tmp.DjIOpA0Qlk /tmp/tmp.O8VMPbpDDx
+++ return 0
+++ kubectl_bin wait pod/curl-23926 -n monitoring-pmm3-11763 --for=condition=Ready --timeout=60s
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.TTYTNTVQMa
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.4MSszfB3iu
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in $(seq 0 2)
+++ set +e
+++ kubectl wait pod/curl-23926 -n monitoring-pmm3-11763 --for=condition=Ready --timeout=60s
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.TTYTNTVQMa
+++ cat /tmp/tmp.4MSszfB3iu
+++ rm /tmp/tmp.TTYTNTVQMa /tmp/tmp.4MSszfB3iu
+++ return 0
+++ local exit_status=0
+++ kubectl_bin exec curl-23926 -n monitoring-pmm3-11763 -- curl --insecure -s -X GET --user 'admin:Os\2'\''.99QS0gs;]f' 'https://35.254.76.43/v1/inventory/nodes?node_type=NODE_TYPE_CONTAINER_NODE'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.u6IIUzBgxm
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.ymsrTnaI9F
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in $(seq 0 2)
+++ set +e
+++ kubectl exec curl-23926 -n monitoring-pmm3-11763 -- curl --insecure -s -X GET --user 'admin:Os\2'\''.99QS0gs;]f' 'https://35.254.76.43/v1/inventory/nodes?node_type=NODE_TYPE_CONTAINER_NODE'
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.u6IIUzBgxm
+++ cat /tmp/tmp.ymsrTnaI9F
+++ rm /tmp/tmp.u6IIUzBgxm /tmp/tmp.ymsrTnaI9F
+++ return 0
+++ kubectl_bin delete pod curl-23926 -n monitoring-pmm3-11763 --ignore-not-found --wait=false
+++ return 0
++ for node_id in "${nodeList[@]}"
++ nodeList_from_pmm+=($(execute_curl --insecure -s -X GET --user "admin:${admin_pass}" "https://${endpoint}/v1/inventory/nodes?node_type=NODE_TYPE_CONTAINER_NODE" | jq -r --arg node_id "$node_id" '.nodes[]? | select(.node_id == $node_id) | .node_id'))
+++ execute_curl --insecure -s -X GET --user 'admin:Os\2'\''.99QS0gs;]f' 'https://35.254.76.43/v1/inventory/nodes?node_type=NODE_TYPE_CONTAINER_NODE'
+++ jq -r --arg node_id b71c4a17-6c43-4666-9275-d206aa45f397 '.nodes[]? | select(.node_id == $node_id) | .node_id'
+++ local name=curl-27633
+++ local image=docker.io/alpine/curl:latest
+++ kubectl_bin run curl-27633 -n monitoring-pmm3-11763 --restart=Never --image=docker.io/alpine/curl:latest --command -- sleep 3600
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.QD7qWe3HNj
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.7bz3jerdi9
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in $(seq 0 2)
+++ set +e
+++ kubectl run curl-27633 -n monitoring-pmm3-11763 --restart=Never --image=docker.io/alpine/curl:latest --command -- sleep 3600
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.QD7qWe3HNj
+++ cat /tmp/tmp.7bz3jerdi9
+++ rm /tmp/tmp.QD7qWe3HNj /tmp/tmp.7bz3jerdi9
+++ return 0
+++ kubectl_bin wait pod/curl-27633 -n monitoring-pmm3-11763 --for=condition=Ready --timeout=60s
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.ipmI4aP9EV
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.gn7F0PZSKe
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in $(seq 0 2)
+++ set +e
+++ kubectl wait pod/curl-27633 -n monitoring-pmm3-11763 --for=condition=Ready --timeout=60s
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.ipmI4aP9EV
+++ cat /tmp/tmp.gn7F0PZSKe
+++ rm /tmp/tmp.ipmI4aP9EV /tmp/tmp.gn7F0PZSKe
+++ return 0
+++ local exit_status=0
+++ kubectl_bin exec curl-27633 -n monitoring-pmm3-11763 -- curl --insecure -s -X GET --user 'admin:Os\2'\''.99QS0gs;]f' 'https://35.254.76.43/v1/inventory/nodes?node_type=NODE_TYPE_CONTAINER_NODE'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.OqUzNhV1t3
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.kjz0MSEdV3
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in $(seq 0 2)
+++ set +e
+++ kubectl exec curl-27633 -n monitoring-pmm3-11763 -- curl --insecure -s -X GET --user 'admin:Os\2'\''.99QS0gs;]f' 'https://35.254.76.43/v1/inventory/nodes?node_type=NODE_TYPE_CONTAINER_NODE'
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.OqUzNhV1t3
+++ cat /tmp/tmp.kjz0MSEdV3
+++ rm /tmp/tmp.OqUzNhV1t3 /tmp/tmp.kjz0MSEdV3
+++ return 0
+++ kubectl_bin delete pod curl-27633 -n monitoring-pmm3-11763 --ignore-not-found --wait=false
+++ return 0
++ for node_id in "${nodeList[@]}"
++ nodeList_from_pmm+=($(execute_curl --insecure -s -X GET --user "admin:${admin_pass}" "https://${endpoint}/v1/inventory/nodes?node_type=NODE_TYPE_CONTAINER_NODE" | jq -r --arg node_id "$node_id" '.nodes[]? | select(.node_id == $node_id) | .node_id'))
+++ execute_curl --insecure -s -X GET --user 'admin:Os\2'\''.99QS0gs;]f' 'https://35.254.76.43/v1/inventory/nodes?node_type=NODE_TYPE_CONTAINER_NODE'
+++ local name=curl-22069
+++ local image=docker.io/alpine/curl:latest
+++ jq -r --arg node_id 7c8bf6ae-311d-46b8-bdd1-8dd64f649283 '.nodes[]? | select(.node_id == $node_id) | .node_id'
+++ kubectl_bin run curl-22069 -n monitoring-pmm3-11763 --restart=Never --image=docker.io/alpine/curl:latest --command -- sleep 3600
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.YAGpHsQ9Rn
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.cwPAK4oTne
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in $(seq 0 2)
+++ set +e
+++ kubectl run curl-22069 -n monitoring-pmm3-11763 --restart=Never --image=docker.io/alpine/curl:latest --command -- sleep 3600
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.YAGpHsQ9Rn
+++ cat /tmp/tmp.cwPAK4oTne
+++ rm /tmp/tmp.YAGpHsQ9Rn /tmp/tmp.cwPAK4oTne
+++ return 0
+++ kubectl_bin wait pod/curl-22069 -n monitoring-pmm3-11763 --for=condition=Ready --timeout=60s
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.cDxFD0TPa5
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.ZTsb04NXVI
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in $(seq 0 2)
+++ set +e
+++ kubectl wait pod/curl-22069 -n monitoring-pmm3-11763 --for=condition=Ready --timeout=60s
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.cDxFD0TPa5
+++ cat /tmp/tmp.ZTsb04NXVI
+++ rm /tmp/tmp.cDxFD0TPa5 /tmp/tmp.ZTsb04NXVI
+++ return 0
+++ local exit_status=0
+++ kubectl_bin exec curl-22069 -n monitoring-pmm3-11763 -- curl --insecure -s -X GET --user 'admin:Os\2'\''.99QS0gs;]f' 'https://35.254.76.43/v1/inventory/nodes?node_type=NODE_TYPE_CONTAINER_NODE'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.K1mU2gAXSD
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.jOu2VHuiuD
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in $(seq 0 2)
+++ set +e
+++ kubectl exec curl-22069 -n monitoring-pmm3-11763 -- curl --insecure -s -X GET --user 'admin:Os\2'\''.99QS0gs;]f' 'https://35.254.76.43/v1/inventory/nodes?node_type=NODE_TYPE_CONTAINER_NODE'
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.K1mU2gAXSD
+++ cat /tmp/tmp.jOu2VHuiuD
+++ rm /tmp/tmp.K1mU2gAXSD /tmp/tmp.jOu2VHuiuD
+++ return 0
+++ kubectl_bin delete pod curl-22069 -n monitoring-pmm3-11763 --ignore-not-found --wait=false
+++ return 0
++ for node_id in "${nodeList[@]}"
++ nodeList_from_pmm+=($(execute_curl --insecure -s -X GET --user "admin:${admin_pass}" "https://${endpoint}/v1/inventory/nodes?node_type=NODE_TYPE_CONTAINER_NODE" | jq -r --arg node_id "$node_id" '.nodes[]? | select(.node_id == $node_id) | .node_id'))
+++ execute_curl --insecure -s -X GET --user 'admin:Os\2'\''.99QS0gs;]f' 'https://35.254.76.43/v1/inventory/nodes?node_type=NODE_TYPE_CONTAINER_NODE'
+++ jq -r --arg node_id e97e0000-593c-4f24-bd3c-ccf938a451bd '.nodes[]? | select(.node_id == $node_id) | .node_id'
+++ local name=curl-16294
+++ local image=docker.io/alpine/curl:latest
+++ kubectl_bin run curl-16294 -n monitoring-pmm3-11763 --restart=Never --image=docker.io/alpine/curl:latest --command -- sleep 3600
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.F07BSRJDm1
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.tJ4MQFODrE
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in $(seq 0 2)
+++ set +e
+++ kubectl run curl-16294 -n monitoring-pmm3-11763 --restart=Never --image=docker.io/alpine/curl:latest --command -- sleep 3600
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.F07BSRJDm1
+++ cat /tmp/tmp.tJ4MQFODrE
+++ rm /tmp/tmp.F07BSRJDm1 /tmp/tmp.tJ4MQFODrE
+++ return 0
+++ kubectl_bin wait pod/curl-16294 -n monitoring-pmm3-11763 --for=condition=Ready --timeout=60s
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.5uzNqztBjQ
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.4BQSBT15mF
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in $(seq 0 2)
+++ set +e
+++ kubectl wait pod/curl-16294 -n monitoring-pmm3-11763 --for=condition=Ready --timeout=60s
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.5uzNqztBjQ
+++ cat /tmp/tmp.4BQSBT15mF
+++ rm /tmp/tmp.5uzNqztBjQ /tmp/tmp.4BQSBT15mF
+++ return 0
+++ local exit_status=0
+++ kubectl_bin exec curl-16294 -n monitoring-pmm3-11763 -- curl --insecure -s -X GET --user 'admin:Os\2'\''.99QS0gs;]f' 'https://35.254.76.43/v1/inventory/nodes?node_type=NODE_TYPE_CONTAINER_NODE'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.IQ6fSpRjEv
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.piX5hwq2st
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in $(seq 0 2)
+++ set +e
+++ kubectl exec curl-16294 -n monitoring-pmm3-11763 -- curl --insecure -s -X GET --user 'admin:Os\2'\''.99QS0gs;]f' 'https://35.254.76.43/v1/inventory/nodes?node_type=NODE_TYPE_CONTAINER_NODE'
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.IQ6fSpRjEv
+++ cat /tmp/tmp.piX5hwq2st
+++ rm /tmp/tmp.IQ6fSpRjEv /tmp/tmp.piX5hwq2st
+++ return 0
+++ kubectl_bin delete pod curl-16294 -n monitoring-pmm3-11763 --ignore-not-found --wait=false
+++ return 0
++ for node_id in "${nodeList[@]}"
++ nodeList_from_pmm+=($(execute_curl --insecure -s -X GET --user "admin:${admin_pass}" "https://${endpoint}/v1/inventory/nodes?node_type=NODE_TYPE_CONTAINER_NODE" | jq -r --arg node_id "$node_id" '.nodes[]? | select(.node_id == $node_id) | .node_id'))
+++ execute_curl --insecure -s -X GET --user 'admin:Os\2'\''.99QS0gs;]f' 'https://35.254.76.43/v1/inventory/nodes?node_type=NODE_TYPE_CONTAINER_NODE'
+++ jq -r --arg node_id fa4bfc6f-11ec-42d2-9cc8-d216645f789d '.nodes[]? | select(.node_id == $node_id) | .node_id'
+++ local name=curl-7106
+++ local image=docker.io/alpine/curl:latest
+++ kubectl_bin run curl-7106 -n monitoring-pmm3-11763 --restart=Never --image=docker.io/alpine/curl:latest --command -- sleep 3600
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.2PiO5XY4E6
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.z10xLK0qhk
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in $(seq 0 2)
+++ set +e
+++ kubectl run curl-7106 -n monitoring-pmm3-11763 --restart=Never --image=docker.io/alpine/curl:latest --command -- sleep 3600
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.2PiO5XY4E6
+++ cat /tmp/tmp.z10xLK0qhk
+++ rm /tmp/tmp.2PiO5XY4E6 /tmp/tmp.z10xLK0qhk
+++ return 0
+++ kubectl_bin wait pod/curl-7106 -n monitoring-pmm3-11763 --for=condition=Ready --timeout=60s
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.pvSmwMXnKg
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.HXhTwxmjnY
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in $(seq 0 2)
+++ set +e
+++ kubectl wait pod/curl-7106 -n monitoring-pmm3-11763 --for=condition=Ready --timeout=60s
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.pvSmwMXnKg
+++ cat /tmp/tmp.HXhTwxmjnY
E0707 20:04:30.230620  680758 reflector.go:227] "Failed to watch" err="Get \"https://34.27.139.243/api/v1/namespaces/monitoring-pmm3-11763/pods?allowWatchBookmarks=true&fieldSelector=metadata.name%3Dcurl-7106&resourceVersion=1783454670013679022&timeoutSeconds=553&watch=true\": context canceled" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" type="*unstructured.Unstructured"
+++ rm /tmp/tmp.pvSmwMXnKg /tmp/tmp.HXhTwxmjnY
+++ return 0
+++ local exit_status=0
+++ kubectl_bin exec curl-7106 -n monitoring-pmm3-11763 -- curl --insecure -s -X GET --user 'admin:Os\2'\''.99QS0gs;]f' 'https://35.254.76.43/v1/inventory/nodes?node_type=NODE_TYPE_CONTAINER_NODE'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.Emu0JOR3ul
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.v6DgdQaYeI
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in $(seq 0 2)
+++ set +e
+++ kubectl exec curl-7106 -n monitoring-pmm3-11763 -- curl --insecure -s -X GET --user 'admin:Os\2'\''.99QS0gs;]f' 'https://35.254.76.43/v1/inventory/nodes?node_type=NODE_TYPE_CONTAINER_NODE'
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.Emu0JOR3ul
+++ cat /tmp/tmp.v6DgdQaYeI
+++ rm /tmp/tmp.Emu0JOR3ul /tmp/tmp.v6DgdQaYeI
+++ return 0
+++ kubectl_bin delete pod curl-7106 -n monitoring-pmm3-11763 --ignore-not-found --wait=false
+++ return 0
++ for node_id in "${nodeList[@]}"
++ nodeList_from_pmm+=($(execute_curl --insecure -s -X GET --user "admin:${admin_pass}" "https://${endpoint}/v1/inventory/nodes?node_type=NODE_TYPE_CONTAINER_NODE" | jq -r --arg node_id "$node_id" '.nodes[]? | select(.node_id == $node_id) | .node_id'))
+++ execute_curl --insecure -s -X GET --user 'admin:Os\2'\''.99QS0gs;]f' 'https://35.254.76.43/v1/inventory/nodes?node_type=NODE_TYPE_CONTAINER_NODE'
+++ jq -r --arg node_id 1c9723e3-915b-48aa-8802-110a65a38734 '.nodes[]? | select(.node_id == $node_id) | .node_id'
+++ local name=curl-26400
+++ local image=docker.io/alpine/curl:latest
+++ kubectl_bin run curl-26400 -n monitoring-pmm3-11763 --restart=Never --image=docker.io/alpine/curl:latest --command -- sleep 3600
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.sv5uC0F1aT
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.OhZevv2nVX
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in $(seq 0 2)
+++ set +e
+++ kubectl run curl-26400 -n monitoring-pmm3-11763 --restart=Never --image=docker.io/alpine/curl:latest --command -- sleep 3600
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.sv5uC0F1aT
+++ cat /tmp/tmp.OhZevv2nVX
+++ rm /tmp/tmp.sv5uC0F1aT /tmp/tmp.OhZevv2nVX
+++ return 0
+++ kubectl_bin wait pod/curl-26400 -n monitoring-pmm3-11763 --for=condition=Ready --timeout=60s
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.EYwnj8xYmj
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.s7QnzSXdCW
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in $(seq 0 2)
+++ set +e
+++ kubectl wait pod/curl-26400 -n monitoring-pmm3-11763 --for=condition=Ready --timeout=60s
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.EYwnj8xYmj
+++ cat /tmp/tmp.s7QnzSXdCW
+++ rm /tmp/tmp.EYwnj8xYmj /tmp/tmp.s7QnzSXdCW
+++ return 0
+++ local exit_status=0
+++ kubectl_bin exec curl-26400 -n monitoring-pmm3-11763 -- curl --insecure -s -X GET --user 'admin:Os\2'\''.99QS0gs;]f' 'https://35.254.76.43/v1/inventory/nodes?node_type=NODE_TYPE_CONTAINER_NODE'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.eO2ARFfFNf
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.IJvSZwV7Cz
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in $(seq 0 2)
+++ set +e
+++ kubectl exec curl-26400 -n monitoring-pmm3-11763 -- curl --insecure -s -X GET --user 'admin:Os\2'\''.99QS0gs;]f' 'https://35.254.76.43/v1/inventory/nodes?node_type=NODE_TYPE_CONTAINER_NODE'
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.eO2ARFfFNf
+++ cat /tmp/tmp.IJvSZwV7Cz
+++ rm /tmp/tmp.eO2ARFfFNf /tmp/tmp.IJvSZwV7Cz
+++ return 0
+++ kubectl_bin delete pod curl-26400 -n monitoring-pmm3-11763 --ignore-not-found --wait=false
+++ return 0
++ for node_id in "${nodeList[@]}"
++ nodeList_from_pmm+=($(execute_curl --insecure -s -X GET --user "admin:${admin_pass}" "https://${endpoint}/v1/inventory/nodes?node_type=NODE_TYPE_CONTAINER_NODE" | jq -r --arg node_id "$node_id" '.nodes[]? | select(.node_id == $node_id) | .node_id'))
+++ execute_curl --insecure -s -X GET --user 'admin:Os\2'\''.99QS0gs;]f' 'https://35.254.76.43/v1/inventory/nodes?node_type=NODE_TYPE_CONTAINER_NODE'
+++ jq -r --arg node_id d67f78f3-1137-46d0-af49-c33ae7ebe927 '.nodes[]? | select(.node_id == $node_id) | .node_id'
+++ local name=curl-2685
+++ local image=docker.io/alpine/curl:latest
+++ kubectl_bin run curl-2685 -n monitoring-pmm3-11763 --restart=Never --image=docker.io/alpine/curl:latest --command -- sleep 3600
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.YvXDH3DAVx
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.xwPI3tpU4M
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in $(seq 0 2)
+++ set +e
+++ kubectl run curl-2685 -n monitoring-pmm3-11763 --restart=Never --image=docker.io/alpine/curl:latest --command -- sleep 3600
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.YvXDH3DAVx
+++ cat /tmp/tmp.xwPI3tpU4M
+++ rm /tmp/tmp.YvXDH3DAVx /tmp/tmp.xwPI3tpU4M
+++ return 0
+++ kubectl_bin wait pod/curl-2685 -n monitoring-pmm3-11763 --for=condition=Ready --timeout=60s
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.FZ9aRYktOl
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.nJyrO2okO6
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in $(seq 0 2)
+++ set +e
+++ kubectl wait pod/curl-2685 -n monitoring-pmm3-11763 --for=condition=Ready --timeout=60s
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.FZ9aRYktOl
+++ cat /tmp/tmp.nJyrO2okO6
+++ rm /tmp/tmp.FZ9aRYktOl /tmp/tmp.nJyrO2okO6
+++ return 0
+++ local exit_status=0
+++ kubectl_bin exec curl-2685 -n monitoring-pmm3-11763 -- curl --insecure -s -X GET --user 'admin:Os\2'\''.99QS0gs;]f' 'https://35.254.76.43/v1/inventory/nodes?node_type=NODE_TYPE_CONTAINER_NODE'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.0G1TS96HGF
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.S0tjTq1wRm
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in $(seq 0 2)
+++ set +e
+++ kubectl exec curl-2685 -n monitoring-pmm3-11763 -- curl --insecure -s -X GET --user 'admin:Os\2'\''.99QS0gs;]f' 'https://35.254.76.43/v1/inventory/nodes?node_type=NODE_TYPE_CONTAINER_NODE'
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.0G1TS96HGF
+++ cat /tmp/tmp.S0tjTq1wRm
+++ rm /tmp/tmp.0G1TS96HGF /tmp/tmp.S0tjTq1wRm
+++ return 0
+++ kubectl_bin delete pod curl-2685 -n monitoring-pmm3-11763 --ignore-not-found --wait=false
+++ return 0
++ for node_id in "${nodeList[@]}"
++ nodeList_from_pmm+=($(execute_curl --insecure -s -X GET --user "admin:${admin_pass}" "https://${endpoint}/v1/inventory/nodes?node_type=NODE_TYPE_CONTAINER_NODE" | jq -r --arg node_id "$node_id" '.nodes[]? | select(.node_id == $node_id) | .node_id'))
+++ execute_curl --insecure -s -X GET --user 'admin:Os\2'\''.99QS0gs;]f' 'https://35.254.76.43/v1/inventory/nodes?node_type=NODE_TYPE_CONTAINER_NODE'
+++ local name=curl-1816
+++ local image=docker.io/alpine/curl:latest
+++ kubectl_bin run curl-1816 -n monitoring-pmm3-11763 --restart=Never --image=docker.io/alpine/curl:latest --command -- sleep 3600
+++ jq -r --arg node_id 13549544-fc7f-45c7-89f3-c3cf312dbf57 '.nodes[]? | select(.node_id == $node_id) | .node_id'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.7kRccT3Vez
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.cqf6Y5XWnL
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in $(seq 0 2)
+++ set +e
+++ kubectl run curl-1816 -n monitoring-pmm3-11763 --restart=Never --image=docker.io/alpine/curl:latest --command -- sleep 3600
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.7kRccT3Vez
+++ cat /tmp/tmp.cqf6Y5XWnL
+++ rm /tmp/tmp.7kRccT3Vez /tmp/tmp.cqf6Y5XWnL
+++ return 0
+++ kubectl_bin wait pod/curl-1816 -n monitoring-pmm3-11763 --for=condition=Ready --timeout=60s
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.g9rSUtsP9C
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.HOZiMKNtjq
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in $(seq 0 2)
+++ set +e
+++ kubectl wait pod/curl-1816 -n monitoring-pmm3-11763 --for=condition=Ready --timeout=60s
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.g9rSUtsP9C
+++ cat /tmp/tmp.HOZiMKNtjq
E0707 20:04:42.470459  681300 reflector.go:227] "Failed to watch" err="Get \"https://34.27.139.243/api/v1/namespaces/monitoring-pmm3-11763/pods?allowWatchBookmarks=true&fieldSelector=metadata.name%3Dcurl-1816&resourceVersion=1783454682335663007&timeoutSeconds=441&watch=true\": context canceled" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" type="*unstructured.Unstructured"
+++ rm /tmp/tmp.g9rSUtsP9C /tmp/tmp.HOZiMKNtjq
+++ return 0
+++ local exit_status=0
+++ kubectl_bin exec curl-1816 -n monitoring-pmm3-11763 -- curl --insecure -s -X GET --user 'admin:Os\2'\''.99QS0gs;]f' 'https://35.254.76.43/v1/inventory/nodes?node_type=NODE_TYPE_CONTAINER_NODE'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.CoKLZCDh8S
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.aqSasxnirn
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in $(seq 0 2)
+++ set +e
+++ kubectl exec curl-1816 -n monitoring-pmm3-11763 -- curl --insecure -s -X GET --user 'admin:Os\2'\''.99QS0gs;]f' 'https://35.254.76.43/v1/inventory/nodes?node_type=NODE_TYPE_CONTAINER_NODE'
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.CoKLZCDh8S
+++ cat /tmp/tmp.aqSasxnirn
+++ rm /tmp/tmp.CoKLZCDh8S /tmp/tmp.aqSasxnirn
+++ return 0
+++ kubectl_bin delete pod curl-1816 -n monitoring-pmm3-11763 --ignore-not-found --wait=false
+++ return 0
++ for node_id in "${nodeList[@]}"
++ nodeList_from_pmm+=($(execute_curl --insecure -s -X GET --user "admin:${admin_pass}" "https://${endpoint}/v1/inventory/nodes?node_type=NODE_TYPE_CONTAINER_NODE" | jq -r --arg node_id "$node_id" '.nodes[]? | select(.node_id == $node_id) | .node_id'))
+++ execute_curl --insecure -s -X GET --user 'admin:Os\2'\''.99QS0gs;]f' 'https://35.254.76.43/v1/inventory/nodes?node_type=NODE_TYPE_CONTAINER_NODE'
+++ jq -r --arg node_id f523c611-01f0-4636-999b-16de7ab7a399 '.nodes[]? | select(.node_id == $node_id) | .node_id'
+++ local name=curl-9776
+++ local image=docker.io/alpine/curl:latest
+++ kubectl_bin run curl-9776 -n monitoring-pmm3-11763 --restart=Never --image=docker.io/alpine/curl:latest --command -- sleep 3600
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.QfXPgWsaSq
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.JP3y9lR6Zu
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in $(seq 0 2)
+++ set +e
+++ kubectl run curl-9776 -n monitoring-pmm3-11763 --restart=Never --image=docker.io/alpine/curl:latest --command -- sleep 3600
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.QfXPgWsaSq
+++ cat /tmp/tmp.JP3y9lR6Zu
+++ rm /tmp/tmp.QfXPgWsaSq /tmp/tmp.JP3y9lR6Zu
+++ return 0
+++ kubectl_bin wait pod/curl-9776 -n monitoring-pmm3-11763 --for=condition=Ready --timeout=60s
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.MD0cQHg9g6
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.5GD9PPcZqR
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in $(seq 0 2)
+++ set +e
+++ kubectl wait pod/curl-9776 -n monitoring-pmm3-11763 --for=condition=Ready --timeout=60s
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.MD0cQHg9g6
+++ cat /tmp/tmp.5GD9PPcZqR
+++ rm /tmp/tmp.MD0cQHg9g6 /tmp/tmp.5GD9PPcZqR
+++ return 0
+++ local exit_status=0
+++ kubectl_bin exec curl-9776 -n monitoring-pmm3-11763 -- curl --insecure -s -X GET --user 'admin:Os\2'\''.99QS0gs;]f' 'https://35.254.76.43/v1/inventory/nodes?node_type=NODE_TYPE_CONTAINER_NODE'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.kHJgGyqSI6
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.a2vjwuqmQA
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in $(seq 0 2)
+++ set +e
+++ kubectl exec curl-9776 -n monitoring-pmm3-11763 -- curl --insecure -s -X GET --user 'admin:Os\2'\''.99QS0gs;]f' 'https://35.254.76.43/v1/inventory/nodes?node_type=NODE_TYPE_CONTAINER_NODE'
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.kHJgGyqSI6
+++ cat /tmp/tmp.a2vjwuqmQA
+++ rm /tmp/tmp.kHJgGyqSI6 /tmp/tmp.a2vjwuqmQA
+++ return 0
+++ kubectl_bin delete pod curl-9776 -n monitoring-pmm3-11763 --ignore-not-found --wait=false
+++ return 0
++ echo
+ desc 'patch cluster to switch QAN source to mongolog'
+ set +o xtrace


-----------------------------------------------------------------------------------
patch cluster to switch QAN source to mongolog
-----------------------------------------------------------------------------------

+ kubectl_bin patch psmdb monitoring-pmm3 --type=json $'-p=[\n  {"op":"add","path":"/spec/pmm/querySource","value":"mongolog"},\n  {"op":"replace","path":"/spec/pmm/mongodParams","value":"--enable-all-collectors --environment=dev-mongod-mongolog"}\n]'
++ mktemp
+ local LAST_OUT=/tmp/tmp.apuR2KOkZ8
++ mktemp
+ local LAST_ERR=/tmp/tmp.A2G2R4uBt8
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl patch psmdb monitoring-pmm3 --type=json $'-p=[\n  {"op":"add","path":"/spec/pmm/querySource","value":"mongolog"},\n  {"op":"replace","path":"/spec/pmm/mongodParams","value":"--enable-all-collectors --environment=dev-mongod-mongolog"}\n]'
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.apuR2KOkZ8
perconaservermongodb.psmdb.percona.com/monitoring-pmm3 patched
+ cat /tmp/tmp.A2G2R4uBt8
+ rm /tmp/tmp.apuR2KOkZ8 /tmp/tmp.A2G2R4uBt8
+ return 0
+ wait_for_running monitoring-pmm3-rs0 3
+ local name=monitoring-pmm3-rs0
+ let last_pod=2
+ local check_cluster_readiness=true
+ set_debug
+ [[ 1 == 1 ]]
+ set -o xtrace
+ local rs_name=rs0
+ local cluster_name=monitoring-pmm3
++ seq 0 2
+ for i in $(seq 0 $last_pod)
+ [[ 0 -eq 2 ]]
+ wait_pod monitoring-pmm3-rs0-0
+ local pod=monitoring-pmm3-rs0-0
+ set +o xtrace
waiting for pod/monitoring-pmm3-rs0-0 to be ready.OK
+ for i in $(seq 0 $last_pod)
+ [[ 1 -eq 2 ]]
+ wait_pod monitoring-pmm3-rs0-1
+ local pod=monitoring-pmm3-rs0-1
+ set +o xtrace
waiting for pod/monitoring-pmm3-rs0-1 to be ready.OK
+ for i in $(seq 0 $last_pod)
+ [[ 2 -eq 2 ]]
++ kubectl_bin get psmdb monitoring-pmm3 -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].arbiter.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.y9kqKlluIe
+++ mktemp
++ local LAST_ERR=/tmp/tmp.VCHPaYTQMA
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb monitoring-pmm3 -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].arbiter.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.y9kqKlluIe
++ cat /tmp/tmp.VCHPaYTQMA
++ rm /tmp/tmp.y9kqKlluIe /tmp/tmp.VCHPaYTQMA
++ return 0
+ [[ '' == true ]]
+ wait_pod monitoring-pmm3-rs0-2
+ local pod=monitoring-pmm3-rs0-2
+ set +o xtrace
waiting for pod/monitoring-pmm3-rs0-2 to be ready.OK
++ kubectl_bin get psmdb monitoring-pmm3 -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].nonvoting.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.N91E8jbgiY
+++ mktemp
++ local LAST_ERR=/tmp/tmp.7FIIYFYlxH
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb monitoring-pmm3 -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].nonvoting.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.N91E8jbgiY
++ cat /tmp/tmp.7FIIYFYlxH
++ rm /tmp/tmp.N91E8jbgiY /tmp/tmp.7FIIYFYlxH
++ return 0
+ [[ '' == true ]]
++ kubectl_bin get psmdb monitoring-pmm3 -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].hidden.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.kxqbmDGhLD
+++ mktemp
++ local LAST_ERR=/tmp/tmp.4zqy1fU3eR
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb monitoring-pmm3 -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].hidden.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.kxqbmDGhLD
++ cat /tmp/tmp.4zqy1fU3eR
++ rm /tmp/tmp.kxqbmDGhLD /tmp/tmp.4zqy1fU3eR
++ return 0
+ [[ '' == true ]]
+ sleep 10
+ [[ true == true ]]
+ set +x
Waiting for cluster monitoring-pmm3 to be ready.........................................................................................................................................................
+ sleep 90
+ desc 'check mongolog QAN data'
+ set +o xtrace


-----------------------------------------------------------------------------------
check mongolog QAN data
-----------------------------------------------------------------------------------

+ get_qan_values mongodb dev-mongod-mongolog glsa_ENtP2ImukR8cr3g1MllgZoNhKS1S8MWd_73f00929
+ local service_type=mongodb
+ local environment=dev-mongod-mongolog
+ local token=glsa_ENtP2ImukR8cr3g1MllgZoNhKS1S8MWd_73f00929
+ local start
+ local end
+ local endpoint
++ /usr/sbin/date -u -d '-12 hour' +%Y-%m-%dT%H:%M:%S%:z
+ start=2026-07-07T08:11:44+00:00
++ /usr/sbin/date -u +%Y-%m-%dT%H:%M:%S%:z
+ end=2026-07-07T20:11:44+00:00
++ get_service_endpoint monitoring-service
++ local service=monitoring-service
+++ kubectl_bin get service/monitoring-service -o json
+++ jq '.status.loadBalancer.ingress[].hostname'
+++ sed -e 's/^"//; s/"$//;'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.XMRb5QcPtU
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.ewVcN1qjmx
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in $(seq 0 2)
+++ set +e
+++ kubectl get service/monitoring-service -o json
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.XMRb5QcPtU
+++ cat /tmp/tmp.ewVcN1qjmx
+++ rm /tmp/tmp.XMRb5QcPtU /tmp/tmp.ewVcN1qjmx
+++ return 0
++ local hostname=null
++ '[' -n null -a null '!=' null ']'
+++ kubectl_bin get service/monitoring-service -o json
+++ jq '.status.loadBalancer.ingress[].ip'
+++ sed -e 's/^"//; s/"$//;'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.xOMAYunmE8
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.OUzFheVW1B
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in $(seq 0 2)
+++ set +e
+++ kubectl get service/monitoring-service -o json
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.xOMAYunmE8
+++ cat /tmp/tmp.OUzFheVW1B
+++ rm /tmp/tmp.xOMAYunmE8 /tmp/tmp.OUzFheVW1B
+++ return 0
++ local ip=35.254.76.43
++ '[' -n 35.254.76.43 -a 35.254.76.43 '!=' null ']'
++ echo 35.254.76.43
++ return
+ endpoint=35.254.76.43
+ cat
+ local response
+ local wait_count=30
+ local retry=0
++ jq -r '(.rows // [])[]? | .fingerprint // empty'
+++ cat payload.json
++ execute_curl -s -k -H 'Authorization: Bearer glsa_ENtP2ImukR8cr3g1MllgZoNhKS1S8MWd_73f00929' -XPOST -d $'{\n  "period_start_from": "2026-07-07T08:11:44+00:00",\n  "period_start_to": "2026-07-07T20:11:44+00:00",\n  "group_by": "queryid",\n  "labels": [\n    {\n      "key": "service_type",\n      "value": [\n        "mongodb"\n      ]\n    },\n    {\n      "key": "environment",\n      "value": [\n        "dev-mongod-mongolog"\n      ]\n    }\n  ],\n  "columns": [\n    "load",\n    "num_queries",\n    "query_time"\n  ],\n  "order_by": "-load",\n  "offset": 0,\n  "limit": 10,\n  "main_metric": "load",\n  "search": ""\n}' https://35.254.76.43/v1/qan/metrics:getReport
++ local name=curl-19949
++ local image=docker.io/alpine/curl:latest
++ kubectl_bin run curl-19949 -n monitoring-pmm3-11763 --restart=Never --image=docker.io/alpine/curl:latest --command -- sleep 3600
+++ mktemp
++ local LAST_OUT=/tmp/tmp.Wzn774bFDM
+++ mktemp
++ local LAST_ERR=/tmp/tmp.diwspmKJvA
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl run curl-19949 -n monitoring-pmm3-11763 --restart=Never --image=docker.io/alpine/curl:latest --command -- sleep 3600
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.Wzn774bFDM
++ cat /tmp/tmp.diwspmKJvA
++ rm /tmp/tmp.Wzn774bFDM /tmp/tmp.diwspmKJvA
++ return 0
++ kubectl_bin wait pod/curl-19949 -n monitoring-pmm3-11763 --for=condition=Ready --timeout=60s
+++ mktemp
++ local LAST_OUT=/tmp/tmp.UdOXmsfous
+++ mktemp
++ local LAST_ERR=/tmp/tmp.C0j1Z2MLsS
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl wait pod/curl-19949 -n monitoring-pmm3-11763 --for=condition=Ready --timeout=60s
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.UdOXmsfous
++ cat /tmp/tmp.C0j1Z2MLsS
++ rm /tmp/tmp.UdOXmsfous /tmp/tmp.C0j1Z2MLsS
++ return 0
++ local exit_status=0
++ kubectl_bin exec curl-19949 -n monitoring-pmm3-11763 -- curl -s -k -H 'Authorization: Bearer glsa_ENtP2ImukR8cr3g1MllgZoNhKS1S8MWd_73f00929' -XPOST -d $'{\n  "period_start_from": "2026-07-07T08:11:44+00:00",\n  "period_start_to": "2026-07-07T20:11:44+00:00",\n  "group_by": "queryid",\n  "labels": [\n    {\n      "key": "service_type",\n      "value": [\n        "mongodb"\n      ]\n    },\n    {\n      "key": "environment",\n      "value": [\n        "dev-mongod-mongolog"\n      ]\n    }\n  ],\n  "columns": [\n    "load",\n    "num_queries",\n    "query_time"\n  ],\n  "order_by": "-load",\n  "offset": 0,\n  "limit": 10,\n  "main_metric": "load",\n  "search": ""\n}' https://35.254.76.43/v1/qan/metrics:getReport
+++ mktemp
++ local LAST_OUT=/tmp/tmp.KT2JvYf1bI
+++ mktemp
++ local LAST_ERR=/tmp/tmp.NwDCcNigph
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl exec curl-19949 -n monitoring-pmm3-11763 -- curl -s -k -H 'Authorization: Bearer glsa_ENtP2ImukR8cr3g1MllgZoNhKS1S8MWd_73f00929' -XPOST -d $'{\n  "period_start_from": "2026-07-07T08:11:44+00:00",\n  "period_start_to": "2026-07-07T20:11:44+00:00",\n  "group_by": "queryid",\n  "labels": [\n    {\n      "key": "service_type",\n      "value": [\n        "mongodb"\n      ]\n    },\n    {\n      "key": "environment",\n      "value": [\n        "dev-mongod-mongolog"\n      ]\n    }\n  ],\n  "columns": [\n    "load",\n    "num_queries",\n    "query_time"\n  ],\n  "order_by": "-load",\n  "offset": 0,\n  "limit": 10,\n  "main_metric": "load",\n  "search": ""\n}' https://35.254.76.43/v1/qan/metrics:getReport
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.KT2JvYf1bI
++ cat /tmp/tmp.NwDCcNigph
++ rm /tmp/tmp.KT2JvYf1bI /tmp/tmp.NwDCcNigph
++ return 0
++ kubectl_bin delete pod curl-19949 -n monitoring-pmm3-11763 --ignore-not-found --wait=false
++ return 0
+ [[ -n TOTAL
db.runCommand({"$audit":"?","$client":"?","$configTime":"?","$topologyTime":"?","awaitData":"?","clientOperationKey":"?","databaseVersion":"?","filter":"?","find":"?","hint":"?","mayBypassWriteBlocking":"?","readConcern":"?","shardVersion":"?","tailable":"?"}) ]]
+ rm -f payload.json
+ kubectl_bin patch psmdb monitoring-pmm3 --type json '-p=[{"op":"add","path":"/spec/pause","value":true}]'
++ mktemp
+ local LAST_OUT=/tmp/tmp.wLOuOptX8M
++ mktemp
+ local LAST_ERR=/tmp/tmp.iR8GqYuS7D
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl patch psmdb monitoring-pmm3 --type json '-p=[{"op":"add","path":"/spec/pause","value":true}]'
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.wLOuOptX8M
perconaservermongodb.psmdb.percona.com/monitoring-pmm3 patched
+ cat /tmp/tmp.iR8GqYuS7D
+ rm /tmp/tmp.wLOuOptX8M /tmp/tmp.iR8GqYuS7D
+ return 0
+ wait_for_delete pod/monitoring-pmm3-mongos-0
+ local res=pod/monitoring-pmm3-mongos-0
+ local wait_time=60
+ set +o xtrace
waiting for pod/monitoring-pmm3-mongos-0 to be deleted..........................Error from server (NotFound): pods "monitoring-pmm3-mongos-0" not found
Error from server (NotFound): pods "monitoring-pmm3-mongos-0" not found
Error from server (NotFound): pods "monitoring-pmm3-mongos-0" not found
Error from server (NotFound): pods "monitoring-pmm3-mongos-0" not found
+ wait_for_delete pod/monitoring-pmm3-rs0-0
+ local res=pod/monitoring-pmm3-rs0-0
+ local wait_time=60
+ set +o xtrace
waiting for pod/monitoring-pmm3-rs0-0 to be deleted......Error from server (NotFound): pods "monitoring-pmm3-rs0-0" not found
Error from server (NotFound): pods "monitoring-pmm3-rs0-0" not found
Error from server (NotFound): pods "monitoring-pmm3-rs0-0" not found
Error from server (NotFound): pods "monitoring-pmm3-rs0-0" not found
+ wait_for_delete pod/monitoring-pmm3-cfg-0
+ local res=pod/monitoring-pmm3-cfg-0
+ local wait_time=60
+ set +o xtrace
waiting for pod/monitoring-pmm3-cfg-0 to be deleted.......Error from server (NotFound): pods "monitoring-pmm3-cfg-0" not found
Error from server (NotFound): pods "monitoring-pmm3-cfg-0" not found
Error from server (NotFound): pods "monitoring-pmm3-cfg-0" not found
Error from server (NotFound): pods "monitoring-pmm3-cfg-0" not found
+ desc 'check if services are not deleted'
+ set +o xtrace


-----------------------------------------------------------------------------------
check if services are not deleted
-----------------------------------------------------------------------------------

+ kubectl_bin get svc monitoring-pmm3-rs0
++ mktemp
+ local LAST_OUT=/tmp/tmp.S2xR3SsKj3
++ mktemp
+ local LAST_ERR=/tmp/tmp.p2EtBQimZ6
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl get svc monitoring-pmm3-rs0
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.S2xR3SsKj3
NAME                  TYPE        CLUSTER-IP   EXTERNAL-IP   PORT(S)     AGE
monitoring-pmm3-rs0   ClusterIP   None         <none>        27019/TCP   32m
+ cat /tmp/tmp.p2EtBQimZ6
+ rm /tmp/tmp.S2xR3SsKj3 /tmp/tmp.p2EtBQimZ6
+ return 0
+ kubectl_bin get svc monitoring-pmm3-cfg
++ mktemp
+ local LAST_OUT=/tmp/tmp.uH3ItmeJH1
++ mktemp
+ local LAST_ERR=/tmp/tmp.w6ETphwZg9
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl get svc monitoring-pmm3-cfg
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.uH3ItmeJH1
NAME                  TYPE        CLUSTER-IP   EXTERNAL-IP   PORT(S)     AGE
monitoring-pmm3-cfg   ClusterIP   None         <none>        27019/TCP   32m
+ cat /tmp/tmp.w6ETphwZg9
+ rm /tmp/tmp.uH3ItmeJH1 /tmp/tmp.w6ETphwZg9
+ return 0
+ kubectl_bin get svc monitoring-pmm3-mongos
++ mktemp
+ local LAST_OUT=/tmp/tmp.a2texv7sBT
++ mktemp
+ local LAST_ERR=/tmp/tmp.vhnxL2wNdC
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl get svc monitoring-pmm3-mongos
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.a2texv7sBT
NAME                     TYPE        CLUSTER-IP       EXTERNAL-IP   PORT(S)     AGE
monitoring-pmm3-mongos   ClusterIP   34.118.230.116   <none>        27019/TCP   32m
+ cat /tmp/tmp.vhnxL2wNdC
+ rm /tmp/tmp.a2texv7sBT /tmp/tmp.vhnxL2wNdC
+ return 0
+ does_node_id_exists_in_pmm=($(does_node_id_exists "${nodeList[@]}"))
++ does_node_id_exists cccaab90-d90a-47fe-b618-5e42a0b21edc b71c4a17-6c43-4666-9275-d206aa45f397 7c8bf6ae-311d-46b8-bdd1-8dd64f649283 e97e0000-593c-4f24-bd3c-ccf938a451bd fa4bfc6f-11ec-42d2-9cc8-d216645f789d 1c9723e3-915b-48aa-8802-110a65a38734 d67f78f3-1137-46d0-af49-c33ae7ebe927 13549544-fc7f-45c7-89f3-c3cf312dbf57 f523c611-01f0-4636-999b-16de7ab7a399
++ nodeList=('cccaab90-d90a-47fe-b618-5e42a0b21edc' 'b71c4a17-6c43-4666-9275-d206aa45f397' '7c8bf6ae-311d-46b8-bdd1-8dd64f649283' 'e97e0000-593c-4f24-bd3c-ccf938a451bd' 'fa4bfc6f-11ec-42d2-9cc8-d216645f789d' '1c9723e3-915b-48aa-8802-110a65a38734' 'd67f78f3-1137-46d0-af49-c33ae7ebe927' '13549544-fc7f-45c7-89f3-c3cf312dbf57' 'f523c611-01f0-4636-999b-16de7ab7a399')
++ local -a nodeList
++ nodeList_from_pmm=()
++ local -a nodeList_from_pmm
+++ kubectl_bin get secret pmm-secret -o 'jsonpath={.data.PMM_ADMIN_PASSWORD}'
+++ base64 --decode
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.WNuJckPmV0
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.rUPQBuCTYv
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in $(seq 0 2)
+++ set +e
+++ kubectl get secret pmm-secret -o 'jsonpath={.data.PMM_ADMIN_PASSWORD}'
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.WNuJckPmV0
+++ cat /tmp/tmp.rUPQBuCTYv
+++ rm /tmp/tmp.WNuJckPmV0 /tmp/tmp.rUPQBuCTYv
+++ return 0
++ local 'admin_pass=Os\2'\''.99QS0gs;]f'
+++ get_service_endpoint monitoring-service
+++ local service=monitoring-service
++++ kubectl_bin get service/monitoring-service -o json
++++ jq '.status.loadBalancer.ingress[].hostname'
++++ sed -e 's/^"//; s/"$//;'
+++++ mktemp
++++ local LAST_OUT=/tmp/tmp.l9gEwmTkBE
+++++ mktemp
++++ local LAST_ERR=/tmp/tmp.YxwXRGBvyB
++++ local exit_status=0
++++ local timeout=4
+++++ seq 0 2
++++ for i in $(seq 0 2)
++++ set +e
++++ kubectl get service/monitoring-service -o json
++++ exit_status=0
++++ set -e
++++ '[' 0 '!=' 0 -a -n 1 ']'
++++ break
++++ cat /tmp/tmp.l9gEwmTkBE
++++ cat /tmp/tmp.YxwXRGBvyB
++++ rm /tmp/tmp.l9gEwmTkBE /tmp/tmp.YxwXRGBvyB
++++ return 0
+++ local hostname=null
+++ '[' -n null -a null '!=' null ']'
++++ kubectl_bin get service/monitoring-service -o json
++++ jq '.status.loadBalancer.ingress[].ip'
++++ sed -e 's/^"//; s/"$//;'
+++++ mktemp
++++ local LAST_OUT=/tmp/tmp.v0XQh7u89r
+++++ mktemp
++++ local LAST_ERR=/tmp/tmp.hxBhKb9MSM
++++ local exit_status=0
++++ local timeout=4
+++++ seq 0 2
++++ for i in $(seq 0 2)
++++ set +e
++++ kubectl get service/monitoring-service -o json
++++ exit_status=0
++++ set -e
++++ '[' 0 '!=' 0 -a -n 1 ']'
++++ break
++++ cat /tmp/tmp.v0XQh7u89r
++++ cat /tmp/tmp.hxBhKb9MSM
++++ rm /tmp/tmp.v0XQh7u89r /tmp/tmp.hxBhKb9MSM
++++ return 0
+++ local ip=35.254.76.43
+++ '[' -n 35.254.76.43 -a 35.254.76.43 '!=' null ']'
+++ echo 35.254.76.43
+++ return
++ local endpoint=35.254.76.43
++ for node_id in "${nodeList[@]}"
++ nodeList_from_pmm+=($(execute_curl --insecure -s -X GET --user "admin:${admin_pass}" "https://${endpoint}/v1/inventory/nodes?node_type=NODE_TYPE_CONTAINER_NODE" | jq -r --arg node_id "$node_id" '.nodes[]? | select(.node_id == $node_id) | .node_id'))
+++ execute_curl --insecure -s -X GET --user 'admin:Os\2'\''.99QS0gs;]f' 'https://35.254.76.43/v1/inventory/nodes?node_type=NODE_TYPE_CONTAINER_NODE'
+++ jq -r --arg node_id cccaab90-d90a-47fe-b618-5e42a0b21edc '.nodes[]? | select(.node_id == $node_id) | .node_id'
+++ local name=curl-21841
+++ local image=docker.io/alpine/curl:latest
+++ kubectl_bin run curl-21841 -n monitoring-pmm3-11763 --restart=Never --image=docker.io/alpine/curl:latest --command -- sleep 3600
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.MODps6uFJh
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.ZoeqC8h0fp
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in $(seq 0 2)
+++ set +e
+++ kubectl run curl-21841 -n monitoring-pmm3-11763 --restart=Never --image=docker.io/alpine/curl:latest --command -- sleep 3600
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.MODps6uFJh
+++ cat /tmp/tmp.ZoeqC8h0fp
+++ rm /tmp/tmp.MODps6uFJh /tmp/tmp.ZoeqC8h0fp
+++ return 0
+++ kubectl_bin wait pod/curl-21841 -n monitoring-pmm3-11763 --for=condition=Ready --timeout=60s
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.LKbt2S3dNm
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.lAhjpQf6oB
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in $(seq 0 2)
+++ set +e
+++ kubectl wait pod/curl-21841 -n monitoring-pmm3-11763 --for=condition=Ready --timeout=60s
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.LKbt2S3dNm
+++ cat /tmp/tmp.lAhjpQf6oB
E0707 20:13:57.585128  697383 reflector.go:227] "Failed to watch" err="Get \"https://34.27.139.243/api/v1/namespaces/monitoring-pmm3-11763/pods?allowWatchBookmarks=true&fieldSelector=metadata.name%3Dcurl-21841&resourceVersion=1783455237405551008&timeoutSeconds=345&watch=true\": context canceled" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" type="*unstructured.Unstructured"
+++ rm /tmp/tmp.LKbt2S3dNm /tmp/tmp.lAhjpQf6oB
+++ return 0
+++ local exit_status=0
+++ kubectl_bin exec curl-21841 -n monitoring-pmm3-11763 -- curl --insecure -s -X GET --user 'admin:Os\2'\''.99QS0gs;]f' 'https://35.254.76.43/v1/inventory/nodes?node_type=NODE_TYPE_CONTAINER_NODE'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.u8LQssT6FG
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.KEwiulM2Az
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in $(seq 0 2)
+++ set +e
+++ kubectl exec curl-21841 -n monitoring-pmm3-11763 -- curl --insecure -s -X GET --user 'admin:Os\2'\''.99QS0gs;]f' 'https://35.254.76.43/v1/inventory/nodes?node_type=NODE_TYPE_CONTAINER_NODE'
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.u8LQssT6FG
+++ cat /tmp/tmp.KEwiulM2Az
+++ rm /tmp/tmp.u8LQssT6FG /tmp/tmp.KEwiulM2Az
+++ return 0
+++ kubectl_bin delete pod curl-21841 -n monitoring-pmm3-11763 --ignore-not-found --wait=false
+++ return 0
++ for node_id in "${nodeList[@]}"
++ nodeList_from_pmm+=($(execute_curl --insecure -s -X GET --user "admin:${admin_pass}" "https://${endpoint}/v1/inventory/nodes?node_type=NODE_TYPE_CONTAINER_NODE" | jq -r --arg node_id "$node_id" '.nodes[]? | select(.node_id == $node_id) | .node_id'))
+++ execute_curl --insecure -s -X GET --user 'admin:Os\2'\''.99QS0gs;]f' 'https://35.254.76.43/v1/inventory/nodes?node_type=NODE_TYPE_CONTAINER_NODE'
+++ local name=curl-22965
+++ local image=docker.io/alpine/curl:latest
+++ kubectl_bin run curl-22965 -n monitoring-pmm3-11763 --restart=Never --image=docker.io/alpine/curl:latest --command -- sleep 3600
+++ jq -r --arg node_id b71c4a17-6c43-4666-9275-d206aa45f397 '.nodes[]? | select(.node_id == $node_id) | .node_id'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.QeTp4GAWqz
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.eKonW1j9WQ
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in $(seq 0 2)
+++ set +e
+++ kubectl run curl-22965 -n monitoring-pmm3-11763 --restart=Never --image=docker.io/alpine/curl:latest --command -- sleep 3600
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.QeTp4GAWqz
+++ cat /tmp/tmp.eKonW1j9WQ
+++ rm /tmp/tmp.QeTp4GAWqz /tmp/tmp.eKonW1j9WQ
+++ return 0
+++ kubectl_bin wait pod/curl-22965 -n monitoring-pmm3-11763 --for=condition=Ready --timeout=60s
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.vGSqTNLbL1
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.yxpggxpBXD
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in $(seq 0 2)
+++ set +e
+++ kubectl wait pod/curl-22965 -n monitoring-pmm3-11763 --for=condition=Ready --timeout=60s
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.vGSqTNLbL1
+++ cat /tmp/tmp.yxpggxpBXD
+++ rm /tmp/tmp.vGSqTNLbL1 /tmp/tmp.yxpggxpBXD
+++ return 0
+++ local exit_status=0
+++ kubectl_bin exec curl-22965 -n monitoring-pmm3-11763 -- curl --insecure -s -X GET --user 'admin:Os\2'\''.99QS0gs;]f' 'https://35.254.76.43/v1/inventory/nodes?node_type=NODE_TYPE_CONTAINER_NODE'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.CxyJ3QLfw3
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.3fizyGKZd4
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in $(seq 0 2)
+++ set +e
+++ kubectl exec curl-22965 -n monitoring-pmm3-11763 -- curl --insecure -s -X GET --user 'admin:Os\2'\''.99QS0gs;]f' 'https://35.254.76.43/v1/inventory/nodes?node_type=NODE_TYPE_CONTAINER_NODE'
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.CxyJ3QLfw3
+++ cat /tmp/tmp.3fizyGKZd4
+++ rm /tmp/tmp.CxyJ3QLfw3 /tmp/tmp.3fizyGKZd4
+++ return 0
+++ kubectl_bin delete pod curl-22965 -n monitoring-pmm3-11763 --ignore-not-found --wait=false
+++ return 0
++ for node_id in "${nodeList[@]}"
++ nodeList_from_pmm+=($(execute_curl --insecure -s -X GET --user "admin:${admin_pass}" "https://${endpoint}/v1/inventory/nodes?node_type=NODE_TYPE_CONTAINER_NODE" | jq -r --arg node_id "$node_id" '.nodes[]? | select(.node_id == $node_id) | .node_id'))
+++ execute_curl --insecure -s -X GET --user 'admin:Os\2'\''.99QS0gs;]f' 'https://35.254.76.43/v1/inventory/nodes?node_type=NODE_TYPE_CONTAINER_NODE'
+++ jq -r --arg node_id 7c8bf6ae-311d-46b8-bdd1-8dd64f649283 '.nodes[]? | select(.node_id == $node_id) | .node_id'
+++ local name=curl-23931
+++ local image=docker.io/alpine/curl:latest
+++ kubectl_bin run curl-23931 -n monitoring-pmm3-11763 --restart=Never --image=docker.io/alpine/curl:latest --command -- sleep 3600
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.opcymDdi6i
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.zOClsMIM7g
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in $(seq 0 2)
+++ set +e
+++ kubectl run curl-23931 -n monitoring-pmm3-11763 --restart=Never --image=docker.io/alpine/curl:latest --command -- sleep 3600
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.opcymDdi6i
+++ cat /tmp/tmp.zOClsMIM7g
+++ rm /tmp/tmp.opcymDdi6i /tmp/tmp.zOClsMIM7g
+++ return 0
+++ kubectl_bin wait pod/curl-23931 -n monitoring-pmm3-11763 --for=condition=Ready --timeout=60s
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.yOdYp9EDM4
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.jamAxSJpef
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in $(seq 0 2)
+++ set +e
+++ kubectl wait pod/curl-23931 -n monitoring-pmm3-11763 --for=condition=Ready --timeout=60s
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.yOdYp9EDM4
+++ cat /tmp/tmp.jamAxSJpef
+++ rm /tmp/tmp.yOdYp9EDM4 /tmp/tmp.jamAxSJpef
+++ return 0
+++ local exit_status=0
+++ kubectl_bin exec curl-23931 -n monitoring-pmm3-11763 -- curl --insecure -s -X GET --user 'admin:Os\2'\''.99QS0gs;]f' 'https://35.254.76.43/v1/inventory/nodes?node_type=NODE_TYPE_CONTAINER_NODE'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.queClJDztV
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.zw3jWuLXU0
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in $(seq 0 2)
+++ set +e
+++ kubectl exec curl-23931 -n monitoring-pmm3-11763 -- curl --insecure -s -X GET --user 'admin:Os\2'\''.99QS0gs;]f' 'https://35.254.76.43/v1/inventory/nodes?node_type=NODE_TYPE_CONTAINER_NODE'
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.queClJDztV
+++ cat /tmp/tmp.zw3jWuLXU0
+++ rm /tmp/tmp.queClJDztV /tmp/tmp.zw3jWuLXU0
+++ return 0
+++ kubectl_bin delete pod curl-23931 -n monitoring-pmm3-11763 --ignore-not-found --wait=false
+++ return 0
++ for node_id in "${nodeList[@]}"
++ nodeList_from_pmm+=($(execute_curl --insecure -s -X GET --user "admin:${admin_pass}" "https://${endpoint}/v1/inventory/nodes?node_type=NODE_TYPE_CONTAINER_NODE" | jq -r --arg node_id "$node_id" '.nodes[]? | select(.node_id == $node_id) | .node_id'))
+++ execute_curl --insecure -s -X GET --user 'admin:Os\2'\''.99QS0gs;]f' 'https://35.254.76.43/v1/inventory/nodes?node_type=NODE_TYPE_CONTAINER_NODE'
+++ local name=curl-15150
+++ local image=docker.io/alpine/curl:latest
+++ kubectl_bin run curl-15150 -n monitoring-pmm3-11763 --restart=Never --image=docker.io/alpine/curl:latest --command -- sleep 3600
+++ jq -r --arg node_id e97e0000-593c-4f24-bd3c-ccf938a451bd '.nodes[]? | select(.node_id == $node_id) | .node_id'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.tOAksrX6EW
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.vHZdXmADXx
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in $(seq 0 2)
+++ set +e
+++ kubectl run curl-15150 -n monitoring-pmm3-11763 --restart=Never --image=docker.io/alpine/curl:latest --command -- sleep 3600
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.tOAksrX6EW
+++ cat /tmp/tmp.vHZdXmADXx
+++ rm /tmp/tmp.tOAksrX6EW /tmp/tmp.vHZdXmADXx
+++ return 0
+++ kubectl_bin wait pod/curl-15150 -n monitoring-pmm3-11763 --for=condition=Ready --timeout=60s
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.jA7lYVApqC
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.2G9AOEDes0
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in $(seq 0 2)
+++ set +e
+++ kubectl wait pod/curl-15150 -n monitoring-pmm3-11763 --for=condition=Ready --timeout=60s
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.jA7lYVApqC
+++ cat /tmp/tmp.2G9AOEDes0
+++ rm /tmp/tmp.jA7lYVApqC /tmp/tmp.2G9AOEDes0
+++ return 0
+++ local exit_status=0
+++ kubectl_bin exec curl-15150 -n monitoring-pmm3-11763 -- curl --insecure -s -X GET --user 'admin:Os\2'\''.99QS0gs;]f' 'https://35.254.76.43/v1/inventory/nodes?node_type=NODE_TYPE_CONTAINER_NODE'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.kaDRcDY1Fq
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.CbO4VSBQEK
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in $(seq 0 2)
+++ set +e
+++ kubectl exec curl-15150 -n monitoring-pmm3-11763 -- curl --insecure -s -X GET --user 'admin:Os\2'\''.99QS0gs;]f' 'https://35.254.76.43/v1/inventory/nodes?node_type=NODE_TYPE_CONTAINER_NODE'
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.kaDRcDY1Fq
+++ cat /tmp/tmp.CbO4VSBQEK
+++ rm /tmp/tmp.kaDRcDY1Fq /tmp/tmp.CbO4VSBQEK
+++ return 0
+++ kubectl_bin delete pod curl-15150 -n monitoring-pmm3-11763 --ignore-not-found --wait=false
+++ return 0
++ for node_id in "${nodeList[@]}"
++ nodeList_from_pmm+=($(execute_curl --insecure -s -X GET --user "admin:${admin_pass}" "https://${endpoint}/v1/inventory/nodes?node_type=NODE_TYPE_CONTAINER_NODE" | jq -r --arg node_id "$node_id" '.nodes[]? | select(.node_id == $node_id) | .node_id'))
+++ execute_curl --insecure -s -X GET --user 'admin:Os\2'\''.99QS0gs;]f' 'https://35.254.76.43/v1/inventory/nodes?node_type=NODE_TYPE_CONTAINER_NODE'
+++ local name=curl-21262
+++ local image=docker.io/alpine/curl:latest
+++ jq -r --arg node_id fa4bfc6f-11ec-42d2-9cc8-d216645f789d '.nodes[]? | select(.node_id == $node_id) | .node_id'
+++ kubectl_bin run curl-21262 -n monitoring-pmm3-11763 --restart=Never --image=docker.io/alpine/curl:latest --command -- sleep 3600
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.hIdoGDHJF3
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.Y9uH3KnDl2
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in $(seq 0 2)
+++ set +e
+++ kubectl run curl-21262 -n monitoring-pmm3-11763 --restart=Never --image=docker.io/alpine/curl:latest --command -- sleep 3600
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.hIdoGDHJF3
+++ cat /tmp/tmp.Y9uH3KnDl2
+++ rm /tmp/tmp.hIdoGDHJF3 /tmp/tmp.Y9uH3KnDl2
+++ return 0
+++ kubectl_bin wait pod/curl-21262 -n monitoring-pmm3-11763 --for=condition=Ready --timeout=60s
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.R2FLQroRSl
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.GlXCFoABun
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in $(seq 0 2)
+++ set +e
+++ kubectl wait pod/curl-21262 -n monitoring-pmm3-11763 --for=condition=Ready --timeout=60s
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.R2FLQroRSl
+++ cat /tmp/tmp.GlXCFoABun
+++ rm /tmp/tmp.R2FLQroRSl /tmp/tmp.GlXCFoABun
+++ return 0
+++ local exit_status=0
+++ kubectl_bin exec curl-21262 -n monitoring-pmm3-11763 -- curl --insecure -s -X GET --user 'admin:Os\2'\''.99QS0gs;]f' 'https://35.254.76.43/v1/inventory/nodes?node_type=NODE_TYPE_CONTAINER_NODE'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.BX9juSqcQh
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.T5bKx96IGX
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in $(seq 0 2)
+++ set +e
+++ kubectl exec curl-21262 -n monitoring-pmm3-11763 -- curl --insecure -s -X GET --user 'admin:Os\2'\''.99QS0gs;]f' 'https://35.254.76.43/v1/inventory/nodes?node_type=NODE_TYPE_CONTAINER_NODE'
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.BX9juSqcQh
+++ cat /tmp/tmp.T5bKx96IGX
+++ rm /tmp/tmp.BX9juSqcQh /tmp/tmp.T5bKx96IGX
+++ return 0
+++ kubectl_bin delete pod curl-21262 -n monitoring-pmm3-11763 --ignore-not-found --wait=false
+++ return 0
++ for node_id in "${nodeList[@]}"
++ nodeList_from_pmm+=($(execute_curl --insecure -s -X GET --user "admin:${admin_pass}" "https://${endpoint}/v1/inventory/nodes?node_type=NODE_TYPE_CONTAINER_NODE" | jq -r --arg node_id "$node_id" '.nodes[]? | select(.node_id == $node_id) | .node_id'))
+++ execute_curl --insecure -s -X GET --user 'admin:Os\2'\''.99QS0gs;]f' 'https://35.254.76.43/v1/inventory/nodes?node_type=NODE_TYPE_CONTAINER_NODE'
+++ jq -r --arg node_id 1c9723e3-915b-48aa-8802-110a65a38734 '.nodes[]? | select(.node_id == $node_id) | .node_id'
+++ local name=curl-19364
+++ local image=docker.io/alpine/curl:latest
+++ kubectl_bin run curl-19364 -n monitoring-pmm3-11763 --restart=Never --image=docker.io/alpine/curl:latest --command -- sleep 3600
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.e36hOdog48
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.zHfhZJSN43
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in $(seq 0 2)
+++ set +e
+++ kubectl run curl-19364 -n monitoring-pmm3-11763 --restart=Never --image=docker.io/alpine/curl:latest --command -- sleep 3600
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.e36hOdog48
+++ cat /tmp/tmp.zHfhZJSN43
+++ rm /tmp/tmp.e36hOdog48 /tmp/tmp.zHfhZJSN43
+++ return 0
+++ kubectl_bin wait pod/curl-19364 -n monitoring-pmm3-11763 --for=condition=Ready --timeout=60s
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.TZTIo2CI1F
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.Bln9QWRBfv
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in $(seq 0 2)
+++ set +e
+++ kubectl wait pod/curl-19364 -n monitoring-pmm3-11763 --for=condition=Ready --timeout=60s
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.TZTIo2CI1F
+++ cat /tmp/tmp.Bln9QWRBfv
E0707 20:14:18.716566  698310 reflector.go:227] "Failed to watch" err="Get \"https://34.27.139.243/api/v1/namespaces/monitoring-pmm3-11763/pods?allowWatchBookmarks=true&fieldSelector=metadata.name%3Dcurl-19364&resourceVersion=1783455258503951021&timeoutSeconds=445&watch=true\": context canceled" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" type="*unstructured.Unstructured"
+++ rm /tmp/tmp.TZTIo2CI1F /tmp/tmp.Bln9QWRBfv
+++ return 0
+++ local exit_status=0
+++ kubectl_bin exec curl-19364 -n monitoring-pmm3-11763 -- curl --insecure -s -X GET --user 'admin:Os\2'\''.99QS0gs;]f' 'https://35.254.76.43/v1/inventory/nodes?node_type=NODE_TYPE_CONTAINER_NODE'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.f7Bf1VdBYp
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.tCUOrWUsDs
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in $(seq 0 2)
+++ set +e
+++ kubectl exec curl-19364 -n monitoring-pmm3-11763 -- curl --insecure -s -X GET --user 'admin:Os\2'\''.99QS0gs;]f' 'https://35.254.76.43/v1/inventory/nodes?node_type=NODE_TYPE_CONTAINER_NODE'
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.f7Bf1VdBYp
+++ cat /tmp/tmp.tCUOrWUsDs
+++ rm /tmp/tmp.f7Bf1VdBYp /tmp/tmp.tCUOrWUsDs
+++ return 0
+++ kubectl_bin delete pod curl-19364 -n monitoring-pmm3-11763 --ignore-not-found --wait=false
+++ return 0
++ for node_id in "${nodeList[@]}"
++ nodeList_from_pmm+=($(execute_curl --insecure -s -X GET --user "admin:${admin_pass}" "https://${endpoint}/v1/inventory/nodes?node_type=NODE_TYPE_CONTAINER_NODE" | jq -r --arg node_id "$node_id" '.nodes[]? | select(.node_id == $node_id) | .node_id'))
+++ execute_curl --insecure -s -X GET --user 'admin:Os\2'\''.99QS0gs;]f' 'https://35.254.76.43/v1/inventory/nodes?node_type=NODE_TYPE_CONTAINER_NODE'
+++ jq -r --arg node_id d67f78f3-1137-46d0-af49-c33ae7ebe927 '.nodes[]? | select(.node_id == $node_id) | .node_id'
+++ local name=curl-27707
+++ local image=docker.io/alpine/curl:latest
+++ kubectl_bin run curl-27707 -n monitoring-pmm3-11763 --restart=Never --image=docker.io/alpine/curl:latest --command -- sleep 3600
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.DPDSF74h28
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.DoJ03gdcwE
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in $(seq 0 2)
+++ set +e
+++ kubectl run curl-27707 -n monitoring-pmm3-11763 --restart=Never --image=docker.io/alpine/curl:latest --command -- sleep 3600
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.DPDSF74h28
+++ cat /tmp/tmp.DoJ03gdcwE
+++ rm /tmp/tmp.DPDSF74h28 /tmp/tmp.DoJ03gdcwE
+++ return 0
+++ kubectl_bin wait pod/curl-27707 -n monitoring-pmm3-11763 --for=condition=Ready --timeout=60s
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.USX2dfHOAK
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.ZWcpsjMwma
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in $(seq 0 2)
+++ set +e
+++ kubectl wait pod/curl-27707 -n monitoring-pmm3-11763 --for=condition=Ready --timeout=60s
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.USX2dfHOAK
+++ cat /tmp/tmp.ZWcpsjMwma
+++ rm /tmp/tmp.USX2dfHOAK /tmp/tmp.ZWcpsjMwma
+++ return 0
+++ local exit_status=0
+++ kubectl_bin exec curl-27707 -n monitoring-pmm3-11763 -- curl --insecure -s -X GET --user 'admin:Os\2'\''.99QS0gs;]f' 'https://35.254.76.43/v1/inventory/nodes?node_type=NODE_TYPE_CONTAINER_NODE'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.8hYOcYHdGV
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.KgIR3sbGY9
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in $(seq 0 2)
+++ set +e
+++ kubectl exec curl-27707 -n monitoring-pmm3-11763 -- curl --insecure -s -X GET --user 'admin:Os\2'\''.99QS0gs;]f' 'https://35.254.76.43/v1/inventory/nodes?node_type=NODE_TYPE_CONTAINER_NODE'
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.8hYOcYHdGV
+++ cat /tmp/tmp.KgIR3sbGY9
+++ rm /tmp/tmp.8hYOcYHdGV /tmp/tmp.KgIR3sbGY9
+++ return 0
+++ kubectl_bin delete pod curl-27707 -n monitoring-pmm3-11763 --ignore-not-found --wait=false
+++ return 0
++ for node_id in "${nodeList[@]}"
++ nodeList_from_pmm+=($(execute_curl --insecure -s -X GET --user "admin:${admin_pass}" "https://${endpoint}/v1/inventory/nodes?node_type=NODE_TYPE_CONTAINER_NODE" | jq -r --arg node_id "$node_id" '.nodes[]? | select(.node_id == $node_id) | .node_id'))
+++ execute_curl --insecure -s -X GET --user 'admin:Os\2'\''.99QS0gs;]f' 'https://35.254.76.43/v1/inventory/nodes?node_type=NODE_TYPE_CONTAINER_NODE'
+++ jq -r --arg node_id 13549544-fc7f-45c7-89f3-c3cf312dbf57 '.nodes[]? | select(.node_id == $node_id) | .node_id'
+++ local name=curl-19774
+++ local image=docker.io/alpine/curl:latest
+++ kubectl_bin run curl-19774 -n monitoring-pmm3-11763 --restart=Never --image=docker.io/alpine/curl:latest --command -- sleep 3600
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.yLy7IHViNX
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.JIAXWqbGxt
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in $(seq 0 2)
+++ set +e
+++ kubectl run curl-19774 -n monitoring-pmm3-11763 --restart=Never --image=docker.io/alpine/curl:latest --command -- sleep 3600
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.yLy7IHViNX
+++ cat /tmp/tmp.JIAXWqbGxt
+++ rm /tmp/tmp.yLy7IHViNX /tmp/tmp.JIAXWqbGxt
+++ return 0
+++ kubectl_bin wait pod/curl-19774 -n monitoring-pmm3-11763 --for=condition=Ready --timeout=60s
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.d7KWEqZ2sk
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.u94O4vElLV
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in $(seq 0 2)
+++ set +e
+++ kubectl wait pod/curl-19774 -n monitoring-pmm3-11763 --for=condition=Ready --timeout=60s
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.d7KWEqZ2sk
+++ cat /tmp/tmp.u94O4vElLV
+++ rm /tmp/tmp.d7KWEqZ2sk /tmp/tmp.u94O4vElLV
+++ return 0
+++ local exit_status=0
+++ kubectl_bin exec curl-19774 -n monitoring-pmm3-11763 -- curl --insecure -s -X GET --user 'admin:Os\2'\''.99QS0gs;]f' 'https://35.254.76.43/v1/inventory/nodes?node_type=NODE_TYPE_CONTAINER_NODE'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.w8NxU5LV4A
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.kbS0FOM0Kl
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in $(seq 0 2)
+++ set +e
+++ kubectl exec curl-19774 -n monitoring-pmm3-11763 -- curl --insecure -s -X GET --user 'admin:Os\2'\''.99QS0gs;]f' 'https://35.254.76.43/v1/inventory/nodes?node_type=NODE_TYPE_CONTAINER_NODE'
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.w8NxU5LV4A
+++ cat /tmp/tmp.kbS0FOM0Kl
+++ rm /tmp/tmp.w8NxU5LV4A /tmp/tmp.kbS0FOM0Kl
+++ return 0
+++ kubectl_bin delete pod curl-19774 -n monitoring-pmm3-11763 --ignore-not-found --wait=false
+++ return 0
++ for node_id in "${nodeList[@]}"
++ nodeList_from_pmm+=($(execute_curl --insecure -s -X GET --user "admin:${admin_pass}" "https://${endpoint}/v1/inventory/nodes?node_type=NODE_TYPE_CONTAINER_NODE" | jq -r --arg node_id "$node_id" '.nodes[]? | select(.node_id == $node_id) | .node_id'))
+++ execute_curl --insecure -s -X GET --user 'admin:Os\2'\''.99QS0gs;]f' 'https://35.254.76.43/v1/inventory/nodes?node_type=NODE_TYPE_CONTAINER_NODE'
+++ local name=curl-17844
+++ local image=docker.io/alpine/curl:latest
+++ kubectl_bin run curl-17844 -n monitoring-pmm3-11763 --restart=Never --image=docker.io/alpine/curl:latest --command -- sleep 3600
+++ jq -r --arg node_id f523c611-01f0-4636-999b-16de7ab7a399 '.nodes[]? | select(.node_id == $node_id) | .node_id'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.WhO5yjob8d
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.437Jhp5WCO
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in $(seq 0 2)
+++ set +e
+++ kubectl run curl-17844 -n monitoring-pmm3-11763 --restart=Never --image=docker.io/alpine/curl:latest --command -- sleep 3600
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.WhO5yjob8d
+++ cat /tmp/tmp.437Jhp5WCO
+++ rm /tmp/tmp.WhO5yjob8d /tmp/tmp.437Jhp5WCO
+++ return 0
+++ kubectl_bin wait pod/curl-17844 -n monitoring-pmm3-11763 --for=condition=Ready --timeout=60s
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.Il4fd4y9j8
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.d4NqN5bmkJ
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in $(seq 0 2)
+++ set +e
+++ kubectl wait pod/curl-17844 -n monitoring-pmm3-11763 --for=condition=Ready --timeout=60s
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.Il4fd4y9j8
+++ cat /tmp/tmp.d4NqN5bmkJ
+++ rm /tmp/tmp.Il4fd4y9j8 /tmp/tmp.d4NqN5bmkJ
+++ return 0
+++ local exit_status=0
+++ kubectl_bin exec curl-17844 -n monitoring-pmm3-11763 -- curl --insecure -s -X GET --user 'admin:Os\2'\''.99QS0gs;]f' 'https://35.254.76.43/v1/inventory/nodes?node_type=NODE_TYPE_CONTAINER_NODE'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.V123BGdBes
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.TBlyQJM0JP
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in $(seq 0 2)
+++ set +e
+++ kubectl exec curl-17844 -n monitoring-pmm3-11763 -- curl --insecure -s -X GET --user 'admin:Os\2'\''.99QS0gs;]f' 'https://35.254.76.43/v1/inventory/nodes?node_type=NODE_TYPE_CONTAINER_NODE'
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.V123BGdBes
+++ cat /tmp/tmp.TBlyQJM0JP
+++ rm /tmp/tmp.V123BGdBes /tmp/tmp.TBlyQJM0JP
+++ return 0
+++ kubectl_bin delete pod curl-17844 -n monitoring-pmm3-11763 --ignore-not-found --wait=false
+++ return 0
++ echo
+ [[ -n '' ]]
++ kubectl_bin logs monitoring-pmm3-rs0-0 pmm-client
++ grep -c 'cannot auto discover databases and collections'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.5Grrd5EbGe
+++ mktemp
++ local LAST_ERR=/tmp/tmp.QxdjGjRYUn
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl logs monitoring-pmm3-rs0-0 pmm-client
++ exit_status=1
++ set -e
++ '[' 1 '!=' 0 -a -n 1 ']'
++ cat /tmp/tmp.5Grrd5EbGe
++ cat /tmp/tmp.QxdjGjRYUn
error: error from server (NotFound): pods "monitoring-pmm3-rs0-0" not found in namespace "monitoring-pmm3-11763"
++ sleep 0
++ for i in $(seq 0 2)
++ set +e
++ kubectl logs monitoring-pmm3-rs0-0 pmm-client
++ exit_status=1
++ set -e
++ '[' 1 '!=' 0 -a -n 1 ']'
++ cat /tmp/tmp.5Grrd5EbGe
++ cat /tmp/tmp.QxdjGjRYUn
error: error from server (NotFound): pods "monitoring-pmm3-rs0-0" not found in namespace "monitoring-pmm3-11763"
++ sleep 4
++ for i in $(seq 0 2)
++ set +e
++ kubectl logs monitoring-pmm3-rs0-0 pmm-client
++ exit_status=1
++ set -e
++ '[' 1 '!=' 0 -a -n 1 ']'
++ cat /tmp/tmp.5Grrd5EbGe
++ cat /tmp/tmp.QxdjGjRYUn
error: error from server (NotFound): pods "monitoring-pmm3-rs0-0" not found in namespace "monitoring-pmm3-11763"
++ sleep 8
++ cat /tmp/tmp.5Grrd5EbGe
++ cat /tmp/tmp.QxdjGjRYUn
error: error from server (NotFound): pods "monitoring-pmm3-rs0-0" not found in namespace "monitoring-pmm3-11763"
++ rm /tmp/tmp.5Grrd5EbGe /tmp/tmp.QxdjGjRYUn
++ return 1
+ [[ 0 != 0 ]]
+ helm uninstall monitoring
release "monitoring" uninstalled
+ destroy monitoring-pmm3-11763
+ local namespace=monitoring-pmm3-11763
+ local ignore_logs=true
+ [[ 0 == 1 ]]
+ desc 'destroy cluster/operator and all other resources'
+ set +o xtrace


-----------------------------------------------------------------------------------
destroy cluster/operator and all other resources
-----------------------------------------------------------------------------------

+ '[' true == false ']'
+ delete_backups
+ desc 'Delete psmdb-backup'
+ set +o xtrace


-----------------------------------------------------------------------------------
Delete psmdb-backup
-----------------------------------------------------------------------------------

++ kubectl_bin get psmdb-backup --no-headers
++ wc -l
+++ mktemp
++ local LAST_OUT=/tmp/tmp.dwFpK29MZM
+++ mktemp
++ local LAST_ERR=/tmp/tmp.6lYChjF4ig
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb-backup --no-headers
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.dwFpK29MZM
++ cat /tmp/tmp.6lYChjF4ig
No resources found in monitoring-pmm3-11763 namespace.
++ rm /tmp/tmp.dwFpK29MZM /tmp/tmp.6lYChjF4ig
++ return 0
+ '[' 0 '!=' 0 ']'
+ delete_crd
+ desc 'get and delete old CRDs and RBAC'
+ set +o xtrace


-----------------------------------------------------------------------------------
get and delete old CRDs and RBAC
-----------------------------------------------------------------------------------

+ kubectl_bin delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2412/deploy/crd.yaml --ignore-not-found --wait=false
++ mktemp
+ local LAST_OUT=/tmp/tmp.ugmf78HN24
++ mktemp
+ local LAST_ERR=/tmp/tmp.hFv5qOWVJb
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2412/deploy/crd.yaml --ignore-not-found --wait=false
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.ugmf78HN24
customresourcedefinition.apiextensions.k8s.io "perconaservermongodbbackups.psmdb.percona.com" deleted
customresourcedefinition.apiextensions.k8s.io "perconaservermongodbclustersyncs.psmdb.percona.com" deleted
customresourcedefinition.apiextensions.k8s.io "perconaservermongodbrestores.psmdb.percona.com" deleted
customresourcedefinition.apiextensions.k8s.io "perconaservermongodbs.psmdb.percona.com" deleted
+ cat /tmp/tmp.hFv5qOWVJb
+ rm /tmp/tmp.ugmf78HN24 /tmp/tmp.hFv5qOWVJb
+ return 0
++ yq eval .metadata.name /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2412/deploy/crd.yaml
++ grep -v '\-\-\-'
grep: warning: stray \ before -
grep: warning: stray \ before -
+ for crd_name in $(yq eval '.metadata.name' "${src_dir}/deploy/crd.yaml" | grep -v '\-\-\-')
+ kubectl get perconaservermongodbbackups.psmdb.percona.com --all-namespaces -o wide
+ grep -v NAMESPACE
+ xargs -L 1 sh -xc 'kubectl patch perconaservermongodbbackups.psmdb.percona.com -n $0 $1 --type=merge -p "{\"metadata\":{\"finalizers\":[]}}"'
error: the server doesn't have a resource type "perconaservermongodbbackups"
+ kubectl patch perconaservermongodbbackups.psmdb.percona.com -n sh --type=merge -p '{"metadata":{"finalizers":[]}}'
error: the server doesn't have a resource type "perconaservermongodbbackups"
+ :
+ kubectl_bin wait --for=delete crd perconaservermongodbbackups.psmdb.percona.com
++ mktemp
+ local LAST_OUT=/tmp/tmp.mhM05Im4Z8
++ mktemp
+ local LAST_ERR=/tmp/tmp.k4LOHNrGt3
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl wait --for=delete crd perconaservermongodbbackups.psmdb.percona.com
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.mhM05Im4Z8
+ cat /tmp/tmp.k4LOHNrGt3
+ rm /tmp/tmp.mhM05Im4Z8 /tmp/tmp.k4LOHNrGt3
+ return 0
+ for crd_name in $(yq eval '.metadata.name' "${src_dir}/deploy/crd.yaml" | grep -v '\-\-\-')
+ kubectl get perconaservermongodbclustersyncs.psmdb.percona.com --all-namespaces -o wide
+ grep -v NAMESPACE
+ xargs -L 1 sh -xc 'kubectl patch perconaservermongodbclustersyncs.psmdb.percona.com -n $0 $1 --type=merge -p "{\"metadata\":{\"finalizers\":[]}}"'
error: the server doesn't have a resource type "perconaservermongodbclustersyncs"
+ kubectl patch perconaservermongodbclustersyncs.psmdb.percona.com -n sh --type=merge -p '{"metadata":{"finalizers":[]}}'
error: the server doesn't have a resource type "perconaservermongodbclustersyncs"
+ :
+ kubectl_bin wait --for=delete crd perconaservermongodbclustersyncs.psmdb.percona.com
++ mktemp
+ local LAST_OUT=/tmp/tmp.S5T59YdtjZ
++ mktemp
+ local LAST_ERR=/tmp/tmp.ZCcee2oIOi
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl wait --for=delete crd perconaservermongodbclustersyncs.psmdb.percona.com
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.S5T59YdtjZ
+ cat /tmp/tmp.ZCcee2oIOi
+ rm /tmp/tmp.S5T59YdtjZ /tmp/tmp.ZCcee2oIOi
+ return 0
+ for crd_name in $(yq eval '.metadata.name' "${src_dir}/deploy/crd.yaml" | grep -v '\-\-\-')
+ kubectl get perconaservermongodbrestores.psmdb.percona.com --all-namespaces -o wide
+ grep -v NAMESPACE
+ xargs -L 1 sh -xc 'kubectl patch perconaservermongodbrestores.psmdb.percona.com -n $0 $1 --type=merge -p "{\"metadata\":{\"finalizers\":[]}}"'
error: the server doesn't have a resource type "perconaservermongodbrestores"
+ kubectl patch perconaservermongodbrestores.psmdb.percona.com -n sh --type=merge -p '{"metadata":{"finalizers":[]}}'
error: the server doesn't have a resource type "perconaservermongodbrestores"
+ :
+ kubectl_bin wait --for=delete crd perconaservermongodbrestores.psmdb.percona.com
++ mktemp
+ local LAST_OUT=/tmp/tmp.XHlqVctIKc
++ mktemp
+ local LAST_ERR=/tmp/tmp.t3wpAGkxZI
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl wait --for=delete crd perconaservermongodbrestores.psmdb.percona.com
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.XHlqVctIKc
+ cat /tmp/tmp.t3wpAGkxZI
+ rm /tmp/tmp.XHlqVctIKc /tmp/tmp.t3wpAGkxZI
+ return 0
+ for crd_name in $(yq eval '.metadata.name' "${src_dir}/deploy/crd.yaml" | grep -v '\-\-\-')
+ kubectl get perconaservermongodbs.psmdb.percona.com --all-namespaces -o wide
+ grep -v NAMESPACE
+ xargs -L 1 sh -xc 'kubectl patch perconaservermongodbs.psmdb.percona.com -n $0 $1 --type=merge -p "{\"metadata\":{\"finalizers\":[]}}"'
error: the server doesn't have a resource type "perconaservermongodbs"
+ kubectl patch perconaservermongodbs.psmdb.percona.com -n sh --type=merge -p '{"metadata":{"finalizers":[]}}'
error: the server doesn't have a resource type "perconaservermongodbs"
+ :
+ kubectl_bin wait --for=delete crd perconaservermongodbs.psmdb.percona.com
++ mktemp
+ local LAST_OUT=/tmp/tmp.cFgnJ5NWbY
++ mktemp
+ local LAST_ERR=/tmp/tmp.URtgw7onD0
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl wait --for=delete crd perconaservermongodbs.psmdb.percona.com
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.cFgnJ5NWbY
+ cat /tmp/tmp.URtgw7onD0
+ rm /tmp/tmp.cFgnJ5NWbY /tmp/tmp.URtgw7onD0
+ return 0
+ local rbac_yaml=rbac.yaml
+ '[' -n psmdb-operator ']'
+ rbac_yaml=cw-rbac.yaml
+ kubectl_bin delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2412/deploy/cw-rbac.yaml --ignore-not-found
++ mktemp
+ local LAST_OUT=/tmp/tmp.RDvnMyglvd
++ mktemp
+ local LAST_ERR=/tmp/tmp.NUPFA31Py5
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2412/deploy/cw-rbac.yaml --ignore-not-found
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.RDvnMyglvd
clusterrole.rbac.authorization.k8s.io "percona-server-mongodb-operator" deleted
clusterrolebinding.rbac.authorization.k8s.io "service-account-percona-server-mongodb-operator" deleted
+ cat /tmp/tmp.NUPFA31Py5
+ rm /tmp/tmp.RDvnMyglvd /tmp/tmp.NUPFA31Py5
+ return 0
+ destroy_cert_manager
+ [[ gke == rancher ]]
+ kubectl_bin delete -f https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml
++ mktemp
+ local LAST_OUT=/tmp/tmp.zhdmZpPjLZ
++ mktemp
+ local LAST_ERR=/tmp/tmp.0ChJl3Mxva
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl delete -f https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.zhdmZpPjLZ
namespace "cert-manager" deleted
customresourcedefinition.apiextensions.k8s.io "challenges.acme.cert-manager.io" deleted
customresourcedefinition.apiextensions.k8s.io "orders.acme.cert-manager.io" deleted
customresourcedefinition.apiextensions.k8s.io "certificaterequests.cert-manager.io" deleted
customresourcedefinition.apiextensions.k8s.io "certificates.cert-manager.io" deleted
customresourcedefinition.apiextensions.k8s.io "clusterissuers.cert-manager.io" deleted
customresourcedefinition.apiextensions.k8s.io "issuers.cert-manager.io" deleted
serviceaccount "cert-manager-cainjector" deleted from cert-manager namespace
serviceaccount "cert-manager" deleted from cert-manager namespace
serviceaccount "cert-manager-webhook" deleted from cert-manager namespace
clusterrole.rbac.authorization.k8s.io "cert-manager-cainjector" deleted
clusterrole.rbac.authorization.k8s.io "cert-manager-controller-issuers" deleted
clusterrole.rbac.authorization.k8s.io "cert-manager-controller-clusterissuers" deleted
clusterrole.rbac.authorization.k8s.io "cert-manager-controller-certificates" deleted
clusterrole.rbac.authorization.k8s.io "cert-manager-controller-orders" deleted
clusterrole.rbac.authorization.k8s.io "cert-manager-controller-challenges" deleted
clusterrole.rbac.authorization.k8s.io "cert-manager-controller-ingress-shim" deleted
clusterrole.rbac.authorization.k8s.io "cert-manager-cluster-view" deleted
clusterrole.rbac.authorization.k8s.io "cert-manager-view" deleted
clusterrole.rbac.authorization.k8s.io "cert-manager-edit" deleted
clusterrole.rbac.authorization.k8s.io "cert-manager-controller-approve:cert-manager-io" deleted
clusterrole.rbac.authorization.k8s.io "cert-manager-controller-certificatesigningrequests" deleted
clusterrole.rbac.authorization.k8s.io "cert-manager-webhook:subjectaccessreviews" deleted
clusterrolebinding.rbac.authorization.k8s.io "cert-manager-cainjector" deleted
clusterrolebinding.rbac.authorization.k8s.io "cert-manager-controller-issuers" deleted
clusterrolebinding.rbac.authorization.k8s.io "cert-manager-controller-clusterissuers" deleted
clusterrolebinding.rbac.authorization.k8s.io "cert-manager-controller-certificates" deleted
clusterrolebinding.rbac.authorization.k8s.io "cert-manager-controller-orders" deleted
clusterrolebinding.rbac.authorization.k8s.io "cert-manager-controller-challenges" deleted
clusterrolebinding.rbac.authorization.k8s.io "cert-manager-controller-ingress-shim" deleted
clusterrolebinding.rbac.authorization.k8s.io "cert-manager-controller-approve:cert-manager-io" deleted
clusterrolebinding.rbac.authorization.k8s.io "cert-manager-controller-certificatesigningrequests" deleted
clusterrolebinding.rbac.authorization.k8s.io "cert-manager-webhook:subjectaccessreviews" deleted
role.rbac.authorization.k8s.io "cert-manager-cainjector:leaderelection" deleted from kube-system namespace
role.rbac.authorization.k8s.io "cert-manager:leaderelection" deleted from kube-system namespace
role.rbac.authorization.k8s.io "cert-manager-tokenrequest" deleted from cert-manager namespace
role.rbac.authorization.k8s.io "cert-manager-webhook:dynamic-serving" deleted from cert-manager namespace
rolebinding.rbac.authorization.k8s.io "cert-manager-cainjector:leaderelection" deleted from kube-system namespace
rolebinding.rbac.authorization.k8s.io "cert-manager:leaderelection" deleted from kube-system namespace
rolebinding.rbac.authorization.k8s.io "cert-manager-tokenrequest" deleted from cert-manager namespace
rolebinding.rbac.authorization.k8s.io "cert-manager-webhook:dynamic-serving" deleted from cert-manager namespace
service "cert-manager-cainjector" deleted from cert-manager namespace
service "cert-manager" deleted from cert-manager namespace
service "cert-manager-webhook" deleted from cert-manager namespace
deployment.apps "cert-manager-cainjector" deleted from cert-manager namespace
deployment.apps "cert-manager" deleted from cert-manager namespace
deployment.apps "cert-manager-webhook" deleted from cert-manager namespace
mutatingwebhookconfiguration.admissionregistration.k8s.io "cert-manager-webhook" deleted
validatingwebhookconfiguration.admissionregistration.k8s.io "cert-manager-webhook" deleted
+ cat /tmp/tmp.0ChJl3Mxva
+ rm /tmp/tmp.zhdmZpPjLZ /tmp/tmp.0ChJl3Mxva
+ return 0
+ [[ -n '' ]]
+ '[' -n psmdb-operator ']'
+ kubectl_bin delete --grace-period=0 --force=true namespace monitoring-pmm3-11763
+ rm -rf /tmp/tmp.5urvO7mwus
+ kubectl_bin delete --grace-period=0 --force=true namespace psmdb-operator
++ mktemp
++ mktemp
+ local LAST_OUT=/tmp/tmp.yQAbK43O3d
++ mktemp
+ desc 'test passed'
+ set +o xtrace


-----------------------------------------------------------------------------------
test passed
-----------------------------------------------------------------------------------

+ local LAST_OUT=/tmp/tmp.ddBoCPn32L
++ mktemp
+ local LAST_ERR=/tmp/tmp.3b0gs638Gd
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ local LAST_ERR=/tmp/tmp.lcwrBsW3cM
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl delete --grace-period=0 --force=true namespace monitoring-pmm3-11763
+ for i in $(seq 0 2)
+ set +e
+ kubectl delete --grace-period=0 --force=true namespace psmdb-operator