Log: /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/e2e-tests/logs/monitoring-pmm3.log Warning: version difference between client (1.36) and server (1.33) exceeds the supported minor version skew of +/-1 Warning: version difference between client (1.36) and server (1.33) exceeds the supported minor version skew of +/-1 Warning: version difference between client (1.36) and server (1.33) exceeds the supported minor version skew of +/-1 + create_infra monitoring-pmm3-6334 + local ns=monitoring-pmm3-6334 + echo monitoring-pmm3-6334 + [[ 0 == 1 ]] + '[' -n psmdb-operator ']' + create_namespace psmdb-operator + local namespace=psmdb-operator + local skip_clean_namespace= + [[ 1 == 1 ]] + [[ -z '' ]] + destroy_chaos_mesh ++ helm list --all-namespaces --filter chaos-mesh ++ tail -n1 ++ sed s/NAMESPACE// ++ awk '-F ' '{print $2}' + local chaos_mesh_ns= + desc 'destroy chaos-mesh' + set +o xtrace ----------------------------------------------------------------------------------- destroy chaos-mesh ----------------------------------------------------------------------------------- + '[' -n '' ']' ++ kubectl get MutatingWebhookConfiguration ++ grep chaos-mesh ++ awk '{print $1}' + timeout 30 kubectl delete MutatingWebhookConfiguration error: resource(s) were provided, but no name was specified + : ++ kubectl get ValidatingWebhookConfiguration ++ grep chaos-mesh ++ awk '{print $1}' + timeout 30 kubectl delete ValidatingWebhookConfiguration error: resource(s) were provided, but no name was specified + : ++ grep validate-auth ++ kubectl get ValidatingWebhookConfiguration ++ awk '{print $1}' + timeout 30 kubectl delete ValidatingWebhookConfiguration error: resource(s) were provided, but no name was specified + : ++ kubectl api-resources ++ grep chaos-mesh ++ awk '{print $1}' ++ kubectl get crd ++ grep chaos-mesh.org ++ awk '{print $1}' + timeout 30 kubectl delete crd error: resource(s) were provided, but no name was specified + : ++ kubectl get clusterrolebinding ++ grep chaos-mesh ++ awk '{print $1}' + timeout 30 kubectl delete clusterrolebinding error: resource(s) were provided, but no name was specified + : ++ kubectl get clusterrole ++ grep chaos-mesh ++ awk '{print $1}' + timeout 30 kubectl delete clusterrole error: resource(s) were provided, but no name was specified + : + desc 'cleaned up all old namespaces' + set +o xtrace ----------------------------------------------------------------------------------- cleaned up all old namespaces ----------------------------------------------------------------------------------- + kubectl_bin get ns + grep -E -v '^kube-|^default|Terminating|psmdb-operator|openshift|^gke-|^gmp-|^NAME' ++ mktemp + awk '{print$1}' + '[' -n '' ']' + desc 'cleaned up old namespaces psmdb-operator' + set +o xtrace ----------------------------------------------------------------------------------- cleaned up old namespaces psmdb-operator ----------------------------------------------------------------------------------- + kubectl_bin delete namespace psmdb-operator --ignore-not-found + xargs kubectl delete ns ++ mktemp + local LAST_OUT=/tmp/tmp.qVzP3thNwh ++ mktemp + local LAST_OUT=/tmp/tmp.wMGgFnvx7E ++ mktemp + local LAST_ERR=/tmp/tmp.cCTI2G8H49 + local exit_status=0 + local timeout=4 + local LAST_ERR=/tmp/tmp.wdiIAVRgzS + local exit_status=0 + local timeout=4 ++ seq 0 2 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl get ns + for i in $(seq 0 2) + set +e + kubectl delete namespace psmdb-operator --ignore-not-found + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.qVzP3thNwh + cat /tmp/tmp.cCTI2G8H49 + rm /tmp/tmp.qVzP3thNwh /tmp/tmp.cCTI2G8H49 + return 0 namespace "demand-backup-incremental-sharded-gcp-s3-11425" deleted + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.wMGgFnvx7E namespace "psmdb-operator" deleted + cat /tmp/tmp.wdiIAVRgzS + rm /tmp/tmp.wMGgFnvx7E /tmp/tmp.wdiIAVRgzS + return 0 + kubectl_bin wait --for=delete namespace psmdb-operator ++ mktemp + local LAST_OUT=/tmp/tmp.PanIxTGdp1 ++ mktemp + local LAST_ERR=/tmp/tmp.uUTgxItUrM + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl wait --for=delete namespace psmdb-operator + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.PanIxTGdp1 + cat /tmp/tmp.uUTgxItUrM + rm /tmp/tmp.PanIxTGdp1 /tmp/tmp.uUTgxItUrM + return 0 + desc 'create namespace psmdb-operator' + set +o xtrace ----------------------------------------------------------------------------------- create namespace psmdb-operator ----------------------------------------------------------------------------------- + kubectl_bin create namespace psmdb-operator ++ mktemp + local LAST_OUT=/tmp/tmp.gg0fExjBJB ++ mktemp + local LAST_ERR=/tmp/tmp.fHbnfntbJo + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl create namespace psmdb-operator + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.gg0fExjBJB namespace/psmdb-operator created + cat /tmp/tmp.fHbnfntbJo + rm /tmp/tmp.gg0fExjBJB /tmp/tmp.fHbnfntbJo + return 0 + set_kube_ctx psmdb-operator + local namespace=psmdb-operator ++ kubectl_bin config current-context +++ mktemp ++ local LAST_OUT=/tmp/tmp.2w35ertGEX +++ mktemp ++ local LAST_ERR=/tmp/tmp.IeD7HEQ8k0 ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl config current-context ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.2w35ertGEX ++ cat /tmp/tmp.IeD7HEQ8k0 ++ rm /tmp/tmp.2w35ertGEX /tmp/tmp.IeD7HEQ8k0 ++ return 0 + kubectl_bin config set-context gke_cloud-dev-112233_us-central1-a_jen-psmdb-2058-fb8cc7187-17-cluster14 --namespace=psmdb-operator ++ mktemp + local LAST_OUT=/tmp/tmp.3i8sm2a2QX ++ mktemp + local LAST_ERR=/tmp/tmp.nog8jo6wzD + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl config set-context gke_cloud-dev-112233_us-central1-a_jen-psmdb-2058-fb8cc7187-17-cluster14 --namespace=psmdb-operator + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.3i8sm2a2QX Context "gke_cloud-dev-112233_us-central1-a_jen-psmdb-2058-fb8cc7187-17-cluster14" modified. + cat /tmp/tmp.nog8jo6wzD + rm /tmp/tmp.3i8sm2a2QX /tmp/tmp.nog8jo6wzD + return 0 + deploy_operator + desc 'start PSMDB operator: docker.io/perconalab/percona-server-mongodb-operator:PR-2058-fb8cc7187' + set +o xtrace ----------------------------------------------------------------------------------- start PSMDB operator: docker.io/perconalab/percona-server-mongodb-operator:PR-2058-fb8cc7187 ----------------------------------------------------------------------------------- + local cr_file + '[' -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/e2e-tests/monitoring-pmm3/conf/crd.yaml ']' + cr_file=/mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/deploy/crd.yaml + kubectl_bin apply --server-side --force-conflicts -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/deploy/crd.yaml ++ mktemp + local LAST_OUT=/tmp/tmp.FCr1QwFckz ++ mktemp + local LAST_ERR=/tmp/tmp.eT5JEQDOSd + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl apply --server-side --force-conflicts -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/deploy/crd.yaml + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.FCr1QwFckz customresourcedefinition.apiextensions.k8s.io/perconaservermongodbbackups.psmdb.percona.com serverside-applied customresourcedefinition.apiextensions.k8s.io/perconaservermongodbrestores.psmdb.percona.com serverside-applied customresourcedefinition.apiextensions.k8s.io/perconaservermongodbs.psmdb.percona.com serverside-applied + cat /tmp/tmp.eT5JEQDOSd + rm /tmp/tmp.FCr1QwFckz /tmp/tmp.eT5JEQDOSd + return 0 + '[' -n psmdb-operator ']' + apply_rbac cw-rbac + local operator_namespace=psmdb-operator + local rbac=cw-rbac + cat /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/deploy/cw-rbac.yaml + sed -e 's^namespace: .*^namespace: psmdb-operator^' + kubectl_bin apply -n psmdb-operator -f - ++ mktemp + local LAST_OUT=/tmp/tmp.LZKQrDCS1F ++ mktemp + local LAST_ERR=/tmp/tmp.61nvGY9YPy + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl apply -n psmdb-operator -f - + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.LZKQrDCS1F clusterrole.rbac.authorization.k8s.io/percona-server-mongodb-operator unchanged serviceaccount/percona-server-mongodb-operator created clusterrolebinding.rbac.authorization.k8s.io/service-account-percona-server-mongodb-operator unchanged + cat /tmp/tmp.61nvGY9YPy + rm /tmp/tmp.LZKQrDCS1F /tmp/tmp.61nvGY9YPy + return 0 + yq eval $'\n\t\t\t(.spec.template.spec.containers[].image = "docker.io/perconalab/percona-server-mongodb-operator:PR-2058-fb8cc7187") |\n\t\t\t((.. | select(.[] == "DISABLE_TELEMETRY")) |= .value="true") |\n\t\t\t((.. | select(.[] == "LOG_LEVEL")) |= .value="DEBUG")' /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/deploy/cw-operator.yaml + kubectl_bin apply -n psmdb-operator -f - ++ mktemp + local LAST_OUT=/tmp/tmp.AnmIv9bBun ++ mktemp + local LAST_ERR=/tmp/tmp.koRw9YvZE7 + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl apply -n psmdb-operator -f - + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.AnmIv9bBun deployment.apps/percona-server-mongodb-operator created + cat /tmp/tmp.koRw9YvZE7 + rm /tmp/tmp.AnmIv9bBun /tmp/tmp.koRw9YvZE7 + return 0 + sleep 20 ++ get_operator_pod ++ kubectl_bin get pods --selector=name=percona-server-mongodb-operator -o 'jsonpath={.items[].metadata.name}' -n psmdb-operator +++ mktemp ++ local LAST_OUT=/tmp/tmp.z1eFqtxuVS +++ mktemp ++ local LAST_ERR=/tmp/tmp.o8bC58ZyI2 ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pods --selector=name=percona-server-mongodb-operator -o 'jsonpath={.items[].metadata.name}' -n psmdb-operator ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.z1eFqtxuVS ++ cat /tmp/tmp.o8bC58ZyI2 ++ rm /tmp/tmp.z1eFqtxuVS /tmp/tmp.o8bC58ZyI2 ++ return 0 + wait_operator_pod percona-server-mongodb-operator-859f4bc6d7-4tjdg + local pod=percona-server-mongodb-operator-859f4bc6d7-4tjdg + set +o xtrace waiting for pod/percona-server-mongodb-operator-859f4bc6d7-4tjdg to be ready.OK + echo 'Print operator info from log' Print operator info from log + grep 'Manager starting up' ++ get_operator_pod ++ kubectl_bin get pods --selector=name=percona-server-mongodb-operator -o 'jsonpath={.items[].metadata.name}' -n psmdb-operator +++ mktemp ++ local LAST_OUT=/tmp/tmp.t2x0hEv1rF +++ mktemp ++ local LAST_ERR=/tmp/tmp.28dlHPJM55 ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pods --selector=name=percona-server-mongodb-operator -o 'jsonpath={.items[].metadata.name}' -n psmdb-operator ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.t2x0hEv1rF ++ cat /tmp/tmp.28dlHPJM55 ++ rm /tmp/tmp.t2x0hEv1rF /tmp/tmp.28dlHPJM55 ++ return 0 + kubectl_bin logs -n psmdb-operator percona-server-mongodb-operator-859f4bc6d7-4tjdg ++ mktemp + local LAST_OUT=/tmp/tmp.KoSCX4AjGN ++ mktemp + local LAST_ERR=/tmp/tmp.Y4GmXeWzm3 + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl logs -n psmdb-operator percona-server-mongodb-operator-859f4bc6d7-4tjdg + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.KoSCX4AjGN + cat /tmp/tmp.Y4GmXeWzm3 + rm /tmp/tmp.KoSCX4AjGN /tmp/tmp.Y4GmXeWzm3 + return 0 2026-06-09T09:14:24.247Z INFO setup Manager starting up {"gitCommit": "fb8cc718776dc8f9e011cd5f23fe48cf7f65143e", "gitBranch": "PR-2058-fb8cc7187", "buildTime": "", "goVersion": "go1.26.4", "os": "linux", "arch": "amd64"} + create_namespace monitoring-pmm3-6334 + local namespace=monitoring-pmm3-6334 + local skip_clean_namespace= + [[ 1 == 1 ]] + [[ -z '' ]] + destroy_chaos_mesh ++ helm list --all-namespaces --filter chaos-mesh ++ tail -n1 ++ awk '-F ' '{print $2}' ++ sed s/NAMESPACE// + local chaos_mesh_ns= + desc 'destroy chaos-mesh' + set +o xtrace ----------------------------------------------------------------------------------- destroy chaos-mesh ----------------------------------------------------------------------------------- + '[' -n '' ']' ++ kubectl get MutatingWebhookConfiguration ++ grep chaos-mesh ++ awk '{print $1}' + timeout 30 kubectl delete MutatingWebhookConfiguration error: resource(s) were provided, but no name was specified + : ++ grep chaos-mesh ++ awk '{print $1}' ++ kubectl get ValidatingWebhookConfiguration + timeout 30 kubectl delete ValidatingWebhookConfiguration error: resource(s) were provided, but no name was specified + : ++ kubectl get ValidatingWebhookConfiguration ++ grep validate-auth ++ awk '{print $1}' + timeout 30 kubectl delete ValidatingWebhookConfiguration error: resource(s) were provided, but no name was specified + : ++ kubectl api-resources ++ grep chaos-mesh ++ awk '{print $1}' ++ kubectl get crd ++ grep chaos-mesh.org ++ awk '{print $1}' + timeout 30 kubectl delete crd error: resource(s) were provided, but no name was specified + : ++ kubectl get clusterrolebinding ++ awk '{print $1}' ++ grep chaos-mesh + timeout 30 kubectl delete clusterrolebinding error: resource(s) were provided, but no name was specified + : ++ kubectl get clusterrole ++ awk '{print $1}' ++ grep chaos-mesh + timeout 30 kubectl delete clusterrole error: resource(s) were provided, but no name was specified + : + desc 'cleaned up all old namespaces' + set +o xtrace ----------------------------------------------------------------------------------- cleaned up all old namespaces ----------------------------------------------------------------------------------- + kubectl_bin get ns + awk '{print$1}' + grep -E -v '^kube-|^default|Terminating|psmdb-operator|openshift|^gke-|^gmp-|^NAME' + '[' -n '' ']' ++ mktemp + desc 'cleaned up old namespaces monitoring-pmm3-6334' + set +o xtrace ----------------------------------------------------------------------------------- cleaned up old namespaces monitoring-pmm3-6334 ----------------------------------------------------------------------------------- + kubectl_bin delete namespace monitoring-pmm3-6334 --ignore-not-found + xargs kubectl delete ns + local LAST_OUT=/tmp/tmp.mUk0aTEmac ++ mktemp ++ mktemp + local LAST_ERR=/tmp/tmp.pBuh3F8kkn + local exit_status=0 + local timeout=4 ++ seq 0 2 + local LAST_OUT=/tmp/tmp.3Ex9XgOq1W ++ mktemp + for i in $(seq 0 2) + set +e + kubectl get ns + local LAST_ERR=/tmp/tmp.H0Y5JkCm7P + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl delete namespace monitoring-pmm3-6334 --ignore-not-found + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.mUk0aTEmac + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.3Ex9XgOq1W + cat /tmp/tmp.pBuh3F8kkn + cat /tmp/tmp.H0Y5JkCm7P + rm /tmp/tmp.mUk0aTEmac /tmp/tmp.pBuh3F8kkn + rm /tmp/tmp.3Ex9XgOq1W /tmp/tmp.H0Y5JkCm7P + return 0 + return 0 + kubectl_bin wait --for=delete namespace monitoring-pmm3-6334 ++ mktemp + local LAST_OUT=/tmp/tmp.xJ35Nnht5c ++ mktemp + local LAST_ERR=/tmp/tmp.7rUCPmApLW + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl wait --for=delete namespace monitoring-pmm3-6334 error: resource(s) were provided, but no name was specified + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.xJ35Nnht5c + cat /tmp/tmp.7rUCPmApLW + rm /tmp/tmp.xJ35Nnht5c /tmp/tmp.7rUCPmApLW + return 0 + desc 'create namespace monitoring-pmm3-6334' + set +o xtrace ----------------------------------------------------------------------------------- create namespace monitoring-pmm3-6334 ----------------------------------------------------------------------------------- + kubectl_bin create namespace monitoring-pmm3-6334 ++ mktemp + local LAST_OUT=/tmp/tmp.JHOx1FUOh2 ++ mktemp + local LAST_ERR=/tmp/tmp.MBZ2fsUUR8 + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl create namespace monitoring-pmm3-6334 + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.JHOx1FUOh2 namespace/monitoring-pmm3-6334 created + cat /tmp/tmp.MBZ2fsUUR8 + rm /tmp/tmp.JHOx1FUOh2 /tmp/tmp.MBZ2fsUUR8 + return 0 + set_kube_ctx monitoring-pmm3-6334 + local namespace=monitoring-pmm3-6334 ++ kubectl_bin config current-context +++ mktemp ++ local LAST_OUT=/tmp/tmp.K6xy3jpJUO +++ mktemp ++ local LAST_ERR=/tmp/tmp.wqGYWbaiVp ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl config current-context ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.K6xy3jpJUO ++ cat /tmp/tmp.wqGYWbaiVp ++ rm /tmp/tmp.K6xy3jpJUO /tmp/tmp.wqGYWbaiVp ++ return 0 + kubectl_bin config set-context gke_cloud-dev-112233_us-central1-a_jen-psmdb-2058-fb8cc7187-17-cluster14 --namespace=monitoring-pmm3-6334 ++ mktemp + local LAST_OUT=/tmp/tmp.GOd5KigytJ ++ mktemp + local LAST_ERR=/tmp/tmp.3cK7xx5bGk + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl config set-context gke_cloud-dev-112233_us-central1-a_jen-psmdb-2058-fb8cc7187-17-cluster14 --namespace=monitoring-pmm3-6334 + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.GOd5KigytJ Context "gke_cloud-dev-112233_us-central1-a_jen-psmdb-2058-fb8cc7187-17-cluster14" modified. + cat /tmp/tmp.3cK7xx5bGk + rm /tmp/tmp.GOd5KigytJ /tmp/tmp.3cK7xx5bGk + return 0 + deploy_cert_manager + desc 'deploy cert manager' + set +o xtrace ----------------------------------------------------------------------------------- deploy cert manager ----------------------------------------------------------------------------------- + kubectl_bin create namespace cert-manager ++ mktemp + local LAST_OUT=/tmp/tmp.9ffkxmevY6 ++ mktemp + local LAST_ERR=/tmp/tmp.Zwsv4zzSyS + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl create namespace cert-manager + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.9ffkxmevY6 namespace/cert-manager created + cat /tmp/tmp.Zwsv4zzSyS + rm /tmp/tmp.9ffkxmevY6 /tmp/tmp.Zwsv4zzSyS + return 0 + kubectl_bin label namespace cert-manager certmanager.k8s.io/disable-validation=true ++ mktemp + local LAST_OUT=/tmp/tmp.qB611Gu0K9 ++ mktemp + local LAST_ERR=/tmp/tmp.WLs0180ZpS + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl label namespace cert-manager certmanager.k8s.io/disable-validation=true + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.qB611Gu0K9 namespace/cert-manager labeled + cat /tmp/tmp.WLs0180ZpS + rm /tmp/tmp.qB611Gu0K9 /tmp/tmp.WLs0180ZpS + return 0 + kubectl_bin apply -f https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml --validate=false ++ mktemp + local LAST_OUT=/tmp/tmp.jscdbpPQlG ++ mktemp + local LAST_ERR=/tmp/tmp.585M5eZGCx + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl apply -f https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml --validate=false + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.jscdbpPQlG namespace/cert-manager configured customresourcedefinition.apiextensions.k8s.io/challenges.acme.cert-manager.io created customresourcedefinition.apiextensions.k8s.io/orders.acme.cert-manager.io created customresourcedefinition.apiextensions.k8s.io/certificaterequests.cert-manager.io created customresourcedefinition.apiextensions.k8s.io/certificates.cert-manager.io created customresourcedefinition.apiextensions.k8s.io/clusterissuers.cert-manager.io created customresourcedefinition.apiextensions.k8s.io/issuers.cert-manager.io created serviceaccount/cert-manager-cainjector created serviceaccount/cert-manager created serviceaccount/cert-manager-webhook created clusterrole.rbac.authorization.k8s.io/cert-manager-cainjector created clusterrole.rbac.authorization.k8s.io/cert-manager-controller-issuers created clusterrole.rbac.authorization.k8s.io/cert-manager-controller-clusterissuers created clusterrole.rbac.authorization.k8s.io/cert-manager-controller-certificates created clusterrole.rbac.authorization.k8s.io/cert-manager-controller-orders created clusterrole.rbac.authorization.k8s.io/cert-manager-controller-challenges created clusterrole.rbac.authorization.k8s.io/cert-manager-controller-ingress-shim created clusterrole.rbac.authorization.k8s.io/cert-manager-cluster-view created clusterrole.rbac.authorization.k8s.io/cert-manager-view created clusterrole.rbac.authorization.k8s.io/cert-manager-edit created clusterrole.rbac.authorization.k8s.io/cert-manager-controller-approve:cert-manager-io created clusterrole.rbac.authorization.k8s.io/cert-manager-controller-certificatesigningrequests created clusterrole.rbac.authorization.k8s.io/cert-manager-webhook:subjectaccessreviews created clusterrolebinding.rbac.authorization.k8s.io/cert-manager-cainjector created clusterrolebinding.rbac.authorization.k8s.io/cert-manager-controller-issuers created clusterrolebinding.rbac.authorization.k8s.io/cert-manager-controller-clusterissuers created clusterrolebinding.rbac.authorization.k8s.io/cert-manager-controller-certificates created clusterrolebinding.rbac.authorization.k8s.io/cert-manager-controller-orders created clusterrolebinding.rbac.authorization.k8s.io/cert-manager-controller-challenges created clusterrolebinding.rbac.authorization.k8s.io/cert-manager-controller-ingress-shim created clusterrolebinding.rbac.authorization.k8s.io/cert-manager-controller-approve:cert-manager-io created clusterrolebinding.rbac.authorization.k8s.io/cert-manager-controller-certificatesigningrequests created clusterrolebinding.rbac.authorization.k8s.io/cert-manager-webhook:subjectaccessreviews created role.rbac.authorization.k8s.io/cert-manager-cainjector:leaderelection created role.rbac.authorization.k8s.io/cert-manager:leaderelection created role.rbac.authorization.k8s.io/cert-manager-tokenrequest created role.rbac.authorization.k8s.io/cert-manager-webhook:dynamic-serving created rolebinding.rbac.authorization.k8s.io/cert-manager-cainjector:leaderelection created rolebinding.rbac.authorization.k8s.io/cert-manager:leaderelection created rolebinding.rbac.authorization.k8s.io/cert-manager-tokenrequest created rolebinding.rbac.authorization.k8s.io/cert-manager-webhook:dynamic-serving created service/cert-manager-cainjector created service/cert-manager created service/cert-manager-webhook created deployment.apps/cert-manager-cainjector created deployment.apps/cert-manager created deployment.apps/cert-manager-webhook created mutatingwebhookconfiguration.admissionregistration.k8s.io/cert-manager-webhook created validatingwebhookconfiguration.admissionregistration.k8s.io/cert-manager-webhook created + cat /tmp/tmp.585M5eZGCx Warning: resource namespaces/cert-manager is missing the kubectl.kubernetes.io/last-applied-configuration annotation which is required by kubectl apply. kubectl apply should only be used on resources created declaratively by either kubectl create --save-config or kubectl apply. The missing annotation will be patched automatically. + rm /tmp/tmp.jscdbpPQlG /tmp/tmp.585M5eZGCx + return 0 + kubectl_bin -n cert-manager wait pod -l app.kubernetes.io/instance=cert-manager --for=condition=ready ++ mktemp + local LAST_OUT=/tmp/tmp.cfrMv1mdXu ++ mktemp + local LAST_ERR=/tmp/tmp.S1Cfrv3b5V + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl -n cert-manager wait pod -l app.kubernetes.io/instance=cert-manager --for=condition=ready + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.cfrMv1mdXu pod/cert-manager-559d798845-q89sf condition met pod/cert-manager-cainjector-64958d9c7c-zkcqb condition met pod/cert-manager-webhook-7fb6f99b56-5g5dt condition met + cat /tmp/tmp.S1Cfrv3b5V E0609 09:15:22.745409 694520 reflector.go:227] "Failed to watch" err="Get \"https://34.123.116.168/api/v1/namespaces/cert-manager/pods?allowWatchBookmarks=true&fieldSelector=metadata.name%3Dcert-manager-cainjector-64958d9c7c-zkcqb&resourceVersion=1780996522361807022&timeoutSeconds=410&watch=true\": context canceled" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" type="*unstructured.Unstructured" + rm /tmp/tmp.cfrMv1mdXu /tmp/tmp.S1Cfrv3b5V + return 0 + sleep 120 + desc 'install PMM Server' + set +o xtrace ----------------------------------------------------------------------------------- install PMM Server ----------------------------------------------------------------------------------- + deploy_pmm3_server + helm uninstall monitoring Error: uninstall: Release not loaded: monitoring: release: not found + : + helm repo remove percona Error: no repo named "percona" found + : + kubectl delete clusterrole monitoring --ignore-not-found + kubectl delete clusterrolebinding monitoring --ignore-not-found + helm repo add percona https://percona.github.io/percona-helm-charts/ "percona" has been added to your repositories + helm repo update Hang tight while we grab the latest from your chart repositories... ...Successfully got an update from the "minio" chart repository ...Successfully got an update from the "percona" chart repository ...Successfully got an update from the "hashicorp" chart repository ...Successfully got an update from the "stable" chart repository Update Complete. ⎈Happy Helming!⎈ + [[ -n '' ]] + retry 10 60 helm install monitoring percona/pmm --set fullnameOverride=monitoring-server --set image.tag=3-dev-latest --set image.repository=docker.io/perconalab/pmm-server --set service.type=LoadBalancer --force + local max=10 + local delay=60 + shift 2 + local n=1 + helm install monitoring percona/pmm --set fullnameOverride=monitoring-server --set image.tag=3-dev-latest --set image.repository=docker.io/perconalab/pmm-server --set service.type=LoadBalancer --force NAME: monitoring LAST DEPLOYED: Tue Jun 9 09:17:35 2026 NAMESPACE: monitoring-pmm3-6334 STATUS: deployed REVISION: 1 TEST SUITE: None NOTES: Percona Monitoring and Management (PMM) An open source database monitoring, observability and management tool Check more info here: https://docs.percona.com/percona-monitoring-and-management/index.html Get the application URL: NOTE: It may take a few minutes for the LoadBalancer IP to be available. You can watch the status of by running 'kubectl get --namespace monitoring-pmm3-6334 svc -w monitoring-service' export SERVICE_IP=$(kubectl get svc --namespace monitoring-pmm3-6334 monitoring-service -o jsonpath="{.status.loadBalancer.ingress[0].ip}") echo https://$SERVICE_IP: Get password for the "admin" user: export ADMIN_PASS=$(kubectl get secret pmm-secret --namespace monitoring-pmm3-6334 -o jsonpath='{.data.PMM_ADMIN_PASSWORD}' | base64 --decode) echo $ADMIN_PASS + sleep 20 + kubectl_bin exec monitoring-server-0 -- bash -c 'ls -l /proc/*/exe 2>/dev/null| grep postgres >/dev/null' ++ mktemp + local LAST_OUT=/tmp/tmp.7CUa5yVdbS ++ mktemp + local LAST_ERR=/tmp/tmp.bKNuOBVYwY + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl exec monitoring-server-0 -- bash -c 'ls -l /proc/*/exe 2>/dev/null| grep postgres >/dev/null' + exit_status=1 + set -e + '[' 1 '!=' 0 -a -n 1 ']' + cat /tmp/tmp.7CUa5yVdbS + cat /tmp/tmp.bKNuOBVYwY error: Internal error occurred: unable to upgrade connection: container not found ("pmm") + sleep 0 + for i in $(seq 0 2) + set +e + kubectl exec monitoring-server-0 -- bash -c 'ls -l /proc/*/exe 2>/dev/null| grep postgres >/dev/null' + exit_status=1 + set -e + '[' 1 '!=' 0 -a -n 1 ']' + cat /tmp/tmp.7CUa5yVdbS + cat /tmp/tmp.bKNuOBVYwY error: Internal error occurred: unable to upgrade connection: container not found ("pmm") + sleep 4 + for i in $(seq 0 2) + set +e + kubectl exec monitoring-server-0 -- bash -c 'ls -l /proc/*/exe 2>/dev/null| grep postgres >/dev/null' + exit_status=1 + set -e + '[' 1 '!=' 0 -a -n 1 ']' + cat /tmp/tmp.7CUa5yVdbS + cat /tmp/tmp.bKNuOBVYwY error: Internal error occurred: unable to upgrade connection: container not found ("pmm") + sleep 8 + cat /tmp/tmp.7CUa5yVdbS + cat /tmp/tmp.bKNuOBVYwY error: Internal error occurred: unable to upgrade connection: container not found ("pmm") + rm /tmp/tmp.7CUa5yVdbS /tmp/tmp.bKNuOBVYwY + return 1 + echo 'Retry 0' Retry 0 + sleep 5 + let retry+=1 + '[' 1 -ge 20 ']' + kubectl_bin exec monitoring-server-0 -- bash -c 'ls -l /proc/*/exe 2>/dev/null| grep postgres >/dev/null' ++ mktemp + local LAST_OUT=/tmp/tmp.WrwOYoBP3f ++ mktemp + local LAST_ERR=/tmp/tmp.5jpVP0fgDB + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl exec monitoring-server-0 -- bash -c 'ls -l /proc/*/exe 2>/dev/null| grep postgres >/dev/null' + exit_status=1 + set -e + '[' 1 '!=' 0 -a -n 1 ']' + cat /tmp/tmp.WrwOYoBP3f + cat /tmp/tmp.5jpVP0fgDB error: Internal error occurred: unable to upgrade connection: container not found ("pmm") + sleep 0 + for i in $(seq 0 2) + set +e + kubectl exec monitoring-server-0 -- bash -c 'ls -l /proc/*/exe 2>/dev/null| grep postgres >/dev/null' + exit_status=1 + set -e + '[' 1 '!=' 0 -a -n 1 ']' + cat /tmp/tmp.WrwOYoBP3f + cat /tmp/tmp.5jpVP0fgDB error: Internal error occurred: unable to upgrade connection: container not found ("pmm") + sleep 4 + for i in $(seq 0 2) + set +e + kubectl exec monitoring-server-0 -- bash -c 'ls -l /proc/*/exe 2>/dev/null| grep postgres >/dev/null' + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.WrwOYoBP3f + cat /tmp/tmp.5jpVP0fgDB + rm /tmp/tmp.WrwOYoBP3f /tmp/tmp.5jpVP0fgDB + return 0 + cluster=monitoring-pmm3 + desc 'create secrets and start client' + set +o xtrace ----------------------------------------------------------------------------------- create secrets and start client ----------------------------------------------------------------------------------- + kubectl_bin apply -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/e2e-tests/conf/secrets.yml -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/e2e-tests/monitoring-pmm3/conf/secrets.yml ++ mktemp + local LAST_OUT=/tmp/tmp.0wKXfM1fyG ++ mktemp + local LAST_ERR=/tmp/tmp.wtMAami7at + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl apply -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/e2e-tests/conf/secrets.yml -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/e2e-tests/monitoring-pmm3/conf/secrets.yml + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.0wKXfM1fyG secret/some-users created secret/some-users unchanged + cat /tmp/tmp.wtMAami7at + rm /tmp/tmp.0wKXfM1fyG /tmp/tmp.wtMAami7at + return 0 + yq '.spec.template.spec.volumes[0].secret.secretName="monitoring-pmm3-ssl"' /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/e2e-tests/conf/client_with_tls.yml + kubectl_bin apply -f - ++ mktemp + local LAST_OUT=/tmp/tmp.DIOPMlrIdK ++ mktemp + local LAST_ERR=/tmp/tmp.MPVbggOkdq + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.DIOPMlrIdK deployment.apps/psmdb-client created + cat /tmp/tmp.MPVbggOkdq + rm /tmp/tmp.DIOPMlrIdK /tmp/tmp.MPVbggOkdq + return 0 + sleep 90 + desc 'create first PSMDB cluster monitoring-pmm3' + set +o xtrace ----------------------------------------------------------------------------------- create first PSMDB cluster monitoring-pmm3 ----------------------------------------------------------------------------------- + custom_cluster_name=super-custom + yq eval '(.spec | select(.image == null)).image = "docker.io/perconalab/percona-server-mongodb-operator:main-mongod8.0"' /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/e2e-tests/monitoring-pmm3/conf/monitoring-pmm3-rs0.yml + yq eval '(.spec | select(has("pmm"))).pmm.image = "docker.io/perconalab/pmm-client:3-dev-latest"' - + yq eval '(.spec | select(has("pmm"))).pmm.customClusterName = "super-custom"' - + yq eval '(.spec | select(has("initImage"))).initImage = "docker.io/perconalab/percona-server-mongodb-operator:PR-2058-fb8cc7187"' - + yq eval '(.spec | select(has("backup"))).backup.image = "docker.io/perconalab/percona-server-mongodb-operator:main-backup"' - + yq eval '.spec.upgradeOptions.apply = "Never"' - + kubectl_bin apply -f - ++ mktemp + local LAST_OUT=/tmp/tmp.3ZwQ9rWCao ++ mktemp + local LAST_ERR=/tmp/tmp.V2QoLwkDEU + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.3ZwQ9rWCao perconaservermongodb.psmdb.percona.com/monitoring-pmm3 created + cat /tmp/tmp.V2QoLwkDEU + rm /tmp/tmp.3ZwQ9rWCao /tmp/tmp.V2QoLwkDEU + return 0 + wait_for_running monitoring-pmm3-rs0 3 + local name=monitoring-pmm3-rs0 + let last_pod=2 + local check_cluster_readyness=true + set_debug + [[ 1 == 1 ]] + set -o xtrace + local rs_name=rs0 + local cluster_name=monitoring-pmm3 ++ seq 0 2 + for i in $(seq 0 $last_pod) + [[ 0 -eq 2 ]] + wait_pod monitoring-pmm3-rs0-0 + local pod=monitoring-pmm3-rs0-0 + set +o xtrace waiting for pod/monitoring-pmm3-rs0-0 to be ready.............OK + for i in $(seq 0 $last_pod) + [[ 1 -eq 2 ]] + wait_pod monitoring-pmm3-rs0-1 + local pod=monitoring-pmm3-rs0-1 + set +o xtrace waiting for pod/monitoring-pmm3-rs0-1 to be ready.............OK + for i in $(seq 0 $last_pod) + [[ 2 -eq 2 ]] ++ kubectl_bin get psmdb monitoring-pmm3 -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].arbiter.enabled}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.olMHs6lPUv +++ mktemp ++ local LAST_ERR=/tmp/tmp.2HuDhYaL3W ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb monitoring-pmm3 -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].arbiter.enabled}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.olMHs6lPUv ++ cat /tmp/tmp.2HuDhYaL3W ++ rm /tmp/tmp.olMHs6lPUv /tmp/tmp.2HuDhYaL3W ++ return 0 + [[ '' == true ]] + wait_pod monitoring-pmm3-rs0-2 + local pod=monitoring-pmm3-rs0-2 + set +o xtrace waiting for pod/monitoring-pmm3-rs0-2 to be ready................OK ++ kubectl_bin get psmdb monitoring-pmm3 -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].nonvoting.enabled}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.To8obMeCUG +++ mktemp ++ local LAST_ERR=/tmp/tmp.s79NsZbnFk ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb monitoring-pmm3 -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].nonvoting.enabled}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.To8obMeCUG ++ cat /tmp/tmp.s79NsZbnFk ++ rm /tmp/tmp.To8obMeCUG /tmp/tmp.s79NsZbnFk ++ return 0 + [[ '' == true ]] ++ kubectl_bin get psmdb monitoring-pmm3 -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].hidden.enabled}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.4YzERMBPvt +++ mktemp ++ local LAST_ERR=/tmp/tmp.qw9NdwcwGB ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb monitoring-pmm3 -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].hidden.enabled}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.4YzERMBPvt ++ cat /tmp/tmp.qw9NdwcwGB ++ rm /tmp/tmp.4YzERMBPvt /tmp/tmp.qw9NdwcwGB ++ return 0 + [[ '' == true ]] + sleep 10 + [[ true == true ]] + set +x Waiting for cluster readyness.................... + desc 'check if pmm-client container is not enabled' + set +o xtrace ----------------------------------------------------------------------------------- check if pmm-client container is not enabled ----------------------------------------------------------------------------------- + compare_kubectl statefulset/monitoring-pmm3-rs0 -no-pmm + local resource=statefulset/monitoring-pmm3-rs0 + local postfix=-no-pmm + local skip_generation_check= + local expected_result=/mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/e2e-tests/monitoring-pmm3/compare/statefulset_monitoring-pmm3-rs0-no-pmm.yml + local new_result=/tmp/tmp.5EJEnDygsZ/statefulset_monitoring-pmm3-rs0.yml + '[' -n '' -a -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/e2e-tests/monitoring-pmm3/compare/statefulset_monitoring-pmm3-rs0-no-pmm-oc.yml ']' + kubectl_bin get -o yaml statefulset/monitoring-pmm3-rs0 + yq eval $'\n\t\t\tdel(.metadata.ownerReferences[].apiVersion) |\n\t\t\tdel(.metadata.managedFields) |\n\t\t\tdel(.. | select(has("creationTimestamp")).creationTimestamp) |\n\t\t\tdel(.. | select(has("namespace")).namespace) |\n\t\t\tdel(.. | select(has("uid")).uid) |\n\t\t\tdel(.metadata.resourceVersion) |\n\t\t\tdel(.spec.template.spec.containers[].env[] | select(.name == "NAMESPACE")) |\n\t\t\tdel(.metadata.selfLink) |\n\t\t\tdel(.metadata.annotations."cloud.google.com/neg") |\n\t\t\tdel(.metadata.annotations."kubectl.kubernetes.io/last-applied-configuration") |\n\t\t\tdel(.. | select(has("image")).image) |\n\t\t\tdel(.. | select(has("clusterIP")).clusterIP) |\n\t\t\tdel(.. | select(has("clusterIPs")).clusterIPs) |\n\t\t\tdel(.. | select(has("dataSource")).dataSource) |\n\t\t\tdel(.. | select(has("procMount")).procMount) |\n\t\t\tdel(.. | select(has("storageClassName")).storageClassName) |\n\t\t\tdel(.. | select(has("finalizers")).finalizers) |\n\t\t\tdel(.. | select(has("kubernetes.io/pvc-protection"))."kubernetes.io/pvc-protection") |\n\t\t\tdel(.. | select(has("volumeName")).volumeName) |\n\t\t\tdel(.. | select(has("volume.beta.kubernetes.io/storage-provisioner"))."volume.beta.kubernetes.io/storage-provisioner") |\n\t\t\tdel(.. | select(has("volume.kubernetes.io/storage-provisioner"))."volume.kubernetes.io/storage-provisioner") |\n\t\t\tdel(.spec.volumeMode) |\n\t\t\tdel(.. | select(has("volume.kubernetes.io/selected-node"))."volume.kubernetes.io/selected-node") |\n\t\t\tdel(.. | select(has("percona.com/last-config-hash"))."percona.com/last-config-hash") |\n\t\t\tdel(.. | select(has("percona.com/configuration-hash"))."percona.com/configuration-hash") |\n\t\t\tdel(.. | select(has("percona.com/ssl-hash"))."percona.com/ssl-hash") |\n\t\t\tdel(.. | select(has("percona.com/ssl-internal-hash"))."percona.com/ssl-internal-hash") |\n\t\t\tdel(.spec.volumeClaimTemplates[].spec.volumeMode | select(. == "Filesystem")) |\n\t\t\tdel(.. | select(has("healthCheckNodePort")).healthCheckNodePort) |\n\t\t\tdel(.. | select(has("nodePort")).nodePort) |\n\t\t\tdel(.status) |\n\t\t\t(.. | select(tag == "!!str")) |= sub("monitoring-pmm3-6334", "NAME_SPACE") |\n\t\t\tdel(.spec.volumeClaimTemplates[].apiVersion) |\n\t\t\tdel(.spec.volumeClaimTemplates[].kind) |\n\t\t\tdel(.spec.ipFamilies) |\n\t\t\tdel(.spec.ipFamilyPolicy) |\n\t\t\t(.. | select(. == "extensions/v1beta1")) = "apps/v1" |\n\t\t\t(.. | select(. == "batch/v1beta1")) = "batch/v1" ' - ++ mktemp + local LAST_OUT=/tmp/tmp.ytuwMsafcB ++ mktemp + local LAST_ERR=/tmp/tmp.WZJeO5pKPC + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl get -o yaml statefulset/monitoring-pmm3-rs0 + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.ytuwMsafcB + cat /tmp/tmp.WZJeO5pKPC + rm /tmp/tmp.ytuwMsafcB /tmp/tmp.WZJeO5pKPC + return 0 + yq -i eval 'del(.spec.persistentVolumeClaimRetentionPolicy)' /tmp/tmp.5EJEnDygsZ/statefulset_monitoring-pmm3-rs0.yml + version_gt 1.22 ++ echo '1.33 >= 1.22' ++ bc -l + '[' 1 -eq 1 ']' + return 0 + yq -i eval 'del(.spec.internalTrafficPolicy)' /tmp/tmp.5EJEnDygsZ/statefulset_monitoring-pmm3-rs0.yml + yq -i eval 'del(.spec.allocateLoadBalancerNodePorts)' /tmp/tmp.5EJEnDygsZ/statefulset_monitoring-pmm3-rs0.yml + [[ /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/e2e-tests/monitoring-pmm3/compare/statefulset_monitoring-pmm3-rs0-no-pmm.yml == */cronjob* ]] + '[' -n '' ']' + [[ 0 -eq 0 ]] + diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/e2e-tests/monitoring-pmm3/compare/statefulset_monitoring-pmm3-rs0-no-pmm.yml /tmp/tmp.5EJEnDygsZ/statefulset_monitoring-pmm3-rs0.yml + log 'compare_kubectl: statefulset/monitoring-pmm3-rs0 OK' + set +o xtrace [2026-06-09T09:22:35+0000] compare_kubectl: statefulset/monitoring-pmm3-rs0 OK + sleep 10 + custom_port=27019 + run_mongos 'db.createUser({user:"myApp",pwd:"myPass",roles:[{db:"myApp",role:"readWrite"}]})' userAdmin:userAdmin123456@monitoring-pmm3-mongos.monitoring-pmm3-6334 '' '' '--tlsCertificateKeyFile /tmp/tls.pem --tlsCAFile /etc/mongodb-ssl/ca.crt --tls' 27019 + local 'command=db.createUser({user:"myApp",pwd:"myPass",roles:[{db:"myApp",role:"readWrite"}]})' + local uri=userAdmin:userAdmin123456@monitoring-pmm3-mongos.monitoring-pmm3-6334 + local driver=mongodb + local suffix=.svc.cluster.local + local 'mongo_flag=--tlsCertificateKeyFile /tmp/tls.pem --tlsCAFile /etc/mongodb-ssl/ca.crt --tls' + local port=27019 + local mongo_bin=mongo ++ echo .svc.cluster.local ++ awk -F: '{print $2}' + suffix_port= + [[ -z '' ]] + suffix=.svc.cluster.local:27019 ++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.IBGQICs45q +++ mktemp ++ local LAST_ERR=/tmp/tmp.319dYWEfZw ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.IBGQICs45q ++ cat /tmp/tmp.319dYWEfZw ++ rm /tmp/tmp.IBGQICs45q /tmp/tmp.319dYWEfZw ++ return 0 + local client_container=psmdb-client-7c8f77fb65-gkcpv + kubectl_bin exec psmdb-client-7c8f77fb65-gkcpv -- bash -c 'printf '\''db.createUser({user:"myApp",pwd:"myPass",roles:[{db:"myApp",role:"readWrite"}]})\n'\'' | mongo mongodb://userAdmin:userAdmin123456@monitoring-pmm3-mongos.monitoring-pmm3-6334.svc.cluster.local:27019/admin --tlsCertificateKeyFile /tmp/tls.pem --tlsCAFile /etc/mongodb-ssl/ca.crt --tls' ++ mktemp + local LAST_OUT=/tmp/tmp.MPbn7Dt9eu ++ mktemp + local LAST_ERR=/tmp/tmp.LVJ0LQSOOC + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl exec psmdb-client-7c8f77fb65-gkcpv -- bash -c 'printf '\''db.createUser({user:"myApp",pwd:"myPass",roles:[{db:"myApp",role:"readWrite"}]})\n'\'' | mongo mongodb://userAdmin:userAdmin123456@monitoring-pmm3-mongos.monitoring-pmm3-6334.svc.cluster.local:27019/admin --tlsCertificateKeyFile /tmp/tls.pem --tlsCAFile /etc/mongodb-ssl/ca.crt --tls' + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.MPbn7Dt9eu Percona Server for MongoDB shell version v4.4.29-28 connecting to: mongodb://monitoring-pmm3-mongos.monitoring-pmm3-6334.svc.cluster.local:27019/admin?compressors=disabled&gssapiServiceName=mongodb {"t":{"$date":"2026-06-09T09:22:48.099Z"},"s":"I", "c":"NETWORK", "id":5490002, "ctx":"thread1","msg":"Started a new thread for the timer service"} Implicit session: session { "id" : UUID("50d45054-4847-485e-818f-43c7f00326b8") } Percona Server for MongoDB server version: v8.0.23-10 WARNING: shell and server versions do not match Successfully added user: { "user" : "myApp", "roles" : [ { "db" : "myApp", "role" : "readWrite" } ] } bye + cat /tmp/tmp.LVJ0LQSOOC + rm /tmp/tmp.MPbn7Dt9eu /tmp/tmp.LVJ0LQSOOC + return 0 + run_mongos 'sh.enableSharding("myApp")' clusterAdmin:clusterAdmin123456@monitoring-pmm3-mongos.monitoring-pmm3-6334 '' '' '--tlsCertificateKeyFile /tmp/tls.pem --tlsCAFile /etc/mongodb-ssl/ca.crt --tls' 27019 + local 'command=sh.enableSharding("myApp")' + local uri=clusterAdmin:clusterAdmin123456@monitoring-pmm3-mongos.monitoring-pmm3-6334 + local driver=mongodb + local suffix=.svc.cluster.local + local 'mongo_flag=--tlsCertificateKeyFile /tmp/tls.pem --tlsCAFile /etc/mongodb-ssl/ca.crt --tls' + local port=27019 + local mongo_bin=mongo ++ echo .svc.cluster.local ++ awk -F: '{print $2}' + suffix_port= + [[ -z '' ]] + suffix=.svc.cluster.local:27019 ++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.5UmRw0xuUS +++ mktemp ++ local LAST_ERR=/tmp/tmp.cKtTP6P2Za ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.5UmRw0xuUS ++ cat /tmp/tmp.cKtTP6P2Za ++ rm /tmp/tmp.5UmRw0xuUS /tmp/tmp.cKtTP6P2Za ++ return 0 + local client_container=psmdb-client-7c8f77fb65-gkcpv + kubectl_bin exec psmdb-client-7c8f77fb65-gkcpv -- bash -c 'printf '\''sh.enableSharding("myApp")\n'\'' | mongo mongodb://clusterAdmin:clusterAdmin123456@monitoring-pmm3-mongos.monitoring-pmm3-6334.svc.cluster.local:27019/admin --tlsCertificateKeyFile /tmp/tls.pem --tlsCAFile /etc/mongodb-ssl/ca.crt --tls' ++ mktemp + local LAST_OUT=/tmp/tmp.toeni6Yt2A ++ mktemp + local LAST_ERR=/tmp/tmp.0IRxU35cjT + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl exec psmdb-client-7c8f77fb65-gkcpv -- bash -c 'printf '\''sh.enableSharding("myApp")\n'\'' | mongo mongodb://clusterAdmin:clusterAdmin123456@monitoring-pmm3-mongos.monitoring-pmm3-6334.svc.cluster.local:27019/admin --tlsCertificateKeyFile /tmp/tls.pem --tlsCAFile /etc/mongodb-ssl/ca.crt --tls' + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.toeni6Yt2A Percona Server for MongoDB shell version v4.4.29-28 connecting to: mongodb://monitoring-pmm3-mongos.monitoring-pmm3-6334.svc.cluster.local:27019/admin?compressors=disabled&gssapiServiceName=mongodb {"t":{"$date":"2026-06-09T09:22:50.866Z"},"s":"I", "c":"NETWORK", "id":5490002, "ctx":"thread1","msg":"Started a new thread for the timer service"} Implicit session: session { "id" : UUID("2ff4fa70-4869-49dd-a1a8-28566615ee21") } Percona Server for MongoDB server version: v8.0.23-10 WARNING: shell and server versions do not match { "ok" : 1, "$clusterTime" : { "clusterTime" : Timestamp(1780996970, 10), "signature" : { "hash" : BinData(0,"i0dglm0q5nwRFuLhd/1wJf/vD7s="), "keyId" : NumberLong("7649323332403200024") } }, "operationTime" : Timestamp(1780996970, 7) } bye + cat /tmp/tmp.0IRxU35cjT + rm /tmp/tmp.toeni6Yt2A /tmp/tmp.0IRxU35cjT + return 0 + insert_data_mongos 100500 myApp '--tlsCertificateKeyFile /tmp/tls.pem --tlsCAFile /etc/mongodb-ssl/ca.crt --tls' 27019 + local data=100500 + local db_name=myApp + local 'flags=--tlsCertificateKeyFile /tmp/tls.pem --tlsCAFile /etc/mongodb-ssl/ca.crt --tls' + local port=27019 + run_mongos 'use myApp\n db.test.insert({ x: 100500 })' myApp:myPass@monitoring-pmm3-mongos.monitoring-pmm3-6334 '' '' '--tlsCertificateKeyFile /tmp/tls.pem --tlsCAFile /etc/mongodb-ssl/ca.crt --tls' 27019 + local 'command=use myApp\n db.test.insert({ x: 100500 })' + local uri=myApp:myPass@monitoring-pmm3-mongos.monitoring-pmm3-6334 + local driver=mongodb + local suffix=.svc.cluster.local + local 'mongo_flag=--tlsCertificateKeyFile /tmp/tls.pem --tlsCAFile /etc/mongodb-ssl/ca.crt --tls' + local port=27019 + local mongo_bin=mongo ++ echo .svc.cluster.local ++ awk -F: '{print $2}' + suffix_port= + [[ -z '' ]] + suffix=.svc.cluster.local:27019 ++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.vW8i9DKakq +++ mktemp ++ local LAST_ERR=/tmp/tmp.7nV1N3hARP ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.vW8i9DKakq ++ cat /tmp/tmp.7nV1N3hARP ++ rm /tmp/tmp.vW8i9DKakq /tmp/tmp.7nV1N3hARP ++ return 0 + local client_container=psmdb-client-7c8f77fb65-gkcpv + kubectl_bin exec psmdb-client-7c8f77fb65-gkcpv -- bash -c 'printf '\''use myApp\n db.test.insert({ x: 100500 })\n'\'' | mongo mongodb://myApp:myPass@monitoring-pmm3-mongos.monitoring-pmm3-6334.svc.cluster.local:27019/admin --tlsCertificateKeyFile /tmp/tls.pem --tlsCAFile /etc/mongodb-ssl/ca.crt --tls' ++ mktemp + local LAST_OUT=/tmp/tmp.Qwg7E5kYAS ++ mktemp + local LAST_ERR=/tmp/tmp.8PNoTnCgmj + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl exec psmdb-client-7c8f77fb65-gkcpv -- bash -c 'printf '\''use myApp\n db.test.insert({ x: 100500 })\n'\'' | mongo mongodb://myApp:myPass@monitoring-pmm3-mongos.monitoring-pmm3-6334.svc.cluster.local:27019/admin --tlsCertificateKeyFile /tmp/tls.pem --tlsCAFile /etc/mongodb-ssl/ca.crt --tls' + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.Qwg7E5kYAS Percona Server for MongoDB shell version v4.4.29-28 connecting to: mongodb://monitoring-pmm3-mongos.monitoring-pmm3-6334.svc.cluster.local:27019/admin?compressors=disabled&gssapiServiceName=mongodb {"t":{"$date":"2026-06-09T09:22:53.668Z"},"s":"I", "c":"NETWORK", "id":5490002, "ctx":"thread1","msg":"Started a new thread for the timer service"} Implicit session: session { "id" : UUID("ca84a875-c12c-4b03-ab1f-a308fcac2aea") } Percona Server for MongoDB server version: v8.0.23-10 WARNING: shell and server versions do not match switched to db myApp WriteResult({ "nInserted" : 1 }) bye + cat /tmp/tmp.8PNoTnCgmj + rm /tmp/tmp.Qwg7E5kYAS /tmp/tmp.8PNoTnCgmj + return 0 + insert_data_mongos 100600 myApp '--tlsCertificateKeyFile /tmp/tls.pem --tlsCAFile /etc/mongodb-ssl/ca.crt --tls' 27019 + local data=100600 + local db_name=myApp + local 'flags=--tlsCertificateKeyFile /tmp/tls.pem --tlsCAFile /etc/mongodb-ssl/ca.crt --tls' + local port=27019 + run_mongos 'use myApp\n db.test.insert({ x: 100600 })' myApp:myPass@monitoring-pmm3-mongos.monitoring-pmm3-6334 '' '' '--tlsCertificateKeyFile /tmp/tls.pem --tlsCAFile /etc/mongodb-ssl/ca.crt --tls' 27019 + local 'command=use myApp\n db.test.insert({ x: 100600 })' + local uri=myApp:myPass@monitoring-pmm3-mongos.monitoring-pmm3-6334 + local driver=mongodb + local suffix=.svc.cluster.local + local 'mongo_flag=--tlsCertificateKeyFile /tmp/tls.pem --tlsCAFile /etc/mongodb-ssl/ca.crt --tls' + local port=27019 + local mongo_bin=mongo ++ echo .svc.cluster.local ++ awk -F: '{print $2}' + suffix_port= + [[ -z '' ]] + suffix=.svc.cluster.local:27019 ++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.J33kJKQzSG +++ mktemp ++ local LAST_ERR=/tmp/tmp.ftmL6i5YEn ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.J33kJKQzSG ++ cat /tmp/tmp.ftmL6i5YEn ++ rm /tmp/tmp.J33kJKQzSG /tmp/tmp.ftmL6i5YEn ++ return 0 + local client_container=psmdb-client-7c8f77fb65-gkcpv + kubectl_bin exec psmdb-client-7c8f77fb65-gkcpv -- bash -c 'printf '\''use myApp\n db.test.insert({ x: 100600 })\n'\'' | mongo mongodb://myApp:myPass@monitoring-pmm3-mongos.monitoring-pmm3-6334.svc.cluster.local:27019/admin --tlsCertificateKeyFile /tmp/tls.pem --tlsCAFile /etc/mongodb-ssl/ca.crt --tls' ++ mktemp + local LAST_OUT=/tmp/tmp.NENtW37Qkd ++ mktemp + local LAST_ERR=/tmp/tmp.o7RPciwxUm + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl exec psmdb-client-7c8f77fb65-gkcpv -- bash -c 'printf '\''use myApp\n db.test.insert({ x: 100600 })\n'\'' | mongo mongodb://myApp:myPass@monitoring-pmm3-mongos.monitoring-pmm3-6334.svc.cluster.local:27019/admin --tlsCertificateKeyFile /tmp/tls.pem --tlsCAFile /etc/mongodb-ssl/ca.crt --tls' + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.NENtW37Qkd Percona Server for MongoDB shell version v4.4.29-28 connecting to: mongodb://monitoring-pmm3-mongos.monitoring-pmm3-6334.svc.cluster.local:27019/admin?compressors=disabled&gssapiServiceName=mongodb {"t":{"$date":"2026-06-09T09:22:55.968Z"},"s":"I", "c":"NETWORK", "id":5490002, "ctx":"thread1","msg":"Started a new thread for the timer service"} Implicit session: session { "id" : UUID("99067d16-b9a6-4ccc-9446-ccc9ff96cf48") } Percona Server for MongoDB server version: v8.0.23-10 WARNING: shell and server versions do not match switched to db myApp WriteResult({ "nInserted" : 1 }) bye + cat /tmp/tmp.o7RPciwxUm + rm /tmp/tmp.NENtW37Qkd /tmp/tmp.o7RPciwxUm + return 0 + insert_data_mongos 100700 myApp '--tlsCertificateKeyFile /tmp/tls.pem --tlsCAFile /etc/mongodb-ssl/ca.crt --tls' 27019 + local data=100700 + local db_name=myApp + local 'flags=--tlsCertificateKeyFile /tmp/tls.pem --tlsCAFile /etc/mongodb-ssl/ca.crt --tls' + local port=27019 + run_mongos 'use myApp\n db.test.insert({ x: 100700 })' myApp:myPass@monitoring-pmm3-mongos.monitoring-pmm3-6334 '' '' '--tlsCertificateKeyFile /tmp/tls.pem --tlsCAFile /etc/mongodb-ssl/ca.crt --tls' 27019 + local 'command=use myApp\n db.test.insert({ x: 100700 })' + local uri=myApp:myPass@monitoring-pmm3-mongos.monitoring-pmm3-6334 + local driver=mongodb + local suffix=.svc.cluster.local + local 'mongo_flag=--tlsCertificateKeyFile /tmp/tls.pem --tlsCAFile /etc/mongodb-ssl/ca.crt --tls' + local port=27019 + local mongo_bin=mongo ++ echo .svc.cluster.local ++ awk -F: '{print $2}' + suffix_port= + [[ -z '' ]] + suffix=.svc.cluster.local:27019 ++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.CwvmPRON4m +++ mktemp ++ local LAST_ERR=/tmp/tmp.spQ2StrCHv ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.CwvmPRON4m ++ cat /tmp/tmp.spQ2StrCHv ++ rm /tmp/tmp.CwvmPRON4m /tmp/tmp.spQ2StrCHv ++ return 0 + local client_container=psmdb-client-7c8f77fb65-gkcpv + kubectl_bin exec psmdb-client-7c8f77fb65-gkcpv -- bash -c 'printf '\''use myApp\n db.test.insert({ x: 100700 })\n'\'' | mongo mongodb://myApp:myPass@monitoring-pmm3-mongos.monitoring-pmm3-6334.svc.cluster.local:27019/admin --tlsCertificateKeyFile /tmp/tls.pem --tlsCAFile /etc/mongodb-ssl/ca.crt --tls' ++ mktemp + local LAST_OUT=/tmp/tmp.wQUzW4VaKp ++ mktemp + local LAST_ERR=/tmp/tmp.vztqcIrZCH + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl exec psmdb-client-7c8f77fb65-gkcpv -- bash -c 'printf '\''use myApp\n db.test.insert({ x: 100700 })\n'\'' | mongo mongodb://myApp:myPass@monitoring-pmm3-mongos.monitoring-pmm3-6334.svc.cluster.local:27019/admin --tlsCertificateKeyFile /tmp/tls.pem --tlsCAFile /etc/mongodb-ssl/ca.crt --tls' + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.wQUzW4VaKp Percona Server for MongoDB shell version v4.4.29-28 connecting to: mongodb://monitoring-pmm3-mongos.monitoring-pmm3-6334.svc.cluster.local:27019/admin?compressors=disabled&gssapiServiceName=mongodb {"t":{"$date":"2026-06-09T09:22:58.741Z"},"s":"I", "c":"NETWORK", "id":5490002, "ctx":"thread1","msg":"Started a new thread for the timer service"} Implicit session: session { "id" : UUID("b9795b53-6ee4-4f19-9666-cc31fcb21cef") } Percona Server for MongoDB server version: v8.0.23-10 WARNING: shell and server versions do not match switched to db myApp WriteResult({ "nInserted" : 1 }) bye + cat /tmp/tmp.vztqcIrZCH + rm /tmp/tmp.wQUzW4VaKp /tmp/tmp.vztqcIrZCH + return 0 + desc 'add PMM3 token to secret' + set +o xtrace ----------------------------------------------------------------------------------- add PMM3 token to secret ----------------------------------------------------------------------------------- ++ get_pmm_server_token operator ++ local key_name=operator ++ [[ -z operator ]] ++ local ADMIN_PASSWORD +++ kubectl get secret pmm-secret -o 'jsonpath={.data.PMM_ADMIN_PASSWORD}' +++ base64 --decode ++ ADMIN_PASSWORD='(0%0rKB/#vj#2ICw' ++ [[ -z (0%0rKB/#vj#2ICw ]] ++ local create_response create_status_code create_json_response ++++ get_service_endpoint monitoring-service ++++ local service=monitoring-service +++++ kubectl_bin get service/monitoring-service -o json +++++ jq '.status.loadBalancer.ingress[].hostname' +++++ sed -e 's/^"//; s/"$//;' ++++++ mktemp +++++ local LAST_OUT=/tmp/tmp.FpHaDGGQ1l ++++++ mktemp +++++ local LAST_ERR=/tmp/tmp.LD6QOP7UwW +++++ local exit_status=0 +++++ local timeout=4 ++++++ seq 0 2 +++++ for i in $(seq 0 2) +++++ set +e +++++ kubectl get service/monitoring-service -o json +++++ exit_status=0 +++++ set -e +++++ '[' 0 '!=' 0 -a -n 1 ']' +++++ break +++++ cat /tmp/tmp.FpHaDGGQ1l +++++ cat /tmp/tmp.LD6QOP7UwW +++++ rm /tmp/tmp.FpHaDGGQ1l /tmp/tmp.LD6QOP7UwW +++++ return 0 ++++ local hostname=null ++++ '[' -n null -a null '!=' null ']' +++++ kubectl_bin get service/monitoring-service -o json +++++ jq '.status.loadBalancer.ingress[].ip' +++++ sed -e 's/^"//; s/"$//;' ++++++ mktemp +++++ local LAST_OUT=/tmp/tmp.XD4ZwoJLcq ++++++ mktemp +++++ local LAST_ERR=/tmp/tmp.VIwsuWkmYt +++++ local exit_status=0 +++++ local timeout=4 ++++++ seq 0 2 +++++ for i in $(seq 0 2) +++++ set +e +++++ kubectl get service/monitoring-service -o json +++++ exit_status=0 +++++ set -e +++++ '[' 0 '!=' 0 -a -n 1 ']' +++++ break +++++ cat /tmp/tmp.XD4ZwoJLcq +++++ cat /tmp/tmp.VIwsuWkmYt +++++ rm /tmp/tmp.XD4ZwoJLcq /tmp/tmp.VIwsuWkmYt +++++ return 0 ++++ local ip=136.115.177.16 ++++ '[' -n 136.115.177.16 -a 136.115.177.16 '!=' null ']' ++++ echo 136.115.177.16 ++++ return +++ curl --insecure -s -X POST -H 'Content-Type: application/json' -H 'Accept: application/json' -d '{"name":"operator", "role":"Admin", "isDisabled":false}' --user 'admin:(0%0rKB/#vj#2ICw' https://136.115.177.16/graph/api/serviceaccounts -w '\n%{http_code}' ++ create_response=$'{"id":2,"uid":"cfolp5guen8cgb","name":"operator","login":"sa-1-operator","orgId":1,"isDisabled":false,"role":"Admin","tokens":0,"avatarUrl":""}\n201' +++ echo $'{"id":2,"uid":"cfolp5guen8cgb","name":"operator","login":"sa-1-operator","orgId":1,"isDisabled":false,"role":"Admin","tokens":0,"avatarUrl":""}\n201' +++ tail -n1 ++ create_status_code=201 +++ echo $'{"id":2,"uid":"cfolp5guen8cgb","name":"operator","login":"sa-1-operator","orgId":1,"isDisabled":false,"role":"Admin","tokens":0,"avatarUrl":""}\n201' +++ sed '$ d' ++ create_json_response='{"id":2,"uid":"cfolp5guen8cgb","name":"operator","login":"sa-1-operator","orgId":1,"isDisabled":false,"role":"Admin","tokens":0,"avatarUrl":""}' ++ [[ 201 -ne 201 ]] ++ local service_account_id +++ echo '{"id":2,"uid":"cfolp5guen8cgb","name":"operator","login":"sa-1-operator","orgId":1,"isDisabled":false,"role":"Admin","tokens":0,"avatarUrl":""}' +++ jq -r .id ++ service_account_id=2 ++ [[ -z 2 ]] ++ [[ 2 == null ]] ++ local token_response token_status_code token_json_response ++++ get_service_endpoint monitoring-service ++++ local service=monitoring-service +++++ kubectl_bin get service/monitoring-service -o json +++++ jq '.status.loadBalancer.ingress[].hostname' +++++ sed -e 's/^"//; s/"$//;' ++++++ mktemp +++++ local LAST_OUT=/tmp/tmp.o8nlYlOM1h ++++++ mktemp +++++ local LAST_ERR=/tmp/tmp.BCESa8l8Lr +++++ local exit_status=0 +++++ local timeout=4 ++++++ seq 0 2 +++++ for i in $(seq 0 2) +++++ set +e +++++ kubectl get service/monitoring-service -o json +++++ exit_status=0 +++++ set -e +++++ '[' 0 '!=' 0 -a -n 1 ']' +++++ break +++++ cat /tmp/tmp.o8nlYlOM1h +++++ cat /tmp/tmp.BCESa8l8Lr +++++ rm /tmp/tmp.o8nlYlOM1h /tmp/tmp.BCESa8l8Lr +++++ return 0 ++++ local hostname=null ++++ '[' -n null -a null '!=' null ']' +++++ kubectl_bin get service/monitoring-service -o json +++++ jq '.status.loadBalancer.ingress[].ip' +++++ sed -e 's/^"//; s/"$//;' ++++++ mktemp +++++ local LAST_OUT=/tmp/tmp.zSts8ifzfa ++++++ mktemp +++++ local LAST_ERR=/tmp/tmp.5ESJ3SOp0A +++++ local exit_status=0 +++++ local timeout=4 ++++++ seq 0 2 +++++ for i in $(seq 0 2) +++++ set +e +++++ kubectl get service/monitoring-service -o json +++++ exit_status=0 +++++ set -e +++++ '[' 0 '!=' 0 -a -n 1 ']' +++++ break +++++ cat /tmp/tmp.zSts8ifzfa +++++ cat /tmp/tmp.5ESJ3SOp0A +++++ rm /tmp/tmp.zSts8ifzfa /tmp/tmp.5ESJ3SOp0A +++++ return 0 ++++ local ip=136.115.177.16 ++++ '[' -n 136.115.177.16 -a 136.115.177.16 '!=' null ']' ++++ echo 136.115.177.16 ++++ return +++ curl --insecure -s -X POST -H 'Content-Type: application/json' -d '{"name":"operator"}' --user 'admin:(0%0rKB/#vj#2ICw' https://136.115.177.16/graph/api/serviceaccounts/2/tokens -w '\n%{http_code}' ++ token_response=$'{"id":1,"name":"operator","key":"glsa_pVqnA2v8JPKzAd1vzYkjAvLperbzMa89_b5644e26"}\n200' +++ echo $'{"id":1,"name":"operator","key":"glsa_pVqnA2v8JPKzAd1vzYkjAvLperbzMa89_b5644e26"}\n200' +++ tail -n1 ++ token_status_code=200 +++ echo $'{"id":1,"name":"operator","key":"glsa_pVqnA2v8JPKzAd1vzYkjAvLperbzMa89_b5644e26"}\n200' +++ sed '$ d' ++ token_json_response='{"id":1,"name":"operator","key":"glsa_pVqnA2v8JPKzAd1vzYkjAvLperbzMa89_b5644e26"}' ++ [[ 200 -ne 200 ]] ++ echo '{"id":1,"name":"operator","key":"glsa_pVqnA2v8JPKzAd1vzYkjAvLperbzMa89_b5644e26"}' ++ jq -r .key + TOKEN=glsa_pVqnA2v8JPKzAd1vzYkjAvLperbzMa89_b5644e26 + kubectl_bin patch secret some-users --type merge --patch '{"stringData": {"PMM_SERVER_TOKEN": "glsa_pVqnA2v8JPKzAd1vzYkjAvLperbzMa89_b5644e26"}}' ++ mktemp + local LAST_OUT=/tmp/tmp.f2CXnBAhsO ++ mktemp + local LAST_ERR=/tmp/tmp.281s6TwruA + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl patch secret some-users --type merge --patch '{"stringData": {"PMM_SERVER_TOKEN": "glsa_pVqnA2v8JPKzAd1vzYkjAvLperbzMa89_b5644e26"}}' + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.f2CXnBAhsO secret/some-users patched + cat /tmp/tmp.281s6TwruA + rm /tmp/tmp.f2CXnBAhsO /tmp/tmp.281s6TwruA + return 0 + desc 'check if all 3 Pods started' + set +o xtrace ----------------------------------------------------------------------------------- check if all 3 Pods started ----------------------------------------------------------------------------------- + wait_for_running monitoring-pmm3-rs0 3 + local name=monitoring-pmm3-rs0 + let last_pod=2 + local check_cluster_readyness=true + set_debug + [[ 1 == 1 ]] + set -o xtrace + local rs_name=rs0 + local cluster_name=monitoring-pmm3 ++ seq 0 2 + for i in $(seq 0 $last_pod) + [[ 0 -eq 2 ]] + wait_pod monitoring-pmm3-rs0-0 + local pod=monitoring-pmm3-rs0-0 + set +o xtrace waiting for pod/monitoring-pmm3-rs0-0 to be ready.OK + for i in $(seq 0 $last_pod) + [[ 1 -eq 2 ]] + wait_pod monitoring-pmm3-rs0-1 + local pod=monitoring-pmm3-rs0-1 + set +o xtrace waiting for pod/monitoring-pmm3-rs0-1 to be ready.OK + for i in $(seq 0 $last_pod) + [[ 2 -eq 2 ]] ++ kubectl_bin get psmdb monitoring-pmm3 -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].arbiter.enabled}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.fKBC6qiJBV +++ mktemp ++ local LAST_ERR=/tmp/tmp.YKXjb026MO ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb monitoring-pmm3 -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].arbiter.enabled}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.fKBC6qiJBV ++ cat /tmp/tmp.YKXjb026MO ++ rm /tmp/tmp.fKBC6qiJBV /tmp/tmp.YKXjb026MO ++ return 0 + [[ '' == true ]] + wait_pod monitoring-pmm3-rs0-2 + local pod=monitoring-pmm3-rs0-2 + set +o xtrace waiting for pod/monitoring-pmm3-rs0-2 to be ready.OK ++ kubectl_bin get psmdb monitoring-pmm3 -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].nonvoting.enabled}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.pYtSOjY6Je +++ mktemp ++ local LAST_ERR=/tmp/tmp.U0qI0KaGJO ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb monitoring-pmm3 -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].nonvoting.enabled}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.pYtSOjY6Je ++ cat /tmp/tmp.U0qI0KaGJO ++ rm /tmp/tmp.pYtSOjY6Je /tmp/tmp.U0qI0KaGJO ++ return 0 + [[ '' == true ]] ++ kubectl_bin get psmdb monitoring-pmm3 -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].hidden.enabled}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.9fvbDrXQFn +++ mktemp ++ local LAST_ERR=/tmp/tmp.GsgXdBn84y ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb monitoring-pmm3 -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].hidden.enabled}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.9fvbDrXQFn ++ cat /tmp/tmp.GsgXdBn84y ++ rm /tmp/tmp.9fvbDrXQFn /tmp/tmp.GsgXdBn84y ++ return 0 + [[ '' == true ]] + sleep 10 + [[ true == true ]] + set +x Waiting for cluster readyness............................................................................................................................................ + sleep 90 + desc 'check if pmm-client container enabled' + set +o xtrace ----------------------------------------------------------------------------------- check if pmm-client container enabled ----------------------------------------------------------------------------------- + compare_kubectl statefulset/monitoring-pmm3-rs0 + local resource=statefulset/monitoring-pmm3-rs0 + local postfix= + local skip_generation_check= + local expected_result=/mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/e2e-tests/monitoring-pmm3/compare/statefulset_monitoring-pmm3-rs0.yml + local new_result=/tmp/tmp.5EJEnDygsZ/statefulset_monitoring-pmm3-rs0.yml + '[' -n '' -a -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/e2e-tests/monitoring-pmm3/compare/statefulset_monitoring-pmm3-rs0-oc.yml ']' + kubectl_bin get -o yaml statefulset/monitoring-pmm3-rs0 + yq eval $'\n\t\t\tdel(.metadata.ownerReferences[].apiVersion) |\n\t\t\tdel(.metadata.managedFields) |\n\t\t\tdel(.. | select(has("creationTimestamp")).creationTimestamp) |\n\t\t\tdel(.. | select(has("namespace")).namespace) |\n\t\t\tdel(.. | select(has("uid")).uid) |\n\t\t\tdel(.metadata.resourceVersion) |\n\t\t\tdel(.spec.template.spec.containers[].env[] | select(.name == "NAMESPACE")) |\n\t\t\tdel(.metadata.selfLink) |\n\t\t\tdel(.metadata.annotations."cloud.google.com/neg") |\n\t\t\tdel(.metadata.annotations."kubectl.kubernetes.io/last-applied-configuration") |\n\t\t\tdel(.. | select(has("image")).image) |\n\t\t\tdel(.. | select(has("clusterIP")).clusterIP) |\n\t\t\tdel(.. | select(has("clusterIPs")).clusterIPs) |\n\t\t\tdel(.. | select(has("dataSource")).dataSource) |\n\t\t\tdel(.. | select(has("procMount")).procMount) |\n\t\t\tdel(.. | select(has("storageClassName")).storageClassName) |\n\t\t\tdel(.. | select(has("finalizers")).finalizers) |\n\t\t\tdel(.. | select(has("kubernetes.io/pvc-protection"))."kubernetes.io/pvc-protection") |\n\t\t\tdel(.. | select(has("volumeName")).volumeName) |\n\t\t\tdel(.. | select(has("volume.beta.kubernetes.io/storage-provisioner"))."volume.beta.kubernetes.io/storage-provisioner") |\n\t\t\tdel(.. | select(has("volume.kubernetes.io/storage-provisioner"))."volume.kubernetes.io/storage-provisioner") |\n\t\t\tdel(.spec.volumeMode) |\n\t\t\tdel(.. | select(has("volume.kubernetes.io/selected-node"))."volume.kubernetes.io/selected-node") |\n\t\t\tdel(.. | select(has("percona.com/last-config-hash"))."percona.com/last-config-hash") |\n\t\t\tdel(.. | select(has("percona.com/configuration-hash"))."percona.com/configuration-hash") |\n\t\t\tdel(.. | select(has("percona.com/ssl-hash"))."percona.com/ssl-hash") |\n\t\t\tdel(.. | select(has("percona.com/ssl-internal-hash"))."percona.com/ssl-internal-hash") |\n\t\t\tdel(.spec.volumeClaimTemplates[].spec.volumeMode | select(. == "Filesystem")) |\n\t\t\tdel(.. | select(has("healthCheckNodePort")).healthCheckNodePort) |\n\t\t\tdel(.. | select(has("nodePort")).nodePort) |\n\t\t\tdel(.status) |\n\t\t\t(.. | select(tag == "!!str")) |= sub("monitoring-pmm3-6334", "NAME_SPACE") |\n\t\t\tdel(.spec.volumeClaimTemplates[].apiVersion) |\n\t\t\tdel(.spec.volumeClaimTemplates[].kind) |\n\t\t\tdel(.spec.ipFamilies) |\n\t\t\tdel(.spec.ipFamilyPolicy) |\n\t\t\t(.. | select(. == "extensions/v1beta1")) = "apps/v1" |\n\t\t\t(.. | select(. == "batch/v1beta1")) = "batch/v1" ' - ++ mktemp + local LAST_OUT=/tmp/tmp.jkyH0KO5S1 ++ mktemp + local LAST_ERR=/tmp/tmp.zPEyiBVNwn + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl get -o yaml statefulset/monitoring-pmm3-rs0 + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.jkyH0KO5S1 + cat /tmp/tmp.zPEyiBVNwn + rm /tmp/tmp.jkyH0KO5S1 /tmp/tmp.zPEyiBVNwn + return 0 + yq -i eval 'del(.spec.persistentVolumeClaimRetentionPolicy)' /tmp/tmp.5EJEnDygsZ/statefulset_monitoring-pmm3-rs0.yml + version_gt 1.22 ++ echo '1.33 >= 1.22' ++ bc -l + '[' 1 -eq 1 ']' + return 0 + yq -i eval 'del(.spec.internalTrafficPolicy)' /tmp/tmp.5EJEnDygsZ/statefulset_monitoring-pmm3-rs0.yml + yq -i eval 'del(.spec.allocateLoadBalancerNodePorts)' /tmp/tmp.5EJEnDygsZ/statefulset_monitoring-pmm3-rs0.yml + [[ /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/e2e-tests/monitoring-pmm3/compare/statefulset_monitoring-pmm3-rs0.yml == */cronjob* ]] + '[' -n '' ']' + [[ 0 -eq 0 ]] + diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/e2e-tests/monitoring-pmm3/compare/statefulset_monitoring-pmm3-rs0.yml /tmp/tmp.5EJEnDygsZ/statefulset_monitoring-pmm3-rs0.yml + log 'compare_kubectl: statefulset/monitoring-pmm3-rs0 OK' + set +o xtrace [2026-06-09T09:29:53+0000] compare_kubectl: statefulset/monitoring-pmm3-rs0 OK + compare_kubectl service/monitoring-pmm3-rs0 + local resource=service/monitoring-pmm3-rs0 + local postfix= + local skip_generation_check= + local expected_result=/mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/e2e-tests/monitoring-pmm3/compare/service_monitoring-pmm3-rs0.yml + local new_result=/tmp/tmp.5EJEnDygsZ/service_monitoring-pmm3-rs0.yml + '[' -n '' -a -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/e2e-tests/monitoring-pmm3/compare/service_monitoring-pmm3-rs0-oc.yml ']' + kubectl_bin get -o yaml service/monitoring-pmm3-rs0 + yq eval $'\n\t\t\tdel(.metadata.ownerReferences[].apiVersion) |\n\t\t\tdel(.metadata.managedFields) |\n\t\t\tdel(.. | select(has("creationTimestamp")).creationTimestamp) |\n\t\t\tdel(.. | select(has("namespace")).namespace) |\n\t\t\tdel(.. | select(has("uid")).uid) |\n\t\t\tdel(.metadata.resourceVersion) |\n\t\t\tdel(.spec.template.spec.containers[].env[] | select(.name == "NAMESPACE")) |\n\t\t\tdel(.metadata.selfLink) |\n\t\t\tdel(.metadata.annotations."cloud.google.com/neg") |\n\t\t\tdel(.metadata.annotations."kubectl.kubernetes.io/last-applied-configuration") |\n\t\t\tdel(.. | select(has("image")).image) |\n\t\t\tdel(.. | select(has("clusterIP")).clusterIP) |\n\t\t\tdel(.. | select(has("clusterIPs")).clusterIPs) |\n\t\t\tdel(.. | select(has("dataSource")).dataSource) |\n\t\t\tdel(.. | select(has("procMount")).procMount) |\n\t\t\tdel(.. | select(has("storageClassName")).storageClassName) |\n\t\t\tdel(.. | select(has("finalizers")).finalizers) |\n\t\t\tdel(.. | select(has("kubernetes.io/pvc-protection"))."kubernetes.io/pvc-protection") |\n\t\t\tdel(.. | select(has("volumeName")).volumeName) |\n\t\t\tdel(.. | select(has("volume.beta.kubernetes.io/storage-provisioner"))."volume.beta.kubernetes.io/storage-provisioner") |\n\t\t\tdel(.. | select(has("volume.kubernetes.io/storage-provisioner"))."volume.kubernetes.io/storage-provisioner") |\n\t\t\tdel(.spec.volumeMode) |\n\t\t\tdel(.. | select(has("volume.kubernetes.io/selected-node"))."volume.kubernetes.io/selected-node") |\n\t\t\tdel(.. | select(has("percona.com/last-config-hash"))."percona.com/last-config-hash") |\n\t\t\tdel(.. | select(has("percona.com/configuration-hash"))."percona.com/configuration-hash") |\n\t\t\tdel(.. | select(has("percona.com/ssl-hash"))."percona.com/ssl-hash") |\n\t\t\tdel(.. | select(has("percona.com/ssl-internal-hash"))."percona.com/ssl-internal-hash") |\n\t\t\tdel(.spec.volumeClaimTemplates[].spec.volumeMode | select(. == "Filesystem")) |\n\t\t\tdel(.. | select(has("healthCheckNodePort")).healthCheckNodePort) |\n\t\t\tdel(.. | select(has("nodePort")).nodePort) |\n\t\t\tdel(.status) |\n\t\t\t(.. | select(tag == "!!str")) |= sub("monitoring-pmm3-6334", "NAME_SPACE") |\n\t\t\tdel(.spec.volumeClaimTemplates[].apiVersion) |\n\t\t\tdel(.spec.volumeClaimTemplates[].kind) |\n\t\t\tdel(.spec.ipFamilies) |\n\t\t\tdel(.spec.ipFamilyPolicy) |\n\t\t\t(.. | select(. == "extensions/v1beta1")) = "apps/v1" |\n\t\t\t(.. | select(. == "batch/v1beta1")) = "batch/v1" ' - ++ mktemp + local LAST_OUT=/tmp/tmp.zsJtEBP02v ++ mktemp + local LAST_ERR=/tmp/tmp.nONBaU8xdJ + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl get -o yaml service/monitoring-pmm3-rs0 + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.zsJtEBP02v + cat /tmp/tmp.nONBaU8xdJ + rm /tmp/tmp.zsJtEBP02v /tmp/tmp.nONBaU8xdJ + return 0 + yq -i eval 'del(.spec.persistentVolumeClaimRetentionPolicy)' /tmp/tmp.5EJEnDygsZ/service_monitoring-pmm3-rs0.yml + version_gt 1.22 ++ echo '1.33 >= 1.22' ++ bc -l + '[' 1 -eq 1 ']' + return 0 + yq -i eval 'del(.spec.internalTrafficPolicy)' /tmp/tmp.5EJEnDygsZ/service_monitoring-pmm3-rs0.yml + yq -i eval 'del(.spec.allocateLoadBalancerNodePorts)' /tmp/tmp.5EJEnDygsZ/service_monitoring-pmm3-rs0.yml + [[ /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/e2e-tests/monitoring-pmm3/compare/service_monitoring-pmm3-rs0.yml == */cronjob* ]] + '[' -n '' ']' + [[ 0 -eq 0 ]] + diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/e2e-tests/monitoring-pmm3/compare/service_monitoring-pmm3-rs0.yml /tmp/tmp.5EJEnDygsZ/service_monitoring-pmm3-rs0.yml + log 'compare_kubectl: service/monitoring-pmm3-rs0 OK' + set +o xtrace [2026-06-09T09:29:54+0000] compare_kubectl: service/monitoring-pmm3-rs0 OK + compare_kubectl service/monitoring-pmm3-mongos + local resource=service/monitoring-pmm3-mongos + local postfix= + local skip_generation_check= + local expected_result=/mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/e2e-tests/monitoring-pmm3/compare/service_monitoring-pmm3-mongos.yml + local new_result=/tmp/tmp.5EJEnDygsZ/service_monitoring-pmm3-mongos.yml + '[' -n '' -a -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/e2e-tests/monitoring-pmm3/compare/service_monitoring-pmm3-mongos-oc.yml ']' + kubectl_bin get -o yaml service/monitoring-pmm3-mongos + yq eval $'\n\t\t\tdel(.metadata.ownerReferences[].apiVersion) |\n\t\t\tdel(.metadata.managedFields) |\n\t\t\tdel(.. | select(has("creationTimestamp")).creationTimestamp) |\n\t\t\tdel(.. | select(has("namespace")).namespace) |\n\t\t\tdel(.. | select(has("uid")).uid) |\n\t\t\tdel(.metadata.resourceVersion) |\n\t\t\tdel(.spec.template.spec.containers[].env[] | select(.name == "NAMESPACE")) |\n\t\t\tdel(.metadata.selfLink) |\n\t\t\tdel(.metadata.annotations."cloud.google.com/neg") |\n\t\t\tdel(.metadata.annotations."kubectl.kubernetes.io/last-applied-configuration") |\n\t\t\tdel(.. | select(has("image")).image) |\n\t\t\tdel(.. | select(has("clusterIP")).clusterIP) |\n\t\t\tdel(.. | select(has("clusterIPs")).clusterIPs) |\n\t\t\tdel(.. | select(has("dataSource")).dataSource) |\n\t\t\tdel(.. | select(has("procMount")).procMount) |\n\t\t\tdel(.. | select(has("storageClassName")).storageClassName) |\n\t\t\tdel(.. | select(has("finalizers")).finalizers) |\n\t\t\tdel(.. | select(has("kubernetes.io/pvc-protection"))."kubernetes.io/pvc-protection") |\n\t\t\tdel(.. | select(has("volumeName")).volumeName) |\n\t\t\tdel(.. | select(has("volume.beta.kubernetes.io/storage-provisioner"))."volume.beta.kubernetes.io/storage-provisioner") |\n\t\t\tdel(.. | select(has("volume.kubernetes.io/storage-provisioner"))."volume.kubernetes.io/storage-provisioner") |\n\t\t\tdel(.spec.volumeMode) |\n\t\t\tdel(.. | select(has("volume.kubernetes.io/selected-node"))."volume.kubernetes.io/selected-node") |\n\t\t\tdel(.. | select(has("percona.com/last-config-hash"))."percona.com/last-config-hash") |\n\t\t\tdel(.. | select(has("percona.com/configuration-hash"))."percona.com/configuration-hash") |\n\t\t\tdel(.. | select(has("percona.com/ssl-hash"))."percona.com/ssl-hash") |\n\t\t\tdel(.. | select(has("percona.com/ssl-internal-hash"))."percona.com/ssl-internal-hash") |\n\t\t\tdel(.spec.volumeClaimTemplates[].spec.volumeMode | select(. == "Filesystem")) |\n\t\t\tdel(.. | select(has("healthCheckNodePort")).healthCheckNodePort) |\n\t\t\tdel(.. | select(has("nodePort")).nodePort) |\n\t\t\tdel(.status) |\n\t\t\t(.. | select(tag == "!!str")) |= sub("monitoring-pmm3-6334", "NAME_SPACE") |\n\t\t\tdel(.spec.volumeClaimTemplates[].apiVersion) |\n\t\t\tdel(.spec.volumeClaimTemplates[].kind) |\n\t\t\tdel(.spec.ipFamilies) |\n\t\t\tdel(.spec.ipFamilyPolicy) |\n\t\t\t(.. | select(. == "extensions/v1beta1")) = "apps/v1" |\n\t\t\t(.. | select(. == "batch/v1beta1")) = "batch/v1" ' - ++ mktemp + local LAST_OUT=/tmp/tmp.P2KZn1HZYW ++ mktemp + local LAST_ERR=/tmp/tmp.39OYuu9e6G + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl get -o yaml service/monitoring-pmm3-mongos + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.P2KZn1HZYW + cat /tmp/tmp.39OYuu9e6G + rm /tmp/tmp.P2KZn1HZYW /tmp/tmp.39OYuu9e6G + return 0 + yq -i eval 'del(.spec.persistentVolumeClaimRetentionPolicy)' /tmp/tmp.5EJEnDygsZ/service_monitoring-pmm3-mongos.yml + version_gt 1.22 ++ echo '1.33 >= 1.22' ++ bc -l + '[' 1 -eq 1 ']' + return 0 + yq -i eval 'del(.spec.internalTrafficPolicy)' /tmp/tmp.5EJEnDygsZ/service_monitoring-pmm3-mongos.yml + yq -i eval 'del(.spec.allocateLoadBalancerNodePorts)' /tmp/tmp.5EJEnDygsZ/service_monitoring-pmm3-mongos.yml + [[ /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/e2e-tests/monitoring-pmm3/compare/service_monitoring-pmm3-mongos.yml == */cronjob* ]] + '[' -n '' ']' + [[ 0 -eq 0 ]] + diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/e2e-tests/monitoring-pmm3/compare/service_monitoring-pmm3-mongos.yml /tmp/tmp.5EJEnDygsZ/service_monitoring-pmm3-mongos.yml + log 'compare_kubectl: service/monitoring-pmm3-mongos OK' + set +o xtrace [2026-06-09T09:29:54+0000] compare_kubectl: service/monitoring-pmm3-mongos OK + compare_kubectl statefulset/monitoring-pmm3-cfg + local resource=statefulset/monitoring-pmm3-cfg + local postfix= + local skip_generation_check= + local expected_result=/mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/e2e-tests/monitoring-pmm3/compare/statefulset_monitoring-pmm3-cfg.yml + local new_result=/tmp/tmp.5EJEnDygsZ/statefulset_monitoring-pmm3-cfg.yml + '[' -n '' -a -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/e2e-tests/monitoring-pmm3/compare/statefulset_monitoring-pmm3-cfg-oc.yml ']' + kubectl_bin get -o yaml statefulset/monitoring-pmm3-cfg + yq eval $'\n\t\t\tdel(.metadata.ownerReferences[].apiVersion) |\n\t\t\tdel(.metadata.managedFields) |\n\t\t\tdel(.. | select(has("creationTimestamp")).creationTimestamp) |\n\t\t\tdel(.. | select(has("namespace")).namespace) |\n\t\t\tdel(.. | select(has("uid")).uid) |\n\t\t\tdel(.metadata.resourceVersion) |\n\t\t\tdel(.spec.template.spec.containers[].env[] | select(.name == "NAMESPACE")) |\n\t\t\tdel(.metadata.selfLink) |\n\t\t\tdel(.metadata.annotations."cloud.google.com/neg") |\n\t\t\tdel(.metadata.annotations."kubectl.kubernetes.io/last-applied-configuration") |\n\t\t\tdel(.. | select(has("image")).image) |\n\t\t\tdel(.. | select(has("clusterIP")).clusterIP) |\n\t\t\tdel(.. | select(has("clusterIPs")).clusterIPs) |\n\t\t\tdel(.. | select(has("dataSource")).dataSource) |\n\t\t\tdel(.. | select(has("procMount")).procMount) |\n\t\t\tdel(.. | select(has("storageClassName")).storageClassName) |\n\t\t\tdel(.. | select(has("finalizers")).finalizers) |\n\t\t\tdel(.. | select(has("kubernetes.io/pvc-protection"))."kubernetes.io/pvc-protection") |\n\t\t\tdel(.. | select(has("volumeName")).volumeName) |\n\t\t\tdel(.. | select(has("volume.beta.kubernetes.io/storage-provisioner"))."volume.beta.kubernetes.io/storage-provisioner") |\n\t\t\tdel(.. | select(has("volume.kubernetes.io/storage-provisioner"))."volume.kubernetes.io/storage-provisioner") |\n\t\t\tdel(.spec.volumeMode) |\n\t\t\tdel(.. | select(has("volume.kubernetes.io/selected-node"))."volume.kubernetes.io/selected-node") |\n\t\t\tdel(.. | select(has("percona.com/last-config-hash"))."percona.com/last-config-hash") |\n\t\t\tdel(.. | select(has("percona.com/configuration-hash"))."percona.com/configuration-hash") |\n\t\t\tdel(.. | select(has("percona.com/ssl-hash"))."percona.com/ssl-hash") |\n\t\t\tdel(.. | select(has("percona.com/ssl-internal-hash"))."percona.com/ssl-internal-hash") |\n\t\t\tdel(.spec.volumeClaimTemplates[].spec.volumeMode | select(. == "Filesystem")) |\n\t\t\tdel(.. | select(has("healthCheckNodePort")).healthCheckNodePort) |\n\t\t\tdel(.. | select(has("nodePort")).nodePort) |\n\t\t\tdel(.status) |\n\t\t\t(.. | select(tag == "!!str")) |= sub("monitoring-pmm3-6334", "NAME_SPACE") |\n\t\t\tdel(.spec.volumeClaimTemplates[].apiVersion) |\n\t\t\tdel(.spec.volumeClaimTemplates[].kind) |\n\t\t\tdel(.spec.ipFamilies) |\n\t\t\tdel(.spec.ipFamilyPolicy) |\n\t\t\t(.. | select(. == "extensions/v1beta1")) = "apps/v1" |\n\t\t\t(.. | select(. == "batch/v1beta1")) = "batch/v1" ' - ++ mktemp + local LAST_OUT=/tmp/tmp.N3cO0LhyeU ++ mktemp + local LAST_ERR=/tmp/tmp.63K43fJbuz + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl get -o yaml statefulset/monitoring-pmm3-cfg + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.N3cO0LhyeU + cat /tmp/tmp.63K43fJbuz + rm /tmp/tmp.N3cO0LhyeU /tmp/tmp.63K43fJbuz + return 0 + yq -i eval 'del(.spec.persistentVolumeClaimRetentionPolicy)' /tmp/tmp.5EJEnDygsZ/statefulset_monitoring-pmm3-cfg.yml + version_gt 1.22 ++ echo '1.33 >= 1.22' ++ bc -l + '[' 1 -eq 1 ']' + return 0 + yq -i eval 'del(.spec.internalTrafficPolicy)' /tmp/tmp.5EJEnDygsZ/statefulset_monitoring-pmm3-cfg.yml + yq -i eval 'del(.spec.allocateLoadBalancerNodePorts)' /tmp/tmp.5EJEnDygsZ/statefulset_monitoring-pmm3-cfg.yml + [[ /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/e2e-tests/monitoring-pmm3/compare/statefulset_monitoring-pmm3-cfg.yml == */cronjob* ]] + '[' -n '' ']' + [[ 0 -eq 0 ]] + diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/e2e-tests/monitoring-pmm3/compare/statefulset_monitoring-pmm3-cfg.yml /tmp/tmp.5EJEnDygsZ/statefulset_monitoring-pmm3-cfg.yml + log 'compare_kubectl: statefulset/monitoring-pmm3-cfg OK' + set +o xtrace [2026-06-09T09:29:56+0000] compare_kubectl: statefulset/monitoring-pmm3-cfg OK + compare_kubectl statefulset/monitoring-pmm3-mongos + local resource=statefulset/monitoring-pmm3-mongos + local postfix= + local skip_generation_check= + local expected_result=/mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/e2e-tests/monitoring-pmm3/compare/statefulset_monitoring-pmm3-mongos.yml + local new_result=/tmp/tmp.5EJEnDygsZ/statefulset_monitoring-pmm3-mongos.yml + '[' -n '' -a -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/e2e-tests/monitoring-pmm3/compare/statefulset_monitoring-pmm3-mongos-oc.yml ']' + kubectl_bin get -o yaml statefulset/monitoring-pmm3-mongos + yq eval $'\n\t\t\tdel(.metadata.ownerReferences[].apiVersion) |\n\t\t\tdel(.metadata.managedFields) |\n\t\t\tdel(.. | select(has("creationTimestamp")).creationTimestamp) |\n\t\t\tdel(.. | select(has("namespace")).namespace) |\n\t\t\tdel(.. | select(has("uid")).uid) |\n\t\t\tdel(.metadata.resourceVersion) |\n\t\t\tdel(.spec.template.spec.containers[].env[] | select(.name == "NAMESPACE")) |\n\t\t\tdel(.metadata.selfLink) |\n\t\t\tdel(.metadata.annotations."cloud.google.com/neg") |\n\t\t\tdel(.metadata.annotations."kubectl.kubernetes.io/last-applied-configuration") |\n\t\t\tdel(.. | select(has("image")).image) |\n\t\t\tdel(.. | select(has("clusterIP")).clusterIP) |\n\t\t\tdel(.. | select(has("clusterIPs")).clusterIPs) |\n\t\t\tdel(.. | select(has("dataSource")).dataSource) |\n\t\t\tdel(.. | select(has("procMount")).procMount) |\n\t\t\tdel(.. | select(has("storageClassName")).storageClassName) |\n\t\t\tdel(.. | select(has("finalizers")).finalizers) |\n\t\t\tdel(.. | select(has("kubernetes.io/pvc-protection"))."kubernetes.io/pvc-protection") |\n\t\t\tdel(.. | select(has("volumeName")).volumeName) |\n\t\t\tdel(.. | select(has("volume.beta.kubernetes.io/storage-provisioner"))."volume.beta.kubernetes.io/storage-provisioner") |\n\t\t\tdel(.. | select(has("volume.kubernetes.io/storage-provisioner"))."volume.kubernetes.io/storage-provisioner") |\n\t\t\tdel(.spec.volumeMode) |\n\t\t\tdel(.. | select(has("volume.kubernetes.io/selected-node"))."volume.kubernetes.io/selected-node") |\n\t\t\tdel(.. | select(has("percona.com/last-config-hash"))."percona.com/last-config-hash") |\n\t\t\tdel(.. | select(has("percona.com/configuration-hash"))."percona.com/configuration-hash") |\n\t\t\tdel(.. | select(has("percona.com/ssl-hash"))."percona.com/ssl-hash") |\n\t\t\tdel(.. | select(has("percona.com/ssl-internal-hash"))."percona.com/ssl-internal-hash") |\n\t\t\tdel(.spec.volumeClaimTemplates[].spec.volumeMode | select(. == "Filesystem")) |\n\t\t\tdel(.. | select(has("healthCheckNodePort")).healthCheckNodePort) |\n\t\t\tdel(.. | select(has("nodePort")).nodePort) |\n\t\t\tdel(.status) |\n\t\t\t(.. | select(tag == "!!str")) |= sub("monitoring-pmm3-6334", "NAME_SPACE") |\n\t\t\tdel(.spec.volumeClaimTemplates[].apiVersion) |\n\t\t\tdel(.spec.volumeClaimTemplates[].kind) |\n\t\t\tdel(.spec.ipFamilies) |\n\t\t\tdel(.spec.ipFamilyPolicy) |\n\t\t\t(.. | select(. == "extensions/v1beta1")) = "apps/v1" |\n\t\t\t(.. | select(. == "batch/v1beta1")) = "batch/v1" ' - ++ mktemp + local LAST_OUT=/tmp/tmp.TkFrCuzajA ++ mktemp + local LAST_ERR=/tmp/tmp.xlvoeKCqqh + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl get -o yaml statefulset/monitoring-pmm3-mongos + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.TkFrCuzajA + cat /tmp/tmp.xlvoeKCqqh + rm /tmp/tmp.TkFrCuzajA /tmp/tmp.xlvoeKCqqh + return 0 + yq -i eval 'del(.spec.persistentVolumeClaimRetentionPolicy)' /tmp/tmp.5EJEnDygsZ/statefulset_monitoring-pmm3-mongos.yml + version_gt 1.22 ++ echo '1.33 >= 1.22' ++ bc -l + '[' 1 -eq 1 ']' + return 0 + yq -i eval 'del(.spec.internalTrafficPolicy)' /tmp/tmp.5EJEnDygsZ/statefulset_monitoring-pmm3-mongos.yml + yq -i eval 'del(.spec.allocateLoadBalancerNodePorts)' /tmp/tmp.5EJEnDygsZ/statefulset_monitoring-pmm3-mongos.yml + [[ /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/e2e-tests/monitoring-pmm3/compare/statefulset_monitoring-pmm3-mongos.yml == */cronjob* ]] + '[' -n '' ']' + [[ 0 -eq 0 ]] + diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/e2e-tests/monitoring-pmm3/compare/statefulset_monitoring-pmm3-mongos.yml /tmp/tmp.5EJEnDygsZ/statefulset_monitoring-pmm3-mongos.yml + log 'compare_kubectl: statefulset/monitoring-pmm3-mongos OK' + set +o xtrace [2026-06-09T09:29:57+0000] compare_kubectl: statefulset/monitoring-pmm3-mongos OK + desc 'create new PMM token and add it to the secret' + set +o xtrace ----------------------------------------------------------------------------------- create new PMM token and add it to the secret ----------------------------------------------------------------------------------- ++ get_pmm_server_token operator_new ++ local key_name=operator_new ++ [[ -z operator_new ]] ++ local ADMIN_PASSWORD +++ kubectl get secret pmm-secret -o 'jsonpath={.data.PMM_ADMIN_PASSWORD}' +++ base64 --decode ++ ADMIN_PASSWORD='(0%0rKB/#vj#2ICw' ++ [[ -z (0%0rKB/#vj#2ICw ]] ++ local create_response create_status_code create_json_response ++++ get_service_endpoint monitoring-service ++++ local service=monitoring-service +++++ kubectl_bin get service/monitoring-service -o json +++++ jq '.status.loadBalancer.ingress[].hostname' +++++ sed -e 's/^"//; s/"$//;' ++++++ mktemp +++++ local LAST_OUT=/tmp/tmp.1IolrLaM2x ++++++ mktemp +++++ local LAST_ERR=/tmp/tmp.o4hcma7wBL +++++ local exit_status=0 +++++ local timeout=4 ++++++ seq 0 2 +++++ for i in $(seq 0 2) +++++ set +e +++++ kubectl get service/monitoring-service -o json +++++ exit_status=0 +++++ set -e +++++ '[' 0 '!=' 0 -a -n 1 ']' +++++ break +++++ cat /tmp/tmp.1IolrLaM2x +++++ cat /tmp/tmp.o4hcma7wBL +++++ rm /tmp/tmp.1IolrLaM2x /tmp/tmp.o4hcma7wBL +++++ return 0 ++++ local hostname=null ++++ '[' -n null -a null '!=' null ']' +++++ kubectl_bin get service/monitoring-service -o json +++++ jq '.status.loadBalancer.ingress[].ip' +++++ sed -e 's/^"//; s/"$//;' ++++++ mktemp +++++ local LAST_OUT=/tmp/tmp.NFx6N3cgTv ++++++ mktemp +++++ local LAST_ERR=/tmp/tmp.dSsaknFtSX +++++ local exit_status=0 +++++ local timeout=4 ++++++ seq 0 2 +++++ for i in $(seq 0 2) +++++ set +e +++++ kubectl get service/monitoring-service -o json +++++ exit_status=0 +++++ set -e +++++ '[' 0 '!=' 0 -a -n 1 ']' +++++ break +++++ cat /tmp/tmp.NFx6N3cgTv +++++ cat /tmp/tmp.dSsaknFtSX +++++ rm /tmp/tmp.NFx6N3cgTv /tmp/tmp.dSsaknFtSX +++++ return 0 ++++ local ip=136.115.177.16 ++++ '[' -n 136.115.177.16 -a 136.115.177.16 '!=' null ']' ++++ echo 136.115.177.16 ++++ return +++ curl --insecure -s -X POST -H 'Content-Type: application/json' -H 'Accept: application/json' -d '{"name":"operator_new", "role":"Admin", "isDisabled":false}' --user 'admin:(0%0rKB/#vj#2ICw' https://136.115.177.16/graph/api/serviceaccounts -w '\n%{http_code}' ++ create_response=$'{"id":3,"uid":"afolprwd1xuyoe","name":"operator_new","login":"sa-1-operator_new","orgId":1,"isDisabled":false,"role":"Admin","tokens":0,"avatarUrl":""}\n201' +++ tail -n1 +++ echo $'{"id":3,"uid":"afolprwd1xuyoe","name":"operator_new","login":"sa-1-operator_new","orgId":1,"isDisabled":false,"role":"Admin","tokens":0,"avatarUrl":""}\n201' ++ create_status_code=201 +++ echo $'{"id":3,"uid":"afolprwd1xuyoe","name":"operator_new","login":"sa-1-operator_new","orgId":1,"isDisabled":false,"role":"Admin","tokens":0,"avatarUrl":""}\n201' +++ sed '$ d' ++ create_json_response='{"id":3,"uid":"afolprwd1xuyoe","name":"operator_new","login":"sa-1-operator_new","orgId":1,"isDisabled":false,"role":"Admin","tokens":0,"avatarUrl":""}' ++ [[ 201 -ne 201 ]] ++ local service_account_id +++ echo '{"id":3,"uid":"afolprwd1xuyoe","name":"operator_new","login":"sa-1-operator_new","orgId":1,"isDisabled":false,"role":"Admin","tokens":0,"avatarUrl":""}' +++ jq -r .id ++ service_account_id=3 ++ [[ -z 3 ]] ++ [[ 3 == null ]] ++ local token_response token_status_code token_json_response ++++ get_service_endpoint monitoring-service ++++ local service=monitoring-service +++++ kubectl_bin get service/monitoring-service -o json +++++ jq '.status.loadBalancer.ingress[].hostname' +++++ sed -e 's/^"//; s/"$//;' ++++++ mktemp +++++ local LAST_OUT=/tmp/tmp.boimme9DR8 ++++++ mktemp +++++ local LAST_ERR=/tmp/tmp.YjvlwLJ6Lb +++++ local exit_status=0 +++++ local timeout=4 ++++++ seq 0 2 +++++ for i in $(seq 0 2) +++++ set +e +++++ kubectl get service/monitoring-service -o json +++++ exit_status=0 +++++ set -e +++++ '[' 0 '!=' 0 -a -n 1 ']' +++++ break +++++ cat /tmp/tmp.boimme9DR8 +++++ cat /tmp/tmp.YjvlwLJ6Lb +++++ rm /tmp/tmp.boimme9DR8 /tmp/tmp.YjvlwLJ6Lb +++++ return 0 ++++ local hostname=null ++++ '[' -n null -a null '!=' null ']' +++++ kubectl_bin get service/monitoring-service -o json +++++ jq '.status.loadBalancer.ingress[].ip' +++++ sed -e 's/^"//; s/"$//;' ++++++ mktemp +++++ local LAST_OUT=/tmp/tmp.WWwrtx5P81 ++++++ mktemp +++++ local LAST_ERR=/tmp/tmp.uxdCfK3Tp4 +++++ local exit_status=0 +++++ local timeout=4 ++++++ seq 0 2 +++++ for i in $(seq 0 2) +++++ set +e +++++ kubectl get service/monitoring-service -o json +++++ exit_status=0 +++++ set -e +++++ '[' 0 '!=' 0 -a -n 1 ']' +++++ break +++++ cat /tmp/tmp.WWwrtx5P81 +++++ cat /tmp/tmp.uxdCfK3Tp4 +++++ rm /tmp/tmp.WWwrtx5P81 /tmp/tmp.uxdCfK3Tp4 +++++ return 0 ++++ local ip=136.115.177.16 ++++ '[' -n 136.115.177.16 -a 136.115.177.16 '!=' null ']' ++++ echo 136.115.177.16 ++++ return +++ curl --insecure -s -X POST -H 'Content-Type: application/json' -d '{"name":"operator_new"}' --user 'admin:(0%0rKB/#vj#2ICw' https://136.115.177.16/graph/api/serviceaccounts/3/tokens -w '\n%{http_code}' ++ token_response=$'{"id":2,"name":"operator_new","key":"glsa_qNTRaLDY4afgDqigeAKU7XgNctvWxJOv_72a0b5bd"}\n200' +++ echo $'{"id":2,"name":"operator_new","key":"glsa_qNTRaLDY4afgDqigeAKU7XgNctvWxJOv_72a0b5bd"}\n200' +++ tail -n1 ++ token_status_code=200 +++ echo $'{"id":2,"name":"operator_new","key":"glsa_qNTRaLDY4afgDqigeAKU7XgNctvWxJOv_72a0b5bd"}\n200' +++ sed '$ d' ++ token_json_response='{"id":2,"name":"operator_new","key":"glsa_qNTRaLDY4afgDqigeAKU7XgNctvWxJOv_72a0b5bd"}' ++ [[ 200 -ne 200 ]] ++ echo '{"id":2,"name":"operator_new","key":"glsa_qNTRaLDY4afgDqigeAKU7XgNctvWxJOv_72a0b5bd"}' ++ jq -r .key + NEW_TOKEN=glsa_qNTRaLDY4afgDqigeAKU7XgNctvWxJOv_72a0b5bd + kubectl_bin patch secret some-users --type merge --patch '{"stringData": {"PMM_SERVER_TOKEN": "glsa_qNTRaLDY4afgDqigeAKU7XgNctvWxJOv_72a0b5bd"}}' ++ mktemp + local LAST_OUT=/tmp/tmp.EgsGTjALZB ++ mktemp + local LAST_ERR=/tmp/tmp.xp8CyH2q4e + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl patch secret some-users --type merge --patch '{"stringData": {"PMM_SERVER_TOKEN": "glsa_qNTRaLDY4afgDqigeAKU7XgNctvWxJOv_72a0b5bd"}}' + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.EgsGTjALZB secret/some-users patched + cat /tmp/tmp.xp8CyH2q4e + rm /tmp/tmp.EgsGTjALZB /tmp/tmp.xp8CyH2q4e + return 0 + desc 'delete old PMM token' + set +o xtrace ----------------------------------------------------------------------------------- delete old PMM token ----------------------------------------------------------------------------------- + delete_pmm_server_token operator + local key_name=operator + [[ -z operator ]] + local ADMIN_PASSWORD ++ kubectl get secret pmm-secret -o 'jsonpath={.data.PMM_ADMIN_PASSWORD}' ++ base64 --decode + ADMIN_PASSWORD='(0%0rKB/#vj#2ICw' + [[ -z (0%0rKB/#vj#2ICw ]] + local 'user_credentials=admin:(0%0rKB/#vj#2ICw' + local service_accounts_response service_accounts_status +++ get_service_endpoint monitoring-service +++ local service=monitoring-service ++++ kubectl_bin get service/monitoring-service -o json ++++ jq '.status.loadBalancer.ingress[].hostname' ++++ sed -e 's/^"//; s/"$//;' +++++ mktemp ++++ local LAST_OUT=/tmp/tmp.PWqfHDpqxw +++++ mktemp ++++ local LAST_ERR=/tmp/tmp.qGvzukrgbP ++++ local exit_status=0 ++++ local timeout=4 +++++ seq 0 2 ++++ for i in $(seq 0 2) ++++ set +e ++++ kubectl get service/monitoring-service -o json ++++ exit_status=0 ++++ set -e ++++ '[' 0 '!=' 0 -a -n 1 ']' ++++ break ++++ cat /tmp/tmp.PWqfHDpqxw ++++ cat /tmp/tmp.qGvzukrgbP ++++ rm /tmp/tmp.PWqfHDpqxw /tmp/tmp.qGvzukrgbP ++++ return 0 +++ local hostname=null +++ '[' -n null -a null '!=' null ']' ++++ kubectl_bin get service/monitoring-service -o json ++++ jq '.status.loadBalancer.ingress[].ip' ++++ sed -e 's/^"//; s/"$//;' +++++ mktemp ++++ local LAST_OUT=/tmp/tmp.NjpZiiVyNy +++++ mktemp ++++ local LAST_ERR=/tmp/tmp.eSXP5Nlsvi ++++ local exit_status=0 ++++ local timeout=4 +++++ seq 0 2 ++++ for i in $(seq 0 2) ++++ set +e ++++ kubectl get service/monitoring-service -o json ++++ exit_status=0 ++++ set -e ++++ '[' 0 '!=' 0 -a -n 1 ']' ++++ break ++++ cat /tmp/tmp.NjpZiiVyNy ++++ cat /tmp/tmp.eSXP5Nlsvi ++++ rm /tmp/tmp.NjpZiiVyNy /tmp/tmp.eSXP5Nlsvi ++++ return 0 +++ local ip=136.115.177.16 +++ '[' -n 136.115.177.16 -a 136.115.177.16 '!=' null ']' +++ echo 136.115.177.16 +++ return ++ curl --insecure -s -X GET --user 'admin:(0%0rKB/#vj#2ICw' https://136.115.177.16/graph/api/serviceaccounts/search -w '\n%{http_code}' + service_accounts_response=$'{"totalCount":2,"serviceAccounts":[{"id":2,"uid":"cfolp5guen8cgb","name":"operator","login":"sa-1-operator","orgId":1,"isDisabled":false,"role":"Admin","tokens":1,"avatarUrl":"/graph/avatar/f741e7abdd8def4ed30a37f602429df3"},{"id":3,"uid":"afolprwd1xuyoe","name":"operator_new","login":"sa-1-operator_new","orgId":1,"isDisabled":false,"role":"Admin","tokens":1,"avatarUrl":"/graph/avatar/6d1b2db67a8f3e06db4ae076406fece5"}],"page":1,"perPage":1000}\n200' ++ echo $'{"totalCount":2,"serviceAccounts":[{"id":2,"uid":"cfolp5guen8cgb","name":"operator","login":"sa-1-operator","orgId":1,"isDisabled":false,"role":"Admin","tokens":1,"avatarUrl":"/graph/avatar/f741e7abdd8def4ed30a37f602429df3"},{"id":3,"uid":"afolprwd1xuyoe","name":"operator_new","login":"sa-1-operator_new","orgId":1,"isDisabled":false,"role":"Admin","tokens":1,"avatarUrl":"/graph/avatar/6d1b2db67a8f3e06db4ae076406fece5"}],"page":1,"perPage":1000}\n200' ++ tail -n1 + service_accounts_status=200 ++ echo $'{"totalCount":2,"serviceAccounts":[{"id":2,"uid":"cfolp5guen8cgb","name":"operator","login":"sa-1-operator","orgId":1,"isDisabled":false,"role":"Admin","tokens":1,"avatarUrl":"/graph/avatar/f741e7abdd8def4ed30a37f602429df3"},{"id":3,"uid":"afolprwd1xuyoe","name":"operator_new","login":"sa-1-operator_new","orgId":1,"isDisabled":false,"role":"Admin","tokens":1,"avatarUrl":"/graph/avatar/6d1b2db67a8f3e06db4ae076406fece5"}],"page":1,"perPage":1000}\n200' ++ sed '$ d' + service_accounts_json='{"totalCount":2,"serviceAccounts":[{"id":2,"uid":"cfolp5guen8cgb","name":"operator","login":"sa-1-operator","orgId":1,"isDisabled":false,"role":"Admin","tokens":1,"avatarUrl":"/graph/avatar/f741e7abdd8def4ed30a37f602429df3"},{"id":3,"uid":"afolprwd1xuyoe","name":"operator_new","login":"sa-1-operator_new","orgId":1,"isDisabled":false,"role":"Admin","tokens":1,"avatarUrl":"/graph/avatar/6d1b2db67a8f3e06db4ae076406fece5"}],"page":1,"perPage":1000}' + [[ 200 -ne 200 ]] + local service_account_id ++ echo '{"totalCount":2,"serviceAccounts":[{"id":2,"uid":"cfolp5guen8cgb","name":"operator","login":"sa-1-operator","orgId":1,"isDisabled":false,"role":"Admin","tokens":1,"avatarUrl":"/graph/avatar/f741e7abdd8def4ed30a37f602429df3"},{"id":3,"uid":"afolprwd1xuyoe","name":"operator_new","login":"sa-1-operator_new","orgId":1,"isDisabled":false,"role":"Admin","tokens":1,"avatarUrl":"/graph/avatar/6d1b2db67a8f3e06db4ae076406fece5"}],"page":1,"perPage":1000}' ++ jq -r '.serviceAccounts[] | select(.name == "operator").id' + service_account_id=2 + [[ -z 2 ]] + [[ 2 == null ]] + local tokens_response tokens_status tokens_json +++ get_service_endpoint monitoring-service +++ local service=monitoring-service ++++ kubectl_bin get service/monitoring-service -o json ++++ jq '.status.loadBalancer.ingress[].hostname' ++++ sed -e 's/^"//; s/"$//;' +++++ mktemp ++++ local LAST_OUT=/tmp/tmp.zrwjziqds2 +++++ mktemp ++++ local LAST_ERR=/tmp/tmp.61JBk4Qh0K ++++ local exit_status=0 ++++ local timeout=4 +++++ seq 0 2 ++++ for i in $(seq 0 2) ++++ set +e ++++ kubectl get service/monitoring-service -o json ++++ exit_status=0 ++++ set -e ++++ '[' 0 '!=' 0 -a -n 1 ']' ++++ break ++++ cat /tmp/tmp.zrwjziqds2 ++++ cat /tmp/tmp.61JBk4Qh0K ++++ rm /tmp/tmp.zrwjziqds2 /tmp/tmp.61JBk4Qh0K ++++ return 0 +++ local hostname=null +++ '[' -n null -a null '!=' null ']' ++++ kubectl_bin get service/monitoring-service -o json ++++ jq '.status.loadBalancer.ingress[].ip' ++++ sed -e 's/^"//; s/"$//;' +++++ mktemp ++++ local LAST_OUT=/tmp/tmp.YvSweHnTf5 +++++ mktemp ++++ local LAST_ERR=/tmp/tmp.4dCoO70sqD ++++ local exit_status=0 ++++ local timeout=4 +++++ seq 0 2 ++++ for i in $(seq 0 2) ++++ set +e ++++ kubectl get service/monitoring-service -o json ++++ exit_status=0 ++++ set -e ++++ '[' 0 '!=' 0 -a -n 1 ']' ++++ break ++++ cat /tmp/tmp.YvSweHnTf5 ++++ cat /tmp/tmp.4dCoO70sqD ++++ rm /tmp/tmp.YvSweHnTf5 /tmp/tmp.4dCoO70sqD ++++ return 0 +++ local ip=136.115.177.16 +++ '[' -n 136.115.177.16 -a 136.115.177.16 '!=' null ']' +++ echo 136.115.177.16 +++ return ++ curl --insecure -s -X GET --user 'admin:(0%0rKB/#vj#2ICw' https://136.115.177.16/graph/api/serviceaccounts/2/tokens -w '\n%{http_code}' + tokens_response=$'[{"id":1,"name":"operator","created":"2026-06-09T09:23:04Z","lastUsedAt":"2026-06-09T09:28:52Z","expiration":null,"secondsUntilExpiration":0,"hasExpired":false,"isRevoked":false}]\n200' ++ echo $'[{"id":1,"name":"operator","created":"2026-06-09T09:23:04Z","lastUsedAt":"2026-06-09T09:28:52Z","expiration":null,"secondsUntilExpiration":0,"hasExpired":false,"isRevoked":false}]\n200' ++ tail -n1 + tokens_status=200 ++ echo $'[{"id":1,"name":"operator","created":"2026-06-09T09:23:04Z","lastUsedAt":"2026-06-09T09:28:52Z","expiration":null,"secondsUntilExpiration":0,"hasExpired":false,"isRevoked":false}]\n200' ++ sed '$ d' + tokens_json='[{"id":1,"name":"operator","created":"2026-06-09T09:23:04Z","lastUsedAt":"2026-06-09T09:28:52Z","expiration":null,"secondsUntilExpiration":0,"hasExpired":false,"isRevoked":false}]' + [[ 200 -ne 200 ]] + local token_id ++ echo '[{"id":1,"name":"operator","created":"2026-06-09T09:23:04Z","lastUsedAt":"2026-06-09T09:28:52Z","expiration":null,"secondsUntilExpiration":0,"hasExpired":false,"isRevoked":false}]' ++ jq -r '.[] | select(.name == "operator").id' + token_id=1 + [[ -z 1 ]] + [[ 1 == null ]] + local delete_response delete_status +++ get_service_endpoint monitoring-service +++ local service=monitoring-service ++++ kubectl_bin get service/monitoring-service -o json ++++ jq '.status.loadBalancer.ingress[].hostname' ++++ sed -e 's/^"//; s/"$//;' +++++ mktemp ++++ local LAST_OUT=/tmp/tmp.316J8vNGgQ +++++ mktemp ++++ local LAST_ERR=/tmp/tmp.Bi6kl9mMyF ++++ local exit_status=0 ++++ local timeout=4 +++++ seq 0 2 ++++ for i in $(seq 0 2) ++++ set +e ++++ kubectl get service/monitoring-service -o json ++++ exit_status=0 ++++ set -e ++++ '[' 0 '!=' 0 -a -n 1 ']' ++++ break ++++ cat /tmp/tmp.316J8vNGgQ ++++ cat /tmp/tmp.Bi6kl9mMyF ++++ rm /tmp/tmp.316J8vNGgQ /tmp/tmp.Bi6kl9mMyF ++++ return 0 +++ local hostname=null +++ '[' -n null -a null '!=' null ']' ++++ kubectl_bin get service/monitoring-service -o json ++++ jq '.status.loadBalancer.ingress[].ip' ++++ sed -e 's/^"//; s/"$//;' +++++ mktemp ++++ local LAST_OUT=/tmp/tmp.e4agswRgXo +++++ mktemp ++++ local LAST_ERR=/tmp/tmp.VBdm9ZQArP ++++ local exit_status=0 ++++ local timeout=4 +++++ seq 0 2 ++++ for i in $(seq 0 2) ++++ set +e ++++ kubectl get service/monitoring-service -o json ++++ exit_status=0 ++++ set -e ++++ '[' 0 '!=' 0 -a -n 1 ']' ++++ break ++++ cat /tmp/tmp.e4agswRgXo ++++ cat /tmp/tmp.VBdm9ZQArP ++++ rm /tmp/tmp.e4agswRgXo /tmp/tmp.VBdm9ZQArP ++++ return 0 +++ local ip=136.115.177.16 +++ '[' -n 136.115.177.16 -a 136.115.177.16 '!=' null ']' +++ echo 136.115.177.16 +++ return ++ curl --insecure -s -X DELETE --user 'admin:(0%0rKB/#vj#2ICw' https://136.115.177.16/graph/api/serviceaccounts/2/tokens/1 -w '\n%{http_code}' + delete_response=$'{"message":"Service account token deleted"}\n200' ++ echo $'{"message":"Service account token deleted"}\n200' ++ tail -n1 + delete_status=200 + [[ 200 -ne 200 ]] + desc 'check for authentication errors in PMM client logs' + set +o xtrace ----------------------------------------------------------------------------------- check for authentication errors in PMM client logs ----------------------------------------------------------------------------------- ++ kubectl get pods --selector=app.kubernetes.io/replset=rs0 -o 'jsonpath={.items[*].metadata.name}' + pods='monitoring-pmm3-rs0-0 monitoring-pmm3-rs0-1 monitoring-pmm3-rs0-2' + for pod in $pods + for i in {1..3} + kubectl logs monitoring-pmm3-rs0-0 pmm-client + grep -q 'Invalid username or password' + sleep 2 + for i in {1..3} + kubectl logs monitoring-pmm3-rs0-0 pmm-client + grep -q 'Invalid username or password' + sleep 2 + for i in {1..3} + kubectl logs monitoring-pmm3-rs0-0 pmm-client + grep -q 'Invalid username or password' + sleep 2 + for pod in $pods + for i in {1..3} + kubectl logs monitoring-pmm3-rs0-1 pmm-client + grep -q 'Invalid username or password' + sleep 2 + for i in {1..3} + kubectl logs monitoring-pmm3-rs0-1 pmm-client + grep -q 'Invalid username or password' + sleep 2 + for i in {1..3} + kubectl logs monitoring-pmm3-rs0-1 pmm-client + grep -q 'Invalid username or password' + sleep 2 + for pod in $pods + for i in {1..3} + kubectl logs monitoring-pmm3-rs0-2 pmm-client + grep -q 'Invalid username or password' + sleep 2 + for i in {1..3} + kubectl logs monitoring-pmm3-rs0-2 pmm-client + grep -q 'Invalid username or password' + sleep 2 + for i in {1..3} + kubectl logs monitoring-pmm3-rs0-2 pmm-client + grep -q 'Invalid username or password' + sleep 2 + desc 'check mongod metrics' + set +o xtrace ----------------------------------------------------------------------------------- check mongod metrics ----------------------------------------------------------------------------------- + get_metric_values node_boot_time_seconds monitoring-pmm3-6334-monitoring-pmm3-rs0-1 glsa_qNTRaLDY4afgDqigeAKU7XgNctvWxJOv_72a0b5bd + local metric=node_boot_time_seconds + local instance=monitoring-pmm3-6334-monitoring-pmm3-rs0-1 + local token=glsa_qNTRaLDY4afgDqigeAKU7XgNctvWxJOv_72a0b5bd ++ /usr/sbin/date -u +%s -d '-1 minute' + local start=1780997384 ++ /usr/sbin/date -u +%s + local end=1780997444 ++ get_service_endpoint monitoring-service ++ local service=monitoring-service +++ kubectl_bin get service/monitoring-service -o json +++ jq '.status.loadBalancer.ingress[].hostname' +++ sed -e 's/^"//; s/"$//;' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.AUMACp9jwT ++++ mktemp +++ local LAST_ERR=/tmp/tmp.815QQG0Mrt +++ local exit_status=0 +++ local timeout=4 ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl get service/monitoring-service -o json +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 -a -n 1 ']' +++ break +++ cat /tmp/tmp.AUMACp9jwT +++ cat /tmp/tmp.815QQG0Mrt +++ rm /tmp/tmp.AUMACp9jwT /tmp/tmp.815QQG0Mrt +++ return 0 ++ local hostname=null ++ '[' -n null -a null '!=' null ']' +++ kubectl_bin get service/monitoring-service -o json +++ jq '.status.loadBalancer.ingress[].ip' +++ sed -e 's/^"//; s/"$//;' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.754IKLeYRU ++++ mktemp +++ local LAST_ERR=/tmp/tmp.nQcoL0pmo3 +++ local exit_status=0 +++ local timeout=4 ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl get service/monitoring-service -o json +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 -a -n 1 ']' +++ break +++ cat /tmp/tmp.754IKLeYRU +++ cat /tmp/tmp.nQcoL0pmo3 +++ rm /tmp/tmp.754IKLeYRU /tmp/tmp.nQcoL0pmo3 +++ return 0 ++ local ip=136.115.177.16 ++ '[' -n 136.115.177.16 -a 136.115.177.16 '!=' null ']' ++ echo 136.115.177.16 ++ return + local endpoint=136.115.177.16 + '[' -z node_boot_time_seconds ']' + '[' -z glsa_qNTRaLDY4afgDqigeAKU7XgNctvWxJOv_72a0b5bd ']' + local wait_count=30 + local retry=0 ++ curl -s -k -H 'Authorization: Bearer glsa_qNTRaLDY4afgDqigeAKU7XgNctvWxJOv_72a0b5bd' 'https://136.115.177.16/graph/api/datasources/proxy/1/api/v1/query_range?query=min%28node_boot_time_seconds%7Bnode_name%3D%7E%22monitoring-pmm3-6334-monitoring-pmm3-rs0-1%22%7d%20or%20node_boot_time_seconds%7Bnode_name%3D%7E%22monitoring-pmm3-6334-monitoring-pmm3-rs0-1%22%7D%29&start=1780997384&end=1780997444&step=60' ++ jq '.data.result[0].values[][1]' ++ grep '^"[0-9]' + [[ -n "1780990196" "1780990196" ]] + get_metric_values mongodb_connections monitoring-pmm3-6334-monitoring-pmm3-rs0-1 glsa_qNTRaLDY4afgDqigeAKU7XgNctvWxJOv_72a0b5bd + local metric=mongodb_connections + local instance=monitoring-pmm3-6334-monitoring-pmm3-rs0-1 + local token=glsa_qNTRaLDY4afgDqigeAKU7XgNctvWxJOv_72a0b5bd ++ /usr/sbin/date -u +%s -d '-1 minute' + local start=1780997386 ++ /usr/sbin/date -u +%s + local end=1780997446 ++ get_service_endpoint monitoring-service ++ local service=monitoring-service +++ kubectl_bin get service/monitoring-service -o json +++ jq '.status.loadBalancer.ingress[].hostname' +++ sed -e 's/^"//; s/"$//;' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.bfp1eCCc6a ++++ mktemp +++ local LAST_ERR=/tmp/tmp.xXc49faPXz +++ local exit_status=0 +++ local timeout=4 ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl get service/monitoring-service -o json +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 -a -n 1 ']' +++ break +++ cat /tmp/tmp.bfp1eCCc6a +++ cat /tmp/tmp.xXc49faPXz +++ rm /tmp/tmp.bfp1eCCc6a /tmp/tmp.xXc49faPXz +++ return 0 ++ local hostname=null ++ '[' -n null -a null '!=' null ']' +++ kubectl_bin get service/monitoring-service -o json +++ jq '.status.loadBalancer.ingress[].ip' +++ sed -e 's/^"//; s/"$//;' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.GZiBAfT87q ++++ mktemp +++ local LAST_ERR=/tmp/tmp.RrwbLGsnxN +++ local exit_status=0 +++ local timeout=4 ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl get service/monitoring-service -o json +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 -a -n 1 ']' +++ break +++ cat /tmp/tmp.GZiBAfT87q +++ cat /tmp/tmp.RrwbLGsnxN +++ rm /tmp/tmp.GZiBAfT87q /tmp/tmp.RrwbLGsnxN +++ return 0 ++ local ip=136.115.177.16 ++ '[' -n 136.115.177.16 -a 136.115.177.16 '!=' null ']' ++ echo 136.115.177.16 ++ return + local endpoint=136.115.177.16 + '[' -z mongodb_connections ']' + '[' -z glsa_qNTRaLDY4afgDqigeAKU7XgNctvWxJOv_72a0b5bd ']' + local wait_count=30 + local retry=0 ++ curl -s -k -H 'Authorization: Bearer glsa_qNTRaLDY4afgDqigeAKU7XgNctvWxJOv_72a0b5bd' 'https://136.115.177.16/graph/api/datasources/proxy/1/api/v1/query_range?query=min%28mongodb_connections%7Bnode_name%3D%7E%22monitoring-pmm3-6334-monitoring-pmm3-rs0-1%22%7d%20or%20mongodb_connections%7Bnode_name%3D%7E%22monitoring-pmm3-6334-monitoring-pmm3-rs0-1%22%7D%29&start=1780997386&end=1780997446&step=60' ++ jq '.data.result[0].values[][1]' ++ grep '^"[0-9]' + [[ -n "0" "0" ]] + desc 'check mongo config metrics' + set +o xtrace ----------------------------------------------------------------------------------- check mongo config metrics ----------------------------------------------------------------------------------- + get_metric_values node_boot_time_seconds monitoring-pmm3-6334-monitoring-pmm3-cfg-1 glsa_qNTRaLDY4afgDqigeAKU7XgNctvWxJOv_72a0b5bd + local metric=node_boot_time_seconds + local instance=monitoring-pmm3-6334-monitoring-pmm3-cfg-1 + local token=glsa_qNTRaLDY4afgDqigeAKU7XgNctvWxJOv_72a0b5bd ++ /usr/sbin/date -u +%s -d '-1 minute' + local start=1780997389 ++ /usr/sbin/date -u +%s + local end=1780997449 ++ get_service_endpoint monitoring-service ++ local service=monitoring-service +++ kubectl_bin get service/monitoring-service -o json +++ jq '.status.loadBalancer.ingress[].hostname' +++ sed -e 's/^"//; s/"$//;' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.9Tf5N4xcuk ++++ mktemp +++ local LAST_ERR=/tmp/tmp.gbCaN4eSZ8 +++ local exit_status=0 +++ local timeout=4 ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl get service/monitoring-service -o json +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 -a -n 1 ']' +++ break +++ cat /tmp/tmp.9Tf5N4xcuk +++ cat /tmp/tmp.gbCaN4eSZ8 +++ rm /tmp/tmp.9Tf5N4xcuk /tmp/tmp.gbCaN4eSZ8 +++ return 0 ++ local hostname=null ++ '[' -n null -a null '!=' null ']' +++ kubectl_bin get service/monitoring-service -o json +++ jq '.status.loadBalancer.ingress[].ip' +++ sed -e 's/^"//; s/"$//;' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.LXrCrRrcYO ++++ mktemp +++ local LAST_ERR=/tmp/tmp.qtYK8uBQpO +++ local exit_status=0 +++ local timeout=4 ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl get service/monitoring-service -o json +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 -a -n 1 ']' +++ break +++ cat /tmp/tmp.LXrCrRrcYO +++ cat /tmp/tmp.qtYK8uBQpO +++ rm /tmp/tmp.LXrCrRrcYO /tmp/tmp.qtYK8uBQpO +++ return 0 ++ local ip=136.115.177.16 ++ '[' -n 136.115.177.16 -a 136.115.177.16 '!=' null ']' ++ echo 136.115.177.16 ++ return + local endpoint=136.115.177.16 + '[' -z node_boot_time_seconds ']' + '[' -z glsa_qNTRaLDY4afgDqigeAKU7XgNctvWxJOv_72a0b5bd ']' + local wait_count=30 + local retry=0 ++ curl -s -k -H 'Authorization: Bearer glsa_qNTRaLDY4afgDqigeAKU7XgNctvWxJOv_72a0b5bd' 'https://136.115.177.16/graph/api/datasources/proxy/1/api/v1/query_range?query=min%28node_boot_time_seconds%7Bnode_name%3D%7E%22monitoring-pmm3-6334-monitoring-pmm3-cfg-1%22%7d%20or%20node_boot_time_seconds%7Bnode_name%3D%7E%22monitoring-pmm3-6334-monitoring-pmm3-cfg-1%22%7D%29&start=1780997389&end=1780997449&step=60' ++ jq '.data.result[0].values[][1]' ++ grep '^"[0-9]' + [[ -n "1780990196" "1780990196" ]] + get_metric_values mongodb_connections monitoring-pmm3-6334-monitoring-pmm3-cfg-1 glsa_qNTRaLDY4afgDqigeAKU7XgNctvWxJOv_72a0b5bd + local metric=mongodb_connections + local instance=monitoring-pmm3-6334-monitoring-pmm3-cfg-1 + local token=glsa_qNTRaLDY4afgDqigeAKU7XgNctvWxJOv_72a0b5bd ++ /usr/sbin/date -u +%s -d '-1 minute' + local start=1780997392 ++ /usr/sbin/date -u +%s + local end=1780997452 ++ get_service_endpoint monitoring-service ++ local service=monitoring-service +++ kubectl_bin get service/monitoring-service -o json +++ jq '.status.loadBalancer.ingress[].hostname' +++ sed -e 's/^"//; s/"$//;' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.UVhrnutjgT ++++ mktemp +++ local LAST_ERR=/tmp/tmp.8GGL7hImL1 +++ local exit_status=0 +++ local timeout=4 ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl get service/monitoring-service -o json +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 -a -n 1 ']' +++ break +++ cat /tmp/tmp.UVhrnutjgT +++ cat /tmp/tmp.8GGL7hImL1 +++ rm /tmp/tmp.UVhrnutjgT /tmp/tmp.8GGL7hImL1 +++ return 0 ++ local hostname=null ++ '[' -n null -a null '!=' null ']' +++ kubectl_bin get service/monitoring-service -o json +++ jq '.status.loadBalancer.ingress[].ip' +++ sed -e 's/^"//; s/"$//;' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.TPhszXoEZR ++++ mktemp +++ local LAST_ERR=/tmp/tmp.gd4rQoTm8R +++ local exit_status=0 +++ local timeout=4 ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl get service/monitoring-service -o json +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 -a -n 1 ']' +++ break +++ cat /tmp/tmp.TPhszXoEZR +++ cat /tmp/tmp.gd4rQoTm8R +++ rm /tmp/tmp.TPhszXoEZR /tmp/tmp.gd4rQoTm8R +++ return 0 ++ local ip=136.115.177.16 ++ '[' -n 136.115.177.16 -a 136.115.177.16 '!=' null ']' ++ echo 136.115.177.16 ++ return + local endpoint=136.115.177.16 + '[' -z mongodb_connections ']' + '[' -z glsa_qNTRaLDY4afgDqigeAKU7XgNctvWxJOv_72a0b5bd ']' + local wait_count=30 + local retry=0 ++ curl -s -k -H 'Authorization: Bearer glsa_qNTRaLDY4afgDqigeAKU7XgNctvWxJOv_72a0b5bd' 'https://136.115.177.16/graph/api/datasources/proxy/1/api/v1/query_range?query=min%28mongodb_connections%7Bnode_name%3D%7E%22monitoring-pmm3-6334-monitoring-pmm3-cfg-1%22%7d%20or%20mongodb_connections%7Bnode_name%3D%7E%22monitoring-pmm3-6334-monitoring-pmm3-cfg-1%22%7D%29&start=1780997392&end=1780997452&step=60' ++ jq '.data.result[0].values[][1]' ++ grep '^"[0-9]' + [[ -n "0" "0" ]] + desc 'check mongos metrics' + set +o xtrace ----------------------------------------------------------------------------------- check mongos metrics ----------------------------------------------------------------------------------- ++ kubectl get pod -l app.kubernetes.io/component=mongos -o 'jsonpath={.items[0].metadata.name}' + MONGOS_POD_NAME=monitoring-pmm3-mongos-0 + get_metric_values node_boot_time_seconds monitoring-pmm3-6334-monitoring-pmm3-mongos-0 glsa_qNTRaLDY4afgDqigeAKU7XgNctvWxJOv_72a0b5bd + local metric=node_boot_time_seconds + local instance=monitoring-pmm3-6334-monitoring-pmm3-mongos-0 + local token=glsa_qNTRaLDY4afgDqigeAKU7XgNctvWxJOv_72a0b5bd ++ /usr/sbin/date -u +%s -d '-1 minute' + local start=1780997396 ++ /usr/sbin/date -u +%s + local end=1780997456 ++ get_service_endpoint monitoring-service ++ local service=monitoring-service +++ kubectl_bin get service/monitoring-service -o json +++ jq '.status.loadBalancer.ingress[].hostname' +++ sed -e 's/^"//; s/"$//;' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.t8wTGBO71X ++++ mktemp +++ local LAST_ERR=/tmp/tmp.sKcqeFd7ZE +++ local exit_status=0 +++ local timeout=4 ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl get service/monitoring-service -o json +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 -a -n 1 ']' +++ break +++ cat /tmp/tmp.t8wTGBO71X +++ cat /tmp/tmp.sKcqeFd7ZE +++ rm /tmp/tmp.t8wTGBO71X /tmp/tmp.sKcqeFd7ZE +++ return 0 ++ local hostname=null ++ '[' -n null -a null '!=' null ']' +++ kubectl_bin get service/monitoring-service -o json +++ jq '.status.loadBalancer.ingress[].ip' +++ sed -e 's/^"//; s/"$//;' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.H7v3T1VRtb ++++ mktemp +++ local LAST_ERR=/tmp/tmp.wUnYlibkeF +++ local exit_status=0 +++ local timeout=4 ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl get service/monitoring-service -o json +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 -a -n 1 ']' +++ break +++ cat /tmp/tmp.H7v3T1VRtb +++ cat /tmp/tmp.wUnYlibkeF +++ rm /tmp/tmp.H7v3T1VRtb /tmp/tmp.wUnYlibkeF +++ return 0 ++ local ip=136.115.177.16 ++ '[' -n 136.115.177.16 -a 136.115.177.16 '!=' null ']' ++ echo 136.115.177.16 ++ return + local endpoint=136.115.177.16 + '[' -z node_boot_time_seconds ']' + '[' -z glsa_qNTRaLDY4afgDqigeAKU7XgNctvWxJOv_72a0b5bd ']' + local wait_count=30 + local retry=0 ++ jq '.data.result[0].values[][1]' ++ curl -s -k -H 'Authorization: Bearer glsa_qNTRaLDY4afgDqigeAKU7XgNctvWxJOv_72a0b5bd' 'https://136.115.177.16/graph/api/datasources/proxy/1/api/v1/query_range?query=min%28node_boot_time_seconds%7Bnode_name%3D%7E%22monitoring-pmm3-6334-monitoring-pmm3-mongos-0%22%7d%20or%20node_boot_time_seconds%7Bnode_name%3D%7E%22monitoring-pmm3-6334-monitoring-pmm3-mongos-0%22%7D%29&start=1780997396&end=1780997456&step=60' ++ grep '^"[0-9]' + [[ -n "1780993352" "1780993352" ]] + sleep 90 + desc 'check QAN data' + set +o xtrace ----------------------------------------------------------------------------------- check QAN data ----------------------------------------------------------------------------------- + get_qan_values mongodb dev-mongod glsa_qNTRaLDY4afgDqigeAKU7XgNctvWxJOv_72a0b5bd + local service_type=mongodb + local environment=dev-mongod + local token=glsa_qNTRaLDY4afgDqigeAKU7XgNctvWxJOv_72a0b5bd + local start + local end + local endpoint ++ /usr/sbin/date -u -d '-12 hour' +%Y-%m-%dT%H:%M:%S%:z + start=2026-06-08T21:32:27+00:00 ++ /usr/sbin/date -u +%Y-%m-%dT%H:%M:%S%:z + end=2026-06-09T09:32:27+00:00 ++ get_service_endpoint monitoring-service ++ local service=monitoring-service +++ kubectl_bin get service/monitoring-service -o json +++ jq '.status.loadBalancer.ingress[].hostname' +++ sed -e 's/^"//; s/"$//;' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.RwaIaQSrst ++++ mktemp +++ local LAST_ERR=/tmp/tmp.XSCDOS9nxJ +++ local exit_status=0 +++ local timeout=4 ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl get service/monitoring-service -o json +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 -a -n 1 ']' +++ break +++ cat /tmp/tmp.RwaIaQSrst +++ cat /tmp/tmp.XSCDOS9nxJ +++ rm /tmp/tmp.RwaIaQSrst /tmp/tmp.XSCDOS9nxJ +++ return 0 ++ local hostname=null ++ '[' -n null -a null '!=' null ']' +++ kubectl_bin get service/monitoring-service -o json +++ jq '.status.loadBalancer.ingress[].ip' +++ sed -e 's/^"//; s/"$//;' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.peq1XMq674 ++++ mktemp +++ local LAST_ERR=/tmp/tmp.Wi1hgoD7t9 +++ local exit_status=0 +++ local timeout=4 ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl get service/monitoring-service -o json +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 -a -n 1 ']' +++ break +++ cat /tmp/tmp.peq1XMq674 +++ cat /tmp/tmp.Wi1hgoD7t9 +++ rm /tmp/tmp.peq1XMq674 /tmp/tmp.Wi1hgoD7t9 +++ return 0 ++ local ip=136.115.177.16 ++ '[' -n 136.115.177.16 -a 136.115.177.16 '!=' null ']' ++ echo 136.115.177.16 ++ return + endpoint=136.115.177.16 + cat + local response + local wait_count=30 + local retry=0 ++ curl -s -k -H 'Authorization: Bearer glsa_qNTRaLDY4afgDqigeAKU7XgNctvWxJOv_72a0b5bd' -XPOST -d @payload.json https://136.115.177.16/v1/qan/metrics:getReport ++ jq '.rows[].fingerprint' + [[ -n "TOTAL" "db.runCommand({\"$audit\":{\"$impersonatedRoles\":[{\"db\":\"?\",\"role\":\"?\"},{\"db\":\"?\",\"role\":\"?\"},{\"db\":\"?\",\"role\":\"?\"},{\"db\":\"?\",\"role\":\"?\"}],\"$impersonatedUser\":{\"db\":\"?\",\"user\":\"?\"}},\"$client\":{\"application\":{\"name\":\"?\"},\"driver\":{\"name\":\"?\",\"version\":\"?\"},\"mongos\":{\"client\":\"?\",\"host\":\"?\",\"version\":\"?\"},\"os\":{\"architecture\":\"?\",\"type\":\"?\"},\"platform\":\"?\"},\"$configTime\":\"?\",\"$topologyTime\":\"?\",\"dbStats\":\"?\",\"mayBypassWriteBlocking\":\"?\",\"readConcern\":{\"level\":\"?\",\"provenance\":\"?\"},\"scale\":\"?\"})" "db.runCommand({\"$configTime\":\"?\",\"$topologyTime\":\"?\",\"batchSize\":\"?\",\"collection\":\"?\",\"getMore\":\"?\",\"maxTimeMS\":\"?\",\"term\":\"?\"})" "db.runCommand({\"usersInfo\":\"?\"})" "db.system.sessions.update({\"_id\":{\"id\":\"?\",\"uid\":\"?\"}}, [{\"$set\":{\"lastUse\":\"?\"}}], {\"upsert\":true})" "db.system.sessions.deleteOne({\"_id\":{\"id\":\"?\",\"uid\":\"?\"}})" "db.runCommand({\"$audit\":{\"$impersonatedRoles\":[{\"db\":\"?\",\"role\":\"?\"},{\"db\":\"?\",\"role\":\"?\"},{\"db\":\"?\",\"role\":\"?\"},{\"db\":\"?\",\"role\":\"?\"}],\"$impersonatedUser\":{\"db\":\"?\",\"user\":\"?\"}},\"$client\":{\"application\":{\"name\":\"?\"},\"driver\":{\"name\":\"?\",\"version\":\"?\"},\"mongos\":{\"client\":\"?\",\"host\":\"?\",\"version\":\"?\"},\"os\":{\"architecture\":\"?\",\"type\":\"?\"},\"platform\":\"?\"},\"$configTime\":\"?\",\"$topologyTime\":\"?\",\"dbStats\":\"?\",\"mayBypassWriteBlocking\":\"?\",\"readConcern\":{\"level\":\"?\",\"provenance\":\"?\"},\"scale\":\"?\"})" "db.system.sessions.update({\"_id\":{\"id\":\"?\",\"uid\":\"?\"}}, [{\"$set\":{\"lastUse\":\"?\"}},{\"$set\":{\"user\":{\"name\":\"?\"}}}], {\"upsert\":true})" ]] + rm -f payload.json + get_qan_values mongodb dev-mongos glsa_qNTRaLDY4afgDqigeAKU7XgNctvWxJOv_72a0b5bd + local service_type=mongodb + local environment=dev-mongos + local token=glsa_qNTRaLDY4afgDqigeAKU7XgNctvWxJOv_72a0b5bd + local start + local end + local endpoint ++ /usr/sbin/date -u -d '-12 hour' +%Y-%m-%dT%H:%M:%S%:z + start=2026-06-08T21:32:30+00:00 ++ /usr/sbin/date -u +%Y-%m-%dT%H:%M:%S%:z + end=2026-06-09T09:32:30+00:00 ++ get_service_endpoint monitoring-service ++ local service=monitoring-service +++ kubectl_bin get service/monitoring-service -o json +++ jq '.status.loadBalancer.ingress[].hostname' +++ sed -e 's/^"//; s/"$//;' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.pcVZgmDziP ++++ mktemp +++ local LAST_ERR=/tmp/tmp.Wi9kUwwX94 +++ local exit_status=0 +++ local timeout=4 ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl get service/monitoring-service -o json +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 -a -n 1 ']' +++ break +++ cat /tmp/tmp.pcVZgmDziP +++ cat /tmp/tmp.Wi9kUwwX94 +++ rm /tmp/tmp.pcVZgmDziP /tmp/tmp.Wi9kUwwX94 +++ return 0 ++ local hostname=null ++ '[' -n null -a null '!=' null ']' +++ kubectl_bin get service/monitoring-service -o json +++ jq '.status.loadBalancer.ingress[].ip' +++ sed -e 's/^"//; s/"$//;' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.LjkXs5k2Rb ++++ mktemp +++ local LAST_ERR=/tmp/tmp.vddHOvG41m +++ local exit_status=0 +++ local timeout=4 ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl get service/monitoring-service -o json +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 -a -n 1 ']' +++ break +++ cat /tmp/tmp.LjkXs5k2Rb +++ cat /tmp/tmp.vddHOvG41m +++ rm /tmp/tmp.LjkXs5k2Rb /tmp/tmp.vddHOvG41m +++ return 0 ++ local ip=136.115.177.16 ++ '[' -n 136.115.177.16 -a 136.115.177.16 '!=' null ']' ++ echo 136.115.177.16 ++ return + endpoint=136.115.177.16 + cat + local response + local wait_count=30 + local retry=0 ++ curl -s -k -H 'Authorization: Bearer glsa_qNTRaLDY4afgDqigeAKU7XgNctvWxJOv_72a0b5bd' -XPOST -d @payload.json https://136.115.177.16/v1/qan/metrics:getReport ++ jq '.rows[].fingerprint' + [[ -n null ]] + rm -f payload.json + desc 'verify that the custom cluster name is configured' + set +o xtrace ----------------------------------------------------------------------------------- verify that the custom cluster name is configured ----------------------------------------------------------------------------------- + verify_custom_cluster_name super-custom glsa_qNTRaLDY4afgDqigeAKU7XgNctvWxJOv_72a0b5bd monitoring-pmm3-6334-monitoring-pmm3-mongos-0 monitoring-pmm3-6334-monitoring-pmm3-cfg-0 monitoring-pmm3-6334-monitoring-pmm3-rs0-0 + local expected_cluster=super-custom + local token=glsa_qNTRaLDY4afgDqigeAKU7XgNctvWxJOv_72a0b5bd + shift 2 + service_names=('monitoring-pmm3-6334-monitoring-pmm3-mongos-0' 'monitoring-pmm3-6334-monitoring-pmm3-cfg-0' 'monitoring-pmm3-6334-monitoring-pmm3-rs0-0') + local service_names + local endpoint ++ get_service_endpoint monitoring-service ++ local service=monitoring-service +++ kubectl_bin get service/monitoring-service -o json +++ jq '.status.loadBalancer.ingress[].hostname' +++ sed -e 's/^"//; s/"$//;' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.6GY1Q3z7HZ ++++ mktemp +++ local LAST_ERR=/tmp/tmp.WglwYZbIYo +++ local exit_status=0 +++ local timeout=4 ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl get service/monitoring-service -o json +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 -a -n 1 ']' +++ break +++ cat /tmp/tmp.6GY1Q3z7HZ +++ cat /tmp/tmp.WglwYZbIYo +++ rm /tmp/tmp.6GY1Q3z7HZ /tmp/tmp.WglwYZbIYo +++ return 0 ++ local hostname=null ++ '[' -n null -a null '!=' null ']' +++ jq '.status.loadBalancer.ingress[].ip' +++ kubectl_bin get service/monitoring-service -o json +++ sed -e 's/^"//; s/"$//;' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.a1BsHBGTAK ++++ mktemp +++ local LAST_ERR=/tmp/tmp.FmB1zzpxLU +++ local exit_status=0 +++ local timeout=4 ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl get service/monitoring-service -o json +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 -a -n 1 ']' +++ break +++ cat /tmp/tmp.a1BsHBGTAK +++ cat /tmp/tmp.FmB1zzpxLU +++ rm /tmp/tmp.a1BsHBGTAK /tmp/tmp.FmB1zzpxLU +++ return 0 ++ local ip=136.115.177.16 ++ '[' -n 136.115.177.16 -a 136.115.177.16 '!=' null ']' ++ echo 136.115.177.16 ++ return + endpoint=136.115.177.16 + local response ++ curl -s -k -H 'Authorization: Bearer glsa_qNTRaLDY4afgDqigeAKU7XgNctvWxJOv_72a0b5bd' 'https://136.115.177.16/v1/inventory/services?service_type=SERVICE_TYPE_MONGODB_SERVICE' + response=$'{\n "mysql": [],\n "mongodb": [\n {\n "service_id": "30f94b01-7681-4734-a719-c35ed1e45f75",\n "service_name": "monitoring-pmm3-6334-monitoring-pmm3-rs0-1",\n "node_id": "de0c1bd1-0160-4675-b54a-34e0d154d8ba",\n "address": "localhost",\n "port": 27019,\n "socket": "",\n "environment": "dev-mongod",\n "cluster": "super-custom",\n "replication_set": "",\n "custom_labels": {},\n "version": ""\n },\n {\n "service_id": "3e41bf45-b811-4eb3-b730-b46f5ce109d7",\n "service_name": "monitoring-pmm3-6334-monitoring-pmm3-mongos-0",\n "node_id": "bd1c8be8-f68c-4841-a902-35492e16b99f",\n "address": "localhost",\n "port": 27019,\n "socket": "",\n "environment": "dev-mongos",\n "cluster": "super-custom",\n "replication_set": "",\n "custom_labels": {},\n "version": ""\n },\n {\n "service_id": "62a34b56-d50d-47de-bf6f-23074d309d0a",\n "service_name": "monitoring-pmm3-6334-monitoring-pmm3-cfg-2",\n "node_id": "2438b450-3893-49bc-8c44-5b423e7c50e7",\n "address": "localhost",\n "port": 27019,\n "socket": "",\n "environment": "dev-mongod",\n "cluster": "super-custom",\n "replication_set": "",\n "custom_labels": {},\n "version": ""\n },\n {\n "service_id": "6e904044-8a8b-4ae0-8b18-4c646e3c13e1",\n "service_name": "monitoring-pmm3-6334-monitoring-pmm3-cfg-1",\n "node_id": "a433bf0f-bd6c-4624-ac8e-66e66ccfbb22",\n "address": "localhost",\n "port": 27019,\n "socket": "",\n "environment": "dev-mongod",\n "cluster": "super-custom",\n "replication_set": "",\n "custom_labels": {},\n "version": ""\n },\n {\n "service_id": "83cfd331-c56d-4307-96a5-bab3240f0d68",\n "service_name": "monitoring-pmm3-6334-monitoring-pmm3-cfg-0",\n "node_id": "0c227efd-580e-4d7f-8616-937e5c5fe091",\n "address": "localhost",\n "port": 27019,\n "socket": "",\n "environment": "dev-mongod",\n "cluster": "super-custom",\n "replication_set": "",\n "custom_labels": {},\n "version": ""\n },\n {\n "service_id": "a053e6d7-5823-4a54-90ce-32dbba58eb88",\n "service_name": "monitoring-pmm3-6334-monitoring-pmm3-rs0-0",\n "node_id": "cc6efe57-5e10-40bc-b773-d30293aeef21",\n "address": "localhost",\n "port": 27019,\n "socket": "",\n "environment": "dev-mongod",\n "cluster": "super-custom",\n "replication_set": "",\n "custom_labels": {},\n "version": ""\n },\n {\n "service_id": "c5051c19-34a1-4aeb-b606-a6fc668e28a2",\n "service_name": "monitoring-pmm3-6334-monitoring-pmm3-rs0-2",\n "node_id": "d5ea0912-0daa-4f9a-add2-98046719381a",\n "address": "localhost",\n "port": 27019,\n "socket": "",\n "environment": "dev-mongod",\n "cluster": "super-custom",\n "replication_set": "",\n "custom_labels": {},\n "version": ""\n },\n {\n "service_id": "d86e2d4b-9422-4aab-ac56-1ab0b8656385",\n "service_name": "monitoring-pmm3-6334-monitoring-pmm3-mongos-1",\n "node_id": "5f2c700d-929d-4cb4-ad1c-58ed99181690",\n "address": "localhost",\n "port": 27019,\n "socket": "",\n "environment": "dev-mongos",\n "cluster": "super-custom",\n "replication_set": "",\n "custom_labels": {},\n "version": ""\n },\n {\n "service_id": "f694a319-c651-4fb5-97ba-9872785bb6b2",\n "service_name": "monitoring-pmm3-6334-monitoring-pmm3-mongos-2",\n "node_id": "8804cd11-6a01-4e80-80e4-0cdad05e8275",\n "address": "localhost",\n "port": 27019,\n "socket": "",\n "environment": "dev-mongos",\n "cluster": "super-custom",\n "replication_set": "",\n "custom_labels": {},\n "version": ""\n }\n ],\n "postgresql": [],\n "proxysql": [],\n "haproxy": [],\n "external": [],\n "valkey": []\n}' + local verified=0 + for service_name in "${service_names[@]}" + local actual_cluster ++ echo $'{\n "mysql": [],\n "mongodb": [\n {\n "service_id": "30f94b01-7681-4734-a719-c35ed1e45f75",\n "service_name": "monitoring-pmm3-6334-monitoring-pmm3-rs0-1",\n "node_id": "de0c1bd1-0160-4675-b54a-34e0d154d8ba",\n "address": "localhost",\n "port": 27019,\n "socket": "",\n "environment": "dev-mongod",\n "cluster": "super-custom",\n "replication_set": "",\n "custom_labels": {},\n "version": ""\n },\n {\n "service_id": "3e41bf45-b811-4eb3-b730-b46f5ce109d7",\n "service_name": "monitoring-pmm3-6334-monitoring-pmm3-mongos-0",\n "node_id": "bd1c8be8-f68c-4841-a902-35492e16b99f",\n "address": "localhost",\n "port": 27019,\n "socket": "",\n "environment": "dev-mongos",\n "cluster": "super-custom",\n "replication_set": "",\n "custom_labels": {},\n "version": ""\n },\n {\n "service_id": "62a34b56-d50d-47de-bf6f-23074d309d0a",\n "service_name": "monitoring-pmm3-6334-monitoring-pmm3-cfg-2",\n "node_id": "2438b450-3893-49bc-8c44-5b423e7c50e7",\n "address": "localhost",\n "port": 27019,\n "socket": "",\n "environment": "dev-mongod",\n "cluster": "super-custom",\n "replication_set": "",\n "custom_labels": {},\n "version": ""\n },\n {\n "service_id": "6e904044-8a8b-4ae0-8b18-4c646e3c13e1",\n "service_name": "monitoring-pmm3-6334-monitoring-pmm3-cfg-1",\n "node_id": "a433bf0f-bd6c-4624-ac8e-66e66ccfbb22",\n "address": "localhost",\n "port": 27019,\n "socket": "",\n "environment": "dev-mongod",\n "cluster": "super-custom",\n "replication_set": "",\n "custom_labels": {},\n "version": ""\n },\n {\n "service_id": "83cfd331-c56d-4307-96a5-bab3240f0d68",\n "service_name": "monitoring-pmm3-6334-monitoring-pmm3-cfg-0",\n "node_id": "0c227efd-580e-4d7f-8616-937e5c5fe091",\n "address": "localhost",\n "port": 27019,\n "socket": "",\n "environment": "dev-mongod",\n "cluster": "super-custom",\n "replication_set": "",\n "custom_labels": {},\n "version": ""\n },\n {\n "service_id": "a053e6d7-5823-4a54-90ce-32dbba58eb88",\n "service_name": "monitoring-pmm3-6334-monitoring-pmm3-rs0-0",\n "node_id": "cc6efe57-5e10-40bc-b773-d30293aeef21",\n "address": "localhost",\n "port": 27019,\n "socket": "",\n "environment": "dev-mongod",\n "cluster": "super-custom",\n "replication_set": "",\n "custom_labels": {},\n "version": ""\n },\n {\n "service_id": "c5051c19-34a1-4aeb-b606-a6fc668e28a2",\n "service_name": "monitoring-pmm3-6334-monitoring-pmm3-rs0-2",\n "node_id": "d5ea0912-0daa-4f9a-add2-98046719381a",\n "address": "localhost",\n "port": 27019,\n "socket": "",\n "environment": "dev-mongod",\n "cluster": "super-custom",\n "replication_set": "",\n "custom_labels": {},\n "version": ""\n },\n {\n "service_id": "d86e2d4b-9422-4aab-ac56-1ab0b8656385",\n "service_name": "monitoring-pmm3-6334-monitoring-pmm3-mongos-1",\n "node_id": "5f2c700d-929d-4cb4-ad1c-58ed99181690",\n "address": "localhost",\n "port": 27019,\n "socket": "",\n "environment": "dev-mongos",\n "cluster": "super-custom",\n "replication_set": "",\n "custom_labels": {},\n "version": ""\n },\n {\n "service_id": "f694a319-c651-4fb5-97ba-9872785bb6b2",\n "service_name": "monitoring-pmm3-6334-monitoring-pmm3-mongos-2",\n "node_id": "8804cd11-6a01-4e80-80e4-0cdad05e8275",\n "address": "localhost",\n "port": 27019,\n "socket": "",\n "environment": "dev-mongos",\n "cluster": "super-custom",\n "replication_set": "",\n "custom_labels": {},\n "version": ""\n }\n ],\n "postgresql": [],\n "proxysql": [],\n "haproxy": [],\n "external": [],\n "valkey": []\n}' ++ jq -r --arg name monitoring-pmm3-6334-monitoring-pmm3-mongos-0 $'\n\t\t\t.mongodb[] | select(.service_name == $name) | .cluster\n\t\t' + actual_cluster=super-custom + [[ -z super-custom ]] + [[ super-custom == null ]] + [[ super-custom != super\-custom ]] + for service_name in "${service_names[@]}" + local actual_cluster ++ echo $'{\n "mysql": [],\n "mongodb": [\n {\n "service_id": "30f94b01-7681-4734-a719-c35ed1e45f75",\n "service_name": "monitoring-pmm3-6334-monitoring-pmm3-rs0-1",\n "node_id": "de0c1bd1-0160-4675-b54a-34e0d154d8ba",\n "address": "localhost",\n "port": 27019,\n "socket": "",\n "environment": "dev-mongod",\n "cluster": "super-custom",\n "replication_set": "",\n "custom_labels": {},\n "version": ""\n },\n {\n "service_id": "3e41bf45-b811-4eb3-b730-b46f5ce109d7",\n "service_name": "monitoring-pmm3-6334-monitoring-pmm3-mongos-0",\n "node_id": "bd1c8be8-f68c-4841-a902-35492e16b99f",\n "address": "localhost",\n "port": 27019,\n "socket": "",\n "environment": "dev-mongos",\n "cluster": "super-custom",\n "replication_set": "",\n "custom_labels": {},\n "version": ""\n },\n {\n "service_id": "62a34b56-d50d-47de-bf6f-23074d309d0a",\n "service_name": "monitoring-pmm3-6334-monitoring-pmm3-cfg-2",\n "node_id": "2438b450-3893-49bc-8c44-5b423e7c50e7",\n "address": "localhost",\n "port": 27019,\n "socket": "",\n "environment": "dev-mongod",\n "cluster": "super-custom",\n "replication_set": "",\n "custom_labels": {},\n "version": ""\n },\n {\n "service_id": "6e904044-8a8b-4ae0-8b18-4c646e3c13e1",\n "service_name": "monitoring-pmm3-6334-monitoring-pmm3-cfg-1",\n "node_id": "a433bf0f-bd6c-4624-ac8e-66e66ccfbb22",\n "address": "localhost",\n "port": 27019,\n "socket": "",\n "environment": "dev-mongod",\n "cluster": "super-custom",\n "replication_set": "",\n "custom_labels": {},\n "version": ""\n },\n {\n "service_id": "83cfd331-c56d-4307-96a5-bab3240f0d68",\n "service_name": "monitoring-pmm3-6334-monitoring-pmm3-cfg-0",\n "node_id": "0c227efd-580e-4d7f-8616-937e5c5fe091",\n "address": "localhost",\n "port": 27019,\n "socket": "",\n "environment": "dev-mongod",\n "cluster": "super-custom",\n "replication_set": "",\n "custom_labels": {},\n "version": ""\n },\n {\n "service_id": "a053e6d7-5823-4a54-90ce-32dbba58eb88",\n "service_name": "monitoring-pmm3-6334-monitoring-pmm3-rs0-0",\n "node_id": "cc6efe57-5e10-40bc-b773-d30293aeef21",\n "address": "localhost",\n "port": 27019,\n "socket": "",\n "environment": "dev-mongod",\n "cluster": "super-custom",\n "replication_set": "",\n "custom_labels": {},\n "version": ""\n },\n {\n "service_id": "c5051c19-34a1-4aeb-b606-a6fc668e28a2",\n "service_name": "monitoring-pmm3-6334-monitoring-pmm3-rs0-2",\n "node_id": "d5ea0912-0daa-4f9a-add2-98046719381a",\n "address": "localhost",\n "port": 27019,\n "socket": "",\n "environment": "dev-mongod",\n "cluster": "super-custom",\n "replication_set": "",\n "custom_labels": {},\n "version": ""\n },\n {\n "service_id": "d86e2d4b-9422-4aab-ac56-1ab0b8656385",\n "service_name": "monitoring-pmm3-6334-monitoring-pmm3-mongos-1",\n "node_id": "5f2c700d-929d-4cb4-ad1c-58ed99181690",\n "address": "localhost",\n "port": 27019,\n "socket": "",\n "environment": "dev-mongos",\n "cluster": "super-custom",\n "replication_set": "",\n "custom_labels": {},\n "version": ""\n },\n {\n "service_id": "f694a319-c651-4fb5-97ba-9872785bb6b2",\n "service_name": "monitoring-pmm3-6334-monitoring-pmm3-mongos-2",\n "node_id": "8804cd11-6a01-4e80-80e4-0cdad05e8275",\n "address": "localhost",\n "port": 27019,\n "socket": "",\n "environment": "dev-mongos",\n "cluster": "super-custom",\n "replication_set": "",\n "custom_labels": {},\n "version": ""\n }\n ],\n "postgresql": [],\n "proxysql": [],\n "haproxy": [],\n "external": [],\n "valkey": []\n}' ++ jq -r --arg name monitoring-pmm3-6334-monitoring-pmm3-cfg-0 $'\n\t\t\t.mongodb[] | select(.service_name == $name) | .cluster\n\t\t' + actual_cluster=super-custom + [[ -z super-custom ]] + [[ super-custom == null ]] + [[ super-custom != super\-custom ]] + for service_name in "${service_names[@]}" + local actual_cluster ++ echo $'{\n "mysql": [],\n "mongodb": [\n {\n "service_id": "30f94b01-7681-4734-a719-c35ed1e45f75",\n "service_name": "monitoring-pmm3-6334-monitoring-pmm3-rs0-1",\n "node_id": "de0c1bd1-0160-4675-b54a-34e0d154d8ba",\n "address": "localhost",\n "port": 27019,\n "socket": "",\n "environment": "dev-mongod",\n "cluster": "super-custom",\n "replication_set": "",\n "custom_labels": {},\n "version": ""\n },\n {\n "service_id": "3e41bf45-b811-4eb3-b730-b46f5ce109d7",\n "service_name": "monitoring-pmm3-6334-monitoring-pmm3-mongos-0",\n "node_id": "bd1c8be8-f68c-4841-a902-35492e16b99f",\n "address": "localhost",\n "port": 27019,\n "socket": "",\n "environment": "dev-mongos",\n "cluster": "super-custom",\n "replication_set": "",\n "custom_labels": {},\n "version": ""\n },\n {\n "service_id": "62a34b56-d50d-47de-bf6f-23074d309d0a",\n "service_name": "monitoring-pmm3-6334-monitoring-pmm3-cfg-2",\n "node_id": "2438b450-3893-49bc-8c44-5b423e7c50e7",\n "address": "localhost",\n "port": 27019,\n "socket": "",\n "environment": "dev-mongod",\n "cluster": "super-custom",\n "replication_set": "",\n "custom_labels": {},\n "version": ""\n },\n {\n "service_id": "6e904044-8a8b-4ae0-8b18-4c646e3c13e1",\n "service_name": "monitoring-pmm3-6334-monitoring-pmm3-cfg-1",\n "node_id": "a433bf0f-bd6c-4624-ac8e-66e66ccfbb22",\n "address": "localhost",\n "port": 27019,\n "socket": "",\n "environment": "dev-mongod",\n "cluster": "super-custom",\n "replication_set": "",\n "custom_labels": {},\n "version": ""\n },\n {\n "service_id": "83cfd331-c56d-4307-96a5-bab3240f0d68",\n "service_name": "monitoring-pmm3-6334-monitoring-pmm3-cfg-0",\n "node_id": "0c227efd-580e-4d7f-8616-937e5c5fe091",\n "address": "localhost",\n "port": 27019,\n "socket": "",\n "environment": "dev-mongod",\n "cluster": "super-custom",\n "replication_set": "",\n "custom_labels": {},\n "version": ""\n },\n {\n "service_id": "a053e6d7-5823-4a54-90ce-32dbba58eb88",\n "service_name": "monitoring-pmm3-6334-monitoring-pmm3-rs0-0",\n "node_id": "cc6efe57-5e10-40bc-b773-d30293aeef21",\n "address": "localhost",\n "port": 27019,\n "socket": "",\n "environment": "dev-mongod",\n "cluster": "super-custom",\n "replication_set": "",\n "custom_labels": {},\n "version": ""\n },\n {\n "service_id": "c5051c19-34a1-4aeb-b606-a6fc668e28a2",\n "service_name": "monitoring-pmm3-6334-monitoring-pmm3-rs0-2",\n "node_id": "d5ea0912-0daa-4f9a-add2-98046719381a",\n "address": "localhost",\n "port": 27019,\n "socket": "",\n "environment": "dev-mongod",\n "cluster": "super-custom",\n "replication_set": "",\n "custom_labels": {},\n "version": ""\n },\n {\n "service_id": "d86e2d4b-9422-4aab-ac56-1ab0b8656385",\n "service_name": "monitoring-pmm3-6334-monitoring-pmm3-mongos-1",\n "node_id": "5f2c700d-929d-4cb4-ad1c-58ed99181690",\n "address": "localhost",\n "port": 27019,\n "socket": "",\n "environment": "dev-mongos",\n "cluster": "super-custom",\n "replication_set": "",\n "custom_labels": {},\n "version": ""\n },\n {\n "service_id": "f694a319-c651-4fb5-97ba-9872785bb6b2",\n "service_name": "monitoring-pmm3-6334-monitoring-pmm3-mongos-2",\n "node_id": "8804cd11-6a01-4e80-80e4-0cdad05e8275",\n "address": "localhost",\n "port": 27019,\n "socket": "",\n "environment": "dev-mongos",\n "cluster": "super-custom",\n "replication_set": "",\n "custom_labels": {},\n "version": ""\n }\n ],\n "postgresql": [],\n "proxysql": [],\n "haproxy": [],\n "external": [],\n "valkey": []\n}' ++ jq -r --arg name monitoring-pmm3-6334-monitoring-pmm3-rs0-0 $'\n\t\t\t.mongodb[] | select(.service_name == $name) | .cluster\n\t\t' + actual_cluster=super-custom + [[ -z super-custom ]] + [[ super-custom == null ]] + [[ super-custom != super\-custom ]] + return 0 + nodeList=($(get_node_id_from_pmm)) ++ get_node_id_from_pmm ++ nodeList=() ++ local -a nodeList +++ kubectl_bin get pods --no-headers -l app.kubernetes.io/name=percona-server-mongodb --output=custom-columns=NAME:.metadata.name ++++ mktemp +++ local LAST_OUT=/tmp/tmp.G1HeFXPTwV ++++ mktemp +++ local LAST_ERR=/tmp/tmp.GD3AcOFbU1 +++ local exit_status=0 +++ local timeout=4 ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl get pods --no-headers -l app.kubernetes.io/name=percona-server-mongodb --output=custom-columns=NAME:.metadata.name +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 -a -n 1 ']' +++ break +++ cat /tmp/tmp.G1HeFXPTwV +++ cat /tmp/tmp.GD3AcOFbU1 +++ rm /tmp/tmp.G1HeFXPTwV /tmp/tmp.GD3AcOFbU1 +++ return 0 ++ for instance in $(kubectl_bin get pods --no-headers -l app.kubernetes.io/name=percona-server-mongodb --output=custom-columns='NAME:.metadata.name') ++ nodeList+=($(kubectl_bin exec -n "$namespace" $instance -c pmm-client -- pmm-admin status --json | jq -r '.pmm_agent_status.node_id')) +++ kubectl_bin exec -n monitoring-pmm3-6334 monitoring-pmm3-cfg-0 -c pmm-client -- pmm-admin status --json +++ jq -r .pmm_agent_status.node_id ++++ mktemp +++ local LAST_OUT=/tmp/tmp.Xvnms4IAUV ++++ mktemp +++ local LAST_ERR=/tmp/tmp.pE4Bdxkm7f +++ local exit_status=0 +++ local timeout=4 ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl exec -n monitoring-pmm3-6334 monitoring-pmm3-cfg-0 -c pmm-client -- pmm-admin status --json +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 -a -n 1 ']' +++ break +++ cat /tmp/tmp.Xvnms4IAUV +++ cat /tmp/tmp.pE4Bdxkm7f +++ rm /tmp/tmp.Xvnms4IAUV /tmp/tmp.pE4Bdxkm7f +++ return 0 ++ for instance in $(kubectl_bin get pods --no-headers -l app.kubernetes.io/name=percona-server-mongodb --output=custom-columns='NAME:.metadata.name') ++ nodeList+=($(kubectl_bin exec -n "$namespace" $instance -c pmm-client -- pmm-admin status --json | jq -r '.pmm_agent_status.node_id')) +++ jq -r .pmm_agent_status.node_id +++ kubectl_bin exec -n monitoring-pmm3-6334 monitoring-pmm3-cfg-1 -c pmm-client -- pmm-admin status --json ++++ mktemp +++ local LAST_OUT=/tmp/tmp.bioGNaTNDF ++++ mktemp +++ local LAST_ERR=/tmp/tmp.NG1UXFaeEY +++ local exit_status=0 +++ local timeout=4 ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl exec -n monitoring-pmm3-6334 monitoring-pmm3-cfg-1 -c pmm-client -- pmm-admin status --json +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 -a -n 1 ']' +++ break +++ cat /tmp/tmp.bioGNaTNDF +++ cat /tmp/tmp.NG1UXFaeEY +++ rm /tmp/tmp.bioGNaTNDF /tmp/tmp.NG1UXFaeEY +++ return 0 ++ for instance in $(kubectl_bin get pods --no-headers -l app.kubernetes.io/name=percona-server-mongodb --output=custom-columns='NAME:.metadata.name') ++ nodeList+=($(kubectl_bin exec -n "$namespace" $instance -c pmm-client -- pmm-admin status --json | jq -r '.pmm_agent_status.node_id')) +++ kubectl_bin exec -n monitoring-pmm3-6334 monitoring-pmm3-cfg-2 -c pmm-client -- pmm-admin status --json +++ jq -r .pmm_agent_status.node_id ++++ mktemp +++ local LAST_OUT=/tmp/tmp.hprpCdpgyE ++++ mktemp +++ local LAST_ERR=/tmp/tmp.s3WihvQesn +++ local exit_status=0 +++ local timeout=4 ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl exec -n monitoring-pmm3-6334 monitoring-pmm3-cfg-2 -c pmm-client -- pmm-admin status --json +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 -a -n 1 ']' +++ break +++ cat /tmp/tmp.hprpCdpgyE +++ cat /tmp/tmp.s3WihvQesn +++ rm /tmp/tmp.hprpCdpgyE /tmp/tmp.s3WihvQesn +++ return 0 ++ for instance in $(kubectl_bin get pods --no-headers -l app.kubernetes.io/name=percona-server-mongodb --output=custom-columns='NAME:.metadata.name') ++ nodeList+=($(kubectl_bin exec -n "$namespace" $instance -c pmm-client -- pmm-admin status --json | jq -r '.pmm_agent_status.node_id')) +++ kubectl_bin exec -n monitoring-pmm3-6334 monitoring-pmm3-mongos-0 -c pmm-client -- pmm-admin status --json +++ jq -r .pmm_agent_status.node_id ++++ mktemp +++ local LAST_OUT=/tmp/tmp.Xc1rGzkXqU ++++ mktemp +++ local LAST_ERR=/tmp/tmp.4hhEEMjQCb +++ local exit_status=0 +++ local timeout=4 ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl exec -n monitoring-pmm3-6334 monitoring-pmm3-mongos-0 -c pmm-client -- pmm-admin status --json +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 -a -n 1 ']' +++ break +++ cat /tmp/tmp.Xc1rGzkXqU +++ cat /tmp/tmp.4hhEEMjQCb +++ rm /tmp/tmp.Xc1rGzkXqU /tmp/tmp.4hhEEMjQCb +++ return 0 ++ for instance in $(kubectl_bin get pods --no-headers -l app.kubernetes.io/name=percona-server-mongodb --output=custom-columns='NAME:.metadata.name') ++ nodeList+=($(kubectl_bin exec -n "$namespace" $instance -c pmm-client -- pmm-admin status --json | jq -r '.pmm_agent_status.node_id')) +++ kubectl_bin exec -n monitoring-pmm3-6334 monitoring-pmm3-mongos-1 -c pmm-client -- pmm-admin status --json +++ jq -r .pmm_agent_status.node_id ++++ mktemp +++ local LAST_OUT=/tmp/tmp.oXYzUkymDX ++++ mktemp +++ local LAST_ERR=/tmp/tmp.av6ouHniat +++ local exit_status=0 +++ local timeout=4 ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl exec -n monitoring-pmm3-6334 monitoring-pmm3-mongos-1 -c pmm-client -- pmm-admin status --json +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 -a -n 1 ']' +++ break +++ cat /tmp/tmp.oXYzUkymDX +++ cat /tmp/tmp.av6ouHniat +++ rm /tmp/tmp.oXYzUkymDX /tmp/tmp.av6ouHniat +++ return 0 ++ for instance in $(kubectl_bin get pods --no-headers -l app.kubernetes.io/name=percona-server-mongodb --output=custom-columns='NAME:.metadata.name') ++ nodeList+=($(kubectl_bin exec -n "$namespace" $instance -c pmm-client -- pmm-admin status --json | jq -r '.pmm_agent_status.node_id')) +++ kubectl_bin exec -n monitoring-pmm3-6334 monitoring-pmm3-mongos-2 -c pmm-client -- pmm-admin status --json +++ jq -r .pmm_agent_status.node_id ++++ mktemp +++ local LAST_OUT=/tmp/tmp.F173uGcGCN ++++ mktemp +++ local LAST_ERR=/tmp/tmp.lfngh9eimA +++ local exit_status=0 +++ local timeout=4 ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl exec -n monitoring-pmm3-6334 monitoring-pmm3-mongos-2 -c pmm-client -- pmm-admin status --json +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 -a -n 1 ']' +++ break +++ cat /tmp/tmp.F173uGcGCN +++ cat /tmp/tmp.lfngh9eimA +++ rm /tmp/tmp.F173uGcGCN /tmp/tmp.lfngh9eimA +++ return 0 ++ for instance in $(kubectl_bin get pods --no-headers -l app.kubernetes.io/name=percona-server-mongodb --output=custom-columns='NAME:.metadata.name') ++ nodeList+=($(kubectl_bin exec -n "$namespace" $instance -c pmm-client -- pmm-admin status --json | jq -r '.pmm_agent_status.node_id')) +++ kubectl_bin exec -n monitoring-pmm3-6334 monitoring-pmm3-rs0-0 -c pmm-client -- pmm-admin status --json +++ jq -r .pmm_agent_status.node_id ++++ mktemp +++ local LAST_OUT=/tmp/tmp.qQO9NoOjVt ++++ mktemp +++ local LAST_ERR=/tmp/tmp.Ry8OS33zDd +++ local exit_status=0 +++ local timeout=4 ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl exec -n monitoring-pmm3-6334 monitoring-pmm3-rs0-0 -c pmm-client -- pmm-admin status --json +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 -a -n 1 ']' +++ break +++ cat /tmp/tmp.qQO9NoOjVt +++ cat /tmp/tmp.Ry8OS33zDd +++ rm /tmp/tmp.qQO9NoOjVt /tmp/tmp.Ry8OS33zDd +++ return 0 ++ for instance in $(kubectl_bin get pods --no-headers -l app.kubernetes.io/name=percona-server-mongodb --output=custom-columns='NAME:.metadata.name') ++ nodeList+=($(kubectl_bin exec -n "$namespace" $instance -c pmm-client -- pmm-admin status --json | jq -r '.pmm_agent_status.node_id')) +++ kubectl_bin exec -n monitoring-pmm3-6334 monitoring-pmm3-rs0-1 -c pmm-client -- pmm-admin status --json +++ jq -r .pmm_agent_status.node_id ++++ mktemp +++ local LAST_OUT=/tmp/tmp.XugzPvkaCG ++++ mktemp +++ local LAST_ERR=/tmp/tmp.XspHqC9wCa +++ local exit_status=0 +++ local timeout=4 ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl exec -n monitoring-pmm3-6334 monitoring-pmm3-rs0-1 -c pmm-client -- pmm-admin status --json +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 -a -n 1 ']' +++ break +++ cat /tmp/tmp.XugzPvkaCG +++ cat /tmp/tmp.XspHqC9wCa +++ rm /tmp/tmp.XugzPvkaCG /tmp/tmp.XspHqC9wCa +++ return 0 ++ for instance in $(kubectl_bin get pods --no-headers -l app.kubernetes.io/name=percona-server-mongodb --output=custom-columns='NAME:.metadata.name') ++ nodeList+=($(kubectl_bin exec -n "$namespace" $instance -c pmm-client -- pmm-admin status --json | jq -r '.pmm_agent_status.node_id')) +++ kubectl_bin exec -n monitoring-pmm3-6334 monitoring-pmm3-rs0-2 -c pmm-client -- pmm-admin status --json +++ jq -r .pmm_agent_status.node_id ++++ mktemp +++ local LAST_OUT=/tmp/tmp.2dCQCHBI1g ++++ mktemp +++ local LAST_ERR=/tmp/tmp.ADaDI5haH1 +++ local exit_status=0 +++ local timeout=4 ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl exec -n monitoring-pmm3-6334 monitoring-pmm3-rs0-2 -c pmm-client -- pmm-admin status --json +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 -a -n 1 ']' +++ break +++ cat /tmp/tmp.2dCQCHBI1g +++ cat /tmp/tmp.ADaDI5haH1 +++ rm /tmp/tmp.2dCQCHBI1g /tmp/tmp.ADaDI5haH1 +++ return 0 ++ echo 0c227efd-580e-4d7f-8616-937e5c5fe091 a433bf0f-bd6c-4624-ac8e-66e66ccfbb22 2438b450-3893-49bc-8c44-5b423e7c50e7 bd1c8be8-f68c-4841-a902-35492e16b99f 5f2c700d-929d-4cb4-ad1c-58ed99181690 8804cd11-6a01-4e80-80e4-0cdad05e8275 cc6efe57-5e10-40bc-b773-d30293aeef21 de0c1bd1-0160-4675-b54a-34e0d154d8ba d5ea0912-0daa-4f9a-add2-98046719381a + nodeList_from_pmm=($(does_node_id_exists "${nodeList[@]}")) ++ does_node_id_exists 0c227efd-580e-4d7f-8616-937e5c5fe091 a433bf0f-bd6c-4624-ac8e-66e66ccfbb22 2438b450-3893-49bc-8c44-5b423e7c50e7 bd1c8be8-f68c-4841-a902-35492e16b99f 5f2c700d-929d-4cb4-ad1c-58ed99181690 8804cd11-6a01-4e80-80e4-0cdad05e8275 cc6efe57-5e10-40bc-b773-d30293aeef21 de0c1bd1-0160-4675-b54a-34e0d154d8ba d5ea0912-0daa-4f9a-add2-98046719381a ++ nodeList=('0c227efd-580e-4d7f-8616-937e5c5fe091' 'a433bf0f-bd6c-4624-ac8e-66e66ccfbb22' '2438b450-3893-49bc-8c44-5b423e7c50e7' 'bd1c8be8-f68c-4841-a902-35492e16b99f' '5f2c700d-929d-4cb4-ad1c-58ed99181690' '8804cd11-6a01-4e80-80e4-0cdad05e8275' 'cc6efe57-5e10-40bc-b773-d30293aeef21' 'de0c1bd1-0160-4675-b54a-34e0d154d8ba' 'd5ea0912-0daa-4f9a-add2-98046719381a') ++ local -a nodeList ++ nodeList_from_pmm=() ++ local -a nodeList_from_pmm ++ for node_id in "${nodeList[@]}" ++ nodeList_from_pmm+=($(kubectl_bin exec -n "${namespace}" monitoring-server-0 -- pmm-admin --server-url=https://admin:admin@$(get_pmm_service_ip monitoring-service)/ --server-insecure-tls inventory list nodes --node-type=CONTAINER_NODE | grep $node_id | awk '{print $4}')) +++ grep 0c227efd-580e-4d7f-8616-937e5c5fe091 +++ awk '{print $4}' ++++ get_pmm_service_ip monitoring-service ++++ local service=monitoring-service ++++ kubectl_bin get service/monitoring-service -o 'jsonpath={.spec.type}' ++++ grep -q NotFound ++++ kubectl_bin get service/monitoring-service -o 'jsonpath={.status.loadBalancer.ingress[]}' ++++ grep -E -q 'hostname|ip' ++++ kubectl_bin get service/monitoring-service -o 'jsonpath={.status.loadBalancer.ingress[].ip}' +++++ mktemp ++++ local LAST_OUT=/tmp/tmp.fIaYpe5Mme +++++ mktemp ++++ local LAST_ERR=/tmp/tmp.PgVCepwsgK ++++ local exit_status=0 ++++ local timeout=4 +++++ seq 0 2 ++++ for i in $(seq 0 2) ++++ set +e ++++ kubectl get service/monitoring-service -o 'jsonpath={.status.loadBalancer.ingress[].ip}' ++++ exit_status=0 ++++ set -e ++++ '[' 0 '!=' 0 -a -n 1 ']' ++++ break ++++ cat /tmp/tmp.fIaYpe5Mme ++++ cat /tmp/tmp.PgVCepwsgK ++++ rm /tmp/tmp.fIaYpe5Mme /tmp/tmp.PgVCepwsgK ++++ return 0 ++++ kubectl_bin get service/monitoring-service -o 'jsonpath={.status.loadBalancer.ingress[].hostname}' +++++ mktemp ++++ local LAST_OUT=/tmp/tmp.IG8ncR7WXt +++++ mktemp ++++ local LAST_ERR=/tmp/tmp.5lnqfuzTij ++++ local exit_status=0 ++++ local timeout=4 +++++ seq 0 2 ++++ for i in $(seq 0 2) ++++ set +e ++++ kubectl get service/monitoring-service -o 'jsonpath={.status.loadBalancer.ingress[].hostname}' ++++ exit_status=0 ++++ set -e ++++ '[' 0 '!=' 0 -a -n 1 ']' ++++ break ++++ cat /tmp/tmp.IG8ncR7WXt ++++ cat /tmp/tmp.5lnqfuzTij ++++ rm /tmp/tmp.IG8ncR7WXt /tmp/tmp.5lnqfuzTij ++++ return 0 +++ kubectl_bin exec -n monitoring-pmm3-6334 monitoring-server-0 -- pmm-admin --server-url=https://admin:admin@136.115.177.16/ --server-insecure-tls inventory list nodes --node-type=CONTAINER_NODE ++++ mktemp +++ local LAST_OUT=/tmp/tmp.bzdXwUcFep ++++ mktemp +++ local LAST_ERR=/tmp/tmp.vZyYQPpeWt +++ local exit_status=0 +++ local timeout=4 ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl exec -n monitoring-pmm3-6334 monitoring-server-0 -- pmm-admin --server-url=https://admin:admin@136.115.177.16/ --server-insecure-tls inventory list nodes --node-type=CONTAINER_NODE +++ exit_status=1 +++ set -e +++ '[' 1 '!=' 0 -a -n 1 ']' +++ cat /tmp/tmp.bzdXwUcFep +++ cat /tmp/tmp.vZyYQPpeWt command terminated with exit code 1 +++ sleep 0 +++ for i in $(seq 0 2) +++ set +e +++ kubectl exec -n monitoring-pmm3-6334 monitoring-server-0 -- pmm-admin --server-url=https://admin:admin@136.115.177.16/ --server-insecure-tls inventory list nodes --node-type=CONTAINER_NODE +++ exit_status=1 +++ set -e +++ '[' 1 '!=' 0 -a -n 1 ']' +++ cat /tmp/tmp.bzdXwUcFep +++ cat /tmp/tmp.vZyYQPpeWt command terminated with exit code 1 +++ sleep 4 +++ for i in $(seq 0 2) +++ set +e +++ kubectl exec -n monitoring-pmm3-6334 monitoring-server-0 -- pmm-admin --server-url=https://admin:admin@136.115.177.16/ --server-insecure-tls inventory list nodes --node-type=CONTAINER_NODE +++ exit_status=1 +++ set -e +++ '[' 1 '!=' 0 -a -n 1 ']' +++ cat /tmp/tmp.bzdXwUcFep +++ cat /tmp/tmp.vZyYQPpeWt command terminated with exit code 1 +++ sleep 8 +++ cat /tmp/tmp.bzdXwUcFep +++ cat /tmp/tmp.vZyYQPpeWt command terminated with exit code 1 +++ rm /tmp/tmp.bzdXwUcFep /tmp/tmp.vZyYQPpeWt +++ return 1 ++ for node_id in "${nodeList[@]}" ++ nodeList_from_pmm+=($(kubectl_bin exec -n "${namespace}" monitoring-server-0 -- pmm-admin --server-url=https://admin:admin@$(get_pmm_service_ip monitoring-service)/ --server-insecure-tls inventory list nodes --node-type=CONTAINER_NODE | grep $node_id | awk '{print $4}')) +++ grep a433bf0f-bd6c-4624-ac8e-66e66ccfbb22 +++ awk '{print $4}' ++++ get_pmm_service_ip monitoring-service ++++ local service=monitoring-service ++++ kubectl_bin get service/monitoring-service -o 'jsonpath={.spec.type}' ++++ grep -q NotFound ++++ kubectl_bin get service/monitoring-service -o 'jsonpath={.status.loadBalancer.ingress[]}' ++++ grep -E -q 'hostname|ip' ++++ kubectl_bin get service/monitoring-service -o 'jsonpath={.status.loadBalancer.ingress[].ip}' +++++ mktemp ++++ local LAST_OUT=/tmp/tmp.REK8YYo1wj +++++ mktemp ++++ local LAST_ERR=/tmp/tmp.N1k9NioHRM ++++ local exit_status=0 ++++ local timeout=4 +++++ seq 0 2 ++++ for i in $(seq 0 2) ++++ set +e ++++ kubectl get service/monitoring-service -o 'jsonpath={.status.loadBalancer.ingress[].ip}' ++++ exit_status=0 ++++ set -e ++++ '[' 0 '!=' 0 -a -n 1 ']' ++++ break ++++ cat /tmp/tmp.REK8YYo1wj ++++ cat /tmp/tmp.N1k9NioHRM ++++ rm /tmp/tmp.REK8YYo1wj /tmp/tmp.N1k9NioHRM ++++ return 0 ++++ kubectl_bin get service/monitoring-service -o 'jsonpath={.status.loadBalancer.ingress[].hostname}' +++++ mktemp ++++ local LAST_OUT=/tmp/tmp.UqSXifL5iT +++++ mktemp ++++ local LAST_ERR=/tmp/tmp.WcBR6Gcyut ++++ local exit_status=0 ++++ local timeout=4 +++++ seq 0 2 ++++ for i in $(seq 0 2) ++++ set +e ++++ kubectl get service/monitoring-service -o 'jsonpath={.status.loadBalancer.ingress[].hostname}' ++++ exit_status=0 ++++ set -e ++++ '[' 0 '!=' 0 -a -n 1 ']' ++++ break ++++ cat /tmp/tmp.UqSXifL5iT ++++ cat /tmp/tmp.WcBR6Gcyut ++++ rm /tmp/tmp.UqSXifL5iT /tmp/tmp.WcBR6Gcyut ++++ return 0 +++ kubectl_bin exec -n monitoring-pmm3-6334 monitoring-server-0 -- pmm-admin --server-url=https://admin:admin@136.115.177.16/ --server-insecure-tls inventory list nodes --node-type=CONTAINER_NODE ++++ mktemp +++ local LAST_OUT=/tmp/tmp.rq72JGN1Bm ++++ mktemp +++ local LAST_ERR=/tmp/tmp.uixOH9m17C +++ local exit_status=0 +++ local timeout=4 ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl exec -n monitoring-pmm3-6334 monitoring-server-0 -- pmm-admin --server-url=https://admin:admin@136.115.177.16/ --server-insecure-tls inventory list nodes --node-type=CONTAINER_NODE +++ exit_status=1 +++ set -e +++ '[' 1 '!=' 0 -a -n 1 ']' +++ cat /tmp/tmp.rq72JGN1Bm +++ cat /tmp/tmp.uixOH9m17C command terminated with exit code 1 +++ sleep 0 +++ for i in $(seq 0 2) +++ set +e +++ kubectl exec -n monitoring-pmm3-6334 monitoring-server-0 -- pmm-admin --server-url=https://admin:admin@136.115.177.16/ --server-insecure-tls inventory list nodes --node-type=CONTAINER_NODE +++ exit_status=1 +++ set -e +++ '[' 1 '!=' 0 -a -n 1 ']' +++ cat /tmp/tmp.rq72JGN1Bm +++ cat /tmp/tmp.uixOH9m17C command terminated with exit code 1 +++ sleep 4 +++ for i in $(seq 0 2) +++ set +e +++ kubectl exec -n monitoring-pmm3-6334 monitoring-server-0 -- pmm-admin --server-url=https://admin:admin@136.115.177.16/ --server-insecure-tls inventory list nodes --node-type=CONTAINER_NODE +++ exit_status=1 +++ set -e +++ '[' 1 '!=' 0 -a -n 1 ']' +++ cat /tmp/tmp.rq72JGN1Bm +++ cat /tmp/tmp.uixOH9m17C command terminated with exit code 1 +++ sleep 8 +++ cat /tmp/tmp.rq72JGN1Bm +++ cat /tmp/tmp.uixOH9m17C command terminated with exit code 1 +++ rm /tmp/tmp.rq72JGN1Bm /tmp/tmp.uixOH9m17C +++ return 1 ++ for node_id in "${nodeList[@]}" ++ nodeList_from_pmm+=($(kubectl_bin exec -n "${namespace}" monitoring-server-0 -- pmm-admin --server-url=https://admin:admin@$(get_pmm_service_ip monitoring-service)/ --server-insecure-tls inventory list nodes --node-type=CONTAINER_NODE | grep $node_id | awk '{print $4}')) +++ grep 2438b450-3893-49bc-8c44-5b423e7c50e7 +++ awk '{print $4}' ++++ get_pmm_service_ip monitoring-service ++++ local service=monitoring-service ++++ kubectl_bin get service/monitoring-service -o 'jsonpath={.spec.type}' ++++ grep -q NotFound ++++ kubectl_bin get service/monitoring-service -o 'jsonpath={.status.loadBalancer.ingress[]}' ++++ grep -E -q 'hostname|ip' ++++ kubectl_bin get service/monitoring-service -o 'jsonpath={.status.loadBalancer.ingress[].ip}' +++++ mktemp ++++ local LAST_OUT=/tmp/tmp.4BqWU2W4xf +++++ mktemp ++++ local LAST_ERR=/tmp/tmp.zdts5oFX4s ++++ local exit_status=0 ++++ local timeout=4 +++++ seq 0 2 ++++ for i in $(seq 0 2) ++++ set +e ++++ kubectl get service/monitoring-service -o 'jsonpath={.status.loadBalancer.ingress[].ip}' ++++ exit_status=0 ++++ set -e ++++ '[' 0 '!=' 0 -a -n 1 ']' ++++ break ++++ cat /tmp/tmp.4BqWU2W4xf ++++ cat /tmp/tmp.zdts5oFX4s ++++ rm /tmp/tmp.4BqWU2W4xf /tmp/tmp.zdts5oFX4s ++++ return 0 ++++ kubectl_bin get service/monitoring-service -o 'jsonpath={.status.loadBalancer.ingress[].hostname}' +++++ mktemp ++++ local LAST_OUT=/tmp/tmp.4HrocNufv0 +++++ mktemp ++++ local LAST_ERR=/tmp/tmp.Mfqp9hz2lj ++++ local exit_status=0 ++++ local timeout=4 +++++ seq 0 2 ++++ for i in $(seq 0 2) ++++ set +e ++++ kubectl get service/monitoring-service -o 'jsonpath={.status.loadBalancer.ingress[].hostname}' ++++ exit_status=0 ++++ set -e ++++ '[' 0 '!=' 0 -a -n 1 ']' ++++ break ++++ cat /tmp/tmp.4HrocNufv0 ++++ cat /tmp/tmp.Mfqp9hz2lj ++++ rm /tmp/tmp.4HrocNufv0 /tmp/tmp.Mfqp9hz2lj ++++ return 0 +++ kubectl_bin exec -n monitoring-pmm3-6334 monitoring-server-0 -- pmm-admin --server-url=https://admin:admin@136.115.177.16/ --server-insecure-tls inventory list nodes --node-type=CONTAINER_NODE ++++ mktemp +++ local LAST_OUT=/tmp/tmp.c5xTNPYNaK ++++ mktemp +++ local LAST_ERR=/tmp/tmp.102W6ICnwZ +++ local exit_status=0 +++ local timeout=4 ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl exec -n monitoring-pmm3-6334 monitoring-server-0 -- pmm-admin --server-url=https://admin:admin@136.115.177.16/ --server-insecure-tls inventory list nodes --node-type=CONTAINER_NODE +++ exit_status=1 +++ set -e +++ '[' 1 '!=' 0 -a -n 1 ']' +++ cat /tmp/tmp.c5xTNPYNaK +++ cat /tmp/tmp.102W6ICnwZ command terminated with exit code 1 +++ sleep 0 +++ for i in $(seq 0 2) +++ set +e +++ kubectl exec -n monitoring-pmm3-6334 monitoring-server-0 -- pmm-admin --server-url=https://admin:admin@136.115.177.16/ --server-insecure-tls inventory list nodes --node-type=CONTAINER_NODE +++ exit_status=1 +++ set -e +++ '[' 1 '!=' 0 -a -n 1 ']' +++ cat /tmp/tmp.c5xTNPYNaK +++ cat /tmp/tmp.102W6ICnwZ command terminated with exit code 1 +++ sleep 4 +++ for i in $(seq 0 2) +++ set +e +++ kubectl exec -n monitoring-pmm3-6334 monitoring-server-0 -- pmm-admin --server-url=https://admin:admin@136.115.177.16/ --server-insecure-tls inventory list nodes --node-type=CONTAINER_NODE +++ exit_status=1 +++ set -e +++ '[' 1 '!=' 0 -a -n 1 ']' +++ cat /tmp/tmp.c5xTNPYNaK +++ cat /tmp/tmp.102W6ICnwZ command terminated with exit code 1 +++ sleep 8 +++ cat /tmp/tmp.c5xTNPYNaK +++ cat /tmp/tmp.102W6ICnwZ command terminated with exit code 1 +++ rm /tmp/tmp.c5xTNPYNaK /tmp/tmp.102W6ICnwZ +++ return 1 ++ for node_id in "${nodeList[@]}" ++ nodeList_from_pmm+=($(kubectl_bin exec -n "${namespace}" monitoring-server-0 -- pmm-admin --server-url=https://admin:admin@$(get_pmm_service_ip monitoring-service)/ --server-insecure-tls inventory list nodes --node-type=CONTAINER_NODE | grep $node_id | awk '{print $4}')) +++ grep bd1c8be8-f68c-4841-a902-35492e16b99f +++ awk '{print $4}' ++++ get_pmm_service_ip monitoring-service ++++ local service=monitoring-service ++++ kubectl_bin get service/monitoring-service -o 'jsonpath={.spec.type}' ++++ grep -q NotFound ++++ kubectl_bin get service/monitoring-service -o 'jsonpath={.status.loadBalancer.ingress[]}' ++++ grep -E -q 'hostname|ip' ++++ kubectl_bin get service/monitoring-service -o 'jsonpath={.status.loadBalancer.ingress[].ip}' +++++ mktemp ++++ local LAST_OUT=/tmp/tmp.ephUvSsY0d +++++ mktemp ++++ local LAST_ERR=/tmp/tmp.jrmgJjdZiC ++++ local exit_status=0 ++++ local timeout=4 +++++ seq 0 2 ++++ for i in $(seq 0 2) ++++ set +e ++++ kubectl get service/monitoring-service -o 'jsonpath={.status.loadBalancer.ingress[].ip}' ++++ exit_status=0 ++++ set -e ++++ '[' 0 '!=' 0 -a -n 1 ']' ++++ break ++++ cat /tmp/tmp.ephUvSsY0d ++++ cat /tmp/tmp.jrmgJjdZiC ++++ rm /tmp/tmp.ephUvSsY0d /tmp/tmp.jrmgJjdZiC ++++ return 0 ++++ kubectl_bin get service/monitoring-service -o 'jsonpath={.status.loadBalancer.ingress[].hostname}' +++++ mktemp ++++ local LAST_OUT=/tmp/tmp.6vZU3QWGad +++++ mktemp ++++ local LAST_ERR=/tmp/tmp.mq4NjYEFol ++++ local exit_status=0 ++++ local timeout=4 +++++ seq 0 2 ++++ for i in $(seq 0 2) ++++ set +e ++++ kubectl get service/monitoring-service -o 'jsonpath={.status.loadBalancer.ingress[].hostname}' ++++ exit_status=0 ++++ set -e ++++ '[' 0 '!=' 0 -a -n 1 ']' ++++ break ++++ cat /tmp/tmp.6vZU3QWGad ++++ cat /tmp/tmp.mq4NjYEFol ++++ rm /tmp/tmp.6vZU3QWGad /tmp/tmp.mq4NjYEFol ++++ return 0 +++ kubectl_bin exec -n monitoring-pmm3-6334 monitoring-server-0 -- pmm-admin --server-url=https://admin:admin@136.115.177.16/ --server-insecure-tls inventory list nodes --node-type=CONTAINER_NODE ++++ mktemp +++ local LAST_OUT=/tmp/tmp.R4dBkKBcRG ++++ mktemp +++ local LAST_ERR=/tmp/tmp.sdkZFi7vHj +++ local exit_status=0 +++ local timeout=4 ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl exec -n monitoring-pmm3-6334 monitoring-server-0 -- pmm-admin --server-url=https://admin:admin@136.115.177.16/ --server-insecure-tls inventory list nodes --node-type=CONTAINER_NODE +++ exit_status=1 +++ set -e +++ '[' 1 '!=' 0 -a -n 1 ']' +++ cat /tmp/tmp.R4dBkKBcRG +++ cat /tmp/tmp.sdkZFi7vHj command terminated with exit code 1 +++ sleep 0 +++ for i in $(seq 0 2) +++ set +e +++ kubectl exec -n monitoring-pmm3-6334 monitoring-server-0 -- pmm-admin --server-url=https://admin:admin@136.115.177.16/ --server-insecure-tls inventory list nodes --node-type=CONTAINER_NODE +++ exit_status=1 +++ set -e +++ '[' 1 '!=' 0 -a -n 1 ']' +++ cat /tmp/tmp.R4dBkKBcRG +++ cat /tmp/tmp.sdkZFi7vHj command terminated with exit code 1 +++ sleep 4 +++ for i in $(seq 0 2) +++ set +e +++ kubectl exec -n monitoring-pmm3-6334 monitoring-server-0 -- pmm-admin --server-url=https://admin:admin@136.115.177.16/ --server-insecure-tls inventory list nodes --node-type=CONTAINER_NODE +++ exit_status=1 +++ set -e +++ '[' 1 '!=' 0 -a -n 1 ']' +++ cat /tmp/tmp.R4dBkKBcRG +++ cat /tmp/tmp.sdkZFi7vHj command terminated with exit code 1 +++ sleep 8 +++ cat /tmp/tmp.R4dBkKBcRG +++ cat /tmp/tmp.sdkZFi7vHj command terminated with exit code 1 +++ rm /tmp/tmp.R4dBkKBcRG /tmp/tmp.sdkZFi7vHj +++ return 1 ++ for node_id in "${nodeList[@]}" ++ nodeList_from_pmm+=($(kubectl_bin exec -n "${namespace}" monitoring-server-0 -- pmm-admin --server-url=https://admin:admin@$(get_pmm_service_ip monitoring-service)/ --server-insecure-tls inventory list nodes --node-type=CONTAINER_NODE | grep $node_id | awk '{print $4}')) +++ grep 5f2c700d-929d-4cb4-ad1c-58ed99181690 +++ awk '{print $4}' ++++ get_pmm_service_ip monitoring-service ++++ local service=monitoring-service ++++ kubectl_bin get service/monitoring-service -o 'jsonpath={.spec.type}' ++++ grep -q NotFound ++++ kubectl_bin get service/monitoring-service -o 'jsonpath={.status.loadBalancer.ingress[]}' ++++ grep -E -q 'hostname|ip' ++++ kubectl_bin get service/monitoring-service -o 'jsonpath={.status.loadBalancer.ingress[].ip}' +++++ mktemp ++++ local LAST_OUT=/tmp/tmp.2w413djgQd +++++ mktemp ++++ local LAST_ERR=/tmp/tmp.2LkYYpATe5 ++++ local exit_status=0 ++++ local timeout=4 +++++ seq 0 2 ++++ for i in $(seq 0 2) ++++ set +e ++++ kubectl get service/monitoring-service -o 'jsonpath={.status.loadBalancer.ingress[].ip}' ++++ exit_status=0 ++++ set -e ++++ '[' 0 '!=' 0 -a -n 1 ']' ++++ break ++++ cat /tmp/tmp.2w413djgQd ++++ cat /tmp/tmp.2LkYYpATe5 ++++ rm /tmp/tmp.2w413djgQd /tmp/tmp.2LkYYpATe5 ++++ return 0 ++++ kubectl_bin get service/monitoring-service -o 'jsonpath={.status.loadBalancer.ingress[].hostname}' +++++ mktemp ++++ local LAST_OUT=/tmp/tmp.1q8z3bAbTt +++++ mktemp ++++ local LAST_ERR=/tmp/tmp.vSHsqFGbO1 ++++ local exit_status=0 ++++ local timeout=4 +++++ seq 0 2 ++++ for i in $(seq 0 2) ++++ set +e ++++ kubectl get service/monitoring-service -o 'jsonpath={.status.loadBalancer.ingress[].hostname}' ++++ exit_status=0 ++++ set -e ++++ '[' 0 '!=' 0 -a -n 1 ']' ++++ break ++++ cat /tmp/tmp.1q8z3bAbTt ++++ cat /tmp/tmp.vSHsqFGbO1 ++++ rm /tmp/tmp.1q8z3bAbTt /tmp/tmp.vSHsqFGbO1 ++++ return 0 +++ kubectl_bin exec -n monitoring-pmm3-6334 monitoring-server-0 -- pmm-admin --server-url=https://admin:admin@136.115.177.16/ --server-insecure-tls inventory list nodes --node-type=CONTAINER_NODE ++++ mktemp +++ local LAST_OUT=/tmp/tmp.ZMDk19DKOb ++++ mktemp +++ local LAST_ERR=/tmp/tmp.unWWeTsvFr +++ local exit_status=0 +++ local timeout=4 ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl exec -n monitoring-pmm3-6334 monitoring-server-0 -- pmm-admin --server-url=https://admin:admin@136.115.177.16/ --server-insecure-tls inventory list nodes --node-type=CONTAINER_NODE +++ exit_status=1 +++ set -e +++ '[' 1 '!=' 0 -a -n 1 ']' +++ cat /tmp/tmp.ZMDk19DKOb +++ cat /tmp/tmp.unWWeTsvFr command terminated with exit code 1 +++ sleep 0 +++ for i in $(seq 0 2) +++ set +e +++ kubectl exec -n monitoring-pmm3-6334 monitoring-server-0 -- pmm-admin --server-url=https://admin:admin@136.115.177.16/ --server-insecure-tls inventory list nodes --node-type=CONTAINER_NODE +++ exit_status=1 +++ set -e +++ '[' 1 '!=' 0 -a -n 1 ']' +++ cat /tmp/tmp.ZMDk19DKOb +++ cat /tmp/tmp.unWWeTsvFr command terminated with exit code 1 +++ sleep 4 +++ for i in $(seq 0 2) +++ set +e +++ kubectl exec -n monitoring-pmm3-6334 monitoring-server-0 -- pmm-admin --server-url=https://admin:admin@136.115.177.16/ --server-insecure-tls inventory list nodes --node-type=CONTAINER_NODE +++ exit_status=1 +++ set -e +++ '[' 1 '!=' 0 -a -n 1 ']' +++ cat /tmp/tmp.ZMDk19DKOb +++ cat /tmp/tmp.unWWeTsvFr command terminated with exit code 1 +++ sleep 8 +++ cat /tmp/tmp.ZMDk19DKOb +++ cat /tmp/tmp.unWWeTsvFr command terminated with exit code 1 +++ rm /tmp/tmp.ZMDk19DKOb /tmp/tmp.unWWeTsvFr +++ return 1 ++ for node_id in "${nodeList[@]}" ++ nodeList_from_pmm+=($(kubectl_bin exec -n "${namespace}" monitoring-server-0 -- pmm-admin --server-url=https://admin:admin@$(get_pmm_service_ip monitoring-service)/ --server-insecure-tls inventory list nodes --node-type=CONTAINER_NODE | grep $node_id | awk '{print $4}')) +++ grep 8804cd11-6a01-4e80-80e4-0cdad05e8275 +++ awk '{print $4}' ++++ get_pmm_service_ip monitoring-service ++++ local service=monitoring-service ++++ kubectl_bin get service/monitoring-service -o 'jsonpath={.spec.type}' ++++ grep -q NotFound ++++ kubectl_bin get service/monitoring-service -o 'jsonpath={.status.loadBalancer.ingress[]}' ++++ grep -E -q 'hostname|ip' ++++ kubectl_bin get service/monitoring-service -o 'jsonpath={.status.loadBalancer.ingress[].ip}' +++++ mktemp ++++ local LAST_OUT=/tmp/tmp.x1o4tKubLX +++++ mktemp ++++ local LAST_ERR=/tmp/tmp.5WaO8sfDH7 ++++ local exit_status=0 ++++ local timeout=4 +++++ seq 0 2 ++++ for i in $(seq 0 2) ++++ set +e ++++ kubectl get service/monitoring-service -o 'jsonpath={.status.loadBalancer.ingress[].ip}' ++++ exit_status=0 ++++ set -e ++++ '[' 0 '!=' 0 -a -n 1 ']' ++++ break ++++ cat /tmp/tmp.x1o4tKubLX ++++ cat /tmp/tmp.5WaO8sfDH7 ++++ rm /tmp/tmp.x1o4tKubLX /tmp/tmp.5WaO8sfDH7 ++++ return 0 ++++ kubectl_bin get service/monitoring-service -o 'jsonpath={.status.loadBalancer.ingress[].hostname}' +++++ mktemp ++++ local LAST_OUT=/tmp/tmp.qoxAcsQyaT +++++ mktemp ++++ local LAST_ERR=/tmp/tmp.Aiz0AtSPwu ++++ local exit_status=0 ++++ local timeout=4 +++++ seq 0 2 ++++ for i in $(seq 0 2) ++++ set +e ++++ kubectl get service/monitoring-service -o 'jsonpath={.status.loadBalancer.ingress[].hostname}' ++++ exit_status=0 ++++ set -e ++++ '[' 0 '!=' 0 -a -n 1 ']' ++++ break ++++ cat /tmp/tmp.qoxAcsQyaT ++++ cat /tmp/tmp.Aiz0AtSPwu ++++ rm /tmp/tmp.qoxAcsQyaT /tmp/tmp.Aiz0AtSPwu ++++ return 0 +++ kubectl_bin exec -n monitoring-pmm3-6334 monitoring-server-0 -- pmm-admin --server-url=https://admin:admin@136.115.177.16/ --server-insecure-tls inventory list nodes --node-type=CONTAINER_NODE ++++ mktemp +++ local LAST_OUT=/tmp/tmp.dkHjgmTLo9 ++++ mktemp +++ local LAST_ERR=/tmp/tmp.cqfnIfmksV +++ local exit_status=0 +++ local timeout=4 ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl exec -n monitoring-pmm3-6334 monitoring-server-0 -- pmm-admin --server-url=https://admin:admin@136.115.177.16/ --server-insecure-tls inventory list nodes --node-type=CONTAINER_NODE +++ exit_status=1 +++ set -e +++ '[' 1 '!=' 0 -a -n 1 ']' +++ cat /tmp/tmp.dkHjgmTLo9 +++ cat /tmp/tmp.cqfnIfmksV command terminated with exit code 1 +++ sleep 0 +++ for i in $(seq 0 2) +++ set +e +++ kubectl exec -n monitoring-pmm3-6334 monitoring-server-0 -- pmm-admin --server-url=https://admin:admin@136.115.177.16/ --server-insecure-tls inventory list nodes --node-type=CONTAINER_NODE +++ exit_status=1 +++ set -e +++ '[' 1 '!=' 0 -a -n 1 ']' +++ cat /tmp/tmp.dkHjgmTLo9 +++ cat /tmp/tmp.cqfnIfmksV command terminated with exit code 1 +++ sleep 4 +++ for i in $(seq 0 2) +++ set +e +++ kubectl exec -n monitoring-pmm3-6334 monitoring-server-0 -- pmm-admin --server-url=https://admin:admin@136.115.177.16/ --server-insecure-tls inventory list nodes --node-type=CONTAINER_NODE +++ exit_status=1 +++ set -e +++ '[' 1 '!=' 0 -a -n 1 ']' +++ cat /tmp/tmp.dkHjgmTLo9 +++ cat /tmp/tmp.cqfnIfmksV command terminated with exit code 1 +++ sleep 8 +++ cat /tmp/tmp.dkHjgmTLo9 +++ cat /tmp/tmp.cqfnIfmksV command terminated with exit code 1 +++ rm /tmp/tmp.dkHjgmTLo9 /tmp/tmp.cqfnIfmksV +++ return 1 ++ for node_id in "${nodeList[@]}" ++ nodeList_from_pmm+=($(kubectl_bin exec -n "${namespace}" monitoring-server-0 -- pmm-admin --server-url=https://admin:admin@$(get_pmm_service_ip monitoring-service)/ --server-insecure-tls inventory list nodes --node-type=CONTAINER_NODE | grep $node_id | awk '{print $4}')) +++ grep cc6efe57-5e10-40bc-b773-d30293aeef21 ++++ get_pmm_service_ip monitoring-service +++ awk '{print $4}' ++++ local service=monitoring-service ++++ kubectl_bin get service/monitoring-service -o 'jsonpath={.spec.type}' ++++ grep -q NotFound ++++ kubectl_bin get service/monitoring-service -o 'jsonpath={.status.loadBalancer.ingress[]}' ++++ grep -E -q 'hostname|ip' ++++ kubectl_bin get service/monitoring-service -o 'jsonpath={.status.loadBalancer.ingress[].ip}' +++++ mktemp ++++ local LAST_OUT=/tmp/tmp.ZQMGLcUBL9 +++++ mktemp ++++ local LAST_ERR=/tmp/tmp.6IK8WU2KFT ++++ local exit_status=0 ++++ local timeout=4 +++++ seq 0 2 ++++ for i in $(seq 0 2) ++++ set +e ++++ kubectl get service/monitoring-service -o 'jsonpath={.status.loadBalancer.ingress[].ip}' ++++ exit_status=0 ++++ set -e ++++ '[' 0 '!=' 0 -a -n 1 ']' ++++ break ++++ cat /tmp/tmp.ZQMGLcUBL9 ++++ cat /tmp/tmp.6IK8WU2KFT ++++ rm /tmp/tmp.ZQMGLcUBL9 /tmp/tmp.6IK8WU2KFT ++++ return 0 ++++ kubectl_bin get service/monitoring-service -o 'jsonpath={.status.loadBalancer.ingress[].hostname}' +++++ mktemp ++++ local LAST_OUT=/tmp/tmp.Cdg6mCQMBD +++++ mktemp ++++ local LAST_ERR=/tmp/tmp.FyDwPuRIRn ++++ local exit_status=0 ++++ local timeout=4 +++++ seq 0 2 ++++ for i in $(seq 0 2) ++++ set +e ++++ kubectl get service/monitoring-service -o 'jsonpath={.status.loadBalancer.ingress[].hostname}' ++++ exit_status=0 ++++ set -e ++++ '[' 0 '!=' 0 -a -n 1 ']' ++++ break ++++ cat /tmp/tmp.Cdg6mCQMBD ++++ cat /tmp/tmp.FyDwPuRIRn ++++ rm /tmp/tmp.Cdg6mCQMBD /tmp/tmp.FyDwPuRIRn ++++ return 0 +++ kubectl_bin exec -n monitoring-pmm3-6334 monitoring-server-0 -- pmm-admin --server-url=https://admin:admin@136.115.177.16/ --server-insecure-tls inventory list nodes --node-type=CONTAINER_NODE ++++ mktemp +++ local LAST_OUT=/tmp/tmp.h2vVVNhYlU ++++ mktemp +++ local LAST_ERR=/tmp/tmp.LyoYJ9wWbp +++ local exit_status=0 +++ local timeout=4 ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl exec -n monitoring-pmm3-6334 monitoring-server-0 -- pmm-admin --server-url=https://admin:admin@136.115.177.16/ --server-insecure-tls inventory list nodes --node-type=CONTAINER_NODE +++ exit_status=1 +++ set -e +++ '[' 1 '!=' 0 -a -n 1 ']' +++ cat /tmp/tmp.h2vVVNhYlU +++ cat /tmp/tmp.LyoYJ9wWbp command terminated with exit code 1 +++ sleep 0 +++ for i in $(seq 0 2) +++ set +e +++ kubectl exec -n monitoring-pmm3-6334 monitoring-server-0 -- pmm-admin --server-url=https://admin:admin@136.115.177.16/ --server-insecure-tls inventory list nodes --node-type=CONTAINER_NODE +++ exit_status=1 +++ set -e +++ '[' 1 '!=' 0 -a -n 1 ']' +++ cat /tmp/tmp.h2vVVNhYlU +++ cat /tmp/tmp.LyoYJ9wWbp command terminated with exit code 1 +++ sleep 4 +++ for i in $(seq 0 2) +++ set +e +++ kubectl exec -n monitoring-pmm3-6334 monitoring-server-0 -- pmm-admin --server-url=https://admin:admin@136.115.177.16/ --server-insecure-tls inventory list nodes --node-type=CONTAINER_NODE +++ exit_status=1 +++ set -e +++ '[' 1 '!=' 0 -a -n 1 ']' +++ cat /tmp/tmp.h2vVVNhYlU +++ cat /tmp/tmp.LyoYJ9wWbp command terminated with exit code 1 +++ sleep 8 +++ cat /tmp/tmp.h2vVVNhYlU +++ cat /tmp/tmp.LyoYJ9wWbp command terminated with exit code 1 +++ rm /tmp/tmp.h2vVVNhYlU /tmp/tmp.LyoYJ9wWbp +++ return 1 ++ for node_id in "${nodeList[@]}" ++ nodeList_from_pmm+=($(kubectl_bin exec -n "${namespace}" monitoring-server-0 -- pmm-admin --server-url=https://admin:admin@$(get_pmm_service_ip monitoring-service)/ --server-insecure-tls inventory list nodes --node-type=CONTAINER_NODE | grep $node_id | awk '{print $4}')) +++ grep de0c1bd1-0160-4675-b54a-34e0d154d8ba +++ awk '{print $4}' ++++ get_pmm_service_ip monitoring-service ++++ local service=monitoring-service ++++ kubectl_bin get service/monitoring-service -o 'jsonpath={.spec.type}' ++++ grep -q NotFound ++++ kubectl_bin get service/monitoring-service -o 'jsonpath={.status.loadBalancer.ingress[]}' ++++ grep -E -q 'hostname|ip' ++++ kubectl_bin get service/monitoring-service -o 'jsonpath={.status.loadBalancer.ingress[].ip}' +++++ mktemp ++++ local LAST_OUT=/tmp/tmp.BXgLYuWjp5 +++++ mktemp ++++ local LAST_ERR=/tmp/tmp.FHVGMpAUwD ++++ local exit_status=0 ++++ local timeout=4 +++++ seq 0 2 ++++ for i in $(seq 0 2) ++++ set +e ++++ kubectl get service/monitoring-service -o 'jsonpath={.status.loadBalancer.ingress[].ip}' ++++ exit_status=0 ++++ set -e ++++ '[' 0 '!=' 0 -a -n 1 ']' ++++ break ++++ cat /tmp/tmp.BXgLYuWjp5 ++++ cat /tmp/tmp.FHVGMpAUwD ++++ rm /tmp/tmp.BXgLYuWjp5 /tmp/tmp.FHVGMpAUwD ++++ return 0 ++++ kubectl_bin get service/monitoring-service -o 'jsonpath={.status.loadBalancer.ingress[].hostname}' +++++ mktemp ++++ local LAST_OUT=/tmp/tmp.1qjyeUr2MJ +++++ mktemp ++++ local LAST_ERR=/tmp/tmp.HynbbFU8B9 ++++ local exit_status=0 ++++ local timeout=4 +++++ seq 0 2 ++++ for i in $(seq 0 2) ++++ set +e ++++ kubectl get service/monitoring-service -o 'jsonpath={.status.loadBalancer.ingress[].hostname}' ++++ exit_status=0 ++++ set -e ++++ '[' 0 '!=' 0 -a -n 1 ']' ++++ break ++++ cat /tmp/tmp.1qjyeUr2MJ ++++ cat /tmp/tmp.HynbbFU8B9 ++++ rm /tmp/tmp.1qjyeUr2MJ /tmp/tmp.HynbbFU8B9 ++++ return 0 +++ kubectl_bin exec -n monitoring-pmm3-6334 monitoring-server-0 -- pmm-admin --server-url=https://admin:admin@136.115.177.16/ --server-insecure-tls inventory list nodes --node-type=CONTAINER_NODE ++++ mktemp +++ local LAST_OUT=/tmp/tmp.atjduBuTsb ++++ mktemp +++ local LAST_ERR=/tmp/tmp.E2PauTcRRH +++ local exit_status=0 +++ local timeout=4 ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl exec -n monitoring-pmm3-6334 monitoring-server-0 -- pmm-admin --server-url=https://admin:admin@136.115.177.16/ --server-insecure-tls inventory list nodes --node-type=CONTAINER_NODE +++ exit_status=1 +++ set -e +++ '[' 1 '!=' 0 -a -n 1 ']' +++ cat /tmp/tmp.atjduBuTsb +++ cat /tmp/tmp.E2PauTcRRH command terminated with exit code 1 +++ sleep 0 +++ for i in $(seq 0 2) +++ set +e +++ kubectl exec -n monitoring-pmm3-6334 monitoring-server-0 -- pmm-admin --server-url=https://admin:admin@136.115.177.16/ --server-insecure-tls inventory list nodes --node-type=CONTAINER_NODE +++ exit_status=1 +++ set -e +++ '[' 1 '!=' 0 -a -n 1 ']' +++ cat /tmp/tmp.atjduBuTsb +++ cat /tmp/tmp.E2PauTcRRH command terminated with exit code 1 +++ sleep 4 +++ for i in $(seq 0 2) +++ set +e +++ kubectl exec -n monitoring-pmm3-6334 monitoring-server-0 -- pmm-admin --server-url=https://admin:admin@136.115.177.16/ --server-insecure-tls inventory list nodes --node-type=CONTAINER_NODE +++ exit_status=1 +++ set -e +++ '[' 1 '!=' 0 -a -n 1 ']' +++ cat /tmp/tmp.atjduBuTsb +++ cat /tmp/tmp.E2PauTcRRH command terminated with exit code 1 +++ sleep 8 +++ cat /tmp/tmp.atjduBuTsb +++ cat /tmp/tmp.E2PauTcRRH command terminated with exit code 1 +++ rm /tmp/tmp.atjduBuTsb /tmp/tmp.E2PauTcRRH +++ return 1 ++ for node_id in "${nodeList[@]}" ++ nodeList_from_pmm+=($(kubectl_bin exec -n "${namespace}" monitoring-server-0 -- pmm-admin --server-url=https://admin:admin@$(get_pmm_service_ip monitoring-service)/ --server-insecure-tls inventory list nodes --node-type=CONTAINER_NODE | grep $node_id | awk '{print $4}')) +++ grep d5ea0912-0daa-4f9a-add2-98046719381a +++ awk '{print $4}' ++++ get_pmm_service_ip monitoring-service ++++ local service=monitoring-service ++++ kubectl_bin get service/monitoring-service -o 'jsonpath={.spec.type}' ++++ grep -q NotFound ++++ kubectl_bin get service/monitoring-service -o 'jsonpath={.status.loadBalancer.ingress[]}' ++++ grep -E -q 'hostname|ip' ++++ kubectl_bin get service/monitoring-service -o 'jsonpath={.status.loadBalancer.ingress[].ip}' +++++ mktemp ++++ local LAST_OUT=/tmp/tmp.IbCcKveqTN +++++ mktemp ++++ local LAST_ERR=/tmp/tmp.ssxF7rKfAA ++++ local exit_status=0 ++++ local timeout=4 +++++ seq 0 2 ++++ for i in $(seq 0 2) ++++ set +e ++++ kubectl get service/monitoring-service -o 'jsonpath={.status.loadBalancer.ingress[].ip}' ++++ exit_status=0 ++++ set -e ++++ '[' 0 '!=' 0 -a -n 1 ']' ++++ break ++++ cat /tmp/tmp.IbCcKveqTN ++++ cat /tmp/tmp.ssxF7rKfAA ++++ rm /tmp/tmp.IbCcKveqTN /tmp/tmp.ssxF7rKfAA ++++ return 0 ++++ kubectl_bin get service/monitoring-service -o 'jsonpath={.status.loadBalancer.ingress[].hostname}' +++++ mktemp ++++ local LAST_OUT=/tmp/tmp.CSo8thpPrG +++++ mktemp ++++ local LAST_ERR=/tmp/tmp.kbosd88Siv ++++ local exit_status=0 ++++ local timeout=4 +++++ seq 0 2 ++++ for i in $(seq 0 2) ++++ set +e ++++ kubectl get service/monitoring-service -o 'jsonpath={.status.loadBalancer.ingress[].hostname}' ++++ exit_status=0 ++++ set -e ++++ '[' 0 '!=' 0 -a -n 1 ']' ++++ break ++++ cat /tmp/tmp.CSo8thpPrG ++++ cat /tmp/tmp.kbosd88Siv ++++ rm /tmp/tmp.CSo8thpPrG /tmp/tmp.kbosd88Siv ++++ return 0 +++ kubectl_bin exec -n monitoring-pmm3-6334 monitoring-server-0 -- pmm-admin --server-url=https://admin:admin@136.115.177.16/ --server-insecure-tls inventory list nodes --node-type=CONTAINER_NODE ++++ mktemp +++ local LAST_OUT=/tmp/tmp.bZLvAZKsZO ++++ mktemp +++ local LAST_ERR=/tmp/tmp.JIMUKejAxH +++ local exit_status=0 +++ local timeout=4 ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl exec -n monitoring-pmm3-6334 monitoring-server-0 -- pmm-admin --server-url=https://admin:admin@136.115.177.16/ --server-insecure-tls inventory list nodes --node-type=CONTAINER_NODE +++ exit_status=1 +++ set -e +++ '[' 1 '!=' 0 -a -n 1 ']' +++ cat /tmp/tmp.bZLvAZKsZO +++ cat /tmp/tmp.JIMUKejAxH command terminated with exit code 1 +++ sleep 0 +++ for i in $(seq 0 2) +++ set +e +++ kubectl exec -n monitoring-pmm3-6334 monitoring-server-0 -- pmm-admin --server-url=https://admin:admin@136.115.177.16/ --server-insecure-tls inventory list nodes --node-type=CONTAINER_NODE +++ exit_status=1 +++ set -e +++ '[' 1 '!=' 0 -a -n 1 ']' +++ cat /tmp/tmp.bZLvAZKsZO +++ cat /tmp/tmp.JIMUKejAxH command terminated with exit code 1 +++ sleep 4 +++ for i in $(seq 0 2) +++ set +e +++ kubectl exec -n monitoring-pmm3-6334 monitoring-server-0 -- pmm-admin --server-url=https://admin:admin@136.115.177.16/ --server-insecure-tls inventory list nodes --node-type=CONTAINER_NODE +++ exit_status=1 +++ set -e +++ '[' 1 '!=' 0 -a -n 1 ']' +++ cat /tmp/tmp.bZLvAZKsZO +++ cat /tmp/tmp.JIMUKejAxH command terminated with exit code 1 +++ sleep 8 +++ cat /tmp/tmp.bZLvAZKsZO +++ cat /tmp/tmp.JIMUKejAxH command terminated with exit code 1 +++ rm /tmp/tmp.bZLvAZKsZO /tmp/tmp.JIMUKejAxH +++ return 1 ++ echo + kubectl_bin patch psmdb monitoring-pmm3 --type json '-p=[{"op":"add","path":"/spec/pause","value":true}]' ++ mktemp + local LAST_OUT=/tmp/tmp.Dh5L5Ujlky ++ mktemp + local LAST_ERR=/tmp/tmp.aKfUBjbW4u + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl patch psmdb monitoring-pmm3 --type json '-p=[{"op":"add","path":"/spec/pause","value":true}]' + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.Dh5L5Ujlky perconaservermongodb.psmdb.percona.com/monitoring-pmm3 patched + cat /tmp/tmp.aKfUBjbW4u + rm /tmp/tmp.Dh5L5Ujlky /tmp/tmp.aKfUBjbW4u + return 0 + wait_for_delete pod/monitoring-pmm3-mongos-0 + local res=pod/monitoring-pmm3-mongos-0 + local wait_time=60 + set +o xtrace waiting for pod/monitoring-pmm3-mongos-0 to be deleted.........................Error from server (NotFound): pods "monitoring-pmm3-mongos-0" not found Error from server (NotFound): pods "monitoring-pmm3-mongos-0" not found Error from server (NotFound): pods "monitoring-pmm3-mongos-0" not found Error from server (NotFound): pods "monitoring-pmm3-mongos-0" not found + wait_for_delete pod/monitoring-pmm3-rs0-0 + local res=pod/monitoring-pmm3-rs0-0 + local wait_time=60 + set +o xtrace waiting for pod/monitoring-pmm3-rs0-0 to be deleted............Error from server (NotFound): pods "monitoring-pmm3-rs0-0" not found Error from server (NotFound): pods "monitoring-pmm3-rs0-0" not found Error from server (NotFound): pods "monitoring-pmm3-rs0-0" not found Error from server (NotFound): pods "monitoring-pmm3-rs0-0" not found + wait_for_delete pod/monitoring-pmm3-cfg-0 + local res=pod/monitoring-pmm3-cfg-0 + local wait_time=60 + set +o xtrace waiting for pod/monitoring-pmm3-cfg-0 to be deleted.......Error from server (NotFound): pods "monitoring-pmm3-cfg-0" not found Error from server (NotFound): pods "monitoring-pmm3-cfg-0" not found Error from server (NotFound): pods "monitoring-pmm3-cfg-0" not found Error from server (NotFound): pods "monitoring-pmm3-cfg-0" not found + desc 'check if services are not deleted' + set +o xtrace ----------------------------------------------------------------------------------- check if services are not deleted ----------------------------------------------------------------------------------- + kubectl_bin get svc monitoring-pmm3-rs0 ++ mktemp + local LAST_OUT=/tmp/tmp.5Wwq3YJFiK ++ mktemp + local LAST_ERR=/tmp/tmp.EgnMfhqyhm + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl get svc monitoring-pmm3-rs0 + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.5Wwq3YJFiK NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE monitoring-pmm3-rs0 ClusterIP None 27019/TCP 18m + cat /tmp/tmp.EgnMfhqyhm + rm /tmp/tmp.5Wwq3YJFiK /tmp/tmp.EgnMfhqyhm + return 0 + kubectl_bin get svc monitoring-pmm3-cfg ++ mktemp + local LAST_OUT=/tmp/tmp.HFbWOHx4n2 ++ mktemp + local LAST_ERR=/tmp/tmp.mtl4uKQxlm + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl get svc monitoring-pmm3-cfg + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.HFbWOHx4n2 NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE monitoring-pmm3-cfg ClusterIP None 27019/TCP 18m + cat /tmp/tmp.mtl4uKQxlm + rm /tmp/tmp.HFbWOHx4n2 /tmp/tmp.mtl4uKQxlm + return 0 + kubectl_bin get svc monitoring-pmm3-mongos ++ mktemp + local LAST_OUT=/tmp/tmp.ye1pZyOkdg ++ mktemp + local LAST_ERR=/tmp/tmp.U46t3tqm9M + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl get svc monitoring-pmm3-mongos + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.ye1pZyOkdg NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE monitoring-pmm3-mongos ClusterIP 34.118.238.184 27019/TCP 18m + cat /tmp/tmp.U46t3tqm9M + rm /tmp/tmp.ye1pZyOkdg /tmp/tmp.U46t3tqm9M + return 0 + does_node_id_exists_in_pmm=($(does_node_id_exists "${nodeList[@]}")) ++ does_node_id_exists 0c227efd-580e-4d7f-8616-937e5c5fe091 a433bf0f-bd6c-4624-ac8e-66e66ccfbb22 2438b450-3893-49bc-8c44-5b423e7c50e7 bd1c8be8-f68c-4841-a902-35492e16b99f 5f2c700d-929d-4cb4-ad1c-58ed99181690 8804cd11-6a01-4e80-80e4-0cdad05e8275 cc6efe57-5e10-40bc-b773-d30293aeef21 de0c1bd1-0160-4675-b54a-34e0d154d8ba d5ea0912-0daa-4f9a-add2-98046719381a ++ nodeList=('0c227efd-580e-4d7f-8616-937e5c5fe091' 'a433bf0f-bd6c-4624-ac8e-66e66ccfbb22' '2438b450-3893-49bc-8c44-5b423e7c50e7' 'bd1c8be8-f68c-4841-a902-35492e16b99f' '5f2c700d-929d-4cb4-ad1c-58ed99181690' '8804cd11-6a01-4e80-80e4-0cdad05e8275' 'cc6efe57-5e10-40bc-b773-d30293aeef21' 'de0c1bd1-0160-4675-b54a-34e0d154d8ba' 'd5ea0912-0daa-4f9a-add2-98046719381a') ++ local -a nodeList ++ nodeList_from_pmm=() ++ local -a nodeList_from_pmm ++ for node_id in "${nodeList[@]}" ++ nodeList_from_pmm+=($(kubectl_bin exec -n "${namespace}" monitoring-server-0 -- pmm-admin --server-url=https://admin:admin@$(get_pmm_service_ip monitoring-service)/ --server-insecure-tls inventory list nodes --node-type=CONTAINER_NODE | grep $node_id | awk '{print $4}')) +++ grep 0c227efd-580e-4d7f-8616-937e5c5fe091 +++ awk '{print $4}' ++++ get_pmm_service_ip monitoring-service ++++ local service=monitoring-service ++++ kubectl_bin get service/monitoring-service -o 'jsonpath={.spec.type}' ++++ grep -q NotFound ++++ kubectl_bin get service/monitoring-service -o 'jsonpath={.status.loadBalancer.ingress[]}' ++++ grep -E -q 'hostname|ip' ++++ kubectl_bin get service/monitoring-service -o 'jsonpath={.status.loadBalancer.ingress[].ip}' +++++ mktemp ++++ local LAST_OUT=/tmp/tmp.UFIqrpycN3 +++++ mktemp ++++ local LAST_ERR=/tmp/tmp.t0RkCunpKB ++++ local exit_status=0 ++++ local timeout=4 +++++ seq 0 2 ++++ for i in $(seq 0 2) ++++ set +e ++++ kubectl get service/monitoring-service -o 'jsonpath={.status.loadBalancer.ingress[].ip}' ++++ exit_status=0 ++++ set -e ++++ '[' 0 '!=' 0 -a -n 1 ']' ++++ break ++++ cat /tmp/tmp.UFIqrpycN3 ++++ cat /tmp/tmp.t0RkCunpKB ++++ rm /tmp/tmp.UFIqrpycN3 /tmp/tmp.t0RkCunpKB ++++ return 0 ++++ kubectl_bin get service/monitoring-service -o 'jsonpath={.status.loadBalancer.ingress[].hostname}' +++++ mktemp ++++ local LAST_OUT=/tmp/tmp.32f6mWIiiu +++++ mktemp ++++ local LAST_ERR=/tmp/tmp.MOPkdkTX6a ++++ local exit_status=0 ++++ local timeout=4 +++++ seq 0 2 ++++ for i in $(seq 0 2) ++++ set +e ++++ kubectl get service/monitoring-service -o 'jsonpath={.status.loadBalancer.ingress[].hostname}' ++++ exit_status=0 ++++ set -e ++++ '[' 0 '!=' 0 -a -n 1 ']' ++++ break ++++ cat /tmp/tmp.32f6mWIiiu ++++ cat /tmp/tmp.MOPkdkTX6a ++++ rm /tmp/tmp.32f6mWIiiu /tmp/tmp.MOPkdkTX6a ++++ return 0 +++ kubectl_bin exec -n monitoring-pmm3-6334 monitoring-server-0 -- pmm-admin --server-url=https://admin:admin@136.115.177.16/ --server-insecure-tls inventory list nodes --node-type=CONTAINER_NODE ++++ mktemp +++ local LAST_OUT=/tmp/tmp.XdnsnPmKtE ++++ mktemp +++ local LAST_ERR=/tmp/tmp.UD4OORQJ0m +++ local exit_status=0 +++ local timeout=4 ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl exec -n monitoring-pmm3-6334 monitoring-server-0 -- pmm-admin --server-url=https://admin:admin@136.115.177.16/ --server-insecure-tls inventory list nodes --node-type=CONTAINER_NODE +++ exit_status=1 +++ set -e +++ '[' 1 '!=' 0 -a -n 1 ']' +++ cat /tmp/tmp.XdnsnPmKtE +++ cat /tmp/tmp.UD4OORQJ0m command terminated with exit code 1 +++ sleep 0 +++ for i in $(seq 0 2) +++ set +e +++ kubectl exec -n monitoring-pmm3-6334 monitoring-server-0 -- pmm-admin --server-url=https://admin:admin@136.115.177.16/ --server-insecure-tls inventory list nodes --node-type=CONTAINER_NODE +++ exit_status=1 +++ set -e +++ '[' 1 '!=' 0 -a -n 1 ']' +++ cat /tmp/tmp.XdnsnPmKtE +++ cat /tmp/tmp.UD4OORQJ0m command terminated with exit code 1 +++ sleep 4 +++ for i in $(seq 0 2) +++ set +e +++ kubectl exec -n monitoring-pmm3-6334 monitoring-server-0 -- pmm-admin --server-url=https://admin:admin@136.115.177.16/ --server-insecure-tls inventory list nodes --node-type=CONTAINER_NODE +++ exit_status=1 +++ set -e +++ '[' 1 '!=' 0 -a -n 1 ']' +++ cat /tmp/tmp.XdnsnPmKtE +++ cat /tmp/tmp.UD4OORQJ0m command terminated with exit code 1 +++ sleep 8 +++ cat /tmp/tmp.XdnsnPmKtE +++ cat /tmp/tmp.UD4OORQJ0m command terminated with exit code 1 +++ rm /tmp/tmp.XdnsnPmKtE /tmp/tmp.UD4OORQJ0m +++ return 1 ++ for node_id in "${nodeList[@]}" ++ nodeList_from_pmm+=($(kubectl_bin exec -n "${namespace}" monitoring-server-0 -- pmm-admin --server-url=https://admin:admin@$(get_pmm_service_ip monitoring-service)/ --server-insecure-tls inventory list nodes --node-type=CONTAINER_NODE | grep $node_id | awk '{print $4}')) +++ grep a433bf0f-bd6c-4624-ac8e-66e66ccfbb22 +++ awk '{print $4}' ++++ get_pmm_service_ip monitoring-service ++++ local service=monitoring-service ++++ kubectl_bin get service/monitoring-service -o 'jsonpath={.spec.type}' ++++ grep -q NotFound ++++ kubectl_bin get service/monitoring-service -o 'jsonpath={.status.loadBalancer.ingress[]}' ++++ grep -E -q 'hostname|ip' ++++ kubectl_bin get service/monitoring-service -o 'jsonpath={.status.loadBalancer.ingress[].ip}' +++++ mktemp ++++ local LAST_OUT=/tmp/tmp.Za9xOmMA6y +++++ mktemp ++++ local LAST_ERR=/tmp/tmp.xswOpDMcBS ++++ local exit_status=0 ++++ local timeout=4 +++++ seq 0 2 ++++ for i in $(seq 0 2) ++++ set +e ++++ kubectl get service/monitoring-service -o 'jsonpath={.status.loadBalancer.ingress[].ip}' ++++ exit_status=0 ++++ set -e ++++ '[' 0 '!=' 0 -a -n 1 ']' ++++ break ++++ cat /tmp/tmp.Za9xOmMA6y ++++ cat /tmp/tmp.xswOpDMcBS ++++ rm /tmp/tmp.Za9xOmMA6y /tmp/tmp.xswOpDMcBS ++++ return 0 ++++ kubectl_bin get service/monitoring-service -o 'jsonpath={.status.loadBalancer.ingress[].hostname}' +++++ mktemp ++++ local LAST_OUT=/tmp/tmp.r6BHnjmGKG +++++ mktemp ++++ local LAST_ERR=/tmp/tmp.ekrY9tAg29 ++++ local exit_status=0 ++++ local timeout=4 +++++ seq 0 2 ++++ for i in $(seq 0 2) ++++ set +e ++++ kubectl get service/monitoring-service -o 'jsonpath={.status.loadBalancer.ingress[].hostname}' ++++ exit_status=0 ++++ set -e ++++ '[' 0 '!=' 0 -a -n 1 ']' ++++ break ++++ cat /tmp/tmp.r6BHnjmGKG ++++ cat /tmp/tmp.ekrY9tAg29 ++++ rm /tmp/tmp.r6BHnjmGKG /tmp/tmp.ekrY9tAg29 ++++ return 0 +++ kubectl_bin exec -n monitoring-pmm3-6334 monitoring-server-0 -- pmm-admin --server-url=https://admin:admin@136.115.177.16/ --server-insecure-tls inventory list nodes --node-type=CONTAINER_NODE ++++ mktemp +++ local LAST_OUT=/tmp/tmp.r3VfCBcnHV ++++ mktemp +++ local LAST_ERR=/tmp/tmp.DGKnDiOV1i +++ local exit_status=0 +++ local timeout=4 ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl exec -n monitoring-pmm3-6334 monitoring-server-0 -- pmm-admin --server-url=https://admin:admin@136.115.177.16/ --server-insecure-tls inventory list nodes --node-type=CONTAINER_NODE +++ exit_status=1 +++ set -e +++ '[' 1 '!=' 0 -a -n 1 ']' +++ cat /tmp/tmp.r3VfCBcnHV +++ cat /tmp/tmp.DGKnDiOV1i command terminated with exit code 1 +++ sleep 0 +++ for i in $(seq 0 2) +++ set +e +++ kubectl exec -n monitoring-pmm3-6334 monitoring-server-0 -- pmm-admin --server-url=https://admin:admin@136.115.177.16/ --server-insecure-tls inventory list nodes --node-type=CONTAINER_NODE +++ exit_status=1 +++ set -e +++ '[' 1 '!=' 0 -a -n 1 ']' +++ cat /tmp/tmp.r3VfCBcnHV +++ cat /tmp/tmp.DGKnDiOV1i command terminated with exit code 1 +++ sleep 4 +++ for i in $(seq 0 2) +++ set +e +++ kubectl exec -n monitoring-pmm3-6334 monitoring-server-0 -- pmm-admin --server-url=https://admin:admin@136.115.177.16/ --server-insecure-tls inventory list nodes --node-type=CONTAINER_NODE +++ exit_status=1 +++ set -e +++ '[' 1 '!=' 0 -a -n 1 ']' +++ cat /tmp/tmp.r3VfCBcnHV +++ cat /tmp/tmp.DGKnDiOV1i command terminated with exit code 1 +++ sleep 8 +++ cat /tmp/tmp.r3VfCBcnHV +++ cat /tmp/tmp.DGKnDiOV1i command terminated with exit code 1 +++ rm /tmp/tmp.r3VfCBcnHV /tmp/tmp.DGKnDiOV1i +++ return 1 ++ for node_id in "${nodeList[@]}" ++ nodeList_from_pmm+=($(kubectl_bin exec -n "${namespace}" monitoring-server-0 -- pmm-admin --server-url=https://admin:admin@$(get_pmm_service_ip monitoring-service)/ --server-insecure-tls inventory list nodes --node-type=CONTAINER_NODE | grep $node_id | awk '{print $4}')) +++ grep 2438b450-3893-49bc-8c44-5b423e7c50e7 +++ awk '{print $4}' ++++ get_pmm_service_ip monitoring-service ++++ local service=monitoring-service ++++ kubectl_bin get service/monitoring-service -o 'jsonpath={.spec.type}' ++++ grep -q NotFound ++++ kubectl_bin get service/monitoring-service -o 'jsonpath={.status.loadBalancer.ingress[]}' ++++ grep -E -q 'hostname|ip' ++++ kubectl_bin get service/monitoring-service -o 'jsonpath={.status.loadBalancer.ingress[].ip}' +++++ mktemp ++++ local LAST_OUT=/tmp/tmp.QkqAJckKaB +++++ mktemp ++++ local LAST_ERR=/tmp/tmp.mbCEB5tyEL ++++ local exit_status=0 ++++ local timeout=4 +++++ seq 0 2 ++++ for i in $(seq 0 2) ++++ set +e ++++ kubectl get service/monitoring-service -o 'jsonpath={.status.loadBalancer.ingress[].ip}' ++++ exit_status=0 ++++ set -e ++++ '[' 0 '!=' 0 -a -n 1 ']' ++++ break ++++ cat /tmp/tmp.QkqAJckKaB ++++ cat /tmp/tmp.mbCEB5tyEL ++++ rm /tmp/tmp.QkqAJckKaB /tmp/tmp.mbCEB5tyEL ++++ return 0 ++++ kubectl_bin get service/monitoring-service -o 'jsonpath={.status.loadBalancer.ingress[].hostname}' +++++ mktemp ++++ local LAST_OUT=/tmp/tmp.WzBlmWXsP8 +++++ mktemp ++++ local LAST_ERR=/tmp/tmp.RwFAlhyNSo ++++ local exit_status=0 ++++ local timeout=4 +++++ seq 0 2 ++++ for i in $(seq 0 2) ++++ set +e ++++ kubectl get service/monitoring-service -o 'jsonpath={.status.loadBalancer.ingress[].hostname}' ++++ exit_status=0 ++++ set -e ++++ '[' 0 '!=' 0 -a -n 1 ']' ++++ break ++++ cat /tmp/tmp.WzBlmWXsP8 ++++ cat /tmp/tmp.RwFAlhyNSo ++++ rm /tmp/tmp.WzBlmWXsP8 /tmp/tmp.RwFAlhyNSo ++++ return 0 +++ kubectl_bin exec -n monitoring-pmm3-6334 monitoring-server-0 -- pmm-admin --server-url=https://admin:admin@136.115.177.16/ --server-insecure-tls inventory list nodes --node-type=CONTAINER_NODE ++++ mktemp +++ local LAST_OUT=/tmp/tmp.gsBpxgDtvN ++++ mktemp +++ local LAST_ERR=/tmp/tmp.wo13sZt2wu +++ local exit_status=0 +++ local timeout=4 ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl exec -n monitoring-pmm3-6334 monitoring-server-0 -- pmm-admin --server-url=https://admin:admin@136.115.177.16/ --server-insecure-tls inventory list nodes --node-type=CONTAINER_NODE +++ exit_status=1 +++ set -e +++ '[' 1 '!=' 0 -a -n 1 ']' +++ cat /tmp/tmp.gsBpxgDtvN +++ cat /tmp/tmp.wo13sZt2wu command terminated with exit code 1 +++ sleep 0 +++ for i in $(seq 0 2) +++ set +e +++ kubectl exec -n monitoring-pmm3-6334 monitoring-server-0 -- pmm-admin --server-url=https://admin:admin@136.115.177.16/ --server-insecure-tls inventory list nodes --node-type=CONTAINER_NODE +++ exit_status=1 +++ set -e +++ '[' 1 '!=' 0 -a -n 1 ']' +++ cat /tmp/tmp.gsBpxgDtvN +++ cat /tmp/tmp.wo13sZt2wu command terminated with exit code 1 +++ sleep 4 +++ for i in $(seq 0 2) +++ set +e +++ kubectl exec -n monitoring-pmm3-6334 monitoring-server-0 -- pmm-admin --server-url=https://admin:admin@136.115.177.16/ --server-insecure-tls inventory list nodes --node-type=CONTAINER_NODE +++ exit_status=1 +++ set -e +++ '[' 1 '!=' 0 -a -n 1 ']' +++ cat /tmp/tmp.gsBpxgDtvN +++ cat /tmp/tmp.wo13sZt2wu command terminated with exit code 1 +++ sleep 8 +++ cat /tmp/tmp.gsBpxgDtvN +++ cat /tmp/tmp.wo13sZt2wu command terminated with exit code 1 +++ rm /tmp/tmp.gsBpxgDtvN /tmp/tmp.wo13sZt2wu +++ return 1 ++ for node_id in "${nodeList[@]}" ++ nodeList_from_pmm+=($(kubectl_bin exec -n "${namespace}" monitoring-server-0 -- pmm-admin --server-url=https://admin:admin@$(get_pmm_service_ip monitoring-service)/ --server-insecure-tls inventory list nodes --node-type=CONTAINER_NODE | grep $node_id | awk '{print $4}')) +++ grep bd1c8be8-f68c-4841-a902-35492e16b99f +++ awk '{print $4}' ++++ get_pmm_service_ip monitoring-service ++++ local service=monitoring-service ++++ kubectl_bin get service/monitoring-service -o 'jsonpath={.spec.type}' ++++ grep -q NotFound ++++ kubectl_bin get service/monitoring-service -o 'jsonpath={.status.loadBalancer.ingress[]}' ++++ grep -E -q 'hostname|ip' ++++ kubectl_bin get service/monitoring-service -o 'jsonpath={.status.loadBalancer.ingress[].ip}' +++++ mktemp ++++ local LAST_OUT=/tmp/tmp.oBmlRlrem1 +++++ mktemp ++++ local LAST_ERR=/tmp/tmp.23uMxDfal3 ++++ local exit_status=0 ++++ local timeout=4 +++++ seq 0 2 ++++ for i in $(seq 0 2) ++++ set +e ++++ kubectl get service/monitoring-service -o 'jsonpath={.status.loadBalancer.ingress[].ip}' ++++ exit_status=0 ++++ set -e ++++ '[' 0 '!=' 0 -a -n 1 ']' ++++ break ++++ cat /tmp/tmp.oBmlRlrem1 ++++ cat /tmp/tmp.23uMxDfal3 ++++ rm /tmp/tmp.oBmlRlrem1 /tmp/tmp.23uMxDfal3 ++++ return 0 ++++ kubectl_bin get service/monitoring-service -o 'jsonpath={.status.loadBalancer.ingress[].hostname}' +++++ mktemp ++++ local LAST_OUT=/tmp/tmp.IsIXRXZlCU +++++ mktemp ++++ local LAST_ERR=/tmp/tmp.tFyQqk1OIS ++++ local exit_status=0 ++++ local timeout=4 +++++ seq 0 2 ++++ for i in $(seq 0 2) ++++ set +e ++++ kubectl get service/monitoring-service -o 'jsonpath={.status.loadBalancer.ingress[].hostname}' ++++ exit_status=0 ++++ set -e ++++ '[' 0 '!=' 0 -a -n 1 ']' ++++ break ++++ cat /tmp/tmp.IsIXRXZlCU ++++ cat /tmp/tmp.tFyQqk1OIS ++++ rm /tmp/tmp.IsIXRXZlCU /tmp/tmp.tFyQqk1OIS ++++ return 0 +++ kubectl_bin exec -n monitoring-pmm3-6334 monitoring-server-0 -- pmm-admin --server-url=https://admin:admin@136.115.177.16/ --server-insecure-tls inventory list nodes --node-type=CONTAINER_NODE ++++ mktemp +++ local LAST_OUT=/tmp/tmp.HFx330Kicm ++++ mktemp +++ local LAST_ERR=/tmp/tmp.N3XLuC8kHp +++ local exit_status=0 +++ local timeout=4 ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl exec -n monitoring-pmm3-6334 monitoring-server-0 -- pmm-admin --server-url=https://admin:admin@136.115.177.16/ --server-insecure-tls inventory list nodes --node-type=CONTAINER_NODE +++ exit_status=1 +++ set -e +++ '[' 1 '!=' 0 -a -n 1 ']' +++ cat /tmp/tmp.HFx330Kicm +++ cat /tmp/tmp.N3XLuC8kHp command terminated with exit code 1 +++ sleep 0 +++ for i in $(seq 0 2) +++ set +e +++ kubectl exec -n monitoring-pmm3-6334 monitoring-server-0 -- pmm-admin --server-url=https://admin:admin@136.115.177.16/ --server-insecure-tls inventory list nodes --node-type=CONTAINER_NODE +++ exit_status=1 +++ set -e +++ '[' 1 '!=' 0 -a -n 1 ']' +++ cat /tmp/tmp.HFx330Kicm +++ cat /tmp/tmp.N3XLuC8kHp command terminated with exit code 1 +++ sleep 4 +++ for i in $(seq 0 2) +++ set +e +++ kubectl exec -n monitoring-pmm3-6334 monitoring-server-0 -- pmm-admin --server-url=https://admin:admin@136.115.177.16/ --server-insecure-tls inventory list nodes --node-type=CONTAINER_NODE +++ exit_status=1 +++ set -e +++ '[' 1 '!=' 0 -a -n 1 ']' +++ cat /tmp/tmp.HFx330Kicm +++ cat /tmp/tmp.N3XLuC8kHp command terminated with exit code 1 +++ sleep 8 +++ cat /tmp/tmp.HFx330Kicm +++ cat /tmp/tmp.N3XLuC8kHp command terminated with exit code 1 +++ rm /tmp/tmp.HFx330Kicm /tmp/tmp.N3XLuC8kHp +++ return 1 ++ for node_id in "${nodeList[@]}" ++ nodeList_from_pmm+=($(kubectl_bin exec -n "${namespace}" monitoring-server-0 -- pmm-admin --server-url=https://admin:admin@$(get_pmm_service_ip monitoring-service)/ --server-insecure-tls inventory list nodes --node-type=CONTAINER_NODE | grep $node_id | awk '{print $4}')) +++ awk '{print $4}' +++ grep 5f2c700d-929d-4cb4-ad1c-58ed99181690 ++++ get_pmm_service_ip monitoring-service ++++ local service=monitoring-service ++++ kubectl_bin get service/monitoring-service -o 'jsonpath={.spec.type}' ++++ grep -q NotFound ++++ kubectl_bin get service/monitoring-service -o 'jsonpath={.status.loadBalancer.ingress[]}' ++++ grep -E -q 'hostname|ip' ++++ kubectl_bin get service/monitoring-service -o 'jsonpath={.status.loadBalancer.ingress[].ip}' +++++ mktemp ++++ local LAST_OUT=/tmp/tmp.qciQtnfuhL +++++ mktemp ++++ local LAST_ERR=/tmp/tmp.buUBWt22er ++++ local exit_status=0 ++++ local timeout=4 +++++ seq 0 2 ++++ for i in $(seq 0 2) ++++ set +e ++++ kubectl get service/monitoring-service -o 'jsonpath={.status.loadBalancer.ingress[].ip}' ++++ exit_status=0 ++++ set -e ++++ '[' 0 '!=' 0 -a -n 1 ']' ++++ break ++++ cat /tmp/tmp.qciQtnfuhL ++++ cat /tmp/tmp.buUBWt22er ++++ rm /tmp/tmp.qciQtnfuhL /tmp/tmp.buUBWt22er ++++ return 0 ++++ kubectl_bin get service/monitoring-service -o 'jsonpath={.status.loadBalancer.ingress[].hostname}' +++++ mktemp ++++ local LAST_OUT=/tmp/tmp.iW54AFqFQ7 +++++ mktemp ++++ local LAST_ERR=/tmp/tmp.cApvgXPtU5 ++++ local exit_status=0 ++++ local timeout=4 +++++ seq 0 2 ++++ for i in $(seq 0 2) ++++ set +e ++++ kubectl get service/monitoring-service -o 'jsonpath={.status.loadBalancer.ingress[].hostname}' ++++ exit_status=0 ++++ set -e ++++ '[' 0 '!=' 0 -a -n 1 ']' ++++ break ++++ cat /tmp/tmp.iW54AFqFQ7 ++++ cat /tmp/tmp.cApvgXPtU5 ++++ rm /tmp/tmp.iW54AFqFQ7 /tmp/tmp.cApvgXPtU5 ++++ return 0 +++ kubectl_bin exec -n monitoring-pmm3-6334 monitoring-server-0 -- pmm-admin --server-url=https://admin:admin@136.115.177.16/ --server-insecure-tls inventory list nodes --node-type=CONTAINER_NODE ++++ mktemp +++ local LAST_OUT=/tmp/tmp.G5Bup0yNzz ++++ mktemp +++ local LAST_ERR=/tmp/tmp.3dKcoROHar +++ local exit_status=0 +++ local timeout=4 ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl exec -n monitoring-pmm3-6334 monitoring-server-0 -- pmm-admin --server-url=https://admin:admin@136.115.177.16/ --server-insecure-tls inventory list nodes --node-type=CONTAINER_NODE +++ exit_status=1 +++ set -e +++ '[' 1 '!=' 0 -a -n 1 ']' +++ cat /tmp/tmp.G5Bup0yNzz +++ cat /tmp/tmp.3dKcoROHar command terminated with exit code 1 +++ sleep 0 +++ for i in $(seq 0 2) +++ set +e +++ kubectl exec -n monitoring-pmm3-6334 monitoring-server-0 -- pmm-admin --server-url=https://admin:admin@136.115.177.16/ --server-insecure-tls inventory list nodes --node-type=CONTAINER_NODE +++ exit_status=1 +++ set -e +++ '[' 1 '!=' 0 -a -n 1 ']' +++ cat /tmp/tmp.G5Bup0yNzz +++ cat /tmp/tmp.3dKcoROHar command terminated with exit code 1 +++ sleep 4 +++ for i in $(seq 0 2) +++ set +e +++ kubectl exec -n monitoring-pmm3-6334 monitoring-server-0 -- pmm-admin --server-url=https://admin:admin@136.115.177.16/ --server-insecure-tls inventory list nodes --node-type=CONTAINER_NODE +++ exit_status=1 +++ set -e +++ '[' 1 '!=' 0 -a -n 1 ']' +++ cat /tmp/tmp.G5Bup0yNzz +++ cat /tmp/tmp.3dKcoROHar command terminated with exit code 1 +++ sleep 8 +++ cat /tmp/tmp.G5Bup0yNzz +++ cat /tmp/tmp.3dKcoROHar command terminated with exit code 1 +++ rm /tmp/tmp.G5Bup0yNzz /tmp/tmp.3dKcoROHar +++ return 1 ++ for node_id in "${nodeList[@]}" ++ nodeList_from_pmm+=($(kubectl_bin exec -n "${namespace}" monitoring-server-0 -- pmm-admin --server-url=https://admin:admin@$(get_pmm_service_ip monitoring-service)/ --server-insecure-tls inventory list nodes --node-type=CONTAINER_NODE | grep $node_id | awk '{print $4}')) +++ grep 8804cd11-6a01-4e80-80e4-0cdad05e8275 +++ awk '{print $4}' ++++ get_pmm_service_ip monitoring-service ++++ local service=monitoring-service ++++ kubectl_bin get service/monitoring-service -o 'jsonpath={.spec.type}' ++++ grep -q NotFound ++++ kubectl_bin get service/monitoring-service -o 'jsonpath={.status.loadBalancer.ingress[]}' ++++ grep -E -q 'hostname|ip' ++++ kubectl_bin get service/monitoring-service -o 'jsonpath={.status.loadBalancer.ingress[].ip}' +++++ mktemp ++++ local LAST_OUT=/tmp/tmp.VDRuirJIDG +++++ mktemp ++++ local LAST_ERR=/tmp/tmp.zK6UZfsikR ++++ local exit_status=0 ++++ local timeout=4 +++++ seq 0 2 ++++ for i in $(seq 0 2) ++++ set +e ++++ kubectl get service/monitoring-service -o 'jsonpath={.status.loadBalancer.ingress[].ip}' ++++ exit_status=0 ++++ set -e ++++ '[' 0 '!=' 0 -a -n 1 ']' ++++ break ++++ cat /tmp/tmp.VDRuirJIDG ++++ cat /tmp/tmp.zK6UZfsikR ++++ rm /tmp/tmp.VDRuirJIDG /tmp/tmp.zK6UZfsikR ++++ return 0 ++++ kubectl_bin get service/monitoring-service -o 'jsonpath={.status.loadBalancer.ingress[].hostname}' +++++ mktemp ++++ local LAST_OUT=/tmp/tmp.HgHZyF6Rt4 +++++ mktemp ++++ local LAST_ERR=/tmp/tmp.5cmmWMbKyM ++++ local exit_status=0 ++++ local timeout=4 +++++ seq 0 2 ++++ for i in $(seq 0 2) ++++ set +e ++++ kubectl get service/monitoring-service -o 'jsonpath={.status.loadBalancer.ingress[].hostname}' ++++ exit_status=0 ++++ set -e ++++ '[' 0 '!=' 0 -a -n 1 ']' ++++ break ++++ cat /tmp/tmp.HgHZyF6Rt4 ++++ cat /tmp/tmp.5cmmWMbKyM ++++ rm /tmp/tmp.HgHZyF6Rt4 /tmp/tmp.5cmmWMbKyM ++++ return 0 +++ kubectl_bin exec -n monitoring-pmm3-6334 monitoring-server-0 -- pmm-admin --server-url=https://admin:admin@136.115.177.16/ --server-insecure-tls inventory list nodes --node-type=CONTAINER_NODE ++++ mktemp +++ local LAST_OUT=/tmp/tmp.o4XrYpAa3a ++++ mktemp +++ local LAST_ERR=/tmp/tmp.gHQdORWIcE +++ local exit_status=0 +++ local timeout=4 ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl exec -n monitoring-pmm3-6334 monitoring-server-0 -- pmm-admin --server-url=https://admin:admin@136.115.177.16/ --server-insecure-tls inventory list nodes --node-type=CONTAINER_NODE +++ exit_status=1 +++ set -e +++ '[' 1 '!=' 0 -a -n 1 ']' +++ cat /tmp/tmp.o4XrYpAa3a +++ cat /tmp/tmp.gHQdORWIcE command terminated with exit code 1 +++ sleep 0 +++ for i in $(seq 0 2) +++ set +e +++ kubectl exec -n monitoring-pmm3-6334 monitoring-server-0 -- pmm-admin --server-url=https://admin:admin@136.115.177.16/ --server-insecure-tls inventory list nodes --node-type=CONTAINER_NODE +++ exit_status=1 +++ set -e +++ '[' 1 '!=' 0 -a -n 1 ']' +++ cat /tmp/tmp.o4XrYpAa3a +++ cat /tmp/tmp.gHQdORWIcE command terminated with exit code 1 +++ sleep 4 +++ for i in $(seq 0 2) +++ set +e +++ kubectl exec -n monitoring-pmm3-6334 monitoring-server-0 -- pmm-admin --server-url=https://admin:admin@136.115.177.16/ --server-insecure-tls inventory list nodes --node-type=CONTAINER_NODE +++ exit_status=1 +++ set -e +++ '[' 1 '!=' 0 -a -n 1 ']' +++ cat /tmp/tmp.o4XrYpAa3a +++ cat /tmp/tmp.gHQdORWIcE command terminated with exit code 1 +++ sleep 8 +++ cat /tmp/tmp.o4XrYpAa3a +++ cat /tmp/tmp.gHQdORWIcE command terminated with exit code 1 +++ rm /tmp/tmp.o4XrYpAa3a /tmp/tmp.gHQdORWIcE +++ return 1 ++ for node_id in "${nodeList[@]}" ++ nodeList_from_pmm+=($(kubectl_bin exec -n "${namespace}" monitoring-server-0 -- pmm-admin --server-url=https://admin:admin@$(get_pmm_service_ip monitoring-service)/ --server-insecure-tls inventory list nodes --node-type=CONTAINER_NODE | grep $node_id | awk '{print $4}')) +++ grep cc6efe57-5e10-40bc-b773-d30293aeef21 +++ awk '{print $4}' ++++ get_pmm_service_ip monitoring-service ++++ local service=monitoring-service ++++ kubectl_bin get service/monitoring-service -o 'jsonpath={.spec.type}' ++++ grep -q NotFound ++++ kubectl_bin get service/monitoring-service -o 'jsonpath={.status.loadBalancer.ingress[]}' ++++ grep -E -q 'hostname|ip' ++++ kubectl_bin get service/monitoring-service -o 'jsonpath={.status.loadBalancer.ingress[].ip}' +++++ mktemp ++++ local LAST_OUT=/tmp/tmp.KWCOdsquhu +++++ mktemp ++++ local LAST_ERR=/tmp/tmp.ZF1IDTZgVf ++++ local exit_status=0 ++++ local timeout=4 +++++ seq 0 2 ++++ for i in $(seq 0 2) ++++ set +e ++++ kubectl get service/monitoring-service -o 'jsonpath={.status.loadBalancer.ingress[].ip}' ++++ exit_status=0 ++++ set -e ++++ '[' 0 '!=' 0 -a -n 1 ']' ++++ break ++++ cat /tmp/tmp.KWCOdsquhu ++++ cat /tmp/tmp.ZF1IDTZgVf ++++ rm /tmp/tmp.KWCOdsquhu /tmp/tmp.ZF1IDTZgVf ++++ return 0 ++++ kubectl_bin get service/monitoring-service -o 'jsonpath={.status.loadBalancer.ingress[].hostname}' +++++ mktemp ++++ local LAST_OUT=/tmp/tmp.eigQT8nrEj +++++ mktemp ++++ local LAST_ERR=/tmp/tmp.5MW4avLNmu ++++ local exit_status=0 ++++ local timeout=4 +++++ seq 0 2 ++++ for i in $(seq 0 2) ++++ set +e ++++ kubectl get service/monitoring-service -o 'jsonpath={.status.loadBalancer.ingress[].hostname}' ++++ exit_status=0 ++++ set -e ++++ '[' 0 '!=' 0 -a -n 1 ']' ++++ break ++++ cat /tmp/tmp.eigQT8nrEj ++++ cat /tmp/tmp.5MW4avLNmu ++++ rm /tmp/tmp.eigQT8nrEj /tmp/tmp.5MW4avLNmu ++++ return 0 +++ kubectl_bin exec -n monitoring-pmm3-6334 monitoring-server-0 -- pmm-admin --server-url=https://admin:admin@136.115.177.16/ --server-insecure-tls inventory list nodes --node-type=CONTAINER_NODE ++++ mktemp +++ local LAST_OUT=/tmp/tmp.WJw8ZeBSJC ++++ mktemp +++ local LAST_ERR=/tmp/tmp.oHpMGoNTTw +++ local exit_status=0 +++ local timeout=4 ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl exec -n monitoring-pmm3-6334 monitoring-server-0 -- pmm-admin --server-url=https://admin:admin@136.115.177.16/ --server-insecure-tls inventory list nodes --node-type=CONTAINER_NODE +++ exit_status=1 +++ set -e +++ '[' 1 '!=' 0 -a -n 1 ']' +++ cat /tmp/tmp.WJw8ZeBSJC +++ cat /tmp/tmp.oHpMGoNTTw command terminated with exit code 1 +++ sleep 0 +++ for i in $(seq 0 2) +++ set +e +++ kubectl exec -n monitoring-pmm3-6334 monitoring-server-0 -- pmm-admin --server-url=https://admin:admin@136.115.177.16/ --server-insecure-tls inventory list nodes --node-type=CONTAINER_NODE +++ exit_status=1 +++ set -e +++ '[' 1 '!=' 0 -a -n 1 ']' +++ cat /tmp/tmp.WJw8ZeBSJC +++ cat /tmp/tmp.oHpMGoNTTw command terminated with exit code 1 +++ sleep 4 +++ for i in $(seq 0 2) +++ set +e +++ kubectl exec -n monitoring-pmm3-6334 monitoring-server-0 -- pmm-admin --server-url=https://admin:admin@136.115.177.16/ --server-insecure-tls inventory list nodes --node-type=CONTAINER_NODE +++ exit_status=1 +++ set -e +++ '[' 1 '!=' 0 -a -n 1 ']' +++ cat /tmp/tmp.WJw8ZeBSJC +++ cat /tmp/tmp.oHpMGoNTTw command terminated with exit code 1 +++ sleep 8 +++ cat /tmp/tmp.WJw8ZeBSJC +++ cat /tmp/tmp.oHpMGoNTTw command terminated with exit code 1 +++ rm /tmp/tmp.WJw8ZeBSJC /tmp/tmp.oHpMGoNTTw +++ return 1 ++ for node_id in "${nodeList[@]}" ++ nodeList_from_pmm+=($(kubectl_bin exec -n "${namespace}" monitoring-server-0 -- pmm-admin --server-url=https://admin:admin@$(get_pmm_service_ip monitoring-service)/ --server-insecure-tls inventory list nodes --node-type=CONTAINER_NODE | grep $node_id | awk '{print $4}')) +++ grep de0c1bd1-0160-4675-b54a-34e0d154d8ba +++ awk '{print $4}' ++++ get_pmm_service_ip monitoring-service ++++ local service=monitoring-service ++++ kubectl_bin get service/monitoring-service -o 'jsonpath={.spec.type}' ++++ grep -q NotFound ++++ kubectl_bin get service/monitoring-service -o 'jsonpath={.status.loadBalancer.ingress[]}' ++++ grep -E -q 'hostname|ip' ++++ kubectl_bin get service/monitoring-service -o 'jsonpath={.status.loadBalancer.ingress[].ip}' +++++ mktemp ++++ local LAST_OUT=/tmp/tmp.vtG19YeDo5 +++++ mktemp ++++ local LAST_ERR=/tmp/tmp.oxPkmILgns ++++ local exit_status=0 ++++ local timeout=4 +++++ seq 0 2 ++++ for i in $(seq 0 2) ++++ set +e ++++ kubectl get service/monitoring-service -o 'jsonpath={.status.loadBalancer.ingress[].ip}' ++++ exit_status=0 ++++ set -e ++++ '[' 0 '!=' 0 -a -n 1 ']' ++++ break ++++ cat /tmp/tmp.vtG19YeDo5 ++++ cat /tmp/tmp.oxPkmILgns ++++ rm /tmp/tmp.vtG19YeDo5 /tmp/tmp.oxPkmILgns ++++ return 0 ++++ kubectl_bin get service/monitoring-service -o 'jsonpath={.status.loadBalancer.ingress[].hostname}' +++++ mktemp ++++ local LAST_OUT=/tmp/tmp.L4TXhV0A3t +++++ mktemp ++++ local LAST_ERR=/tmp/tmp.HwGvvUetaC ++++ local exit_status=0 ++++ local timeout=4 +++++ seq 0 2 ++++ for i in $(seq 0 2) ++++ set +e ++++ kubectl get service/monitoring-service -o 'jsonpath={.status.loadBalancer.ingress[].hostname}' ++++ exit_status=0 ++++ set -e ++++ '[' 0 '!=' 0 -a -n 1 ']' ++++ break ++++ cat /tmp/tmp.L4TXhV0A3t ++++ cat /tmp/tmp.HwGvvUetaC ++++ rm /tmp/tmp.L4TXhV0A3t /tmp/tmp.HwGvvUetaC ++++ return 0 +++ kubectl_bin exec -n monitoring-pmm3-6334 monitoring-server-0 -- pmm-admin --server-url=https://admin:admin@136.115.177.16/ --server-insecure-tls inventory list nodes --node-type=CONTAINER_NODE ++++ mktemp +++ local LAST_OUT=/tmp/tmp.dORhYVZwhI ++++ mktemp +++ local LAST_ERR=/tmp/tmp.8TeKOnyQqk +++ local exit_status=0 +++ local timeout=4 ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl exec -n monitoring-pmm3-6334 monitoring-server-0 -- pmm-admin --server-url=https://admin:admin@136.115.177.16/ --server-insecure-tls inventory list nodes --node-type=CONTAINER_NODE +++ exit_status=1 +++ set -e +++ '[' 1 '!=' 0 -a -n 1 ']' +++ cat /tmp/tmp.dORhYVZwhI +++ cat /tmp/tmp.8TeKOnyQqk command terminated with exit code 1 +++ sleep 0 +++ for i in $(seq 0 2) +++ set +e +++ kubectl exec -n monitoring-pmm3-6334 monitoring-server-0 -- pmm-admin --server-url=https://admin:admin@136.115.177.16/ --server-insecure-tls inventory list nodes --node-type=CONTAINER_NODE +++ exit_status=1 +++ set -e +++ '[' 1 '!=' 0 -a -n 1 ']' +++ cat /tmp/tmp.dORhYVZwhI +++ cat /tmp/tmp.8TeKOnyQqk command terminated with exit code 1 +++ sleep 4 +++ for i in $(seq 0 2) +++ set +e +++ kubectl exec -n monitoring-pmm3-6334 monitoring-server-0 -- pmm-admin --server-url=https://admin:admin@136.115.177.16/ --server-insecure-tls inventory list nodes --node-type=CONTAINER_NODE +++ exit_status=1 +++ set -e +++ '[' 1 '!=' 0 -a -n 1 ']' +++ cat /tmp/tmp.dORhYVZwhI +++ cat /tmp/tmp.8TeKOnyQqk command terminated with exit code 1 +++ sleep 8 +++ cat /tmp/tmp.dORhYVZwhI +++ cat /tmp/tmp.8TeKOnyQqk command terminated with exit code 1 +++ rm /tmp/tmp.dORhYVZwhI /tmp/tmp.8TeKOnyQqk +++ return 1 ++ for node_id in "${nodeList[@]}" ++ nodeList_from_pmm+=($(kubectl_bin exec -n "${namespace}" monitoring-server-0 -- pmm-admin --server-url=https://admin:admin@$(get_pmm_service_ip monitoring-service)/ --server-insecure-tls inventory list nodes --node-type=CONTAINER_NODE | grep $node_id | awk '{print $4}')) +++ grep d5ea0912-0daa-4f9a-add2-98046719381a +++ awk '{print $4}' ++++ get_pmm_service_ip monitoring-service ++++ local service=monitoring-service ++++ kubectl_bin get service/monitoring-service -o 'jsonpath={.spec.type}' ++++ grep -q NotFound ++++ kubectl_bin get service/monitoring-service -o 'jsonpath={.status.loadBalancer.ingress[]}' ++++ grep -E -q 'hostname|ip' ++++ kubectl_bin get service/monitoring-service -o 'jsonpath={.status.loadBalancer.ingress[].ip}' +++++ mktemp ++++ local LAST_OUT=/tmp/tmp.YOddQoxh5z +++++ mktemp ++++ local LAST_ERR=/tmp/tmp.PS4ouHdV8E ++++ local exit_status=0 ++++ local timeout=4 +++++ seq 0 2 ++++ for i in $(seq 0 2) ++++ set +e ++++ kubectl get service/monitoring-service -o 'jsonpath={.status.loadBalancer.ingress[].ip}' ++++ exit_status=0 ++++ set -e ++++ '[' 0 '!=' 0 -a -n 1 ']' ++++ break ++++ cat /tmp/tmp.YOddQoxh5z ++++ cat /tmp/tmp.PS4ouHdV8E ++++ rm /tmp/tmp.YOddQoxh5z /tmp/tmp.PS4ouHdV8E ++++ return 0 ++++ kubectl_bin get service/monitoring-service -o 'jsonpath={.status.loadBalancer.ingress[].hostname}' +++++ mktemp ++++ local LAST_OUT=/tmp/tmp.ttG3tPKJ9k +++++ mktemp ++++ local LAST_ERR=/tmp/tmp.pNNZARpWg9 ++++ local exit_status=0 ++++ local timeout=4 +++++ seq 0 2 ++++ for i in $(seq 0 2) ++++ set +e ++++ kubectl get service/monitoring-service -o 'jsonpath={.status.loadBalancer.ingress[].hostname}' ++++ exit_status=0 ++++ set -e ++++ '[' 0 '!=' 0 -a -n 1 ']' ++++ break ++++ cat /tmp/tmp.ttG3tPKJ9k ++++ cat /tmp/tmp.pNNZARpWg9 ++++ rm /tmp/tmp.ttG3tPKJ9k /tmp/tmp.pNNZARpWg9 ++++ return 0 +++ kubectl_bin exec -n monitoring-pmm3-6334 monitoring-server-0 -- pmm-admin --server-url=https://admin:admin@136.115.177.16/ --server-insecure-tls inventory list nodes --node-type=CONTAINER_NODE ++++ mktemp +++ local LAST_OUT=/tmp/tmp.rcQTO0RZKC ++++ mktemp +++ local LAST_ERR=/tmp/tmp.MQWI3GTbrE +++ local exit_status=0 +++ local timeout=4 ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl exec -n monitoring-pmm3-6334 monitoring-server-0 -- pmm-admin --server-url=https://admin:admin@136.115.177.16/ --server-insecure-tls inventory list nodes --node-type=CONTAINER_NODE +++ exit_status=1 +++ set -e +++ '[' 1 '!=' 0 -a -n 1 ']' +++ cat /tmp/tmp.rcQTO0RZKC +++ cat /tmp/tmp.MQWI3GTbrE command terminated with exit code 1 +++ sleep 0 +++ for i in $(seq 0 2) +++ set +e +++ kubectl exec -n monitoring-pmm3-6334 monitoring-server-0 -- pmm-admin --server-url=https://admin:admin@136.115.177.16/ --server-insecure-tls inventory list nodes --node-type=CONTAINER_NODE +++ exit_status=1 +++ set -e +++ '[' 1 '!=' 0 -a -n 1 ']' +++ cat /tmp/tmp.rcQTO0RZKC +++ cat /tmp/tmp.MQWI3GTbrE command terminated with exit code 1 +++ sleep 4 +++ for i in $(seq 0 2) +++ set +e +++ kubectl exec -n monitoring-pmm3-6334 monitoring-server-0 -- pmm-admin --server-url=https://admin:admin@136.115.177.16/ --server-insecure-tls inventory list nodes --node-type=CONTAINER_NODE +++ exit_status=1 +++ set -e +++ '[' 1 '!=' 0 -a -n 1 ']' +++ cat /tmp/tmp.rcQTO0RZKC +++ cat /tmp/tmp.MQWI3GTbrE command terminated with exit code 1 +++ sleep 8 +++ cat /tmp/tmp.rcQTO0RZKC +++ cat /tmp/tmp.MQWI3GTbrE command terminated with exit code 1 +++ rm /tmp/tmp.rcQTO0RZKC /tmp/tmp.MQWI3GTbrE +++ return 1 ++ echo + [[ -n '' ]] ++ kubectl_bin logs monitoring-pmm3-rs0-0 pmm-client ++ grep -c 'cannot auto discover databases and collections' +++ mktemp ++ local LAST_OUT=/tmp/tmp.OmaqmjQ1Xg +++ mktemp ++ local LAST_ERR=/tmp/tmp.TkycVQPv6a ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl logs monitoring-pmm3-rs0-0 pmm-client ++ exit_status=1 ++ set -e ++ '[' 1 '!=' 0 -a -n 1 ']' ++ cat /tmp/tmp.OmaqmjQ1Xg ++ cat /tmp/tmp.TkycVQPv6a error: error from server (NotFound): pods "monitoring-pmm3-rs0-0" not found in namespace "monitoring-pmm3-6334" ++ sleep 0 ++ for i in $(seq 0 2) ++ set +e ++ kubectl logs monitoring-pmm3-rs0-0 pmm-client ++ exit_status=1 ++ set -e ++ '[' 1 '!=' 0 -a -n 1 ']' ++ cat /tmp/tmp.OmaqmjQ1Xg ++ cat /tmp/tmp.TkycVQPv6a error: error from server (NotFound): pods "monitoring-pmm3-rs0-0" not found in namespace "monitoring-pmm3-6334" ++ sleep 4 ++ for i in $(seq 0 2) ++ set +e ++ kubectl logs monitoring-pmm3-rs0-0 pmm-client ++ exit_status=1 ++ set -e ++ '[' 1 '!=' 0 -a -n 1 ']' ++ cat /tmp/tmp.OmaqmjQ1Xg ++ cat /tmp/tmp.TkycVQPv6a error: error from server (NotFound): pods "monitoring-pmm3-rs0-0" not found in namespace "monitoring-pmm3-6334" ++ sleep 8 ++ cat /tmp/tmp.OmaqmjQ1Xg ++ cat /tmp/tmp.TkycVQPv6a error: error from server (NotFound): pods "monitoring-pmm3-rs0-0" not found in namespace "monitoring-pmm3-6334" ++ rm /tmp/tmp.OmaqmjQ1Xg /tmp/tmp.TkycVQPv6a ++ return 1 + [[ 0 != 0 ]] + helm uninstall monitoring release "monitoring" uninstalled + destroy monitoring-pmm3-6334 + local namespace=monitoring-pmm3-6334 + local ignore_logs=true + [[ 1 == 1 ]] + echo 'SKIP_DELETE=1, not destroying monitoring-pmm3-6334' SKIP_DELETE=1, not destroying monitoring-pmm3-6334 + return + desc 'test passed' + set +o xtrace ----------------------------------------------------------------------------------- test passed -----------------------------------------------------------------------------------