Log: /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2465/e2e-tests/monitoring-2-0/../logs/monitoring-2-0.log + create_infra monitoring-2-0-3915 + local ns=monitoring-2-0-3915 + [[ 1 == 1 ]] + delete_crd + desc 'get and delete old CRDs and RBAC' + set +o xtrace ----------------------------------------------------------------------------------- get and delete old CRDs and RBAC ----------------------------------------------------------------------------------- + kubectl_bin delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2465/deploy/crd.yaml --ignore-not-found --wait=false ++ mktemp + local LAST_OUT=/tmp/tmp.VntcIFXBq2 ++ mktemp + local LAST_ERR=/tmp/tmp.g2ZUm1lds3 + local exit_status=0 + local timeout=4 + args=('delete' '-f' '/mnt/jenkins/workspace/cloud-psmdb-operator_PR-2465/deploy/crd.yaml' '--ignore-not-found' '--wait=false') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2465/deploy/crd.yaml --ignore-not-found --wait=false + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.VntcIFXBq2 customresourcedefinition.apiextensions.k8s.io "perconaservermongodbbackups.psmdb.percona.com" deleted customresourcedefinition.apiextensions.k8s.io "perconaservermongodbclustersyncs.psmdb.percona.com" deleted customresourcedefinition.apiextensions.k8s.io "perconaservermongodbrestores.psmdb.percona.com" deleted customresourcedefinition.apiextensions.k8s.io "perconaservermongodbs.psmdb.percona.com" deleted + cat /tmp/tmp.g2ZUm1lds3 + rm /tmp/tmp.VntcIFXBq2 /tmp/tmp.g2ZUm1lds3 + return 0 ++ yq eval .metadata.name /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2465/deploy/crd.yaml ++ grep -v '\-\-\-' grep: warning: stray \ before - grep: warning: stray \ before - + for crd_name in $(yq eval '.metadata.name' "${src_dir}/deploy/crd.yaml" | grep -v '\-\-\-') + kubectl get perconaservermongodbbackups.psmdb.percona.com --all-namespaces -o wide + grep -v NAMESPACE + xargs -L 1 sh -xc 'kubectl patch perconaservermongodbbackups.psmdb.percona.com -n $0 $1 --type=merge -p "{\"metadata\":{\"finalizers\":[]}}"' error: the server doesn't have a resource type "perconaservermongodbbackups" + kubectl patch perconaservermongodbbackups.psmdb.percona.com -n sh --type=merge -p '{"metadata":{"finalizers":[]}}' error: the server doesn't have a resource type "perconaservermongodbbackups" + : + kubectl_bin wait --for=delete crd perconaservermongodbbackups.psmdb.percona.com ++ mktemp + local LAST_OUT=/tmp/tmp.av3feqQdth ++ mktemp + local LAST_ERR=/tmp/tmp.d6EZMygDd3 + local exit_status=0 + local timeout=4 + args=('wait' '--for=delete' 'crd' 'perconaservermongodbbackups.psmdb.percona.com') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl wait --for=delete crd perconaservermongodbbackups.psmdb.percona.com + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.av3feqQdth + cat /tmp/tmp.d6EZMygDd3 + rm /tmp/tmp.av3feqQdth /tmp/tmp.d6EZMygDd3 + return 0 + for crd_name in $(yq eval '.metadata.name' "${src_dir}/deploy/crd.yaml" | grep -v '\-\-\-') + kubectl get perconaservermongodbclustersyncs.psmdb.percona.com --all-namespaces -o wide + grep -v NAMESPACE + xargs -L 1 sh -xc 'kubectl patch perconaservermongodbclustersyncs.psmdb.percona.com -n $0 $1 --type=merge -p "{\"metadata\":{\"finalizers\":[]}}"' error: the server doesn't have a resource type "perconaservermongodbclustersyncs" + kubectl patch perconaservermongodbclustersyncs.psmdb.percona.com -n sh --type=merge -p '{"metadata":{"finalizers":[]}}' error: the server doesn't have a resource type "perconaservermongodbclustersyncs" + : + kubectl_bin wait --for=delete crd perconaservermongodbclustersyncs.psmdb.percona.com ++ mktemp + local LAST_OUT=/tmp/tmp.qG2SSoP2OW ++ mktemp + local LAST_ERR=/tmp/tmp.O3AgffuGph + local exit_status=0 + local timeout=4 + args=('wait' '--for=delete' 'crd' 'perconaservermongodbclustersyncs.psmdb.percona.com') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl wait --for=delete crd perconaservermongodbclustersyncs.psmdb.percona.com + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.qG2SSoP2OW + cat /tmp/tmp.O3AgffuGph + rm /tmp/tmp.qG2SSoP2OW /tmp/tmp.O3AgffuGph + return 0 + for crd_name in $(yq eval '.metadata.name' "${src_dir}/deploy/crd.yaml" | grep -v '\-\-\-') + kubectl get perconaservermongodbrestores.psmdb.percona.com --all-namespaces -o wide + grep -v NAMESPACE + xargs -L 1 sh -xc 'kubectl patch perconaservermongodbrestores.psmdb.percona.com -n $0 $1 --type=merge -p "{\"metadata\":{\"finalizers\":[]}}"' error: the server doesn't have a resource type "perconaservermongodbrestores" + kubectl patch perconaservermongodbrestores.psmdb.percona.com -n sh --type=merge -p '{"metadata":{"finalizers":[]}}' error: the server doesn't have a resource type "perconaservermongodbrestores" + : + kubectl_bin wait --for=delete crd perconaservermongodbrestores.psmdb.percona.com ++ mktemp + local LAST_OUT=/tmp/tmp.QuLcJUGcCs ++ mktemp + local LAST_ERR=/tmp/tmp.cEBw9nP0md + local exit_status=0 + local timeout=4 + args=('wait' '--for=delete' 'crd' 'perconaservermongodbrestores.psmdb.percona.com') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl wait --for=delete crd perconaservermongodbrestores.psmdb.percona.com + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.QuLcJUGcCs + cat /tmp/tmp.cEBw9nP0md + rm /tmp/tmp.QuLcJUGcCs /tmp/tmp.cEBw9nP0md + return 0 + for crd_name in $(yq eval '.metadata.name' "${src_dir}/deploy/crd.yaml" | grep -v '\-\-\-') + kubectl get perconaservermongodbs.psmdb.percona.com --all-namespaces -o wide + grep -v NAMESPACE + xargs -L 1 sh -xc 'kubectl patch perconaservermongodbs.psmdb.percona.com -n $0 $1 --type=merge -p "{\"metadata\":{\"finalizers\":[]}}"' error: the server doesn't have a resource type "perconaservermongodbs" + kubectl patch perconaservermongodbs.psmdb.percona.com -n sh --type=merge -p '{"metadata":{"finalizers":[]}}' error: the server doesn't have a resource type "perconaservermongodbs" + : + kubectl_bin wait --for=delete crd perconaservermongodbs.psmdb.percona.com ++ mktemp + local LAST_OUT=/tmp/tmp.pvQcXrQoym ++ mktemp + local LAST_ERR=/tmp/tmp.CxzFBB30Rv + local exit_status=0 + local timeout=4 + args=('wait' '--for=delete' 'crd' 'perconaservermongodbs.psmdb.percona.com') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl wait --for=delete crd perconaservermongodbs.psmdb.percona.com + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.pvQcXrQoym + cat /tmp/tmp.CxzFBB30Rv + rm /tmp/tmp.pvQcXrQoym /tmp/tmp.CxzFBB30Rv + return 0 + local rbac_yaml=rbac.yaml + '[' -n psmdb-operator ']' + rbac_yaml=cw-rbac.yaml + kubectl_bin delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2465/deploy/cw-rbac.yaml --ignore-not-found ++ mktemp + local LAST_OUT=/tmp/tmp.FLF2FLWZig ++ mktemp + local LAST_ERR=/tmp/tmp.TEsvt4PwOY + local exit_status=0 + local timeout=4 + args=('delete' '-f' '/mnt/jenkins/workspace/cloud-psmdb-operator_PR-2465/deploy/cw-rbac.yaml' '--ignore-not-found') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2465/deploy/cw-rbac.yaml --ignore-not-found + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.FLF2FLWZig clusterrole.rbac.authorization.k8s.io "percona-server-mongodb-operator" deleted clusterrolebinding.rbac.authorization.k8s.io "service-account-percona-server-mongodb-operator" deleted + cat /tmp/tmp.TEsvt4PwOY + rm /tmp/tmp.FLF2FLWZig /tmp/tmp.TEsvt4PwOY + return 0 + check_crd_for_deletion PR-2465-a177d53a8 + local git_tag=PR-2465-a177d53a8 ++ curl -s https://raw.githubusercontent.com/percona/percona-server-mongodb-operator/PR-2465-a177d53a8/deploy/crd.yaml ++ yq eval .metadata.name ++ /usr/sbin/sed s/---//g ++ /usr/sbin/sed ':a;N;$!ba;s/\n/ /g' + for crd_name in $(curl -s https://raw.githubusercontent.com/percona/percona-server-mongodb-operator/${git_tag}/deploy/crd.yaml | yq eval '.metadata.name' | $sed 's/---//g' | $sed ':a;N;$!ba;s/\n/ /g') ++ kubectl_bin get crd/null -o 'jsonpath={.status.conditions[-1].type}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.x5j42XigIF +++ mktemp ++ local LAST_ERR=/tmp/tmp.vdzp7erWfR ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'crd/null' '-o' 'jsonpath={.status.conditions[-1].type}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get crd/null -o 'jsonpath={.status.conditions[-1].type}' ++ exit_status=1 ++ set -e ++ '[' 1 '!=' 0 -a -n 1 ']' ++ cat /tmp/tmp.x5j42XigIF ++ cat /tmp/tmp.vdzp7erWfR Error from server (NotFound): customresourcedefinitions.apiextensions.k8s.io "null" not found ++ sleep 0 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get crd/null -o 'jsonpath={.status.conditions[-1].type}' ++ exit_status=1 ++ set -e ++ '[' 1 '!=' 0 -a -n 1 ']' ++ cat /tmp/tmp.x5j42XigIF ++ cat /tmp/tmp.vdzp7erWfR Error from server (NotFound): customresourcedefinitions.apiextensions.k8s.io "null" not found ++ sleep 4 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get crd/null -o 'jsonpath={.status.conditions[-1].type}' ++ exit_status=1 ++ set -e ++ '[' 1 '!=' 0 -a -n 1 ']' ++ cat /tmp/tmp.x5j42XigIF ++ cat /tmp/tmp.vdzp7erWfR Error from server (NotFound): customresourcedefinitions.apiextensions.k8s.io "null" not found ++ sleep 8 ++ cat /tmp/tmp.x5j42XigIF ++ cat /tmp/tmp.vdzp7erWfR Error from server (NotFound): customresourcedefinitions.apiextensions.k8s.io "null" not found ++ rm /tmp/tmp.x5j42XigIF /tmp/tmp.vdzp7erWfR ++ return 1 + [[ '' == Terminating ]] + '[' -n psmdb-operator ']' + create_namespace psmdb-operator + local namespace=psmdb-operator + local skip_clean_namespace= + [[ 1 == 1 ]] + [[ -z '' ]] + destroy_chaos_mesh ++ helm list --all-namespaces --filter chaos-mesh ++ awk '-F ' '{print $2}' ++ sed s/NAMESPACE// ++ tail -n1 + local chaos_mesh_ns= + desc 'destroy chaos-mesh' + set +o xtrace ----------------------------------------------------------------------------------- destroy chaos-mesh ----------------------------------------------------------------------------------- + '[' -n '' ']' ++ kubectl get MutatingWebhookConfiguration ++ grep chaos-mesh ++ awk '{print $1}' + timeout 30 kubectl delete MutatingWebhookConfiguration error: resource(s) were provided, but no name was specified + : ++ kubectl get ValidatingWebhookConfiguration ++ grep chaos-mesh ++ awk '{print $1}' + timeout 30 kubectl delete ValidatingWebhookConfiguration error: resource(s) were provided, but no name was specified + : ++ kubectl get ValidatingWebhookConfiguration ++ awk '{print $1}' ++ grep validate-auth + timeout 30 kubectl delete ValidatingWebhookConfiguration error: resource(s) were provided, but no name was specified + : ++ kubectl api-resources ++ awk '{print $1}' ++ grep chaos-mesh ++ kubectl get crd ++ grep chaos-mesh.org ++ awk '{print $1}' + timeout 30 kubectl delete crd error: resource(s) were provided, but no name was specified + : ++ kubectl get clusterrolebinding ++ grep chaos-mesh ++ awk '{print $1}' + timeout 30 kubectl delete clusterrolebinding error: resource(s) were provided, but no name was specified + : ++ kubectl get clusterrole ++ awk '{print $1}' ++ grep chaos-mesh + timeout 30 kubectl delete clusterrole error: resource(s) were provided, but no name was specified + : + desc 'cleaned up all old namespaces' + set +o xtrace ----------------------------------------------------------------------------------- cleaned up all old namespaces ----------------------------------------------------------------------------------- + kubectl_bin get ns + grep -E -v '^kube-|^default|Terminating|psmdb-operator|openshift|^gke-|^gmp-|^NAME' + awk '{print$1}' + [[ -n '' ]] ++ mktemp + desc 'cleaned up old namespaces psmdb-operator' + set +o xtrace ----------------------------------------------------------------------------------- cleaned up old namespaces psmdb-operator ----------------------------------------------------------------------------------- + kubectl_bin delete namespace psmdb-operator --ignore-not-found + xargs kubectl delete ns + local LAST_OUT=/tmp/tmp.fGD53gXvS0 ++ mktemp ++ mktemp + local LAST_ERR=/tmp/tmp.rc0PCle85x + local exit_status=0 + local timeout=4 + args=('get' 'ns') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] + local LAST_OUT=/tmp/tmp.0RVXOrOoUx ++ seq 0 2 ++ mktemp + for i in $(seq 0 2) + set +e + local LAST_ERR=/tmp/tmp.YoXwoTTPOO + kubectl get ns + local exit_status=0 + local timeout=4 + args=('delete' 'namespace' 'psmdb-operator' '--ignore-not-found') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl delete namespace psmdb-operator --ignore-not-found + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.fGD53gXvS0 + cat /tmp/tmp.rc0PCle85x + rm /tmp/tmp.fGD53gXvS0 /tmp/tmp.rc0PCle85x + return 0 namespace "cert-manager" deleted namespace "monitoring-2-0-7068" deleted + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.0RVXOrOoUx namespace "psmdb-operator" deleted + cat /tmp/tmp.YoXwoTTPOO + rm /tmp/tmp.0RVXOrOoUx /tmp/tmp.YoXwoTTPOO + return 0 + kubectl_bin wait --for=delete namespace psmdb-operator ++ mktemp + local LAST_OUT=/tmp/tmp.3RURgQKix6 ++ mktemp + local LAST_ERR=/tmp/tmp.wA6NUc0UN1 + local exit_status=0 + local timeout=4 + args=('wait' '--for=delete' 'namespace' 'psmdb-operator') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl wait --for=delete namespace psmdb-operator + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.3RURgQKix6 + cat /tmp/tmp.wA6NUc0UN1 + rm /tmp/tmp.3RURgQKix6 /tmp/tmp.wA6NUc0UN1 + return 0 + desc 'create namespace psmdb-operator' + set +o xtrace ----------------------------------------------------------------------------------- create namespace psmdb-operator ----------------------------------------------------------------------------------- + kubectl_bin create namespace psmdb-operator ++ mktemp + local LAST_OUT=/tmp/tmp.J9gTJfWEcr ++ mktemp + local LAST_ERR=/tmp/tmp.ie6GlphGUk + local exit_status=0 + local timeout=4 + args=('create' 'namespace' 'psmdb-operator') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl create namespace psmdb-operator + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.J9gTJfWEcr namespace/psmdb-operator created + cat /tmp/tmp.ie6GlphGUk + rm /tmp/tmp.J9gTJfWEcr /tmp/tmp.ie6GlphGUk + return 0 + set_kube_ctx psmdb-operator + local namespace=psmdb-operator ++ kubectl_bin config current-context +++ mktemp ++ local LAST_OUT=/tmp/tmp.qzfO7IFXxb +++ mktemp ++ local LAST_ERR=/tmp/tmp.VBCN8DLqrb ++ local exit_status=0 ++ local timeout=4 ++ args=('config' 'current-context') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl config current-context ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.qzfO7IFXxb ++ cat /tmp/tmp.VBCN8DLqrb ++ rm /tmp/tmp.qzfO7IFXxb /tmp/tmp.VBCN8DLqrb ++ return 0 + kubectl_bin config set-context gke_cloud-dev-112233_us-central1-a_jen-psmdb-2465-a177d53a8-9-cluster7 --namespace=psmdb-operator ++ mktemp + local LAST_OUT=/tmp/tmp.2Qzhp0Qt4J ++ mktemp + local LAST_ERR=/tmp/tmp.Xnn51kxnAX + local exit_status=0 + local timeout=4 + args=('config' 'set-context' 'gke_cloud-dev-112233_us-central1-a_jen-psmdb-2465-a177d53a8-9-cluster7' '--namespace=psmdb-operator') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl config set-context gke_cloud-dev-112233_us-central1-a_jen-psmdb-2465-a177d53a8-9-cluster7 --namespace=psmdb-operator + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.2Qzhp0Qt4J Context "gke_cloud-dev-112233_us-central1-a_jen-psmdb-2465-a177d53a8-9-cluster7" modified. + cat /tmp/tmp.Xnn51kxnAX + rm /tmp/tmp.2Qzhp0Qt4J /tmp/tmp.Xnn51kxnAX + return 0 + deploy_operator + desc 'start PSMDB operator: docker.io/perconalab/percona-server-mongodb-operator:PR-2465-a177d53a8' + set +o xtrace ----------------------------------------------------------------------------------- start PSMDB operator: docker.io/perconalab/percona-server-mongodb-operator:PR-2465-a177d53a8 ----------------------------------------------------------------------------------- + local cr_file + '[' -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2465/e2e-tests/monitoring-2-0/conf/crd.yaml ']' + cr_file=/mnt/jenkins/workspace/cloud-psmdb-operator_PR-2465/deploy/crd.yaml + kubectl_bin apply --server-side --force-conflicts -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2465/deploy/crd.yaml ++ mktemp + local LAST_OUT=/tmp/tmp.JbcQ8QMzdr ++ mktemp + local LAST_ERR=/tmp/tmp.Jk8Zk8D8qK + local exit_status=0 + local timeout=4 + args=('apply' '--server-side' '--force-conflicts' '-f' '/mnt/jenkins/workspace/cloud-psmdb-operator_PR-2465/deploy/crd.yaml') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl apply --server-side --force-conflicts -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2465/deploy/crd.yaml + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.JbcQ8QMzdr customresourcedefinition.apiextensions.k8s.io/perconaservermongodbbackups.psmdb.percona.com serverside-applied customresourcedefinition.apiextensions.k8s.io/perconaservermongodbclustersyncs.psmdb.percona.com serverside-applied customresourcedefinition.apiextensions.k8s.io/perconaservermongodbrestores.psmdb.percona.com serverside-applied customresourcedefinition.apiextensions.k8s.io/perconaservermongodbs.psmdb.percona.com serverside-applied + cat /tmp/tmp.Jk8Zk8D8qK + rm /tmp/tmp.JbcQ8QMzdr /tmp/tmp.Jk8Zk8D8qK + return 0 + '[' -n psmdb-operator ']' + apply_rbac cw-rbac + local operator_namespace=psmdb-operator + local rbac=cw-rbac + cat /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2465/deploy/cw-rbac.yaml + sed -e 's^namespace: .*^namespace: psmdb-operator^' + kubectl_bin apply -n psmdb-operator -f - ++ mktemp + local LAST_OUT=/tmp/tmp.9dXdXFLMNm ++ mktemp + local LAST_ERR=/tmp/tmp.1QpTvMd0pD + local exit_status=0 + local timeout=4 + args=('apply' '-n' 'psmdb-operator' '-f' '-') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl apply -n psmdb-operator -f - + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.9dXdXFLMNm clusterrole.rbac.authorization.k8s.io/percona-server-mongodb-operator created serviceaccount/percona-server-mongodb-operator created clusterrolebinding.rbac.authorization.k8s.io/service-account-percona-server-mongodb-operator created + cat /tmp/tmp.1QpTvMd0pD + rm /tmp/tmp.9dXdXFLMNm /tmp/tmp.1QpTvMd0pD + return 0 + add_arch_to_podspecs - + yq eval $'\n\t\t\t(.spec.template.spec.containers[].image = "docker.io/perconalab/percona-server-mongodb-operator:PR-2465-a177d53a8") |\n\t\t\t((.. | select(.[] == "DISABLE_TELEMETRY")) |= .value="true") |\n\t\t\t((.. | select(.[] == "LOG_LEVEL")) |= .value="DEBUG")' /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2465/deploy/cw-operator.yaml + arch_scheduling_enabled + [[ -n '' ]] + cat - + kubectl_bin apply -n psmdb-operator -f - ++ mktemp + local LAST_OUT=/tmp/tmp.44lrXjX5vP ++ mktemp + local LAST_ERR=/tmp/tmp.WBV60PF92d + local exit_status=0 + local timeout=4 + args=('apply' '-n' 'psmdb-operator' '-f' '-') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl apply -n psmdb-operator -f - + return + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.44lrXjX5vP deployment.apps/percona-server-mongodb-operator created + cat /tmp/tmp.WBV60PF92d + rm /tmp/tmp.44lrXjX5vP /tmp/tmp.WBV60PF92d + return 0 + sleep 20 ++ get_operator_pod ++ kubectl_bin get pods --selector=name=percona-server-mongodb-operator -o 'jsonpath={.items[].metadata.name}' -n psmdb-operator +++ mktemp ++ local LAST_OUT=/tmp/tmp.Cmzn87Iy8g +++ mktemp ++ local LAST_ERR=/tmp/tmp.fx82lzXGJF ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'pods' '--selector=name=percona-server-mongodb-operator' '-o' 'jsonpath={.items[].metadata.name}' '-n' 'psmdb-operator') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pods --selector=name=percona-server-mongodb-operator -o 'jsonpath={.items[].metadata.name}' -n psmdb-operator ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.Cmzn87Iy8g ++ cat /tmp/tmp.fx82lzXGJF ++ rm /tmp/tmp.Cmzn87Iy8g /tmp/tmp.fx82lzXGJF ++ return 0 + wait_operator_pod percona-server-mongodb-operator-6cb789f874-6j2lh + local pod=percona-server-mongodb-operator-6cb789f874-6j2lh + set +o xtrace waiting for pod/percona-server-mongodb-operator-6cb789f874-6j2lh to be ready.OK + echo 'Print operator info from log' Print operator info from log + grep 'Manager starting up' ++ get_operator_pod ++ kubectl_bin get pods --selector=name=percona-server-mongodb-operator -o 'jsonpath={.items[].metadata.name}' -n psmdb-operator +++ mktemp ++ local LAST_OUT=/tmp/tmp.vHsiQOPt4I +++ mktemp ++ local LAST_ERR=/tmp/tmp.m95kSX7UBC ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'pods' '--selector=name=percona-server-mongodb-operator' '-o' 'jsonpath={.items[].metadata.name}' '-n' 'psmdb-operator') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pods --selector=name=percona-server-mongodb-operator -o 'jsonpath={.items[].metadata.name}' -n psmdb-operator ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.vHsiQOPt4I ++ cat /tmp/tmp.m95kSX7UBC ++ rm /tmp/tmp.vHsiQOPt4I /tmp/tmp.m95kSX7UBC ++ return 0 + kubectl_bin logs -n psmdb-operator percona-server-mongodb-operator-6cb789f874-6j2lh ++ mktemp + local LAST_OUT=/tmp/tmp.mN9er8c7G4 ++ mktemp + local LAST_ERR=/tmp/tmp.AYg6xPq9lt + local exit_status=0 + local timeout=4 + args=('logs' '-n' 'psmdb-operator' 'percona-server-mongodb-operator-6cb789f874-6j2lh') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl logs -n psmdb-operator percona-server-mongodb-operator-6cb789f874-6j2lh + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.mN9er8c7G4 + cat /tmp/tmp.AYg6xPq9lt + rm /tmp/tmp.mN9er8c7G4 /tmp/tmp.AYg6xPq9lt + return 0 2026-07-22T18:29:23.750Z INFO setup Manager starting up {"gitCommit": "a177d53a8f1a53d235b5094c0cbe20626ddb0066", "gitBranch": "PR-2465-a177d53a8", "buildTime": "", "goVersion": "go1.26.5", "os": "linux", "arch": "amd64"} + create_namespace monitoring-2-0-3915 + local namespace=monitoring-2-0-3915 + local skip_clean_namespace= + [[ 1 == 1 ]] + [[ -z '' ]] + destroy_chaos_mesh ++ helm list --all-namespaces --filter chaos-mesh ++ tail -n1 ++ awk '-F ' '{print $2}' ++ sed s/NAMESPACE// + local chaos_mesh_ns= + desc 'destroy chaos-mesh' + set +o xtrace ----------------------------------------------------------------------------------- destroy chaos-mesh ----------------------------------------------------------------------------------- + '[' -n '' ']' ++ kubectl get MutatingWebhookConfiguration ++ grep chaos-mesh ++ awk '{print $1}' + timeout 30 kubectl delete MutatingWebhookConfiguration error: resource(s) were provided, but no name was specified + : ++ kubectl get ValidatingWebhookConfiguration ++ grep chaos-mesh ++ awk '{print $1}' + timeout 30 kubectl delete ValidatingWebhookConfiguration error: resource(s) were provided, but no name was specified + : ++ kubectl get ValidatingWebhookConfiguration ++ grep validate-auth ++ awk '{print $1}' + timeout 30 kubectl delete ValidatingWebhookConfiguration error: resource(s) were provided, but no name was specified + : ++ kubectl api-resources ++ grep chaos-mesh ++ awk '{print $1}' ++ kubectl get crd ++ grep chaos-mesh.org ++ awk '{print $1}' + timeout 30 kubectl delete crd error: resource(s) were provided, but no name was specified + : ++ awk '{print $1}' ++ grep chaos-mesh ++ kubectl get clusterrolebinding + timeout 30 kubectl delete clusterrolebinding error: resource(s) were provided, but no name was specified + : ++ kubectl get clusterrole ++ awk '{print $1}' ++ grep chaos-mesh + timeout 30 kubectl delete clusterrole error: resource(s) were provided, but no name was specified + : + desc 'cleaned up all old namespaces' + set +o xtrace ----------------------------------------------------------------------------------- cleaned up all old namespaces ----------------------------------------------------------------------------------- + kubectl_bin get ns + [[ -n '' ]] + desc 'cleaned up old namespaces monitoring-2-0-3915' + set +o xtrace ----------------------------------------------------------------------------------- cleaned up old namespaces monitoring-2-0-3915 ----------------------------------------------------------------------------------- + kubectl_bin delete namespace monitoring-2-0-3915 --ignore-not-found ++ mktemp + grep -E -v '^kube-|^default|Terminating|psmdb-operator|openshift|^gke-|^gmp-|^NAME' + awk '{print$1}' + xargs kubectl delete ns ++ mktemp + local LAST_OUT=/tmp/tmp.2Mfh4wkCZJ ++ mktemp + local LAST_OUT=/tmp/tmp.8fnlYzioIf + local LAST_ERR=/tmp/tmp.NNEWewlTW4 + local exit_status=0 + local timeout=4 + args=('get' 'ns') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 ++ mktemp + local LAST_ERR=/tmp/tmp.HK0j9tffAc + local exit_status=0 + local timeout=4 + args=('delete' 'namespace' 'monitoring-2-0-3915' '--ignore-not-found') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] + for i in $(seq 0 2) + set +e + kubectl get ns ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl delete namespace monitoring-2-0-3915 --ignore-not-found + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.2Mfh4wkCZJ + cat /tmp/tmp.NNEWewlTW4 + rm /tmp/tmp.2Mfh4wkCZJ /tmp/tmp.NNEWewlTW4 + return 0 error: resource(s) were provided, but no name was specified + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.8fnlYzioIf + cat /tmp/tmp.HK0j9tffAc + rm /tmp/tmp.8fnlYzioIf /tmp/tmp.HK0j9tffAc + return 0 + kubectl_bin wait --for=delete namespace monitoring-2-0-3915 ++ mktemp + local LAST_OUT=/tmp/tmp.9VhN2xGOlQ ++ mktemp + local LAST_ERR=/tmp/tmp.0eOx5kXYwU + local exit_status=0 + local timeout=4 + args=('wait' '--for=delete' 'namespace' 'monitoring-2-0-3915') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl wait --for=delete namespace monitoring-2-0-3915 + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.9VhN2xGOlQ + cat /tmp/tmp.0eOx5kXYwU + rm /tmp/tmp.9VhN2xGOlQ /tmp/tmp.0eOx5kXYwU + return 0 + desc 'create namespace monitoring-2-0-3915' + set +o xtrace ----------------------------------------------------------------------------------- create namespace monitoring-2-0-3915 ----------------------------------------------------------------------------------- + kubectl_bin create namespace monitoring-2-0-3915 ++ mktemp + local LAST_OUT=/tmp/tmp.hi8c5ktb8o ++ mktemp + local LAST_ERR=/tmp/tmp.rM1hitCZbK + local exit_status=0 + local timeout=4 + args=('create' 'namespace' 'monitoring-2-0-3915') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl create namespace monitoring-2-0-3915 + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.hi8c5ktb8o namespace/monitoring-2-0-3915 created + cat /tmp/tmp.rM1hitCZbK + rm /tmp/tmp.hi8c5ktb8o /tmp/tmp.rM1hitCZbK + return 0 + set_kube_ctx monitoring-2-0-3915 + local namespace=monitoring-2-0-3915 ++ kubectl_bin config current-context +++ mktemp ++ local LAST_OUT=/tmp/tmp.LmRP1tkYHh +++ mktemp ++ local LAST_ERR=/tmp/tmp.A6kvMXVBdq ++ local exit_status=0 ++ local timeout=4 ++ args=('config' 'current-context') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl config current-context ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.LmRP1tkYHh ++ cat /tmp/tmp.A6kvMXVBdq ++ rm /tmp/tmp.LmRP1tkYHh /tmp/tmp.A6kvMXVBdq ++ return 0 + kubectl_bin config set-context gke_cloud-dev-112233_us-central1-a_jen-psmdb-2465-a177d53a8-9-cluster7 --namespace=monitoring-2-0-3915 ++ mktemp + local LAST_OUT=/tmp/tmp.WnID6EpPQT ++ mktemp + local LAST_ERR=/tmp/tmp.jtaMxJTRRG + local exit_status=0 + local timeout=4 + args=('config' 'set-context' 'gke_cloud-dev-112233_us-central1-a_jen-psmdb-2465-a177d53a8-9-cluster7' '--namespace=monitoring-2-0-3915') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl config set-context gke_cloud-dev-112233_us-central1-a_jen-psmdb-2465-a177d53a8-9-cluster7 --namespace=monitoring-2-0-3915 + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.WnID6EpPQT Context "gke_cloud-dev-112233_us-central1-a_jen-psmdb-2465-a177d53a8-9-cluster7" modified. + cat /tmp/tmp.jtaMxJTRRG + rm /tmp/tmp.WnID6EpPQT /tmp/tmp.jtaMxJTRRG + return 0 + deploy_cert_manager + [[ gke == rancher ]] + desc 'deploy cert manager' + set +o xtrace ----------------------------------------------------------------------------------- deploy cert manager ----------------------------------------------------------------------------------- + kubectl_bin -n cert-manager delete clusterissuer --all ++ mktemp + local LAST_OUT=/tmp/tmp.vvf8Hn2Y7P ++ mktemp + local LAST_ERR=/tmp/tmp.A4uydMDOIi + local exit_status=0 + local timeout=4 + args=('-n' 'cert-manager' 'delete' 'clusterissuer' '--all') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl -n cert-manager delete clusterissuer --all + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.vvf8Hn2Y7P No resources found + cat /tmp/tmp.A4uydMDOIi + rm /tmp/tmp.vvf8Hn2Y7P /tmp/tmp.A4uydMDOIi + return 0 + kubectl_bin -n cert-manager delete certificate --all ++ mktemp + local LAST_OUT=/tmp/tmp.q7avhkBcFR ++ mktemp + local LAST_ERR=/tmp/tmp.NDEfgase5h + local exit_status=0 + local timeout=4 + args=('-n' 'cert-manager' 'delete' 'certificate' '--all') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl -n cert-manager delete certificate --all + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.q7avhkBcFR No resources found + cat /tmp/tmp.NDEfgase5h + rm /tmp/tmp.q7avhkBcFR /tmp/tmp.NDEfgase5h + return 0 + kubectl_bin create namespace cert-manager ++ mktemp + local LAST_OUT=/tmp/tmp.xmgH2WmyPJ ++ mktemp + local LAST_ERR=/tmp/tmp.VnA2sz0BuQ + local exit_status=0 + local timeout=4 + args=('create' 'namespace' 'cert-manager') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl create namespace cert-manager + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.xmgH2WmyPJ namespace/cert-manager created + cat /tmp/tmp.VnA2sz0BuQ + rm /tmp/tmp.xmgH2WmyPJ /tmp/tmp.VnA2sz0BuQ + return 0 + kubectl_bin label namespace cert-manager certmanager.k8s.io/disable-validation=true ++ mktemp + local LAST_OUT=/tmp/tmp.csZinLNZol ++ mktemp + local LAST_ERR=/tmp/tmp.MY2X8xxjet + local exit_status=0 + local timeout=4 + args=('label' 'namespace' 'cert-manager' 'certmanager.k8s.io/disable-validation=true') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl label namespace cert-manager certmanager.k8s.io/disable-validation=true + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.csZinLNZol namespace/cert-manager labeled + cat /tmp/tmp.MY2X8xxjet + rm /tmp/tmp.csZinLNZol /tmp/tmp.MY2X8xxjet + return 0 + curl -sL https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml + add_arch_to_podspecs - + arch_scheduling_enabled + [[ -n '' ]] + cat - + kubectl_bin apply -f - --validate=false + return namespace/cert-manager configured customresourcedefinition.apiextensions.k8s.io/challenges.acme.cert-manager.io unchanged customresourcedefinition.apiextensions.k8s.io/orders.acme.cert-manager.io unchanged customresourcedefinition.apiextensions.k8s.io/certificaterequests.cert-manager.io unchanged customresourcedefinition.apiextensions.k8s.io/certificates.cert-manager.io unchanged customresourcedefinition.apiextensions.k8s.io/clusterissuers.cert-manager.io unchanged customresourcedefinition.apiextensions.k8s.io/issuers.cert-manager.io unchanged serviceaccount/cert-manager-cainjector created serviceaccount/cert-manager created serviceaccount/cert-manager-webhook created clusterrole.rbac.authorization.k8s.io/cert-manager-cainjector unchanged clusterrole.rbac.authorization.k8s.io/cert-manager-controller-issuers unchanged clusterrole.rbac.authorization.k8s.io/cert-manager-controller-clusterissuers unchanged clusterrole.rbac.authorization.k8s.io/cert-manager-controller-certificates unchanged clusterrole.rbac.authorization.k8s.io/cert-manager-controller-orders unchanged clusterrole.rbac.authorization.k8s.io/cert-manager-controller-challenges unchanged clusterrole.rbac.authorization.k8s.io/cert-manager-controller-ingress-shim unchanged clusterrole.rbac.authorization.k8s.io/cert-manager-cluster-view unchanged clusterrole.rbac.authorization.k8s.io/cert-manager-view unchanged clusterrole.rbac.authorization.k8s.io/cert-manager-edit unchanged clusterrole.rbac.authorization.k8s.io/cert-manager-controller-approve:cert-manager-io unchanged clusterrole.rbac.authorization.k8s.io/cert-manager-controller-certificatesigningrequests unchanged clusterrole.rbac.authorization.k8s.io/cert-manager-webhook:subjectaccessreviews unchanged clusterrolebinding.rbac.authorization.k8s.io/cert-manager-cainjector unchanged clusterrolebinding.rbac.authorization.k8s.io/cert-manager-controller-issuers unchanged clusterrolebinding.rbac.authorization.k8s.io/cert-manager-controller-clusterissuers unchanged clusterrolebinding.rbac.authorization.k8s.io/cert-manager-controller-certificates unchanged clusterrolebinding.rbac.authorization.k8s.io/cert-manager-controller-orders unchanged clusterrolebinding.rbac.authorization.k8s.io/cert-manager-controller-challenges unchanged clusterrolebinding.rbac.authorization.k8s.io/cert-manager-controller-ingress-shim unchanged clusterrolebinding.rbac.authorization.k8s.io/cert-manager-controller-approve:cert-manager-io unchanged clusterrolebinding.rbac.authorization.k8s.io/cert-manager-controller-certificatesigningrequests unchanged clusterrolebinding.rbac.authorization.k8s.io/cert-manager-webhook:subjectaccessreviews unchanged role.rbac.authorization.k8s.io/cert-manager-cainjector:leaderelection unchanged role.rbac.authorization.k8s.io/cert-manager:leaderelection unchanged role.rbac.authorization.k8s.io/cert-manager-tokenrequest created role.rbac.authorization.k8s.io/cert-manager-webhook:dynamic-serving created rolebinding.rbac.authorization.k8s.io/cert-manager-cainjector:leaderelection unchanged rolebinding.rbac.authorization.k8s.io/cert-manager:leaderelection unchanged rolebinding.rbac.authorization.k8s.io/cert-manager-tokenrequest created rolebinding.rbac.authorization.k8s.io/cert-manager-webhook:dynamic-serving created service/cert-manager-cainjector created service/cert-manager created service/cert-manager-webhook created deployment.apps/cert-manager-cainjector created deployment.apps/cert-manager created deployment.apps/cert-manager-webhook created mutatingwebhookconfiguration.admissionregistration.k8s.io/cert-manager-webhook configured validatingwebhookconfiguration.admissionregistration.k8s.io/cert-manager-webhook configured + kubectl_bin -n cert-manager wait pod -l app.kubernetes.io/instance=cert-manager --for=condition=ready ++ mktemp + local LAST_OUT=/tmp/tmp.S3yhIpxvOj ++ mktemp + local LAST_ERR=/tmp/tmp.Tmth97KTNP + local exit_status=0 + local timeout=4 + args=('-n' 'cert-manager' 'wait' 'pod' '-l' 'app.kubernetes.io/instance=cert-manager' '--for=condition=ready') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl -n cert-manager wait pod -l app.kubernetes.io/instance=cert-manager --for=condition=ready + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.S3yhIpxvOj pod/cert-manager-559d798845-8kxj8 condition met pod/cert-manager-cainjector-64958d9c7c-tfd7s condition met pod/cert-manager-webhook-7fb6f99b56-qckdv condition met + cat /tmp/tmp.Tmth97KTNP E0722 18:30:29.474824 383808 reflector.go:227] "Failed to watch" err="Get \"https://34.135.50.243/api/v1/namespaces/cert-manager/pods?allowWatchBookmarks=true&fieldSelector=metadata.name%3Dcert-manager-webhook-7fb6f99b56-qckdv&resourceVersion=1784745029084511019&timeoutSeconds=315&watch=true\": context canceled" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" type="*unstructured.Unstructured" + rm /tmp/tmp.S3yhIpxvOj /tmp/tmp.Tmth97KTNP + return 0 + sleep 120 + desc 'install PMM Server' + set +o xtrace ----------------------------------------------------------------------------------- install PMM Server ----------------------------------------------------------------------------------- + deploy_pmm_server + helm uninstall monitoring Error: uninstall: Release not loaded: monitoring: release: not found + : + helm repo remove stable "stable" has been removed from your repositories + helm repo add stable https://charts.helm.sh/stable "stable" has been added to your repositories + [[ -n '' ]] + pmm_args=('--set' 'imageTag=dev-latest' '--set' 'imageRepo=docker.io/perconalab/pmm-server') + local pmm_args + arch_scheduling_enabled + [[ -n '' ]] + retry 10 60 helm install monitoring --set imageTag=dev-latest --set imageRepo=docker.io/perconalab/pmm-server https://percona-charts.storage.googleapis.com/pmm-server-9.9.9.tgz + local max=10 + local delay=60 + shift 2 + local n=1 + helm install monitoring --set imageTag=dev-latest --set imageRepo=docker.io/perconalab/pmm-server https://percona-charts.storage.googleapis.com/pmm-server-9.9.9.tgz NAME: monitoring LAST DEPLOYED: Wed Jul 22 18:32:35 2026 NAMESPACE: monitoring-2-0-3915 STATUS: deployed REVISION: 1 TEST SUITE: None NOTES: PMM server can be accessed via HTTPS (port 443) on the following DNS name from within your cluster: endpoint: https://monitoring-service.monitoring-2-0-3915.svc.cluster.local:443 login: admin password: admin + sleep 40 + kubectl_bin exec monitoring-0 -- bash -c 'ls -l /proc/*/exe 2>/dev/null| grep postgres >/dev/null' ++ mktemp + local LAST_OUT=/tmp/tmp.0iLhCHtz1X ++ mktemp + local LAST_ERR=/tmp/tmp.J2bQ0BcMBA + local exit_status=0 + local timeout=4 + args=('exec' 'monitoring-0' '--' 'bash' '-c' 'ls -l /proc/*/exe 2>/dev/null| grep postgres >/dev/null') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl exec monitoring-0 -- bash -c 'ls -l /proc/*/exe 2>/dev/null| grep postgres >/dev/null' + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.0iLhCHtz1X + cat /tmp/tmp.J2bQ0BcMBA + rm /tmp/tmp.0iLhCHtz1X /tmp/tmp.J2bQ0BcMBA + return 0 + cluster=monitoring + desc 'create secrets and start client' + set +o xtrace ----------------------------------------------------------------------------------- create secrets and start client ----------------------------------------------------------------------------------- + kubectl_bin apply -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2465/e2e-tests/conf/secrets.yml -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2465/e2e-tests/monitoring-2-0/conf/secrets.yml ++ mktemp + local LAST_OUT=/tmp/tmp.JHyZrVM4bB ++ mktemp + local LAST_ERR=/tmp/tmp.Z9eDDUrU9k + local exit_status=0 + local timeout=4 + args=('apply' '-f' '/mnt/jenkins/workspace/cloud-psmdb-operator_PR-2465/e2e-tests/conf/secrets.yml' '-f' '/mnt/jenkins/workspace/cloud-psmdb-operator_PR-2465/e2e-tests/monitoring-2-0/conf/secrets.yml') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl apply -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2465/e2e-tests/conf/secrets.yml -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2465/e2e-tests/monitoring-2-0/conf/secrets.yml + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.JHyZrVM4bB secret/some-users created secret/some-users unchanged + cat /tmp/tmp.Z9eDDUrU9k + rm /tmp/tmp.JHyZrVM4bB /tmp/tmp.Z9eDDUrU9k + return 0 + desc 'create first PSMDB cluster monitoring' + set +o xtrace ----------------------------------------------------------------------------------- create first PSMDB cluster monitoring ----------------------------------------------------------------------------------- + apply_cluster /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2465/e2e-tests/monitoring-2-0/conf/monitoring-rs0.yml + '[' -z '' ']' + cat_config /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2465/e2e-tests/monitoring-2-0/conf/monitoring-rs0.yml + kubectl_bin apply -f - + cat /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2465/e2e-tests/monitoring-2-0/conf/monitoring-rs0.yml + yq eval '(.spec | select(.image == null)).image = "docker.io/perconalab/percona-server-mongodb-operator:main-mongod8.0"' + yq eval '(.spec | select(has("pmm"))).pmm.image = "docker.io/percona/pmm-client:2.44.1-1"' ++ mktemp + yq eval '(.spec | select(has("initImage"))).initImage = "docker.io/perconalab/percona-server-mongodb-operator:PR-2465-a177d53a8"' + yq eval '(.spec | select(has("backup"))).backup.image = "docker.io/perconalab/percona-server-mongodb-operator:main-backup"' + [[ -n docker.io/perconalab/fluentbit:main-logcollector ]] + IMAGE_LOGCOLLECTOR=docker.io/perconalab/fluentbit:main-logcollector + yq eval '(.spec | select(has("logcollector"))).logcollector.image = strenv(IMAGE_LOGCOLLECTOR)' + yq eval '(.spec | select(has("search"))).search.image = "perconalab/percona-server-mongodb-operator:main-mongot"' + /usr/sbin/sed -e s/NAME_SPACE/monitoring-2-0-3915/g + add_arch_to_psmdb - + arch_scheduling_enabled + [[ -n '' ]] + cat - + yq eval '.spec.upgradeOptions.apply="Never"' + local LAST_OUT=/tmp/tmp.VbjI8HYfdO ++ mktemp + local LAST_ERR=/tmp/tmp.pCjYicUndC + local exit_status=0 + local timeout=4 + args=('apply' '-f' '-') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl apply -f - + return + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.VbjI8HYfdO perconaservermongodb.psmdb.percona.com/monitoring created + cat /tmp/tmp.pCjYicUndC + rm /tmp/tmp.VbjI8HYfdO /tmp/tmp.pCjYicUndC + return 0 + wait_for_running monitoring-rs0 3 + local name=monitoring-rs0 + let last_pod=2 + local check_cluster_readiness=true + set_debug + [[ 1 == 1 ]] + set -o xtrace + local rs_name=rs0 + local cluster_name=monitoring ++ seq 0 2 + for i in $(seq 0 $last_pod) + [[ 0 -eq 2 ]] + wait_pod monitoring-rs0-0 + local pod=monitoring-rs0-0 + set +o xtrace waiting for pod/monitoring-rs0-0 to be ready............OK + for i in $(seq 0 $last_pod) + [[ 1 -eq 2 ]] + wait_pod monitoring-rs0-1 + local pod=monitoring-rs0-1 + set +o xtrace waiting for pod/monitoring-rs0-1 to be ready............OK + for i in $(seq 0 $last_pod) + [[ 2 -eq 2 ]] ++ kubectl_bin get psmdb monitoring -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].arbiter.enabled}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.janS3qSSHJ +++ mktemp ++ local LAST_ERR=/tmp/tmp.4Evk2hlVIF ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'psmdb' 'monitoring' '-o' 'jsonpath={.spec.replsets[?(@.name=="rs0")].arbiter.enabled}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb monitoring -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].arbiter.enabled}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.janS3qSSHJ ++ cat /tmp/tmp.4Evk2hlVIF ++ rm /tmp/tmp.janS3qSSHJ /tmp/tmp.4Evk2hlVIF ++ return 0 + [[ '' == true ]] + wait_pod monitoring-rs0-2 + local pod=monitoring-rs0-2 + set +o xtrace waiting for pod/monitoring-rs0-2 to be ready..............OK ++ kubectl_bin get psmdb monitoring -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].nonvoting.enabled}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.2TW1oVPEnW +++ mktemp ++ local LAST_ERR=/tmp/tmp.mh3twJ9NYR ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'psmdb' 'monitoring' '-o' 'jsonpath={.spec.replsets[?(@.name=="rs0")].nonvoting.enabled}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb monitoring -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].nonvoting.enabled}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.2TW1oVPEnW ++ cat /tmp/tmp.mh3twJ9NYR ++ rm /tmp/tmp.2TW1oVPEnW /tmp/tmp.mh3twJ9NYR ++ return 0 + [[ '' == true ]] ++ kubectl_bin get psmdb monitoring -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].hidden.enabled}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.p3qLYDrVJy +++ mktemp ++ local LAST_ERR=/tmp/tmp.zgK9fkI6x6 ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'psmdb' 'monitoring' '-o' 'jsonpath={.spec.replsets[?(@.name=="rs0")].hidden.enabled}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb monitoring -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].hidden.enabled}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.p3qLYDrVJy ++ cat /tmp/tmp.zgK9fkI6x6 ++ rm /tmp/tmp.p3qLYDrVJy /tmp/tmp.zgK9fkI6x6 ++ return 0 + [[ '' == true ]] + sleep 10 + [[ true == true ]] + set +x Waiting for cluster monitoring to be ready..................... + apply_client_with_tls monitoring + local cluster=monitoring + kubectl_bin apply -f - + yq '.spec.template.spec.volumes[0].secret.secretName="monitoring-ssl"' /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2465/e2e-tests/conf/client_with_tls.yml ++ mktemp + local LAST_OUT=/tmp/tmp.n7oc12XePN ++ mktemp + local LAST_ERR=/tmp/tmp.wFYtUkJsjx + local exit_status=0 + local timeout=4 + args=('apply' '-f' '-') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.n7oc12XePN deployment.apps/psmdb-client created + cat /tmp/tmp.wFYtUkJsjx + rm /tmp/tmp.n7oc12XePN /tmp/tmp.wFYtUkJsjx + return 0 + kubectl_bin rollout status deployment/psmdb-client --timeout=360s ++ mktemp + local LAST_OUT=/tmp/tmp.KUsKDInMl7 ++ mktemp + local LAST_ERR=/tmp/tmp.T8SL1tOEba + local exit_status=0 + local timeout=4 + args=('rollout' 'status' 'deployment/psmdb-client' '--timeout=360s') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl rollout status deployment/psmdb-client --timeout=360s + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.KUsKDInMl7 deployment "psmdb-client" successfully rolled out + cat /tmp/tmp.T8SL1tOEba + rm /tmp/tmp.KUsKDInMl7 /tmp/tmp.T8SL1tOEba + return 0 + desc 'check if pmm-client container is not enabled' + set +o xtrace ----------------------------------------------------------------------------------- check if pmm-client container is not enabled ----------------------------------------------------------------------------------- + compare_kubectl statefulset/monitoring-rs0 -no-pmm + local resource=statefulset/monitoring-rs0 + local postfix=-no-pmm + local skip_generation_check= + local expected_result=/mnt/jenkins/workspace/cloud-psmdb-operator_PR-2465/e2e-tests/monitoring-2-0/compare/statefulset_monitoring-rs0-no-pmm.yml + local new_result=/tmp/tmp.G9Wq8CVNYh/statefulset_monitoring-rs0.yml + [[ -n '' ]] + kubectl_bin get -o yaml statefulset/monitoring-rs0 + yq eval $'\n\t\t\tdel(.metadata.ownerReferences[].apiVersion) |\n\t\t\tdel(.metadata.managedFields) |\n\t\t\tdel(.. | select(has("creationTimestamp")).creationTimestamp) |\n\t\t\tdel(.. | select(has("namespace")).namespace) |\n\t\t\tdel(.. | select(has("uid")).uid) |\n\t\t\tdel(.metadata.resourceVersion) |\n\t\t\tdel(.spec.template.spec.containers[].env[] | select(.name == "NAMESPACE")) |\n\t\t\tdel(.metadata.selfLink) |\n\t\t\tdel(.metadata.annotations."cloud.google.com/neg") |\n\t\t\tdel(.metadata.annotations."field.cattle.io/publicEndpoints") |\n\t\t\tdel(.metadata.annotations."metallb.io/ip-allocated-from-pool") |\n\t\t\tdel(.metadata.annotations."networking.gke.io/target-pool") |\n\t\t\tdel(.metadata.annotations."kubectl.kubernetes.io/last-applied-configuration") |\n\t\t\tdel(.. | select(has("image")).image) |\n\t\t\tdel(.. | select(has("clusterIP")).clusterIP) |\n\t\t\tdel(.. | select(has("clusterIPs")).clusterIPs) |\n\t\t\tdel(.. | select(has("dataSource")).dataSource) |\n\t\t\tdel(.. | select(has("procMount")).procMount) |\n\t\t\tdel(.. | select(has("storageClassName")).storageClassName) |\n\t\t\tdel(.. | select(has("finalizers")).finalizers) |\n\t\t\tdel(.. | select(has("kubernetes.io/pvc-protection"))."kubernetes.io/pvc-protection") |\n\t\t\tdel(.. | select(has("volumeName")).volumeName) |\n\t\t\tdel(.. | select(has("volume.beta.kubernetes.io/storage-provisioner"))."volume.beta.kubernetes.io/storage-provisioner") |\n\t\t\tdel(.. | select(has("volume.kubernetes.io/storage-provisioner"))."volume.kubernetes.io/storage-provisioner") |\n\t\t\tdel(.spec.volumeMode) |\n\t\t\tdel(.. | select(has("volume.kubernetes.io/selected-node"))."volume.kubernetes.io/selected-node") |\n\t\t\tdel(.. | select(has("percona.com/last-config-hash"))."percona.com/last-config-hash") |\n\t\t\tdel(.. | select(has("percona.com/configuration-hash"))."percona.com/configuration-hash") |\n\t\t\tdel(.. | select(has("percona.com/ssl-hash"))."percona.com/ssl-hash") |\n\t\t\tdel(.. | select(has("percona.com/ssl-internal-hash"))."percona.com/ssl-internal-hash") |\n\t\t\tdel(.spec.volumeClaimTemplates[].spec.volumeMode | select(. == "Filesystem")) |\n\t\t\tdel(.. | select(has("healthCheckNodePort")).healthCheckNodePort) |\n\t\t\tdel(.. | select(has("nodePort")).nodePort) |\n\t\t\tdel(.status) |\n\t\t\t(.. | select(tag == "!!str")) |= sub("monitoring-2-0-3915", "NAME_SPACE") |\n\t\t\tdel(.spec.volumeClaimTemplates[].apiVersion) |\n\t\t\tdel(.spec.volumeClaimTemplates[].kind) |\n\t\t\tdel(.spec.ipFamilies) |\n\t\t\tdel(.spec.ipFamilyPolicy) |\n\t\t\t(.. | select(. == "extensions/v1beta1")) = "apps/v1" |\n\t\t\t(.. | select(. == "batch/v1beta1")) = "batch/v1" ' - ++ mktemp + local LAST_OUT=/tmp/tmp.GYaDxJMJf6 ++ mktemp + local LAST_ERR=/tmp/tmp.SORn3MU1CD + local exit_status=0 + local timeout=4 + args=('get' '-o' 'yaml' 'statefulset/monitoring-rs0') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl get -o yaml statefulset/monitoring-rs0 + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.GYaDxJMJf6 + cat /tmp/tmp.SORn3MU1CD + rm /tmp/tmp.GYaDxJMJf6 /tmp/tmp.SORn3MU1CD + return 0 + yq -i eval 'del(.spec.persistentVolumeClaimRetentionPolicy)' /tmp/tmp.G9Wq8CVNYh/statefulset_monitoring-rs0.yml + arch_scheduling_enabled + [[ -n '' ]] + version_gt 1.22 ++ bc -l ++ echo '1.33 >= 1.22' + '[' 1 -eq 1 ']' + return 0 + yq -i eval 'del(.spec.internalTrafficPolicy)' /tmp/tmp.G9Wq8CVNYh/statefulset_monitoring-rs0.yml + yq -i eval 'del(.spec.allocateLoadBalancerNodePorts)' /tmp/tmp.G9Wq8CVNYh/statefulset_monitoring-rs0.yml + [[ /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2465/e2e-tests/monitoring-2-0/compare/statefulset_monitoring-rs0-no-pmm.yml == */cronjob* ]] + '[' -n '' ']' + [[ 0 -eq 0 ]] + diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2465/e2e-tests/monitoring-2-0/compare/statefulset_monitoring-rs0-no-pmm.yml /tmp/tmp.G9Wq8CVNYh/statefulset_monitoring-rs0.yml + log 'compare_kubectl: statefulset/monitoring-rs0 OK' + set +o xtrace [2026-07-22T18:35:52+0000] compare_kubectl: statefulset/monitoring-rs0 OK + sleep 10 + custom_port=27019 + run_mongos 'db.createUser({user:"myApp",pwd:"myPass",roles:[{db:"myApp",role:"readWrite"}]})' userAdmin:userAdmin123456@monitoring-mongos.monitoring-2-0-3915 '' '' '--tlsCertificateKeyFile /tmp/tls.pem --tlsCAFile /etc/mongodb-ssl/ca.crt --tls' 27019 + local 'command=db.createUser({user:"myApp",pwd:"myPass",roles:[{db:"myApp",role:"readWrite"}]})' + local uri=userAdmin:userAdmin123456@monitoring-mongos.monitoring-2-0-3915 + local driver=mongodb + local suffix=.svc.cluster.local + local 'mongo_flag=--tlsCertificateKeyFile /tmp/tls.pem --tlsCAFile /etc/mongodb-ssl/ca.crt --tls' + local port=27019 + local mongo_bin=mongo ++ echo .svc.cluster.local ++ awk -F: '{print $2}' + suffix_port= + [[ -z '' ]] + suffix=.svc.cluster.local:27019 ++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.SlxNqAIYRu +++ mktemp ++ local LAST_ERR=/tmp/tmp.9oc2Pn4uq1 ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'pods' '--selector=name=psmdb-client' '-o' 'jsonpath={.items[].metadata.name}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.SlxNqAIYRu ++ cat /tmp/tmp.9oc2Pn4uq1 ++ rm /tmp/tmp.SlxNqAIYRu /tmp/tmp.9oc2Pn4uq1 ++ return 0 + local client_container=psmdb-client-5b9745b9f-ncwjh + kubectl_bin exec psmdb-client-5b9745b9f-ncwjh -- bash -c 'export HOME=/tmp; printf '\''db.createUser({user:"myApp",pwd:"myPass",roles:[{db:"myApp",role:"readWrite"}]})\n'\'' | mongo mongodb://userAdmin:userAdmin123456@monitoring-mongos.monitoring-2-0-3915.svc.cluster.local:27019/admin --tlsCertificateKeyFile /tmp/tls.pem --tlsCAFile /etc/mongodb-ssl/ca.crt --tls' ++ mktemp + local LAST_OUT=/tmp/tmp.CLRHv6MxkM ++ mktemp + local LAST_ERR=/tmp/tmp.hSQbphcVcs + local exit_status=0 + local timeout=4 + args=('exec' 'psmdb-client-5b9745b9f-ncwjh' '--' 'bash' '-c' 'export HOME=/tmp; printf '\''db.createUser({user:"myApp",pwd:"myPass",roles:[{db:"myApp",role:"readWrite"}]})\n'\'' | mongo mongodb://userAdmin:userAdmin123456@monitoring-mongos.monitoring-2-0-3915.svc.cluster.local:27019/admin --tlsCertificateKeyFile /tmp/tls.pem --tlsCAFile /etc/mongodb-ssl/ca.crt --tls') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl exec psmdb-client-5b9745b9f-ncwjh -- bash -c 'export HOME=/tmp; printf '\''db.createUser({user:"myApp",pwd:"myPass",roles:[{db:"myApp",role:"readWrite"}]})\n'\'' | mongo mongodb://userAdmin:userAdmin123456@monitoring-mongos.monitoring-2-0-3915.svc.cluster.local:27019/admin --tlsCertificateKeyFile /tmp/tls.pem --tlsCAFile /etc/mongodb-ssl/ca.crt --tls' + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.CLRHv6MxkM Percona Server for MongoDB shell version v4.4.29-28 connecting to: mongodb://monitoring-mongos.monitoring-2-0-3915.svc.cluster.local:27019/admin?compressors=disabled&gssapiServiceName=mongodb {"t":{"$date":"2026-07-22T18:36:05.136Z"},"s":"I", "c":"NETWORK", "id":5490002, "ctx":"thread1","msg":"Started a new thread for the timer service"} Implicit session: session { "id" : UUID("84cbe02b-f118-4a56-a2c4-379c797ba064") } Percona Server for MongoDB server version: v8.0.26-11 WARNING: shell and server versions do not match Successfully added user: { "user" : "myApp", "roles" : [ { "db" : "myApp", "role" : "readWrite" } ] } bye + cat /tmp/tmp.hSQbphcVcs + rm /tmp/tmp.CLRHv6MxkM /tmp/tmp.hSQbphcVcs + return 0 + run_mongos 'sh.enableSharding("myApp")' clusterAdmin:clusterAdmin123456@monitoring-mongos.monitoring-2-0-3915 '' '' '--tlsCertificateKeyFile /tmp/tls.pem --tlsCAFile /etc/mongodb-ssl/ca.crt --tls' 27019 + local 'command=sh.enableSharding("myApp")' + local uri=clusterAdmin:clusterAdmin123456@monitoring-mongos.monitoring-2-0-3915 + local driver=mongodb + local suffix=.svc.cluster.local + local 'mongo_flag=--tlsCertificateKeyFile /tmp/tls.pem --tlsCAFile /etc/mongodb-ssl/ca.crt --tls' + local port=27019 + local mongo_bin=mongo ++ awk -F: '{print $2}' ++ echo .svc.cluster.local + suffix_port= + [[ -z '' ]] + suffix=.svc.cluster.local:27019 ++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.k8PglYl2cc +++ mktemp ++ local LAST_ERR=/tmp/tmp.mTctnYonIA ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'pods' '--selector=name=psmdb-client' '-o' 'jsonpath={.items[].metadata.name}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.k8PglYl2cc ++ cat /tmp/tmp.mTctnYonIA ++ rm /tmp/tmp.k8PglYl2cc /tmp/tmp.mTctnYonIA ++ return 0 + local client_container=psmdb-client-5b9745b9f-ncwjh + kubectl_bin exec psmdb-client-5b9745b9f-ncwjh -- bash -c 'export HOME=/tmp; printf '\''sh.enableSharding("myApp")\n'\'' | mongo mongodb://clusterAdmin:clusterAdmin123456@monitoring-mongos.monitoring-2-0-3915.svc.cluster.local:27019/admin --tlsCertificateKeyFile /tmp/tls.pem --tlsCAFile /etc/mongodb-ssl/ca.crt --tls' ++ mktemp + local LAST_OUT=/tmp/tmp.rvZ9nO4BzS ++ mktemp + local LAST_ERR=/tmp/tmp.7280tUnCpg + local exit_status=0 + local timeout=4 + args=('exec' 'psmdb-client-5b9745b9f-ncwjh' '--' 'bash' '-c' 'export HOME=/tmp; printf '\''sh.enableSharding("myApp")\n'\'' | mongo mongodb://clusterAdmin:clusterAdmin123456@monitoring-mongos.monitoring-2-0-3915.svc.cluster.local:27019/admin --tlsCertificateKeyFile /tmp/tls.pem --tlsCAFile /etc/mongodb-ssl/ca.crt --tls') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl exec psmdb-client-5b9745b9f-ncwjh -- bash -c 'export HOME=/tmp; printf '\''sh.enableSharding("myApp")\n'\'' | mongo mongodb://clusterAdmin:clusterAdmin123456@monitoring-mongos.monitoring-2-0-3915.svc.cluster.local:27019/admin --tlsCertificateKeyFile /tmp/tls.pem --tlsCAFile /etc/mongodb-ssl/ca.crt --tls' + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.rvZ9nO4BzS Percona Server for MongoDB shell version v4.4.29-28 connecting to: mongodb://monitoring-mongos.monitoring-2-0-3915.svc.cluster.local:27019/admin?compressors=disabled&gssapiServiceName=mongodb {"t":{"$date":"2026-07-22T18:36:08.002Z"},"s":"I", "c":"NETWORK", "id":5490002, "ctx":"thread1","msg":"Started a new thread for the timer service"} Implicit session: session { "id" : UUID("1784ade9-869c-46c5-85e2-dc14c9b6e535") } Percona Server for MongoDB server version: v8.0.26-11 WARNING: shell and server versions do not match { "ok" : 1, "$clusterTime" : { "clusterTime" : Timestamp(1784745368, 8), "signature" : { "hash" : BinData(0,"LEo9H+R9Htr2I8jl7G+l5l2I63c="), "keyId" : NumberLong("7665422587815526424") } }, "operationTime" : Timestamp(1784745368, 5) } bye + cat /tmp/tmp.7280tUnCpg + rm /tmp/tmp.rvZ9nO4BzS /tmp/tmp.7280tUnCpg + return 0 + insert_data_mongos 100500 myApp '--tlsCertificateKeyFile /tmp/tls.pem --tlsCAFile /etc/mongodb-ssl/ca.crt --tls' 27019 + local data=100500 + local db_name=myApp + local 'flags=--tlsCertificateKeyFile /tmp/tls.pem --tlsCAFile /etc/mongodb-ssl/ca.crt --tls' + local port=27019 + run_mongos 'use myApp\n db.test.insert({ x: 100500 })' myApp:myPass@monitoring-mongos.monitoring-2-0-3915 '' '' '--tlsCertificateKeyFile /tmp/tls.pem --tlsCAFile /etc/mongodb-ssl/ca.crt --tls' 27019 + local 'command=use myApp\n db.test.insert({ x: 100500 })' + local uri=myApp:myPass@monitoring-mongos.monitoring-2-0-3915 + local driver=mongodb + local suffix=.svc.cluster.local + local 'mongo_flag=--tlsCertificateKeyFile /tmp/tls.pem --tlsCAFile /etc/mongodb-ssl/ca.crt --tls' + local port=27019 + local mongo_bin=mongo ++ echo .svc.cluster.local ++ awk -F: '{print $2}' + suffix_port= + [[ -z '' ]] + suffix=.svc.cluster.local:27019 ++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.c7uiFSEIcr +++ mktemp ++ local LAST_ERR=/tmp/tmp.0bYqvjYjXi ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'pods' '--selector=name=psmdb-client' '-o' 'jsonpath={.items[].metadata.name}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.c7uiFSEIcr ++ cat /tmp/tmp.0bYqvjYjXi ++ rm /tmp/tmp.c7uiFSEIcr /tmp/tmp.0bYqvjYjXi ++ return 0 + local client_container=psmdb-client-5b9745b9f-ncwjh + kubectl_bin exec psmdb-client-5b9745b9f-ncwjh -- bash -c 'export HOME=/tmp; printf '\''use myApp\n db.test.insert({ x: 100500 })\n'\'' | mongo mongodb://myApp:myPass@monitoring-mongos.monitoring-2-0-3915.svc.cluster.local:27019/admin --tlsCertificateKeyFile /tmp/tls.pem --tlsCAFile /etc/mongodb-ssl/ca.crt --tls' ++ mktemp + local LAST_OUT=/tmp/tmp.pLiEsiI7LY ++ mktemp + local LAST_ERR=/tmp/tmp.4C7Th7YHm3 + local exit_status=0 + local timeout=4 + args=('exec' 'psmdb-client-5b9745b9f-ncwjh' '--' 'bash' '-c' 'export HOME=/tmp; printf '\''use myApp\n db.test.insert({ x: 100500 })\n'\'' | mongo mongodb://myApp:myPass@monitoring-mongos.monitoring-2-0-3915.svc.cluster.local:27019/admin --tlsCertificateKeyFile /tmp/tls.pem --tlsCAFile /etc/mongodb-ssl/ca.crt --tls') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl exec psmdb-client-5b9745b9f-ncwjh -- bash -c 'export HOME=/tmp; printf '\''use myApp\n db.test.insert({ x: 100500 })\n'\'' | mongo mongodb://myApp:myPass@monitoring-mongos.monitoring-2-0-3915.svc.cluster.local:27019/admin --tlsCertificateKeyFile /tmp/tls.pem --tlsCAFile /etc/mongodb-ssl/ca.crt --tls' + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.pLiEsiI7LY Percona Server for MongoDB shell version v4.4.29-28 connecting to: mongodb://monitoring-mongos.monitoring-2-0-3915.svc.cluster.local:27019/admin?compressors=disabled&gssapiServiceName=mongodb {"t":{"$date":"2026-07-22T18:36:11.098Z"},"s":"I", "c":"NETWORK", "id":5490002, "ctx":"thread1","msg":"Started a new thread for the timer service"} Implicit session: session { "id" : UUID("97ea3f16-dbb4-4bfe-a275-2a45a1c61ff9") } Percona Server for MongoDB server version: v8.0.26-11 WARNING: shell and server versions do not match switched to db myApp WriteResult({ "nInserted" : 1 }) bye + cat /tmp/tmp.4C7Th7YHm3 + rm /tmp/tmp.pLiEsiI7LY /tmp/tmp.4C7Th7YHm3 + return 0 + insert_data_mongos 100600 myApp '--tlsCertificateKeyFile /tmp/tls.pem --tlsCAFile /etc/mongodb-ssl/ca.crt --tls' 27019 + local data=100600 + local db_name=myApp + local 'flags=--tlsCertificateKeyFile /tmp/tls.pem --tlsCAFile /etc/mongodb-ssl/ca.crt --tls' + local port=27019 + run_mongos 'use myApp\n db.test.insert({ x: 100600 })' myApp:myPass@monitoring-mongos.monitoring-2-0-3915 '' '' '--tlsCertificateKeyFile /tmp/tls.pem --tlsCAFile /etc/mongodb-ssl/ca.crt --tls' 27019 + local 'command=use myApp\n db.test.insert({ x: 100600 })' + local uri=myApp:myPass@monitoring-mongos.monitoring-2-0-3915 + local driver=mongodb + local suffix=.svc.cluster.local + local 'mongo_flag=--tlsCertificateKeyFile /tmp/tls.pem --tlsCAFile /etc/mongodb-ssl/ca.crt --tls' + local port=27019 + local mongo_bin=mongo ++ echo .svc.cluster.local ++ awk -F: '{print $2}' + suffix_port= + [[ -z '' ]] + suffix=.svc.cluster.local:27019 ++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.PIiiZ1P84Y +++ mktemp ++ local LAST_ERR=/tmp/tmp.e3MHIf7E3B ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'pods' '--selector=name=psmdb-client' '-o' 'jsonpath={.items[].metadata.name}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.PIiiZ1P84Y ++ cat /tmp/tmp.e3MHIf7E3B ++ rm /tmp/tmp.PIiiZ1P84Y /tmp/tmp.e3MHIf7E3B ++ return 0 + local client_container=psmdb-client-5b9745b9f-ncwjh + kubectl_bin exec psmdb-client-5b9745b9f-ncwjh -- bash -c 'export HOME=/tmp; printf '\''use myApp\n db.test.insert({ x: 100600 })\n'\'' | mongo mongodb://myApp:myPass@monitoring-mongos.monitoring-2-0-3915.svc.cluster.local:27019/admin --tlsCertificateKeyFile /tmp/tls.pem --tlsCAFile /etc/mongodb-ssl/ca.crt --tls' ++ mktemp + local LAST_OUT=/tmp/tmp.XSf2FH6pFf ++ mktemp + local LAST_ERR=/tmp/tmp.TaurSP4iA3 + local exit_status=0 + local timeout=4 + args=('exec' 'psmdb-client-5b9745b9f-ncwjh' '--' 'bash' '-c' 'export HOME=/tmp; printf '\''use myApp\n db.test.insert({ x: 100600 })\n'\'' | mongo mongodb://myApp:myPass@monitoring-mongos.monitoring-2-0-3915.svc.cluster.local:27019/admin --tlsCertificateKeyFile /tmp/tls.pem --tlsCAFile /etc/mongodb-ssl/ca.crt --tls') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl exec psmdb-client-5b9745b9f-ncwjh -- bash -c 'export HOME=/tmp; printf '\''use myApp\n db.test.insert({ x: 100600 })\n'\'' | mongo mongodb://myApp:myPass@monitoring-mongos.monitoring-2-0-3915.svc.cluster.local:27019/admin --tlsCertificateKeyFile /tmp/tls.pem --tlsCAFile /etc/mongodb-ssl/ca.crt --tls' + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.XSf2FH6pFf Percona Server for MongoDB shell version v4.4.29-28 connecting to: mongodb://monitoring-mongos.monitoring-2-0-3915.svc.cluster.local:27019/admin?compressors=disabled&gssapiServiceName=mongodb {"t":{"$date":"2026-07-22T18:36:14.151Z"},"s":"I", "c":"NETWORK", "id":5490002, "ctx":"thread1","msg":"Started a new thread for the timer service"} Implicit session: session { "id" : UUID("587d8900-e716-49f7-b274-b2f207354b6f") } Percona Server for MongoDB server version: v8.0.26-11 WARNING: shell and server versions do not match switched to db myApp WriteResult({ "nInserted" : 1 }) bye + cat /tmp/tmp.TaurSP4iA3 + rm /tmp/tmp.XSf2FH6pFf /tmp/tmp.TaurSP4iA3 + return 0 + insert_data_mongos 100700 myApp '--tlsCertificateKeyFile /tmp/tls.pem --tlsCAFile /etc/mongodb-ssl/ca.crt --tls' 27019 + local data=100700 + local db_name=myApp + local 'flags=--tlsCertificateKeyFile /tmp/tls.pem --tlsCAFile /etc/mongodb-ssl/ca.crt --tls' + local port=27019 + run_mongos 'use myApp\n db.test.insert({ x: 100700 })' myApp:myPass@monitoring-mongos.monitoring-2-0-3915 '' '' '--tlsCertificateKeyFile /tmp/tls.pem --tlsCAFile /etc/mongodb-ssl/ca.crt --tls' 27019 + local 'command=use myApp\n db.test.insert({ x: 100700 })' + local uri=myApp:myPass@monitoring-mongos.monitoring-2-0-3915 + local driver=mongodb + local suffix=.svc.cluster.local + local 'mongo_flag=--tlsCertificateKeyFile /tmp/tls.pem --tlsCAFile /etc/mongodb-ssl/ca.crt --tls' + local port=27019 + local mongo_bin=mongo ++ echo .svc.cluster.local ++ awk -F: '{print $2}' + suffix_port= + [[ -z '' ]] + suffix=.svc.cluster.local:27019 ++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.zMhRmslhcE +++ mktemp ++ local LAST_ERR=/tmp/tmp.qzrhICyyRc ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'pods' '--selector=name=psmdb-client' '-o' 'jsonpath={.items[].metadata.name}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.zMhRmslhcE ++ cat /tmp/tmp.qzrhICyyRc ++ rm /tmp/tmp.zMhRmslhcE /tmp/tmp.qzrhICyyRc ++ return 0 + local client_container=psmdb-client-5b9745b9f-ncwjh + kubectl_bin exec psmdb-client-5b9745b9f-ncwjh -- bash -c 'export HOME=/tmp; printf '\''use myApp\n db.test.insert({ x: 100700 })\n'\'' | mongo mongodb://myApp:myPass@monitoring-mongos.monitoring-2-0-3915.svc.cluster.local:27019/admin --tlsCertificateKeyFile /tmp/tls.pem --tlsCAFile /etc/mongodb-ssl/ca.crt --tls' ++ mktemp + local LAST_OUT=/tmp/tmp.fP1ylRkrjn ++ mktemp + local LAST_ERR=/tmp/tmp.qfNMJlT1gR + local exit_status=0 + local timeout=4 + args=('exec' 'psmdb-client-5b9745b9f-ncwjh' '--' 'bash' '-c' 'export HOME=/tmp; printf '\''use myApp\n db.test.insert({ x: 100700 })\n'\'' | mongo mongodb://myApp:myPass@monitoring-mongos.monitoring-2-0-3915.svc.cluster.local:27019/admin --tlsCertificateKeyFile /tmp/tls.pem --tlsCAFile /etc/mongodb-ssl/ca.crt --tls') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl exec psmdb-client-5b9745b9f-ncwjh -- bash -c 'export HOME=/tmp; printf '\''use myApp\n db.test.insert({ x: 100700 })\n'\'' | mongo mongodb://myApp:myPass@monitoring-mongos.monitoring-2-0-3915.svc.cluster.local:27019/admin --tlsCertificateKeyFile /tmp/tls.pem --tlsCAFile /etc/mongodb-ssl/ca.crt --tls' + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.fP1ylRkrjn Percona Server for MongoDB shell version v4.4.29-28 connecting to: mongodb://monitoring-mongos.monitoring-2-0-3915.svc.cluster.local:27019/admin?compressors=disabled&gssapiServiceName=mongodb {"t":{"$date":"2026-07-22T18:36:17.128Z"},"s":"I", "c":"NETWORK", "id":5490002, "ctx":"thread1","msg":"Started a new thread for the timer service"} Implicit session: session { "id" : UUID("cd2b08e7-41f1-4344-af97-118796b35eaa") } Percona Server for MongoDB server version: v8.0.26-11 WARNING: shell and server versions do not match switched to db myApp WriteResult({ "nInserted" : 1 }) bye + cat /tmp/tmp.qfNMJlT1gR + rm /tmp/tmp.fP1ylRkrjn /tmp/tmp.qfNMJlT1gR + return 0 + desc 'add PMM_SERVER_API_KEY for secret some-users' + set +o xtrace ----------------------------------------------------------------------------------- add PMM_SERVER_API_KEY for secret some-users ----------------------------------------------------------------------------------- ++ get_service_endpoint monitoring-service ++ local service=monitoring-service ++ local endpoint ++ local service_type +++ kubectl_bin get service/monitoring-service -o 'jsonpath={.spec.type}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.6vNPqe5Rlg ++++ mktemp +++ local LAST_ERR=/tmp/tmp.szNkSEfPx4 +++ local exit_status=0 +++ local timeout=4 +++ args=('get' 'service/monitoring-service' '-o' 'jsonpath={.spec.type}') +++ local args +++ transformed_files=() +++ local transformed_files +++ arch_scheduling_enabled +++ [[ -n '' ]] +++ arch_scheduling_enabled +++ [[ -n '' ]] ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl get service/monitoring-service -o 'jsonpath={.spec.type}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 -a -n 1 ']' +++ break +++ cat /tmp/tmp.6vNPqe5Rlg +++ cat /tmp/tmp.szNkSEfPx4 +++ rm /tmp/tmp.6vNPqe5Rlg /tmp/tmp.szNkSEfPx4 +++ return 0 ++ service_type=LoadBalancer ++ [[ LoadBalancer == LoadBalancer ]] +++ get_load_balancer_endpoint monitoring-service +++ local service=monitoring-service +++ local endpoint ++++ kubectl_bin get service/monitoring-service -o json ++++ jq -r '[.status.loadBalancer.ingress[]?.hostname, .status.loadBalancer.ingress[]?.ip] | map(select(. != null and . != "")) | first // empty' +++++ mktemp ++++ local LAST_OUT=/tmp/tmp.U1fmtO5ZP8 +++++ mktemp ++++ local LAST_ERR=/tmp/tmp.j2u8MF2dE5 ++++ local exit_status=0 ++++ local timeout=4 ++++ args=('get' 'service/monitoring-service' '-o' 'json') ++++ local args ++++ transformed_files=() ++++ local transformed_files ++++ arch_scheduling_enabled ++++ [[ -n '' ]] ++++ arch_scheduling_enabled ++++ [[ -n '' ]] +++++ seq 0 2 ++++ for i in $(seq 0 2) ++++ set +e ++++ kubectl get service/monitoring-service -o json ++++ exit_status=0 ++++ set -e ++++ '[' 0 '!=' 0 -a -n 1 ']' ++++ break ++++ cat /tmp/tmp.U1fmtO5ZP8 ++++ cat /tmp/tmp.j2u8MF2dE5 ++++ rm /tmp/tmp.U1fmtO5ZP8 /tmp/tmp.j2u8MF2dE5 ++++ return 0 +++ endpoint=34.28.217.84 +++ '[' -n 34.28.217.84 -a 34.28.217.84 '!=' null ']' +++ echo 34.28.217.84 +++ return ++ endpoint=34.28.217.84 ++ echo 34.28.217.84 ++ return + monitoring_endpoint=34.28.217.84 ++ urlencode_userinfo admin:admin ++ local user_pass=admin:admin ++ local username=admin ++ local password=admin +++ urlencode admin +++ uri=admin +++ echo -n admin +++ jq -s -R -r @uri +++ urlencode admin +++ uri=admin +++ echo -n admin +++ jq -s -R -r @uri ++ printf %s:%s admin admin + encoded_admin_userinfo=admin:admin ++ execute_curl --insecure -sS -X POST -H 'Content-Type: application/json' -d '{"name":"operator", "role": "Admin"}' https://admin:admin@34.28.217.84/graph/api/auth/keys ++ local name=curl-12100 ++ jq -r .key ++ local image=docker.io/alpine/curl:latest ++ local phase= ++ local exit_status=0 ++ kubectl_bin run curl-12100 -n monitoring-2-0-3915 --restart=Never --image=docker.io/alpine/curl:latest --command -- curl -sS --insecure -sS -X POST -H 'Content-Type: application/json' -d '{"name":"operator", "role": "Admin"}' https://admin:admin@34.28.217.84/graph/api/auth/keys +++ mktemp ++ local LAST_OUT=/tmp/tmp.ZDI0ukuGVy +++ mktemp ++ local LAST_ERR=/tmp/tmp.26Zd0q8Atg ++ local exit_status=0 ++ local timeout=4 ++ args=('run' 'curl-12100' '-n' 'monitoring-2-0-3915' '--restart=Never' '--image=docker.io/alpine/curl:latest' '--command' '--' 'curl' '-sS' '--insecure' '-sS' '-X' 'POST' '-H' 'Content-Type: application/json' '-d' '{"name":"operator", "role": "Admin"}' 'https://admin:admin@34.28.217.84/graph/api/auth/keys') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl run curl-12100 -n monitoring-2-0-3915 --restart=Never --image=docker.io/alpine/curl:latest --command -- curl -sS --insecure -sS -X POST -H 'Content-Type: application/json' -d '{"name":"operator", "role": "Admin"}' https://admin:admin@34.28.217.84/graph/api/auth/keys ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.ZDI0ukuGVy ++ cat /tmp/tmp.26Zd0q8Atg ++ rm /tmp/tmp.ZDI0ukuGVy /tmp/tmp.26Zd0q8Atg ++ return 0 +++ seq 1 60 ++ for _ in $(seq 1 60) +++ kubectl_bin get pod curl-12100 -n monitoring-2-0-3915 -o 'jsonpath={.status.phase}' ++ phase=Pending ++ [[ Pending == Succeeded ]] ++ [[ Pending == Failed ]] ++ sleep 1 ++ for _ in $(seq 1 60) +++ kubectl_bin get pod curl-12100 -n monitoring-2-0-3915 -o 'jsonpath={.status.phase}' ++ phase=Succeeded ++ [[ Succeeded == Succeeded ]] ++ break ++ kubectl_bin logs curl-12100 -n monitoring-2-0-3915 +++ mktemp ++ local LAST_OUT=/tmp/tmp.mMkbNLjRy5 +++ mktemp ++ local LAST_ERR=/tmp/tmp.qDXQWWVcrK ++ local exit_status=0 ++ local timeout=4 ++ args=('logs' 'curl-12100' '-n' 'monitoring-2-0-3915') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl logs curl-12100 -n monitoring-2-0-3915 ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.mMkbNLjRy5 ++ cat /tmp/tmp.qDXQWWVcrK ++ rm /tmp/tmp.mMkbNLjRy5 /tmp/tmp.qDXQWWVcrK ++ return 0 ++ kubectl_bin delete pod curl-12100 -n monitoring-2-0-3915 --ignore-not-found --wait=false ++ [[ Succeeded != Succeeded ]] ++ return 0 + API_KEY=eyJrIjoiWUtWVXU2VFhKR2tHUXdId3JwenZEVzlNVzcyQ2tuWnQiLCJuIjoib3BlcmF0b3IiLCJpZCI6MX0= + kubectl_bin patch secret some-users --type merge --patch '{"stringData": {"PMM_SERVER_API_KEY": "eyJrIjoiWUtWVXU2VFhKR2tHUXdId3JwenZEVzlNVzcyQ2tuWnQiLCJuIjoib3BlcmF0b3IiLCJpZCI6MX0="}}' ++ mktemp + local LAST_OUT=/tmp/tmp.uc9aymiT8g ++ mktemp + local LAST_ERR=/tmp/tmp.9EOW14BZLJ + local exit_status=0 + local timeout=4 + args=('patch' 'secret' 'some-users' '--type' 'merge' '--patch' '{"stringData": {"PMM_SERVER_API_KEY": "eyJrIjoiWUtWVXU2VFhKR2tHUXdId3JwenZEVzlNVzcyQ2tuWnQiLCJuIjoib3BlcmF0b3IiLCJpZCI6MX0="}}') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl patch secret some-users --type merge --patch '{"stringData": {"PMM_SERVER_API_KEY": "eyJrIjoiWUtWVXU2VFhKR2tHUXdId3JwenZEVzlNVzcyQ2tuWnQiLCJuIjoib3BlcmF0b3IiLCJpZCI6MX0="}}' + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.uc9aymiT8g secret/some-users patched + cat /tmp/tmp.9EOW14BZLJ + rm /tmp/tmp.uc9aymiT8g /tmp/tmp.9EOW14BZLJ + return 0 + desc 'check if all 3 Pods started' + set +o xtrace ----------------------------------------------------------------------------------- check if all 3 Pods started ----------------------------------------------------------------------------------- + wait_for_running monitoring-rs0 3 + local name=monitoring-rs0 + let last_pod=2 + local check_cluster_readiness=true + set_debug + [[ 1 == 1 ]] + set -o xtrace + local rs_name=rs0 + local cluster_name=monitoring ++ seq 0 2 + for i in $(seq 0 $last_pod) + [[ 0 -eq 2 ]] + wait_pod monitoring-rs0-0 + local pod=monitoring-rs0-0 + set +o xtrace waiting for pod/monitoring-rs0-0 to be ready.OK + for i in $(seq 0 $last_pod) + [[ 1 -eq 2 ]] + wait_pod monitoring-rs0-1 + local pod=monitoring-rs0-1 + set +o xtrace waiting for pod/monitoring-rs0-1 to be ready.OK + for i in $(seq 0 $last_pod) + [[ 2 -eq 2 ]] ++ kubectl_bin get psmdb monitoring -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].arbiter.enabled}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.wUFEOlH3gC +++ mktemp ++ local LAST_ERR=/tmp/tmp.ra1HOZTJbD ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'psmdb' 'monitoring' '-o' 'jsonpath={.spec.replsets[?(@.name=="rs0")].arbiter.enabled}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb monitoring -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].arbiter.enabled}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.wUFEOlH3gC ++ cat /tmp/tmp.ra1HOZTJbD ++ rm /tmp/tmp.wUFEOlH3gC /tmp/tmp.ra1HOZTJbD ++ return 0 + [[ '' == true ]] + wait_pod monitoring-rs0-2 + local pod=monitoring-rs0-2 + set +o xtrace waiting for pod/monitoring-rs0-2 to be ready.OK ++ kubectl_bin get psmdb monitoring -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].nonvoting.enabled}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.mii9gj4oEo +++ mktemp ++ local LAST_ERR=/tmp/tmp.gRnjSQza11 ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'psmdb' 'monitoring' '-o' 'jsonpath={.spec.replsets[?(@.name=="rs0")].nonvoting.enabled}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb monitoring -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].nonvoting.enabled}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.mii9gj4oEo ++ cat /tmp/tmp.gRnjSQza11 ++ rm /tmp/tmp.mii9gj4oEo /tmp/tmp.gRnjSQza11 ++ return 0 + [[ '' == true ]] ++ kubectl_bin get psmdb monitoring -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].hidden.enabled}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.WFxpaTlyhD +++ mktemp ++ local LAST_ERR=/tmp/tmp.XH0GaeJeTF ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'psmdb' 'monitoring' '-o' 'jsonpath={.spec.replsets[?(@.name=="rs0")].hidden.enabled}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb monitoring -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].hidden.enabled}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.WFxpaTlyhD ++ cat /tmp/tmp.XH0GaeJeTF ++ rm /tmp/tmp.WFxpaTlyhD /tmp/tmp.XH0GaeJeTF ++ return 0 + [[ '' == true ]] + sleep 10 + [[ true == true ]] + set +x Waiting for cluster monitoring to be ready...................................................................................................................................... + sleep 90 + desc 'check if pmm-client container enabled' + set +o xtrace ----------------------------------------------------------------------------------- check if pmm-client container enabled ----------------------------------------------------------------------------------- + compare_kubectl statefulset/monitoring-rs0 + local resource=statefulset/monitoring-rs0 + local postfix= + local skip_generation_check= + local expected_result=/mnt/jenkins/workspace/cloud-psmdb-operator_PR-2465/e2e-tests/monitoring-2-0/compare/statefulset_monitoring-rs0.yml + local new_result=/tmp/tmp.G9Wq8CVNYh/statefulset_monitoring-rs0.yml + [[ -n '' ]] + kubectl_bin get -o yaml statefulset/monitoring-rs0 + yq eval $'\n\t\t\tdel(.metadata.ownerReferences[].apiVersion) |\n\t\t\tdel(.metadata.managedFields) |\n\t\t\tdel(.. | select(has("creationTimestamp")).creationTimestamp) |\n\t\t\tdel(.. | select(has("namespace")).namespace) |\n\t\t\tdel(.. | select(has("uid")).uid) |\n\t\t\tdel(.metadata.resourceVersion) |\n\t\t\tdel(.spec.template.spec.containers[].env[] | select(.name == "NAMESPACE")) |\n\t\t\tdel(.metadata.selfLink) |\n\t\t\tdel(.metadata.annotations."cloud.google.com/neg") |\n\t\t\tdel(.metadata.annotations."field.cattle.io/publicEndpoints") |\n\t\t\tdel(.metadata.annotations."metallb.io/ip-allocated-from-pool") |\n\t\t\tdel(.metadata.annotations."networking.gke.io/target-pool") |\n\t\t\tdel(.metadata.annotations."kubectl.kubernetes.io/last-applied-configuration") |\n\t\t\tdel(.. | select(has("image")).image) |\n\t\t\tdel(.. | select(has("clusterIP")).clusterIP) |\n\t\t\tdel(.. | select(has("clusterIPs")).clusterIPs) |\n\t\t\tdel(.. | select(has("dataSource")).dataSource) |\n\t\t\tdel(.. | select(has("procMount")).procMount) |\n\t\t\tdel(.. | select(has("storageClassName")).storageClassName) |\n\t\t\tdel(.. | select(has("finalizers")).finalizers) |\n\t\t\tdel(.. | select(has("kubernetes.io/pvc-protection"))."kubernetes.io/pvc-protection") |\n\t\t\tdel(.. | select(has("volumeName")).volumeName) |\n\t\t\tdel(.. | select(has("volume.beta.kubernetes.io/storage-provisioner"))."volume.beta.kubernetes.io/storage-provisioner") |\n\t\t\tdel(.. | select(has("volume.kubernetes.io/storage-provisioner"))."volume.kubernetes.io/storage-provisioner") |\n\t\t\tdel(.spec.volumeMode) |\n\t\t\tdel(.. | select(has("volume.kubernetes.io/selected-node"))."volume.kubernetes.io/selected-node") |\n\t\t\tdel(.. | select(has("percona.com/last-config-hash"))."percona.com/last-config-hash") |\n\t\t\tdel(.. | select(has("percona.com/configuration-hash"))."percona.com/configuration-hash") |\n\t\t\tdel(.. | select(has("percona.com/ssl-hash"))."percona.com/ssl-hash") |\n\t\t\tdel(.. | select(has("percona.com/ssl-internal-hash"))."percona.com/ssl-internal-hash") |\n\t\t\tdel(.spec.volumeClaimTemplates[].spec.volumeMode | select(. == "Filesystem")) |\n\t\t\tdel(.. | select(has("healthCheckNodePort")).healthCheckNodePort) |\n\t\t\tdel(.. | select(has("nodePort")).nodePort) |\n\t\t\tdel(.status) |\n\t\t\t(.. | select(tag == "!!str")) |= sub("monitoring-2-0-3915", "NAME_SPACE") |\n\t\t\tdel(.spec.volumeClaimTemplates[].apiVersion) |\n\t\t\tdel(.spec.volumeClaimTemplates[].kind) |\n\t\t\tdel(.spec.ipFamilies) |\n\t\t\tdel(.spec.ipFamilyPolicy) |\n\t\t\t(.. | select(. == "extensions/v1beta1")) = "apps/v1" |\n\t\t\t(.. | select(. == "batch/v1beta1")) = "batch/v1" ' - ++ mktemp + local LAST_OUT=/tmp/tmp.Htnmc8hhxV ++ mktemp + local LAST_ERR=/tmp/tmp.NRtXx6YFT3 + local exit_status=0 + local timeout=4 + args=('get' '-o' 'yaml' 'statefulset/monitoring-rs0') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl get -o yaml statefulset/monitoring-rs0 + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.Htnmc8hhxV + cat /tmp/tmp.NRtXx6YFT3 + rm /tmp/tmp.Htnmc8hhxV /tmp/tmp.NRtXx6YFT3 + return 0 + yq -i eval 'del(.spec.persistentVolumeClaimRetentionPolicy)' /tmp/tmp.G9Wq8CVNYh/statefulset_monitoring-rs0.yml + arch_scheduling_enabled + [[ -n '' ]] + version_gt 1.22 ++ echo '1.33 >= 1.22' ++ bc -l + '[' 1 -eq 1 ']' + return 0 + yq -i eval 'del(.spec.internalTrafficPolicy)' /tmp/tmp.G9Wq8CVNYh/statefulset_monitoring-rs0.yml + yq -i eval 'del(.spec.allocateLoadBalancerNodePorts)' /tmp/tmp.G9Wq8CVNYh/statefulset_monitoring-rs0.yml + [[ /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2465/e2e-tests/monitoring-2-0/compare/statefulset_monitoring-rs0.yml == */cronjob* ]] + '[' -n '' ']' + [[ 0 -eq 0 ]] + diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2465/e2e-tests/monitoring-2-0/compare/statefulset_monitoring-rs0.yml /tmp/tmp.G9Wq8CVNYh/statefulset_monitoring-rs0.yml + log 'compare_kubectl: statefulset/monitoring-rs0 OK' + set +o xtrace [2026-07-22T18:42:54+0000] compare_kubectl: statefulset/monitoring-rs0 OK + compare_kubectl service/monitoring-rs0 + local resource=service/monitoring-rs0 + local postfix= + local skip_generation_check= + local expected_result=/mnt/jenkins/workspace/cloud-psmdb-operator_PR-2465/e2e-tests/monitoring-2-0/compare/service_monitoring-rs0.yml + local new_result=/tmp/tmp.G9Wq8CVNYh/service_monitoring-rs0.yml + [[ -n '' ]] + kubectl_bin get -o yaml service/monitoring-rs0 + yq eval $'\n\t\t\tdel(.metadata.ownerReferences[].apiVersion) |\n\t\t\tdel(.metadata.managedFields) |\n\t\t\tdel(.. | select(has("creationTimestamp")).creationTimestamp) |\n\t\t\tdel(.. | select(has("namespace")).namespace) |\n\t\t\tdel(.. | select(has("uid")).uid) |\n\t\t\tdel(.metadata.resourceVersion) |\n\t\t\tdel(.spec.template.spec.containers[].env[] | select(.name == "NAMESPACE")) |\n\t\t\tdel(.metadata.selfLink) |\n\t\t\tdel(.metadata.annotations."cloud.google.com/neg") |\n\t\t\tdel(.metadata.annotations."field.cattle.io/publicEndpoints") |\n\t\t\tdel(.metadata.annotations."metallb.io/ip-allocated-from-pool") |\n\t\t\tdel(.metadata.annotations."networking.gke.io/target-pool") |\n\t\t\tdel(.metadata.annotations."kubectl.kubernetes.io/last-applied-configuration") |\n\t\t\tdel(.. | select(has("image")).image) |\n\t\t\tdel(.. | select(has("clusterIP")).clusterIP) |\n\t\t\tdel(.. | select(has("clusterIPs")).clusterIPs) |\n\t\t\tdel(.. | select(has("dataSource")).dataSource) |\n\t\t\tdel(.. | select(has("procMount")).procMount) |\n\t\t\tdel(.. | select(has("storageClassName")).storageClassName) |\n\t\t\tdel(.. | select(has("finalizers")).finalizers) |\n\t\t\tdel(.. | select(has("kubernetes.io/pvc-protection"))."kubernetes.io/pvc-protection") |\n\t\t\tdel(.. | select(has("volumeName")).volumeName) |\n\t\t\tdel(.. | select(has("volume.beta.kubernetes.io/storage-provisioner"))."volume.beta.kubernetes.io/storage-provisioner") |\n\t\t\tdel(.. | select(has("volume.kubernetes.io/storage-provisioner"))."volume.kubernetes.io/storage-provisioner") |\n\t\t\tdel(.spec.volumeMode) |\n\t\t\tdel(.. | select(has("volume.kubernetes.io/selected-node"))."volume.kubernetes.io/selected-node") |\n\t\t\tdel(.. | select(has("percona.com/last-config-hash"))."percona.com/last-config-hash") |\n\t\t\tdel(.. | select(has("percona.com/configuration-hash"))."percona.com/configuration-hash") |\n\t\t\tdel(.. | select(has("percona.com/ssl-hash"))."percona.com/ssl-hash") |\n\t\t\tdel(.. | select(has("percona.com/ssl-internal-hash"))."percona.com/ssl-internal-hash") |\n\t\t\tdel(.spec.volumeClaimTemplates[].spec.volumeMode | select(. == "Filesystem")) |\n\t\t\tdel(.. | select(has("healthCheckNodePort")).healthCheckNodePort) |\n\t\t\tdel(.. | select(has("nodePort")).nodePort) |\n\t\t\tdel(.status) |\n\t\t\t(.. | select(tag == "!!str")) |= sub("monitoring-2-0-3915", "NAME_SPACE") |\n\t\t\tdel(.spec.volumeClaimTemplates[].apiVersion) |\n\t\t\tdel(.spec.volumeClaimTemplates[].kind) |\n\t\t\tdel(.spec.ipFamilies) |\n\t\t\tdel(.spec.ipFamilyPolicy) |\n\t\t\t(.. | select(. == "extensions/v1beta1")) = "apps/v1" |\n\t\t\t(.. | select(. == "batch/v1beta1")) = "batch/v1" ' - ++ mktemp + local LAST_OUT=/tmp/tmp.NMRvQ9qWIK ++ mktemp + local LAST_ERR=/tmp/tmp.7OWdb6pUtT + local exit_status=0 + local timeout=4 + args=('get' '-o' 'yaml' 'service/monitoring-rs0') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl get -o yaml service/monitoring-rs0 + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.NMRvQ9qWIK + cat /tmp/tmp.7OWdb6pUtT + rm /tmp/tmp.NMRvQ9qWIK /tmp/tmp.7OWdb6pUtT + return 0 + yq -i eval 'del(.spec.persistentVolumeClaimRetentionPolicy)' /tmp/tmp.G9Wq8CVNYh/service_monitoring-rs0.yml + arch_scheduling_enabled + [[ -n '' ]] + version_gt 1.22 ++ bc -l ++ echo '1.33 >= 1.22' + '[' 1 -eq 1 ']' + return 0 + yq -i eval 'del(.spec.internalTrafficPolicy)' /tmp/tmp.G9Wq8CVNYh/service_monitoring-rs0.yml + yq -i eval 'del(.spec.allocateLoadBalancerNodePorts)' /tmp/tmp.G9Wq8CVNYh/service_monitoring-rs0.yml + [[ /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2465/e2e-tests/monitoring-2-0/compare/service_monitoring-rs0.yml == */cronjob* ]] + '[' -n '' ']' + [[ 0 -eq 0 ]] + diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2465/e2e-tests/monitoring-2-0/compare/service_monitoring-rs0.yml /tmp/tmp.G9Wq8CVNYh/service_monitoring-rs0.yml + log 'compare_kubectl: service/monitoring-rs0 OK' + set +o xtrace [2026-07-22T18:42:55+0000] compare_kubectl: service/monitoring-rs0 OK + compare_kubectl service/monitoring-mongos + local resource=service/monitoring-mongos + local postfix= + local skip_generation_check= + local expected_result=/mnt/jenkins/workspace/cloud-psmdb-operator_PR-2465/e2e-tests/monitoring-2-0/compare/service_monitoring-mongos.yml + local new_result=/tmp/tmp.G9Wq8CVNYh/service_monitoring-mongos.yml + [[ -n '' ]] + kubectl_bin get -o yaml service/monitoring-mongos + yq eval $'\n\t\t\tdel(.metadata.ownerReferences[].apiVersion) |\n\t\t\tdel(.metadata.managedFields) |\n\t\t\tdel(.. | select(has("creationTimestamp")).creationTimestamp) |\n\t\t\tdel(.. | select(has("namespace")).namespace) |\n\t\t\tdel(.. | select(has("uid")).uid) |\n\t\t\tdel(.metadata.resourceVersion) |\n\t\t\tdel(.spec.template.spec.containers[].env[] | select(.name == "NAMESPACE")) |\n\t\t\tdel(.metadata.selfLink) |\n\t\t\tdel(.metadata.annotations."cloud.google.com/neg") |\n\t\t\tdel(.metadata.annotations."field.cattle.io/publicEndpoints") |\n\t\t\tdel(.metadata.annotations."metallb.io/ip-allocated-from-pool") |\n\t\t\tdel(.metadata.annotations."networking.gke.io/target-pool") |\n\t\t\tdel(.metadata.annotations."kubectl.kubernetes.io/last-applied-configuration") |\n\t\t\tdel(.. | select(has("image")).image) |\n\t\t\tdel(.. | select(has("clusterIP")).clusterIP) |\n\t\t\tdel(.. | select(has("clusterIPs")).clusterIPs) |\n\t\t\tdel(.. | select(has("dataSource")).dataSource) |\n\t\t\tdel(.. | select(has("procMount")).procMount) |\n\t\t\tdel(.. | select(has("storageClassName")).storageClassName) |\n\t\t\tdel(.. | select(has("finalizers")).finalizers) |\n\t\t\tdel(.. | select(has("kubernetes.io/pvc-protection"))."kubernetes.io/pvc-protection") |\n\t\t\tdel(.. | select(has("volumeName")).volumeName) |\n\t\t\tdel(.. | select(has("volume.beta.kubernetes.io/storage-provisioner"))."volume.beta.kubernetes.io/storage-provisioner") |\n\t\t\tdel(.. | select(has("volume.kubernetes.io/storage-provisioner"))."volume.kubernetes.io/storage-provisioner") |\n\t\t\tdel(.spec.volumeMode) |\n\t\t\tdel(.. | select(has("volume.kubernetes.io/selected-node"))."volume.kubernetes.io/selected-node") |\n\t\t\tdel(.. | select(has("percona.com/last-config-hash"))."percona.com/last-config-hash") |\n\t\t\tdel(.. | select(has("percona.com/configuration-hash"))."percona.com/configuration-hash") |\n\t\t\tdel(.. | select(has("percona.com/ssl-hash"))."percona.com/ssl-hash") |\n\t\t\tdel(.. | select(has("percona.com/ssl-internal-hash"))."percona.com/ssl-internal-hash") |\n\t\t\tdel(.spec.volumeClaimTemplates[].spec.volumeMode | select(. == "Filesystem")) |\n\t\t\tdel(.. | select(has("healthCheckNodePort")).healthCheckNodePort) |\n\t\t\tdel(.. | select(has("nodePort")).nodePort) |\n\t\t\tdel(.status) |\n\t\t\t(.. | select(tag == "!!str")) |= sub("monitoring-2-0-3915", "NAME_SPACE") |\n\t\t\tdel(.spec.volumeClaimTemplates[].apiVersion) |\n\t\t\tdel(.spec.volumeClaimTemplates[].kind) |\n\t\t\tdel(.spec.ipFamilies) |\n\t\t\tdel(.spec.ipFamilyPolicy) |\n\t\t\t(.. | select(. == "extensions/v1beta1")) = "apps/v1" |\n\t\t\t(.. | select(. == "batch/v1beta1")) = "batch/v1" ' - ++ mktemp + local LAST_OUT=/tmp/tmp.SXoYaBAdPD ++ mktemp + local LAST_ERR=/tmp/tmp.xYE7nNtz5w + local exit_status=0 + local timeout=4 + args=('get' '-o' 'yaml' 'service/monitoring-mongos') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl get -o yaml service/monitoring-mongos + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.SXoYaBAdPD + cat /tmp/tmp.xYE7nNtz5w + rm /tmp/tmp.SXoYaBAdPD /tmp/tmp.xYE7nNtz5w + return 0 + yq -i eval 'del(.spec.persistentVolumeClaimRetentionPolicy)' /tmp/tmp.G9Wq8CVNYh/service_monitoring-mongos.yml + arch_scheduling_enabled + [[ -n '' ]] + version_gt 1.22 ++ echo '1.33 >= 1.22' ++ bc -l + '[' 1 -eq 1 ']' + return 0 + yq -i eval 'del(.spec.internalTrafficPolicy)' /tmp/tmp.G9Wq8CVNYh/service_monitoring-mongos.yml + yq -i eval 'del(.spec.allocateLoadBalancerNodePorts)' /tmp/tmp.G9Wq8CVNYh/service_monitoring-mongos.yml + [[ /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2465/e2e-tests/monitoring-2-0/compare/service_monitoring-mongos.yml == */cronjob* ]] + '[' -n '' ']' + [[ 0 -eq 0 ]] + diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2465/e2e-tests/monitoring-2-0/compare/service_monitoring-mongos.yml /tmp/tmp.G9Wq8CVNYh/service_monitoring-mongos.yml + log 'compare_kubectl: service/monitoring-mongos OK' + set +o xtrace [2026-07-22T18:42:56+0000] compare_kubectl: service/monitoring-mongos OK + compare_kubectl statefulset/monitoring-cfg + local resource=statefulset/monitoring-cfg + local postfix= + local skip_generation_check= + local expected_result=/mnt/jenkins/workspace/cloud-psmdb-operator_PR-2465/e2e-tests/monitoring-2-0/compare/statefulset_monitoring-cfg.yml + local new_result=/tmp/tmp.G9Wq8CVNYh/statefulset_monitoring-cfg.yml + [[ -n '' ]] + kubectl_bin get -o yaml statefulset/monitoring-cfg + yq eval $'\n\t\t\tdel(.metadata.ownerReferences[].apiVersion) |\n\t\t\tdel(.metadata.managedFields) |\n\t\t\tdel(.. | select(has("creationTimestamp")).creationTimestamp) |\n\t\t\tdel(.. | select(has("namespace")).namespace) |\n\t\t\tdel(.. | select(has("uid")).uid) |\n\t\t\tdel(.metadata.resourceVersion) |\n\t\t\tdel(.spec.template.spec.containers[].env[] | select(.name == "NAMESPACE")) |\n\t\t\tdel(.metadata.selfLink) |\n\t\t\tdel(.metadata.annotations."cloud.google.com/neg") |\n\t\t\tdel(.metadata.annotations."field.cattle.io/publicEndpoints") |\n\t\t\tdel(.metadata.annotations."metallb.io/ip-allocated-from-pool") |\n\t\t\tdel(.metadata.annotations."networking.gke.io/target-pool") |\n\t\t\tdel(.metadata.annotations."kubectl.kubernetes.io/last-applied-configuration") |\n\t\t\tdel(.. | select(has("image")).image) |\n\t\t\tdel(.. | select(has("clusterIP")).clusterIP) |\n\t\t\tdel(.. | select(has("clusterIPs")).clusterIPs) |\n\t\t\tdel(.. | select(has("dataSource")).dataSource) |\n\t\t\tdel(.. | select(has("procMount")).procMount) |\n\t\t\tdel(.. | select(has("storageClassName")).storageClassName) |\n\t\t\tdel(.. | select(has("finalizers")).finalizers) |\n\t\t\tdel(.. | select(has("kubernetes.io/pvc-protection"))."kubernetes.io/pvc-protection") |\n\t\t\tdel(.. | select(has("volumeName")).volumeName) |\n\t\t\tdel(.. | select(has("volume.beta.kubernetes.io/storage-provisioner"))."volume.beta.kubernetes.io/storage-provisioner") |\n\t\t\tdel(.. | select(has("volume.kubernetes.io/storage-provisioner"))."volume.kubernetes.io/storage-provisioner") |\n\t\t\tdel(.spec.volumeMode) |\n\t\t\tdel(.. | select(has("volume.kubernetes.io/selected-node"))."volume.kubernetes.io/selected-node") |\n\t\t\tdel(.. | select(has("percona.com/last-config-hash"))."percona.com/last-config-hash") |\n\t\t\tdel(.. | select(has("percona.com/configuration-hash"))."percona.com/configuration-hash") |\n\t\t\tdel(.. | select(has("percona.com/ssl-hash"))."percona.com/ssl-hash") |\n\t\t\tdel(.. | select(has("percona.com/ssl-internal-hash"))."percona.com/ssl-internal-hash") |\n\t\t\tdel(.spec.volumeClaimTemplates[].spec.volumeMode | select(. == "Filesystem")) |\n\t\t\tdel(.. | select(has("healthCheckNodePort")).healthCheckNodePort) |\n\t\t\tdel(.. | select(has("nodePort")).nodePort) |\n\t\t\tdel(.status) |\n\t\t\t(.. | select(tag == "!!str")) |= sub("monitoring-2-0-3915", "NAME_SPACE") |\n\t\t\tdel(.spec.volumeClaimTemplates[].apiVersion) |\n\t\t\tdel(.spec.volumeClaimTemplates[].kind) |\n\t\t\tdel(.spec.ipFamilies) |\n\t\t\tdel(.spec.ipFamilyPolicy) |\n\t\t\t(.. | select(. == "extensions/v1beta1")) = "apps/v1" |\n\t\t\t(.. | select(. == "batch/v1beta1")) = "batch/v1" ' - ++ mktemp + local LAST_OUT=/tmp/tmp.5hJmKjmA1b ++ mktemp + local LAST_ERR=/tmp/tmp.Z13RnjX7bD + local exit_status=0 + local timeout=4 + args=('get' '-o' 'yaml' 'statefulset/monitoring-cfg') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl get -o yaml statefulset/monitoring-cfg + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.5hJmKjmA1b + cat /tmp/tmp.Z13RnjX7bD + rm /tmp/tmp.5hJmKjmA1b /tmp/tmp.Z13RnjX7bD + return 0 + yq -i eval 'del(.spec.persistentVolumeClaimRetentionPolicy)' /tmp/tmp.G9Wq8CVNYh/statefulset_monitoring-cfg.yml + arch_scheduling_enabled + [[ -n '' ]] + version_gt 1.22 ++ echo '1.33 >= 1.22' ++ bc -l + '[' 1 -eq 1 ']' + return 0 + yq -i eval 'del(.spec.internalTrafficPolicy)' /tmp/tmp.G9Wq8CVNYh/statefulset_monitoring-cfg.yml + yq -i eval 'del(.spec.allocateLoadBalancerNodePorts)' /tmp/tmp.G9Wq8CVNYh/statefulset_monitoring-cfg.yml + [[ /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2465/e2e-tests/monitoring-2-0/compare/statefulset_monitoring-cfg.yml == */cronjob* ]] + '[' -n '' ']' + [[ 0 -eq 0 ]] + diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2465/e2e-tests/monitoring-2-0/compare/statefulset_monitoring-cfg.yml /tmp/tmp.G9Wq8CVNYh/statefulset_monitoring-cfg.yml + log 'compare_kubectl: statefulset/monitoring-cfg OK' + set +o xtrace [2026-07-22T18:42:57+0000] compare_kubectl: statefulset/monitoring-cfg OK + compare_kubectl statefulset/monitoring-mongos + local resource=statefulset/monitoring-mongos + local postfix= + local skip_generation_check= + local expected_result=/mnt/jenkins/workspace/cloud-psmdb-operator_PR-2465/e2e-tests/monitoring-2-0/compare/statefulset_monitoring-mongos.yml + local new_result=/tmp/tmp.G9Wq8CVNYh/statefulset_monitoring-mongos.yml + [[ -n '' ]] + kubectl_bin get -o yaml statefulset/monitoring-mongos + yq eval $'\n\t\t\tdel(.metadata.ownerReferences[].apiVersion) |\n\t\t\tdel(.metadata.managedFields) |\n\t\t\tdel(.. | select(has("creationTimestamp")).creationTimestamp) |\n\t\t\tdel(.. | select(has("namespace")).namespace) |\n\t\t\tdel(.. | select(has("uid")).uid) |\n\t\t\tdel(.metadata.resourceVersion) |\n\t\t\tdel(.spec.template.spec.containers[].env[] | select(.name == "NAMESPACE")) |\n\t\t\tdel(.metadata.selfLink) |\n\t\t\tdel(.metadata.annotations."cloud.google.com/neg") |\n\t\t\tdel(.metadata.annotations."field.cattle.io/publicEndpoints") |\n\t\t\tdel(.metadata.annotations."metallb.io/ip-allocated-from-pool") |\n\t\t\tdel(.metadata.annotations."networking.gke.io/target-pool") |\n\t\t\tdel(.metadata.annotations."kubectl.kubernetes.io/last-applied-configuration") |\n\t\t\tdel(.. | select(has("image")).image) |\n\t\t\tdel(.. | select(has("clusterIP")).clusterIP) |\n\t\t\tdel(.. | select(has("clusterIPs")).clusterIPs) |\n\t\t\tdel(.. | select(has("dataSource")).dataSource) |\n\t\t\tdel(.. | select(has("procMount")).procMount) |\n\t\t\tdel(.. | select(has("storageClassName")).storageClassName) |\n\t\t\tdel(.. | select(has("finalizers")).finalizers) |\n\t\t\tdel(.. | select(has("kubernetes.io/pvc-protection"))."kubernetes.io/pvc-protection") |\n\t\t\tdel(.. | select(has("volumeName")).volumeName) |\n\t\t\tdel(.. | select(has("volume.beta.kubernetes.io/storage-provisioner"))."volume.beta.kubernetes.io/storage-provisioner") |\n\t\t\tdel(.. | select(has("volume.kubernetes.io/storage-provisioner"))."volume.kubernetes.io/storage-provisioner") |\n\t\t\tdel(.spec.volumeMode) |\n\t\t\tdel(.. | select(has("volume.kubernetes.io/selected-node"))."volume.kubernetes.io/selected-node") |\n\t\t\tdel(.. | select(has("percona.com/last-config-hash"))."percona.com/last-config-hash") |\n\t\t\tdel(.. | select(has("percona.com/configuration-hash"))."percona.com/configuration-hash") |\n\t\t\tdel(.. | select(has("percona.com/ssl-hash"))."percona.com/ssl-hash") |\n\t\t\tdel(.. | select(has("percona.com/ssl-internal-hash"))."percona.com/ssl-internal-hash") |\n\t\t\tdel(.spec.volumeClaimTemplates[].spec.volumeMode | select(. == "Filesystem")) |\n\t\t\tdel(.. | select(has("healthCheckNodePort")).healthCheckNodePort) |\n\t\t\tdel(.. | select(has("nodePort")).nodePort) |\n\t\t\tdel(.status) |\n\t\t\t(.. | select(tag == "!!str")) |= sub("monitoring-2-0-3915", "NAME_SPACE") |\n\t\t\tdel(.spec.volumeClaimTemplates[].apiVersion) |\n\t\t\tdel(.spec.volumeClaimTemplates[].kind) |\n\t\t\tdel(.spec.ipFamilies) |\n\t\t\tdel(.spec.ipFamilyPolicy) |\n\t\t\t(.. | select(. == "extensions/v1beta1")) = "apps/v1" |\n\t\t\t(.. | select(. == "batch/v1beta1")) = "batch/v1" ' - ++ mktemp + local LAST_OUT=/tmp/tmp.DNAmylGNBO ++ mktemp + local LAST_ERR=/tmp/tmp.0FOoeqyiuZ + local exit_status=0 + local timeout=4 + args=('get' '-o' 'yaml' 'statefulset/monitoring-mongos') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl get -o yaml statefulset/monitoring-mongos + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.DNAmylGNBO + cat /tmp/tmp.0FOoeqyiuZ + rm /tmp/tmp.DNAmylGNBO /tmp/tmp.0FOoeqyiuZ + return 0 + yq -i eval 'del(.spec.persistentVolumeClaimRetentionPolicy)' /tmp/tmp.G9Wq8CVNYh/statefulset_monitoring-mongos.yml + arch_scheduling_enabled + [[ -n '' ]] + version_gt 1.22 ++ echo '1.33 >= 1.22' ++ bc -l + '[' 1 -eq 1 ']' + return 0 + yq -i eval 'del(.spec.internalTrafficPolicy)' /tmp/tmp.G9Wq8CVNYh/statefulset_monitoring-mongos.yml + yq -i eval 'del(.spec.allocateLoadBalancerNodePorts)' /tmp/tmp.G9Wq8CVNYh/statefulset_monitoring-mongos.yml + [[ /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2465/e2e-tests/monitoring-2-0/compare/statefulset_monitoring-mongos.yml == */cronjob* ]] + '[' -n '' ']' + [[ 0 -eq 0 ]] + diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2465/e2e-tests/monitoring-2-0/compare/statefulset_monitoring-mongos.yml /tmp/tmp.G9Wq8CVNYh/statefulset_monitoring-mongos.yml + log 'compare_kubectl: statefulset/monitoring-mongos OK' + set +o xtrace [2026-07-22T18:42:58+0000] compare_kubectl: statefulset/monitoring-mongos OK + desc 'check mongod metrics' + set +o xtrace ----------------------------------------------------------------------------------- check mongod metrics ----------------------------------------------------------------------------------- + get_metric_values node_boot_time_seconds monitoring-2-0-3915-monitoring-rs0-1 admin:admin + local metric=node_boot_time_seconds + local instance=monitoring-2-0-3915-monitoring-rs0-1 + local user_pass=admin:admin + local auth_type=basic + local endpoint ++ get_service_endpoint monitoring-service ++ local service=monitoring-service ++ local endpoint ++ local service_type +++ kubectl_bin get service/monitoring-service -o 'jsonpath={.spec.type}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.4WSi5bQwNc ++++ mktemp +++ local LAST_ERR=/tmp/tmp.PD28m9vBHT +++ local exit_status=0 +++ local timeout=4 +++ args=('get' 'service/monitoring-service' '-o' 'jsonpath={.spec.type}') +++ local args +++ transformed_files=() +++ local transformed_files +++ arch_scheduling_enabled +++ [[ -n '' ]] +++ arch_scheduling_enabled +++ [[ -n '' ]] ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl get service/monitoring-service -o 'jsonpath={.spec.type}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 -a -n 1 ']' +++ break +++ cat /tmp/tmp.4WSi5bQwNc +++ cat /tmp/tmp.PD28m9vBHT +++ rm /tmp/tmp.4WSi5bQwNc /tmp/tmp.PD28m9vBHT +++ return 0 ++ service_type=LoadBalancer ++ [[ LoadBalancer == LoadBalancer ]] +++ get_load_balancer_endpoint monitoring-service +++ local service=monitoring-service +++ local endpoint ++++ kubectl_bin get service/monitoring-service -o json ++++ jq -r '[.status.loadBalancer.ingress[]?.hostname, .status.loadBalancer.ingress[]?.ip] | map(select(. != null and . != "")) | first // empty' +++++ mktemp ++++ local LAST_OUT=/tmp/tmp.huhmsioF1p +++++ mktemp ++++ local LAST_ERR=/tmp/tmp.TUMRggDUmI ++++ local exit_status=0 ++++ local timeout=4 ++++ args=('get' 'service/monitoring-service' '-o' 'json') ++++ local args ++++ transformed_files=() ++++ local transformed_files ++++ arch_scheduling_enabled ++++ [[ -n '' ]] ++++ arch_scheduling_enabled ++++ [[ -n '' ]] +++++ seq 0 2 ++++ for i in $(seq 0 2) ++++ set +e ++++ kubectl get service/monitoring-service -o json ++++ exit_status=0 ++++ set -e ++++ '[' 0 '!=' 0 -a -n 1 ']' ++++ break ++++ cat /tmp/tmp.huhmsioF1p ++++ cat /tmp/tmp.TUMRggDUmI ++++ rm /tmp/tmp.huhmsioF1p /tmp/tmp.TUMRggDUmI ++++ return 0 +++ endpoint=34.28.217.84 +++ '[' -n 34.28.217.84 -a 34.28.217.84 '!=' null ']' +++ echo 34.28.217.84 +++ return ++ endpoint=34.28.217.84 ++ echo 34.28.217.84 ++ return + endpoint=34.28.217.84 + retry 30 2 curl_metric_values node_boot_time_seconds monitoring-2-0-3915-monitoring-rs0-1 34.28.217.84 admin:admin basic + local max=30 + local delay=2 + shift 2 + local n=1 + curl_metric_values node_boot_time_seconds monitoring-2-0-3915-monitoring-rs0-1 34.28.217.84 admin:admin basic + local metric=node_boot_time_seconds + local instance=monitoring-2-0-3915-monitoring-rs0-1 + local endpoint=34.28.217.84 + local credentials=admin:admin + local auth_type=basic + local start + local end + local url + curl_args=() + local curl_args ++ /usr/sbin/date -u +%s -d '-1 minute' + start=1784745720 ++ /usr/sbin/date -u +%s + end=1784745780 + url='https://34.28.217.84/graph/api/datasources/proxy/1/api/v1/query_range?query=min%28node_boot_time_seconds%7Bnode_name%3D%7E%22monitoring-2-0-3915-monitoring-rs0-1%22%7D%20or%20node_boot_time_seconds%7Bnode_name%3D%7E%22monitoring-2-0-3915-monitoring-rs0-1%22%7D%29&start=1784745720&end=1784745780&step=60' + [[ basic == bearer ]] ++ urlencode_userinfo admin:admin ++ local user_pass=admin:admin ++ local username=admin ++ local password=admin +++ urlencode admin +++ uri=admin +++ jq -s -R -r @uri +++ echo -n admin +++ urlencode admin +++ uri=admin +++ echo -n admin +++ jq -s -R -r @uri ++ printf %s:%s admin admin + credentials=admin:admin + curl_args=("https://${credentials}@${url#https://}") + jq -r '(.data.result[0].values // [])[] | .[1] // empty' + execute_curl -s -k 'https://admin:admin@34.28.217.84/graph/api/datasources/proxy/1/api/v1/query_range?query=min%28node_boot_time_seconds%7Bnode_name%3D%7E%22monitoring-2-0-3915-monitoring-rs0-1%22%7D%20or%20node_boot_time_seconds%7Bnode_name%3D%7E%22monitoring-2-0-3915-monitoring-rs0-1%22%7D%29&start=1784745720&end=1784745780&step=60' + local name=curl-18786 + local image=docker.io/alpine/curl:latest + local phase= + local exit_status=0 + kubectl_bin run curl-18786 -n monitoring-2-0-3915 --restart=Never --image=docker.io/alpine/curl:latest --command -- curl -sS -s -k 'https://admin:admin@34.28.217.84/graph/api/datasources/proxy/1/api/v1/query_range?query=min%28node_boot_time_seconds%7Bnode_name%3D%7E%22monitoring-2-0-3915-monitoring-rs0-1%22%7D%20or%20node_boot_time_seconds%7Bnode_name%3D%7E%22monitoring-2-0-3915-monitoring-rs0-1%22%7D%29&start=1784745720&end=1784745780&step=60' + grep '^[0-9]' ++ mktemp + local LAST_OUT=/tmp/tmp.ALGCuOa1Nn ++ mktemp + local LAST_ERR=/tmp/tmp.52goK6GMZK + local exit_status=0 + local timeout=4 + args=('run' 'curl-18786' '-n' 'monitoring-2-0-3915' '--restart=Never' '--image=docker.io/alpine/curl:latest' '--command' '--' 'curl' '-sS' '-s' '-k' 'https://admin:admin@34.28.217.84/graph/api/datasources/proxy/1/api/v1/query_range?query=min%28node_boot_time_seconds%7Bnode_name%3D%7E%22monitoring-2-0-3915-monitoring-rs0-1%22%7D%20or%20node_boot_time_seconds%7Bnode_name%3D%7E%22monitoring-2-0-3915-monitoring-rs0-1%22%7D%29&start=1784745720&end=1784745780&step=60') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl run curl-18786 -n monitoring-2-0-3915 --restart=Never --image=docker.io/alpine/curl:latest --command -- curl -sS -s -k 'https://admin:admin@34.28.217.84/graph/api/datasources/proxy/1/api/v1/query_range?query=min%28node_boot_time_seconds%7Bnode_name%3D%7E%22monitoring-2-0-3915-monitoring-rs0-1%22%7D%20or%20node_boot_time_seconds%7Bnode_name%3D%7E%22monitoring-2-0-3915-monitoring-rs0-1%22%7D%29&start=1784745720&end=1784745780&step=60' + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.ALGCuOa1Nn + cat /tmp/tmp.52goK6GMZK + rm /tmp/tmp.ALGCuOa1Nn /tmp/tmp.52goK6GMZK + return 0 ++ seq 1 60 + for _ in $(seq 1 60) ++ kubectl_bin get pod curl-18786 -n monitoring-2-0-3915 -o 'jsonpath={.status.phase}' + phase=Pending + [[ Pending == Succeeded ]] + [[ Pending == Failed ]] + sleep 1 + for _ in $(seq 1 60) ++ kubectl_bin get pod curl-18786 -n monitoring-2-0-3915 -o 'jsonpath={.status.phase}' + phase=Succeeded + [[ Succeeded == Succeeded ]] + break + kubectl_bin logs curl-18786 -n monitoring-2-0-3915 ++ mktemp + local LAST_OUT=/tmp/tmp.yhb944Jr6o ++ mktemp + local LAST_ERR=/tmp/tmp.R4ZRT4eqXo + local exit_status=0 + local timeout=4 + args=('logs' 'curl-18786' '-n' 'monitoring-2-0-3915') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl logs curl-18786 -n monitoring-2-0-3915 + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.yhb944Jr6o + cat /tmp/tmp.R4ZRT4eqXo + rm /tmp/tmp.yhb944Jr6o /tmp/tmp.R4ZRT4eqXo + return 0 + kubectl_bin delete pod curl-18786 -n monitoring-2-0-3915 --ignore-not-found --wait=false + [[ Succeeded != Succeeded ]] + return 0 1784743984 1784743984 + get_metric_values mongodb_connections monitoring-2-0-3915-monitoring-rs0-1 admin:admin + local metric=mongodb_connections + local instance=monitoring-2-0-3915-monitoring-rs0-1 + local user_pass=admin:admin + local auth_type=basic + local endpoint ++ get_service_endpoint monitoring-service ++ local service=monitoring-service ++ local endpoint ++ local service_type +++ kubectl_bin get service/monitoring-service -o 'jsonpath={.spec.type}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.wqekkXcPb3 ++++ mktemp +++ local LAST_ERR=/tmp/tmp.a5NVtF17xA +++ local exit_status=0 +++ local timeout=4 +++ args=('get' 'service/monitoring-service' '-o' 'jsonpath={.spec.type}') +++ local args +++ transformed_files=() +++ local transformed_files +++ arch_scheduling_enabled +++ [[ -n '' ]] +++ arch_scheduling_enabled +++ [[ -n '' ]] ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl get service/monitoring-service -o 'jsonpath={.spec.type}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 -a -n 1 ']' +++ break +++ cat /tmp/tmp.wqekkXcPb3 +++ cat /tmp/tmp.a5NVtF17xA +++ rm /tmp/tmp.wqekkXcPb3 /tmp/tmp.a5NVtF17xA +++ return 0 ++ service_type=LoadBalancer ++ [[ LoadBalancer == LoadBalancer ]] +++ get_load_balancer_endpoint monitoring-service +++ local service=monitoring-service +++ local endpoint ++++ kubectl_bin get service/monitoring-service -o json ++++ jq -r '[.status.loadBalancer.ingress[]?.hostname, .status.loadBalancer.ingress[]?.ip] | map(select(. != null and . != "")) | first // empty' +++++ mktemp ++++ local LAST_OUT=/tmp/tmp.NOHsgACyoJ +++++ mktemp ++++ local LAST_ERR=/tmp/tmp.jw9PZihXWQ ++++ local exit_status=0 ++++ local timeout=4 ++++ args=('get' 'service/monitoring-service' '-o' 'json') ++++ local args ++++ transformed_files=() ++++ local transformed_files ++++ arch_scheduling_enabled ++++ [[ -n '' ]] ++++ arch_scheduling_enabled ++++ [[ -n '' ]] +++++ seq 0 2 ++++ for i in $(seq 0 2) ++++ set +e ++++ kubectl get service/monitoring-service -o json ++++ exit_status=0 ++++ set -e ++++ '[' 0 '!=' 0 -a -n 1 ']' ++++ break ++++ cat /tmp/tmp.NOHsgACyoJ ++++ cat /tmp/tmp.jw9PZihXWQ ++++ rm /tmp/tmp.NOHsgACyoJ /tmp/tmp.jw9PZihXWQ ++++ return 0 +++ endpoint=34.28.217.84 +++ '[' -n 34.28.217.84 -a 34.28.217.84 '!=' null ']' +++ echo 34.28.217.84 +++ return ++ endpoint=34.28.217.84 ++ echo 34.28.217.84 ++ return + endpoint=34.28.217.84 + retry 30 2 curl_metric_values mongodb_connections monitoring-2-0-3915-monitoring-rs0-1 34.28.217.84 admin:admin basic + local max=30 + local delay=2 + shift 2 + local n=1 + curl_metric_values mongodb_connections monitoring-2-0-3915-monitoring-rs0-1 34.28.217.84 admin:admin basic + local metric=mongodb_connections + local instance=monitoring-2-0-3915-monitoring-rs0-1 + local endpoint=34.28.217.84 + local credentials=admin:admin + local auth_type=basic + local start + local end + local url + curl_args=() + local curl_args ++ /usr/sbin/date -u +%s -d '-1 minute' + start=1784745729 ++ /usr/sbin/date -u +%s + end=1784745789 + url='https://34.28.217.84/graph/api/datasources/proxy/1/api/v1/query_range?query=min%28mongodb_connections%7Bnode_name%3D%7E%22monitoring-2-0-3915-monitoring-rs0-1%22%7D%20or%20mongodb_connections%7Bnode_name%3D%7E%22monitoring-2-0-3915-monitoring-rs0-1%22%7D%29&start=1784745729&end=1784745789&step=60' + [[ basic == bearer ]] ++ urlencode_userinfo admin:admin ++ local user_pass=admin:admin ++ local username=admin ++ local password=admin +++ urlencode admin +++ uri=admin +++ echo -n admin +++ jq -s -R -r @uri +++ urlencode admin +++ uri=admin +++ echo -n admin +++ jq -s -R -r @uri ++ printf %s:%s admin admin + credentials=admin:admin + curl_args=("https://${credentials}@${url#https://}") + execute_curl -s -k 'https://admin:admin@34.28.217.84/graph/api/datasources/proxy/1/api/v1/query_range?query=min%28mongodb_connections%7Bnode_name%3D%7E%22monitoring-2-0-3915-monitoring-rs0-1%22%7D%20or%20mongodb_connections%7Bnode_name%3D%7E%22monitoring-2-0-3915-monitoring-rs0-1%22%7D%29&start=1784745729&end=1784745789&step=60' + local name=curl-30890 + local image=docker.io/alpine/curl:latest + local phase= + local exit_status=0 + jq -r '(.data.result[0].values // [])[] | .[1] // empty' + kubectl_bin run curl-30890 -n monitoring-2-0-3915 --restart=Never --image=docker.io/alpine/curl:latest --command -- curl -sS -s -k 'https://admin:admin@34.28.217.84/graph/api/datasources/proxy/1/api/v1/query_range?query=min%28mongodb_connections%7Bnode_name%3D%7E%22monitoring-2-0-3915-monitoring-rs0-1%22%7D%20or%20mongodb_connections%7Bnode_name%3D%7E%22monitoring-2-0-3915-monitoring-rs0-1%22%7D%29&start=1784745729&end=1784745789&step=60' + grep '^[0-9]' ++ mktemp + local LAST_OUT=/tmp/tmp.hE2KmjTJYP ++ mktemp + local LAST_ERR=/tmp/tmp.SAYYCPhsA2 + local exit_status=0 + local timeout=4 + args=('run' 'curl-30890' '-n' 'monitoring-2-0-3915' '--restart=Never' '--image=docker.io/alpine/curl:latest' '--command' '--' 'curl' '-sS' '-s' '-k' 'https://admin:admin@34.28.217.84/graph/api/datasources/proxy/1/api/v1/query_range?query=min%28mongodb_connections%7Bnode_name%3D%7E%22monitoring-2-0-3915-monitoring-rs0-1%22%7D%20or%20mongodb_connections%7Bnode_name%3D%7E%22monitoring-2-0-3915-monitoring-rs0-1%22%7D%29&start=1784745729&end=1784745789&step=60') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl run curl-30890 -n monitoring-2-0-3915 --restart=Never --image=docker.io/alpine/curl:latest --command -- curl -sS -s -k 'https://admin:admin@34.28.217.84/graph/api/datasources/proxy/1/api/v1/query_range?query=min%28mongodb_connections%7Bnode_name%3D%7E%22monitoring-2-0-3915-monitoring-rs0-1%22%7D%20or%20mongodb_connections%7Bnode_name%3D%7E%22monitoring-2-0-3915-monitoring-rs0-1%22%7D%29&start=1784745729&end=1784745789&step=60' + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.hE2KmjTJYP + cat /tmp/tmp.SAYYCPhsA2 + rm /tmp/tmp.hE2KmjTJYP /tmp/tmp.SAYYCPhsA2 + return 0 ++ seq 1 60 + for _ in $(seq 1 60) ++ kubectl_bin get pod curl-30890 -n monitoring-2-0-3915 -o 'jsonpath={.status.phase}' + phase=Pending + [[ Pending == Succeeded ]] + [[ Pending == Failed ]] + sleep 1 + for _ in $(seq 1 60) ++ kubectl_bin get pod curl-30890 -n monitoring-2-0-3915 -o 'jsonpath={.status.phase}' + phase=Succeeded + [[ Succeeded == Succeeded ]] + break + kubectl_bin logs curl-30890 -n monitoring-2-0-3915 ++ mktemp + local LAST_OUT=/tmp/tmp.F2S4oL3LoN ++ mktemp + local LAST_ERR=/tmp/tmp.MQ2b8ZBA2I + local exit_status=0 + local timeout=4 + args=('logs' 'curl-30890' '-n' 'monitoring-2-0-3915') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl logs curl-30890 -n monitoring-2-0-3915 + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.F2S4oL3LoN + cat /tmp/tmp.MQ2b8ZBA2I + rm /tmp/tmp.F2S4oL3LoN /tmp/tmp.MQ2b8ZBA2I + return 0 + kubectl_bin delete pod curl-30890 -n monitoring-2-0-3915 --ignore-not-found --wait=false + [[ Succeeded != Succeeded ]] + return 0 0 0 + desc 'check mongo config metrics' + set +o xtrace ----------------------------------------------------------------------------------- check mongo config metrics ----------------------------------------------------------------------------------- + get_metric_values node_boot_time_seconds monitoring-2-0-3915-monitoring-cfg-1 admin:admin + local metric=node_boot_time_seconds + local instance=monitoring-2-0-3915-monitoring-cfg-1 + local user_pass=admin:admin + local auth_type=basic + local endpoint ++ get_service_endpoint monitoring-service ++ local service=monitoring-service ++ local endpoint ++ local service_type +++ kubectl_bin get service/monitoring-service -o 'jsonpath={.spec.type}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.d0IWNd6xfC ++++ mktemp +++ local LAST_ERR=/tmp/tmp.M6MC9PCU7T +++ local exit_status=0 +++ local timeout=4 +++ args=('get' 'service/monitoring-service' '-o' 'jsonpath={.spec.type}') +++ local args +++ transformed_files=() +++ local transformed_files +++ arch_scheduling_enabled +++ [[ -n '' ]] +++ arch_scheduling_enabled +++ [[ -n '' ]] ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl get service/monitoring-service -o 'jsonpath={.spec.type}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 -a -n 1 ']' +++ break +++ cat /tmp/tmp.d0IWNd6xfC +++ cat /tmp/tmp.M6MC9PCU7T +++ rm /tmp/tmp.d0IWNd6xfC /tmp/tmp.M6MC9PCU7T +++ return 0 ++ service_type=LoadBalancer ++ [[ LoadBalancer == LoadBalancer ]] +++ get_load_balancer_endpoint monitoring-service +++ local service=monitoring-service +++ local endpoint ++++ jq -r '[.status.loadBalancer.ingress[]?.hostname, .status.loadBalancer.ingress[]?.ip] | map(select(. != null and . != "")) | first // empty' ++++ kubectl_bin get service/monitoring-service -o json +++++ mktemp ++++ local LAST_OUT=/tmp/tmp.rrUz0TH9xH +++++ mktemp ++++ local LAST_ERR=/tmp/tmp.MY0zZX3wKX ++++ local exit_status=0 ++++ local timeout=4 ++++ args=('get' 'service/monitoring-service' '-o' 'json') ++++ local args ++++ transformed_files=() ++++ local transformed_files ++++ arch_scheduling_enabled ++++ [[ -n '' ]] ++++ arch_scheduling_enabled ++++ [[ -n '' ]] +++++ seq 0 2 ++++ for i in $(seq 0 2) ++++ set +e ++++ kubectl get service/monitoring-service -o json ++++ exit_status=0 ++++ set -e ++++ '[' 0 '!=' 0 -a -n 1 ']' ++++ break ++++ cat /tmp/tmp.rrUz0TH9xH ++++ cat /tmp/tmp.MY0zZX3wKX ++++ rm /tmp/tmp.rrUz0TH9xH /tmp/tmp.MY0zZX3wKX ++++ return 0 +++ endpoint=34.28.217.84 +++ '[' -n 34.28.217.84 -a 34.28.217.84 '!=' null ']' +++ echo 34.28.217.84 +++ return ++ endpoint=34.28.217.84 ++ echo 34.28.217.84 ++ return + endpoint=34.28.217.84 + retry 30 2 curl_metric_values node_boot_time_seconds monitoring-2-0-3915-monitoring-cfg-1 34.28.217.84 admin:admin basic + local max=30 + local delay=2 + shift 2 + local n=1 + curl_metric_values node_boot_time_seconds monitoring-2-0-3915-monitoring-cfg-1 34.28.217.84 admin:admin basic + local metric=node_boot_time_seconds + local instance=monitoring-2-0-3915-monitoring-cfg-1 + local endpoint=34.28.217.84 + local credentials=admin:admin + local auth_type=basic + local start + local end + local url + curl_args=() + local curl_args ++ /usr/sbin/date -u +%s -d '-1 minute' + start=1784745737 ++ /usr/sbin/date -u +%s + end=1784745797 + url='https://34.28.217.84/graph/api/datasources/proxy/1/api/v1/query_range?query=min%28node_boot_time_seconds%7Bnode_name%3D%7E%22monitoring-2-0-3915-monitoring-cfg-1%22%7D%20or%20node_boot_time_seconds%7Bnode_name%3D%7E%22monitoring-2-0-3915-monitoring-cfg-1%22%7D%29&start=1784745737&end=1784745797&step=60' + [[ basic == bearer ]] ++ urlencode_userinfo admin:admin ++ local user_pass=admin:admin ++ local username=admin ++ local password=admin +++ urlencode admin +++ uri=admin +++ echo -n admin +++ jq -s -R -r @uri +++ urlencode admin +++ uri=admin +++ echo -n admin +++ jq -s -R -r @uri ++ printf %s:%s admin admin + credentials=admin:admin + curl_args=("https://${credentials}@${url#https://}") + jq -r '(.data.result[0].values // [])[] | .[1] // empty' + execute_curl -s -k 'https://admin:admin@34.28.217.84/graph/api/datasources/proxy/1/api/v1/query_range?query=min%28node_boot_time_seconds%7Bnode_name%3D%7E%22monitoring-2-0-3915-monitoring-cfg-1%22%7D%20or%20node_boot_time_seconds%7Bnode_name%3D%7E%22monitoring-2-0-3915-monitoring-cfg-1%22%7D%29&start=1784745737&end=1784745797&step=60' + local name=curl-15799 + local image=docker.io/alpine/curl:latest + local phase= + local exit_status=0 + grep '^[0-9]' + kubectl_bin run curl-15799 -n monitoring-2-0-3915 --restart=Never --image=docker.io/alpine/curl:latest --command -- curl -sS -s -k 'https://admin:admin@34.28.217.84/graph/api/datasources/proxy/1/api/v1/query_range?query=min%28node_boot_time_seconds%7Bnode_name%3D%7E%22monitoring-2-0-3915-monitoring-cfg-1%22%7D%20or%20node_boot_time_seconds%7Bnode_name%3D%7E%22monitoring-2-0-3915-monitoring-cfg-1%22%7D%29&start=1784745737&end=1784745797&step=60' ++ mktemp + local LAST_OUT=/tmp/tmp.pZSUdkgqFd ++ mktemp + local LAST_ERR=/tmp/tmp.AYSOerG8cg + local exit_status=0 + local timeout=4 + args=('run' 'curl-15799' '-n' 'monitoring-2-0-3915' '--restart=Never' '--image=docker.io/alpine/curl:latest' '--command' '--' 'curl' '-sS' '-s' '-k' 'https://admin:admin@34.28.217.84/graph/api/datasources/proxy/1/api/v1/query_range?query=min%28node_boot_time_seconds%7Bnode_name%3D%7E%22monitoring-2-0-3915-monitoring-cfg-1%22%7D%20or%20node_boot_time_seconds%7Bnode_name%3D%7E%22monitoring-2-0-3915-monitoring-cfg-1%22%7D%29&start=1784745737&end=1784745797&step=60') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl run curl-15799 -n monitoring-2-0-3915 --restart=Never --image=docker.io/alpine/curl:latest --command -- curl -sS -s -k 'https://admin:admin@34.28.217.84/graph/api/datasources/proxy/1/api/v1/query_range?query=min%28node_boot_time_seconds%7Bnode_name%3D%7E%22monitoring-2-0-3915-monitoring-cfg-1%22%7D%20or%20node_boot_time_seconds%7Bnode_name%3D%7E%22monitoring-2-0-3915-monitoring-cfg-1%22%7D%29&start=1784745737&end=1784745797&step=60' + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.pZSUdkgqFd + cat /tmp/tmp.AYSOerG8cg + rm /tmp/tmp.pZSUdkgqFd /tmp/tmp.AYSOerG8cg + return 0 ++ seq 1 60 + for _ in $(seq 1 60) ++ kubectl_bin get pod curl-15799 -n monitoring-2-0-3915 -o 'jsonpath={.status.phase}' + phase=Pending + [[ Pending == Succeeded ]] + [[ Pending == Failed ]] + sleep 1 + for _ in $(seq 1 60) ++ kubectl_bin get pod curl-15799 -n monitoring-2-0-3915 -o 'jsonpath={.status.phase}' + phase=Succeeded + [[ Succeeded == Succeeded ]] + break + kubectl_bin logs curl-15799 -n monitoring-2-0-3915 ++ mktemp + local LAST_OUT=/tmp/tmp.Ybv2bU2Pee ++ mktemp + local LAST_ERR=/tmp/tmp.oFZqweSffn + local exit_status=0 + local timeout=4 + args=('logs' 'curl-15799' '-n' 'monitoring-2-0-3915') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl logs curl-15799 -n monitoring-2-0-3915 + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.Ybv2bU2Pee + cat /tmp/tmp.oFZqweSffn + rm /tmp/tmp.Ybv2bU2Pee /tmp/tmp.oFZqweSffn + return 0 + kubectl_bin delete pod curl-15799 -n monitoring-2-0-3915 --ignore-not-found --wait=false + [[ Succeeded != Succeeded ]] + return 0 1784744352 1784744352 + get_metric_values mongodb_connections monitoring-2-0-3915-monitoring-cfg-1 admin:admin + local metric=mongodb_connections + local instance=monitoring-2-0-3915-monitoring-cfg-1 + local user_pass=admin:admin + local auth_type=basic + local endpoint ++ get_service_endpoint monitoring-service ++ local service=monitoring-service ++ local endpoint ++ local service_type +++ kubectl_bin get service/monitoring-service -o 'jsonpath={.spec.type}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.ld7iXY3hQv ++++ mktemp +++ local LAST_ERR=/tmp/tmp.jUXo2n5Rkj +++ local exit_status=0 +++ local timeout=4 +++ args=('get' 'service/monitoring-service' '-o' 'jsonpath={.spec.type}') +++ local args +++ transformed_files=() +++ local transformed_files +++ arch_scheduling_enabled +++ [[ -n '' ]] +++ arch_scheduling_enabled +++ [[ -n '' ]] ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl get service/monitoring-service -o 'jsonpath={.spec.type}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 -a -n 1 ']' +++ break +++ cat /tmp/tmp.ld7iXY3hQv +++ cat /tmp/tmp.jUXo2n5Rkj +++ rm /tmp/tmp.ld7iXY3hQv /tmp/tmp.jUXo2n5Rkj +++ return 0 ++ service_type=LoadBalancer ++ [[ LoadBalancer == LoadBalancer ]] +++ get_load_balancer_endpoint monitoring-service +++ local service=monitoring-service +++ local endpoint ++++ kubectl_bin get service/monitoring-service -o json ++++ jq -r '[.status.loadBalancer.ingress[]?.hostname, .status.loadBalancer.ingress[]?.ip] | map(select(. != null and . != "")) | first // empty' +++++ mktemp ++++ local LAST_OUT=/tmp/tmp.jTJrU8FzPn +++++ mktemp ++++ local LAST_ERR=/tmp/tmp.gkTlnrWVRH ++++ local exit_status=0 ++++ local timeout=4 ++++ args=('get' 'service/monitoring-service' '-o' 'json') ++++ local args ++++ transformed_files=() ++++ local transformed_files ++++ arch_scheduling_enabled ++++ [[ -n '' ]] ++++ arch_scheduling_enabled ++++ [[ -n '' ]] +++++ seq 0 2 ++++ for i in $(seq 0 2) ++++ set +e ++++ kubectl get service/monitoring-service -o json ++++ exit_status=0 ++++ set -e ++++ '[' 0 '!=' 0 -a -n 1 ']' ++++ break ++++ cat /tmp/tmp.jTJrU8FzPn ++++ cat /tmp/tmp.gkTlnrWVRH ++++ rm /tmp/tmp.jTJrU8FzPn /tmp/tmp.gkTlnrWVRH ++++ return 0 +++ endpoint=34.28.217.84 +++ '[' -n 34.28.217.84 -a 34.28.217.84 '!=' null ']' +++ echo 34.28.217.84 +++ return ++ endpoint=34.28.217.84 ++ echo 34.28.217.84 ++ return + endpoint=34.28.217.84 + retry 30 2 curl_metric_values mongodb_connections monitoring-2-0-3915-monitoring-cfg-1 34.28.217.84 admin:admin basic + local max=30 + local delay=2 + shift 2 + local n=1 + curl_metric_values mongodb_connections monitoring-2-0-3915-monitoring-cfg-1 34.28.217.84 admin:admin basic + local metric=mongodb_connections + local instance=monitoring-2-0-3915-monitoring-cfg-1 + local endpoint=34.28.217.84 + local credentials=admin:admin + local auth_type=basic + local start + local end + local url + curl_args=() + local curl_args ++ /usr/sbin/date -u +%s -d '-1 minute' + start=1784745746 ++ /usr/sbin/date -u +%s + end=1784745806 + url='https://34.28.217.84/graph/api/datasources/proxy/1/api/v1/query_range?query=min%28mongodb_connections%7Bnode_name%3D%7E%22monitoring-2-0-3915-monitoring-cfg-1%22%7D%20or%20mongodb_connections%7Bnode_name%3D%7E%22monitoring-2-0-3915-monitoring-cfg-1%22%7D%29&start=1784745746&end=1784745806&step=60' + [[ basic == bearer ]] ++ urlencode_userinfo admin:admin ++ local user_pass=admin:admin ++ local username=admin ++ local password=admin +++ urlencode admin +++ uri=admin +++ echo -n admin +++ jq -s -R -r @uri +++ urlencode admin +++ uri=admin +++ jq -s -R -r @uri +++ echo -n admin ++ printf %s:%s admin admin + credentials=admin:admin + curl_args=("https://${credentials}@${url#https://}") + execute_curl -s -k 'https://admin:admin@34.28.217.84/graph/api/datasources/proxy/1/api/v1/query_range?query=min%28mongodb_connections%7Bnode_name%3D%7E%22monitoring-2-0-3915-monitoring-cfg-1%22%7D%20or%20mongodb_connections%7Bnode_name%3D%7E%22monitoring-2-0-3915-monitoring-cfg-1%22%7D%29&start=1784745746&end=1784745806&step=60' + local name=curl-29381 + local image=docker.io/alpine/curl:latest + local phase= + local exit_status=0 + kubectl_bin run curl-29381 -n monitoring-2-0-3915 --restart=Never --image=docker.io/alpine/curl:latest --command -- curl -sS -s -k 'https://admin:admin@34.28.217.84/graph/api/datasources/proxy/1/api/v1/query_range?query=min%28mongodb_connections%7Bnode_name%3D%7E%22monitoring-2-0-3915-monitoring-cfg-1%22%7D%20or%20mongodb_connections%7Bnode_name%3D%7E%22monitoring-2-0-3915-monitoring-cfg-1%22%7D%29&start=1784745746&end=1784745806&step=60' + jq -r '(.data.result[0].values // [])[] | .[1] // empty' + grep '^[0-9]' ++ mktemp + local LAST_OUT=/tmp/tmp.E2zftwKsLr ++ mktemp + local LAST_ERR=/tmp/tmp.4yQ7WXVdSu + local exit_status=0 + local timeout=4 + args=('run' 'curl-29381' '-n' 'monitoring-2-0-3915' '--restart=Never' '--image=docker.io/alpine/curl:latest' '--command' '--' 'curl' '-sS' '-s' '-k' 'https://admin:admin@34.28.217.84/graph/api/datasources/proxy/1/api/v1/query_range?query=min%28mongodb_connections%7Bnode_name%3D%7E%22monitoring-2-0-3915-monitoring-cfg-1%22%7D%20or%20mongodb_connections%7Bnode_name%3D%7E%22monitoring-2-0-3915-monitoring-cfg-1%22%7D%29&start=1784745746&end=1784745806&step=60') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl run curl-29381 -n monitoring-2-0-3915 --restart=Never --image=docker.io/alpine/curl:latest --command -- curl -sS -s -k 'https://admin:admin@34.28.217.84/graph/api/datasources/proxy/1/api/v1/query_range?query=min%28mongodb_connections%7Bnode_name%3D%7E%22monitoring-2-0-3915-monitoring-cfg-1%22%7D%20or%20mongodb_connections%7Bnode_name%3D%7E%22monitoring-2-0-3915-monitoring-cfg-1%22%7D%29&start=1784745746&end=1784745806&step=60' + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.E2zftwKsLr + cat /tmp/tmp.4yQ7WXVdSu + rm /tmp/tmp.E2zftwKsLr /tmp/tmp.4yQ7WXVdSu + return 0 ++ seq 1 60 + for _ in $(seq 1 60) ++ kubectl_bin get pod curl-29381 -n monitoring-2-0-3915 -o 'jsonpath={.status.phase}' + phase=Pending + [[ Pending == Succeeded ]] + [[ Pending == Failed ]] + sleep 1 + for _ in $(seq 1 60) ++ kubectl_bin get pod curl-29381 -n monitoring-2-0-3915 -o 'jsonpath={.status.phase}' + phase=Running + [[ Running == Succeeded ]] + [[ Running == Failed ]] + sleep 1 + for _ in $(seq 1 60) ++ kubectl_bin get pod curl-29381 -n monitoring-2-0-3915 -o 'jsonpath={.status.phase}' + phase=Succeeded + [[ Succeeded == Succeeded ]] + break + kubectl_bin logs curl-29381 -n monitoring-2-0-3915 ++ mktemp + local LAST_OUT=/tmp/tmp.OF9kP6AipX ++ mktemp + local LAST_ERR=/tmp/tmp.WtMfyWNbZa + local exit_status=0 + local timeout=4 + args=('logs' 'curl-29381' '-n' 'monitoring-2-0-3915') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl logs curl-29381 -n monitoring-2-0-3915 + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.OF9kP6AipX + cat /tmp/tmp.WtMfyWNbZa + rm /tmp/tmp.OF9kP6AipX /tmp/tmp.WtMfyWNbZa + return 0 + kubectl_bin delete pod curl-29381 -n monitoring-2-0-3915 --ignore-not-found --wait=false + [[ Succeeded != Succeeded ]] + return 0 0 0 + desc 'check mongos metrics' + set +o xtrace ----------------------------------------------------------------------------------- check mongos metrics ----------------------------------------------------------------------------------- ++ kubectl get pod -l app.kubernetes.io/component=mongos -o 'jsonpath={.items[0].metadata.name}' + MONGOS_POD_NAME=monitoring-mongos-0 + get_metric_values node_boot_time_seconds monitoring-2-0-3915-monitoring-mongos-0 admin:admin + local metric=node_boot_time_seconds + local instance=monitoring-2-0-3915-monitoring-mongos-0 + local user_pass=admin:admin + local auth_type=basic + local endpoint ++ get_service_endpoint monitoring-service ++ local service=monitoring-service ++ local endpoint ++ local service_type +++ kubectl_bin get service/monitoring-service -o 'jsonpath={.spec.type}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.4CU0JqT72P ++++ mktemp +++ local LAST_ERR=/tmp/tmp.tb7il5Zhm6 +++ local exit_status=0 +++ local timeout=4 +++ args=('get' 'service/monitoring-service' '-o' 'jsonpath={.spec.type}') +++ local args +++ transformed_files=() +++ local transformed_files +++ arch_scheduling_enabled +++ [[ -n '' ]] +++ arch_scheduling_enabled +++ [[ -n '' ]] ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl get service/monitoring-service -o 'jsonpath={.spec.type}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 -a -n 1 ']' +++ break +++ cat /tmp/tmp.4CU0JqT72P +++ cat /tmp/tmp.tb7il5Zhm6 +++ rm /tmp/tmp.4CU0JqT72P /tmp/tmp.tb7il5Zhm6 +++ return 0 ++ service_type=LoadBalancer ++ [[ LoadBalancer == LoadBalancer ]] +++ get_load_balancer_endpoint monitoring-service +++ local service=monitoring-service +++ local endpoint ++++ kubectl_bin get service/monitoring-service -o json ++++ jq -r '[.status.loadBalancer.ingress[]?.hostname, .status.loadBalancer.ingress[]?.ip] | map(select(. != null and . != "")) | first // empty' +++++ mktemp ++++ local LAST_OUT=/tmp/tmp.vINkRB3hzt +++++ mktemp ++++ local LAST_ERR=/tmp/tmp.hwGRaVujTh ++++ local exit_status=0 ++++ local timeout=4 ++++ args=('get' 'service/monitoring-service' '-o' 'json') ++++ local args ++++ transformed_files=() ++++ local transformed_files ++++ arch_scheduling_enabled ++++ [[ -n '' ]] ++++ arch_scheduling_enabled ++++ [[ -n '' ]] +++++ seq 0 2 ++++ for i in $(seq 0 2) ++++ set +e ++++ kubectl get service/monitoring-service -o json ++++ exit_status=0 ++++ set -e ++++ '[' 0 '!=' 0 -a -n 1 ']' ++++ break ++++ cat /tmp/tmp.vINkRB3hzt ++++ cat /tmp/tmp.hwGRaVujTh ++++ rm /tmp/tmp.vINkRB3hzt /tmp/tmp.hwGRaVujTh ++++ return 0 +++ endpoint=34.28.217.84 +++ '[' -n 34.28.217.84 -a 34.28.217.84 '!=' null ']' +++ echo 34.28.217.84 +++ return ++ endpoint=34.28.217.84 ++ echo 34.28.217.84 ++ return + endpoint=34.28.217.84 + retry 30 2 curl_metric_values node_boot_time_seconds monitoring-2-0-3915-monitoring-mongos-0 34.28.217.84 admin:admin basic + local max=30 + local delay=2 + shift 2 + local n=1 + curl_metric_values node_boot_time_seconds monitoring-2-0-3915-monitoring-mongos-0 34.28.217.84 admin:admin basic + local metric=node_boot_time_seconds + local instance=monitoring-2-0-3915-monitoring-mongos-0 + local endpoint=34.28.217.84 + local credentials=admin:admin + local auth_type=basic + local start + local end + local url + curl_args=() + local curl_args ++ /usr/sbin/date -u +%s -d '-1 minute' + start=1784745757 ++ /usr/sbin/date -u +%s + end=1784745817 + url='https://34.28.217.84/graph/api/datasources/proxy/1/api/v1/query_range?query=min%28node_boot_time_seconds%7Bnode_name%3D%7E%22monitoring-2-0-3915-monitoring-mongos-0%22%7D%20or%20node_boot_time_seconds%7Bnode_name%3D%7E%22monitoring-2-0-3915-monitoring-mongos-0%22%7D%29&start=1784745757&end=1784745817&step=60' + [[ basic == bearer ]] ++ urlencode_userinfo admin:admin ++ local user_pass=admin:admin ++ local username=admin ++ local password=admin +++ urlencode admin +++ uri=admin +++ echo -n admin +++ jq -s -R -r @uri +++ urlencode admin +++ uri=admin +++ echo -n admin +++ jq -s -R -r @uri ++ printf %s:%s admin admin + credentials=admin:admin + curl_args=("https://${credentials}@${url#https://}") + execute_curl -s -k 'https://admin:admin@34.28.217.84/graph/api/datasources/proxy/1/api/v1/query_range?query=min%28node_boot_time_seconds%7Bnode_name%3D%7E%22monitoring-2-0-3915-monitoring-mongos-0%22%7D%20or%20node_boot_time_seconds%7Bnode_name%3D%7E%22monitoring-2-0-3915-monitoring-mongos-0%22%7D%29&start=1784745757&end=1784745817&step=60' + local name=curl-81 + jq -r '(.data.result[0].values // [])[] | .[1] // empty' + local image=docker.io/alpine/curl:latest + local phase= + local exit_status=0 + kubectl_bin run curl-81 -n monitoring-2-0-3915 --restart=Never --image=docker.io/alpine/curl:latest --command -- curl -sS -s -k 'https://admin:admin@34.28.217.84/graph/api/datasources/proxy/1/api/v1/query_range?query=min%28node_boot_time_seconds%7Bnode_name%3D%7E%22monitoring-2-0-3915-monitoring-mongos-0%22%7D%20or%20node_boot_time_seconds%7Bnode_name%3D%7E%22monitoring-2-0-3915-monitoring-mongos-0%22%7D%29&start=1784745757&end=1784745817&step=60' + grep '^[0-9]' ++ mktemp + local LAST_OUT=/tmp/tmp.or5JFmoH5x ++ mktemp + local LAST_ERR=/tmp/tmp.sOXYiVWUWP + local exit_status=0 + local timeout=4 + args=('run' 'curl-81' '-n' 'monitoring-2-0-3915' '--restart=Never' '--image=docker.io/alpine/curl:latest' '--command' '--' 'curl' '-sS' '-s' '-k' 'https://admin:admin@34.28.217.84/graph/api/datasources/proxy/1/api/v1/query_range?query=min%28node_boot_time_seconds%7Bnode_name%3D%7E%22monitoring-2-0-3915-monitoring-mongos-0%22%7D%20or%20node_boot_time_seconds%7Bnode_name%3D%7E%22monitoring-2-0-3915-monitoring-mongos-0%22%7D%29&start=1784745757&end=1784745817&step=60') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl run curl-81 -n monitoring-2-0-3915 --restart=Never --image=docker.io/alpine/curl:latest --command -- curl -sS -s -k 'https://admin:admin@34.28.217.84/graph/api/datasources/proxy/1/api/v1/query_range?query=min%28node_boot_time_seconds%7Bnode_name%3D%7E%22monitoring-2-0-3915-monitoring-mongos-0%22%7D%20or%20node_boot_time_seconds%7Bnode_name%3D%7E%22monitoring-2-0-3915-monitoring-mongos-0%22%7D%29&start=1784745757&end=1784745817&step=60' + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.or5JFmoH5x + cat /tmp/tmp.sOXYiVWUWP + rm /tmp/tmp.or5JFmoH5x /tmp/tmp.sOXYiVWUWP + return 0 ++ seq 1 60 + for _ in $(seq 1 60) ++ kubectl_bin get pod curl-81 -n monitoring-2-0-3915 -o 'jsonpath={.status.phase}' + phase=Pending + [[ Pending == Succeeded ]] + [[ Pending == Failed ]] + sleep 1 + for _ in $(seq 1 60) ++ kubectl_bin get pod curl-81 -n monitoring-2-0-3915 -o 'jsonpath={.status.phase}' + phase=Succeeded + [[ Succeeded == Succeeded ]] + break + kubectl_bin logs curl-81 -n monitoring-2-0-3915 ++ mktemp + local LAST_OUT=/tmp/tmp.ZNZh2guWeF ++ mktemp + local LAST_ERR=/tmp/tmp.oSqAuPy0XZ + local exit_status=0 + local timeout=4 + args=('logs' 'curl-81' '-n' 'monitoring-2-0-3915') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl logs curl-81 -n monitoring-2-0-3915 + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.ZNZh2guWeF + cat /tmp/tmp.oSqAuPy0XZ + rm /tmp/tmp.ZNZh2guWeF /tmp/tmp.oSqAuPy0XZ + return 0 + kubectl_bin delete pod curl-81 -n monitoring-2-0-3915 --ignore-not-found --wait=false + [[ Succeeded != Succeeded ]] + return 0 1784744352 1784744352 + sleep 90 + desc 'check QAN data' + set +o xtrace ----------------------------------------------------------------------------------- check QAN data ----------------------------------------------------------------------------------- + get_qan_values mongodb dev-mongod admin:admin + local service_type=mongodb + local environment=dev-mongod + local user_pass=admin:admin + local auth_type=basic + local start + local end + local endpoint ++ /usr/sbin/date -u -d '-12 hour' +%Y-%m-%dT%H:%M:%S%:z + start=2026-07-22T06:45:13+00:00 ++ /usr/sbin/date -u +%Y-%m-%dT%H:%M:%S%:z + end=2026-07-22T18:45:13+00:00 ++ get_service_endpoint monitoring-service ++ local service=monitoring-service ++ local endpoint ++ local service_type +++ kubectl_bin get service/monitoring-service -o 'jsonpath={.spec.type}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.lLlF3sBRDa ++++ mktemp +++ local LAST_ERR=/tmp/tmp.HpszOlDuR6 +++ local exit_status=0 +++ local timeout=4 +++ args=('get' 'service/monitoring-service' '-o' 'jsonpath={.spec.type}') +++ local args +++ transformed_files=() +++ local transformed_files +++ arch_scheduling_enabled +++ [[ -n '' ]] +++ arch_scheduling_enabled +++ [[ -n '' ]] ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl get service/monitoring-service -o 'jsonpath={.spec.type}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 -a -n 1 ']' +++ break +++ cat /tmp/tmp.lLlF3sBRDa +++ cat /tmp/tmp.HpszOlDuR6 +++ rm /tmp/tmp.lLlF3sBRDa /tmp/tmp.HpszOlDuR6 +++ return 0 ++ service_type=LoadBalancer ++ [[ LoadBalancer == LoadBalancer ]] +++ get_load_balancer_endpoint monitoring-service +++ local service=monitoring-service +++ local endpoint ++++ jq -r '[.status.loadBalancer.ingress[]?.hostname, .status.loadBalancer.ingress[]?.ip] | map(select(. != null and . != "")) | first // empty' ++++ kubectl_bin get service/monitoring-service -o json +++++ mktemp ++++ local LAST_OUT=/tmp/tmp.o9S9afEQgP +++++ mktemp ++++ local LAST_ERR=/tmp/tmp.wjgqPQOY5v ++++ local exit_status=0 ++++ local timeout=4 ++++ args=('get' 'service/monitoring-service' '-o' 'json') ++++ local args ++++ transformed_files=() ++++ local transformed_files ++++ arch_scheduling_enabled ++++ [[ -n '' ]] ++++ arch_scheduling_enabled ++++ [[ -n '' ]] +++++ seq 0 2 ++++ for i in $(seq 0 2) ++++ set +e ++++ kubectl get service/monitoring-service -o json ++++ exit_status=0 ++++ set -e ++++ '[' 0 '!=' 0 -a -n 1 ']' ++++ break ++++ cat /tmp/tmp.o9S9afEQgP ++++ cat /tmp/tmp.wjgqPQOY5v ++++ rm /tmp/tmp.o9S9afEQgP /tmp/tmp.wjgqPQOY5v ++++ return 0 +++ endpoint=34.28.217.84 +++ '[' -n 34.28.217.84 -a 34.28.217.84 '!=' null ']' +++ echo 34.28.217.84 +++ return ++ endpoint=34.28.217.84 ++ echo 34.28.217.84 ++ return + endpoint=34.28.217.84 + cat + retry 30 2 curl_qan_values payload.json 34.28.217.84 admin:admin basic + local max=30 + local delay=2 + shift 2 + local n=1 + curl_qan_values payload.json 34.28.217.84 admin:admin basic + local payload_file=payload.json + local endpoint=34.28.217.84 + local credentials=admin:admin + local auth_type=basic + local url + curl_args=() + local curl_args + [[ basic == bearer ]] + url=https://34.28.217.84/v0/qan/GetReport ++ urlencode_userinfo admin:admin ++ local user_pass=admin:admin ++ local username=admin ++ local password=admin +++ urlencode admin +++ uri=admin +++ echo -n admin +++ jq -s -R -r @uri +++ urlencode admin +++ uri=admin +++ echo -n admin +++ jq -s -R -r @uri ++ printf %s:%s admin admin + credentials=admin:admin + curl_args=("https://${credentials}@${url#https://}") + jq -e 'any(.rows[]?; .sparkline != null)' ++ cat payload.json + execute_curl -s -k -XPOST -d $'{\n "period_start_from": "2026-07-22T06:45:13+00:00",\n "period_start_to": "2026-07-22T18:45:13+00:00",\n "group_by": "queryid",\n "labels": [\n {\n "key": "service_type",\n "value": [\n "mongodb"\n ]\n },\n {\n "key": "environment",\n "value": [\n "dev-mongod"\n ]\n }\n ],\n "columns": [\n "load",\n "num_queries",\n "query_time"\n ],\n "order_by": "-load",\n "offset": 0,\n "limit": 10,\n "main_metric": "load",\n "search": ""\n}' https://admin:admin@34.28.217.84/v0/qan/GetReport + local name=curl-11418 + local image=docker.io/alpine/curl:latest + local phase= + local exit_status=0 + kubectl_bin run curl-11418 -n monitoring-2-0-3915 --restart=Never --image=docker.io/alpine/curl:latest --command -- curl -sS -s -k -XPOST -d $'{\n "period_start_from": "2026-07-22T06:45:13+00:00",\n "period_start_to": "2026-07-22T18:45:13+00:00",\n "group_by": "queryid",\n "labels": [\n {\n "key": "service_type",\n "value": [\n "mongodb"\n ]\n },\n {\n "key": "environment",\n "value": [\n "dev-mongod"\n ]\n }\n ],\n "columns": [\n "load",\n "num_queries",\n "query_time"\n ],\n "order_by": "-load",\n "offset": 0,\n "limit": 10,\n "main_metric": "load",\n "search": ""\n}' https://admin:admin@34.28.217.84/v0/qan/GetReport ++ mktemp + local LAST_OUT=/tmp/tmp.TFNG77lFo1 ++ mktemp + local LAST_ERR=/tmp/tmp.5dRTEjRiS2 + local exit_status=0 + local timeout=4 + args=('run' 'curl-11418' '-n' 'monitoring-2-0-3915' '--restart=Never' '--image=docker.io/alpine/curl:latest' '--command' '--' 'curl' '-sS' '-s' '-k' '-XPOST' '-d' '{ "period_start_from": "2026-07-22T06:45:13+00:00", "period_start_to": "2026-07-22T18:45:13+00:00", "group_by": "queryid", "labels": [ { "key": "service_type", "value": [ "mongodb" ] }, { "key": "environment", "value": [ "dev-mongod" ] } ], "columns": [ "load", "num_queries", "query_time" ], "order_by": "-load", "offset": 0, "limit": 10, "main_metric": "load", "search": "" }' 'https://admin:admin@34.28.217.84/v0/qan/GetReport') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl run curl-11418 -n monitoring-2-0-3915 --restart=Never --image=docker.io/alpine/curl:latest --command -- curl -sS -s -k -XPOST -d $'{\n "period_start_from": "2026-07-22T06:45:13+00:00",\n "period_start_to": "2026-07-22T18:45:13+00:00",\n "group_by": "queryid",\n "labels": [\n {\n "key": "service_type",\n "value": [\n "mongodb"\n ]\n },\n {\n "key": "environment",\n "value": [\n "dev-mongod"\n ]\n }\n ],\n "columns": [\n "load",\n "num_queries",\n "query_time"\n ],\n "order_by": "-load",\n "offset": 0,\n "limit": 10,\n "main_metric": "load",\n "search": ""\n}' https://admin:admin@34.28.217.84/v0/qan/GetReport + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.TFNG77lFo1 + cat /tmp/tmp.5dRTEjRiS2 + rm /tmp/tmp.TFNG77lFo1 /tmp/tmp.5dRTEjRiS2 + return 0 ++ seq 1 60 + for _ in $(seq 1 60) ++ kubectl_bin get pod curl-11418 -n monitoring-2-0-3915 -o 'jsonpath={.status.phase}' + phase=Pending + [[ Pending == Succeeded ]] + [[ Pending == Failed ]] + sleep 1 + for _ in $(seq 1 60) ++ kubectl_bin get pod curl-11418 -n monitoring-2-0-3915 -o 'jsonpath={.status.phase}' + phase=Succeeded + [[ Succeeded == Succeeded ]] + break + kubectl_bin logs curl-11418 -n monitoring-2-0-3915 ++ mktemp + local LAST_OUT=/tmp/tmp.E0KLpsoo7O ++ mktemp + local LAST_ERR=/tmp/tmp.8jVuji42iE + local exit_status=0 + local timeout=4 + args=('logs' 'curl-11418' '-n' 'monitoring-2-0-3915') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl logs curl-11418 -n monitoring-2-0-3915 + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.E0KLpsoo7O + cat /tmp/tmp.8jVuji42iE + rm /tmp/tmp.E0KLpsoo7O /tmp/tmp.8jVuji42iE + return 0 + kubectl_bin delete pod curl-11418 -n monitoring-2-0-3915 --ignore-not-found --wait=false + [[ Succeeded != Succeeded ]] + return 0 true + rm -f payload.json + get_qan_values mongodb dev-mongos admin:admin + local service_type=mongodb + local environment=dev-mongos + local user_pass=admin:admin + local auth_type=basic + local start + local end + local endpoint ++ /usr/sbin/date -u -d '-12 hour' +%Y-%m-%dT%H:%M:%S%:z + start=2026-07-22T06:45:22+00:00 ++ /usr/sbin/date -u +%Y-%m-%dT%H:%M:%S%:z + end=2026-07-22T18:45:22+00:00 ++ get_service_endpoint monitoring-service ++ local service=monitoring-service ++ local endpoint ++ local service_type +++ kubectl_bin get service/monitoring-service -o 'jsonpath={.spec.type}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.zapS17gzgY ++++ mktemp +++ local LAST_ERR=/tmp/tmp.kJFulVN71R +++ local exit_status=0 +++ local timeout=4 +++ args=('get' 'service/monitoring-service' '-o' 'jsonpath={.spec.type}') +++ local args +++ transformed_files=() +++ local transformed_files +++ arch_scheduling_enabled +++ [[ -n '' ]] +++ arch_scheduling_enabled +++ [[ -n '' ]] ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl get service/monitoring-service -o 'jsonpath={.spec.type}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 -a -n 1 ']' +++ break +++ cat /tmp/tmp.zapS17gzgY +++ cat /tmp/tmp.kJFulVN71R +++ rm /tmp/tmp.zapS17gzgY /tmp/tmp.kJFulVN71R +++ return 0 ++ service_type=LoadBalancer ++ [[ LoadBalancer == LoadBalancer ]] +++ get_load_balancer_endpoint monitoring-service +++ local service=monitoring-service +++ local endpoint ++++ jq -r '[.status.loadBalancer.ingress[]?.hostname, .status.loadBalancer.ingress[]?.ip] | map(select(. != null and . != "")) | first // empty' ++++ kubectl_bin get service/monitoring-service -o json +++++ mktemp ++++ local LAST_OUT=/tmp/tmp.gRXeKSfsOU +++++ mktemp ++++ local LAST_ERR=/tmp/tmp.3cmyXGf0Du ++++ local exit_status=0 ++++ local timeout=4 ++++ args=('get' 'service/monitoring-service' '-o' 'json') ++++ local args ++++ transformed_files=() ++++ local transformed_files ++++ arch_scheduling_enabled ++++ [[ -n '' ]] ++++ arch_scheduling_enabled ++++ [[ -n '' ]] +++++ seq 0 2 ++++ for i in $(seq 0 2) ++++ set +e ++++ kubectl get service/monitoring-service -o json ++++ exit_status=0 ++++ set -e ++++ '[' 0 '!=' 0 -a -n 1 ']' ++++ break ++++ cat /tmp/tmp.gRXeKSfsOU ++++ cat /tmp/tmp.3cmyXGf0Du ++++ rm /tmp/tmp.gRXeKSfsOU /tmp/tmp.3cmyXGf0Du ++++ return 0 +++ endpoint=34.28.217.84 +++ '[' -n 34.28.217.84 -a 34.28.217.84 '!=' null ']' +++ echo 34.28.217.84 +++ return ++ endpoint=34.28.217.84 ++ echo 34.28.217.84 ++ return + endpoint=34.28.217.84 + cat + retry 30 2 curl_qan_values payload.json 34.28.217.84 admin:admin basic + local max=30 + local delay=2 + shift 2 + local n=1 + curl_qan_values payload.json 34.28.217.84 admin:admin basic + local payload_file=payload.json + local endpoint=34.28.217.84 + local credentials=admin:admin + local auth_type=basic + local url + curl_args=() + local curl_args + [[ basic == bearer ]] + url=https://34.28.217.84/v0/qan/GetReport ++ urlencode_userinfo admin:admin ++ local user_pass=admin:admin ++ local username=admin ++ local password=admin +++ urlencode admin +++ uri=admin +++ echo -n admin +++ jq -s -R -r @uri +++ urlencode admin +++ uri=admin +++ echo -n admin +++ jq -s -R -r @uri ++ printf %s:%s admin admin + credentials=admin:admin + curl_args=("https://${credentials}@${url#https://}") + jq -e 'any(.rows[]?; .sparkline != null)' ++ cat payload.json + execute_curl -s -k -XPOST -d $'{\n "period_start_from": "2026-07-22T06:45:22+00:00",\n "period_start_to": "2026-07-22T18:45:22+00:00",\n "group_by": "queryid",\n "labels": [\n {\n "key": "service_type",\n "value": [\n "mongodb"\n ]\n },\n {\n "key": "environment",\n "value": [\n "dev-mongos"\n ]\n }\n ],\n "columns": [\n "load",\n "num_queries",\n "query_time"\n ],\n "order_by": "-load",\n "offset": 0,\n "limit": 10,\n "main_metric": "load",\n "search": ""\n}' https://admin:admin@34.28.217.84/v0/qan/GetReport + local name=curl-30528 + local image=docker.io/alpine/curl:latest + local phase= + local exit_status=0 + kubectl_bin run curl-30528 -n monitoring-2-0-3915 --restart=Never --image=docker.io/alpine/curl:latest --command -- curl -sS -s -k -XPOST -d $'{\n "period_start_from": "2026-07-22T06:45:22+00:00",\n "period_start_to": "2026-07-22T18:45:22+00:00",\n "group_by": "queryid",\n "labels": [\n {\n "key": "service_type",\n "value": [\n "mongodb"\n ]\n },\n {\n "key": "environment",\n "value": [\n "dev-mongos"\n ]\n }\n ],\n "columns": [\n "load",\n "num_queries",\n "query_time"\n ],\n "order_by": "-load",\n "offset": 0,\n "limit": 10,\n "main_metric": "load",\n "search": ""\n}' https://admin:admin@34.28.217.84/v0/qan/GetReport ++ mktemp + local LAST_OUT=/tmp/tmp.7UfhEV13l5 ++ mktemp + local LAST_ERR=/tmp/tmp.QCS7hRZFsA + local exit_status=0 + local timeout=4 + args=('run' 'curl-30528' '-n' 'monitoring-2-0-3915' '--restart=Never' '--image=docker.io/alpine/curl:latest' '--command' '--' 'curl' '-sS' '-s' '-k' '-XPOST' '-d' '{ "period_start_from": "2026-07-22T06:45:22+00:00", "period_start_to": "2026-07-22T18:45:22+00:00", "group_by": "queryid", "labels": [ { "key": "service_type", "value": [ "mongodb" ] }, { "key": "environment", "value": [ "dev-mongos" ] } ], "columns": [ "load", "num_queries", "query_time" ], "order_by": "-load", "offset": 0, "limit": 10, "main_metric": "load", "search": "" }' 'https://admin:admin@34.28.217.84/v0/qan/GetReport') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl run curl-30528 -n monitoring-2-0-3915 --restart=Never --image=docker.io/alpine/curl:latest --command -- curl -sS -s -k -XPOST -d $'{\n "period_start_from": "2026-07-22T06:45:22+00:00",\n "period_start_to": "2026-07-22T18:45:22+00:00",\n "group_by": "queryid",\n "labels": [\n {\n "key": "service_type",\n "value": [\n "mongodb"\n ]\n },\n {\n "key": "environment",\n "value": [\n "dev-mongos"\n ]\n }\n ],\n "columns": [\n "load",\n "num_queries",\n "query_time"\n ],\n "order_by": "-load",\n "offset": 0,\n "limit": 10,\n "main_metric": "load",\n "search": ""\n}' https://admin:admin@34.28.217.84/v0/qan/GetReport + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.7UfhEV13l5 + cat /tmp/tmp.QCS7hRZFsA + rm /tmp/tmp.7UfhEV13l5 /tmp/tmp.QCS7hRZFsA + return 0 ++ seq 1 60 + for _ in $(seq 1 60) ++ kubectl_bin get pod curl-30528 -n monitoring-2-0-3915 -o 'jsonpath={.status.phase}' + phase=Pending + [[ Pending == Succeeded ]] + [[ Pending == Failed ]] + sleep 1 + for _ in $(seq 1 60) ++ kubectl_bin get pod curl-30528 -n monitoring-2-0-3915 -o 'jsonpath={.status.phase}' + phase=Succeeded + [[ Succeeded == Succeeded ]] + break + kubectl_bin logs curl-30528 -n monitoring-2-0-3915 ++ mktemp + local LAST_OUT=/tmp/tmp.6brtPLQ998 ++ mktemp + local LAST_ERR=/tmp/tmp.vd6HS8sq34 + local exit_status=0 + local timeout=4 + args=('logs' 'curl-30528' '-n' 'monitoring-2-0-3915') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl logs curl-30528 -n monitoring-2-0-3915 + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.6brtPLQ998 + cat /tmp/tmp.vd6HS8sq34 + rm /tmp/tmp.6brtPLQ998 /tmp/tmp.vd6HS8sq34 + return 0 + kubectl_bin delete pod curl-30528 -n monitoring-2-0-3915 --ignore-not-found --wait=false + [[ Succeeded != Succeeded ]] + return 0 true + rm -f payload.json + nodeList=($(get_node_id_from_pmm)) ++ get_node_id_from_pmm ++ nodeList=() ++ local -a nodeList +++ kubectl_bin get pods --no-headers -l app.kubernetes.io/name=percona-server-mongodb --output=custom-columns=NAME:.metadata.name ++++ mktemp +++ local LAST_OUT=/tmp/tmp.Rj0dlKe8tJ ++++ mktemp +++ local LAST_ERR=/tmp/tmp.tM15K2FVZN +++ local exit_status=0 +++ local timeout=4 +++ args=('get' 'pods' '--no-headers' '-l' 'app.kubernetes.io/name=percona-server-mongodb' '--output=custom-columns=NAME:.metadata.name') +++ local args +++ transformed_files=() +++ local transformed_files +++ arch_scheduling_enabled +++ [[ -n '' ]] +++ arch_scheduling_enabled +++ [[ -n '' ]] ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl get pods --no-headers -l app.kubernetes.io/name=percona-server-mongodb --output=custom-columns=NAME:.metadata.name +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 -a -n 1 ']' +++ break +++ cat /tmp/tmp.Rj0dlKe8tJ +++ cat /tmp/tmp.tM15K2FVZN +++ rm /tmp/tmp.Rj0dlKe8tJ /tmp/tmp.tM15K2FVZN +++ return 0 ++ for instance in $(kubectl_bin get pods --no-headers -l app.kubernetes.io/name=percona-server-mongodb --output=custom-columns='NAME:.metadata.name') ++ nodeList+=($(kubectl_bin exec -n "$namespace" $instance -c pmm-client -- pmm-admin status --json | jq -r '.pmm_agent_status.node_id')) +++ kubectl_bin exec -n monitoring-2-0-3915 monitoring-cfg-0 -c pmm-client -- pmm-admin status --json +++ jq -r .pmm_agent_status.node_id ++++ mktemp +++ local LAST_OUT=/tmp/tmp.6Yz9zIAFqk ++++ mktemp +++ local LAST_ERR=/tmp/tmp.kxEv0Us38A +++ local exit_status=0 +++ local timeout=4 +++ args=('exec' '-n' 'monitoring-2-0-3915' 'monitoring-cfg-0' '-c' 'pmm-client' '--' 'pmm-admin' 'status' '--json') +++ local args +++ transformed_files=() +++ local transformed_files +++ arch_scheduling_enabled +++ [[ -n '' ]] +++ arch_scheduling_enabled +++ [[ -n '' ]] ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl exec -n monitoring-2-0-3915 monitoring-cfg-0 -c pmm-client -- pmm-admin status --json +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 -a -n 1 ']' +++ break +++ cat /tmp/tmp.6Yz9zIAFqk +++ cat /tmp/tmp.kxEv0Us38A +++ rm /tmp/tmp.6Yz9zIAFqk /tmp/tmp.kxEv0Us38A +++ return 0 ++ for instance in $(kubectl_bin get pods --no-headers -l app.kubernetes.io/name=percona-server-mongodb --output=custom-columns='NAME:.metadata.name') ++ nodeList+=($(kubectl_bin exec -n "$namespace" $instance -c pmm-client -- pmm-admin status --json | jq -r '.pmm_agent_status.node_id')) +++ jq -r .pmm_agent_status.node_id +++ kubectl_bin exec -n monitoring-2-0-3915 monitoring-cfg-1 -c pmm-client -- pmm-admin status --json ++++ mktemp +++ local LAST_OUT=/tmp/tmp.SKYIwumWgu ++++ mktemp +++ local LAST_ERR=/tmp/tmp.jWFrxT8Tib +++ local exit_status=0 +++ local timeout=4 +++ args=('exec' '-n' 'monitoring-2-0-3915' 'monitoring-cfg-1' '-c' 'pmm-client' '--' 'pmm-admin' 'status' '--json') +++ local args +++ transformed_files=() +++ local transformed_files +++ arch_scheduling_enabled +++ [[ -n '' ]] +++ arch_scheduling_enabled +++ [[ -n '' ]] ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl exec -n monitoring-2-0-3915 monitoring-cfg-1 -c pmm-client -- pmm-admin status --json +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 -a -n 1 ']' +++ break +++ cat /tmp/tmp.SKYIwumWgu +++ cat /tmp/tmp.jWFrxT8Tib +++ rm /tmp/tmp.SKYIwumWgu /tmp/tmp.jWFrxT8Tib +++ return 0 ++ for instance in $(kubectl_bin get pods --no-headers -l app.kubernetes.io/name=percona-server-mongodb --output=custom-columns='NAME:.metadata.name') ++ nodeList+=($(kubectl_bin exec -n "$namespace" $instance -c pmm-client -- pmm-admin status --json | jq -r '.pmm_agent_status.node_id')) +++ jq -r .pmm_agent_status.node_id +++ kubectl_bin exec -n monitoring-2-0-3915 monitoring-cfg-2 -c pmm-client -- pmm-admin status --json ++++ mktemp +++ local LAST_OUT=/tmp/tmp.Qpa9J6sIef ++++ mktemp +++ local LAST_ERR=/tmp/tmp.U8TooD8Amk +++ local exit_status=0 +++ local timeout=4 +++ args=('exec' '-n' 'monitoring-2-0-3915' 'monitoring-cfg-2' '-c' 'pmm-client' '--' 'pmm-admin' 'status' '--json') +++ local args +++ transformed_files=() +++ local transformed_files +++ arch_scheduling_enabled +++ [[ -n '' ]] +++ arch_scheduling_enabled +++ [[ -n '' ]] ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl exec -n monitoring-2-0-3915 monitoring-cfg-2 -c pmm-client -- pmm-admin status --json +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 -a -n 1 ']' +++ break +++ cat /tmp/tmp.Qpa9J6sIef +++ cat /tmp/tmp.U8TooD8Amk +++ rm /tmp/tmp.Qpa9J6sIef /tmp/tmp.U8TooD8Amk +++ return 0 ++ for instance in $(kubectl_bin get pods --no-headers -l app.kubernetes.io/name=percona-server-mongodb --output=custom-columns='NAME:.metadata.name') ++ nodeList+=($(kubectl_bin exec -n "$namespace" $instance -c pmm-client -- pmm-admin status --json | jq -r '.pmm_agent_status.node_id')) +++ kubectl_bin exec -n monitoring-2-0-3915 monitoring-mongos-0 -c pmm-client -- pmm-admin status --json +++ jq -r .pmm_agent_status.node_id ++++ mktemp +++ local LAST_OUT=/tmp/tmp.WOi7mUFzTq ++++ mktemp +++ local LAST_ERR=/tmp/tmp.BtAKV5Vy6a +++ local exit_status=0 +++ local timeout=4 +++ args=('exec' '-n' 'monitoring-2-0-3915' 'monitoring-mongos-0' '-c' 'pmm-client' '--' 'pmm-admin' 'status' '--json') +++ local args +++ transformed_files=() +++ local transformed_files +++ arch_scheduling_enabled +++ [[ -n '' ]] +++ arch_scheduling_enabled +++ [[ -n '' ]] ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl exec -n monitoring-2-0-3915 monitoring-mongos-0 -c pmm-client -- pmm-admin status --json +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 -a -n 1 ']' +++ break +++ cat /tmp/tmp.WOi7mUFzTq +++ cat /tmp/tmp.BtAKV5Vy6a +++ rm /tmp/tmp.WOi7mUFzTq /tmp/tmp.BtAKV5Vy6a +++ return 0 ++ for instance in $(kubectl_bin get pods --no-headers -l app.kubernetes.io/name=percona-server-mongodb --output=custom-columns='NAME:.metadata.name') ++ nodeList+=($(kubectl_bin exec -n "$namespace" $instance -c pmm-client -- pmm-admin status --json | jq -r '.pmm_agent_status.node_id')) +++ kubectl_bin exec -n monitoring-2-0-3915 monitoring-mongos-1 -c pmm-client -- pmm-admin status --json +++ jq -r .pmm_agent_status.node_id ++++ mktemp +++ local LAST_OUT=/tmp/tmp.4SpmX4RXcc ++++ mktemp +++ local LAST_ERR=/tmp/tmp.mxN3IhUIAn +++ local exit_status=0 +++ local timeout=4 +++ args=('exec' '-n' 'monitoring-2-0-3915' 'monitoring-mongos-1' '-c' 'pmm-client' '--' 'pmm-admin' 'status' '--json') +++ local args +++ transformed_files=() +++ local transformed_files +++ arch_scheduling_enabled +++ [[ -n '' ]] +++ arch_scheduling_enabled +++ [[ -n '' ]] ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl exec -n monitoring-2-0-3915 monitoring-mongos-1 -c pmm-client -- pmm-admin status --json +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 -a -n 1 ']' +++ break +++ cat /tmp/tmp.4SpmX4RXcc +++ cat /tmp/tmp.mxN3IhUIAn +++ rm /tmp/tmp.4SpmX4RXcc /tmp/tmp.mxN3IhUIAn +++ return 0 ++ for instance in $(kubectl_bin get pods --no-headers -l app.kubernetes.io/name=percona-server-mongodb --output=custom-columns='NAME:.metadata.name') ++ nodeList+=($(kubectl_bin exec -n "$namespace" $instance -c pmm-client -- pmm-admin status --json | jq -r '.pmm_agent_status.node_id')) +++ kubectl_bin exec -n monitoring-2-0-3915 monitoring-mongos-2 -c pmm-client -- pmm-admin status --json +++ jq -r .pmm_agent_status.node_id ++++ mktemp +++ local LAST_OUT=/tmp/tmp.B39qsYElLL ++++ mktemp +++ local LAST_ERR=/tmp/tmp.EkIJCArWRv +++ local exit_status=0 +++ local timeout=4 +++ args=('exec' '-n' 'monitoring-2-0-3915' 'monitoring-mongos-2' '-c' 'pmm-client' '--' 'pmm-admin' 'status' '--json') +++ local args +++ transformed_files=() +++ local transformed_files +++ arch_scheduling_enabled +++ [[ -n '' ]] +++ arch_scheduling_enabled +++ [[ -n '' ]] ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl exec -n monitoring-2-0-3915 monitoring-mongos-2 -c pmm-client -- pmm-admin status --json +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 -a -n 1 ']' +++ break +++ cat /tmp/tmp.B39qsYElLL +++ cat /tmp/tmp.EkIJCArWRv +++ rm /tmp/tmp.B39qsYElLL /tmp/tmp.EkIJCArWRv +++ return 0 ++ for instance in $(kubectl_bin get pods --no-headers -l app.kubernetes.io/name=percona-server-mongodb --output=custom-columns='NAME:.metadata.name') ++ nodeList+=($(kubectl_bin exec -n "$namespace" $instance -c pmm-client -- pmm-admin status --json | jq -r '.pmm_agent_status.node_id')) +++ kubectl_bin exec -n monitoring-2-0-3915 monitoring-rs0-0 -c pmm-client -- pmm-admin status --json +++ jq -r .pmm_agent_status.node_id ++++ mktemp +++ local LAST_OUT=/tmp/tmp.PtMJPEYICL ++++ mktemp +++ local LAST_ERR=/tmp/tmp.vZK7WLXcJr +++ local exit_status=0 +++ local timeout=4 +++ args=('exec' '-n' 'monitoring-2-0-3915' 'monitoring-rs0-0' '-c' 'pmm-client' '--' 'pmm-admin' 'status' '--json') +++ local args +++ transformed_files=() +++ local transformed_files +++ arch_scheduling_enabled +++ [[ -n '' ]] +++ arch_scheduling_enabled +++ [[ -n '' ]] ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl exec -n monitoring-2-0-3915 monitoring-rs0-0 -c pmm-client -- pmm-admin status --json +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 -a -n 1 ']' +++ break +++ cat /tmp/tmp.PtMJPEYICL +++ cat /tmp/tmp.vZK7WLXcJr +++ rm /tmp/tmp.PtMJPEYICL /tmp/tmp.vZK7WLXcJr +++ return 0 ++ for instance in $(kubectl_bin get pods --no-headers -l app.kubernetes.io/name=percona-server-mongodb --output=custom-columns='NAME:.metadata.name') ++ nodeList+=($(kubectl_bin exec -n "$namespace" $instance -c pmm-client -- pmm-admin status --json | jq -r '.pmm_agent_status.node_id')) +++ kubectl_bin exec -n monitoring-2-0-3915 monitoring-rs0-1 -c pmm-client -- pmm-admin status --json +++ jq -r .pmm_agent_status.node_id ++++ mktemp +++ local LAST_OUT=/tmp/tmp.neR4WjHCNw ++++ mktemp +++ local LAST_ERR=/tmp/tmp.nNqptLjSAg +++ local exit_status=0 +++ local timeout=4 +++ args=('exec' '-n' 'monitoring-2-0-3915' 'monitoring-rs0-1' '-c' 'pmm-client' '--' 'pmm-admin' 'status' '--json') +++ local args +++ transformed_files=() +++ local transformed_files +++ arch_scheduling_enabled +++ [[ -n '' ]] +++ arch_scheduling_enabled +++ [[ -n '' ]] ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl exec -n monitoring-2-0-3915 monitoring-rs0-1 -c pmm-client -- pmm-admin status --json +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 -a -n 1 ']' +++ break +++ cat /tmp/tmp.neR4WjHCNw +++ cat /tmp/tmp.nNqptLjSAg +++ rm /tmp/tmp.neR4WjHCNw /tmp/tmp.nNqptLjSAg +++ return 0 ++ for instance in $(kubectl_bin get pods --no-headers -l app.kubernetes.io/name=percona-server-mongodb --output=custom-columns='NAME:.metadata.name') ++ nodeList+=($(kubectl_bin exec -n "$namespace" $instance -c pmm-client -- pmm-admin status --json | jq -r '.pmm_agent_status.node_id')) +++ kubectl_bin exec -n monitoring-2-0-3915 monitoring-rs0-2 -c pmm-client -- pmm-admin status --json +++ jq -r .pmm_agent_status.node_id ++++ mktemp +++ local LAST_OUT=/tmp/tmp.MRt1kfzpmM ++++ mktemp +++ local LAST_ERR=/tmp/tmp.LgMA4dKfxP +++ local exit_status=0 +++ local timeout=4 +++ args=('exec' '-n' 'monitoring-2-0-3915' 'monitoring-rs0-2' '-c' 'pmm-client' '--' 'pmm-admin' 'status' '--json') +++ local args +++ transformed_files=() +++ local transformed_files +++ arch_scheduling_enabled +++ [[ -n '' ]] +++ arch_scheduling_enabled +++ [[ -n '' ]] ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl exec -n monitoring-2-0-3915 monitoring-rs0-2 -c pmm-client -- pmm-admin status --json +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 -a -n 1 ']' +++ break +++ cat /tmp/tmp.MRt1kfzpmM +++ cat /tmp/tmp.LgMA4dKfxP +++ rm /tmp/tmp.MRt1kfzpmM /tmp/tmp.LgMA4dKfxP +++ return 0 ++ echo /node_id/e08d87ef-3039-42e6-9092-b224c41e105c /node_id/1eb31565-cd10-4bb8-8ecd-1976ed118ffc /node_id/22074519-ebbf-4a11-a176-b9e4e542890e /node_id/48bc7d6e-7a03-4b5e-bb39-9406e8e3d1a1 /node_id/a11fcb41-9ff3-48c6-b42f-5dd7fa564072 /node_id/97149e9a-6662-4e19-93b7-af0d52a1a931 /node_id/1c34ee8d-5a3a-4e4d-ad5f-0cec3f08d361 /node_id/93f9f49a-3b54-42df-bc82-c83b0377f714 /node_id/d811f888-5dac-4bbb-9685-2d9ebc0a7a62 + nodeList_from_pmm=($(does_node_id_exists "${nodeList[@]}")) ++ does_node_id_exists /node_id/e08d87ef-3039-42e6-9092-b224c41e105c /node_id/1eb31565-cd10-4bb8-8ecd-1976ed118ffc /node_id/22074519-ebbf-4a11-a176-b9e4e542890e /node_id/48bc7d6e-7a03-4b5e-bb39-9406e8e3d1a1 /node_id/a11fcb41-9ff3-48c6-b42f-5dd7fa564072 /node_id/97149e9a-6662-4e19-93b7-af0d52a1a931 /node_id/1c34ee8d-5a3a-4e4d-ad5f-0cec3f08d361 /node_id/93f9f49a-3b54-42df-bc82-c83b0377f714 /node_id/d811f888-5dac-4bbb-9685-2d9ebc0a7a62 ++ nodeList=('/node_id/e08d87ef-3039-42e6-9092-b224c41e105c' '/node_id/1eb31565-cd10-4bb8-8ecd-1976ed118ffc' '/node_id/22074519-ebbf-4a11-a176-b9e4e542890e' '/node_id/48bc7d6e-7a03-4b5e-bb39-9406e8e3d1a1' '/node_id/a11fcb41-9ff3-48c6-b42f-5dd7fa564072' '/node_id/97149e9a-6662-4e19-93b7-af0d52a1a931' '/node_id/1c34ee8d-5a3a-4e4d-ad5f-0cec3f08d361' '/node_id/93f9f49a-3b54-42df-bc82-c83b0377f714' '/node_id/d811f888-5dac-4bbb-9685-2d9ebc0a7a62') ++ local -a nodeList ++ nodeList_from_pmm=() ++ local -a nodeList_from_pmm ++ local endpoint ++ local encoded_admin_userinfo +++ get_service_endpoint monitoring-service +++ local service=monitoring-service +++ local endpoint +++ local service_type ++++ kubectl_bin get service/monitoring-service -o 'jsonpath={.spec.type}' +++++ mktemp ++++ local LAST_OUT=/tmp/tmp.MSEsIj7Brc +++++ mktemp ++++ local LAST_ERR=/tmp/tmp.fCr9dGXoVN ++++ local exit_status=0 ++++ local timeout=4 ++++ args=('get' 'service/monitoring-service' '-o' 'jsonpath={.spec.type}') ++++ local args ++++ transformed_files=() ++++ local transformed_files ++++ arch_scheduling_enabled ++++ [[ -n '' ]] ++++ arch_scheduling_enabled ++++ [[ -n '' ]] +++++ seq 0 2 ++++ for i in $(seq 0 2) ++++ set +e ++++ kubectl get service/monitoring-service -o 'jsonpath={.spec.type}' ++++ exit_status=0 ++++ set -e ++++ '[' 0 '!=' 0 -a -n 1 ']' ++++ break ++++ cat /tmp/tmp.MSEsIj7Brc ++++ cat /tmp/tmp.fCr9dGXoVN ++++ rm /tmp/tmp.MSEsIj7Brc /tmp/tmp.fCr9dGXoVN ++++ return 0 +++ service_type=LoadBalancer +++ [[ LoadBalancer == LoadBalancer ]] ++++ get_load_balancer_endpoint monitoring-service ++++ local service=monitoring-service ++++ local endpoint +++++ kubectl_bin get service/monitoring-service -o json +++++ jq -r '[.status.loadBalancer.ingress[]?.hostname, .status.loadBalancer.ingress[]?.ip] | map(select(. != null and . != "")) | first // empty' ++++++ mktemp +++++ local LAST_OUT=/tmp/tmp.XOcUBbeGmu ++++++ mktemp +++++ local LAST_ERR=/tmp/tmp.mQd6IazjDS +++++ local exit_status=0 +++++ local timeout=4 +++++ args=('get' 'service/monitoring-service' '-o' 'json') +++++ local args +++++ transformed_files=() +++++ local transformed_files +++++ arch_scheduling_enabled +++++ [[ -n '' ]] +++++ arch_scheduling_enabled +++++ [[ -n '' ]] ++++++ seq 0 2 +++++ for i in $(seq 0 2) +++++ set +e +++++ kubectl get service/monitoring-service -o json +++++ exit_status=0 +++++ set -e +++++ '[' 0 '!=' 0 -a -n 1 ']' +++++ break +++++ cat /tmp/tmp.XOcUBbeGmu +++++ cat /tmp/tmp.mQd6IazjDS +++++ rm /tmp/tmp.XOcUBbeGmu /tmp/tmp.mQd6IazjDS +++++ return 0 ++++ endpoint=34.28.217.84 ++++ '[' -n 34.28.217.84 -a 34.28.217.84 '!=' null ']' ++++ echo 34.28.217.84 ++++ return +++ endpoint=34.28.217.84 +++ echo 34.28.217.84 +++ return ++ endpoint=34.28.217.84 +++ urlencode_userinfo admin:admin +++ local user_pass=admin:admin +++ local username=admin +++ local password=admin ++++ urlencode admin ++++ uri=admin ++++ echo -n admin ++++ jq -s -R -r @uri ++++ urlencode admin ++++ uri=admin ++++ echo -n admin ++++ jq -s -R -r @uri +++ printf %s:%s admin admin ++ encoded_admin_userinfo=admin:admin ++ for node_id in "${nodeList[@]}" ++ nodeList_from_pmm+=($(kubectl_bin exec -n "${namespace}" monitoring-0 -- pmm-admin --server-url=https://${encoded_admin_userinfo}@${endpoint}/ --server-insecure-tls inventory list nodes --node-type=CONTAINER_NODE | grep $node_id | awk '{print $4}')) +++ kubectl_bin exec -n monitoring-2-0-3915 monitoring-0 -- pmm-admin --server-url=https://admin:admin@34.28.217.84/ --server-insecure-tls inventory list nodes --node-type=CONTAINER_NODE +++ grep /node_id/e08d87ef-3039-42e6-9092-b224c41e105c +++ awk '{print $4}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.WB89gYouTI ++++ mktemp +++ local LAST_ERR=/tmp/tmp.ibDiYqr6cg +++ local exit_status=0 +++ local timeout=4 +++ args=('exec' '-n' 'monitoring-2-0-3915' 'monitoring-0' '--' 'pmm-admin' '--server-url=https://admin:admin@34.28.217.84/' '--server-insecure-tls' 'inventory' 'list' 'nodes' '--node-type=CONTAINER_NODE') +++ local args +++ transformed_files=() +++ local transformed_files +++ arch_scheduling_enabled +++ [[ -n '' ]] +++ arch_scheduling_enabled +++ [[ -n '' ]] ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl exec -n monitoring-2-0-3915 monitoring-0 -- pmm-admin --server-url=https://admin:admin@34.28.217.84/ --server-insecure-tls inventory list nodes --node-type=CONTAINER_NODE +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 -a -n 1 ']' +++ break +++ cat /tmp/tmp.WB89gYouTI +++ cat /tmp/tmp.ibDiYqr6cg +++ rm /tmp/tmp.WB89gYouTI /tmp/tmp.ibDiYqr6cg +++ return 0 ++ for node_id in "${nodeList[@]}" ++ nodeList_from_pmm+=($(kubectl_bin exec -n "${namespace}" monitoring-0 -- pmm-admin --server-url=https://${encoded_admin_userinfo}@${endpoint}/ --server-insecure-tls inventory list nodes --node-type=CONTAINER_NODE | grep $node_id | awk '{print $4}')) +++ grep /node_id/1eb31565-cd10-4bb8-8ecd-1976ed118ffc +++ kubectl_bin exec -n monitoring-2-0-3915 monitoring-0 -- pmm-admin --server-url=https://admin:admin@34.28.217.84/ --server-insecure-tls inventory list nodes --node-type=CONTAINER_NODE +++ awk '{print $4}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.37LHIZ4KeW ++++ mktemp +++ local LAST_ERR=/tmp/tmp.4X7JoeXPyp +++ local exit_status=0 +++ local timeout=4 +++ args=('exec' '-n' 'monitoring-2-0-3915' 'monitoring-0' '--' 'pmm-admin' '--server-url=https://admin:admin@34.28.217.84/' '--server-insecure-tls' 'inventory' 'list' 'nodes' '--node-type=CONTAINER_NODE') +++ local args +++ transformed_files=() +++ local transformed_files +++ arch_scheduling_enabled +++ [[ -n '' ]] +++ arch_scheduling_enabled +++ [[ -n '' ]] ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl exec -n monitoring-2-0-3915 monitoring-0 -- pmm-admin --server-url=https://admin:admin@34.28.217.84/ --server-insecure-tls inventory list nodes --node-type=CONTAINER_NODE +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 -a -n 1 ']' +++ break +++ cat /tmp/tmp.37LHIZ4KeW +++ cat /tmp/tmp.4X7JoeXPyp +++ rm /tmp/tmp.37LHIZ4KeW /tmp/tmp.4X7JoeXPyp +++ return 0 ++ for node_id in "${nodeList[@]}" ++ nodeList_from_pmm+=($(kubectl_bin exec -n "${namespace}" monitoring-0 -- pmm-admin --server-url=https://${encoded_admin_userinfo}@${endpoint}/ --server-insecure-tls inventory list nodes --node-type=CONTAINER_NODE | grep $node_id | awk '{print $4}')) +++ kubectl_bin exec -n monitoring-2-0-3915 monitoring-0 -- pmm-admin --server-url=https://admin:admin@34.28.217.84/ --server-insecure-tls inventory list nodes --node-type=CONTAINER_NODE +++ grep /node_id/22074519-ebbf-4a11-a176-b9e4e542890e +++ awk '{print $4}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.CkVuabfH39 ++++ mktemp +++ local LAST_ERR=/tmp/tmp.OFeBMWnXjt +++ local exit_status=0 +++ local timeout=4 +++ args=('exec' '-n' 'monitoring-2-0-3915' 'monitoring-0' '--' 'pmm-admin' '--server-url=https://admin:admin@34.28.217.84/' '--server-insecure-tls' 'inventory' 'list' 'nodes' '--node-type=CONTAINER_NODE') +++ local args +++ transformed_files=() +++ local transformed_files +++ arch_scheduling_enabled +++ [[ -n '' ]] +++ arch_scheduling_enabled +++ [[ -n '' ]] ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl exec -n monitoring-2-0-3915 monitoring-0 -- pmm-admin --server-url=https://admin:admin@34.28.217.84/ --server-insecure-tls inventory list nodes --node-type=CONTAINER_NODE +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 -a -n 1 ']' +++ break +++ cat /tmp/tmp.CkVuabfH39 +++ cat /tmp/tmp.OFeBMWnXjt +++ rm /tmp/tmp.CkVuabfH39 /tmp/tmp.OFeBMWnXjt +++ return 0 ++ for node_id in "${nodeList[@]}" ++ nodeList_from_pmm+=($(kubectl_bin exec -n "${namespace}" monitoring-0 -- pmm-admin --server-url=https://${encoded_admin_userinfo}@${endpoint}/ --server-insecure-tls inventory list nodes --node-type=CONTAINER_NODE | grep $node_id | awk '{print $4}')) +++ grep /node_id/48bc7d6e-7a03-4b5e-bb39-9406e8e3d1a1 +++ kubectl_bin exec -n monitoring-2-0-3915 monitoring-0 -- pmm-admin --server-url=https://admin:admin@34.28.217.84/ --server-insecure-tls inventory list nodes --node-type=CONTAINER_NODE +++ awk '{print $4}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.2g93lWI7wC ++++ mktemp +++ local LAST_ERR=/tmp/tmp.qySpwgPC0u +++ local exit_status=0 +++ local timeout=4 +++ args=('exec' '-n' 'monitoring-2-0-3915' 'monitoring-0' '--' 'pmm-admin' '--server-url=https://admin:admin@34.28.217.84/' '--server-insecure-tls' 'inventory' 'list' 'nodes' '--node-type=CONTAINER_NODE') +++ local args +++ transformed_files=() +++ local transformed_files +++ arch_scheduling_enabled +++ [[ -n '' ]] +++ arch_scheduling_enabled +++ [[ -n '' ]] ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl exec -n monitoring-2-0-3915 monitoring-0 -- pmm-admin --server-url=https://admin:admin@34.28.217.84/ --server-insecure-tls inventory list nodes --node-type=CONTAINER_NODE +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 -a -n 1 ']' +++ break +++ cat /tmp/tmp.2g93lWI7wC +++ cat /tmp/tmp.qySpwgPC0u +++ rm /tmp/tmp.2g93lWI7wC /tmp/tmp.qySpwgPC0u +++ return 0 ++ for node_id in "${nodeList[@]}" ++ nodeList_from_pmm+=($(kubectl_bin exec -n "${namespace}" monitoring-0 -- pmm-admin --server-url=https://${encoded_admin_userinfo}@${endpoint}/ --server-insecure-tls inventory list nodes --node-type=CONTAINER_NODE | grep $node_id | awk '{print $4}')) +++ kubectl_bin exec -n monitoring-2-0-3915 monitoring-0 -- pmm-admin --server-url=https://admin:admin@34.28.217.84/ --server-insecure-tls inventory list nodes --node-type=CONTAINER_NODE +++ grep /node_id/a11fcb41-9ff3-48c6-b42f-5dd7fa564072 +++ awk '{print $4}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.M2g8J8mY4B ++++ mktemp +++ local LAST_ERR=/tmp/tmp.R7bFsa76Wv +++ local exit_status=0 +++ local timeout=4 +++ args=('exec' '-n' 'monitoring-2-0-3915' 'monitoring-0' '--' 'pmm-admin' '--server-url=https://admin:admin@34.28.217.84/' '--server-insecure-tls' 'inventory' 'list' 'nodes' '--node-type=CONTAINER_NODE') +++ local args +++ transformed_files=() +++ local transformed_files +++ arch_scheduling_enabled +++ [[ -n '' ]] +++ arch_scheduling_enabled +++ [[ -n '' ]] ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl exec -n monitoring-2-0-3915 monitoring-0 -- pmm-admin --server-url=https://admin:admin@34.28.217.84/ --server-insecure-tls inventory list nodes --node-type=CONTAINER_NODE +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 -a -n 1 ']' +++ break +++ cat /tmp/tmp.M2g8J8mY4B +++ cat /tmp/tmp.R7bFsa76Wv +++ rm /tmp/tmp.M2g8J8mY4B /tmp/tmp.R7bFsa76Wv +++ return 0 ++ for node_id in "${nodeList[@]}" ++ nodeList_from_pmm+=($(kubectl_bin exec -n "${namespace}" monitoring-0 -- pmm-admin --server-url=https://${encoded_admin_userinfo}@${endpoint}/ --server-insecure-tls inventory list nodes --node-type=CONTAINER_NODE | grep $node_id | awk '{print $4}')) +++ kubectl_bin exec -n monitoring-2-0-3915 monitoring-0 -- pmm-admin --server-url=https://admin:admin@34.28.217.84/ --server-insecure-tls inventory list nodes --node-type=CONTAINER_NODE +++ grep /node_id/97149e9a-6662-4e19-93b7-af0d52a1a931 +++ awk '{print $4}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.axXKC3uRuc ++++ mktemp +++ local LAST_ERR=/tmp/tmp.mWL9O36XI6 +++ local exit_status=0 +++ local timeout=4 +++ args=('exec' '-n' 'monitoring-2-0-3915' 'monitoring-0' '--' 'pmm-admin' '--server-url=https://admin:admin@34.28.217.84/' '--server-insecure-tls' 'inventory' 'list' 'nodes' '--node-type=CONTAINER_NODE') +++ local args +++ transformed_files=() +++ local transformed_files +++ arch_scheduling_enabled +++ [[ -n '' ]] +++ arch_scheduling_enabled +++ [[ -n '' ]] ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl exec -n monitoring-2-0-3915 monitoring-0 -- pmm-admin --server-url=https://admin:admin@34.28.217.84/ --server-insecure-tls inventory list nodes --node-type=CONTAINER_NODE +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 -a -n 1 ']' +++ break +++ cat /tmp/tmp.axXKC3uRuc +++ cat /tmp/tmp.mWL9O36XI6 +++ rm /tmp/tmp.axXKC3uRuc /tmp/tmp.mWL9O36XI6 +++ return 0 ++ for node_id in "${nodeList[@]}" ++ nodeList_from_pmm+=($(kubectl_bin exec -n "${namespace}" monitoring-0 -- pmm-admin --server-url=https://${encoded_admin_userinfo}@${endpoint}/ --server-insecure-tls inventory list nodes --node-type=CONTAINER_NODE | grep $node_id | awk '{print $4}')) +++ kubectl_bin exec -n monitoring-2-0-3915 monitoring-0 -- pmm-admin --server-url=https://admin:admin@34.28.217.84/ --server-insecure-tls inventory list nodes --node-type=CONTAINER_NODE +++ grep /node_id/1c34ee8d-5a3a-4e4d-ad5f-0cec3f08d361 +++ awk '{print $4}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.CzIe3RbrO9 ++++ mktemp +++ local LAST_ERR=/tmp/tmp.BSOWG8rzOF +++ local exit_status=0 +++ local timeout=4 +++ args=('exec' '-n' 'monitoring-2-0-3915' 'monitoring-0' '--' 'pmm-admin' '--server-url=https://admin:admin@34.28.217.84/' '--server-insecure-tls' 'inventory' 'list' 'nodes' '--node-type=CONTAINER_NODE') +++ local args +++ transformed_files=() +++ local transformed_files +++ arch_scheduling_enabled +++ [[ -n '' ]] +++ arch_scheduling_enabled +++ [[ -n '' ]] ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl exec -n monitoring-2-0-3915 monitoring-0 -- pmm-admin --server-url=https://admin:admin@34.28.217.84/ --server-insecure-tls inventory list nodes --node-type=CONTAINER_NODE +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 -a -n 1 ']' +++ break +++ cat /tmp/tmp.CzIe3RbrO9 +++ cat /tmp/tmp.BSOWG8rzOF +++ rm /tmp/tmp.CzIe3RbrO9 /tmp/tmp.BSOWG8rzOF +++ return 0 ++ for node_id in "${nodeList[@]}" ++ nodeList_from_pmm+=($(kubectl_bin exec -n "${namespace}" monitoring-0 -- pmm-admin --server-url=https://${encoded_admin_userinfo}@${endpoint}/ --server-insecure-tls inventory list nodes --node-type=CONTAINER_NODE | grep $node_id | awk '{print $4}')) +++ grep /node_id/93f9f49a-3b54-42df-bc82-c83b0377f714 +++ kubectl_bin exec -n monitoring-2-0-3915 monitoring-0 -- pmm-admin --server-url=https://admin:admin@34.28.217.84/ --server-insecure-tls inventory list nodes --node-type=CONTAINER_NODE +++ awk '{print $4}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.0OIm51N6LP ++++ mktemp +++ local LAST_ERR=/tmp/tmp.CSnXkuD095 +++ local exit_status=0 +++ local timeout=4 +++ args=('exec' '-n' 'monitoring-2-0-3915' 'monitoring-0' '--' 'pmm-admin' '--server-url=https://admin:admin@34.28.217.84/' '--server-insecure-tls' 'inventory' 'list' 'nodes' '--node-type=CONTAINER_NODE') +++ local args +++ transformed_files=() +++ local transformed_files +++ arch_scheduling_enabled +++ [[ -n '' ]] +++ arch_scheduling_enabled +++ [[ -n '' ]] ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl exec -n monitoring-2-0-3915 monitoring-0 -- pmm-admin --server-url=https://admin:admin@34.28.217.84/ --server-insecure-tls inventory list nodes --node-type=CONTAINER_NODE +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 -a -n 1 ']' +++ break +++ cat /tmp/tmp.0OIm51N6LP +++ cat /tmp/tmp.CSnXkuD095 +++ rm /tmp/tmp.0OIm51N6LP /tmp/tmp.CSnXkuD095 +++ return 0 ++ for node_id in "${nodeList[@]}" ++ nodeList_from_pmm+=($(kubectl_bin exec -n "${namespace}" monitoring-0 -- pmm-admin --server-url=https://${encoded_admin_userinfo}@${endpoint}/ --server-insecure-tls inventory list nodes --node-type=CONTAINER_NODE | grep $node_id | awk '{print $4}')) +++ kubectl_bin exec -n monitoring-2-0-3915 monitoring-0 -- pmm-admin --server-url=https://admin:admin@34.28.217.84/ --server-insecure-tls inventory list nodes --node-type=CONTAINER_NODE +++ grep /node_id/d811f888-5dac-4bbb-9685-2d9ebc0a7a62 +++ awk '{print $4}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.kIEKAetSuQ ++++ mktemp +++ local LAST_ERR=/tmp/tmp.P0qD2M8A7U +++ local exit_status=0 +++ local timeout=4 +++ args=('exec' '-n' 'monitoring-2-0-3915' 'monitoring-0' '--' 'pmm-admin' '--server-url=https://admin:admin@34.28.217.84/' '--server-insecure-tls' 'inventory' 'list' 'nodes' '--node-type=CONTAINER_NODE') +++ local args +++ transformed_files=() +++ local transformed_files +++ arch_scheduling_enabled +++ [[ -n '' ]] +++ arch_scheduling_enabled +++ [[ -n '' ]] ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl exec -n monitoring-2-0-3915 monitoring-0 -- pmm-admin --server-url=https://admin:admin@34.28.217.84/ --server-insecure-tls inventory list nodes --node-type=CONTAINER_NODE +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 -a -n 1 ']' +++ break +++ cat /tmp/tmp.kIEKAetSuQ +++ cat /tmp/tmp.P0qD2M8A7U +++ rm /tmp/tmp.kIEKAetSuQ /tmp/tmp.P0qD2M8A7U +++ return 0 ++ echo /node_id/e08d87ef-3039-42e6-9092-b224c41e105c /node_id/1eb31565-cd10-4bb8-8ecd-1976ed118ffc /node_id/22074519-ebbf-4a11-a176-b9e4e542890e /node_id/48bc7d6e-7a03-4b5e-bb39-9406e8e3d1a1 /node_id/a11fcb41-9ff3-48c6-b42f-5dd7fa564072 /node_id/97149e9a-6662-4e19-93b7-af0d52a1a931 /node_id/1c34ee8d-5a3a-4e4d-ad5f-0cec3f08d361 /node_id/93f9f49a-3b54-42df-bc82-c83b0377f714 /node_id/d811f888-5dac-4bbb-9685-2d9ebc0a7a62 + for node_id in "${nodeList_from_pmm[@]}" + '[' -z /node_id/e08d87ef-3039-42e6-9092-b224c41e105c ']' + for node_id in "${nodeList_from_pmm[@]}" + '[' -z /node_id/1eb31565-cd10-4bb8-8ecd-1976ed118ffc ']' + for node_id in "${nodeList_from_pmm[@]}" + '[' -z /node_id/22074519-ebbf-4a11-a176-b9e4e542890e ']' + for node_id in "${nodeList_from_pmm[@]}" + '[' -z /node_id/48bc7d6e-7a03-4b5e-bb39-9406e8e3d1a1 ']' + for node_id in "${nodeList_from_pmm[@]}" + '[' -z /node_id/a11fcb41-9ff3-48c6-b42f-5dd7fa564072 ']' + for node_id in "${nodeList_from_pmm[@]}" + '[' -z /node_id/97149e9a-6662-4e19-93b7-af0d52a1a931 ']' + for node_id in "${nodeList_from_pmm[@]}" + '[' -z /node_id/1c34ee8d-5a3a-4e4d-ad5f-0cec3f08d361 ']' + for node_id in "${nodeList_from_pmm[@]}" + '[' -z /node_id/93f9f49a-3b54-42df-bc82-c83b0377f714 ']' + for node_id in "${nodeList_from_pmm[@]}" + '[' -z /node_id/d811f888-5dac-4bbb-9685-2d9ebc0a7a62 ']' + kubectl_bin patch psmdb monitoring --type json '-p=[{"op":"add","path":"/spec/pause","value":true}]' ++ mktemp + local LAST_OUT=/tmp/tmp.Z8HqQcrSDU ++ mktemp + local LAST_ERR=/tmp/tmp.fAO6nsfC6Q + local exit_status=0 + local timeout=4 + args=('patch' 'psmdb' 'monitoring' '--type' 'json' '-p=[{"op":"add","path":"/spec/pause","value":true}]') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl patch psmdb monitoring --type json '-p=[{"op":"add","path":"/spec/pause","value":true}]' + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.Z8HqQcrSDU perconaservermongodb.psmdb.percona.com/monitoring patched + cat /tmp/tmp.fAO6nsfC6Q + rm /tmp/tmp.Z8HqQcrSDU /tmp/tmp.fAO6nsfC6Q + return 0 + wait_for_delete pod/monitoring-mongos-0 + local res=pod/monitoring-mongos-0 + local wait_time=60 + set +o xtrace waiting for pod/monitoring-mongos-0 to be deleted......................Error from server (NotFound): pods "monitoring-mongos-0" not found Error from server (NotFound): pods "monitoring-mongos-0" not found Error from server (NotFound): pods "monitoring-mongos-0" not found Error from server (NotFound): pods "monitoring-mongos-0" not found + wait_for_delete pod/monitoring-rs0-0 + local res=pod/monitoring-rs0-0 + local wait_time=60 + set +o xtrace waiting for pod/monitoring-rs0-0 to be deleted.......Error from server (NotFound): pods "monitoring-rs0-0" not found Error from server (NotFound): pods "monitoring-rs0-0" not found Error from server (NotFound): pods "monitoring-rs0-0" not found Error from server (NotFound): pods "monitoring-rs0-0" not found + wait_for_delete pod/monitoring-cfg-0 + local res=pod/monitoring-cfg-0 + local wait_time=60 + set +o xtrace waiting for pod/monitoring-cfg-0 to be deleted.....Error from server (NotFound): pods "monitoring-cfg-0" not found Error from server (NotFound): pods "monitoring-cfg-0" not found Error from server (NotFound): pods "monitoring-cfg-0" not found Error from server (NotFound): pods "monitoring-cfg-0" not found + desc 'check if services are not deleted' + set +o xtrace ----------------------------------------------------------------------------------- check if services are not deleted ----------------------------------------------------------------------------------- + kubectl_bin get svc monitoring-rs0 ++ mktemp + local LAST_OUT=/tmp/tmp.iqLKPPtrNB ++ mktemp + local LAST_ERR=/tmp/tmp.sNmEMiue9C + local exit_status=0 + local timeout=4 + args=('get' 'svc' 'monitoring-rs0') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl get svc monitoring-rs0 + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.iqLKPPtrNB NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE monitoring-rs0 ClusterIP None 27019/TCP 14m + cat /tmp/tmp.sNmEMiue9C + rm /tmp/tmp.iqLKPPtrNB /tmp/tmp.sNmEMiue9C + return 0 + kubectl_bin get svc monitoring-cfg ++ mktemp + local LAST_OUT=/tmp/tmp.HanNcTFlnV ++ mktemp + local LAST_ERR=/tmp/tmp.qWDI1Qto7G + local exit_status=0 + local timeout=4 + args=('get' 'svc' 'monitoring-cfg') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl get svc monitoring-cfg + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.HanNcTFlnV NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE monitoring-cfg ClusterIP None 27019/TCP 14m + cat /tmp/tmp.qWDI1Qto7G + rm /tmp/tmp.HanNcTFlnV /tmp/tmp.qWDI1Qto7G + return 0 + kubectl_bin get svc monitoring-mongos ++ mktemp + local LAST_OUT=/tmp/tmp.TUbHEZ4dsX ++ mktemp + local LAST_ERR=/tmp/tmp.XECNKPTDkQ + local exit_status=0 + local timeout=4 + args=('get' 'svc' 'monitoring-mongos') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl get svc monitoring-mongos + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.TUbHEZ4dsX NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE monitoring-mongos ClusterIP 34.118.228.44 27019/TCP 14m + cat /tmp/tmp.XECNKPTDkQ + rm /tmp/tmp.TUbHEZ4dsX /tmp/tmp.XECNKPTDkQ + return 0 + does_node_id_exists_in_pmm=($(does_node_id_exists "${nodeList[@]}")) ++ does_node_id_exists /node_id/e08d87ef-3039-42e6-9092-b224c41e105c /node_id/1eb31565-cd10-4bb8-8ecd-1976ed118ffc /node_id/22074519-ebbf-4a11-a176-b9e4e542890e /node_id/48bc7d6e-7a03-4b5e-bb39-9406e8e3d1a1 /node_id/a11fcb41-9ff3-48c6-b42f-5dd7fa564072 /node_id/97149e9a-6662-4e19-93b7-af0d52a1a931 /node_id/1c34ee8d-5a3a-4e4d-ad5f-0cec3f08d361 /node_id/93f9f49a-3b54-42df-bc82-c83b0377f714 /node_id/d811f888-5dac-4bbb-9685-2d9ebc0a7a62 ++ nodeList=('/node_id/e08d87ef-3039-42e6-9092-b224c41e105c' '/node_id/1eb31565-cd10-4bb8-8ecd-1976ed118ffc' '/node_id/22074519-ebbf-4a11-a176-b9e4e542890e' '/node_id/48bc7d6e-7a03-4b5e-bb39-9406e8e3d1a1' '/node_id/a11fcb41-9ff3-48c6-b42f-5dd7fa564072' '/node_id/97149e9a-6662-4e19-93b7-af0d52a1a931' '/node_id/1c34ee8d-5a3a-4e4d-ad5f-0cec3f08d361' '/node_id/93f9f49a-3b54-42df-bc82-c83b0377f714' '/node_id/d811f888-5dac-4bbb-9685-2d9ebc0a7a62') ++ local -a nodeList ++ nodeList_from_pmm=() ++ local -a nodeList_from_pmm ++ local endpoint ++ local encoded_admin_userinfo +++ get_service_endpoint monitoring-service +++ local service=monitoring-service +++ local endpoint +++ local service_type ++++ kubectl_bin get service/monitoring-service -o 'jsonpath={.spec.type}' +++++ mktemp ++++ local LAST_OUT=/tmp/tmp.4f3t24U4BK +++++ mktemp ++++ local LAST_ERR=/tmp/tmp.D3YOT0aagF ++++ local exit_status=0 ++++ local timeout=4 ++++ args=('get' 'service/monitoring-service' '-o' 'jsonpath={.spec.type}') ++++ local args ++++ transformed_files=() ++++ local transformed_files ++++ arch_scheduling_enabled ++++ [[ -n '' ]] ++++ arch_scheduling_enabled ++++ [[ -n '' ]] +++++ seq 0 2 ++++ for i in $(seq 0 2) ++++ set +e ++++ kubectl get service/monitoring-service -o 'jsonpath={.spec.type}' ++++ exit_status=0 ++++ set -e ++++ '[' 0 '!=' 0 -a -n 1 ']' ++++ break ++++ cat /tmp/tmp.4f3t24U4BK ++++ cat /tmp/tmp.D3YOT0aagF ++++ rm /tmp/tmp.4f3t24U4BK /tmp/tmp.D3YOT0aagF ++++ return 0 +++ service_type=LoadBalancer +++ [[ LoadBalancer == LoadBalancer ]] ++++ get_load_balancer_endpoint monitoring-service ++++ local service=monitoring-service ++++ local endpoint +++++ kubectl_bin get service/monitoring-service -o json +++++ jq -r '[.status.loadBalancer.ingress[]?.hostname, .status.loadBalancer.ingress[]?.ip] | map(select(. != null and . != "")) | first // empty' ++++++ mktemp +++++ local LAST_OUT=/tmp/tmp.C8BY1Ebq32 ++++++ mktemp +++++ local LAST_ERR=/tmp/tmp.vgwao6e9AB +++++ local exit_status=0 +++++ local timeout=4 +++++ args=('get' 'service/monitoring-service' '-o' 'json') +++++ local args +++++ transformed_files=() +++++ local transformed_files +++++ arch_scheduling_enabled +++++ [[ -n '' ]] +++++ arch_scheduling_enabled +++++ [[ -n '' ]] ++++++ seq 0 2 +++++ for i in $(seq 0 2) +++++ set +e +++++ kubectl get service/monitoring-service -o json +++++ exit_status=0 +++++ set -e +++++ '[' 0 '!=' 0 -a -n 1 ']' +++++ break +++++ cat /tmp/tmp.C8BY1Ebq32 +++++ cat /tmp/tmp.vgwao6e9AB +++++ rm /tmp/tmp.C8BY1Ebq32 /tmp/tmp.vgwao6e9AB +++++ return 0 ++++ endpoint=34.28.217.84 ++++ '[' -n 34.28.217.84 -a 34.28.217.84 '!=' null ']' ++++ echo 34.28.217.84 ++++ return +++ endpoint=34.28.217.84 +++ echo 34.28.217.84 +++ return ++ endpoint=34.28.217.84 +++ urlencode_userinfo admin:admin +++ local user_pass=admin:admin +++ local username=admin +++ local password=admin ++++ urlencode admin ++++ uri=admin ++++ echo -n admin ++++ jq -s -R -r @uri ++++ urlencode admin ++++ uri=admin ++++ echo -n admin ++++ jq -s -R -r @uri +++ printf %s:%s admin admin ++ encoded_admin_userinfo=admin:admin ++ for node_id in "${nodeList[@]}" ++ nodeList_from_pmm+=($(kubectl_bin exec -n "${namespace}" monitoring-0 -- pmm-admin --server-url=https://${encoded_admin_userinfo}@${endpoint}/ --server-insecure-tls inventory list nodes --node-type=CONTAINER_NODE | grep $node_id | awk '{print $4}')) +++ kubectl_bin exec -n monitoring-2-0-3915 monitoring-0 -- pmm-admin --server-url=https://admin:admin@34.28.217.84/ --server-insecure-tls inventory list nodes --node-type=CONTAINER_NODE +++ grep /node_id/e08d87ef-3039-42e6-9092-b224c41e105c ++++ mktemp +++ awk '{print $4}' +++ local LAST_OUT=/tmp/tmp.4wvMr3rgbd ++++ mktemp +++ local LAST_ERR=/tmp/tmp.xh7HhRvjce +++ local exit_status=0 +++ local timeout=4 +++ args=('exec' '-n' 'monitoring-2-0-3915' 'monitoring-0' '--' 'pmm-admin' '--server-url=https://admin:admin@34.28.217.84/' '--server-insecure-tls' 'inventory' 'list' 'nodes' '--node-type=CONTAINER_NODE') +++ local args +++ transformed_files=() +++ local transformed_files +++ arch_scheduling_enabled +++ [[ -n '' ]] +++ arch_scheduling_enabled +++ [[ -n '' ]] ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl exec -n monitoring-2-0-3915 monitoring-0 -- pmm-admin --server-url=https://admin:admin@34.28.217.84/ --server-insecure-tls inventory list nodes --node-type=CONTAINER_NODE +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 -a -n 1 ']' +++ break +++ cat /tmp/tmp.4wvMr3rgbd +++ cat /tmp/tmp.xh7HhRvjce +++ rm /tmp/tmp.4wvMr3rgbd /tmp/tmp.xh7HhRvjce +++ return 0 ++ for node_id in "${nodeList[@]}" ++ nodeList_from_pmm+=($(kubectl_bin exec -n "${namespace}" monitoring-0 -- pmm-admin --server-url=https://${encoded_admin_userinfo}@${endpoint}/ --server-insecure-tls inventory list nodes --node-type=CONTAINER_NODE | grep $node_id | awk '{print $4}')) +++ kubectl_bin exec -n monitoring-2-0-3915 monitoring-0 -- pmm-admin --server-url=https://admin:admin@34.28.217.84/ --server-insecure-tls inventory list nodes --node-type=CONTAINER_NODE +++ grep /node_id/1eb31565-cd10-4bb8-8ecd-1976ed118ffc +++ awk '{print $4}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.PXKTJOKvpH ++++ mktemp +++ local LAST_ERR=/tmp/tmp.xjKWafW9EN +++ local exit_status=0 +++ local timeout=4 +++ args=('exec' '-n' 'monitoring-2-0-3915' 'monitoring-0' '--' 'pmm-admin' '--server-url=https://admin:admin@34.28.217.84/' '--server-insecure-tls' 'inventory' 'list' 'nodes' '--node-type=CONTAINER_NODE') +++ local args +++ transformed_files=() +++ local transformed_files +++ arch_scheduling_enabled +++ [[ -n '' ]] +++ arch_scheduling_enabled +++ [[ -n '' ]] ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl exec -n monitoring-2-0-3915 monitoring-0 -- pmm-admin --server-url=https://admin:admin@34.28.217.84/ --server-insecure-tls inventory list nodes --node-type=CONTAINER_NODE +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 -a -n 1 ']' +++ break +++ cat /tmp/tmp.PXKTJOKvpH +++ cat /tmp/tmp.xjKWafW9EN +++ rm /tmp/tmp.PXKTJOKvpH /tmp/tmp.xjKWafW9EN +++ return 0 ++ for node_id in "${nodeList[@]}" ++ nodeList_from_pmm+=($(kubectl_bin exec -n "${namespace}" monitoring-0 -- pmm-admin --server-url=https://${encoded_admin_userinfo}@${endpoint}/ --server-insecure-tls inventory list nodes --node-type=CONTAINER_NODE | grep $node_id | awk '{print $4}')) +++ kubectl_bin exec -n monitoring-2-0-3915 monitoring-0 -- pmm-admin --server-url=https://admin:admin@34.28.217.84/ --server-insecure-tls inventory list nodes --node-type=CONTAINER_NODE +++ grep /node_id/22074519-ebbf-4a11-a176-b9e4e542890e +++ awk '{print $4}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.mMlSGQhwNJ ++++ mktemp +++ local LAST_ERR=/tmp/tmp.SzHICHumvL +++ local exit_status=0 +++ local timeout=4 +++ args=('exec' '-n' 'monitoring-2-0-3915' 'monitoring-0' '--' 'pmm-admin' '--server-url=https://admin:admin@34.28.217.84/' '--server-insecure-tls' 'inventory' 'list' 'nodes' '--node-type=CONTAINER_NODE') +++ local args +++ transformed_files=() +++ local transformed_files +++ arch_scheduling_enabled +++ [[ -n '' ]] +++ arch_scheduling_enabled +++ [[ -n '' ]] ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl exec -n monitoring-2-0-3915 monitoring-0 -- pmm-admin --server-url=https://admin:admin@34.28.217.84/ --server-insecure-tls inventory list nodes --node-type=CONTAINER_NODE +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 -a -n 1 ']' +++ break +++ cat /tmp/tmp.mMlSGQhwNJ +++ cat /tmp/tmp.SzHICHumvL +++ rm /tmp/tmp.mMlSGQhwNJ /tmp/tmp.SzHICHumvL +++ return 0 ++ for node_id in "${nodeList[@]}" ++ nodeList_from_pmm+=($(kubectl_bin exec -n "${namespace}" monitoring-0 -- pmm-admin --server-url=https://${encoded_admin_userinfo}@${endpoint}/ --server-insecure-tls inventory list nodes --node-type=CONTAINER_NODE | grep $node_id | awk '{print $4}')) +++ kubectl_bin exec -n monitoring-2-0-3915 monitoring-0 -- pmm-admin --server-url=https://admin:admin@34.28.217.84/ --server-insecure-tls inventory list nodes --node-type=CONTAINER_NODE +++ grep /node_id/48bc7d6e-7a03-4b5e-bb39-9406e8e3d1a1 +++ awk '{print $4}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.KiyTB6KpN1 ++++ mktemp +++ local LAST_ERR=/tmp/tmp.8VsXKPTVHJ +++ local exit_status=0 +++ local timeout=4 +++ args=('exec' '-n' 'monitoring-2-0-3915' 'monitoring-0' '--' 'pmm-admin' '--server-url=https://admin:admin@34.28.217.84/' '--server-insecure-tls' 'inventory' 'list' 'nodes' '--node-type=CONTAINER_NODE') +++ local args +++ transformed_files=() +++ local transformed_files +++ arch_scheduling_enabled +++ [[ -n '' ]] +++ arch_scheduling_enabled +++ [[ -n '' ]] ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl exec -n monitoring-2-0-3915 monitoring-0 -- pmm-admin --server-url=https://admin:admin@34.28.217.84/ --server-insecure-tls inventory list nodes --node-type=CONTAINER_NODE +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 -a -n 1 ']' +++ break +++ cat /tmp/tmp.KiyTB6KpN1 +++ cat /tmp/tmp.8VsXKPTVHJ +++ rm /tmp/tmp.KiyTB6KpN1 /tmp/tmp.8VsXKPTVHJ +++ return 0 ++ for node_id in "${nodeList[@]}" ++ nodeList_from_pmm+=($(kubectl_bin exec -n "${namespace}" monitoring-0 -- pmm-admin --server-url=https://${encoded_admin_userinfo}@${endpoint}/ --server-insecure-tls inventory list nodes --node-type=CONTAINER_NODE | grep $node_id | awk '{print $4}')) +++ kubectl_bin exec -n monitoring-2-0-3915 monitoring-0 -- pmm-admin --server-url=https://admin:admin@34.28.217.84/ --server-insecure-tls inventory list nodes --node-type=CONTAINER_NODE +++ grep /node_id/a11fcb41-9ff3-48c6-b42f-5dd7fa564072 +++ awk '{print $4}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.shpqeq4wVO ++++ mktemp +++ local LAST_ERR=/tmp/tmp.TCj5SWN7kC +++ local exit_status=0 +++ local timeout=4 +++ args=('exec' '-n' 'monitoring-2-0-3915' 'monitoring-0' '--' 'pmm-admin' '--server-url=https://admin:admin@34.28.217.84/' '--server-insecure-tls' 'inventory' 'list' 'nodes' '--node-type=CONTAINER_NODE') +++ local args +++ transformed_files=() +++ local transformed_files +++ arch_scheduling_enabled +++ [[ -n '' ]] +++ arch_scheduling_enabled +++ [[ -n '' ]] ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl exec -n monitoring-2-0-3915 monitoring-0 -- pmm-admin --server-url=https://admin:admin@34.28.217.84/ --server-insecure-tls inventory list nodes --node-type=CONTAINER_NODE +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 -a -n 1 ']' +++ break +++ cat /tmp/tmp.shpqeq4wVO +++ cat /tmp/tmp.TCj5SWN7kC +++ rm /tmp/tmp.shpqeq4wVO /tmp/tmp.TCj5SWN7kC +++ return 0 ++ for node_id in "${nodeList[@]}" ++ nodeList_from_pmm+=($(kubectl_bin exec -n "${namespace}" monitoring-0 -- pmm-admin --server-url=https://${encoded_admin_userinfo}@${endpoint}/ --server-insecure-tls inventory list nodes --node-type=CONTAINER_NODE | grep $node_id | awk '{print $4}')) +++ kubectl_bin exec -n monitoring-2-0-3915 monitoring-0 -- pmm-admin --server-url=https://admin:admin@34.28.217.84/ --server-insecure-tls inventory list nodes --node-type=CONTAINER_NODE +++ grep /node_id/97149e9a-6662-4e19-93b7-af0d52a1a931 +++ awk '{print $4}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.WcQglePdUx ++++ mktemp +++ local LAST_ERR=/tmp/tmp.1R5RupcqY6 +++ local exit_status=0 +++ local timeout=4 +++ args=('exec' '-n' 'monitoring-2-0-3915' 'monitoring-0' '--' 'pmm-admin' '--server-url=https://admin:admin@34.28.217.84/' '--server-insecure-tls' 'inventory' 'list' 'nodes' '--node-type=CONTAINER_NODE') +++ local args +++ transformed_files=() +++ local transformed_files +++ arch_scheduling_enabled +++ [[ -n '' ]] +++ arch_scheduling_enabled +++ [[ -n '' ]] ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl exec -n monitoring-2-0-3915 monitoring-0 -- pmm-admin --server-url=https://admin:admin@34.28.217.84/ --server-insecure-tls inventory list nodes --node-type=CONTAINER_NODE +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 -a -n 1 ']' +++ break +++ cat /tmp/tmp.WcQglePdUx +++ cat /tmp/tmp.1R5RupcqY6 +++ rm /tmp/tmp.WcQglePdUx /tmp/tmp.1R5RupcqY6 +++ return 0 ++ for node_id in "${nodeList[@]}" ++ nodeList_from_pmm+=($(kubectl_bin exec -n "${namespace}" monitoring-0 -- pmm-admin --server-url=https://${encoded_admin_userinfo}@${endpoint}/ --server-insecure-tls inventory list nodes --node-type=CONTAINER_NODE | grep $node_id | awk '{print $4}')) +++ kubectl_bin exec -n monitoring-2-0-3915 monitoring-0 -- pmm-admin --server-url=https://admin:admin@34.28.217.84/ --server-insecure-tls inventory list nodes --node-type=CONTAINER_NODE +++ grep /node_id/1c34ee8d-5a3a-4e4d-ad5f-0cec3f08d361 +++ awk '{print $4}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.elsVRWbgfM ++++ mktemp +++ local LAST_ERR=/tmp/tmp.t3ww2H77Bs +++ local exit_status=0 +++ local timeout=4 +++ args=('exec' '-n' 'monitoring-2-0-3915' 'monitoring-0' '--' 'pmm-admin' '--server-url=https://admin:admin@34.28.217.84/' '--server-insecure-tls' 'inventory' 'list' 'nodes' '--node-type=CONTAINER_NODE') +++ local args +++ transformed_files=() +++ local transformed_files +++ arch_scheduling_enabled +++ [[ -n '' ]] +++ arch_scheduling_enabled +++ [[ -n '' ]] ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl exec -n monitoring-2-0-3915 monitoring-0 -- pmm-admin --server-url=https://admin:admin@34.28.217.84/ --server-insecure-tls inventory list nodes --node-type=CONTAINER_NODE +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 -a -n 1 ']' +++ break +++ cat /tmp/tmp.elsVRWbgfM +++ cat /tmp/tmp.t3ww2H77Bs +++ rm /tmp/tmp.elsVRWbgfM /tmp/tmp.t3ww2H77Bs +++ return 0 ++ for node_id in "${nodeList[@]}" ++ nodeList_from_pmm+=($(kubectl_bin exec -n "${namespace}" monitoring-0 -- pmm-admin --server-url=https://${encoded_admin_userinfo}@${endpoint}/ --server-insecure-tls inventory list nodes --node-type=CONTAINER_NODE | grep $node_id | awk '{print $4}')) +++ kubectl_bin exec -n monitoring-2-0-3915 monitoring-0 -- pmm-admin --server-url=https://admin:admin@34.28.217.84/ --server-insecure-tls inventory list nodes --node-type=CONTAINER_NODE +++ grep /node_id/93f9f49a-3b54-42df-bc82-c83b0377f714 +++ awk '{print $4}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.JTqQcHeSEo ++++ mktemp +++ local LAST_ERR=/tmp/tmp.BJ9e3HAbt2 +++ local exit_status=0 +++ local timeout=4 +++ args=('exec' '-n' 'monitoring-2-0-3915' 'monitoring-0' '--' 'pmm-admin' '--server-url=https://admin:admin@34.28.217.84/' '--server-insecure-tls' 'inventory' 'list' 'nodes' '--node-type=CONTAINER_NODE') +++ local args +++ transformed_files=() +++ local transformed_files +++ arch_scheduling_enabled +++ [[ -n '' ]] +++ arch_scheduling_enabled +++ [[ -n '' ]] ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl exec -n monitoring-2-0-3915 monitoring-0 -- pmm-admin --server-url=https://admin:admin@34.28.217.84/ --server-insecure-tls inventory list nodes --node-type=CONTAINER_NODE +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 -a -n 1 ']' +++ break +++ cat /tmp/tmp.JTqQcHeSEo +++ cat /tmp/tmp.BJ9e3HAbt2 +++ rm /tmp/tmp.JTqQcHeSEo /tmp/tmp.BJ9e3HAbt2 +++ return 0 ++ for node_id in "${nodeList[@]}" ++ nodeList_from_pmm+=($(kubectl_bin exec -n "${namespace}" monitoring-0 -- pmm-admin --server-url=https://${encoded_admin_userinfo}@${endpoint}/ --server-insecure-tls inventory list nodes --node-type=CONTAINER_NODE | grep $node_id | awk '{print $4}')) +++ kubectl_bin exec -n monitoring-2-0-3915 monitoring-0 -- pmm-admin --server-url=https://admin:admin@34.28.217.84/ --server-insecure-tls inventory list nodes --node-type=CONTAINER_NODE +++ grep /node_id/d811f888-5dac-4bbb-9685-2d9ebc0a7a62 +++ awk '{print $4}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.3P7SnqmLXJ ++++ mktemp +++ local LAST_ERR=/tmp/tmp.EBqzqJF0hg +++ local exit_status=0 +++ local timeout=4 +++ args=('exec' '-n' 'monitoring-2-0-3915' 'monitoring-0' '--' 'pmm-admin' '--server-url=https://admin:admin@34.28.217.84/' '--server-insecure-tls' 'inventory' 'list' 'nodes' '--node-type=CONTAINER_NODE') +++ local args +++ transformed_files=() +++ local transformed_files +++ arch_scheduling_enabled +++ [[ -n '' ]] +++ arch_scheduling_enabled +++ [[ -n '' ]] ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl exec -n monitoring-2-0-3915 monitoring-0 -- pmm-admin --server-url=https://admin:admin@34.28.217.84/ --server-insecure-tls inventory list nodes --node-type=CONTAINER_NODE +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 -a -n 1 ']' +++ break +++ cat /tmp/tmp.3P7SnqmLXJ +++ cat /tmp/tmp.EBqzqJF0hg +++ rm /tmp/tmp.3P7SnqmLXJ /tmp/tmp.EBqzqJF0hg +++ return 0 ++ echo + desc 'check customClusterName for pmm' + set +o xtrace ----------------------------------------------------------------------------------- check customClusterName for pmm ----------------------------------------------------------------------------------- + custom_name=custom-cluster-name + kubectl_bin patch psmdb monitoring --type json '-p=[{"op":"add","path":"/spec/pause","value":false}, {"op":"add","path":"/spec/pmm/customClusterName","value":custom-cluster-name}]' ++ mktemp + local LAST_OUT=/tmp/tmp.mF5sSRRbKp ++ mktemp + local LAST_ERR=/tmp/tmp.ixJXwxM8mn + local exit_status=0 + local timeout=4 + args=('patch' 'psmdb' 'monitoring' '--type' 'json' '-p=[{"op":"add","path":"/spec/pause","value":false}, {"op":"add","path":"/spec/pmm/customClusterName","value":custom-cluster-name}]') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl patch psmdb monitoring --type json '-p=[{"op":"add","path":"/spec/pause","value":false}, {"op":"add","path":"/spec/pmm/customClusterName","value":custom-cluster-name}]' + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.mF5sSRRbKp perconaservermongodb.psmdb.percona.com/monitoring patched + cat /tmp/tmp.ixJXwxM8mn + rm /tmp/tmp.mF5sSRRbKp /tmp/tmp.ixJXwxM8mn + return 0 + wait_for_running monitoring-rs0 3 + local name=monitoring-rs0 + let last_pod=2 + local check_cluster_readiness=true + set_debug + [[ 1 == 1 ]] + set -o xtrace + local rs_name=rs0 + local cluster_name=monitoring ++ seq 0 2 + for i in $(seq 0 $last_pod) + [[ 0 -eq 2 ]] + wait_pod monitoring-rs0-0 + local pod=monitoring-rs0-0 + set +o xtrace waiting for pod/monitoring-rs0-0 to be ready............OK + for i in $(seq 0 $last_pod) + [[ 1 -eq 2 ]] + wait_pod monitoring-rs0-1 + local pod=monitoring-rs0-1 + set +o xtrace waiting for pod/monitoring-rs0-1 to be ready............OK + for i in $(seq 0 $last_pod) + [[ 2 -eq 2 ]] ++ kubectl_bin get psmdb monitoring -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].arbiter.enabled}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.dokG3XlWVe +++ mktemp ++ local LAST_ERR=/tmp/tmp.aBgTDcN3no ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'psmdb' 'monitoring' '-o' 'jsonpath={.spec.replsets[?(@.name=="rs0")].arbiter.enabled}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb monitoring -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].arbiter.enabled}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.dokG3XlWVe ++ cat /tmp/tmp.aBgTDcN3no ++ rm /tmp/tmp.dokG3XlWVe /tmp/tmp.aBgTDcN3no ++ return 0 + [[ '' == true ]] + wait_pod monitoring-rs0-2 + local pod=monitoring-rs0-2 + set +o xtrace waiting for pod/monitoring-rs0-2 to be ready...........OK ++ kubectl_bin get psmdb monitoring -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].nonvoting.enabled}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.oRMiLbsg68 +++ mktemp ++ local LAST_ERR=/tmp/tmp.vs5qZsc2F9 ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'psmdb' 'monitoring' '-o' 'jsonpath={.spec.replsets[?(@.name=="rs0")].nonvoting.enabled}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb monitoring -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].nonvoting.enabled}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.oRMiLbsg68 ++ cat /tmp/tmp.vs5qZsc2F9 ++ rm /tmp/tmp.oRMiLbsg68 /tmp/tmp.vs5qZsc2F9 ++ return 0 + [[ '' == true ]] ++ kubectl_bin get psmdb monitoring -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].hidden.enabled}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.1tNWbuYr7R +++ mktemp ++ local LAST_ERR=/tmp/tmp.nBO5qSZyLt ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'psmdb' 'monitoring' '-o' 'jsonpath={.spec.replsets[?(@.name=="rs0")].hidden.enabled}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb monitoring -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].hidden.enabled}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.1tNWbuYr7R ++ cat /tmp/tmp.nBO5qSZyLt ++ rm /tmp/tmp.1tNWbuYr7R /tmp/tmp.nBO5qSZyLt ++ return 0 + [[ '' == true ]] + sleep 10 + [[ true == true ]] + set +x Waiting for cluster monitoring to be ready.. + execute_curl -s -k -d '{"service_type":"MONGODB_SERVICE"}' https://admin:admin@34.28.217.84/v1/inventory/Services/List + local name=curl-11003 + local image=docker.io/alpine/curl:latest + local phase= + local exit_status=0 + kubectl_bin run curl-11003 -n monitoring-2-0-3915 --restart=Never --image=docker.io/alpine/curl:latest --command -- curl -sS -s -k -d '{"service_type":"MONGODB_SERVICE"}' https://admin:admin@34.28.217.84/v1/inventory/Services/List ++ mktemp + local LAST_OUT=/tmp/tmp.vaF121Zrdi ++ mktemp + local LAST_ERR=/tmp/tmp.TM5jCF5U3z + local exit_status=0 + local timeout=4 + args=('run' 'curl-11003' '-n' 'monitoring-2-0-3915' '--restart=Never' '--image=docker.io/alpine/curl:latest' '--command' '--' 'curl' '-sS' '-s' '-k' '-d' '{"service_type":"MONGODB_SERVICE"}' 'https://admin:admin@34.28.217.84/v1/inventory/Services/List') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl run curl-11003 -n monitoring-2-0-3915 --restart=Never --image=docker.io/alpine/curl:latest --command -- curl -sS -s -k -d '{"service_type":"MONGODB_SERVICE"}' https://admin:admin@34.28.217.84/v1/inventory/Services/List + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.vaF121Zrdi + cat /tmp/tmp.TM5jCF5U3z + rm /tmp/tmp.vaF121Zrdi /tmp/tmp.TM5jCF5U3z + return 0 ++ seq 1 60 + for _ in $(seq 1 60) ++ kubectl_bin get pod curl-11003 -n monitoring-2-0-3915 -o 'jsonpath={.status.phase}' + phase=Pending + [[ Pending == Succeeded ]] + [[ Pending == Failed ]] + sleep 1 + for _ in $(seq 1 60) ++ kubectl_bin get pod curl-11003 -n monitoring-2-0-3915 -o 'jsonpath={.status.phase}' + phase=Succeeded + [[ Succeeded == Succeeded ]] + break + kubectl_bin logs curl-11003 -n monitoring-2-0-3915 ++ mktemp + local LAST_OUT=/tmp/tmp.NPYFqC1XbC ++ mktemp + local LAST_ERR=/tmp/tmp.dneZm604zu + local exit_status=0 + local timeout=4 + args=('logs' 'curl-11003' '-n' 'monitoring-2-0-3915') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl logs curl-11003 -n monitoring-2-0-3915 + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.NPYFqC1XbC + cat /tmp/tmp.dneZm604zu + rm /tmp/tmp.NPYFqC1XbC /tmp/tmp.dneZm604zu + return 0 + kubectl_bin delete pod curl-11003 -n monitoring-2-0-3915 --ignore-not-found --wait=false + [[ Succeeded != Succeeded ]] + return 0 + check_custom_cluster_name monitoring-2-0-3915-monitoring-mongos-0 /tmp/tmp.G9Wq8CVNYh/pmm_service_list.json + local pod_service_name=monitoring-2-0-3915-monitoring-mongos-0 + local pmm_services_file=/tmp/tmp.G9Wq8CVNYh/pmm_service_list.json + echo 'Checking monitoring-2-0-3915-monitoring-mongos-0' Checking monitoring-2-0-3915-monitoring-mongos-0 ++ jq -r '.mongodb[] | select(.service_name=="monitoring-2-0-3915-monitoring-mongos-0") | .cluster' /tmp/tmp.G9Wq8CVNYh/pmm_service_list.json + pmm_service_cluster=custom-cluster-name + [[ custom-cluster-name != custom-cluster-name ]] + check_custom_cluster_name monitoring-2-0-3915-monitoring-rs0-0 /tmp/tmp.G9Wq8CVNYh/pmm_service_list.json + local pod_service_name=monitoring-2-0-3915-monitoring-rs0-0 + local pmm_services_file=/tmp/tmp.G9Wq8CVNYh/pmm_service_list.json + echo 'Checking monitoring-2-0-3915-monitoring-rs0-0' Checking monitoring-2-0-3915-monitoring-rs0-0 ++ jq -r '.mongodb[] | select(.service_name=="monitoring-2-0-3915-monitoring-rs0-0") | .cluster' /tmp/tmp.G9Wq8CVNYh/pmm_service_list.json + pmm_service_cluster=custom-cluster-name + [[ custom-cluster-name != custom-cluster-name ]] + check_custom_cluster_name monitoring-2-0-3915-monitoring-cfg-0 /tmp/tmp.G9Wq8CVNYh/pmm_service_list.json + local pod_service_name=monitoring-2-0-3915-monitoring-cfg-0 + local pmm_services_file=/tmp/tmp.G9Wq8CVNYh/pmm_service_list.json + echo 'Checking monitoring-2-0-3915-monitoring-cfg-0' Checking monitoring-2-0-3915-monitoring-cfg-0 ++ jq -r '.mongodb[] | select(.service_name=="monitoring-2-0-3915-monitoring-cfg-0") | .cluster' /tmp/tmp.G9Wq8CVNYh/pmm_service_list.json + pmm_service_cluster=custom-cluster-name + [[ custom-cluster-name != custom-cluster-name ]] + [[ -n '' ]] ++ kubectl_bin logs monitoring-rs0-0 pmm-client ++ grep -c 'cannot auto discover databases and collections' +++ mktemp ++ local LAST_OUT=/tmp/tmp.jUcmByGAJq +++ mktemp ++ local LAST_ERR=/tmp/tmp.VCL4MUfWS7 ++ local exit_status=0 ++ local timeout=4 ++ args=('logs' 'monitoring-rs0-0' 'pmm-client') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl logs monitoring-rs0-0 pmm-client ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.jUcmByGAJq ++ cat /tmp/tmp.VCL4MUfWS7 ++ rm /tmp/tmp.jUcmByGAJq /tmp/tmp.VCL4MUfWS7 ++ return 0 + [[ 0 != 0 ]] + helm uninstall monitoring release "monitoring" uninstalled + destroy monitoring-2-0-3915 + local namespace=monitoring-2-0-3915 + local ignore_logs=true + [[ 0 == 1 ]] + desc 'destroy cluster/operator and all other resources' + set +o xtrace ----------------------------------------------------------------------------------- destroy cluster/operator and all other resources ----------------------------------------------------------------------------------- + '[' true == false ']' + delete_backups + desc 'Delete psmdb-backup' + set +o xtrace ----------------------------------------------------------------------------------- Delete psmdb-backup ----------------------------------------------------------------------------------- ++ kubectl_bin get psmdb-backup --no-headers ++ wc -l +++ mktemp ++ local LAST_OUT=/tmp/tmp.wP8Xe9pfZg +++ mktemp ++ local LAST_ERR=/tmp/tmp.9vUXSlnSqI ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'psmdb-backup' '--no-headers') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb-backup --no-headers ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.wP8Xe9pfZg ++ cat /tmp/tmp.9vUXSlnSqI No resources found in monitoring-2-0-3915 namespace. ++ rm /tmp/tmp.wP8Xe9pfZg /tmp/tmp.9vUXSlnSqI ++ return 0 + '[' 0 '!=' 0 ']' + delete_crd + desc 'get and delete old CRDs and RBAC' + set +o xtrace ----------------------------------------------------------------------------------- get and delete old CRDs and RBAC ----------------------------------------------------------------------------------- + kubectl_bin delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2465/deploy/crd.yaml --ignore-not-found --wait=false ++ mktemp + local LAST_OUT=/tmp/tmp.pfTQE1yneF ++ mktemp + local LAST_ERR=/tmp/tmp.JHRAuSA3gs + local exit_status=0 + local timeout=4 + args=('delete' '-f' '/mnt/jenkins/workspace/cloud-psmdb-operator_PR-2465/deploy/crd.yaml' '--ignore-not-found' '--wait=false') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2465/deploy/crd.yaml --ignore-not-found --wait=false + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.pfTQE1yneF customresourcedefinition.apiextensions.k8s.io "perconaservermongodbbackups.psmdb.percona.com" deleted customresourcedefinition.apiextensions.k8s.io "perconaservermongodbclustersyncs.psmdb.percona.com" deleted customresourcedefinition.apiextensions.k8s.io "perconaservermongodbrestores.psmdb.percona.com" deleted customresourcedefinition.apiextensions.k8s.io "perconaservermongodbs.psmdb.percona.com" deleted + cat /tmp/tmp.JHRAuSA3gs + rm /tmp/tmp.pfTQE1yneF /tmp/tmp.JHRAuSA3gs + return 0 ++ yq eval .metadata.name /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2465/deploy/crd.yaml ++ grep -v '\-\-\-' grep: warning: stray \ before - grep: warning: stray \ before - + for crd_name in $(yq eval '.metadata.name' "${src_dir}/deploy/crd.yaml" | grep -v '\-\-\-') + kubectl get perconaservermongodbbackups.psmdb.percona.com --all-namespaces -o wide + grep -v NAMESPACE + xargs -L 1 sh -xc 'kubectl patch perconaservermongodbbackups.psmdb.percona.com -n $0 $1 --type=merge -p "{\"metadata\":{\"finalizers\":[]}}"' error: the server doesn't have a resource type "perconaservermongodbbackups" + kubectl patch perconaservermongodbbackups.psmdb.percona.com -n sh --type=merge -p '{"metadata":{"finalizers":[]}}' error: the server doesn't have a resource type "perconaservermongodbbackups" + : + kubectl_bin wait --for=delete crd perconaservermongodbbackups.psmdb.percona.com ++ mktemp + local LAST_OUT=/tmp/tmp.UPVNGZNpoD ++ mktemp + local LAST_ERR=/tmp/tmp.JaAweD1hkj + local exit_status=0 + local timeout=4 + args=('wait' '--for=delete' 'crd' 'perconaservermongodbbackups.psmdb.percona.com') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl wait --for=delete crd perconaservermongodbbackups.psmdb.percona.com + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.UPVNGZNpoD + cat /tmp/tmp.JaAweD1hkj + rm /tmp/tmp.UPVNGZNpoD /tmp/tmp.JaAweD1hkj + return 0 + for crd_name in $(yq eval '.metadata.name' "${src_dir}/deploy/crd.yaml" | grep -v '\-\-\-') + kubectl get perconaservermongodbclustersyncs.psmdb.percona.com --all-namespaces -o wide + grep -v NAMESPACE + xargs -L 1 sh -xc 'kubectl patch perconaservermongodbclustersyncs.psmdb.percona.com -n $0 $1 --type=merge -p "{\"metadata\":{\"finalizers\":[]}}"' error: the server doesn't have a resource type "perconaservermongodbclustersyncs" + kubectl patch perconaservermongodbclustersyncs.psmdb.percona.com -n sh --type=merge -p '{"metadata":{"finalizers":[]}}' error: the server doesn't have a resource type "perconaservermongodbclustersyncs" + : + kubectl_bin wait --for=delete crd perconaservermongodbclustersyncs.psmdb.percona.com ++ mktemp + local LAST_OUT=/tmp/tmp.S56DFFXujl ++ mktemp + local LAST_ERR=/tmp/tmp.dE5ZsQRX36 + local exit_status=0 + local timeout=4 + args=('wait' '--for=delete' 'crd' 'perconaservermongodbclustersyncs.psmdb.percona.com') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl wait --for=delete crd perconaservermongodbclustersyncs.psmdb.percona.com + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.S56DFFXujl + cat /tmp/tmp.dE5ZsQRX36 + rm /tmp/tmp.S56DFFXujl /tmp/tmp.dE5ZsQRX36 + return 0 + for crd_name in $(yq eval '.metadata.name' "${src_dir}/deploy/crd.yaml" | grep -v '\-\-\-') + grep -v NAMESPACE + kubectl get perconaservermongodbrestores.psmdb.percona.com --all-namespaces -o wide + xargs -L 1 sh -xc 'kubectl patch perconaservermongodbrestores.psmdb.percona.com -n $0 $1 --type=merge -p "{\"metadata\":{\"finalizers\":[]}}"' error: the server doesn't have a resource type "perconaservermongodbrestores" + kubectl patch perconaservermongodbrestores.psmdb.percona.com -n sh --type=merge -p '{"metadata":{"finalizers":[]}}' error: the server doesn't have a resource type "perconaservermongodbrestores" + : + kubectl_bin wait --for=delete crd perconaservermongodbrestores.psmdb.percona.com ++ mktemp + local LAST_OUT=/tmp/tmp.QuY5hGP4NJ ++ mktemp + local LAST_ERR=/tmp/tmp.mrfh36zGMU + local exit_status=0 + local timeout=4 + args=('wait' '--for=delete' 'crd' 'perconaservermongodbrestores.psmdb.percona.com') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl wait --for=delete crd perconaservermongodbrestores.psmdb.percona.com + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.QuY5hGP4NJ + cat /tmp/tmp.mrfh36zGMU + rm /tmp/tmp.QuY5hGP4NJ /tmp/tmp.mrfh36zGMU + return 0 + for crd_name in $(yq eval '.metadata.name' "${src_dir}/deploy/crd.yaml" | grep -v '\-\-\-') + kubectl get perconaservermongodbs.psmdb.percona.com --all-namespaces -o wide + grep -v NAMESPACE + xargs -L 1 sh -xc 'kubectl patch perconaservermongodbs.psmdb.percona.com -n $0 $1 --type=merge -p "{\"metadata\":{\"finalizers\":[]}}"' error: the server doesn't have a resource type "perconaservermongodbs" + kubectl patch perconaservermongodbs.psmdb.percona.com -n sh --type=merge -p '{"metadata":{"finalizers":[]}}' error: the server doesn't have a resource type "perconaservermongodbs" + : + kubectl_bin wait --for=delete crd perconaservermongodbs.psmdb.percona.com ++ mktemp + local LAST_OUT=/tmp/tmp.EXJ1bwL2gI ++ mktemp + local LAST_ERR=/tmp/tmp.m3xTTnXF5z + local exit_status=0 + local timeout=4 + args=('wait' '--for=delete' 'crd' 'perconaservermongodbs.psmdb.percona.com') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl wait --for=delete crd perconaservermongodbs.psmdb.percona.com + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.EXJ1bwL2gI + cat /tmp/tmp.m3xTTnXF5z + rm /tmp/tmp.EXJ1bwL2gI /tmp/tmp.m3xTTnXF5z + return 0 + local rbac_yaml=rbac.yaml + '[' -n psmdb-operator ']' + rbac_yaml=cw-rbac.yaml + kubectl_bin delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2465/deploy/cw-rbac.yaml --ignore-not-found ++ mktemp + local LAST_OUT=/tmp/tmp.2oGQP4zR0e ++ mktemp + local LAST_ERR=/tmp/tmp.clDWVNTUZa + local exit_status=0 + local timeout=4 + args=('delete' '-f' '/mnt/jenkins/workspace/cloud-psmdb-operator_PR-2465/deploy/cw-rbac.yaml' '--ignore-not-found') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2465/deploy/cw-rbac.yaml --ignore-not-found + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.2oGQP4zR0e clusterrole.rbac.authorization.k8s.io "percona-server-mongodb-operator" deleted clusterrolebinding.rbac.authorization.k8s.io "service-account-percona-server-mongodb-operator" deleted + cat /tmp/tmp.clDWVNTUZa + rm /tmp/tmp.2oGQP4zR0e /tmp/tmp.clDWVNTUZa + return 0 + destroy_cert_manager + [[ gke == rancher ]] + kubectl_bin delete -f https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml ++ mktemp + local LAST_OUT=/tmp/tmp.KhTdtJ5bwQ ++ mktemp + local LAST_ERR=/tmp/tmp.O1QRJLWts1 + local exit_status=0 + local timeout=4 + args=('delete' '-f' 'https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl delete -f https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.KhTdtJ5bwQ namespace "cert-manager" deleted customresourcedefinition.apiextensions.k8s.io "challenges.acme.cert-manager.io" deleted customresourcedefinition.apiextensions.k8s.io "orders.acme.cert-manager.io" deleted customresourcedefinition.apiextensions.k8s.io "certificaterequests.cert-manager.io" deleted customresourcedefinition.apiextensions.k8s.io "certificates.cert-manager.io" deleted customresourcedefinition.apiextensions.k8s.io "clusterissuers.cert-manager.io" deleted customresourcedefinition.apiextensions.k8s.io "issuers.cert-manager.io" deleted serviceaccount "cert-manager-cainjector" deleted from cert-manager namespace serviceaccount "cert-manager" deleted from cert-manager namespace serviceaccount "cert-manager-webhook" deleted from cert-manager namespace clusterrole.rbac.authorization.k8s.io "cert-manager-cainjector" deleted clusterrole.rbac.authorization.k8s.io "cert-manager-controller-issuers" deleted clusterrole.rbac.authorization.k8s.io "cert-manager-controller-clusterissuers" deleted clusterrole.rbac.authorization.k8s.io "cert-manager-controller-certificates" deleted clusterrole.rbac.authorization.k8s.io "cert-manager-controller-orders" deleted clusterrole.rbac.authorization.k8s.io "cert-manager-controller-challenges" deleted clusterrole.rbac.authorization.k8s.io "cert-manager-controller-ingress-shim" deleted clusterrole.rbac.authorization.k8s.io "cert-manager-cluster-view" deleted clusterrole.rbac.authorization.k8s.io "cert-manager-view" deleted clusterrole.rbac.authorization.k8s.io "cert-manager-edit" deleted clusterrole.rbac.authorization.k8s.io "cert-manager-controller-approve:cert-manager-io" deleted clusterrole.rbac.authorization.k8s.io "cert-manager-controller-certificatesigningrequests" deleted clusterrole.rbac.authorization.k8s.io "cert-manager-webhook:subjectaccessreviews" deleted clusterrolebinding.rbac.authorization.k8s.io "cert-manager-cainjector" deleted clusterrolebinding.rbac.authorization.k8s.io "cert-manager-controller-issuers" deleted clusterrolebinding.rbac.authorization.k8s.io "cert-manager-controller-clusterissuers" deleted clusterrolebinding.rbac.authorization.k8s.io "cert-manager-controller-certificates" deleted clusterrolebinding.rbac.authorization.k8s.io "cert-manager-controller-orders" deleted clusterrolebinding.rbac.authorization.k8s.io "cert-manager-controller-challenges" deleted clusterrolebinding.rbac.authorization.k8s.io "cert-manager-controller-ingress-shim" deleted clusterrolebinding.rbac.authorization.k8s.io "cert-manager-controller-approve:cert-manager-io" deleted clusterrolebinding.rbac.authorization.k8s.io "cert-manager-controller-certificatesigningrequests" deleted clusterrolebinding.rbac.authorization.k8s.io "cert-manager-webhook:subjectaccessreviews" deleted role.rbac.authorization.k8s.io "cert-manager-cainjector:leaderelection" deleted from kube-system namespace role.rbac.authorization.k8s.io "cert-manager:leaderelection" deleted from kube-system namespace role.rbac.authorization.k8s.io "cert-manager-tokenrequest" deleted from cert-manager namespace role.rbac.authorization.k8s.io "cert-manager-webhook:dynamic-serving" deleted from cert-manager namespace rolebinding.rbac.authorization.k8s.io "cert-manager-cainjector:leaderelection" deleted from kube-system namespace rolebinding.rbac.authorization.k8s.io "cert-manager:leaderelection" deleted from kube-system namespace rolebinding.rbac.authorization.k8s.io "cert-manager-tokenrequest" deleted from cert-manager namespace rolebinding.rbac.authorization.k8s.io "cert-manager-webhook:dynamic-serving" deleted from cert-manager namespace service "cert-manager-cainjector" deleted from cert-manager namespace service "cert-manager" deleted from cert-manager namespace service "cert-manager-webhook" deleted from cert-manager namespace deployment.apps "cert-manager-cainjector" deleted from cert-manager namespace deployment.apps "cert-manager" deleted from cert-manager namespace deployment.apps "cert-manager-webhook" deleted from cert-manager namespace mutatingwebhookconfiguration.admissionregistration.k8s.io "cert-manager-webhook" deleted validatingwebhookconfiguration.admissionregistration.k8s.io "cert-manager-webhook" deleted + cat /tmp/tmp.O1QRJLWts1 + rm /tmp/tmp.KhTdtJ5bwQ /tmp/tmp.O1QRJLWts1 + return 0 + [[ -n '' ]] + '[' -n psmdb-operator ']' + kubectl_bin delete --grace-period=0 --force=true namespace monitoring-2-0-3915 + rm -rf /tmp/tmp.G9Wq8CVNYh + kubectl_bin delete --grace-period=0 --force=true namespace psmdb-operator ++ mktemp ++ mktemp + desc 'test passed' + set +o xtrace ----------------------------------------------------------------------------------- test passed ----------------------------------------------------------------------------------- + local LAST_OUT=/tmp/tmp.HadrW3HEig + local LAST_OUT=/tmp/tmp.6uCHLQ5oMP ++ mktemp ++ mktemp + local LAST_ERR=/tmp/tmp.yvbu8UPc3X + local exit_status=0 + local timeout=4 + args=('delete' '--grace-period=0' '--force=true' 'namespace' 'psmdb-operator') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] + local LAST_ERR=/tmp/tmp.1tMWqbYW1l + local exit_status=0 + local timeout=4 + args=('delete' '--grace-period=0' '--force=true' 'namespace' 'monitoring-2-0-3915') ++ seq 0 2 + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl delete --grace-period=0 --force=true namespace psmdb-operator + for i in $(seq 0 2) + set +e + kubectl delete --grace-period=0 --force=true namespace monitoring-2-0-3915