Log: /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1596/e2e-tests/logs/monitoring-2-0.log
WARNING: version difference between client (1.30) and server (1.27) exceeds the supported minor version skew of +/-1
WARNING: version difference between client (1.30) and server (1.27) exceeds the supported minor version skew of +/-1
WARNING: version difference between client (1.30) and server (1.27) exceeds the supported minor version skew of +/-1
+ create_infra monitoring-2-0-16090
+ local ns=monitoring-2-0-16090
+ delete_crd
+ desc 'get and delete old CRDs and RBAC'
+ set +o xtrace


-----------------------------------------------------------------------------------
get and delete old CRDs and RBAC
-----------------------------------------------------------------------------------

+ kubectl_bin delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1596/deploy/crd.yaml --ignore-not-found --wait=false
++ mktemp
+ local LAST_OUT=/tmp/tmp.Od9o2OyPgL
++ mktemp
+ local LAST_ERR=/tmp/tmp.gLORlbEykI
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1596/deploy/crd.yaml --ignore-not-found --wait=false
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.Od9o2OyPgL
customresourcedefinition.apiextensions.k8s.io "perconaservermongodbbackups.psmdb.percona.com" deleted
customresourcedefinition.apiextensions.k8s.io "perconaservermongodbrestores.psmdb.percona.com" deleted
customresourcedefinition.apiextensions.k8s.io "perconaservermongodbs.psmdb.percona.com" deleted
+ cat /tmp/tmp.gLORlbEykI
+ rm /tmp/tmp.Od9o2OyPgL /tmp/tmp.gLORlbEykI
+ return 0
++ yq eval .metadata.name /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1596/deploy/crd.yaml
++ grep -v '\-\-\-'
+ for crd_name in '$(yq eval '\''.metadata.name'\'' "${src_dir}/deploy/crd.yaml" | grep -v '\''\-\-\-'\'')'
+ kubectl get perconaservermongodbbackups.psmdb.percona.com --all-namespaces -o wide
+ grep -v NAMESPACE
+ xargs -L 1 sh -xc 'kubectl patch perconaservermongodbbackups.psmdb.percona.com -n $0 $1 --type=merge -p "{\"metadata\":{\"finalizers\":[]}}"'
error: the server doesn't have a resource type "perconaservermongodbbackups"
+ kubectl patch perconaservermongodbbackups.psmdb.percona.com -n sh --type=merge -p '{"metadata":{"finalizers":[]}}'
error: the server doesn't have a resource type "perconaservermongodbbackups"
+ :
+ kubectl_bin wait --for=delete crd perconaservermongodbbackups.psmdb.percona.com
++ mktemp
+ local LAST_OUT=/tmp/tmp.aokyjfftsA
++ mktemp
+ local LAST_ERR=/tmp/tmp.C83eZtfKGj
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl wait --for=delete crd perconaservermongodbbackups.psmdb.percona.com
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.aokyjfftsA
+ cat /tmp/tmp.C83eZtfKGj
+ rm /tmp/tmp.aokyjfftsA /tmp/tmp.C83eZtfKGj
+ return 0
+ for crd_name in '$(yq eval '\''.metadata.name'\'' "${src_dir}/deploy/crd.yaml" | grep -v '\''\-\-\-'\'')'
+ kubectl get perconaservermongodbrestores.psmdb.percona.com --all-namespaces -o wide
+ grep -v NAMESPACE
+ xargs -L 1 sh -xc 'kubectl patch perconaservermongodbrestores.psmdb.percona.com -n $0 $1 --type=merge -p "{\"metadata\":{\"finalizers\":[]}}"'
error: the server doesn't have a resource type "perconaservermongodbrestores"
+ kubectl patch perconaservermongodbrestores.psmdb.percona.com -n sh --type=merge -p '{"metadata":{"finalizers":[]}}'
error: the server doesn't have a resource type "perconaservermongodbrestores"
+ :
+ kubectl_bin wait --for=delete crd perconaservermongodbrestores.psmdb.percona.com
++ mktemp
+ local LAST_OUT=/tmp/tmp.Z0ycslu6tK
++ mktemp
+ local LAST_ERR=/tmp/tmp.OOE8RdlLl2
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl wait --for=delete crd perconaservermongodbrestores.psmdb.percona.com
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.Z0ycslu6tK
+ cat /tmp/tmp.OOE8RdlLl2
+ rm /tmp/tmp.Z0ycslu6tK /tmp/tmp.OOE8RdlLl2
+ return 0
+ for crd_name in '$(yq eval '\''.metadata.name'\'' "${src_dir}/deploy/crd.yaml" | grep -v '\''\-\-\-'\'')'
+ kubectl get perconaservermongodbs.psmdb.percona.com --all-namespaces -o wide
+ grep -v NAMESPACE
+ xargs -L 1 sh -xc 'kubectl patch perconaservermongodbs.psmdb.percona.com -n $0 $1 --type=merge -p "{\"metadata\":{\"finalizers\":[]}}"'
error: the server doesn't have a resource type "perconaservermongodbs"
+ kubectl patch perconaservermongodbs.psmdb.percona.com -n sh --type=merge -p '{"metadata":{"finalizers":[]}}'
error: the server doesn't have a resource type "perconaservermongodbs"
+ :
+ kubectl_bin wait --for=delete crd perconaservermongodbs.psmdb.percona.com
++ mktemp
+ local LAST_OUT=/tmp/tmp.mcYpKGhpgd
++ mktemp
+ local LAST_ERR=/tmp/tmp.nuLf5ZhCbC
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl wait --for=delete crd perconaservermongodbs.psmdb.percona.com
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.mcYpKGhpgd
+ cat /tmp/tmp.nuLf5ZhCbC
+ rm /tmp/tmp.mcYpKGhpgd /tmp/tmp.nuLf5ZhCbC
+ return 0
+ local rbac_yaml=rbac.yaml
+ '[' -n psmdb-operator ']'
+ rbac_yaml=cw-rbac.yaml
+ kubectl_bin delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1596/deploy/cw-rbac.yaml --ignore-not-found
++ mktemp
+ local LAST_OUT=/tmp/tmp.J69WSMAPSG
++ mktemp
+ local LAST_ERR=/tmp/tmp.0qDxtXZuYp
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1596/deploy/cw-rbac.yaml --ignore-not-found
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.J69WSMAPSG
clusterrole.rbac.authorization.k8s.io "percona-server-mongodb-operator" deleted
clusterrolebinding.rbac.authorization.k8s.io "service-account-percona-server-mongodb-operator" deleted
+ cat /tmp/tmp.0qDxtXZuYp
+ rm /tmp/tmp.J69WSMAPSG /tmp/tmp.0qDxtXZuYp
+ return 0
+ check_crd_for_deletion PR-1596-83fb9e77
+ local git_tag=PR-1596-83fb9e77
++ curl -s https://raw.githubusercontent.com/percona/percona-server-mongodb-operator/PR-1596-83fb9e77/deploy/crd.yaml
++ /usr/bin/sed ':a;N;$!ba;s/\n/ /g'
++ yq eval .metadata.name
++ /usr/bin/sed s/---//g
+ for crd_name in '$(curl -s https://raw.githubusercontent.com/percona/percona-server-mongodb-operator/${git_tag}/deploy/crd.yaml | yq eval '\''.metadata.name'\'' | $sed '\''s/---//g'\'' | $sed '\'':a;N;$!ba;s/\n/ /g'\'')'
++ kubectl_bin get crd/null -o 'jsonpath={.status.conditions[-1].type}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.rvIfRhm4pR
+++ mktemp
++ local LAST_ERR=/tmp/tmp.5lDs37uNfI
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get crd/null -o 'jsonpath={.status.conditions[-1].type}'
++ exit_status=1
++ set -e
++ '[' 1 '!=' 0 -a -n 1 ']'
++ cat /tmp/tmp.rvIfRhm4pR
++ cat /tmp/tmp.5lDs37uNfI
Error from server (NotFound): customresourcedefinitions.apiextensions.k8s.io "null" not found
++ sleep 0
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get crd/null -o 'jsonpath={.status.conditions[-1].type}'
++ exit_status=1
++ set -e
++ '[' 1 '!=' 0 -a -n 1 ']'
++ cat /tmp/tmp.rvIfRhm4pR
++ cat /tmp/tmp.5lDs37uNfI
Error from server (NotFound): customresourcedefinitions.apiextensions.k8s.io "null" not found
++ sleep 4
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get crd/null -o 'jsonpath={.status.conditions[-1].type}'
++ exit_status=1
++ set -e
++ '[' 1 '!=' 0 -a -n 1 ']'
++ cat /tmp/tmp.rvIfRhm4pR
++ cat /tmp/tmp.5lDs37uNfI
Error from server (NotFound): customresourcedefinitions.apiextensions.k8s.io "null" not found
++ sleep 8
++ cat /tmp/tmp.rvIfRhm4pR
++ cat /tmp/tmp.5lDs37uNfI
Error from server (NotFound): customresourcedefinitions.apiextensions.k8s.io "null" not found
++ rm /tmp/tmp.rvIfRhm4pR /tmp/tmp.5lDs37uNfI
++ return 1
+ [[ '' == \T\e\r\m\i\n\a\t\i\n\g ]]
+ '[' -n psmdb-operator ']'
+ create_namespace psmdb-operator
+ local namespace=psmdb-operator
+ local skip_clean_namespace=
+ [[ 1 == 1 ]]
+ [[ -z '' ]]
+ destroy_chaos_mesh
++ helm list --all-namespaces --filter chaos-mesh
++ tail -n1
++ awk '-F ' '{print $2}'
++ sed s/NAMESPACE//
+ local chaos_mesh_ns=
+ desc 'destroy chaos-mesh'
+ set +o xtrace


-----------------------------------------------------------------------------------
destroy chaos-mesh
-----------------------------------------------------------------------------------

+ '[' -n '' ']'
++ grep chaos-mesh
++ awk '{print $1}'
++ kubectl get MutatingWebhookConfiguration
+ timeout 30 kubectl delete MutatingWebhookConfiguration
error: resource(s) were provided, but no name was specified
+ :
++ kubectl get ValidatingWebhookConfiguration
++ grep chaos-mesh
++ awk '{print $1}'
+ timeout 30 kubectl delete ValidatingWebhookConfiguration
error: resource(s) were provided, but no name was specified
+ :
++ kubectl get ValidatingWebhookConfiguration
++ grep validate-auth
++ awk '{print $1}'
+ timeout 30 kubectl delete ValidatingWebhookConfiguration
error: resource(s) were provided, but no name was specified
+ :
++ kubectl api-resources
++ awk '{print $1}'
++ grep chaos-mesh
++ kubectl get crd
++ grep chaos-mesh.org
++ awk '{print $1}'
+ timeout 30 kubectl delete crd
error: resource(s) were provided, but no name was specified
+ :
++ kubectl get clusterrolebinding
++ grep chaos-mesh
++ awk '{print $1}'
+ timeout 30 kubectl delete clusterrolebinding
error: resource(s) were provided, but no name was specified
+ :
++ kubectl get clusterrole
++ grep chaos-mesh
++ awk '{print $1}'
+ timeout 30 kubectl delete clusterrole
error: resource(s) were provided, but no name was specified
+ :
+ desc 'cleaned up all old namespaces'
+ set +o xtrace


-----------------------------------------------------------------------------------
cleaned up all old namespaces
-----------------------------------------------------------------------------------

+ kubectl_bin get ns
+ egrep -v '^kube-|^default|Terminating|psmdb-operator|openshift|gke-mcs|^NAME'
+ '[' -n '' ']'
+ desc 'cleaned up old namespaces psmdb-operator'
+ set +o xtrace


-----------------------------------------------------------------------------------
cleaned up old namespaces psmdb-operator
-----------------------------------------------------------------------------------

+ kubectl_bin delete namespace psmdb-operator --ignore-not-found
+ awk '{print$1}'
++ mktemp
++ mktemp
+ local LAST_OUT=/tmp/tmp.DrcGyTthn3
+ local LAST_OUT=/tmp/tmp.zuekUFFvNq
+ xargs kubectl delete ns
++ mktemp
+ local LAST_ERR=/tmp/tmp.Ug8nOif3AT
++ mktemp
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ local LAST_ERR=/tmp/tmp.CDwE5x1y4V
+ local exit_status=0
+ local timeout=4
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl get ns
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl delete namespace psmdb-operator --ignore-not-found
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.DrcGyTthn3
+ cat /tmp/tmp.Ug8nOif3AT
+ rm /tmp/tmp.DrcGyTthn3 /tmp/tmp.Ug8nOif3AT
+ return 0
namespace "cert-manager" deleted
namespace "gmp-public" deleted
namespace "gmp-system" deleted
namespace "monitoring-2-0-23882" deleted
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.zuekUFFvNq
namespace "psmdb-operator" deleted
+ cat /tmp/tmp.CDwE5x1y4V
+ rm /tmp/tmp.zuekUFFvNq /tmp/tmp.CDwE5x1y4V
+ return 0
+ kubectl_bin wait --for=delete namespace psmdb-operator
++ mktemp
+ local LAST_OUT=/tmp/tmp.I9EleoDMOB
++ mktemp
+ local LAST_ERR=/tmp/tmp.kuB2UKvfNd
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl wait --for=delete namespace psmdb-operator
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.I9EleoDMOB
+ cat /tmp/tmp.kuB2UKvfNd
+ rm /tmp/tmp.I9EleoDMOB /tmp/tmp.kuB2UKvfNd
+ return 0
+ desc 'create namespace psmdb-operator'
+ set +o xtrace


-----------------------------------------------------------------------------------
create namespace psmdb-operator
-----------------------------------------------------------------------------------

+ kubectl_bin create namespace psmdb-operator
++ mktemp
+ local LAST_OUT=/tmp/tmp.1ioX6evf5a
++ mktemp
+ local LAST_ERR=/tmp/tmp.kOP60Gjnpy
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl create namespace psmdb-operator
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.1ioX6evf5a
namespace/psmdb-operator created
+ cat /tmp/tmp.kOP60Gjnpy
+ rm /tmp/tmp.1ioX6evf5a /tmp/tmp.kOP60Gjnpy
+ return 0
++ kubectl_bin config current-context
+++ mktemp
++ local LAST_OUT=/tmp/tmp.6wC3ZtWUd0
+++ mktemp
++ local LAST_ERR=/tmp/tmp.rz47aIoCzs
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl config current-context
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.6wC3ZtWUd0
++ cat /tmp/tmp.rz47aIoCzs
++ rm /tmp/tmp.6wC3ZtWUd0 /tmp/tmp.rz47aIoCzs
++ return 0
+ kubectl_bin config set-context gke_cloud-dev-112233_us-central1-a_jen-psmdb-1596-83fb9e77-16-cluster4 --namespace=psmdb-operator
++ mktemp
+ local LAST_OUT=/tmp/tmp.xemNvQ6JOx
++ mktemp
+ local LAST_ERR=/tmp/tmp.sEOaH1Ziu8
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl config set-context gke_cloud-dev-112233_us-central1-a_jen-psmdb-1596-83fb9e77-16-cluster4 --namespace=psmdb-operator
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.xemNvQ6JOx
Context "gke_cloud-dev-112233_us-central1-a_jen-psmdb-1596-83fb9e77-16-cluster4" modified.
+ cat /tmp/tmp.sEOaH1Ziu8
+ rm /tmp/tmp.xemNvQ6JOx /tmp/tmp.sEOaH1Ziu8
+ return 0
+ deploy_operator
+ desc 'start PSMDB operator'
+ set +o xtrace


-----------------------------------------------------------------------------------
start PSMDB operator
-----------------------------------------------------------------------------------

+ local cr_file
+ '[' -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1596/e2e-tests/monitoring-2-0/conf/crd.yaml ']'
+ cr_file=/mnt/jenkins/workspace/cloud-psmdb-operator_PR-1596/deploy/crd.yaml
+ kubectl_bin apply --server-side --force-conflicts -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1596/deploy/crd.yaml
++ mktemp
+ local LAST_OUT=/tmp/tmp.VRj4InzRxi
++ mktemp
+ local LAST_ERR=/tmp/tmp.uGPRywvm2B
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl apply --server-side --force-conflicts -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1596/deploy/crd.yaml
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.VRj4InzRxi
customresourcedefinition.apiextensions.k8s.io/perconaservermongodbbackups.psmdb.percona.com serverside-applied
customresourcedefinition.apiextensions.k8s.io/perconaservermongodbrestores.psmdb.percona.com serverside-applied
customresourcedefinition.apiextensions.k8s.io/perconaservermongodbs.psmdb.percona.com serverside-applied
+ cat /tmp/tmp.uGPRywvm2B
+ rm /tmp/tmp.VRj4InzRxi /tmp/tmp.uGPRywvm2B
+ return 0
+ '[' -n psmdb-operator ']'
+ apply_rbac cw-rbac
+ local operator_namespace=psmdb-operator
+ local rbac=cw-rbac
+ cat /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1596/deploy/cw-rbac.yaml
+ sed -e 's^namespace: .*^namespace: psmdb-operator^'
+ kubectl_bin apply -n psmdb-operator -f -
++ mktemp
+ local LAST_OUT=/tmp/tmp.eP4m4eJ13c
++ mktemp
+ local LAST_ERR=/tmp/tmp.8XNzyc3FMd
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl apply -n psmdb-operator -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.eP4m4eJ13c
clusterrole.rbac.authorization.k8s.io/percona-server-mongodb-operator created
serviceaccount/percona-server-mongodb-operator created
clusterrolebinding.rbac.authorization.k8s.io/service-account-percona-server-mongodb-operator created
+ cat /tmp/tmp.8XNzyc3FMd
+ rm /tmp/tmp.eP4m4eJ13c /tmp/tmp.8XNzyc3FMd
+ return 0
+ kubectl_bin apply -f -
+ yq eval '
			(.spec.template.spec.containers[].image = "perconalab/percona-server-mongodb-operator:PR-1596-83fb9e77") |
			((.. | select(.[] == "DISABLE_TELEMETRY")) |= .value="true") |
			((.. | select(.[] == "LOG_LEVEL")) |= .value="DEBUG")' /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1596/deploy/cw-operator.yaml
++ mktemp
+ local LAST_OUT=/tmp/tmp.1Gz7oKLOhd
++ mktemp
+ local LAST_ERR=/tmp/tmp.8SOXdSuzdl
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl apply -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.1Gz7oKLOhd
deployment.apps/percona-server-mongodb-operator created
+ cat /tmp/tmp.8SOXdSuzdl
+ rm /tmp/tmp.1Gz7oKLOhd /tmp/tmp.8SOXdSuzdl
+ return 0
+ sleep 2
++ get_operator_pod
++ kubectl_bin get pods --selector=name=percona-server-mongodb-operator -o 'jsonpath={.items[].metadata.name}' -n psmdb-operator
+++ mktemp
++ local LAST_OUT=/tmp/tmp.BgtFdD4MZd
+++ mktemp
++ local LAST_ERR=/tmp/tmp.WMDeDabwgZ
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=percona-server-mongodb-operator -o 'jsonpath={.items[].metadata.name}' -n psmdb-operator
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.BgtFdD4MZd
++ cat /tmp/tmp.WMDeDabwgZ
++ rm /tmp/tmp.BgtFdD4MZd /tmp/tmp.WMDeDabwgZ
++ return 0
+ wait_pod percona-server-mongodb-operator-64975ffb9f-w95gh
+ local pod=percona-server-mongodb-operator-64975ffb9f-w95gh
+ set +o xtrace
waiting for pod/percona-server-mongodb-operator-64975ffb9f-w95gh to be ready..OK
+ create_namespace monitoring-2-0-16090
+ local namespace=monitoring-2-0-16090
+ local skip_clean_namespace=
+ [[ 1 == 1 ]]
+ [[ -z '' ]]
+ destroy_chaos_mesh
++ helm list --all-namespaces --filter chaos-mesh
++ tail -n1
++ sed s/NAMESPACE//
++ awk '-F ' '{print $2}'
+ local chaos_mesh_ns=
+ desc 'destroy chaos-mesh'
+ set +o xtrace


-----------------------------------------------------------------------------------
destroy chaos-mesh
-----------------------------------------------------------------------------------

+ '[' -n '' ']'
++ kubectl get MutatingWebhookConfiguration
++ grep chaos-mesh
++ awk '{print $1}'
+ timeout 30 kubectl delete MutatingWebhookConfiguration
error: resource(s) were provided, but no name was specified
+ :
++ kubectl get ValidatingWebhookConfiguration
++ grep chaos-mesh
++ awk '{print $1}'
+ timeout 30 kubectl delete ValidatingWebhookConfiguration
error: resource(s) were provided, but no name was specified
+ :
++ kubectl get ValidatingWebhookConfiguration
++ grep validate-auth
++ awk '{print $1}'
+ timeout 30 kubectl delete ValidatingWebhookConfiguration
error: resource(s) were provided, but no name was specified
+ :
++ kubectl api-resources
++ grep chaos-mesh
++ awk '{print $1}'
++ kubectl get crd
++ awk '{print $1}'
++ grep chaos-mesh.org
+ timeout 30 kubectl delete crd
error: resource(s) were provided, but no name was specified
+ :
++ kubectl get clusterrolebinding
++ grep chaos-mesh
++ awk '{print $1}'
+ timeout 30 kubectl delete clusterrolebinding
error: resource(s) were provided, but no name was specified
+ :
++ kubectl get clusterrole
++ grep chaos-mesh
++ awk '{print $1}'
+ timeout 30 kubectl delete clusterrole
error: resource(s) were provided, but no name was specified
+ :
+ desc 'cleaned up all old namespaces'
+ set +o xtrace


-----------------------------------------------------------------------------------
cleaned up all old namespaces
-----------------------------------------------------------------------------------

+ kubectl_bin get ns
+ awk '{print$1}'
+ '[' -n '' ']'
+ xargs kubectl delete ns
+ desc 'cleaned up old namespaces monitoring-2-0-16090'
+ set +o xtrace


-----------------------------------------------------------------------------------
cleaned up old namespaces monitoring-2-0-16090
-----------------------------------------------------------------------------------

+ kubectl_bin delete namespace monitoring-2-0-16090 --ignore-not-found
++ mktemp
+ egrep -v '^kube-|^default|Terminating|psmdb-operator|openshift|gke-mcs|^NAME'
++ mktemp
+ local LAST_OUT=/tmp/tmp.0upgASYRuu
++ mktemp
+ local LAST_OUT=/tmp/tmp.br2SmAVyOD
++ mktemp
+ local LAST_ERR=/tmp/tmp.3thiNqQeJv
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ local LAST_ERR=/tmp/tmp.wiAFrxvkQf
+ local exit_status=0
+ local timeout=4
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl delete namespace monitoring-2-0-16090 --ignore-not-found
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl get ns
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.br2SmAVyOD
+ cat /tmp/tmp.wiAFrxvkQf
+ rm /tmp/tmp.br2SmAVyOD /tmp/tmp.wiAFrxvkQf
+ return 0
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.0upgASYRuu
+ cat /tmp/tmp.3thiNqQeJv
+ rm /tmp/tmp.0upgASYRuu /tmp/tmp.3thiNqQeJv
+ return 0
+ kubectl_bin wait --for=delete namespace monitoring-2-0-16090
++ mktemp
+ local LAST_OUT=/tmp/tmp.Kxv2imwfwy
++ mktemp
+ local LAST_ERR=/tmp/tmp.fROA9MJQzi
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl wait --for=delete namespace monitoring-2-0-16090
namespace "gmp-public" deleted
namespace "gmp-system" deleted
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.Kxv2imwfwy
+ cat /tmp/tmp.fROA9MJQzi
+ rm /tmp/tmp.Kxv2imwfwy /tmp/tmp.fROA9MJQzi
+ return 0
+ desc 'create namespace monitoring-2-0-16090'
+ set +o xtrace


-----------------------------------------------------------------------------------
create namespace monitoring-2-0-16090
-----------------------------------------------------------------------------------

+ kubectl_bin create namespace monitoring-2-0-16090
++ mktemp
+ local LAST_OUT=/tmp/tmp.PO4tz6Umxb
++ mktemp
+ local LAST_ERR=/tmp/tmp.nXWn4r0KRI
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl create namespace monitoring-2-0-16090
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.PO4tz6Umxb
namespace/monitoring-2-0-16090 created
+ cat /tmp/tmp.nXWn4r0KRI
+ rm /tmp/tmp.PO4tz6Umxb /tmp/tmp.nXWn4r0KRI
+ return 0
++ kubectl_bin config current-context
+++ mktemp
++ local LAST_OUT=/tmp/tmp.t44NQiajd2
+++ mktemp
++ local LAST_ERR=/tmp/tmp.1sAzA3KmmJ
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl config current-context
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.t44NQiajd2
++ cat /tmp/tmp.1sAzA3KmmJ
++ rm /tmp/tmp.t44NQiajd2 /tmp/tmp.1sAzA3KmmJ
++ return 0
+ kubectl_bin config set-context gke_cloud-dev-112233_us-central1-a_jen-psmdb-1596-83fb9e77-16-cluster4 --namespace=monitoring-2-0-16090
++ mktemp
+ local LAST_OUT=/tmp/tmp.RpobJZRNFU
++ mktemp
+ local LAST_ERR=/tmp/tmp.vlHdhFASUL
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl config set-context gke_cloud-dev-112233_us-central1-a_jen-psmdb-1596-83fb9e77-16-cluster4 --namespace=monitoring-2-0-16090
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.RpobJZRNFU
Context "gke_cloud-dev-112233_us-central1-a_jen-psmdb-1596-83fb9e77-16-cluster4" modified.
+ cat /tmp/tmp.vlHdhFASUL
+ rm /tmp/tmp.RpobJZRNFU /tmp/tmp.vlHdhFASUL
+ return 0
+ deploy_cert_manager
+ desc 'deploy cert manager'
+ set +o xtrace


-----------------------------------------------------------------------------------
deploy cert manager
-----------------------------------------------------------------------------------

+ kubectl_bin create namespace cert-manager
++ mktemp
+ local LAST_OUT=/tmp/tmp.rGY41hYOV7
++ mktemp
+ local LAST_ERR=/tmp/tmp.zF9XzbwVHA
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl create namespace cert-manager
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.rGY41hYOV7
namespace/cert-manager created
+ cat /tmp/tmp.zF9XzbwVHA
+ rm /tmp/tmp.rGY41hYOV7 /tmp/tmp.zF9XzbwVHA
+ return 0
+ kubectl_bin label namespace cert-manager certmanager.k8s.io/disable-validation=true
++ mktemp
+ local LAST_OUT=/tmp/tmp.IQMbNXgHZ2
++ mktemp
+ local LAST_ERR=/tmp/tmp.ZDzwtQZE5M
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl label namespace cert-manager certmanager.k8s.io/disable-validation=true
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.IQMbNXgHZ2
namespace/cert-manager labeled
+ cat /tmp/tmp.ZDzwtQZE5M
+ rm /tmp/tmp.IQMbNXgHZ2 /tmp/tmp.ZDzwtQZE5M
+ return 0
+ kubectl_bin apply -f https://github.com/cert-manager/cert-manager/releases/download/v1.15.1/cert-manager.yaml --validate=false
++ mktemp
+ local LAST_OUT=/tmp/tmp.RoGfxPfFr4
++ mktemp
+ local LAST_ERR=/tmp/tmp.R8hJNfq1MU
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl apply -f https://github.com/cert-manager/cert-manager/releases/download/v1.15.1/cert-manager.yaml --validate=false
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.RoGfxPfFr4
namespace/cert-manager configured
customresourcedefinition.apiextensions.k8s.io/certificaterequests.cert-manager.io unchanged
customresourcedefinition.apiextensions.k8s.io/certificates.cert-manager.io unchanged
customresourcedefinition.apiextensions.k8s.io/challenges.acme.cert-manager.io unchanged
customresourcedefinition.apiextensions.k8s.io/clusterissuers.cert-manager.io unchanged
customresourcedefinition.apiextensions.k8s.io/issuers.cert-manager.io unchanged
customresourcedefinition.apiextensions.k8s.io/orders.acme.cert-manager.io unchanged
serviceaccount/cert-manager-cainjector created
serviceaccount/cert-manager created
serviceaccount/cert-manager-webhook created
clusterrole.rbac.authorization.k8s.io/cert-manager-cainjector unchanged
clusterrole.rbac.authorization.k8s.io/cert-manager-controller-issuers unchanged
clusterrole.rbac.authorization.k8s.io/cert-manager-controller-clusterissuers unchanged
clusterrole.rbac.authorization.k8s.io/cert-manager-controller-certificates unchanged
clusterrole.rbac.authorization.k8s.io/cert-manager-controller-orders unchanged
clusterrole.rbac.authorization.k8s.io/cert-manager-controller-challenges unchanged
clusterrole.rbac.authorization.k8s.io/cert-manager-controller-ingress-shim unchanged
clusterrole.rbac.authorization.k8s.io/cert-manager-cluster-view unchanged
clusterrole.rbac.authorization.k8s.io/cert-manager-view unchanged
clusterrole.rbac.authorization.k8s.io/cert-manager-edit unchanged
clusterrole.rbac.authorization.k8s.io/cert-manager-controller-approve:cert-manager-io unchanged
clusterrole.rbac.authorization.k8s.io/cert-manager-controller-certificatesigningrequests unchanged
clusterrole.rbac.authorization.k8s.io/cert-manager-webhook:subjectaccessreviews unchanged
clusterrolebinding.rbac.authorization.k8s.io/cert-manager-cainjector unchanged
clusterrolebinding.rbac.authorization.k8s.io/cert-manager-controller-issuers unchanged
clusterrolebinding.rbac.authorization.k8s.io/cert-manager-controller-clusterissuers unchanged
clusterrolebinding.rbac.authorization.k8s.io/cert-manager-controller-certificates unchanged
clusterrolebinding.rbac.authorization.k8s.io/cert-manager-controller-orders unchanged
clusterrolebinding.rbac.authorization.k8s.io/cert-manager-controller-challenges unchanged
clusterrolebinding.rbac.authorization.k8s.io/cert-manager-controller-ingress-shim unchanged
clusterrolebinding.rbac.authorization.k8s.io/cert-manager-controller-approve:cert-manager-io unchanged
clusterrolebinding.rbac.authorization.k8s.io/cert-manager-controller-certificatesigningrequests unchanged
clusterrolebinding.rbac.authorization.k8s.io/cert-manager-webhook:subjectaccessreviews configured
role.rbac.authorization.k8s.io/cert-manager-cainjector:leaderelection unchanged
role.rbac.authorization.k8s.io/cert-manager:leaderelection unchanged
role.rbac.authorization.k8s.io/cert-manager-webhook:dynamic-serving created
rolebinding.rbac.authorization.k8s.io/cert-manager-cainjector:leaderelection unchanged
rolebinding.rbac.authorization.k8s.io/cert-manager:leaderelection configured
rolebinding.rbac.authorization.k8s.io/cert-manager-webhook:dynamic-serving created
service/cert-manager created
service/cert-manager-webhook created
deployment.apps/cert-manager-cainjector created
deployment.apps/cert-manager created
deployment.apps/cert-manager-webhook created
mutatingwebhookconfiguration.admissionregistration.k8s.io/cert-manager-webhook configured
validatingwebhookconfiguration.admissionregistration.k8s.io/cert-manager-webhook configured
+ cat /tmp/tmp.R8hJNfq1MU
Warning: resource namespaces/cert-manager is missing the kubectl.kubernetes.io/last-applied-configuration annotation which is required by kubectl apply. kubectl apply should only be used on resources created declaratively by either kubectl create --save-config or kubectl apply. The missing annotation will be patched automatically.
+ rm /tmp/tmp.RoGfxPfFr4 /tmp/tmp.R8hJNfq1MU
+ return 0
+ kubectl_bin -n cert-manager wait pod -l app.kubernetes.io/instance=cert-manager --for=condition=ready
++ mktemp
+ local LAST_OUT=/tmp/tmp.PczVkffkE7
++ mktemp
+ local LAST_ERR=/tmp/tmp.o8v9ut4SiS
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl -n cert-manager wait pod -l app.kubernetes.io/instance=cert-manager --for=condition=ready
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.PczVkffkE7
pod/cert-manager-bd44d64d-wpggw condition met
pod/cert-manager-cainjector-7dcddbd8b9-95wlx condition met
pod/cert-manager-webhook-6cc8fdfd7-lp99c condition met
+ cat /tmp/tmp.o8v9ut4SiS
+ rm /tmp/tmp.PczVkffkE7 /tmp/tmp.o8v9ut4SiS
+ return 0
+ sleep 120
+ desc 'install PMM Server'
+ set +o xtrace


-----------------------------------------------------------------------------------
install PMM Server
-----------------------------------------------------------------------------------

+ deploy_pmm_server
+ helm uninstall monitoring
Error: uninstall: Release not loaded: monitoring: release: not found
+ :
+ helm repo remove stable
"stable" has been removed from your repositories
+ helm repo add stable https://charts.helm.sh/stable
"stable" has been added to your repositories
+ [[ -n '' ]]
+ retry 10 60 helm install monitoring --set imageTag=dev-latest --set imageRepo=perconalab/pmm-server https://percona-charts.storage.googleapis.com/pmm-server-9.9.9.tgz
+ local max=10
+ local delay=60
+ shift 2
+ local n=1
+ helm install monitoring --set imageTag=dev-latest --set imageRepo=perconalab/pmm-server https://percona-charts.storage.googleapis.com/pmm-server-9.9.9.tgz
NAME: monitoring
LAST DEPLOYED: Fri Jul 26 06:47:24 2024
NAMESPACE: monitoring-2-0-16090
STATUS: deployed
REVISION: 1
TEST SUITE: None
NOTES:
PMM server can be accessed via HTTPS (port 443) on the following DNS name from within your cluster:

endpoint: https://monitoring-service.monitoring-2-0-16090.svc.cluster.local:443
login:    admin
password: admin
+ sleep 20
+ kubectl_bin exec monitoring-0 -- bash -c 'ls -l /proc/*/exe 2>/dev/null| grep postgres >/dev/null'
++ mktemp
+ local LAST_OUT=/tmp/tmp.ct6BG0lOxU
++ mktemp
+ local LAST_ERR=/tmp/tmp.YCfsGiTTO7
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl exec monitoring-0 -- bash -c 'ls -l /proc/*/exe 2>/dev/null| grep postgres >/dev/null'
+ exit_status=1
+ set -e
+ '[' 1 '!=' 0 -a -n 1 ']'
+ cat /tmp/tmp.ct6BG0lOxU
+ cat /tmp/tmp.YCfsGiTTO7
error: unable to upgrade connection: container not found ("monitoring")
+ sleep 0
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl exec monitoring-0 -- bash -c 'ls -l /proc/*/exe 2>/dev/null| grep postgres >/dev/null'
+ exit_status=1
+ set -e
+ '[' 1 '!=' 0 -a -n 1 ']'
+ cat /tmp/tmp.ct6BG0lOxU
+ cat /tmp/tmp.YCfsGiTTO7
error: unable to upgrade connection: container not found ("monitoring")
+ sleep 4
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl exec monitoring-0 -- bash -c 'ls -l /proc/*/exe 2>/dev/null| grep postgres >/dev/null'
+ exit_status=1
+ set -e
+ '[' 1 '!=' 0 -a -n 1 ']'
+ cat /tmp/tmp.ct6BG0lOxU
+ cat /tmp/tmp.YCfsGiTTO7
error: unable to upgrade connection: container not found ("monitoring")
+ sleep 8
+ cat /tmp/tmp.ct6BG0lOxU
+ cat /tmp/tmp.YCfsGiTTO7
error: unable to upgrade connection: container not found ("monitoring")
+ rm /tmp/tmp.ct6BG0lOxU /tmp/tmp.YCfsGiTTO7
+ return 1
+ echo 'Retry 1'
Retry 1
+ sleep 5
+ let retry+=1
+ '[' 2 -ge 20 ']'
+ kubectl_bin exec monitoring-0 -- bash -c 'ls -l /proc/*/exe 2>/dev/null| grep postgres >/dev/null'
++ mktemp
+ local LAST_OUT=/tmp/tmp.AHlTyXutw8
++ mktemp
+ local LAST_ERR=/tmp/tmp.IIp2Ne817x
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl exec monitoring-0 -- bash -c 'ls -l /proc/*/exe 2>/dev/null| grep postgres >/dev/null'
+ exit_status=1
+ set -e
+ '[' 1 '!=' 0 -a -n 1 ']'
+ cat /tmp/tmp.AHlTyXutw8
+ cat /tmp/tmp.IIp2Ne817x
error: unable to upgrade connection: container not found ("monitoring")
+ sleep 0
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl exec monitoring-0 -- bash -c 'ls -l /proc/*/exe 2>/dev/null| grep postgres >/dev/null'
+ exit_status=1
+ set -e
+ '[' 1 '!=' 0 -a -n 1 ']'
+ cat /tmp/tmp.AHlTyXutw8
+ cat /tmp/tmp.IIp2Ne817x
error: unable to upgrade connection: container not found ("monitoring")
+ sleep 4
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl exec monitoring-0 -- bash -c 'ls -l /proc/*/exe 2>/dev/null| grep postgres >/dev/null'
+ exit_status=1
+ set -e
+ '[' 1 '!=' 0 -a -n 1 ']'
+ cat /tmp/tmp.AHlTyXutw8
+ cat /tmp/tmp.IIp2Ne817x
error: unable to upgrade connection: container not found ("monitoring")
+ sleep 8
+ cat /tmp/tmp.AHlTyXutw8
+ cat /tmp/tmp.IIp2Ne817x
error: unable to upgrade connection: container not found ("monitoring")
+ rm /tmp/tmp.AHlTyXutw8 /tmp/tmp.IIp2Ne817x
+ return 1
+ echo 'Retry 2'
Retry 2
+ sleep 5
+ let retry+=1
+ '[' 3 -ge 20 ']'
+ kubectl_bin exec monitoring-0 -- bash -c 'ls -l /proc/*/exe 2>/dev/null| grep postgres >/dev/null'
++ mktemp
+ local LAST_OUT=/tmp/tmp.xuUgmeb6Bm
++ mktemp
+ local LAST_ERR=/tmp/tmp.ZzyKeyNb7n
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl exec monitoring-0 -- bash -c 'ls -l /proc/*/exe 2>/dev/null| grep postgres >/dev/null'
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.xuUgmeb6Bm
+ cat /tmp/tmp.ZzyKeyNb7n
+ rm /tmp/tmp.xuUgmeb6Bm /tmp/tmp.ZzyKeyNb7n
+ return 0
+ cluster=monitoring
+ desc 'create secrets and start client'
+ set +o xtrace


-----------------------------------------------------------------------------------
create secrets and start client
-----------------------------------------------------------------------------------

+ kubectl_bin apply -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1596/e2e-tests/conf/secrets.yml -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1596/e2e-tests/monitoring-2-0/conf/secrets.yml
++ mktemp
+ local LAST_OUT=/tmp/tmp.xZfr67xr1J
++ mktemp
+ local LAST_ERR=/tmp/tmp.jryqpoTr5k
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl apply -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1596/e2e-tests/conf/secrets.yml -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1596/e2e-tests/monitoring-2-0/conf/secrets.yml
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.xZfr67xr1J
secret/some-users created
secret/some-users unchanged
+ cat /tmp/tmp.jryqpoTr5k
+ rm /tmp/tmp.xZfr67xr1J /tmp/tmp.jryqpoTr5k
+ return 0
+ kubectl_bin apply -f -
++ mktemp
+ local LAST_OUT=/tmp/tmp.GOwwL9BlbA
++ mktemp
+ yq '.spec.template.spec.volumes[0].secret.secretName="monitoring-ssl"' /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1596/e2e-tests/conf/client_with_tls.yml
+ local LAST_ERR=/tmp/tmp.BdO9LFYGDa
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl apply -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.GOwwL9BlbA
deployment.apps/psmdb-client created
+ cat /tmp/tmp.BdO9LFYGDa
+ rm /tmp/tmp.GOwwL9BlbA /tmp/tmp.BdO9LFYGDa
+ return 0
+ sleep 90
+ desc 'create first PSMDB cluster monitoring'
+ set +o xtrace


-----------------------------------------------------------------------------------
create first PSMDB cluster monitoring
-----------------------------------------------------------------------------------

+ apply_cluster /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1596/e2e-tests/monitoring-2-0/conf/monitoring-rs0.yml
+ '[' -z '' ']'
+ cat_config /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1596/e2e-tests/monitoring-2-0/conf/monitoring-rs0.yml
+ kubectl_bin apply -f -
+ cat /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1596/e2e-tests/monitoring-2-0/conf/monitoring-rs0.yml
+ yq eval '(.spec | select(.image == null)).image = "perconalab/percona-server-mongodb-operator:main-mongod7.0"'
++ mktemp
+ yq eval '(.spec | select(has("pmm"))).pmm.image = "perconalab/pmm-client:dev-latest"'
+ yq eval '(.spec | select(has("initImage"))).initImage = "perconalab/percona-server-mongodb-operator:PR-1596-83fb9e77"'
+ yq eval '.spec.upgradeOptions.apply="Never"'
+ local LAST_OUT=/tmp/tmp.GfGbAPNgAk
+ yq eval '(.spec | select(has("backup"))).backup.image = "perconalab/percona-server-mongodb-operator:main-backup"'
++ mktemp
+ local LAST_ERR=/tmp/tmp.6KEL726HAW
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl apply -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.GfGbAPNgAk
perconaservermongodb.psmdb.percona.com/monitoring created
+ cat /tmp/tmp.6KEL726HAW
+ rm /tmp/tmp.GfGbAPNgAk /tmp/tmp.6KEL726HAW
+ return 0
+ wait_for_running monitoring-rs0 3
+ local name=monitoring-rs0
+ let last_pod=2
+ local check_cluster_readyness=true
+ set_debug
+ [[ 1 == 1 ]]
+ set -o xtrace
+ local rs_name=rs0
+ local cluster_name=monitoring
++ seq 0 2
+ for i in '$(seq 0 $last_pod)'
+ [[ 0 -eq 2 ]]
+ wait_pod monitoring-rs0-0
+ local pod=monitoring-rs0-0
+ set +o xtrace
waiting for pod/monitoring-rs0-0 to be ready.............OK
+ for i in '$(seq 0 $last_pod)'
+ [[ 1 -eq 2 ]]
+ wait_pod monitoring-rs0-1
+ local pod=monitoring-rs0-1
+ set +o xtrace
waiting for pod/monitoring-rs0-1 to be ready..............OK
+ for i in '$(seq 0 $last_pod)'
+ [[ 2 -eq 2 ]]
++ kubectl_bin get psmdb monitoring -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].arbiter.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.BV4uWxhINY
+++ mktemp
++ local LAST_ERR=/tmp/tmp.NgLdNQmBHM
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb monitoring -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].arbiter.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.BV4uWxhINY
++ cat /tmp/tmp.NgLdNQmBHM
++ rm /tmp/tmp.BV4uWxhINY /tmp/tmp.NgLdNQmBHM
++ return 0
+ [[ '' == \t\r\u\e ]]
+ wait_pod monitoring-rs0-2
+ local pod=monitoring-rs0-2
+ set +o xtrace
waiting for pod/monitoring-rs0-2 to be ready............OK
++ kubectl_bin get psmdb monitoring -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].non_voting.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.3HtObymP65
+++ mktemp
++ local LAST_ERR=/tmp/tmp.JBPikJWtyG
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb monitoring -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].non_voting.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.3HtObymP65
++ cat /tmp/tmp.JBPikJWtyG
++ rm /tmp/tmp.3HtObymP65 /tmp/tmp.JBPikJWtyG
++ return 0
+ [[ '' == \t\r\u\e ]]
+ sleep 10
+ [[ true == \t\r\u\e ]]
+ set +x
Waiting for cluster readyness....................
+ desc 'check if pmm-client container is not enabled'
+ set +o xtrace


-----------------------------------------------------------------------------------
check if pmm-client container is not enabled
-----------------------------------------------------------------------------------

+ compare_kubectl statefulset/monitoring-rs0 -no-pmm
+ local resource=statefulset/monitoring-rs0
+ local postfix=-no-pmm
+ local expected_result=/mnt/jenkins/workspace/cloud-psmdb-operator_PR-1596/e2e-tests/monitoring-2-0/compare/statefulset_monitoring-rs0-no-pmm.yml
+ local new_result=/tmp/tmp.39yPQ2cQ0t/statefulset_monitoring-rs0.yml
+ '[' -n '' -a -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1596/e2e-tests/monitoring-2-0/compare/statefulset_monitoring-rs0-no-pmm-oc.yml ']'
+ kubectl_bin get -o yaml statefulset/monitoring-rs0
++ mktemp
+ yq eval '
			del(.metadata.ownerReferences[].apiVersion) |
			del(.metadata.managedFields) |
			del(.. | select(has("creationTimestamp")).creationTimestamp) |
			del(.. | select(has("namespace")).namespace) |
			del(.. | select(has("uid")).uid) |
			del(.metadata.resourceVersion) |
			del(.spec.template.spec.containers[].env[] | select(.name == "NAMESPACE")) |
			del(.metadata.selfLink) |
			del(.metadata.annotations."cloud.google.com/neg") |
			del(.metadata.annotations."kubectl.kubernetes.io/last-applied-configuration") |
			del(.. | select(has("image")).image) |
			del(.. | select(has("clusterIP")).clusterIP) |
			del(.. | select(has("clusterIPs")).clusterIPs) |
			del(.. | select(has("dataSource")).dataSource) |
			del(.. | select(has("procMount")).procMount) |
			del(.. | select(has("storageClassName")).storageClassName) |
			del(.. | select(has("finalizers")).finalizers) |
			del(.. | select(has("kubernetes.io/pvc-protection"))."kubernetes.io/pvc-protection") |
			del(.. | select(has("volumeName")).volumeName) |
			del(.. | select(has("volume.beta.kubernetes.io/storage-provisioner"))."volume.beta.kubernetes.io/storage-provisioner") |
			del(.. | select(has("volume.kubernetes.io/storage-provisioner"))."volume.kubernetes.io/storage-provisioner") |
			del(.spec.volumeMode) |
			del(.. | select(has("volume.kubernetes.io/selected-node"))."volume.kubernetes.io/selected-node") |
			del(.. | select(has("percona.com/last-config-hash"))."percona.com/last-config-hash") |
			del(.. | select(has("percona.com/configuration-hash"))."percona.com/configuration-hash") |
			del(.. | select(has("percona.com/ssl-hash"))."percona.com/ssl-hash") |
			del(.. | select(has("percona.com/ssl-internal-hash"))."percona.com/ssl-internal-hash") |
			del(.spec.volumeClaimTemplates[].spec.volumeMode | select(. == "Filesystem")) |
			del(.. | select(has("healthCheckNodePort")).healthCheckNodePort) |
			del(.. | select(has("nodePort")).nodePort) |
			del(.status) |
			(.. | select(tag == "!!str")) |= sub("monitoring-2-0-16090", "NAME_SPACE") |
			del(.spec.volumeClaimTemplates[].apiVersion) |
			del(.spec.volumeClaimTemplates[].kind) |
			del(.spec.ipFamilies) |
			del(.spec.ipFamilyPolicy) |
			(.. | select(. == "extensions/v1beta1")) = "apps/v1" |
			(.. | select(. == "batch/v1beta1")) = "batch/v1" ' -
+ local LAST_OUT=/tmp/tmp.QLpAUpPVyJ
++ mktemp
+ local LAST_ERR=/tmp/tmp.t1Vm1fQbXW
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl get -o yaml statefulset/monitoring-rs0
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.QLpAUpPVyJ
+ cat /tmp/tmp.t1Vm1fQbXW
+ rm /tmp/tmp.QLpAUpPVyJ /tmp/tmp.t1Vm1fQbXW
+ return 0
+ yq -i eval 'del(.spec.persistentVolumeClaimRetentionPolicy)' /tmp/tmp.39yPQ2cQ0t/statefulset_monitoring-rs0.yml
+ version_gt 1.22
++ echo '1.27 >= 1.22'
++ bc -l
+ '[' 1 -eq 1 ']'
+ return 0
+ yq -i eval 'del(.spec.internalTrafficPolicy)' /tmp/tmp.39yPQ2cQ0t/statefulset_monitoring-rs0.yml
+ yq -i eval 'del(.spec.allocateLoadBalancerNodePorts)' /tmp/tmp.39yPQ2cQ0t/statefulset_monitoring-rs0.yml
+ [[ /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1596/e2e-tests/monitoring-2-0/compare/statefulset_monitoring-rs0-no-pmm.yml == */cronjob* ]]
+ diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1596/e2e-tests/monitoring-2-0/compare/statefulset_monitoring-rs0-no-pmm.yml /tmp/tmp.39yPQ2cQ0t/statefulset_monitoring-rs0.yml
+ sleep 10
+ run_mongos 'db.createUser({user:"myApp",pwd:"myPass",roles:[{db:"myApp",role:"readWrite"}]})' userAdmin:userAdmin123456@monitoring-mongos.monitoring-2-0-16090 '' '' '--tlsCertificateKeyFile /tmp/tls.pem --tlsCAFile /etc/mongodb-ssl/ca.crt --tls'
+ local 'command=db.createUser({user:"myApp",pwd:"myPass",roles:[{db:"myApp",role:"readWrite"}]})'
+ local uri=userAdmin:userAdmin123456@monitoring-mongos.monitoring-2-0-16090
+ local driver=mongodb
+ local suffix=.svc.cluster.local
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.aegtUPftGz
+++ mktemp
++ local LAST_ERR=/tmp/tmp.KUwTJ7Hemv
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.aegtUPftGz
++ cat /tmp/tmp.KUwTJ7Hemv
++ rm /tmp/tmp.aegtUPftGz /tmp/tmp.KUwTJ7Hemv
++ return 0
+ local client_container=psmdb-client-6f7f47c4b-gdrlr
+ local 'mongo_flag=--tlsCertificateKeyFile /tmp/tls.pem --tlsCAFile /etc/mongodb-ssl/ca.crt --tls'
+ kubectl_bin exec psmdb-client-6f7f47c4b-gdrlr -- bash -c 'printf '\''db.createUser({user:"myApp",pwd:"myPass",roles:[{db:"myApp",role:"readWrite"}]})\n'\'' | mongo mongodb://userAdmin:userAdmin123456@monitoring-mongos.monitoring-2-0-16090.svc.cluster.local/admin --tlsCertificateKeyFile /tmp/tls.pem --tlsCAFile /etc/mongodb-ssl/ca.crt --tls'
++ mktemp
+ local LAST_OUT=/tmp/tmp.vUfhqlCSjR
++ mktemp
+ local LAST_ERR=/tmp/tmp.QlNTOguSNK
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl exec psmdb-client-6f7f47c4b-gdrlr -- bash -c 'printf '\''db.createUser({user:"myApp",pwd:"myPass",roles:[{db:"myApp",role:"readWrite"}]})\n'\'' | mongo mongodb://userAdmin:userAdmin123456@monitoring-mongos.monitoring-2-0-16090.svc.cluster.local/admin --tlsCertificateKeyFile /tmp/tls.pem --tlsCAFile /etc/mongodb-ssl/ca.crt --tls'
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.vUfhqlCSjR
Percona Server for MongoDB shell version v4.4.29-28
connecting to: mongodb://monitoring-mongos.monitoring-2-0-16090.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb
{"t":{"$date":"2024-07-26T06:52:36.778Z"},"s":"I",  "c":"NETWORK",  "id":5490002, "ctx":"thread1","msg":"Started a new thread for the timer service"}
Implicit session: session { "id" : UUID("3cf4bd95-0320-40ef-942b-4a75d47e06e4") }
Percona Server for MongoDB server version: v7.0.12-7
WARNING: shell and server versions do not match
Successfully added user: {
	"user" : "myApp",
	"roles" : [
		{
			"db" : "myApp",
			"role" : "readWrite"
		}
	]
}
bye
+ cat /tmp/tmp.QlNTOguSNK
+ rm /tmp/tmp.vUfhqlCSjR /tmp/tmp.QlNTOguSNK
+ return 0
+ run_mongos 'sh.enableSharding("myApp")' clusterAdmin:clusterAdmin123456@monitoring-mongos.monitoring-2-0-16090 '' '' '--tlsCertificateKeyFile /tmp/tls.pem --tlsCAFile /etc/mongodb-ssl/ca.crt --tls'
+ local 'command=sh.enableSharding("myApp")'
+ local uri=clusterAdmin:clusterAdmin123456@monitoring-mongos.monitoring-2-0-16090
+ local driver=mongodb
+ local suffix=.svc.cluster.local
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.PqnhZgtz2a
+++ mktemp
++ local LAST_ERR=/tmp/tmp.rbmo4FhV4J
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.PqnhZgtz2a
++ cat /tmp/tmp.rbmo4FhV4J
++ rm /tmp/tmp.PqnhZgtz2a /tmp/tmp.rbmo4FhV4J
++ return 0
+ local client_container=psmdb-client-6f7f47c4b-gdrlr
+ local 'mongo_flag=--tlsCertificateKeyFile /tmp/tls.pem --tlsCAFile /etc/mongodb-ssl/ca.crt --tls'
+ kubectl_bin exec psmdb-client-6f7f47c4b-gdrlr -- bash -c 'printf '\''sh.enableSharding("myApp")\n'\'' | mongo mongodb://clusterAdmin:clusterAdmin123456@monitoring-mongos.monitoring-2-0-16090.svc.cluster.local/admin --tlsCertificateKeyFile /tmp/tls.pem --tlsCAFile /etc/mongodb-ssl/ca.crt --tls'
++ mktemp
+ local LAST_OUT=/tmp/tmp.tfBxBYDW25
++ mktemp
+ local LAST_ERR=/tmp/tmp.LjOvdQT989
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl exec psmdb-client-6f7f47c4b-gdrlr -- bash -c 'printf '\''sh.enableSharding("myApp")\n'\'' | mongo mongodb://clusterAdmin:clusterAdmin123456@monitoring-mongos.monitoring-2-0-16090.svc.cluster.local/admin --tlsCertificateKeyFile /tmp/tls.pem --tlsCAFile /etc/mongodb-ssl/ca.crt --tls'
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.tfBxBYDW25
Percona Server for MongoDB shell version v4.4.29-28
connecting to: mongodb://monitoring-mongos.monitoring-2-0-16090.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb
{"t":{"$date":"2024-07-26T06:52:39.776Z"},"s":"I",  "c":"NETWORK",  "id":5490002, "ctx":"thread1","msg":"Started a new thread for the timer service"}
Implicit session: session { "id" : UUID("9ee00274-c4bb-4922-8a76-5392fbaaffd3") }
Percona Server for MongoDB server version: v7.0.12-7
WARNING: shell and server versions do not match
{
	"ok" : 1,
	"$clusterTime" : {
		"clusterTime" : Timestamp(1721976759, 9),
		"signature" : {
			"hash" : BinData(0,"TyplqOFFdhGzbzEu9jb6VODfibA="),
			"keyId" : NumberLong("7395833520779690007")
		}
	},
	"operationTime" : Timestamp(1721976759, 3)
}
bye
+ cat /tmp/tmp.LjOvdQT989
+ rm /tmp/tmp.tfBxBYDW25 /tmp/tmp.LjOvdQT989
+ return 0
+ insert_data_mongos 100500 myApp '--tlsCertificateKeyFile /tmp/tls.pem --tlsCAFile /etc/mongodb-ssl/ca.crt --tls'
+ local data=100500
+ local db_name=myApp
+ local 'flags=--tlsCertificateKeyFile /tmp/tls.pem --tlsCAFile /etc/mongodb-ssl/ca.crt --tls'
+ run_mongos 'use myApp\n db.test.insert({ x: 100500 })' myApp:myPass@monitoring-mongos.monitoring-2-0-16090 '' '' '--tlsCertificateKeyFile /tmp/tls.pem --tlsCAFile /etc/mongodb-ssl/ca.crt --tls'
+ local 'command=use myApp\n db.test.insert({ x: 100500 })'
+ local uri=myApp:myPass@monitoring-mongos.monitoring-2-0-16090
+ local driver=mongodb
+ local suffix=.svc.cluster.local
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.9zEb2yJcvb
+++ mktemp
++ local LAST_ERR=/tmp/tmp.OvR6QkKupA
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.9zEb2yJcvb
++ cat /tmp/tmp.OvR6QkKupA
++ rm /tmp/tmp.9zEb2yJcvb /tmp/tmp.OvR6QkKupA
++ return 0
+ local client_container=psmdb-client-6f7f47c4b-gdrlr
+ local 'mongo_flag=--tlsCertificateKeyFile /tmp/tls.pem --tlsCAFile /etc/mongodb-ssl/ca.crt --tls'
+ kubectl_bin exec psmdb-client-6f7f47c4b-gdrlr -- bash -c 'printf '\''use myApp\n db.test.insert({ x: 100500 })\n'\'' | mongo mongodb://myApp:myPass@monitoring-mongos.monitoring-2-0-16090.svc.cluster.local/admin --tlsCertificateKeyFile /tmp/tls.pem --tlsCAFile /etc/mongodb-ssl/ca.crt --tls'
++ mktemp
+ local LAST_OUT=/tmp/tmp.Xacu8LjAXo
++ mktemp
+ local LAST_ERR=/tmp/tmp.HPN2ON2qBl
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl exec psmdb-client-6f7f47c4b-gdrlr -- bash -c 'printf '\''use myApp\n db.test.insert({ x: 100500 })\n'\'' | mongo mongodb://myApp:myPass@monitoring-mongos.monitoring-2-0-16090.svc.cluster.local/admin --tlsCertificateKeyFile /tmp/tls.pem --tlsCAFile /etc/mongodb-ssl/ca.crt --tls'
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.Xacu8LjAXo
Percona Server for MongoDB shell version v4.4.29-28
connecting to: mongodb://monitoring-mongos.monitoring-2-0-16090.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb
{"t":{"$date":"2024-07-26T06:52:42.295Z"},"s":"I",  "c":"NETWORK",  "id":5490002, "ctx":"thread1","msg":"Started a new thread for the timer service"}
Implicit session: session { "id" : UUID("8f492f77-d721-4bda-8cea-c321e2630600") }
Percona Server for MongoDB server version: v7.0.12-7
WARNING: shell and server versions do not match
switched to db myApp
WriteResult({ "nInserted" : 1 })
bye
+ cat /tmp/tmp.HPN2ON2qBl
+ rm /tmp/tmp.Xacu8LjAXo /tmp/tmp.HPN2ON2qBl
+ return 0
+ insert_data_mongos 100600 myApp '--tlsCertificateKeyFile /tmp/tls.pem --tlsCAFile /etc/mongodb-ssl/ca.crt --tls'
+ local data=100600
+ local db_name=myApp
+ local 'flags=--tlsCertificateKeyFile /tmp/tls.pem --tlsCAFile /etc/mongodb-ssl/ca.crt --tls'
+ run_mongos 'use myApp\n db.test.insert({ x: 100600 })' myApp:myPass@monitoring-mongos.monitoring-2-0-16090 '' '' '--tlsCertificateKeyFile /tmp/tls.pem --tlsCAFile /etc/mongodb-ssl/ca.crt --tls'
+ local 'command=use myApp\n db.test.insert({ x: 100600 })'
+ local uri=myApp:myPass@monitoring-mongos.monitoring-2-0-16090
+ local driver=mongodb
+ local suffix=.svc.cluster.local
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.3eAltaAsVV
+++ mktemp
++ local LAST_ERR=/tmp/tmp.i4t0K0eB5U
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.3eAltaAsVV
++ cat /tmp/tmp.i4t0K0eB5U
++ rm /tmp/tmp.3eAltaAsVV /tmp/tmp.i4t0K0eB5U
++ return 0
+ local client_container=psmdb-client-6f7f47c4b-gdrlr
+ local 'mongo_flag=--tlsCertificateKeyFile /tmp/tls.pem --tlsCAFile /etc/mongodb-ssl/ca.crt --tls'
+ kubectl_bin exec psmdb-client-6f7f47c4b-gdrlr -- bash -c 'printf '\''use myApp\n db.test.insert({ x: 100600 })\n'\'' | mongo mongodb://myApp:myPass@monitoring-mongos.monitoring-2-0-16090.svc.cluster.local/admin --tlsCertificateKeyFile /tmp/tls.pem --tlsCAFile /etc/mongodb-ssl/ca.crt --tls'
++ mktemp
+ local LAST_OUT=/tmp/tmp.IEq15GxrB4
++ mktemp
+ local LAST_ERR=/tmp/tmp.aYZQ4Th4E6
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl exec psmdb-client-6f7f47c4b-gdrlr -- bash -c 'printf '\''use myApp\n db.test.insert({ x: 100600 })\n'\'' | mongo mongodb://myApp:myPass@monitoring-mongos.monitoring-2-0-16090.svc.cluster.local/admin --tlsCertificateKeyFile /tmp/tls.pem --tlsCAFile /etc/mongodb-ssl/ca.crt --tls'
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.IEq15GxrB4
Percona Server for MongoDB shell version v4.4.29-28
connecting to: mongodb://monitoring-mongos.monitoring-2-0-16090.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb
{"t":{"$date":"2024-07-26T06:52:45.582Z"},"s":"I",  "c":"NETWORK",  "id":5490002, "ctx":"thread1","msg":"Started a new thread for the timer service"}
Implicit session: session { "id" : UUID("33000a8f-8591-4e33-977d-441946576d46") }
Percona Server for MongoDB server version: v7.0.12-7
WARNING: shell and server versions do not match
switched to db myApp
WriteResult({ "nInserted" : 1 })
bye
+ cat /tmp/tmp.aYZQ4Th4E6
+ rm /tmp/tmp.IEq15GxrB4 /tmp/tmp.aYZQ4Th4E6
+ return 0
+ insert_data_mongos 100700 myApp '--tlsCertificateKeyFile /tmp/tls.pem --tlsCAFile /etc/mongodb-ssl/ca.crt --tls'
+ local data=100700
+ local db_name=myApp
+ local 'flags=--tlsCertificateKeyFile /tmp/tls.pem --tlsCAFile /etc/mongodb-ssl/ca.crt --tls'
+ run_mongos 'use myApp\n db.test.insert({ x: 100700 })' myApp:myPass@monitoring-mongos.monitoring-2-0-16090 '' '' '--tlsCertificateKeyFile /tmp/tls.pem --tlsCAFile /etc/mongodb-ssl/ca.crt --tls'
+ local 'command=use myApp\n db.test.insert({ x: 100700 })'
+ local uri=myApp:myPass@monitoring-mongos.monitoring-2-0-16090
+ local driver=mongodb
+ local suffix=.svc.cluster.local
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.c58lRSnWP8
+++ mktemp
++ local LAST_ERR=/tmp/tmp.XAOIiBu9eL
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.c58lRSnWP8
++ cat /tmp/tmp.XAOIiBu9eL
++ rm /tmp/tmp.c58lRSnWP8 /tmp/tmp.XAOIiBu9eL
++ return 0
+ local client_container=psmdb-client-6f7f47c4b-gdrlr
+ local 'mongo_flag=--tlsCertificateKeyFile /tmp/tls.pem --tlsCAFile /etc/mongodb-ssl/ca.crt --tls'
+ kubectl_bin exec psmdb-client-6f7f47c4b-gdrlr -- bash -c 'printf '\''use myApp\n db.test.insert({ x: 100700 })\n'\'' | mongo mongodb://myApp:myPass@monitoring-mongos.monitoring-2-0-16090.svc.cluster.local/admin --tlsCertificateKeyFile /tmp/tls.pem --tlsCAFile /etc/mongodb-ssl/ca.crt --tls'
++ mktemp
+ local LAST_OUT=/tmp/tmp.4ZYJagw3gP
++ mktemp
+ local LAST_ERR=/tmp/tmp.aICC5Bi93o
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl exec psmdb-client-6f7f47c4b-gdrlr -- bash -c 'printf '\''use myApp\n db.test.insert({ x: 100700 })\n'\'' | mongo mongodb://myApp:myPass@monitoring-mongos.monitoring-2-0-16090.svc.cluster.local/admin --tlsCertificateKeyFile /tmp/tls.pem --tlsCAFile /etc/mongodb-ssl/ca.crt --tls'
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.4ZYJagw3gP
Percona Server for MongoDB shell version v4.4.29-28
connecting to: mongodb://monitoring-mongos.monitoring-2-0-16090.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb
{"t":{"$date":"2024-07-26T06:52:48.228Z"},"s":"I",  "c":"NETWORK",  "id":5490002, "ctx":"thread1","msg":"Started a new thread for the timer service"}
Implicit session: session { "id" : UUID("53e5bc00-5c09-4a00-95e0-3bd53dbd9d30") }
Percona Server for MongoDB server version: v7.0.12-7
WARNING: shell and server versions do not match
switched to db myApp
WriteResult({ "nInserted" : 1 })
bye
+ cat /tmp/tmp.aICC5Bi93o
+ rm /tmp/tmp.4ZYJagw3gP /tmp/tmp.aICC5Bi93o
+ return 0
+ desc 'add PMM_SERVER_API_KEY for secret some-users'
+ set +o xtrace


-----------------------------------------------------------------------------------
add PMM_SERVER_API_KEY for secret some-users
-----------------------------------------------------------------------------------

++ jq .key
+++ get_service_endpoint monitoring-service
+++ local service=monitoring-service
++++ kubectl_bin get service/monitoring-service -o json
++++ jq '.status.loadBalancer.ingress[].hostname'
++++ sed -e 's/^"//; s/"$//;'
+++++ mktemp
++++ local LAST_OUT=/tmp/tmp.PUeAJAMPpI
+++++ mktemp
++++ local LAST_ERR=/tmp/tmp.mESUvxe8Hd
++++ local exit_status=0
++++ local timeout=4
+++++ seq 0 2
++++ for i in '$(seq 0 2)'
++++ set +e
++++ kubectl get service/monitoring-service -o json
++++ exit_status=0
++++ set -e
++++ '[' 0 '!=' 0 -a -n 1 ']'
++++ break
++++ cat /tmp/tmp.PUeAJAMPpI
++++ cat /tmp/tmp.mESUvxe8Hd
++++ rm /tmp/tmp.PUeAJAMPpI /tmp/tmp.mESUvxe8Hd
++++ return 0
+++ local hostname=null
+++ '[' -n null -a null '!=' null ']'
++++ kubectl_bin get service/monitoring-service -o json
++++ sed -e 's/^"//; s/"$//;'
+++++ mktemp
++++ local LAST_OUT=/tmp/tmp.uIhJHFC5ZA
++++ jq '.status.loadBalancer.ingress[].ip'
+++++ mktemp
++++ local LAST_ERR=/tmp/tmp.NsCmjwRe5S
++++ local exit_status=0
++++ local timeout=4
+++++ seq 0 2
++++ for i in '$(seq 0 2)'
++++ set +e
++++ kubectl get service/monitoring-service -o json
++++ exit_status=0
++++ set -e
++++ '[' 0 '!=' 0 -a -n 1 ']'
++++ break
++++ cat /tmp/tmp.uIhJHFC5ZA
++++ cat /tmp/tmp.NsCmjwRe5S
++++ rm /tmp/tmp.uIhJHFC5ZA /tmp/tmp.NsCmjwRe5S
++++ return 0
+++ local ip=35.192.90.18
+++ '[' -n 35.192.90.18 -a 35.192.90.18 '!=' null ']'
+++ echo 35.192.90.18
+++ return
++ curl --insecure -X POST -H 'Content-Type: application/json' -d '{"name":"operator", "role": "Admin"}' https://admin:admin@35.192.90.18/graph/api/auth/keys
  % Total    % Received % Xferd  Average Speed   Time    Time     Time  Current
                                 Dload  Upload   Total   Spent    Left  Speed

  0     0    0     0    0     0      0      0 --:--:-- --:--:-- --:--:--     0
100   155  100   119  100    36    249     75 --:--:-- --:--:-- --:--:--   324
+ API_KEY='"eyJrIjoiRW5FWmdGTFYydEhHN1hoV1RrZEkzRFVSRGI5YVNXa2QiLCJuIjoib3BlcmF0b3IiLCJpZCI6MX0="'
+ kubectl_bin patch secret some-users --type merge --patch '{"stringData": {"PMM_SERVER_API_KEY": "eyJrIjoiRW5FWmdGTFYydEhHN1hoV1RrZEkzRFVSRGI5YVNXa2QiLCJuIjoib3BlcmF0b3IiLCJpZCI6MX0="}}'
++ mktemp
+ local LAST_OUT=/tmp/tmp.7MqRxIeNCN
++ mktemp
+ local LAST_ERR=/tmp/tmp.U09ddLhfgI
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl patch secret some-users --type merge --patch '{"stringData": {"PMM_SERVER_API_KEY": "eyJrIjoiRW5FWmdGTFYydEhHN1hoV1RrZEkzRFVSRGI5YVNXa2QiLCJuIjoib3BlcmF0b3IiLCJpZCI6MX0="}}'
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.7MqRxIeNCN
secret/some-users patched
+ cat /tmp/tmp.U09ddLhfgI
+ rm /tmp/tmp.7MqRxIeNCN /tmp/tmp.U09ddLhfgI
+ return 0
+ desc 'check if all 3 Pods started'
+ set +o xtrace


-----------------------------------------------------------------------------------
check if all 3 Pods started
-----------------------------------------------------------------------------------

+ wait_for_running monitoring-rs0 3
+ local name=monitoring-rs0
+ let last_pod=2
+ local check_cluster_readyness=true
+ set_debug
+ [[ 1 == 1 ]]
+ set -o xtrace
+ local rs_name=rs0
+ local cluster_name=monitoring
++ seq 0 2
+ for i in '$(seq 0 $last_pod)'
+ [[ 0 -eq 2 ]]
+ wait_pod monitoring-rs0-0
+ local pod=monitoring-rs0-0
+ set +o xtrace
waiting for pod/monitoring-rs0-0 to be ready.OK
+ for i in '$(seq 0 $last_pod)'
+ [[ 1 -eq 2 ]]
+ wait_pod monitoring-rs0-1
+ local pod=monitoring-rs0-1
+ set +o xtrace
waiting for pod/monitoring-rs0-1 to be ready.OK
+ for i in '$(seq 0 $last_pod)'
+ [[ 2 -eq 2 ]]
++ kubectl_bin get psmdb monitoring -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].arbiter.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.ENCbZDAMLM
+++ mktemp
++ local LAST_ERR=/tmp/tmp.d1hPLG1Cte
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb monitoring -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].arbiter.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.ENCbZDAMLM
++ cat /tmp/tmp.d1hPLG1Cte
++ rm /tmp/tmp.ENCbZDAMLM /tmp/tmp.d1hPLG1Cte
++ return 0
+ [[ '' == \t\r\u\e ]]
+ wait_pod monitoring-rs0-2
+ local pod=monitoring-rs0-2
+ set +o xtrace
waiting for pod/monitoring-rs0-2 to be ready.OK
++ kubectl_bin get psmdb monitoring -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].non_voting.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.yu8bB6uI6k
+++ mktemp
++ local LAST_ERR=/tmp/tmp.rAAkzzdMU8
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb monitoring -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].non_voting.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.yu8bB6uI6k
++ cat /tmp/tmp.rAAkzzdMU8
++ rm /tmp/tmp.yu8bB6uI6k /tmp/tmp.rAAkzzdMU8
++ return 0
+ [[ '' == \t\r\u\e ]]
+ sleep 10
+ [[ true == \t\r\u\e ]]
+ set +x
Waiting for cluster readyness...........................................................................................
+ sleep 90
+ desc 'check if pmm-client container enabled'
+ set +o xtrace


-----------------------------------------------------------------------------------
check if pmm-client container enabled
-----------------------------------------------------------------------------------

+ compare_kubectl statefulset/monitoring-rs0
+ local resource=statefulset/monitoring-rs0
+ local postfix=
+ local expected_result=/mnt/jenkins/workspace/cloud-psmdb-operator_PR-1596/e2e-tests/monitoring-2-0/compare/statefulset_monitoring-rs0.yml
+ local new_result=/tmp/tmp.39yPQ2cQ0t/statefulset_monitoring-rs0.yml
+ '[' -n '' -a -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1596/e2e-tests/monitoring-2-0/compare/statefulset_monitoring-rs0-oc.yml ']'
+ yq eval '
			del(.metadata.ownerReferences[].apiVersion) |
			del(.metadata.managedFields) |
			del(.. | select(has("creationTimestamp")).creationTimestamp) |
			del(.. | select(has("namespace")).namespace) |
			del(.. | select(has("uid")).uid) |
			del(.metadata.resourceVersion) |
			del(.spec.template.spec.containers[].env[] | select(.name == "NAMESPACE")) |
			del(.metadata.selfLink) |
			del(.metadata.annotations."cloud.google.com/neg") |
			del(.metadata.annotations."kubectl.kubernetes.io/last-applied-configuration") |
			del(.. | select(has("image")).image) |
			del(.. | select(has("clusterIP")).clusterIP) |
			del(.. | select(has("clusterIPs")).clusterIPs) |
			del(.. | select(has("dataSource")).dataSource) |
			del(.. | select(has("procMount")).procMount) |
			del(.. | select(has("storageClassName")).storageClassName) |
			del(.. | select(has("finalizers")).finalizers) |
			del(.. | select(has("kubernetes.io/pvc-protection"))."kubernetes.io/pvc-protection") |
			del(.. | select(has("volumeName")).volumeName) |
			del(.. | select(has("volume.beta.kubernetes.io/storage-provisioner"))."volume.beta.kubernetes.io/storage-provisioner") |
			del(.. | select(has("volume.kubernetes.io/storage-provisioner"))."volume.kubernetes.io/storage-provisioner") |
			del(.spec.volumeMode) |
			del(.. | select(has("volume.kubernetes.io/selected-node"))."volume.kubernetes.io/selected-node") |
			del(.. | select(has("percona.com/last-config-hash"))."percona.com/last-config-hash") |
			del(.. | select(has("percona.com/configuration-hash"))."percona.com/configuration-hash") |
			del(.. | select(has("percona.com/ssl-hash"))."percona.com/ssl-hash") |
			del(.. | select(has("percona.com/ssl-internal-hash"))."percona.com/ssl-internal-hash") |
			del(.spec.volumeClaimTemplates[].spec.volumeMode | select(. == "Filesystem")) |
			del(.. | select(has("healthCheckNodePort")).healthCheckNodePort) |
			del(.. | select(has("nodePort")).nodePort) |
			del(.status) |
			(.. | select(tag == "!!str")) |= sub("monitoring-2-0-16090", "NAME_SPACE") |
			del(.spec.volumeClaimTemplates[].apiVersion) |
			del(.spec.volumeClaimTemplates[].kind) |
			del(.spec.ipFamilies) |
			del(.spec.ipFamilyPolicy) |
			(.. | select(. == "extensions/v1beta1")) = "apps/v1" |
			(.. | select(. == "batch/v1beta1")) = "batch/v1" ' -
+ kubectl_bin get -o yaml statefulset/monitoring-rs0
++ mktemp
+ local LAST_OUT=/tmp/tmp.wVIaKhaGZ9
++ mktemp
+ local LAST_ERR=/tmp/tmp.gUuqoJHqkk
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl get -o yaml statefulset/monitoring-rs0
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.wVIaKhaGZ9
+ cat /tmp/tmp.gUuqoJHqkk
+ rm /tmp/tmp.wVIaKhaGZ9 /tmp/tmp.gUuqoJHqkk
+ return 0
+ yq -i eval 'del(.spec.persistentVolumeClaimRetentionPolicy)' /tmp/tmp.39yPQ2cQ0t/statefulset_monitoring-rs0.yml
+ version_gt 1.22
++ bc -l
++ echo '1.27 >= 1.22'
+ '[' 1 -eq 1 ']'
+ return 0
+ yq -i eval 'del(.spec.internalTrafficPolicy)' /tmp/tmp.39yPQ2cQ0t/statefulset_monitoring-rs0.yml
+ yq -i eval 'del(.spec.allocateLoadBalancerNodePorts)' /tmp/tmp.39yPQ2cQ0t/statefulset_monitoring-rs0.yml
+ [[ /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1596/e2e-tests/monitoring-2-0/compare/statefulset_monitoring-rs0.yml == */cronjob* ]]
+ diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1596/e2e-tests/monitoring-2-0/compare/statefulset_monitoring-rs0.yml /tmp/tmp.39yPQ2cQ0t/statefulset_monitoring-rs0.yml
+ compare_kubectl service/monitoring-rs0
+ local resource=service/monitoring-rs0
+ local postfix=
+ local expected_result=/mnt/jenkins/workspace/cloud-psmdb-operator_PR-1596/e2e-tests/monitoring-2-0/compare/service_monitoring-rs0.yml
+ local new_result=/tmp/tmp.39yPQ2cQ0t/service_monitoring-rs0.yml
+ '[' -n '' -a -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1596/e2e-tests/monitoring-2-0/compare/service_monitoring-rs0-oc.yml ']'
+ yq eval '
			del(.metadata.ownerReferences[].apiVersion) |
			del(.metadata.managedFields) |
			del(.. | select(has("creationTimestamp")).creationTimestamp) |
			del(.. | select(has("namespace")).namespace) |
			del(.. | select(has("uid")).uid) |
			del(.metadata.resourceVersion) |
			del(.spec.template.spec.containers[].env[] | select(.name == "NAMESPACE")) |
			del(.metadata.selfLink) |
			del(.metadata.annotations."cloud.google.com/neg") |
			del(.metadata.annotations."kubectl.kubernetes.io/last-applied-configuration") |
			del(.. | select(has("image")).image) |
			del(.. | select(has("clusterIP")).clusterIP) |
			del(.. | select(has("clusterIPs")).clusterIPs) |
			del(.. | select(has("dataSource")).dataSource) |
			del(.. | select(has("procMount")).procMount) |
			del(.. | select(has("storageClassName")).storageClassName) |
			del(.. | select(has("finalizers")).finalizers) |
			del(.. | select(has("kubernetes.io/pvc-protection"))."kubernetes.io/pvc-protection") |
			del(.. | select(has("volumeName")).volumeName) |
			del(.. | select(has("volume.beta.kubernetes.io/storage-provisioner"))."volume.beta.kubernetes.io/storage-provisioner") |
			del(.. | select(has("volume.kubernetes.io/storage-provisioner"))."volume.kubernetes.io/storage-provisioner") |
			del(.spec.volumeMode) |
			del(.. | select(has("volume.kubernetes.io/selected-node"))."volume.kubernetes.io/selected-node") |
			del(.. | select(has("percona.com/last-config-hash"))."percona.com/last-config-hash") |
			del(.. | select(has("percona.com/configuration-hash"))."percona.com/configuration-hash") |
			del(.. | select(has("percona.com/ssl-hash"))."percona.com/ssl-hash") |
			del(.. | select(has("percona.com/ssl-internal-hash"))."percona.com/ssl-internal-hash") |
			del(.spec.volumeClaimTemplates[].spec.volumeMode | select(. == "Filesystem")) |
			del(.. | select(has("healthCheckNodePort")).healthCheckNodePort) |
			del(.. | select(has("nodePort")).nodePort) |
			del(.status) |
			(.. | select(tag == "!!str")) |= sub("monitoring-2-0-16090", "NAME_SPACE") |
			del(.spec.volumeClaimTemplates[].apiVersion) |
			del(.spec.volumeClaimTemplates[].kind) |
			del(.spec.ipFamilies) |
			del(.spec.ipFamilyPolicy) |
			(.. | select(. == "extensions/v1beta1")) = "apps/v1" |
			(.. | select(. == "batch/v1beta1")) = "batch/v1" ' -
+ kubectl_bin get -o yaml service/monitoring-rs0
++ mktemp
+ local LAST_OUT=/tmp/tmp.CG7Aa2WwHX
++ mktemp
+ local LAST_ERR=/tmp/tmp.W7WidMLPub
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl get -o yaml service/monitoring-rs0
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.CG7Aa2WwHX
+ cat /tmp/tmp.W7WidMLPub
+ rm /tmp/tmp.CG7Aa2WwHX /tmp/tmp.W7WidMLPub
+ return 0
+ yq -i eval 'del(.spec.persistentVolumeClaimRetentionPolicy)' /tmp/tmp.39yPQ2cQ0t/service_monitoring-rs0.yml
+ version_gt 1.22
++ echo '1.27 >= 1.22'
++ bc -l
+ '[' 1 -eq 1 ']'
+ return 0
+ yq -i eval 'del(.spec.internalTrafficPolicy)' /tmp/tmp.39yPQ2cQ0t/service_monitoring-rs0.yml
+ yq -i eval 'del(.spec.allocateLoadBalancerNodePorts)' /tmp/tmp.39yPQ2cQ0t/service_monitoring-rs0.yml
+ [[ /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1596/e2e-tests/monitoring-2-0/compare/service_monitoring-rs0.yml == */cronjob* ]]
+ diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1596/e2e-tests/monitoring-2-0/compare/service_monitoring-rs0.yml /tmp/tmp.39yPQ2cQ0t/service_monitoring-rs0.yml
+ compare_kubectl service/monitoring-mongos
+ local resource=service/monitoring-mongos
+ local postfix=
+ local expected_result=/mnt/jenkins/workspace/cloud-psmdb-operator_PR-1596/e2e-tests/monitoring-2-0/compare/service_monitoring-mongos.yml
+ local new_result=/tmp/tmp.39yPQ2cQ0t/service_monitoring-mongos.yml
+ '[' -n '' -a -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1596/e2e-tests/monitoring-2-0/compare/service_monitoring-mongos-oc.yml ']'
+ kubectl_bin get -o yaml service/monitoring-mongos
+ yq eval '
			del(.metadata.ownerReferences[].apiVersion) |
			del(.metadata.managedFields) |
			del(.. | select(has("creationTimestamp")).creationTimestamp) |
			del(.. | select(has("namespace")).namespace) |
			del(.. | select(has("uid")).uid) |
			del(.metadata.resourceVersion) |
			del(.spec.template.spec.containers[].env[] | select(.name == "NAMESPACE")) |
			del(.metadata.selfLink) |
			del(.metadata.annotations."cloud.google.com/neg") |
			del(.metadata.annotations."kubectl.kubernetes.io/last-applied-configuration") |
			del(.. | select(has("image")).image) |
			del(.. | select(has("clusterIP")).clusterIP) |
			del(.. | select(has("clusterIPs")).clusterIPs) |
			del(.. | select(has("dataSource")).dataSource) |
			del(.. | select(has("procMount")).procMount) |
			del(.. | select(has("storageClassName")).storageClassName) |
			del(.. | select(has("finalizers")).finalizers) |
			del(.. | select(has("kubernetes.io/pvc-protection"))."kubernetes.io/pvc-protection") |
			del(.. | select(has("volumeName")).volumeName) |
			del(.. | select(has("volume.beta.kubernetes.io/storage-provisioner"))."volume.beta.kubernetes.io/storage-provisioner") |
			del(.. | select(has("volume.kubernetes.io/storage-provisioner"))."volume.kubernetes.io/storage-provisioner") |
			del(.spec.volumeMode) |
			del(.. | select(has("volume.kubernetes.io/selected-node"))."volume.kubernetes.io/selected-node") |
			del(.. | select(has("percona.com/last-config-hash"))."percona.com/last-config-hash") |
			del(.. | select(has("percona.com/configuration-hash"))."percona.com/configuration-hash") |
			del(.. | select(has("percona.com/ssl-hash"))."percona.com/ssl-hash") |
			del(.. | select(has("percona.com/ssl-internal-hash"))."percona.com/ssl-internal-hash") |
			del(.spec.volumeClaimTemplates[].spec.volumeMode | select(. == "Filesystem")) |
			del(.. | select(has("healthCheckNodePort")).healthCheckNodePort) |
			del(.. | select(has("nodePort")).nodePort) |
			del(.status) |
			(.. | select(tag == "!!str")) |= sub("monitoring-2-0-16090", "NAME_SPACE") |
			del(.spec.volumeClaimTemplates[].apiVersion) |
			del(.spec.volumeClaimTemplates[].kind) |
			del(.spec.ipFamilies) |
			del(.spec.ipFamilyPolicy) |
			(.. | select(. == "extensions/v1beta1")) = "apps/v1" |
			(.. | select(. == "batch/v1beta1")) = "batch/v1" ' -
++ mktemp
+ local LAST_OUT=/tmp/tmp.z3E9LXVTXj
++ mktemp
+ local LAST_ERR=/tmp/tmp.SGJKleTQbC
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl get -o yaml service/monitoring-mongos
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.z3E9LXVTXj
+ cat /tmp/tmp.SGJKleTQbC
+ rm /tmp/tmp.z3E9LXVTXj /tmp/tmp.SGJKleTQbC
+ return 0
+ yq -i eval 'del(.spec.persistentVolumeClaimRetentionPolicy)' /tmp/tmp.39yPQ2cQ0t/service_monitoring-mongos.yml
+ version_gt 1.22
++ echo '1.27 >= 1.22'
++ bc -l
+ '[' 1 -eq 1 ']'
+ return 0
+ yq -i eval 'del(.spec.internalTrafficPolicy)' /tmp/tmp.39yPQ2cQ0t/service_monitoring-mongos.yml
+ yq -i eval 'del(.spec.allocateLoadBalancerNodePorts)' /tmp/tmp.39yPQ2cQ0t/service_monitoring-mongos.yml
+ [[ /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1596/e2e-tests/monitoring-2-0/compare/service_monitoring-mongos.yml == */cronjob* ]]
+ diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1596/e2e-tests/monitoring-2-0/compare/service_monitoring-mongos.yml /tmp/tmp.39yPQ2cQ0t/service_monitoring-mongos.yml
+ compare_kubectl statefulset/monitoring-cfg
+ local resource=statefulset/monitoring-cfg
+ local postfix=
+ local expected_result=/mnt/jenkins/workspace/cloud-psmdb-operator_PR-1596/e2e-tests/monitoring-2-0/compare/statefulset_monitoring-cfg.yml
+ local new_result=/tmp/tmp.39yPQ2cQ0t/statefulset_monitoring-cfg.yml
+ '[' -n '' -a -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1596/e2e-tests/monitoring-2-0/compare/statefulset_monitoring-cfg-oc.yml ']'
+ kubectl_bin get -o yaml statefulset/monitoring-cfg
+ yq eval '
			del(.metadata.ownerReferences[].apiVersion) |
			del(.metadata.managedFields) |
			del(.. | select(has("creationTimestamp")).creationTimestamp) |
			del(.. | select(has("namespace")).namespace) |
			del(.. | select(has("uid")).uid) |
			del(.metadata.resourceVersion) |
			del(.spec.template.spec.containers[].env[] | select(.name == "NAMESPACE")) |
			del(.metadata.selfLink) |
			del(.metadata.annotations."cloud.google.com/neg") |
			del(.metadata.annotations."kubectl.kubernetes.io/last-applied-configuration") |
			del(.. | select(has("image")).image) |
			del(.. | select(has("clusterIP")).clusterIP) |
			del(.. | select(has("clusterIPs")).clusterIPs) |
			del(.. | select(has("dataSource")).dataSource) |
			del(.. | select(has("procMount")).procMount) |
			del(.. | select(has("storageClassName")).storageClassName) |
			del(.. | select(has("finalizers")).finalizers) |
			del(.. | select(has("kubernetes.io/pvc-protection"))."kubernetes.io/pvc-protection") |
			del(.. | select(has("volumeName")).volumeName) |
			del(.. | select(has("volume.beta.kubernetes.io/storage-provisioner"))."volume.beta.kubernetes.io/storage-provisioner") |
			del(.. | select(has("volume.kubernetes.io/storage-provisioner"))."volume.kubernetes.io/storage-provisioner") |
			del(.spec.volumeMode) |
			del(.. | select(has("volume.kubernetes.io/selected-node"))."volume.kubernetes.io/selected-node") |
			del(.. | select(has("percona.com/last-config-hash"))."percona.com/last-config-hash") |
			del(.. | select(has("percona.com/configuration-hash"))."percona.com/configuration-hash") |
			del(.. | select(has("percona.com/ssl-hash"))."percona.com/ssl-hash") |
			del(.. | select(has("percona.com/ssl-internal-hash"))."percona.com/ssl-internal-hash") |
			del(.spec.volumeClaimTemplates[].spec.volumeMode | select(. == "Filesystem")) |
			del(.. | select(has("healthCheckNodePort")).healthCheckNodePort) |
			del(.. | select(has("nodePort")).nodePort) |
			del(.status) |
			(.. | select(tag == "!!str")) |= sub("monitoring-2-0-16090", "NAME_SPACE") |
			del(.spec.volumeClaimTemplates[].apiVersion) |
			del(.spec.volumeClaimTemplates[].kind) |
			del(.spec.ipFamilies) |
			del(.spec.ipFamilyPolicy) |
			(.. | select(. == "extensions/v1beta1")) = "apps/v1" |
			(.. | select(. == "batch/v1beta1")) = "batch/v1" ' -
++ mktemp
+ local LAST_OUT=/tmp/tmp.VoXm8R9lqU
++ mktemp
+ local LAST_ERR=/tmp/tmp.G3UDRyAGGA
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl get -o yaml statefulset/monitoring-cfg
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.VoXm8R9lqU
+ cat /tmp/tmp.G3UDRyAGGA
+ rm /tmp/tmp.VoXm8R9lqU /tmp/tmp.G3UDRyAGGA
+ return 0
+ yq -i eval 'del(.spec.persistentVolumeClaimRetentionPolicy)' /tmp/tmp.39yPQ2cQ0t/statefulset_monitoring-cfg.yml
+ version_gt 1.22
++ echo '1.27 >= 1.22'
++ bc -l
+ '[' 1 -eq 1 ']'
+ return 0
+ yq -i eval 'del(.spec.internalTrafficPolicy)' /tmp/tmp.39yPQ2cQ0t/statefulset_monitoring-cfg.yml
+ yq -i eval 'del(.spec.allocateLoadBalancerNodePorts)' /tmp/tmp.39yPQ2cQ0t/statefulset_monitoring-cfg.yml
+ [[ /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1596/e2e-tests/monitoring-2-0/compare/statefulset_monitoring-cfg.yml == */cronjob* ]]
+ diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1596/e2e-tests/monitoring-2-0/compare/statefulset_monitoring-cfg.yml /tmp/tmp.39yPQ2cQ0t/statefulset_monitoring-cfg.yml
+ compare_kubectl statefulset/monitoring-mongos
+ local resource=statefulset/monitoring-mongos
+ local postfix=
+ local expected_result=/mnt/jenkins/workspace/cloud-psmdb-operator_PR-1596/e2e-tests/monitoring-2-0/compare/statefulset_monitoring-mongos.yml
+ local new_result=/tmp/tmp.39yPQ2cQ0t/statefulset_monitoring-mongos.yml
+ '[' -n '' -a -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1596/e2e-tests/monitoring-2-0/compare/statefulset_monitoring-mongos-oc.yml ']'
+ kubectl_bin get -o yaml statefulset/monitoring-mongos
+ yq eval '
			del(.metadata.ownerReferences[].apiVersion) |
			del(.metadata.managedFields) |
			del(.. | select(has("creationTimestamp")).creationTimestamp) |
			del(.. | select(has("namespace")).namespace) |
			del(.. | select(has("uid")).uid) |
			del(.metadata.resourceVersion) |
			del(.spec.template.spec.containers[].env[] | select(.name == "NAMESPACE")) |
			del(.metadata.selfLink) |
			del(.metadata.annotations."cloud.google.com/neg") |
			del(.metadata.annotations."kubectl.kubernetes.io/last-applied-configuration") |
			del(.. | select(has("image")).image) |
			del(.. | select(has("clusterIP")).clusterIP) |
			del(.. | select(has("clusterIPs")).clusterIPs) |
			del(.. | select(has("dataSource")).dataSource) |
			del(.. | select(has("procMount")).procMount) |
			del(.. | select(has("storageClassName")).storageClassName) |
			del(.. | select(has("finalizers")).finalizers) |
			del(.. | select(has("kubernetes.io/pvc-protection"))."kubernetes.io/pvc-protection") |
			del(.. | select(has("volumeName")).volumeName) |
			del(.. | select(has("volume.beta.kubernetes.io/storage-provisioner"))."volume.beta.kubernetes.io/storage-provisioner") |
			del(.. | select(has("volume.kubernetes.io/storage-provisioner"))."volume.kubernetes.io/storage-provisioner") |
			del(.spec.volumeMode) |
			del(.. | select(has("volume.kubernetes.io/selected-node"))."volume.kubernetes.io/selected-node") |
			del(.. | select(has("percona.com/last-config-hash"))."percona.com/last-config-hash") |
			del(.. | select(has("percona.com/configuration-hash"))."percona.com/configuration-hash") |
			del(.. | select(has("percona.com/ssl-hash"))."percona.com/ssl-hash") |
			del(.. | select(has("percona.com/ssl-internal-hash"))."percona.com/ssl-internal-hash") |
			del(.spec.volumeClaimTemplates[].spec.volumeMode | select(. == "Filesystem")) |
			del(.. | select(has("healthCheckNodePort")).healthCheckNodePort) |
			del(.. | select(has("nodePort")).nodePort) |
			del(.status) |
			(.. | select(tag == "!!str")) |= sub("monitoring-2-0-16090", "NAME_SPACE") |
			del(.spec.volumeClaimTemplates[].apiVersion) |
			del(.spec.volumeClaimTemplates[].kind) |
			del(.spec.ipFamilies) |
			del(.spec.ipFamilyPolicy) |
			(.. | select(. == "extensions/v1beta1")) = "apps/v1" |
			(.. | select(. == "batch/v1beta1")) = "batch/v1" ' -
++ mktemp
+ local LAST_OUT=/tmp/tmp.6HV1i8hnsQ
++ mktemp
+ local LAST_ERR=/tmp/tmp.mfoFzpdOKi
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl get -o yaml statefulset/monitoring-mongos
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.6HV1i8hnsQ
+ cat /tmp/tmp.mfoFzpdOKi
+ rm /tmp/tmp.6HV1i8hnsQ /tmp/tmp.mfoFzpdOKi
+ return 0
+ yq -i eval 'del(.spec.persistentVolumeClaimRetentionPolicy)' /tmp/tmp.39yPQ2cQ0t/statefulset_monitoring-mongos.yml
+ version_gt 1.22
++ echo '1.27 >= 1.22'
++ bc -l
+ '[' 1 -eq 1 ']'
+ return 0
+ yq -i eval 'del(.spec.internalTrafficPolicy)' /tmp/tmp.39yPQ2cQ0t/statefulset_monitoring-mongos.yml
+ yq -i eval 'del(.spec.allocateLoadBalancerNodePorts)' /tmp/tmp.39yPQ2cQ0t/statefulset_monitoring-mongos.yml
+ [[ /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1596/e2e-tests/monitoring-2-0/compare/statefulset_monitoring-mongos.yml == */cronjob* ]]
+ diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1596/e2e-tests/monitoring-2-0/compare/statefulset_monitoring-mongos.yml /tmp/tmp.39yPQ2cQ0t/statefulset_monitoring-mongos.yml
+ desc 'check mongod metrics'
+ set +o xtrace


-----------------------------------------------------------------------------------
check mongod metrics
-----------------------------------------------------------------------------------

+ get_metric_values node_boot_time_seconds monitoring-2-0-16090-monitoring-rs0-1 admin:admin
+ local metric=node_boot_time_seconds
+ local instance=monitoring-2-0-16090-monitoring-rs0-1
+ local user_pass=admin:admin
++ /usr/bin/date -u +%s -d '-1 minute'
+ local start=1721977009
++ /usr/bin/date -u +%s
+ local end=1721977069
++ get_service_endpoint monitoring-service
++ local service=monitoring-service
+++ jq '.status.loadBalancer.ingress[].hostname'
+++ kubectl_bin get service/monitoring-service -o json
+++ sed -e 's/^"//; s/"$//;'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.YgmVX6iZsE
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.BJuVdSskNW
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in '$(seq 0 2)'
+++ set +e
+++ kubectl get service/monitoring-service -o json
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.YgmVX6iZsE
+++ cat /tmp/tmp.BJuVdSskNW
+++ rm /tmp/tmp.YgmVX6iZsE /tmp/tmp.BJuVdSskNW
+++ return 0
++ local hostname=null
++ '[' -n null -a null '!=' null ']'
+++ kubectl_bin get service/monitoring-service -o json
+++ jq '.status.loadBalancer.ingress[].ip'
+++ sed -e 's/^"//; s/"$//;'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.zDDrIkEB6K
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.3YWah4SGjn
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in '$(seq 0 2)'
+++ set +e
+++ kubectl get service/monitoring-service -o json
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.zDDrIkEB6K
+++ cat /tmp/tmp.3YWah4SGjn
+++ rm /tmp/tmp.zDDrIkEB6K /tmp/tmp.3YWah4SGjn
+++ return 0
++ local ip=35.192.90.18
++ '[' -n 35.192.90.18 -a 35.192.90.18 '!=' null ']'
++ echo 35.192.90.18
++ return
+ local endpoint=35.192.90.18
+ curl -s -k 'https://admin:admin@35.192.90.18/graph/api/datasources/proxy/1/api/v1/query_range?query=min%28node_boot_time_seconds%7Bnode_name%3D%7E%22monitoring-2-0-16090-monitoring-rs0-1%22%7d%20or%20node_boot_time_seconds%7Bnode_name%3D%7E%22monitoring-2-0-16090-monitoring-rs0-1%22%7D%29&start=1721977009&end=1721977069&step=60'
+ jq '.data.result[0].values[][1]'
+ grep '^"[0-9]'
"1721973228"
"1721973228"
+ get_metric_values mongodb_connections monitoring-2-0-16090-monitoring-rs0-1 admin:admin
+ local metric=mongodb_connections
+ local instance=monitoring-2-0-16090-monitoring-rs0-1
+ local user_pass=admin:admin
++ /usr/bin/date -u +%s -d '-1 minute'
+ local start=1721977012
++ /usr/bin/date -u +%s
+ local end=1721977072
++ get_service_endpoint monitoring-service
++ local service=monitoring-service
+++ kubectl_bin get service/monitoring-service -o json
+++ jq '.status.loadBalancer.ingress[].hostname'
+++ sed -e 's/^"//; s/"$//;'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.1rojLltj5V
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.HPrDvN2kvs
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in '$(seq 0 2)'
+++ set +e
+++ kubectl get service/monitoring-service -o json
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.1rojLltj5V
+++ cat /tmp/tmp.HPrDvN2kvs
+++ rm /tmp/tmp.1rojLltj5V /tmp/tmp.HPrDvN2kvs
+++ return 0
++ local hostname=null
++ '[' -n null -a null '!=' null ']'
+++ kubectl_bin get service/monitoring-service -o json
+++ sed -e 's/^"//; s/"$//;'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.v7JF8izDtl
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.onT36EBAPd
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in '$(seq 0 2)'
+++ set +e
+++ kubectl get service/monitoring-service -o json
+++ jq '.status.loadBalancer.ingress[].ip'
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.v7JF8izDtl
+++ cat /tmp/tmp.onT36EBAPd
+++ rm /tmp/tmp.v7JF8izDtl /tmp/tmp.onT36EBAPd
+++ return 0
++ local ip=35.192.90.18
++ '[' -n 35.192.90.18 -a 35.192.90.18 '!=' null ']'
++ echo 35.192.90.18
++ return
+ local endpoint=35.192.90.18
+ grep '^"[0-9]'
+ jq '.data.result[0].values[][1]'
+ curl -s -k 'https://admin:admin@35.192.90.18/graph/api/datasources/proxy/1/api/v1/query_range?query=min%28mongodb_connections%7Bnode_name%3D%7E%22monitoring-2-0-16090-monitoring-rs0-1%22%7d%20or%20mongodb_connections%7Bnode_name%3D%7E%22monitoring-2-0-16090-monitoring-rs0-1%22%7D%29&start=1721977012&end=1721977072&step=60'
"0"
"0"
+ desc 'check mongo config  metrics'
+ set +o xtrace


-----------------------------------------------------------------------------------
check mongo config  metrics
-----------------------------------------------------------------------------------

+ get_metric_values node_boot_time_seconds monitoring-2-0-16090-monitoring-cfg-1 admin:admin
+ local metric=node_boot_time_seconds
+ local instance=monitoring-2-0-16090-monitoring-cfg-1
+ local user_pass=admin:admin
++ /usr/bin/date -u +%s -d '-1 minute'
+ local start=1721977014
++ /usr/bin/date -u +%s
+ local end=1721977074
++ get_service_endpoint monitoring-service
++ local service=monitoring-service
+++ kubectl_bin get service/monitoring-service -o json
+++ jq '.status.loadBalancer.ingress[].hostname'
+++ sed -e 's/^"//; s/"$//;'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.kVlk6UFjFr
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.rM1HgoQLdj
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in '$(seq 0 2)'
+++ set +e
+++ kubectl get service/monitoring-service -o json
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.kVlk6UFjFr
+++ cat /tmp/tmp.rM1HgoQLdj
+++ rm /tmp/tmp.kVlk6UFjFr /tmp/tmp.rM1HgoQLdj
+++ return 0
++ local hostname=null
++ '[' -n null -a null '!=' null ']'
+++ sed -e 's/^"//; s/"$//;'
+++ kubectl_bin get service/monitoring-service -o json
++++ mktemp
+++ jq '.status.loadBalancer.ingress[].ip'
+++ local LAST_OUT=/tmp/tmp.ZuNr30ydzH
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.rGarwGOOQC
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in '$(seq 0 2)'
+++ set +e
+++ kubectl get service/monitoring-service -o json
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.ZuNr30ydzH
+++ cat /tmp/tmp.rGarwGOOQC
+++ rm /tmp/tmp.ZuNr30ydzH /tmp/tmp.rGarwGOOQC
+++ return 0
++ local ip=35.192.90.18
++ '[' -n 35.192.90.18 -a 35.192.90.18 '!=' null ']'
++ echo 35.192.90.18
++ return
+ local endpoint=35.192.90.18
+ curl -s -k 'https://admin:admin@35.192.90.18/graph/api/datasources/proxy/1/api/v1/query_range?query=min%28node_boot_time_seconds%7Bnode_name%3D%7E%22monitoring-2-0-16090-monitoring-cfg-1%22%7d%20or%20node_boot_time_seconds%7Bnode_name%3D%7E%22monitoring-2-0-16090-monitoring-cfg-1%22%7D%29&start=1721977014&end=1721977074&step=60'
+ grep '^"[0-9]'
+ jq '.data.result[0].values[][1]'
"1721973228"
"1721973228"
+ get_metric_values mongodb_connections monitoring-2-0-16090-monitoring-cfg-1 admin:admin
+ local metric=mongodb_connections
+ local instance=monitoring-2-0-16090-monitoring-cfg-1
+ local user_pass=admin:admin
++ /usr/bin/date -u +%s -d '-1 minute'
+ local start=1721977017
++ /usr/bin/date -u +%s
+ local end=1721977077
++ get_service_endpoint monitoring-service
++ local service=monitoring-service
+++ kubectl_bin get service/monitoring-service -o json
+++ sed -e 's/^"//; s/"$//;'
++++ mktemp
+++ jq '.status.loadBalancer.ingress[].hostname'
+++ local LAST_OUT=/tmp/tmp.VfR3v4b25v
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.KI3hHLol4p
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in '$(seq 0 2)'
+++ set +e
+++ kubectl get service/monitoring-service -o json
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.VfR3v4b25v
+++ cat /tmp/tmp.KI3hHLol4p
+++ rm /tmp/tmp.VfR3v4b25v /tmp/tmp.KI3hHLol4p
+++ return 0
++ local hostname=null
++ '[' -n null -a null '!=' null ']'
+++ kubectl_bin get service/monitoring-service -o json
+++ jq '.status.loadBalancer.ingress[].ip'
++++ mktemp
+++ sed -e 's/^"//; s/"$//;'
+++ local LAST_OUT=/tmp/tmp.h20jROapvU
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.fCBwrqmLUp
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in '$(seq 0 2)'
+++ set +e
+++ kubectl get service/monitoring-service -o json
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.h20jROapvU
+++ cat /tmp/tmp.fCBwrqmLUp
+++ rm /tmp/tmp.h20jROapvU /tmp/tmp.fCBwrqmLUp
+++ return 0
++ local ip=35.192.90.18
++ '[' -n 35.192.90.18 -a 35.192.90.18 '!=' null ']'
++ echo 35.192.90.18
++ return
+ local endpoint=35.192.90.18
+ jq '.data.result[0].values[][1]'
+ grep '^"[0-9]'
+ curl -s -k 'https://admin:admin@35.192.90.18/graph/api/datasources/proxy/1/api/v1/query_range?query=min%28mongodb_connections%7Bnode_name%3D%7E%22monitoring-2-0-16090-monitoring-cfg-1%22%7d%20or%20mongodb_connections%7Bnode_name%3D%7E%22monitoring-2-0-16090-monitoring-cfg-1%22%7D%29&start=1721977017&end=1721977077&step=60'
"0"
"0"
+ desc 'check mongos metrics'
+ set +o xtrace


-----------------------------------------------------------------------------------
check mongos metrics
-----------------------------------------------------------------------------------

++ kubectl get pod -l app.kubernetes.io/component=mongos -o 'jsonpath={.items[0].metadata.name}'
+ MONGOS_POD_NAME=monitoring-mongos-0
+ get_metric_values node_boot_time_seconds monitoring-2-0-16090-monitoring-mongos-0 admin:admin
+ local metric=node_boot_time_seconds
+ local instance=monitoring-2-0-16090-monitoring-mongos-0
+ local user_pass=admin:admin
++ /usr/bin/date -u +%s -d '-1 minute'
+ local start=1721977021
++ /usr/bin/date -u +%s
+ local end=1721977081
++ get_service_endpoint monitoring-service
++ local service=monitoring-service
+++ kubectl_bin get service/monitoring-service -o json
+++ jq '.status.loadBalancer.ingress[].hostname'
+++ sed -e 's/^"//; s/"$//;'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.loIq8A0Pem
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.cBKk2CuyyM
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in '$(seq 0 2)'
+++ set +e
+++ kubectl get service/monitoring-service -o json
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.loIq8A0Pem
+++ cat /tmp/tmp.cBKk2CuyyM
+++ rm /tmp/tmp.loIq8A0Pem /tmp/tmp.cBKk2CuyyM
+++ return 0
++ local hostname=null
++ '[' -n null -a null '!=' null ']'
+++ kubectl_bin get service/monitoring-service -o json
+++ jq '.status.loadBalancer.ingress[].ip'
+++ sed -e 's/^"//; s/"$//;'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.LlWUThW7oy
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.anAXcvInEh
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in '$(seq 0 2)'
+++ set +e
+++ kubectl get service/monitoring-service -o json
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.LlWUThW7oy
+++ cat /tmp/tmp.anAXcvInEh
+++ rm /tmp/tmp.LlWUThW7oy /tmp/tmp.anAXcvInEh
+++ return 0
++ local ip=35.192.90.18
++ '[' -n 35.192.90.18 -a 35.192.90.18 '!=' null ']'
++ echo 35.192.90.18
++ return
+ local endpoint=35.192.90.18
+ curl -s -k 'https://admin:admin@35.192.90.18/graph/api/datasources/proxy/1/api/v1/query_range?query=min%28node_boot_time_seconds%7Bnode_name%3D%7E%22monitoring-2-0-16090-monitoring-mongos-0%22%7d%20or%20node_boot_time_seconds%7Bnode_name%3D%7E%22monitoring-2-0-16090-monitoring-mongos-0%22%7D%29&start=1721977021&end=1721977081&step=60'
+ jq '.data.result[0].values[][1]'
+ grep '^"[0-9]'
"1721973228"
"1721973228"
+ sleep 90
+ desc 'check QAN data'
+ set +o xtrace


-----------------------------------------------------------------------------------
check QAN data
-----------------------------------------------------------------------------------

+ get_qan_values mongodb dev-mongod admin:admin
+ local service_type=mongodb
+ local environment=dev-mongod
+ local user_pass=admin:admin
+ local start
+ local end
+ local endpoint
++ /usr/bin/date -u -d '-12 hour' +%Y-%m-%dT%H:%M:%S%:z
+ start=2024-07-25T18:59:33+00:00
++ /usr/bin/date -u +%Y-%m-%dT%H:%M:%S%:z
+ end=2024-07-26T06:59:33+00:00
++ get_service_endpoint monitoring-service
++ local service=monitoring-service
+++ kubectl_bin get service/monitoring-service -o json
+++ jq '.status.loadBalancer.ingress[].hostname'
+++ sed -e 's/^"//; s/"$//;'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.aT7qa1kOzO
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.8ldruv1Vcd
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in '$(seq 0 2)'
+++ set +e
+++ kubectl get service/monitoring-service -o json
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.aT7qa1kOzO
+++ cat /tmp/tmp.8ldruv1Vcd
+++ rm /tmp/tmp.aT7qa1kOzO /tmp/tmp.8ldruv1Vcd
+++ return 0
++ local hostname=null
++ '[' -n null -a null '!=' null ']'
+++ kubectl_bin get service/monitoring-service -o json
+++ jq '.status.loadBalancer.ingress[].ip'
++++ mktemp
+++ sed -e 's/^"//; s/"$//;'
+++ local LAST_OUT=/tmp/tmp.4Ym4F5o4bm
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.Ez78Eocni1
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in '$(seq 0 2)'
+++ set +e
+++ kubectl get service/monitoring-service -o json
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.4Ym4F5o4bm
+++ cat /tmp/tmp.Ez78Eocni1
+++ rm /tmp/tmp.4Ym4F5o4bm /tmp/tmp.Ez78Eocni1
+++ return 0
++ local ip=35.192.90.18
++ '[' -n 35.192.90.18 -a 35.192.90.18 '!=' null ']'
++ echo 35.192.90.18
++ return
+ endpoint=35.192.90.18
+ cat
+ local response
+ retry=0
++ curl -s -k -XPOST -d @payload.json https://admin:admin@35.192.90.18/v0/qan/GetReport
++ jq '.rows[].fingerprint'
+ [[ "TOTAL"
"FIND version"
"FIND oplog.rs" != \n\u\l\l ]]
+ rm -f payload.json
+ get_qan_values mongodb dev-mongos admin:admin
+ local service_type=mongodb
+ local environment=dev-mongos
+ local user_pass=admin:admin
+ local start
+ local end
+ local endpoint
++ /usr/bin/date -u -d '-12 hour' +%Y-%m-%dT%H:%M:%S%:z
+ start=2024-07-25T18:59:36+00:00
++ /usr/bin/date -u +%Y-%m-%dT%H:%M:%S%:z
+ end=2024-07-26T06:59:36+00:00
++ get_service_endpoint monitoring-service
++ local service=monitoring-service
+++ kubectl_bin get service/monitoring-service -o json
+++ jq '.status.loadBalancer.ingress[].hostname'
+++ sed -e 's/^"//; s/"$//;'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.nOlOu2m8p6
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.cS3YUdIBVx
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in '$(seq 0 2)'
+++ set +e
+++ kubectl get service/monitoring-service -o json
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.nOlOu2m8p6
+++ cat /tmp/tmp.cS3YUdIBVx
+++ rm /tmp/tmp.nOlOu2m8p6 /tmp/tmp.cS3YUdIBVx
+++ return 0
++ local hostname=null
++ '[' -n null -a null '!=' null ']'
+++ kubectl_bin get service/monitoring-service -o json
+++ sed -e 's/^"//; s/"$//;'
++++ mktemp
+++ jq '.status.loadBalancer.ingress[].ip'
+++ local LAST_OUT=/tmp/tmp.lSuSxpVyCm
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.6zdESggduZ
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in '$(seq 0 2)'
+++ set +e
+++ kubectl get service/monitoring-service -o json
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.lSuSxpVyCm
+++ cat /tmp/tmp.6zdESggduZ
+++ rm /tmp/tmp.lSuSxpVyCm /tmp/tmp.6zdESggduZ
+++ return 0
++ local ip=35.192.90.18
++ '[' -n 35.192.90.18 -a 35.192.90.18 '!=' null ']'
++ echo 35.192.90.18
++ return
+ endpoint=35.192.90.18
+ cat
+ local response
+ retry=0
++ curl -s -k -XPOST -d @payload.json https://admin:admin@35.192.90.18/v0/qan/GetReport
++ jq '.rows[].fingerprint'
+ [[ "TOTAL"
"DBSTATS application,architecture,client,clusterTime,db,dbStats,driver,hash,host,id,keyId,level,lsid,mayBypassWriteBlocking,mongos,name,os,platform,provenance,readConcern,role,scale,signature,type,uid,user,version" != \n\u\l\l ]]
+ rm -f payload.json
+ nodeList=($(get_node_id_from_pmm))
++ get_node_id_from_pmm
++ nodeList=()
++ local -a nodeList
+++ kubectl_bin get pods --no-headers -l app.kubernetes.io/name=percona-server-mongodb --output=custom-columns=NAME:.metadata.name
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.56nG4D01GL
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.qzwqJkpRHJ
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in '$(seq 0 2)'
+++ set +e
+++ kubectl get pods --no-headers -l app.kubernetes.io/name=percona-server-mongodb --output=custom-columns=NAME:.metadata.name
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.56nG4D01GL
+++ cat /tmp/tmp.qzwqJkpRHJ
+++ rm /tmp/tmp.56nG4D01GL /tmp/tmp.qzwqJkpRHJ
+++ return 0
++ for instance in '$(kubectl_bin get pods --no-headers -l app.kubernetes.io/name=percona-server-mongodb --output=custom-columns='\''NAME:.metadata.name'\'')'
++ nodeList+=($(kubectl_bin exec -n "$namespace" $instance -c pmm-client -- pmm-admin status --json | jq -r '.pmm_agent_status.node_id'))
+++ kubectl_bin exec -n monitoring-2-0-16090 monitoring-cfg-0 -c pmm-client -- pmm-admin status --json
++++ mktemp
+++ jq -r .pmm_agent_status.node_id
+++ local LAST_OUT=/tmp/tmp.nrC8vXr6tI
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.senZ5A6SSD
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in '$(seq 0 2)'
+++ set +e
+++ kubectl exec -n monitoring-2-0-16090 monitoring-cfg-0 -c pmm-client -- pmm-admin status --json
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.nrC8vXr6tI
+++ cat /tmp/tmp.senZ5A6SSD
+++ rm /tmp/tmp.nrC8vXr6tI /tmp/tmp.senZ5A6SSD
+++ return 0
++ for instance in '$(kubectl_bin get pods --no-headers -l app.kubernetes.io/name=percona-server-mongodb --output=custom-columns='\''NAME:.metadata.name'\'')'
++ nodeList+=($(kubectl_bin exec -n "$namespace" $instance -c pmm-client -- pmm-admin status --json | jq -r '.pmm_agent_status.node_id'))
+++ kubectl_bin exec -n monitoring-2-0-16090 monitoring-cfg-1 -c pmm-client -- pmm-admin status --json
+++ jq -r .pmm_agent_status.node_id
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.8dBPzB3SmV
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.yUXgHKqUPt
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in '$(seq 0 2)'
+++ set +e
+++ kubectl exec -n monitoring-2-0-16090 monitoring-cfg-1 -c pmm-client -- pmm-admin status --json
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.8dBPzB3SmV
+++ cat /tmp/tmp.yUXgHKqUPt
+++ rm /tmp/tmp.8dBPzB3SmV /tmp/tmp.yUXgHKqUPt
+++ return 0
++ for instance in '$(kubectl_bin get pods --no-headers -l app.kubernetes.io/name=percona-server-mongodb --output=custom-columns='\''NAME:.metadata.name'\'')'
++ nodeList+=($(kubectl_bin exec -n "$namespace" $instance -c pmm-client -- pmm-admin status --json | jq -r '.pmm_agent_status.node_id'))
+++ jq -r .pmm_agent_status.node_id
+++ kubectl_bin exec -n monitoring-2-0-16090 monitoring-cfg-2 -c pmm-client -- pmm-admin status --json
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.mHtMJ624ji
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.ExHA4XVlvY
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in '$(seq 0 2)'
+++ set +e
+++ kubectl exec -n monitoring-2-0-16090 monitoring-cfg-2 -c pmm-client -- pmm-admin status --json
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.mHtMJ624ji
+++ cat /tmp/tmp.ExHA4XVlvY
+++ rm /tmp/tmp.mHtMJ624ji /tmp/tmp.ExHA4XVlvY
+++ return 0
++ for instance in '$(kubectl_bin get pods --no-headers -l app.kubernetes.io/name=percona-server-mongodb --output=custom-columns='\''NAME:.metadata.name'\'')'
++ nodeList+=($(kubectl_bin exec -n "$namespace" $instance -c pmm-client -- pmm-admin status --json | jq -r '.pmm_agent_status.node_id'))
+++ kubectl_bin exec -n monitoring-2-0-16090 monitoring-mongos-0 -c pmm-client -- pmm-admin status --json
+++ jq -r .pmm_agent_status.node_id
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.b1Aga8snR5
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.IYwxg3YUfG
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in '$(seq 0 2)'
+++ set +e
+++ kubectl exec -n monitoring-2-0-16090 monitoring-mongos-0 -c pmm-client -- pmm-admin status --json
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.b1Aga8snR5
+++ cat /tmp/tmp.IYwxg3YUfG
+++ rm /tmp/tmp.b1Aga8snR5 /tmp/tmp.IYwxg3YUfG
+++ return 0
++ for instance in '$(kubectl_bin get pods --no-headers -l app.kubernetes.io/name=percona-server-mongodb --output=custom-columns='\''NAME:.metadata.name'\'')'
++ nodeList+=($(kubectl_bin exec -n "$namespace" $instance -c pmm-client -- pmm-admin status --json | jq -r '.pmm_agent_status.node_id'))
+++ kubectl_bin exec -n monitoring-2-0-16090 monitoring-mongos-1 -c pmm-client -- pmm-admin status --json
++++ mktemp
+++ jq -r .pmm_agent_status.node_id
+++ local LAST_OUT=/tmp/tmp.GgO742OPE9
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.2pmAGzDcTH
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in '$(seq 0 2)'
+++ set +e
+++ kubectl exec -n monitoring-2-0-16090 monitoring-mongos-1 -c pmm-client -- pmm-admin status --json
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.GgO742OPE9
+++ cat /tmp/tmp.2pmAGzDcTH
+++ rm /tmp/tmp.GgO742OPE9 /tmp/tmp.2pmAGzDcTH
+++ return 0
++ for instance in '$(kubectl_bin get pods --no-headers -l app.kubernetes.io/name=percona-server-mongodb --output=custom-columns='\''NAME:.metadata.name'\'')'
++ nodeList+=($(kubectl_bin exec -n "$namespace" $instance -c pmm-client -- pmm-admin status --json | jq -r '.pmm_agent_status.node_id'))
+++ kubectl_bin exec -n monitoring-2-0-16090 monitoring-mongos-2 -c pmm-client -- pmm-admin status --json
+++ jq -r .pmm_agent_status.node_id
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.Iw0mSj7VOi
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.diinqvPxdy
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in '$(seq 0 2)'
+++ set +e
+++ kubectl exec -n monitoring-2-0-16090 monitoring-mongos-2 -c pmm-client -- pmm-admin status --json
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.Iw0mSj7VOi
+++ cat /tmp/tmp.diinqvPxdy
+++ rm /tmp/tmp.Iw0mSj7VOi /tmp/tmp.diinqvPxdy
+++ return 0
++ for instance in '$(kubectl_bin get pods --no-headers -l app.kubernetes.io/name=percona-server-mongodb --output=custom-columns='\''NAME:.metadata.name'\'')'
++ nodeList+=($(kubectl_bin exec -n "$namespace" $instance -c pmm-client -- pmm-admin status --json | jq -r '.pmm_agent_status.node_id'))
+++ kubectl_bin exec -n monitoring-2-0-16090 monitoring-rs0-0 -c pmm-client -- pmm-admin status --json
+++ jq -r .pmm_agent_status.node_id
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.Ehk0lWy3kR
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.UIDhvu4Qu8
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in '$(seq 0 2)'
+++ set +e
+++ kubectl exec -n monitoring-2-0-16090 monitoring-rs0-0 -c pmm-client -- pmm-admin status --json
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.Ehk0lWy3kR
+++ cat /tmp/tmp.UIDhvu4Qu8
+++ rm /tmp/tmp.Ehk0lWy3kR /tmp/tmp.UIDhvu4Qu8
+++ return 0
++ for instance in '$(kubectl_bin get pods --no-headers -l app.kubernetes.io/name=percona-server-mongodb --output=custom-columns='\''NAME:.metadata.name'\'')'
++ nodeList+=($(kubectl_bin exec -n "$namespace" $instance -c pmm-client -- pmm-admin status --json | jq -r '.pmm_agent_status.node_id'))
+++ kubectl_bin exec -n monitoring-2-0-16090 monitoring-rs0-1 -c pmm-client -- pmm-admin status --json
+++ jq -r .pmm_agent_status.node_id
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.ScDe0EMUOz
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.ozmrccly84
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in '$(seq 0 2)'
+++ set +e
+++ kubectl exec -n monitoring-2-0-16090 monitoring-rs0-1 -c pmm-client -- pmm-admin status --json
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.ScDe0EMUOz
+++ cat /tmp/tmp.ozmrccly84
+++ rm /tmp/tmp.ScDe0EMUOz /tmp/tmp.ozmrccly84
+++ return 0
++ for instance in '$(kubectl_bin get pods --no-headers -l app.kubernetes.io/name=percona-server-mongodb --output=custom-columns='\''NAME:.metadata.name'\'')'
++ nodeList+=($(kubectl_bin exec -n "$namespace" $instance -c pmm-client -- pmm-admin status --json | jq -r '.pmm_agent_status.node_id'))
+++ kubectl_bin exec -n monitoring-2-0-16090 monitoring-rs0-2 -c pmm-client -- pmm-admin status --json
+++ jq -r .pmm_agent_status.node_id
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.MBzEHtOVWO
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.tcb8AZVVlA
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in '$(seq 0 2)'
+++ set +e
+++ kubectl exec -n monitoring-2-0-16090 monitoring-rs0-2 -c pmm-client -- pmm-admin status --json
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.MBzEHtOVWO
+++ cat /tmp/tmp.tcb8AZVVlA
+++ rm /tmp/tmp.MBzEHtOVWO /tmp/tmp.tcb8AZVVlA
+++ return 0
++ echo /node_id/def17f27-15a0-4240-ac39-0614beb243e6 /node_id/efac200c-f35c-407e-815d-3728aea09fbd /node_id/6eb9ff8f-5f1f-433b-a759-ff8ca749f427 /node_id/54bc59c3-da6b-42c2-bc81-63bf5c67a979 /node_id/ebeaa667-fe43-4326-ba0e-e6e852c734ce /node_id/0af1fe4a-8b09-452c-82a1-ed469e898790 /node_id/c03612ee-2d8b-4228-b363-041e3ae52a53 /node_id/fcb0e3b7-0883-4a6b-bd4e-f093cba949d9 /node_id/9817611e-c22a-442a-bf13-131d02ea6b93
+ nodeList_from_pmm=($(does_node_id_exists "${nodeList[@]}"))
++ does_node_id_exists /node_id/def17f27-15a0-4240-ac39-0614beb243e6 /node_id/efac200c-f35c-407e-815d-3728aea09fbd /node_id/6eb9ff8f-5f1f-433b-a759-ff8ca749f427 /node_id/54bc59c3-da6b-42c2-bc81-63bf5c67a979 /node_id/ebeaa667-fe43-4326-ba0e-e6e852c734ce /node_id/0af1fe4a-8b09-452c-82a1-ed469e898790 /node_id/c03612ee-2d8b-4228-b363-041e3ae52a53 /node_id/fcb0e3b7-0883-4a6b-bd4e-f093cba949d9 /node_id/9817611e-c22a-442a-bf13-131d02ea6b93
++ nodeList=("$@")
++ local -a nodeList
++ nodeList_from_pmm=()
++ local -a nodeList_from_pmm
++ for node_id in '"${nodeList[@]}"'
++ nodeList_from_pmm+=($(kubectl_bin exec -n "${namespace}" monitoring-0 -- pmm-admin --server-url=https://admin:admin@$(get_service_ip monitoring-service)/ --server-insecure-tls inventory list nodes --node-type=CONTAINER_NODE | grep $node_id | awk '{print $4}'))
++++ get_service_ip monitoring-service
++++ local service=monitoring-service
++++ local server_type=rs0
+++ grep /node_id/def17f27-15a0-4240-ac39-0614beb243e6
+++++ kubectl_bin get psmdb/monitoring-service -o 'jsonpath={.spec.replsets[].expose.enabled}'
++++++ mktemp
+++++ local LAST_OUT=/tmp/tmp.TVQTJcME3Y
+++ awk '{print $4}'
++++++ mktemp
+++++ local LAST_ERR=/tmp/tmp.RR4QWlZRFV
+++++ local exit_status=0
+++++ local timeout=4
++++++ seq 0 2
+++++ for i in '$(seq 0 2)'
+++++ set +e
+++++ kubectl get psmdb/monitoring-service -o 'jsonpath={.spec.replsets[].expose.enabled}'
+++++ exit_status=1
+++++ set -e
+++++ '[' 1 '!=' 0 -a -n 1 ']'
+++++ cat /tmp/tmp.TVQTJcME3Y
+++++ cat /tmp/tmp.RR4QWlZRFV
Error from server (NotFound): perconaservermongodbs.psmdb.percona.com "monitoring-service" not found
+++++ sleep 0
+++++ for i in '$(seq 0 2)'
+++++ set +e
+++++ kubectl get psmdb/monitoring-service -o 'jsonpath={.spec.replsets[].expose.enabled}'
+++++ exit_status=1
+++++ set -e
+++++ '[' 1 '!=' 0 -a -n 1 ']'
+++++ cat /tmp/tmp.TVQTJcME3Y
+++++ cat /tmp/tmp.RR4QWlZRFV
Error from server (NotFound): perconaservermongodbs.psmdb.percona.com "monitoring-service" not found
+++++ sleep 4
+++++ for i in '$(seq 0 2)'
+++++ set +e
+++++ kubectl get psmdb/monitoring-service -o 'jsonpath={.spec.replsets[].expose.enabled}'
+++++ exit_status=1
+++++ set -e
+++++ '[' 1 '!=' 0 -a -n 1 ']'
+++++ cat /tmp/tmp.TVQTJcME3Y
+++++ cat /tmp/tmp.RR4QWlZRFV
Error from server (NotFound): perconaservermongodbs.psmdb.percona.com "monitoring-service" not found
+++++ sleep 8
+++++ cat /tmp/tmp.TVQTJcME3Y
+++++ cat /tmp/tmp.RR4QWlZRFV
Error from server (NotFound): perconaservermongodbs.psmdb.percona.com "monitoring-service" not found
+++++ rm /tmp/tmp.TVQTJcME3Y /tmp/tmp.RR4QWlZRFV
+++++ return 1
++++ '[' '' '!=' true ']'
++++ echo -n monitoring-service.monitoring-service-rs0
++++ return
+++ kubectl_bin exec -n monitoring-2-0-16090 monitoring-0 -- pmm-admin --server-url=https://admin:admin@monitoring-service.monitoring-service-rs0/ --server-insecure-tls inventory list nodes --node-type=CONTAINER_NODE
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.Uuph91HHNH
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.fkQa3n5WW3
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in '$(seq 0 2)'
+++ set +e
+++ kubectl exec -n monitoring-2-0-16090 monitoring-0 -- pmm-admin --server-url=https://admin:admin@monitoring-service.monitoring-service-rs0/ --server-insecure-tls inventory list nodes --node-type=CONTAINER_NODE
+++ exit_status=1
+++ set -e
+++ '[' 1 '!=' 0 -a -n 1 ']'
+++ cat /tmp/tmp.Uuph91HHNH
+++ cat /tmp/tmp.fkQa3n5WW3
command terminated with exit code 1
+++ sleep 0
+++ for i in '$(seq 0 2)'
+++ set +e
+++ kubectl exec -n monitoring-2-0-16090 monitoring-0 -- pmm-admin --server-url=https://admin:admin@monitoring-service.monitoring-service-rs0/ --server-insecure-tls inventory list nodes --node-type=CONTAINER_NODE
+++ exit_status=1
+++ set -e
+++ '[' 1 '!=' 0 -a -n 1 ']'
+++ cat /tmp/tmp.Uuph91HHNH
+++ cat /tmp/tmp.fkQa3n5WW3
command terminated with exit code 1
+++ sleep 4
+++ for i in '$(seq 0 2)'
+++ set +e
+++ kubectl exec -n monitoring-2-0-16090 monitoring-0 -- pmm-admin --server-url=https://admin:admin@monitoring-service.monitoring-service-rs0/ --server-insecure-tls inventory list nodes --node-type=CONTAINER_NODE
+++ exit_status=1
+++ set -e
+++ '[' 1 '!=' 0 -a -n 1 ']'
+++ cat /tmp/tmp.Uuph91HHNH
+++ cat /tmp/tmp.fkQa3n5WW3
command terminated with exit code 1
+++ sleep 8
+++ cat /tmp/tmp.Uuph91HHNH
+++ cat /tmp/tmp.fkQa3n5WW3
command terminated with exit code 1
+++ rm /tmp/tmp.Uuph91HHNH /tmp/tmp.fkQa3n5WW3
+++ return 1
++ for node_id in '"${nodeList[@]}"'
++ nodeList_from_pmm+=($(kubectl_bin exec -n "${namespace}" monitoring-0 -- pmm-admin --server-url=https://admin:admin@$(get_service_ip monitoring-service)/ --server-insecure-tls inventory list nodes --node-type=CONTAINER_NODE | grep $node_id | awk '{print $4}'))
+++ grep /node_id/efac200c-f35c-407e-815d-3728aea09fbd
+++ awk '{print $4}'
++++ get_service_ip monitoring-service
++++ local service=monitoring-service
++++ local server_type=rs0
+++++ kubectl_bin get psmdb/monitoring-service -o 'jsonpath={.spec.replsets[].expose.enabled}'
++++++ mktemp
+++++ local LAST_OUT=/tmp/tmp.KVVAKcMCOT
++++++ mktemp
+++++ local LAST_ERR=/tmp/tmp.GB7gRKqeRV
+++++ local exit_status=0
+++++ local timeout=4
++++++ seq 0 2
+++++ for i in '$(seq 0 2)'
+++++ set +e
+++++ kubectl get psmdb/monitoring-service -o 'jsonpath={.spec.replsets[].expose.enabled}'
+++++ exit_status=1
+++++ set -e
+++++ '[' 1 '!=' 0 -a -n 1 ']'
+++++ cat /tmp/tmp.KVVAKcMCOT
+++++ cat /tmp/tmp.GB7gRKqeRV
Error from server (NotFound): perconaservermongodbs.psmdb.percona.com "monitoring-service" not found
+++++ sleep 0
+++++ for i in '$(seq 0 2)'
+++++ set +e
+++++ kubectl get psmdb/monitoring-service -o 'jsonpath={.spec.replsets[].expose.enabled}'
+++++ exit_status=1
+++++ set -e
+++++ '[' 1 '!=' 0 -a -n 1 ']'
+++++ cat /tmp/tmp.KVVAKcMCOT
+++++ cat /tmp/tmp.GB7gRKqeRV
Error from server (NotFound): perconaservermongodbs.psmdb.percona.com "monitoring-service" not found
+++++ sleep 4
+++++ for i in '$(seq 0 2)'
+++++ set +e
+++++ kubectl get psmdb/monitoring-service -o 'jsonpath={.spec.replsets[].expose.enabled}'
+++++ exit_status=1
+++++ set -e
+++++ '[' 1 '!=' 0 -a -n 1 ']'
+++++ cat /tmp/tmp.KVVAKcMCOT
+++++ cat /tmp/tmp.GB7gRKqeRV
Error from server (NotFound): perconaservermongodbs.psmdb.percona.com "monitoring-service" not found
+++++ sleep 8
+++++ cat /tmp/tmp.KVVAKcMCOT
+++++ cat /tmp/tmp.GB7gRKqeRV
Error from server (NotFound): perconaservermongodbs.psmdb.percona.com "monitoring-service" not found
+++++ rm /tmp/tmp.KVVAKcMCOT /tmp/tmp.GB7gRKqeRV
+++++ return 1
++++ '[' '' '!=' true ']'
++++ echo -n monitoring-service.monitoring-service-rs0
++++ return
+++ kubectl_bin exec -n monitoring-2-0-16090 monitoring-0 -- pmm-admin --server-url=https://admin:admin@monitoring-service.monitoring-service-rs0/ --server-insecure-tls inventory list nodes --node-type=CONTAINER_NODE
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.JIjMCQ6Fe2
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.NXOR6mUN5T
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in '$(seq 0 2)'
+++ set +e
+++ kubectl exec -n monitoring-2-0-16090 monitoring-0 -- pmm-admin --server-url=https://admin:admin@monitoring-service.monitoring-service-rs0/ --server-insecure-tls inventory list nodes --node-type=CONTAINER_NODE
+++ exit_status=1
+++ set -e
+++ '[' 1 '!=' 0 -a -n 1 ']'
+++ cat /tmp/tmp.JIjMCQ6Fe2
+++ cat /tmp/tmp.NXOR6mUN5T
command terminated with exit code 1
+++ sleep 0
+++ for i in '$(seq 0 2)'
+++ set +e
+++ kubectl exec -n monitoring-2-0-16090 monitoring-0 -- pmm-admin --server-url=https://admin:admin@monitoring-service.monitoring-service-rs0/ --server-insecure-tls inventory list nodes --node-type=CONTAINER_NODE
+++ exit_status=1
+++ set -e
+++ '[' 1 '!=' 0 -a -n 1 ']'
+++ cat /tmp/tmp.JIjMCQ6Fe2
+++ cat /tmp/tmp.NXOR6mUN5T
command terminated with exit code 1
+++ sleep 4
+++ for i in '$(seq 0 2)'
+++ set +e
+++ kubectl exec -n monitoring-2-0-16090 monitoring-0 -- pmm-admin --server-url=https://admin:admin@monitoring-service.monitoring-service-rs0/ --server-insecure-tls inventory list nodes --node-type=CONTAINER_NODE
+++ exit_status=1
+++ set -e
+++ '[' 1 '!=' 0 -a -n 1 ']'
+++ cat /tmp/tmp.JIjMCQ6Fe2
+++ cat /tmp/tmp.NXOR6mUN5T
command terminated with exit code 1
+++ sleep 8
+++ cat /tmp/tmp.JIjMCQ6Fe2
+++ cat /tmp/tmp.NXOR6mUN5T
command terminated with exit code 1
+++ rm /tmp/tmp.JIjMCQ6Fe2 /tmp/tmp.NXOR6mUN5T
+++ return 1
++ for node_id in '"${nodeList[@]}"'
++ nodeList_from_pmm+=($(kubectl_bin exec -n "${namespace}" monitoring-0 -- pmm-admin --server-url=https://admin:admin@$(get_service_ip monitoring-service)/ --server-insecure-tls inventory list nodes --node-type=CONTAINER_NODE | grep $node_id | awk '{print $4}'))
+++ grep /node_id/6eb9ff8f-5f1f-433b-a759-ff8ca749f427
+++ awk '{print $4}'
++++ get_service_ip monitoring-service
++++ local service=monitoring-service
++++ local server_type=rs0
+++++ kubectl_bin get psmdb/monitoring-service -o 'jsonpath={.spec.replsets[].expose.enabled}'
++++++ mktemp
+++++ local LAST_OUT=/tmp/tmp.a0vjxmd0V0
++++++ mktemp
+++++ local LAST_ERR=/tmp/tmp.AqYE2knK2X
+++++ local exit_status=0
+++++ local timeout=4
++++++ seq 0 2
+++++ for i in '$(seq 0 2)'
+++++ set +e
+++++ kubectl get psmdb/monitoring-service -o 'jsonpath={.spec.replsets[].expose.enabled}'
+++++ exit_status=1
+++++ set -e
+++++ '[' 1 '!=' 0 -a -n 1 ']'
+++++ cat /tmp/tmp.a0vjxmd0V0
+++++ cat /tmp/tmp.AqYE2knK2X
Error from server (NotFound): perconaservermongodbs.psmdb.percona.com "monitoring-service" not found
+++++ sleep 0
+++++ for i in '$(seq 0 2)'
+++++ set +e
+++++ kubectl get psmdb/monitoring-service -o 'jsonpath={.spec.replsets[].expose.enabled}'
+++++ exit_status=1
+++++ set -e
+++++ '[' 1 '!=' 0 -a -n 1 ']'
+++++ cat /tmp/tmp.a0vjxmd0V0
+++++ cat /tmp/tmp.AqYE2knK2X
Error from server (NotFound): perconaservermongodbs.psmdb.percona.com "monitoring-service" not found
+++++ sleep 4
+++++ for i in '$(seq 0 2)'
+++++ set +e
+++++ kubectl get psmdb/monitoring-service -o 'jsonpath={.spec.replsets[].expose.enabled}'
+++++ exit_status=1
+++++ set -e
+++++ '[' 1 '!=' 0 -a -n 1 ']'
+++++ cat /tmp/tmp.a0vjxmd0V0
+++++ cat /tmp/tmp.AqYE2knK2X
Error from server (NotFound): perconaservermongodbs.psmdb.percona.com "monitoring-service" not found
+++++ sleep 8
+++++ cat /tmp/tmp.a0vjxmd0V0
+++++ cat /tmp/tmp.AqYE2knK2X
Error from server (NotFound): perconaservermongodbs.psmdb.percona.com "monitoring-service" not found
+++++ rm /tmp/tmp.a0vjxmd0V0 /tmp/tmp.AqYE2knK2X
+++++ return 1
++++ '[' '' '!=' true ']'
++++ echo -n monitoring-service.monitoring-service-rs0
++++ return
+++ kubectl_bin exec -n monitoring-2-0-16090 monitoring-0 -- pmm-admin --server-url=https://admin:admin@monitoring-service.monitoring-service-rs0/ --server-insecure-tls inventory list nodes --node-type=CONTAINER_NODE
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.Enip8RNPH2
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.4M7gJtnLeF
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in '$(seq 0 2)'
+++ set +e
+++ kubectl exec -n monitoring-2-0-16090 monitoring-0 -- pmm-admin --server-url=https://admin:admin@monitoring-service.monitoring-service-rs0/ --server-insecure-tls inventory list nodes --node-type=CONTAINER_NODE
+++ exit_status=1
+++ set -e
+++ '[' 1 '!=' 0 -a -n 1 ']'
+++ cat /tmp/tmp.Enip8RNPH2
+++ cat /tmp/tmp.4M7gJtnLeF
command terminated with exit code 1
+++ sleep 0
+++ for i in '$(seq 0 2)'
+++ set +e
+++ kubectl exec -n monitoring-2-0-16090 monitoring-0 -- pmm-admin --server-url=https://admin:admin@monitoring-service.monitoring-service-rs0/ --server-insecure-tls inventory list nodes --node-type=CONTAINER_NODE
+++ exit_status=1
+++ set -e
+++ '[' 1 '!=' 0 -a -n 1 ']'
+++ cat /tmp/tmp.Enip8RNPH2
+++ cat /tmp/tmp.4M7gJtnLeF
command terminated with exit code 1
+++ sleep 4
+++ for i in '$(seq 0 2)'
+++ set +e
+++ kubectl exec -n monitoring-2-0-16090 monitoring-0 -- pmm-admin --server-url=https://admin:admin@monitoring-service.monitoring-service-rs0/ --server-insecure-tls inventory list nodes --node-type=CONTAINER_NODE
+++ exit_status=1
+++ set -e
+++ '[' 1 '!=' 0 -a -n 1 ']'
+++ cat /tmp/tmp.Enip8RNPH2
+++ cat /tmp/tmp.4M7gJtnLeF
command terminated with exit code 1
+++ sleep 8
+++ cat /tmp/tmp.Enip8RNPH2
+++ cat /tmp/tmp.4M7gJtnLeF
command terminated with exit code 1
+++ rm /tmp/tmp.Enip8RNPH2 /tmp/tmp.4M7gJtnLeF
+++ return 1
++ for node_id in '"${nodeList[@]}"'
++ nodeList_from_pmm+=($(kubectl_bin exec -n "${namespace}" monitoring-0 -- pmm-admin --server-url=https://admin:admin@$(get_service_ip monitoring-service)/ --server-insecure-tls inventory list nodes --node-type=CONTAINER_NODE | grep $node_id | awk '{print $4}'))
+++ grep /node_id/54bc59c3-da6b-42c2-bc81-63bf5c67a979
+++ awk '{print $4}'
++++ get_service_ip monitoring-service
++++ local service=monitoring-service
++++ local server_type=rs0
+++++ kubectl_bin get psmdb/monitoring-service -o 'jsonpath={.spec.replsets[].expose.enabled}'
++++++ mktemp
+++++ local LAST_OUT=/tmp/tmp.NOy8yqvbqU
++++++ mktemp
+++++ local LAST_ERR=/tmp/tmp.ERRzbsuoK6
+++++ local exit_status=0
+++++ local timeout=4
++++++ seq 0 2
+++++ for i in '$(seq 0 2)'
+++++ set +e
+++++ kubectl get psmdb/monitoring-service -o 'jsonpath={.spec.replsets[].expose.enabled}'
+++++ exit_status=1
+++++ set -e
+++++ '[' 1 '!=' 0 -a -n 1 ']'
+++++ cat /tmp/tmp.NOy8yqvbqU
+++++ cat /tmp/tmp.ERRzbsuoK6
Error from server (NotFound): perconaservermongodbs.psmdb.percona.com "monitoring-service" not found
+++++ sleep 0
+++++ for i in '$(seq 0 2)'
+++++ set +e
+++++ kubectl get psmdb/monitoring-service -o 'jsonpath={.spec.replsets[].expose.enabled}'
+++++ exit_status=1
+++++ set -e
+++++ '[' 1 '!=' 0 -a -n 1 ']'
+++++ cat /tmp/tmp.NOy8yqvbqU
+++++ cat /tmp/tmp.ERRzbsuoK6
Error from server (NotFound): perconaservermongodbs.psmdb.percona.com "monitoring-service" not found
+++++ sleep 4
+++++ for i in '$(seq 0 2)'
+++++ set +e
+++++ kubectl get psmdb/monitoring-service -o 'jsonpath={.spec.replsets[].expose.enabled}'
+++++ exit_status=1
+++++ set -e
+++++ '[' 1 '!=' 0 -a -n 1 ']'
+++++ cat /tmp/tmp.NOy8yqvbqU
+++++ cat /tmp/tmp.ERRzbsuoK6
Error from server (NotFound): perconaservermongodbs.psmdb.percona.com "monitoring-service" not found
+++++ sleep 8
+++++ cat /tmp/tmp.NOy8yqvbqU
+++++ cat /tmp/tmp.ERRzbsuoK6
Error from server (NotFound): perconaservermongodbs.psmdb.percona.com "monitoring-service" not found
+++++ rm /tmp/tmp.NOy8yqvbqU /tmp/tmp.ERRzbsuoK6
+++++ return 1
++++ '[' '' '!=' true ']'
++++ echo -n monitoring-service.monitoring-service-rs0
++++ return
+++ kubectl_bin exec -n monitoring-2-0-16090 monitoring-0 -- pmm-admin --server-url=https://admin:admin@monitoring-service.monitoring-service-rs0/ --server-insecure-tls inventory list nodes --node-type=CONTAINER_NODE
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.UoBIgK3Ick
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.rrrbW8AoLU
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in '$(seq 0 2)'
+++ set +e
+++ kubectl exec -n monitoring-2-0-16090 monitoring-0 -- pmm-admin --server-url=https://admin:admin@monitoring-service.monitoring-service-rs0/ --server-insecure-tls inventory list nodes --node-type=CONTAINER_NODE
+++ exit_status=1
+++ set -e
+++ '[' 1 '!=' 0 -a -n 1 ']'
+++ cat /tmp/tmp.UoBIgK3Ick
+++ cat /tmp/tmp.rrrbW8AoLU
command terminated with exit code 1
+++ sleep 0
+++ for i in '$(seq 0 2)'
+++ set +e
+++ kubectl exec -n monitoring-2-0-16090 monitoring-0 -- pmm-admin --server-url=https://admin:admin@monitoring-service.monitoring-service-rs0/ --server-insecure-tls inventory list nodes --node-type=CONTAINER_NODE
+++ exit_status=1
+++ set -e
+++ '[' 1 '!=' 0 -a -n 1 ']'
+++ cat /tmp/tmp.UoBIgK3Ick
+++ cat /tmp/tmp.rrrbW8AoLU
command terminated with exit code 1
+++ sleep 4
+++ for i in '$(seq 0 2)'
+++ set +e
+++ kubectl exec -n monitoring-2-0-16090 monitoring-0 -- pmm-admin --server-url=https://admin:admin@monitoring-service.monitoring-service-rs0/ --server-insecure-tls inventory list nodes --node-type=CONTAINER_NODE
+++ exit_status=1
+++ set -e
+++ '[' 1 '!=' 0 -a -n 1 ']'
+++ cat /tmp/tmp.UoBIgK3Ick
+++ cat /tmp/tmp.rrrbW8AoLU
command terminated with exit code 1
+++ sleep 8
+++ cat /tmp/tmp.UoBIgK3Ick
+++ cat /tmp/tmp.rrrbW8AoLU
command terminated with exit code 1
+++ rm /tmp/tmp.UoBIgK3Ick /tmp/tmp.rrrbW8AoLU
+++ return 1
++ for node_id in '"${nodeList[@]}"'
++ nodeList_from_pmm+=($(kubectl_bin exec -n "${namespace}" monitoring-0 -- pmm-admin --server-url=https://admin:admin@$(get_service_ip monitoring-service)/ --server-insecure-tls inventory list nodes --node-type=CONTAINER_NODE | grep $node_id | awk '{print $4}'))
+++ grep /node_id/ebeaa667-fe43-4326-ba0e-e6e852c734ce
+++ awk '{print $4}'
++++ get_service_ip monitoring-service
++++ local service=monitoring-service
++++ local server_type=rs0
+++++ kubectl_bin get psmdb/monitoring-service -o 'jsonpath={.spec.replsets[].expose.enabled}'
++++++ mktemp
+++++ local LAST_OUT=/tmp/tmp.hkT9JftiG9
++++++ mktemp
+++++ local LAST_ERR=/tmp/tmp.EBLCom3wZ3
+++++ local exit_status=0
+++++ local timeout=4
++++++ seq 0 2
+++++ for i in '$(seq 0 2)'
+++++ set +e
+++++ kubectl get psmdb/monitoring-service -o 'jsonpath={.spec.replsets[].expose.enabled}'
+++++ exit_status=1
+++++ set -e
+++++ '[' 1 '!=' 0 -a -n 1 ']'
+++++ cat /tmp/tmp.hkT9JftiG9
+++++ cat /tmp/tmp.EBLCom3wZ3
Error from server (NotFound): perconaservermongodbs.psmdb.percona.com "monitoring-service" not found
+++++ sleep 0
+++++ for i in '$(seq 0 2)'
+++++ set +e
+++++ kubectl get psmdb/monitoring-service -o 'jsonpath={.spec.replsets[].expose.enabled}'
+++++ exit_status=1
+++++ set -e
+++++ '[' 1 '!=' 0 -a -n 1 ']'
+++++ cat /tmp/tmp.hkT9JftiG9
+++++ cat /tmp/tmp.EBLCom3wZ3
Error from server (NotFound): perconaservermongodbs.psmdb.percona.com "monitoring-service" not found
+++++ sleep 4
+++++ for i in '$(seq 0 2)'
+++++ set +e
+++++ kubectl get psmdb/monitoring-service -o 'jsonpath={.spec.replsets[].expose.enabled}'
+++++ exit_status=1
+++++ set -e
+++++ '[' 1 '!=' 0 -a -n 1 ']'
+++++ cat /tmp/tmp.hkT9JftiG9
+++++ cat /tmp/tmp.EBLCom3wZ3
Error from server (NotFound): perconaservermongodbs.psmdb.percona.com "monitoring-service" not found
+++++ sleep 8
+++++ cat /tmp/tmp.hkT9JftiG9
+++++ cat /tmp/tmp.EBLCom3wZ3
Error from server (NotFound): perconaservermongodbs.psmdb.percona.com "monitoring-service" not found
+++++ rm /tmp/tmp.hkT9JftiG9 /tmp/tmp.EBLCom3wZ3
+++++ return 1
++++ '[' '' '!=' true ']'
++++ echo -n monitoring-service.monitoring-service-rs0
++++ return
+++ kubectl_bin exec -n monitoring-2-0-16090 monitoring-0 -- pmm-admin --server-url=https://admin:admin@monitoring-service.monitoring-service-rs0/ --server-insecure-tls inventory list nodes --node-type=CONTAINER_NODE
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.V2pcD92DPe
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.f1oEKh1hfd
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in '$(seq 0 2)'
+++ set +e
+++ kubectl exec -n monitoring-2-0-16090 monitoring-0 -- pmm-admin --server-url=https://admin:admin@monitoring-service.monitoring-service-rs0/ --server-insecure-tls inventory list nodes --node-type=CONTAINER_NODE
+++ exit_status=1
+++ set -e
+++ '[' 1 '!=' 0 -a -n 1 ']'
+++ cat /tmp/tmp.V2pcD92DPe
+++ cat /tmp/tmp.f1oEKh1hfd
command terminated with exit code 1
+++ sleep 0
+++ for i in '$(seq 0 2)'
+++ set +e
+++ kubectl exec -n monitoring-2-0-16090 monitoring-0 -- pmm-admin --server-url=https://admin:admin@monitoring-service.monitoring-service-rs0/ --server-insecure-tls inventory list nodes --node-type=CONTAINER_NODE
+++ exit_status=1
+++ set -e
+++ '[' 1 '!=' 0 -a -n 1 ']'
+++ cat /tmp/tmp.V2pcD92DPe
+++ cat /tmp/tmp.f1oEKh1hfd
command terminated with exit code 1
+++ sleep 4
+++ for i in '$(seq 0 2)'
+++ set +e
+++ kubectl exec -n monitoring-2-0-16090 monitoring-0 -- pmm-admin --server-url=https://admin:admin@monitoring-service.monitoring-service-rs0/ --server-insecure-tls inventory list nodes --node-type=CONTAINER_NODE
+++ exit_status=1
+++ set -e
+++ '[' 1 '!=' 0 -a -n 1 ']'
+++ cat /tmp/tmp.V2pcD92DPe
+++ cat /tmp/tmp.f1oEKh1hfd
command terminated with exit code 1
+++ sleep 8
+++ cat /tmp/tmp.V2pcD92DPe
+++ cat /tmp/tmp.f1oEKh1hfd
command terminated with exit code 1
+++ rm /tmp/tmp.V2pcD92DPe /tmp/tmp.f1oEKh1hfd
+++ return 1
++ for node_id in '"${nodeList[@]}"'
++ nodeList_from_pmm+=($(kubectl_bin exec -n "${namespace}" monitoring-0 -- pmm-admin --server-url=https://admin:admin@$(get_service_ip monitoring-service)/ --server-insecure-tls inventory list nodes --node-type=CONTAINER_NODE | grep $node_id | awk '{print $4}'))
+++ grep /node_id/0af1fe4a-8b09-452c-82a1-ed469e898790
+++ awk '{print $4}'
++++ get_service_ip monitoring-service
++++ local service=monitoring-service
++++ local server_type=rs0
+++++ kubectl_bin get psmdb/monitoring-service -o 'jsonpath={.spec.replsets[].expose.enabled}'
++++++ mktemp
+++++ local LAST_OUT=/tmp/tmp.QSOrcd08vj
++++++ mktemp
+++++ local LAST_ERR=/tmp/tmp.KORRJIhNE1
+++++ local exit_status=0
+++++ local timeout=4
++++++ seq 0 2
+++++ for i in '$(seq 0 2)'
+++++ set +e
+++++ kubectl get psmdb/monitoring-service -o 'jsonpath={.spec.replsets[].expose.enabled}'
+++++ exit_status=1
+++++ set -e
+++++ '[' 1 '!=' 0 -a -n 1 ']'
+++++ cat /tmp/tmp.QSOrcd08vj
+++++ cat /tmp/tmp.KORRJIhNE1
Error from server (NotFound): perconaservermongodbs.psmdb.percona.com "monitoring-service" not found
+++++ sleep 0
+++++ for i in '$(seq 0 2)'
+++++ set +e
+++++ kubectl get psmdb/monitoring-service -o 'jsonpath={.spec.replsets[].expose.enabled}'
+++++ exit_status=1
+++++ set -e
+++++ '[' 1 '!=' 0 -a -n 1 ']'
+++++ cat /tmp/tmp.QSOrcd08vj
+++++ cat /tmp/tmp.KORRJIhNE1
Error from server (NotFound): perconaservermongodbs.psmdb.percona.com "monitoring-service" not found
+++++ sleep 4
+++++ for i in '$(seq 0 2)'
+++++ set +e
+++++ kubectl get psmdb/monitoring-service -o 'jsonpath={.spec.replsets[].expose.enabled}'
+++++ exit_status=1
+++++ set -e
+++++ '[' 1 '!=' 0 -a -n 1 ']'
+++++ cat /tmp/tmp.QSOrcd08vj
+++++ cat /tmp/tmp.KORRJIhNE1
Error from server (NotFound): perconaservermongodbs.psmdb.percona.com "monitoring-service" not found
+++++ sleep 8
+++++ cat /tmp/tmp.QSOrcd08vj
+++++ cat /tmp/tmp.KORRJIhNE1
Error from server (NotFound): perconaservermongodbs.psmdb.percona.com "monitoring-service" not found
+++++ rm /tmp/tmp.QSOrcd08vj /tmp/tmp.KORRJIhNE1
+++++ return 1
++++ '[' '' '!=' true ']'
++++ echo -n monitoring-service.monitoring-service-rs0
++++ return
+++ kubectl_bin exec -n monitoring-2-0-16090 monitoring-0 -- pmm-admin --server-url=https://admin:admin@monitoring-service.monitoring-service-rs0/ --server-insecure-tls inventory list nodes --node-type=CONTAINER_NODE
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.zqifMX6U6P
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.m2N1HL838m
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in '$(seq 0 2)'
+++ set +e
+++ kubectl exec -n monitoring-2-0-16090 monitoring-0 -- pmm-admin --server-url=https://admin:admin@monitoring-service.monitoring-service-rs0/ --server-insecure-tls inventory list nodes --node-type=CONTAINER_NODE
+++ exit_status=1
+++ set -e
+++ '[' 1 '!=' 0 -a -n 1 ']'
+++ cat /tmp/tmp.zqifMX6U6P
+++ cat /tmp/tmp.m2N1HL838m
command terminated with exit code 1
+++ sleep 0
+++ for i in '$(seq 0 2)'
+++ set +e
+++ kubectl exec -n monitoring-2-0-16090 monitoring-0 -- pmm-admin --server-url=https://admin:admin@monitoring-service.monitoring-service-rs0/ --server-insecure-tls inventory list nodes --node-type=CONTAINER_NODE
+++ exit_status=1
+++ set -e
+++ '[' 1 '!=' 0 -a -n 1 ']'
+++ cat /tmp/tmp.zqifMX6U6P
+++ cat /tmp/tmp.m2N1HL838m
command terminated with exit code 1
+++ sleep 4
+++ for i in '$(seq 0 2)'
+++ set +e
+++ kubectl exec -n monitoring-2-0-16090 monitoring-0 -- pmm-admin --server-url=https://admin:admin@monitoring-service.monitoring-service-rs0/ --server-insecure-tls inventory list nodes --node-type=CONTAINER_NODE
+++ exit_status=1
+++ set -e
+++ '[' 1 '!=' 0 -a -n 1 ']'
+++ cat /tmp/tmp.zqifMX6U6P
+++ cat /tmp/tmp.m2N1HL838m
command terminated with exit code 1
+++ sleep 8
+++ cat /tmp/tmp.zqifMX6U6P
+++ cat /tmp/tmp.m2N1HL838m
command terminated with exit code 1
+++ rm /tmp/tmp.zqifMX6U6P /tmp/tmp.m2N1HL838m
+++ return 1
++ for node_id in '"${nodeList[@]}"'
++ nodeList_from_pmm+=($(kubectl_bin exec -n "${namespace}" monitoring-0 -- pmm-admin --server-url=https://admin:admin@$(get_service_ip monitoring-service)/ --server-insecure-tls inventory list nodes --node-type=CONTAINER_NODE | grep $node_id | awk '{print $4}'))
+++ grep /node_id/c03612ee-2d8b-4228-b363-041e3ae52a53
+++ awk '{print $4}'
++++ get_service_ip monitoring-service
++++ local service=monitoring-service
++++ local server_type=rs0
+++++ kubectl_bin get psmdb/monitoring-service -o 'jsonpath={.spec.replsets[].expose.enabled}'
++++++ mktemp
+++++ local LAST_OUT=/tmp/tmp.BEwbpTSelT
++++++ mktemp
+++++ local LAST_ERR=/tmp/tmp.HHiHTY9OPT
+++++ local exit_status=0
+++++ local timeout=4
++++++ seq 0 2
+++++ for i in '$(seq 0 2)'
+++++ set +e
+++++ kubectl get psmdb/monitoring-service -o 'jsonpath={.spec.replsets[].expose.enabled}'
+++++ exit_status=1
+++++ set -e
+++++ '[' 1 '!=' 0 -a -n 1 ']'
+++++ cat /tmp/tmp.BEwbpTSelT
+++++ cat /tmp/tmp.HHiHTY9OPT
Error from server (NotFound): perconaservermongodbs.psmdb.percona.com "monitoring-service" not found
+++++ sleep 0
+++++ for i in '$(seq 0 2)'
+++++ set +e
+++++ kubectl get psmdb/monitoring-service -o 'jsonpath={.spec.replsets[].expose.enabled}'
+++++ exit_status=1
+++++ set -e
+++++ '[' 1 '!=' 0 -a -n 1 ']'
+++++ cat /tmp/tmp.BEwbpTSelT
+++++ cat /tmp/tmp.HHiHTY9OPT
Error from server (NotFound): perconaservermongodbs.psmdb.percona.com "monitoring-service" not found
+++++ sleep 4
+++++ for i in '$(seq 0 2)'
+++++ set +e
+++++ kubectl get psmdb/monitoring-service -o 'jsonpath={.spec.replsets[].expose.enabled}'
+++++ exit_status=1
+++++ set -e
+++++ '[' 1 '!=' 0 -a -n 1 ']'
+++++ cat /tmp/tmp.BEwbpTSelT
+++++ cat /tmp/tmp.HHiHTY9OPT
Error from server (NotFound): perconaservermongodbs.psmdb.percona.com "monitoring-service" not found
+++++ sleep 8
+++++ cat /tmp/tmp.BEwbpTSelT
+++++ cat /tmp/tmp.HHiHTY9OPT
Error from server (NotFound): perconaservermongodbs.psmdb.percona.com "monitoring-service" not found
+++++ rm /tmp/tmp.BEwbpTSelT /tmp/tmp.HHiHTY9OPT
+++++ return 1
++++ '[' '' '!=' true ']'
++++ echo -n monitoring-service.monitoring-service-rs0
++++ return
+++ kubectl_bin exec -n monitoring-2-0-16090 monitoring-0 -- pmm-admin --server-url=https://admin:admin@monitoring-service.monitoring-service-rs0/ --server-insecure-tls inventory list nodes --node-type=CONTAINER_NODE
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.iGMFDQOBzs
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.IttUX04N08
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in '$(seq 0 2)'
+++ set +e
+++ kubectl exec -n monitoring-2-0-16090 monitoring-0 -- pmm-admin --server-url=https://admin:admin@monitoring-service.monitoring-service-rs0/ --server-insecure-tls inventory list nodes --node-type=CONTAINER_NODE
+++ exit_status=1
+++ set -e
+++ '[' 1 '!=' 0 -a -n 1 ']'
+++ cat /tmp/tmp.iGMFDQOBzs
+++ cat /tmp/tmp.IttUX04N08
command terminated with exit code 1
+++ sleep 0
+++ for i in '$(seq 0 2)'
+++ set +e
+++ kubectl exec -n monitoring-2-0-16090 monitoring-0 -- pmm-admin --server-url=https://admin:admin@monitoring-service.monitoring-service-rs0/ --server-insecure-tls inventory list nodes --node-type=CONTAINER_NODE
+++ exit_status=1
+++ set -e
+++ '[' 1 '!=' 0 -a -n 1 ']'
+++ cat /tmp/tmp.iGMFDQOBzs
+++ cat /tmp/tmp.IttUX04N08
command terminated with exit code 1
+++ sleep 4
+++ for i in '$(seq 0 2)'
+++ set +e
+++ kubectl exec -n monitoring-2-0-16090 monitoring-0 -- pmm-admin --server-url=https://admin:admin@monitoring-service.monitoring-service-rs0/ --server-insecure-tls inventory list nodes --node-type=CONTAINER_NODE
+++ exit_status=1
+++ set -e
+++ '[' 1 '!=' 0 -a -n 1 ']'
+++ cat /tmp/tmp.iGMFDQOBzs
+++ cat /tmp/tmp.IttUX04N08
command terminated with exit code 1
+++ sleep 8
+++ cat /tmp/tmp.iGMFDQOBzs
+++ cat /tmp/tmp.IttUX04N08
command terminated with exit code 1
+++ rm /tmp/tmp.iGMFDQOBzs /tmp/tmp.IttUX04N08
+++ return 1
++ for node_id in '"${nodeList[@]}"'
++ nodeList_from_pmm+=($(kubectl_bin exec -n "${namespace}" monitoring-0 -- pmm-admin --server-url=https://admin:admin@$(get_service_ip monitoring-service)/ --server-insecure-tls inventory list nodes --node-type=CONTAINER_NODE | grep $node_id | awk '{print $4}'))
+++ grep /node_id/fcb0e3b7-0883-4a6b-bd4e-f093cba949d9
+++ awk '{print $4}'
++++ get_service_ip monitoring-service
++++ local service=monitoring-service
++++ local server_type=rs0
+++++ kubectl_bin get psmdb/monitoring-service -o 'jsonpath={.spec.replsets[].expose.enabled}'
++++++ mktemp
+++++ local LAST_OUT=/tmp/tmp.XkqOyOBNlZ
++++++ mktemp
+++++ local LAST_ERR=/tmp/tmp.Zyl6UM4j1w
+++++ local exit_status=0
+++++ local timeout=4
++++++ seq 0 2
+++++ for i in '$(seq 0 2)'
+++++ set +e
+++++ kubectl get psmdb/monitoring-service -o 'jsonpath={.spec.replsets[].expose.enabled}'
+++++ exit_status=1
+++++ set -e
+++++ '[' 1 '!=' 0 -a -n 1 ']'
+++++ cat /tmp/tmp.XkqOyOBNlZ
+++++ cat /tmp/tmp.Zyl6UM4j1w
Error from server (NotFound): perconaservermongodbs.psmdb.percona.com "monitoring-service" not found
+++++ sleep 0
+++++ for i in '$(seq 0 2)'
+++++ set +e
+++++ kubectl get psmdb/monitoring-service -o 'jsonpath={.spec.replsets[].expose.enabled}'
+++++ exit_status=1
+++++ set -e
+++++ '[' 1 '!=' 0 -a -n 1 ']'
+++++ cat /tmp/tmp.XkqOyOBNlZ
+++++ cat /tmp/tmp.Zyl6UM4j1w
Error from server (NotFound): perconaservermongodbs.psmdb.percona.com "monitoring-service" not found
+++++ sleep 4
+++++ for i in '$(seq 0 2)'
+++++ set +e
+++++ kubectl get psmdb/monitoring-service -o 'jsonpath={.spec.replsets[].expose.enabled}'
+++++ exit_status=1
+++++ set -e
+++++ '[' 1 '!=' 0 -a -n 1 ']'
+++++ cat /tmp/tmp.XkqOyOBNlZ
+++++ cat /tmp/tmp.Zyl6UM4j1w
Error from server (NotFound): perconaservermongodbs.psmdb.percona.com "monitoring-service" not found
+++++ sleep 8
+++++ cat /tmp/tmp.XkqOyOBNlZ
+++++ cat /tmp/tmp.Zyl6UM4j1w
Error from server (NotFound): perconaservermongodbs.psmdb.percona.com "monitoring-service" not found
+++++ rm /tmp/tmp.XkqOyOBNlZ /tmp/tmp.Zyl6UM4j1w
+++++ return 1
++++ '[' '' '!=' true ']'
++++ echo -n monitoring-service.monitoring-service-rs0
++++ return
+++ kubectl_bin exec -n monitoring-2-0-16090 monitoring-0 -- pmm-admin --server-url=https://admin:admin@monitoring-service.monitoring-service-rs0/ --server-insecure-tls inventory list nodes --node-type=CONTAINER_NODE
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.e6QETE98Qo
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.7UoXhrQ7we
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in '$(seq 0 2)'
+++ set +e
+++ kubectl exec -n monitoring-2-0-16090 monitoring-0 -- pmm-admin --server-url=https://admin:admin@monitoring-service.monitoring-service-rs0/ --server-insecure-tls inventory list nodes --node-type=CONTAINER_NODE
+++ exit_status=1
+++ set -e
+++ '[' 1 '!=' 0 -a -n 1 ']'
+++ cat /tmp/tmp.e6QETE98Qo
+++ cat /tmp/tmp.7UoXhrQ7we
command terminated with exit code 1
+++ sleep 0
+++ for i in '$(seq 0 2)'
+++ set +e
+++ kubectl exec -n monitoring-2-0-16090 monitoring-0 -- pmm-admin --server-url=https://admin:admin@monitoring-service.monitoring-service-rs0/ --server-insecure-tls inventory list nodes --node-type=CONTAINER_NODE
+++ exit_status=1
+++ set -e
+++ '[' 1 '!=' 0 -a -n 1 ']'
+++ cat /tmp/tmp.e6QETE98Qo
+++ cat /tmp/tmp.7UoXhrQ7we
command terminated with exit code 1
+++ sleep 4
+++ for i in '$(seq 0 2)'
+++ set +e
+++ kubectl exec -n monitoring-2-0-16090 monitoring-0 -- pmm-admin --server-url=https://admin:admin@monitoring-service.monitoring-service-rs0/ --server-insecure-tls inventory list nodes --node-type=CONTAINER_NODE
+++ exit_status=1
+++ set -e
+++ '[' 1 '!=' 0 -a -n 1 ']'
+++ cat /tmp/tmp.e6QETE98Qo
+++ cat /tmp/tmp.7UoXhrQ7we
command terminated with exit code 1
+++ sleep 8
+++ cat /tmp/tmp.e6QETE98Qo
+++ cat /tmp/tmp.7UoXhrQ7we
command terminated with exit code 1
+++ rm /tmp/tmp.e6QETE98Qo /tmp/tmp.7UoXhrQ7we
+++ return 1
++ for node_id in '"${nodeList[@]}"'
++ nodeList_from_pmm+=($(kubectl_bin exec -n "${namespace}" monitoring-0 -- pmm-admin --server-url=https://admin:admin@$(get_service_ip monitoring-service)/ --server-insecure-tls inventory list nodes --node-type=CONTAINER_NODE | grep $node_id | awk '{print $4}'))
++++ get_service_ip monitoring-service
++++ local service=monitoring-service
++++ local server_type=rs0
+++++ kubectl_bin get psmdb/monitoring-service -o 'jsonpath={.spec.replsets[].expose.enabled}'
++++++ mktemp
+++ awk '{print $4}'
+++++ local LAST_OUT=/tmp/tmp.FClNLKo6EF
+++ grep /node_id/9817611e-c22a-442a-bf13-131d02ea6b93
++++++ mktemp
+++++ local LAST_ERR=/tmp/tmp.u3zpyE4oBn
+++++ local exit_status=0
+++++ local timeout=4
++++++ seq 0 2
+++++ for i in '$(seq 0 2)'
+++++ set +e
+++++ kubectl get psmdb/monitoring-service -o 'jsonpath={.spec.replsets[].expose.enabled}'
+++++ exit_status=1
+++++ set -e
+++++ '[' 1 '!=' 0 -a -n 1 ']'
+++++ cat /tmp/tmp.FClNLKo6EF
+++++ cat /tmp/tmp.u3zpyE4oBn
Error from server (NotFound): perconaservermongodbs.psmdb.percona.com "monitoring-service" not found
+++++ sleep 0
+++++ for i in '$(seq 0 2)'
+++++ set +e
+++++ kubectl get psmdb/monitoring-service -o 'jsonpath={.spec.replsets[].expose.enabled}'
+++++ exit_status=1
+++++ set -e
+++++ '[' 1 '!=' 0 -a -n 1 ']'
+++++ cat /tmp/tmp.FClNLKo6EF
+++++ cat /tmp/tmp.u3zpyE4oBn
Error from server (NotFound): perconaservermongodbs.psmdb.percona.com "monitoring-service" not found
+++++ sleep 4
+++++ for i in '$(seq 0 2)'
+++++ set +e
+++++ kubectl get psmdb/monitoring-service -o 'jsonpath={.spec.replsets[].expose.enabled}'
+++++ exit_status=1
+++++ set -e
+++++ '[' 1 '!=' 0 -a -n 1 ']'
+++++ cat /tmp/tmp.FClNLKo6EF
+++++ cat /tmp/tmp.u3zpyE4oBn
Error from server (NotFound): perconaservermongodbs.psmdb.percona.com "monitoring-service" not found
+++++ sleep 8
+++++ cat /tmp/tmp.FClNLKo6EF
+++++ cat /tmp/tmp.u3zpyE4oBn
Error from server (NotFound): perconaservermongodbs.psmdb.percona.com "monitoring-service" not found
+++++ rm /tmp/tmp.FClNLKo6EF /tmp/tmp.u3zpyE4oBn
+++++ return 1
++++ '[' '' '!=' true ']'
++++ echo -n monitoring-service.monitoring-service-rs0
++++ return
+++ kubectl_bin exec -n monitoring-2-0-16090 monitoring-0 -- pmm-admin --server-url=https://admin:admin@monitoring-service.monitoring-service-rs0/ --server-insecure-tls inventory list nodes --node-type=CONTAINER_NODE
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.Iuj6RTET6Q
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.OiQKAwR9OP
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in '$(seq 0 2)'
+++ set +e
+++ kubectl exec -n monitoring-2-0-16090 monitoring-0 -- pmm-admin --server-url=https://admin:admin@monitoring-service.monitoring-service-rs0/ --server-insecure-tls inventory list nodes --node-type=CONTAINER_NODE
+++ exit_status=1
+++ set -e
+++ '[' 1 '!=' 0 -a -n 1 ']'
+++ cat /tmp/tmp.Iuj6RTET6Q
+++ cat /tmp/tmp.OiQKAwR9OP
command terminated with exit code 1
+++ sleep 0
+++ for i in '$(seq 0 2)'
+++ set +e
+++ kubectl exec -n monitoring-2-0-16090 monitoring-0 -- pmm-admin --server-url=https://admin:admin@monitoring-service.monitoring-service-rs0/ --server-insecure-tls inventory list nodes --node-type=CONTAINER_NODE
+++ exit_status=1
+++ set -e
+++ '[' 1 '!=' 0 -a -n 1 ']'
+++ cat /tmp/tmp.Iuj6RTET6Q
+++ cat /tmp/tmp.OiQKAwR9OP
command terminated with exit code 1
+++ sleep 4
+++ for i in '$(seq 0 2)'
+++ set +e
+++ kubectl exec -n monitoring-2-0-16090 monitoring-0 -- pmm-admin --server-url=https://admin:admin@monitoring-service.monitoring-service-rs0/ --server-insecure-tls inventory list nodes --node-type=CONTAINER_NODE
+++ exit_status=1
+++ set -e
+++ '[' 1 '!=' 0 -a -n 1 ']'
+++ cat /tmp/tmp.Iuj6RTET6Q
+++ cat /tmp/tmp.OiQKAwR9OP
command terminated with exit code 1
+++ sleep 8
+++ cat /tmp/tmp.Iuj6RTET6Q
+++ cat /tmp/tmp.OiQKAwR9OP
command terminated with exit code 1
+++ rm /tmp/tmp.Iuj6RTET6Q /tmp/tmp.OiQKAwR9OP
+++ return 1
++ echo
+ kubectl_bin patch psmdb monitoring --type json '-p=[{"op":"add","path":"/spec/pause","value":true}]'
++ mktemp
+ local LAST_OUT=/tmp/tmp.EBfRtbEtYm
++ mktemp
+ local LAST_ERR=/tmp/tmp.Qz8pGvOzGo
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl patch psmdb monitoring --type json '-p=[{"op":"add","path":"/spec/pause","value":true}]'
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.EBfRtbEtYm
perconaservermongodb.psmdb.percona.com/monitoring patched
+ cat /tmp/tmp.Qz8pGvOzGo
+ rm /tmp/tmp.EBfRtbEtYm /tmp/tmp.Qz8pGvOzGo
+ return 0
+ wait_for_delete pod/monitoring-mongos-0
+ local res=pod/monitoring-mongos-0
+ local wait_time=60
+ set +o xtrace
waiting for pod/monitoring-mongos-0 to be deleted..........................Error from server (NotFound): pods "monitoring-mongos-0" not found
Error from server (NotFound): pods "monitoring-mongos-0" not found
Error from server (NotFound): pods "monitoring-mongos-0" not found
Error from server (NotFound): pods "monitoring-mongos-0" not found
+ wait_for_delete pod/monitoring-rs0-0
+ local res=pod/monitoring-rs0-0
+ local wait_time=60
+ set +o xtrace
waiting for pod/monitoring-rs0-0 to be deleted......Error from server (NotFound): pods "monitoring-rs0-0" not found
Error from server (NotFound): pods "monitoring-rs0-0" not found
Error from server (NotFound): pods "monitoring-rs0-0" not found
Error from server (NotFound): pods "monitoring-rs0-0" not found
+ wait_for_delete pod/monitoring-cfg-0
+ local res=pod/monitoring-cfg-0
+ local wait_time=60
+ set +o xtrace
waiting for pod/monitoring-cfg-0 to be deleted.......Error from server (NotFound): pods "monitoring-cfg-0" not found
Error from server (NotFound): pods "monitoring-cfg-0" not found
Error from server (NotFound): pods "monitoring-cfg-0" not found
Error from server (NotFound): pods "monitoring-cfg-0" not found
+ desc 'check if services are not deleted'
+ set +o xtrace


-----------------------------------------------------------------------------------
check if services are not deleted
-----------------------------------------------------------------------------------

+ kubectl_bin get svc monitoring-rs0
++ mktemp
+ local LAST_OUT=/tmp/tmp.oUxyoiE4N4
++ mktemp
+ local LAST_ERR=/tmp/tmp.LYPvMopumb
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl get svc monitoring-rs0
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.oUxyoiE4N4
NAME             TYPE        CLUSTER-IP   EXTERNAL-IP   PORT(S)     AGE
monitoring-rs0   ClusterIP   None         <none>        27017/TCP   17m
+ cat /tmp/tmp.LYPvMopumb
+ rm /tmp/tmp.oUxyoiE4N4 /tmp/tmp.LYPvMopumb
+ return 0
+ kubectl_bin get svc monitoring-cfg
++ mktemp
+ local LAST_OUT=/tmp/tmp.vF1YHKzT3x
++ mktemp
+ local LAST_ERR=/tmp/tmp.JkPRvWanrE
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl get svc monitoring-cfg
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.vF1YHKzT3x
NAME             TYPE        CLUSTER-IP   EXTERNAL-IP   PORT(S)     AGE
monitoring-cfg   ClusterIP   None         <none>        27017/TCP   17m
+ cat /tmp/tmp.JkPRvWanrE
+ rm /tmp/tmp.vF1YHKzT3x /tmp/tmp.JkPRvWanrE
+ return 0
+ kubectl_bin get svc monitoring-mongos
++ mktemp
+ local LAST_OUT=/tmp/tmp.s0Q4FQytzQ
++ mktemp
+ local LAST_ERR=/tmp/tmp.dmYTUZDaoP
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl get svc monitoring-mongos
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.s0Q4FQytzQ
NAME                TYPE        CLUSTER-IP       EXTERNAL-IP   PORT(S)     AGE
monitoring-mongos   ClusterIP   10.137.185.248   <none>        27017/TCP   15m
+ cat /tmp/tmp.dmYTUZDaoP
+ rm /tmp/tmp.s0Q4FQytzQ /tmp/tmp.dmYTUZDaoP
+ return 0
+ does_node_id_exists_in_pmm=($(does_node_id_exists "${nodeList[@]}"))
++ does_node_id_exists /node_id/def17f27-15a0-4240-ac39-0614beb243e6 /node_id/efac200c-f35c-407e-815d-3728aea09fbd /node_id/6eb9ff8f-5f1f-433b-a759-ff8ca749f427 /node_id/54bc59c3-da6b-42c2-bc81-63bf5c67a979 /node_id/ebeaa667-fe43-4326-ba0e-e6e852c734ce /node_id/0af1fe4a-8b09-452c-82a1-ed469e898790 /node_id/c03612ee-2d8b-4228-b363-041e3ae52a53 /node_id/fcb0e3b7-0883-4a6b-bd4e-f093cba949d9 /node_id/9817611e-c22a-442a-bf13-131d02ea6b93
++ nodeList=("$@")
++ local -a nodeList
++ nodeList_from_pmm=()
++ local -a nodeList_from_pmm
++ for node_id in '"${nodeList[@]}"'
++ nodeList_from_pmm+=($(kubectl_bin exec -n "${namespace}" monitoring-0 -- pmm-admin --server-url=https://admin:admin@$(get_service_ip monitoring-service)/ --server-insecure-tls inventory list nodes --node-type=CONTAINER_NODE | grep $node_id | awk '{print $4}'))
+++ awk '{print $4}'
++++ get_service_ip monitoring-service
++++ local service=monitoring-service
++++ local server_type=rs0
+++ grep /node_id/def17f27-15a0-4240-ac39-0614beb243e6
+++++ kubectl_bin get psmdb/monitoring-service -o 'jsonpath={.spec.replsets[].expose.enabled}'
++++++ mktemp
+++++ local LAST_OUT=/tmp/tmp.ctEL8u66hE
++++++ mktemp
+++++ local LAST_ERR=/tmp/tmp.Jya4cZ29QB
+++++ local exit_status=0
+++++ local timeout=4
++++++ seq 0 2
+++++ for i in '$(seq 0 2)'
+++++ set +e
+++++ kubectl get psmdb/monitoring-service -o 'jsonpath={.spec.replsets[].expose.enabled}'
+++++ exit_status=1
+++++ set -e
+++++ '[' 1 '!=' 0 -a -n 1 ']'
+++++ cat /tmp/tmp.ctEL8u66hE
+++++ cat /tmp/tmp.Jya4cZ29QB
Error from server (NotFound): perconaservermongodbs.psmdb.percona.com "monitoring-service" not found
+++++ sleep 0
+++++ for i in '$(seq 0 2)'
+++++ set +e
+++++ kubectl get psmdb/monitoring-service -o 'jsonpath={.spec.replsets[].expose.enabled}'
+++++ exit_status=1
+++++ set -e
+++++ '[' 1 '!=' 0 -a -n 1 ']'
+++++ cat /tmp/tmp.ctEL8u66hE
+++++ cat /tmp/tmp.Jya4cZ29QB
Error from server (NotFound): perconaservermongodbs.psmdb.percona.com "monitoring-service" not found
+++++ sleep 4
+++++ for i in '$(seq 0 2)'
+++++ set +e
+++++ kubectl get psmdb/monitoring-service -o 'jsonpath={.spec.replsets[].expose.enabled}'
+++++ exit_status=1
+++++ set -e
+++++ '[' 1 '!=' 0 -a -n 1 ']'
+++++ cat /tmp/tmp.ctEL8u66hE
+++++ cat /tmp/tmp.Jya4cZ29QB
Error from server (NotFound): perconaservermongodbs.psmdb.percona.com "monitoring-service" not found
+++++ sleep 8
+++++ cat /tmp/tmp.ctEL8u66hE
+++++ cat /tmp/tmp.Jya4cZ29QB
Error from server (NotFound): perconaservermongodbs.psmdb.percona.com "monitoring-service" not found
+++++ rm /tmp/tmp.ctEL8u66hE /tmp/tmp.Jya4cZ29QB
+++++ return 1
++++ '[' '' '!=' true ']'
++++ echo -n monitoring-service.monitoring-service-rs0
++++ return
+++ kubectl_bin exec -n monitoring-2-0-16090 monitoring-0 -- pmm-admin --server-url=https://admin:admin@monitoring-service.monitoring-service-rs0/ --server-insecure-tls inventory list nodes --node-type=CONTAINER_NODE
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.2eLEjg54gv
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.3o9c4JKbCF
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in '$(seq 0 2)'
+++ set +e
+++ kubectl exec -n monitoring-2-0-16090 monitoring-0 -- pmm-admin --server-url=https://admin:admin@monitoring-service.monitoring-service-rs0/ --server-insecure-tls inventory list nodes --node-type=CONTAINER_NODE
+++ exit_status=1
+++ set -e
+++ '[' 1 '!=' 0 -a -n 1 ']'
+++ cat /tmp/tmp.2eLEjg54gv
+++ cat /tmp/tmp.3o9c4JKbCF
command terminated with exit code 1
+++ sleep 0
+++ for i in '$(seq 0 2)'
+++ set +e
+++ kubectl exec -n monitoring-2-0-16090 monitoring-0 -- pmm-admin --server-url=https://admin:admin@monitoring-service.monitoring-service-rs0/ --server-insecure-tls inventory list nodes --node-type=CONTAINER_NODE
+++ exit_status=1
+++ set -e
+++ '[' 1 '!=' 0 -a -n 1 ']'
+++ cat /tmp/tmp.2eLEjg54gv
+++ cat /tmp/tmp.3o9c4JKbCF
command terminated with exit code 1
+++ sleep 4
+++ for i in '$(seq 0 2)'
+++ set +e
+++ kubectl exec -n monitoring-2-0-16090 monitoring-0 -- pmm-admin --server-url=https://admin:admin@monitoring-service.monitoring-service-rs0/ --server-insecure-tls inventory list nodes --node-type=CONTAINER_NODE
+++ exit_status=1
+++ set -e
+++ '[' 1 '!=' 0 -a -n 1 ']'
+++ cat /tmp/tmp.2eLEjg54gv
+++ cat /tmp/tmp.3o9c4JKbCF
command terminated with exit code 1
+++ sleep 8
+++ cat /tmp/tmp.2eLEjg54gv
+++ cat /tmp/tmp.3o9c4JKbCF
command terminated with exit code 1
+++ rm /tmp/tmp.2eLEjg54gv /tmp/tmp.3o9c4JKbCF
+++ return 1
++ for node_id in '"${nodeList[@]}"'
++ nodeList_from_pmm+=($(kubectl_bin exec -n "${namespace}" monitoring-0 -- pmm-admin --server-url=https://admin:admin@$(get_service_ip monitoring-service)/ --server-insecure-tls inventory list nodes --node-type=CONTAINER_NODE | grep $node_id | awk '{print $4}'))
+++ awk '{print $4}'
+++ grep /node_id/efac200c-f35c-407e-815d-3728aea09fbd
++++ get_service_ip monitoring-service
++++ local service=monitoring-service
++++ local server_type=rs0
+++++ kubectl_bin get psmdb/monitoring-service -o 'jsonpath={.spec.replsets[].expose.enabled}'
++++++ mktemp
+++++ local LAST_OUT=/tmp/tmp.VcCTCUGk3K
++++++ mktemp
+++++ local LAST_ERR=/tmp/tmp.lJLkJBIAWr
+++++ local exit_status=0
+++++ local timeout=4
++++++ seq 0 2
+++++ for i in '$(seq 0 2)'
+++++ set +e
+++++ kubectl get psmdb/monitoring-service -o 'jsonpath={.spec.replsets[].expose.enabled}'
+++++ exit_status=1
+++++ set -e
+++++ '[' 1 '!=' 0 -a -n 1 ']'
+++++ cat /tmp/tmp.VcCTCUGk3K
+++++ cat /tmp/tmp.lJLkJBIAWr
Error from server (NotFound): perconaservermongodbs.psmdb.percona.com "monitoring-service" not found
+++++ sleep 0
+++++ for i in '$(seq 0 2)'
+++++ set +e
+++++ kubectl get psmdb/monitoring-service -o 'jsonpath={.spec.replsets[].expose.enabled}'
+++++ exit_status=1
+++++ set -e
+++++ '[' 1 '!=' 0 -a -n 1 ']'
+++++ cat /tmp/tmp.VcCTCUGk3K
+++++ cat /tmp/tmp.lJLkJBIAWr
Error from server (NotFound): perconaservermongodbs.psmdb.percona.com "monitoring-service" not found
+++++ sleep 4
+++++ for i in '$(seq 0 2)'
+++++ set +e
+++++ kubectl get psmdb/monitoring-service -o 'jsonpath={.spec.replsets[].expose.enabled}'
+++++ exit_status=1
+++++ set -e
+++++ '[' 1 '!=' 0 -a -n 1 ']'
+++++ cat /tmp/tmp.VcCTCUGk3K
+++++ cat /tmp/tmp.lJLkJBIAWr
Error from server (NotFound): perconaservermongodbs.psmdb.percona.com "monitoring-service" not found
+++++ sleep 8
+++++ cat /tmp/tmp.VcCTCUGk3K
+++++ cat /tmp/tmp.lJLkJBIAWr
Error from server (NotFound): perconaservermongodbs.psmdb.percona.com "monitoring-service" not found
+++++ rm /tmp/tmp.VcCTCUGk3K /tmp/tmp.lJLkJBIAWr
+++++ return 1
++++ '[' '' '!=' true ']'
++++ echo -n monitoring-service.monitoring-service-rs0
++++ return
+++ kubectl_bin exec -n monitoring-2-0-16090 monitoring-0 -- pmm-admin --server-url=https://admin:admin@monitoring-service.monitoring-service-rs0/ --server-insecure-tls inventory list nodes --node-type=CONTAINER_NODE
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.sOgXedNBmw
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.mr92oOxkLy
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in '$(seq 0 2)'
+++ set +e
+++ kubectl exec -n monitoring-2-0-16090 monitoring-0 -- pmm-admin --server-url=https://admin:admin@monitoring-service.monitoring-service-rs0/ --server-insecure-tls inventory list nodes --node-type=CONTAINER_NODE
+++ exit_status=1
+++ set -e
+++ '[' 1 '!=' 0 -a -n 1 ']'
+++ cat /tmp/tmp.sOgXedNBmw
+++ cat /tmp/tmp.mr92oOxkLy
command terminated with exit code 1
+++ sleep 0
+++ for i in '$(seq 0 2)'
+++ set +e
+++ kubectl exec -n monitoring-2-0-16090 monitoring-0 -- pmm-admin --server-url=https://admin:admin@monitoring-service.monitoring-service-rs0/ --server-insecure-tls inventory list nodes --node-type=CONTAINER_NODE
+++ exit_status=1
+++ set -e
+++ '[' 1 '!=' 0 -a -n 1 ']'
+++ cat /tmp/tmp.sOgXedNBmw
+++ cat /tmp/tmp.mr92oOxkLy
command terminated with exit code 1
+++ sleep 4
+++ for i in '$(seq 0 2)'
+++ set +e
+++ kubectl exec -n monitoring-2-0-16090 monitoring-0 -- pmm-admin --server-url=https://admin:admin@monitoring-service.monitoring-service-rs0/ --server-insecure-tls inventory list nodes --node-type=CONTAINER_NODE
+++ exit_status=1
+++ set -e
+++ '[' 1 '!=' 0 -a -n 1 ']'
+++ cat /tmp/tmp.sOgXedNBmw
+++ cat /tmp/tmp.mr92oOxkLy
command terminated with exit code 1
+++ sleep 8
+++ cat /tmp/tmp.sOgXedNBmw
+++ cat /tmp/tmp.mr92oOxkLy
command terminated with exit code 1
+++ rm /tmp/tmp.sOgXedNBmw /tmp/tmp.mr92oOxkLy
+++ return 1
++ for node_id in '"${nodeList[@]}"'
++ nodeList_from_pmm+=($(kubectl_bin exec -n "${namespace}" monitoring-0 -- pmm-admin --server-url=https://admin:admin@$(get_service_ip monitoring-service)/ --server-insecure-tls inventory list nodes --node-type=CONTAINER_NODE | grep $node_id | awk '{print $4}'))
+++ grep /node_id/6eb9ff8f-5f1f-433b-a759-ff8ca749f427
++++ get_service_ip monitoring-service
++++ local service=monitoring-service
++++ local server_type=rs0
+++ awk '{print $4}'
+++++ kubectl_bin get psmdb/monitoring-service -o 'jsonpath={.spec.replsets[].expose.enabled}'
++++++ mktemp
+++++ local LAST_OUT=/tmp/tmp.Jdnt0Pc59J
++++++ mktemp
+++++ local LAST_ERR=/tmp/tmp.fpDSzaqyAI
+++++ local exit_status=0
+++++ local timeout=4
++++++ seq 0 2
+++++ for i in '$(seq 0 2)'
+++++ set +e
+++++ kubectl get psmdb/monitoring-service -o 'jsonpath={.spec.replsets[].expose.enabled}'
+++++ exit_status=1
+++++ set -e
+++++ '[' 1 '!=' 0 -a -n 1 ']'
+++++ cat /tmp/tmp.Jdnt0Pc59J
+++++ cat /tmp/tmp.fpDSzaqyAI
Error from server (NotFound): perconaservermongodbs.psmdb.percona.com "monitoring-service" not found
+++++ sleep 0
+++++ for i in '$(seq 0 2)'
+++++ set +e
+++++ kubectl get psmdb/monitoring-service -o 'jsonpath={.spec.replsets[].expose.enabled}'
+++++ exit_status=1
+++++ set -e
+++++ '[' 1 '!=' 0 -a -n 1 ']'
+++++ cat /tmp/tmp.Jdnt0Pc59J
+++++ cat /tmp/tmp.fpDSzaqyAI
Error from server (NotFound): perconaservermongodbs.psmdb.percona.com "monitoring-service" not found
+++++ sleep 4
+++++ for i in '$(seq 0 2)'
+++++ set +e
+++++ kubectl get psmdb/monitoring-service -o 'jsonpath={.spec.replsets[].expose.enabled}'
+++++ exit_status=1
+++++ set -e
+++++ '[' 1 '!=' 0 -a -n 1 ']'
+++++ cat /tmp/tmp.Jdnt0Pc59J
+++++ cat /tmp/tmp.fpDSzaqyAI
Error from server (NotFound): perconaservermongodbs.psmdb.percona.com "monitoring-service" not found
+++++ sleep 8
+++++ cat /tmp/tmp.Jdnt0Pc59J
+++++ cat /tmp/tmp.fpDSzaqyAI
Error from server (NotFound): perconaservermongodbs.psmdb.percona.com "monitoring-service" not found
+++++ rm /tmp/tmp.Jdnt0Pc59J /tmp/tmp.fpDSzaqyAI
+++++ return 1
++++ '[' '' '!=' true ']'
++++ echo -n monitoring-service.monitoring-service-rs0
++++ return
+++ kubectl_bin exec -n monitoring-2-0-16090 monitoring-0 -- pmm-admin --server-url=https://admin:admin@monitoring-service.monitoring-service-rs0/ --server-insecure-tls inventory list nodes --node-type=CONTAINER_NODE
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.3TaOV293kM
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.04b9aRHrg0
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in '$(seq 0 2)'
+++ set +e
+++ kubectl exec -n monitoring-2-0-16090 monitoring-0 -- pmm-admin --server-url=https://admin:admin@monitoring-service.monitoring-service-rs0/ --server-insecure-tls inventory list nodes --node-type=CONTAINER_NODE
+++ exit_status=1
+++ set -e
+++ '[' 1 '!=' 0 -a -n 1 ']'
+++ cat /tmp/tmp.3TaOV293kM
+++ cat /tmp/tmp.04b9aRHrg0
command terminated with exit code 1
+++ sleep 0
+++ for i in '$(seq 0 2)'
+++ set +e
+++ kubectl exec -n monitoring-2-0-16090 monitoring-0 -- pmm-admin --server-url=https://admin:admin@monitoring-service.monitoring-service-rs0/ --server-insecure-tls inventory list nodes --node-type=CONTAINER_NODE
+++ exit_status=1
+++ set -e
+++ '[' 1 '!=' 0 -a -n 1 ']'
+++ cat /tmp/tmp.3TaOV293kM
+++ cat /tmp/tmp.04b9aRHrg0
command terminated with exit code 1
+++ sleep 4
+++ for i in '$(seq 0 2)'
+++ set +e
+++ kubectl exec -n monitoring-2-0-16090 monitoring-0 -- pmm-admin --server-url=https://admin:admin@monitoring-service.monitoring-service-rs0/ --server-insecure-tls inventory list nodes --node-type=CONTAINER_NODE
+++ exit_status=1
+++ set -e
+++ '[' 1 '!=' 0 -a -n 1 ']'
+++ cat /tmp/tmp.3TaOV293kM
+++ cat /tmp/tmp.04b9aRHrg0
command terminated with exit code 1
+++ sleep 8
+++ cat /tmp/tmp.3TaOV293kM
+++ cat /tmp/tmp.04b9aRHrg0
command terminated with exit code 1
+++ rm /tmp/tmp.3TaOV293kM /tmp/tmp.04b9aRHrg0
+++ return 1
++ for node_id in '"${nodeList[@]}"'
++ nodeList_from_pmm+=($(kubectl_bin exec -n "${namespace}" monitoring-0 -- pmm-admin --server-url=https://admin:admin@$(get_service_ip monitoring-service)/ --server-insecure-tls inventory list nodes --node-type=CONTAINER_NODE | grep $node_id | awk '{print $4}'))
++++ get_service_ip monitoring-service
+++ grep /node_id/54bc59c3-da6b-42c2-bc81-63bf5c67a979
++++ local service=monitoring-service
++++ local server_type=rs0
+++++ kubectl_bin get psmdb/monitoring-service -o 'jsonpath={.spec.replsets[].expose.enabled}'
+++ awk '{print $4}'
++++++ mktemp
+++++ local LAST_OUT=/tmp/tmp.q4vO8urjjh
++++++ mktemp
+++++ local LAST_ERR=/tmp/tmp.TigmWlUljE
+++++ local exit_status=0
+++++ local timeout=4
++++++ seq 0 2
+++++ for i in '$(seq 0 2)'
+++++ set +e
+++++ kubectl get psmdb/monitoring-service -o 'jsonpath={.spec.replsets[].expose.enabled}'
+++++ exit_status=1
+++++ set -e
+++++ '[' 1 '!=' 0 -a -n 1 ']'
+++++ cat /tmp/tmp.q4vO8urjjh
+++++ cat /tmp/tmp.TigmWlUljE
Error from server (NotFound): perconaservermongodbs.psmdb.percona.com "monitoring-service" not found
+++++ sleep 0
+++++ for i in '$(seq 0 2)'
+++++ set +e
+++++ kubectl get psmdb/monitoring-service -o 'jsonpath={.spec.replsets[].expose.enabled}'
+++++ exit_status=1
+++++ set -e
+++++ '[' 1 '!=' 0 -a -n 1 ']'
+++++ cat /tmp/tmp.q4vO8urjjh
+++++ cat /tmp/tmp.TigmWlUljE
Error from server (NotFound): perconaservermongodbs.psmdb.percona.com "monitoring-service" not found
+++++ sleep 4
+++++ for i in '$(seq 0 2)'
+++++ set +e
+++++ kubectl get psmdb/monitoring-service -o 'jsonpath={.spec.replsets[].expose.enabled}'
+++++ exit_status=1
+++++ set -e
+++++ '[' 1 '!=' 0 -a -n 1 ']'
+++++ cat /tmp/tmp.q4vO8urjjh
+++++ cat /tmp/tmp.TigmWlUljE
Error from server (NotFound): perconaservermongodbs.psmdb.percona.com "monitoring-service" not found
+++++ sleep 8
+++++ cat /tmp/tmp.q4vO8urjjh
+++++ cat /tmp/tmp.TigmWlUljE
Error from server (NotFound): perconaservermongodbs.psmdb.percona.com "monitoring-service" not found
+++++ rm /tmp/tmp.q4vO8urjjh /tmp/tmp.TigmWlUljE
+++++ return 1
++++ '[' '' '!=' true ']'
++++ echo -n monitoring-service.monitoring-service-rs0
++++ return
+++ kubectl_bin exec -n monitoring-2-0-16090 monitoring-0 -- pmm-admin --server-url=https://admin:admin@monitoring-service.monitoring-service-rs0/ --server-insecure-tls inventory list nodes --node-type=CONTAINER_NODE
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.0kMwDRIvyq
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.CVJgxaA9kP
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in '$(seq 0 2)'
+++ set +e
+++ kubectl exec -n monitoring-2-0-16090 monitoring-0 -- pmm-admin --server-url=https://admin:admin@monitoring-service.monitoring-service-rs0/ --server-insecure-tls inventory list nodes --node-type=CONTAINER_NODE
+++ exit_status=1
+++ set -e
+++ '[' 1 '!=' 0 -a -n 1 ']'
+++ cat /tmp/tmp.0kMwDRIvyq
+++ cat /tmp/tmp.CVJgxaA9kP
command terminated with exit code 1
+++ sleep 0
+++ for i in '$(seq 0 2)'
+++ set +e
+++ kubectl exec -n monitoring-2-0-16090 monitoring-0 -- pmm-admin --server-url=https://admin:admin@monitoring-service.monitoring-service-rs0/ --server-insecure-tls inventory list nodes --node-type=CONTAINER_NODE
+++ exit_status=1
+++ set -e
+++ '[' 1 '!=' 0 -a -n 1 ']'
+++ cat /tmp/tmp.0kMwDRIvyq
+++ cat /tmp/tmp.CVJgxaA9kP
command terminated with exit code 1
+++ sleep 4
+++ for i in '$(seq 0 2)'
+++ set +e
+++ kubectl exec -n monitoring-2-0-16090 monitoring-0 -- pmm-admin --server-url=https://admin:admin@monitoring-service.monitoring-service-rs0/ --server-insecure-tls inventory list nodes --node-type=CONTAINER_NODE
+++ exit_status=1
+++ set -e
+++ '[' 1 '!=' 0 -a -n 1 ']'
+++ cat /tmp/tmp.0kMwDRIvyq
+++ cat /tmp/tmp.CVJgxaA9kP
command terminated with exit code 1
+++ sleep 8
+++ cat /tmp/tmp.0kMwDRIvyq
+++ cat /tmp/tmp.CVJgxaA9kP
command terminated with exit code 1
+++ rm /tmp/tmp.0kMwDRIvyq /tmp/tmp.CVJgxaA9kP
+++ return 1
++ for node_id in '"${nodeList[@]}"'
++ nodeList_from_pmm+=($(kubectl_bin exec -n "${namespace}" monitoring-0 -- pmm-admin --server-url=https://admin:admin@$(get_service_ip monitoring-service)/ --server-insecure-tls inventory list nodes --node-type=CONTAINER_NODE | grep $node_id | awk '{print $4}'))
+++ awk '{print $4}'
+++ grep /node_id/ebeaa667-fe43-4326-ba0e-e6e852c734ce
++++ get_service_ip monitoring-service
++++ local service=monitoring-service
++++ local server_type=rs0
+++++ kubectl_bin get psmdb/monitoring-service -o 'jsonpath={.spec.replsets[].expose.enabled}'
++++++ mktemp
+++++ local LAST_OUT=/tmp/tmp.heAeSDb8dZ
++++++ mktemp
+++++ local LAST_ERR=/tmp/tmp.ybthgWhg6S
+++++ local exit_status=0
+++++ local timeout=4
++++++ seq 0 2
+++++ for i in '$(seq 0 2)'
+++++ set +e
+++++ kubectl get psmdb/monitoring-service -o 'jsonpath={.spec.replsets[].expose.enabled}'
+++++ exit_status=1
+++++ set -e
+++++ '[' 1 '!=' 0 -a -n 1 ']'
+++++ cat /tmp/tmp.heAeSDb8dZ
+++++ cat /tmp/tmp.ybthgWhg6S
Error from server (NotFound): perconaservermongodbs.psmdb.percona.com "monitoring-service" not found
+++++ sleep 0
+++++ for i in '$(seq 0 2)'
+++++ set +e
+++++ kubectl get psmdb/monitoring-service -o 'jsonpath={.spec.replsets[].expose.enabled}'
+++++ exit_status=1
+++++ set -e
+++++ '[' 1 '!=' 0 -a -n 1 ']'
+++++ cat /tmp/tmp.heAeSDb8dZ
+++++ cat /tmp/tmp.ybthgWhg6S
Error from server (NotFound): perconaservermongodbs.psmdb.percona.com "monitoring-service" not found
+++++ sleep 4
+++++ for i in '$(seq 0 2)'
+++++ set +e
+++++ kubectl get psmdb/monitoring-service -o 'jsonpath={.spec.replsets[].expose.enabled}'
+++++ exit_status=1
+++++ set -e
+++++ '[' 1 '!=' 0 -a -n 1 ']'
+++++ cat /tmp/tmp.heAeSDb8dZ
+++++ cat /tmp/tmp.ybthgWhg6S
Error from server (NotFound): perconaservermongodbs.psmdb.percona.com "monitoring-service" not found
+++++ sleep 8
+++++ cat /tmp/tmp.heAeSDb8dZ
+++++ cat /tmp/tmp.ybthgWhg6S
Error from server (NotFound): perconaservermongodbs.psmdb.percona.com "monitoring-service" not found
+++++ rm /tmp/tmp.heAeSDb8dZ /tmp/tmp.ybthgWhg6S
+++++ return 1
++++ '[' '' '!=' true ']'
++++ echo -n monitoring-service.monitoring-service-rs0
++++ return
+++ kubectl_bin exec -n monitoring-2-0-16090 monitoring-0 -- pmm-admin --server-url=https://admin:admin@monitoring-service.monitoring-service-rs0/ --server-insecure-tls inventory list nodes --node-type=CONTAINER_NODE
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.3dwVstQGN3
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.zZ4rH79hrH
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in '$(seq 0 2)'
+++ set +e
+++ kubectl exec -n monitoring-2-0-16090 monitoring-0 -- pmm-admin --server-url=https://admin:admin@monitoring-service.monitoring-service-rs0/ --server-insecure-tls inventory list nodes --node-type=CONTAINER_NODE
+++ exit_status=1
+++ set -e
+++ '[' 1 '!=' 0 -a -n 1 ']'
+++ cat /tmp/tmp.3dwVstQGN3
+++ cat /tmp/tmp.zZ4rH79hrH
command terminated with exit code 1
+++ sleep 0
+++ for i in '$(seq 0 2)'
+++ set +e
+++ kubectl exec -n monitoring-2-0-16090 monitoring-0 -- pmm-admin --server-url=https://admin:admin@monitoring-service.monitoring-service-rs0/ --server-insecure-tls inventory list nodes --node-type=CONTAINER_NODE
+++ exit_status=1
+++ set -e
+++ '[' 1 '!=' 0 -a -n 1 ']'
+++ cat /tmp/tmp.3dwVstQGN3
+++ cat /tmp/tmp.zZ4rH79hrH
command terminated with exit code 1
+++ sleep 4
+++ for i in '$(seq 0 2)'
+++ set +e
+++ kubectl exec -n monitoring-2-0-16090 monitoring-0 -- pmm-admin --server-url=https://admin:admin@monitoring-service.monitoring-service-rs0/ --server-insecure-tls inventory list nodes --node-type=CONTAINER_NODE
+++ exit_status=1
+++ set -e
+++ '[' 1 '!=' 0 -a -n 1 ']'
+++ cat /tmp/tmp.3dwVstQGN3
+++ cat /tmp/tmp.zZ4rH79hrH
command terminated with exit code 1
+++ sleep 8
+++ cat /tmp/tmp.3dwVstQGN3
+++ cat /tmp/tmp.zZ4rH79hrH
command terminated with exit code 1
+++ rm /tmp/tmp.3dwVstQGN3 /tmp/tmp.zZ4rH79hrH
+++ return 1
++ for node_id in '"${nodeList[@]}"'
++ nodeList_from_pmm+=($(kubectl_bin exec -n "${namespace}" monitoring-0 -- pmm-admin --server-url=https://admin:admin@$(get_service_ip monitoring-service)/ --server-insecure-tls inventory list nodes --node-type=CONTAINER_NODE | grep $node_id | awk '{print $4}'))
+++ awk '{print $4}'
++++ get_service_ip monitoring-service
++++ local service=monitoring-service
++++ local server_type=rs0
+++ grep /node_id/0af1fe4a-8b09-452c-82a1-ed469e898790
+++++ kubectl_bin get psmdb/monitoring-service -o 'jsonpath={.spec.replsets[].expose.enabled}'
++++++ mktemp
+++++ local LAST_OUT=/tmp/tmp.NxHegPOD2B
++++++ mktemp
+++++ local LAST_ERR=/tmp/tmp.ybbw0VCAfL
+++++ local exit_status=0
+++++ local timeout=4
++++++ seq 0 2
+++++ for i in '$(seq 0 2)'
+++++ set +e
+++++ kubectl get psmdb/monitoring-service -o 'jsonpath={.spec.replsets[].expose.enabled}'
+++++ exit_status=1
+++++ set -e
+++++ '[' 1 '!=' 0 -a -n 1 ']'
+++++ cat /tmp/tmp.NxHegPOD2B
+++++ cat /tmp/tmp.ybbw0VCAfL
Error from server (NotFound): perconaservermongodbs.psmdb.percona.com "monitoring-service" not found
+++++ sleep 0
+++++ for i in '$(seq 0 2)'
+++++ set +e
+++++ kubectl get psmdb/monitoring-service -o 'jsonpath={.spec.replsets[].expose.enabled}'
+++++ exit_status=1
+++++ set -e
+++++ '[' 1 '!=' 0 -a -n 1 ']'
+++++ cat /tmp/tmp.NxHegPOD2B
+++++ cat /tmp/tmp.ybbw0VCAfL
Error from server (NotFound): perconaservermongodbs.psmdb.percona.com "monitoring-service" not found
+++++ sleep 4
+++++ for i in '$(seq 0 2)'
+++++ set +e
+++++ kubectl get psmdb/monitoring-service -o 'jsonpath={.spec.replsets[].expose.enabled}'
+++++ exit_status=1
+++++ set -e
+++++ '[' 1 '!=' 0 -a -n 1 ']'
+++++ cat /tmp/tmp.NxHegPOD2B
+++++ cat /tmp/tmp.ybbw0VCAfL
Error from server (NotFound): perconaservermongodbs.psmdb.percona.com "monitoring-service" not found
+++++ sleep 8
+++++ cat /tmp/tmp.NxHegPOD2B
+++++ cat /tmp/tmp.ybbw0VCAfL
Error from server (NotFound): perconaservermongodbs.psmdb.percona.com "monitoring-service" not found
+++++ rm /tmp/tmp.NxHegPOD2B /tmp/tmp.ybbw0VCAfL
+++++ return 1
++++ '[' '' '!=' true ']'
++++ echo -n monitoring-service.monitoring-service-rs0
++++ return
+++ kubectl_bin exec -n monitoring-2-0-16090 monitoring-0 -- pmm-admin --server-url=https://admin:admin@monitoring-service.monitoring-service-rs0/ --server-insecure-tls inventory list nodes --node-type=CONTAINER_NODE
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.cl25Dzzt2O
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.VeRSOar0oJ
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in '$(seq 0 2)'
+++ set +e
+++ kubectl exec -n monitoring-2-0-16090 monitoring-0 -- pmm-admin --server-url=https://admin:admin@monitoring-service.monitoring-service-rs0/ --server-insecure-tls inventory list nodes --node-type=CONTAINER_NODE
+++ exit_status=1
+++ set -e
+++ '[' 1 '!=' 0 -a -n 1 ']'
+++ cat /tmp/tmp.cl25Dzzt2O
+++ cat /tmp/tmp.VeRSOar0oJ
command terminated with exit code 1
+++ sleep 0
+++ for i in '$(seq 0 2)'
+++ set +e
+++ kubectl exec -n monitoring-2-0-16090 monitoring-0 -- pmm-admin --server-url=https://admin:admin@monitoring-service.monitoring-service-rs0/ --server-insecure-tls inventory list nodes --node-type=CONTAINER_NODE
+++ exit_status=1
+++ set -e
+++ '[' 1 '!=' 0 -a -n 1 ']'
+++ cat /tmp/tmp.cl25Dzzt2O
+++ cat /tmp/tmp.VeRSOar0oJ
command terminated with exit code 1
+++ sleep 4
+++ for i in '$(seq 0 2)'
+++ set +e
+++ kubectl exec -n monitoring-2-0-16090 monitoring-0 -- pmm-admin --server-url=https://admin:admin@monitoring-service.monitoring-service-rs0/ --server-insecure-tls inventory list nodes --node-type=CONTAINER_NODE
+++ exit_status=1
+++ set -e
+++ '[' 1 '!=' 0 -a -n 1 ']'
+++ cat /tmp/tmp.cl25Dzzt2O
+++ cat /tmp/tmp.VeRSOar0oJ
command terminated with exit code 1
+++ sleep 8
+++ cat /tmp/tmp.cl25Dzzt2O
+++ cat /tmp/tmp.VeRSOar0oJ
command terminated with exit code 1
+++ rm /tmp/tmp.cl25Dzzt2O /tmp/tmp.VeRSOar0oJ
+++ return 1
++ for node_id in '"${nodeList[@]}"'
++ nodeList_from_pmm+=($(kubectl_bin exec -n "${namespace}" monitoring-0 -- pmm-admin --server-url=https://admin:admin@$(get_service_ip monitoring-service)/ --server-insecure-tls inventory list nodes --node-type=CONTAINER_NODE | grep $node_id | awk '{print $4}'))
+++ grep /node_id/c03612ee-2d8b-4228-b363-041e3ae52a53
+++ awk '{print $4}'
++++ get_service_ip monitoring-service
++++ local service=monitoring-service
++++ local server_type=rs0
+++++ kubectl_bin get psmdb/monitoring-service -o 'jsonpath={.spec.replsets[].expose.enabled}'
++++++ mktemp
+++++ local LAST_OUT=/tmp/tmp.MI3l54IkLT
++++++ mktemp
+++++ local LAST_ERR=/tmp/tmp.Q1LJ3uvZLK
+++++ local exit_status=0
+++++ local timeout=4
++++++ seq 0 2
+++++ for i in '$(seq 0 2)'
+++++ set +e
+++++ kubectl get psmdb/monitoring-service -o 'jsonpath={.spec.replsets[].expose.enabled}'
+++++ exit_status=1
+++++ set -e
+++++ '[' 1 '!=' 0 -a -n 1 ']'
+++++ cat /tmp/tmp.MI3l54IkLT
+++++ cat /tmp/tmp.Q1LJ3uvZLK
Error from server (NotFound): perconaservermongodbs.psmdb.percona.com "monitoring-service" not found
+++++ sleep 0
+++++ for i in '$(seq 0 2)'
+++++ set +e
+++++ kubectl get psmdb/monitoring-service -o 'jsonpath={.spec.replsets[].expose.enabled}'
+++++ exit_status=1
+++++ set -e
+++++ '[' 1 '!=' 0 -a -n 1 ']'
+++++ cat /tmp/tmp.MI3l54IkLT
+++++ cat /tmp/tmp.Q1LJ3uvZLK
Error from server (NotFound): perconaservermongodbs.psmdb.percona.com "monitoring-service" not found
+++++ sleep 4
+++++ for i in '$(seq 0 2)'
+++++ set +e
+++++ kubectl get psmdb/monitoring-service -o 'jsonpath={.spec.replsets[].expose.enabled}'
+++++ exit_status=1
+++++ set -e
+++++ '[' 1 '!=' 0 -a -n 1 ']'
+++++ cat /tmp/tmp.MI3l54IkLT
+++++ cat /tmp/tmp.Q1LJ3uvZLK
Error from server (NotFound): perconaservermongodbs.psmdb.percona.com "monitoring-service" not found
+++++ sleep 8
+++++ cat /tmp/tmp.MI3l54IkLT
+++++ cat /tmp/tmp.Q1LJ3uvZLK
Error from server (NotFound): perconaservermongodbs.psmdb.percona.com "monitoring-service" not found
+++++ rm /tmp/tmp.MI3l54IkLT /tmp/tmp.Q1LJ3uvZLK
+++++ return 1
++++ '[' '' '!=' true ']'
++++ echo -n monitoring-service.monitoring-service-rs0
++++ return
+++ kubectl_bin exec -n monitoring-2-0-16090 monitoring-0 -- pmm-admin --server-url=https://admin:admin@monitoring-service.monitoring-service-rs0/ --server-insecure-tls inventory list nodes --node-type=CONTAINER_NODE
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.TCSej6QMiS
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.72Nbgj0Uyj
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in '$(seq 0 2)'
+++ set +e
+++ kubectl exec -n monitoring-2-0-16090 monitoring-0 -- pmm-admin --server-url=https://admin:admin@monitoring-service.monitoring-service-rs0/ --server-insecure-tls inventory list nodes --node-type=CONTAINER_NODE
+++ exit_status=1
+++ set -e
+++ '[' 1 '!=' 0 -a -n 1 ']'
+++ cat /tmp/tmp.TCSej6QMiS
+++ cat /tmp/tmp.72Nbgj0Uyj
command terminated with exit code 1
+++ sleep 0
+++ for i in '$(seq 0 2)'
+++ set +e
+++ kubectl exec -n monitoring-2-0-16090 monitoring-0 -- pmm-admin --server-url=https://admin:admin@monitoring-service.monitoring-service-rs0/ --server-insecure-tls inventory list nodes --node-type=CONTAINER_NODE
+++ exit_status=1
+++ set -e
+++ '[' 1 '!=' 0 -a -n 1 ']'
+++ cat /tmp/tmp.TCSej6QMiS
+++ cat /tmp/tmp.72Nbgj0Uyj
command terminated with exit code 1
+++ sleep 4
+++ for i in '$(seq 0 2)'
+++ set +e
+++ kubectl exec -n monitoring-2-0-16090 monitoring-0 -- pmm-admin --server-url=https://admin:admin@monitoring-service.monitoring-service-rs0/ --server-insecure-tls inventory list nodes --node-type=CONTAINER_NODE
+++ exit_status=1
+++ set -e
+++ '[' 1 '!=' 0 -a -n 1 ']'
+++ cat /tmp/tmp.TCSej6QMiS
+++ cat /tmp/tmp.72Nbgj0Uyj
command terminated with exit code 1
+++ sleep 8
+++ cat /tmp/tmp.TCSej6QMiS
+++ cat /tmp/tmp.72Nbgj0Uyj
command terminated with exit code 1
+++ rm /tmp/tmp.TCSej6QMiS /tmp/tmp.72Nbgj0Uyj
+++ return 1
++ for node_id in '"${nodeList[@]}"'
++ nodeList_from_pmm+=($(kubectl_bin exec -n "${namespace}" monitoring-0 -- pmm-admin --server-url=https://admin:admin@$(get_service_ip monitoring-service)/ --server-insecure-tls inventory list nodes --node-type=CONTAINER_NODE | grep $node_id | awk '{print $4}'))
+++ grep /node_id/fcb0e3b7-0883-4a6b-bd4e-f093cba949d9
+++ awk '{print $4}'
++++ get_service_ip monitoring-service
++++ local service=monitoring-service
++++ local server_type=rs0
+++++ kubectl_bin get psmdb/monitoring-service -o 'jsonpath={.spec.replsets[].expose.enabled}'
++++++ mktemp
+++++ local LAST_OUT=/tmp/tmp.pMTCKiGpae
++++++ mktemp
+++++ local LAST_ERR=/tmp/tmp.VRVqLSrzpa
+++++ local exit_status=0
+++++ local timeout=4
++++++ seq 0 2
+++++ for i in '$(seq 0 2)'
+++++ set +e
+++++ kubectl get psmdb/monitoring-service -o 'jsonpath={.spec.replsets[].expose.enabled}'
+++++ exit_status=1
+++++ set -e
+++++ '[' 1 '!=' 0 -a -n 1 ']'
+++++ cat /tmp/tmp.pMTCKiGpae
+++++ cat /tmp/tmp.VRVqLSrzpa
Error from server (NotFound): perconaservermongodbs.psmdb.percona.com "monitoring-service" not found
+++++ sleep 0
+++++ for i in '$(seq 0 2)'
+++++ set +e
+++++ kubectl get psmdb/monitoring-service -o 'jsonpath={.spec.replsets[].expose.enabled}'
+++++ exit_status=1
+++++ set -e
+++++ '[' 1 '!=' 0 -a -n 1 ']'
+++++ cat /tmp/tmp.pMTCKiGpae
+++++ cat /tmp/tmp.VRVqLSrzpa
Error from server (NotFound): perconaservermongodbs.psmdb.percona.com "monitoring-service" not found
+++++ sleep 4
+++++ for i in '$(seq 0 2)'
+++++ set +e
+++++ kubectl get psmdb/monitoring-service -o 'jsonpath={.spec.replsets[].expose.enabled}'
+++++ exit_status=1
+++++ set -e
+++++ '[' 1 '!=' 0 -a -n 1 ']'
+++++ cat /tmp/tmp.pMTCKiGpae
+++++ cat /tmp/tmp.VRVqLSrzpa
Error from server (NotFound): perconaservermongodbs.psmdb.percona.com "monitoring-service" not found
+++++ sleep 8
+++++ cat /tmp/tmp.pMTCKiGpae
+++++ cat /tmp/tmp.VRVqLSrzpa
Error from server (NotFound): perconaservermongodbs.psmdb.percona.com "monitoring-service" not found
+++++ rm /tmp/tmp.pMTCKiGpae /tmp/tmp.VRVqLSrzpa
+++++ return 1
++++ '[' '' '!=' true ']'
++++ echo -n monitoring-service.monitoring-service-rs0
++++ return
+++ kubectl_bin exec -n monitoring-2-0-16090 monitoring-0 -- pmm-admin --server-url=https://admin:admin@monitoring-service.monitoring-service-rs0/ --server-insecure-tls inventory list nodes --node-type=CONTAINER_NODE
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.V8qEd6nAaa
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.IdsoVRn9uB
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in '$(seq 0 2)'
+++ set +e
+++ kubectl exec -n monitoring-2-0-16090 monitoring-0 -- pmm-admin --server-url=https://admin:admin@monitoring-service.monitoring-service-rs0/ --server-insecure-tls inventory list nodes --node-type=CONTAINER_NODE
+++ exit_status=1
+++ set -e
+++ '[' 1 '!=' 0 -a -n 1 ']'
+++ cat /tmp/tmp.V8qEd6nAaa
+++ cat /tmp/tmp.IdsoVRn9uB
command terminated with exit code 1
+++ sleep 0
+++ for i in '$(seq 0 2)'
+++ set +e
+++ kubectl exec -n monitoring-2-0-16090 monitoring-0 -- pmm-admin --server-url=https://admin:admin@monitoring-service.monitoring-service-rs0/ --server-insecure-tls inventory list nodes --node-type=CONTAINER_NODE
+++ exit_status=1
+++ set -e
+++ '[' 1 '!=' 0 -a -n 1 ']'
+++ cat /tmp/tmp.V8qEd6nAaa
+++ cat /tmp/tmp.IdsoVRn9uB
command terminated with exit code 1
+++ sleep 4
+++ for i in '$(seq 0 2)'
+++ set +e
+++ kubectl exec -n monitoring-2-0-16090 monitoring-0 -- pmm-admin --server-url=https://admin:admin@monitoring-service.monitoring-service-rs0/ --server-insecure-tls inventory list nodes --node-type=CONTAINER_NODE
+++ exit_status=1
+++ set -e
+++ '[' 1 '!=' 0 -a -n 1 ']'
+++ cat /tmp/tmp.V8qEd6nAaa
+++ cat /tmp/tmp.IdsoVRn9uB
command terminated with exit code 1
+++ sleep 8
+++ cat /tmp/tmp.V8qEd6nAaa
+++ cat /tmp/tmp.IdsoVRn9uB
command terminated with exit code 1
+++ rm /tmp/tmp.V8qEd6nAaa /tmp/tmp.IdsoVRn9uB
+++ return 1
++ for node_id in '"${nodeList[@]}"'
++ nodeList_from_pmm+=($(kubectl_bin exec -n "${namespace}" monitoring-0 -- pmm-admin --server-url=https://admin:admin@$(get_service_ip monitoring-service)/ --server-insecure-tls inventory list nodes --node-type=CONTAINER_NODE | grep $node_id | awk '{print $4}'))
+++ grep /node_id/9817611e-c22a-442a-bf13-131d02ea6b93
++++ get_service_ip monitoring-service
++++ local service=monitoring-service
++++ local server_type=rs0
+++ awk '{print $4}'
+++++ kubectl_bin get psmdb/monitoring-service -o 'jsonpath={.spec.replsets[].expose.enabled}'
++++++ mktemp
+++++ local LAST_OUT=/tmp/tmp.JSDBaWkfwT
++++++ mktemp
+++++ local LAST_ERR=/tmp/tmp.zZnB5t9aYt
+++++ local exit_status=0
+++++ local timeout=4
++++++ seq 0 2
+++++ for i in '$(seq 0 2)'
+++++ set +e
+++++ kubectl get psmdb/monitoring-service -o 'jsonpath={.spec.replsets[].expose.enabled}'
+++++ exit_status=1
+++++ set -e
+++++ '[' 1 '!=' 0 -a -n 1 ']'
+++++ cat /tmp/tmp.JSDBaWkfwT
+++++ cat /tmp/tmp.zZnB5t9aYt
Error from server (NotFound): perconaservermongodbs.psmdb.percona.com "monitoring-service" not found
+++++ sleep 0
+++++ for i in '$(seq 0 2)'
+++++ set +e
+++++ kubectl get psmdb/monitoring-service -o 'jsonpath={.spec.replsets[].expose.enabled}'
+++++ exit_status=1
+++++ set -e
+++++ '[' 1 '!=' 0 -a -n 1 ']'
+++++ cat /tmp/tmp.JSDBaWkfwT
+++++ cat /tmp/tmp.zZnB5t9aYt
Error from server (NotFound): perconaservermongodbs.psmdb.percona.com "monitoring-service" not found
+++++ sleep 4
+++++ for i in '$(seq 0 2)'
+++++ set +e
+++++ kubectl get psmdb/monitoring-service -o 'jsonpath={.spec.replsets[].expose.enabled}'
+++++ exit_status=1
+++++ set -e
+++++ '[' 1 '!=' 0 -a -n 1 ']'
+++++ cat /tmp/tmp.JSDBaWkfwT
+++++ cat /tmp/tmp.zZnB5t9aYt
Error from server (NotFound): perconaservermongodbs.psmdb.percona.com "monitoring-service" not found
+++++ sleep 8
+++++ cat /tmp/tmp.JSDBaWkfwT
+++++ cat /tmp/tmp.zZnB5t9aYt
Error from server (NotFound): perconaservermongodbs.psmdb.percona.com "monitoring-service" not found
+++++ rm /tmp/tmp.JSDBaWkfwT /tmp/tmp.zZnB5t9aYt
+++++ return 1
++++ '[' '' '!=' true ']'
++++ echo -n monitoring-service.monitoring-service-rs0
++++ return
+++ kubectl_bin exec -n monitoring-2-0-16090 monitoring-0 -- pmm-admin --server-url=https://admin:admin@monitoring-service.monitoring-service-rs0/ --server-insecure-tls inventory list nodes --node-type=CONTAINER_NODE
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.Vj5plmUjPK
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.Qc39rA6vEy
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in '$(seq 0 2)'
+++ set +e
+++ kubectl exec -n monitoring-2-0-16090 monitoring-0 -- pmm-admin --server-url=https://admin:admin@monitoring-service.monitoring-service-rs0/ --server-insecure-tls inventory list nodes --node-type=CONTAINER_NODE
+++ exit_status=1
+++ set -e
+++ '[' 1 '!=' 0 -a -n 1 ']'
+++ cat /tmp/tmp.Vj5plmUjPK
+++ cat /tmp/tmp.Qc39rA6vEy
command terminated with exit code 1
+++ sleep 0
+++ for i in '$(seq 0 2)'
+++ set +e
+++ kubectl exec -n monitoring-2-0-16090 monitoring-0 -- pmm-admin --server-url=https://admin:admin@monitoring-service.monitoring-service-rs0/ --server-insecure-tls inventory list nodes --node-type=CONTAINER_NODE
+++ exit_status=1
+++ set -e
+++ '[' 1 '!=' 0 -a -n 1 ']'
+++ cat /tmp/tmp.Vj5plmUjPK
+++ cat /tmp/tmp.Qc39rA6vEy
command terminated with exit code 1
+++ sleep 4
+++ for i in '$(seq 0 2)'
+++ set +e
+++ kubectl exec -n monitoring-2-0-16090 monitoring-0 -- pmm-admin --server-url=https://admin:admin@monitoring-service.monitoring-service-rs0/ --server-insecure-tls inventory list nodes --node-type=CONTAINER_NODE
+++ exit_status=1
+++ set -e
+++ '[' 1 '!=' 0 -a -n 1 ']'
+++ cat /tmp/tmp.Vj5plmUjPK
+++ cat /tmp/tmp.Qc39rA6vEy
command terminated with exit code 1
+++ sleep 8
+++ cat /tmp/tmp.Vj5plmUjPK
+++ cat /tmp/tmp.Qc39rA6vEy
command terminated with exit code 1
+++ rm /tmp/tmp.Vj5plmUjPK /tmp/tmp.Qc39rA6vEy
+++ return 1
++ echo
+ [[ -n '' ]]
++ kubectl_bin logs monitoring-rs0-0 pmm-client
++ grep -c 'cannot auto discover databases and collections'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.LtlLPUZGn8
+++ mktemp
++ local LAST_ERR=/tmp/tmp.hfsxzXrllq
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl logs monitoring-rs0-0 pmm-client
++ exit_status=1
++ set -e
++ '[' 1 '!=' 0 -a -n 1 ']'
++ cat /tmp/tmp.LtlLPUZGn8
++ cat /tmp/tmp.hfsxzXrllq
error: error from server (NotFound): pods "monitoring-rs0-0" not found in namespace "monitoring-2-0-16090"
++ sleep 0
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl logs monitoring-rs0-0 pmm-client
++ exit_status=1
++ set -e
++ '[' 1 '!=' 0 -a -n 1 ']'
++ cat /tmp/tmp.LtlLPUZGn8
++ cat /tmp/tmp.hfsxzXrllq
error: error from server (NotFound): pods "monitoring-rs0-0" not found in namespace "monitoring-2-0-16090"
++ sleep 4
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl logs monitoring-rs0-0 pmm-client
++ exit_status=1
++ set -e
++ '[' 1 '!=' 0 -a -n 1 ']'
++ cat /tmp/tmp.LtlLPUZGn8
++ cat /tmp/tmp.hfsxzXrllq
error: error from server (NotFound): pods "monitoring-rs0-0" not found in namespace "monitoring-2-0-16090"
++ sleep 8
++ cat /tmp/tmp.LtlLPUZGn8
++ cat /tmp/tmp.hfsxzXrllq
error: error from server (NotFound): pods "monitoring-rs0-0" not found in namespace "monitoring-2-0-16090"
++ rm /tmp/tmp.LtlLPUZGn8 /tmp/tmp.hfsxzXrllq
++ return 1
+ [[ 0 != 0 ]]
+ desc 'check for passwords leak'
+ set +o xtrace


-----------------------------------------------------------------------------------
check for passwords leak
-----------------------------------------------------------------------------------

+ check_passwords_leak
+ local secrets
+ local passwords
+ local pods
++ kubectl_bin get secrets -o json
+++ mktemp
++ local LAST_OUT=/tmp/tmp.MGuhxUTAvr
+++ mktemp
++ local LAST_ERR=/tmp/tmp.WGqcmlHT9M
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get secrets -o json
++ jq -r '.items[].data | to_entries | .[] | select(.key | (contains("_PASSWORD"))) | .value'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.MGuhxUTAvr
++ cat /tmp/tmp.WGqcmlHT9M
++ rm /tmp/tmp.MGuhxUTAvr /tmp/tmp.WGqcmlHT9M
++ return 0
+ secrets='YmFja3VwMTIzNDU2
Y2x1c3RlckFkbWluMTIzNDU2
Y2x1c3Rlck1vbml0b3IxMjM0NTY=
ZGF0YWJhc2VBZG1pbjEyMzQ1Ng==
dXNlckFkbWluMTIzNDU2
YmFja3VwMTIzNDU2
Y2x1c3RlckFkbWluMTIzNDU2
Y2x1c3Rlck1vbml0b3IxMjM0NTY=
ZGF0YWJhc2VBZG1pbjEyMzQ1Ng==
dXNlckFkbWluMTIzNDU2'
+ echo secrets=YmFja3VwMTIzNDU2 Y2x1c3RlckFkbWluMTIzNDU2 Y2x1c3Rlck1vbml0b3IxMjM0NTY= ZGF0YWJhc2VBZG1pbjEyMzQ1Ng== dXNlckFkbWluMTIzNDU2 YmFja3VwMTIzNDU2 Y2x1c3RlckFkbWluMTIzNDU2 Y2x1c3Rlck1vbml0b3IxMjM0NTY= ZGF0YWJhc2VBZG1pbjEyMzQ1Ng== dXNlckFkbWluMTIzNDU2
secrets=YmFja3VwMTIzNDU2 Y2x1c3RlckFkbWluMTIzNDU2 Y2x1c3Rlck1vbml0b3IxMjM0NTY= ZGF0YWJhc2VBZG1pbjEyMzQ1Ng== dXNlckFkbWluMTIzNDU2 YmFja3VwMTIzNDU2 Y2x1c3RlckFkbWluMTIzNDU2 Y2x1c3Rlck1vbml0b3IxMjM0NTY= ZGF0YWJhc2VBZG1pbjEyMzQ1Ng== dXNlckFkbWluMTIzNDU2
++ for i in '$secrets'
++ base64 -d
++ echo
++ for i in '$secrets'
++ base64 -d
++ echo
++ for i in '$secrets'
++ base64 -d
++ echo
++ for i in '$secrets'
++ base64 -d
++ echo
++ for i in '$secrets'
++ base64 -d
++ echo
++ for i in '$secrets'
++ base64 -d
++ echo
++ for i in '$secrets'
++ base64 -d
++ echo
++ for i in '$secrets'
++ base64 -d
++ echo
++ for i in '$secrets'
++ base64 -d
++ echo
++ for i in '$secrets'
++ base64 -d
++ echo
+ passwords='backup123456
clusterAdmin123456
clusterMonitor123456
databaseAdmin123456
userAdmin123456
backup123456
clusterAdmin123456
clusterMonitor123456
databaseAdmin123456
userAdmin123456 YmFja3VwMTIzNDU2
Y2x1c3RlckFkbWluMTIzNDU2
Y2x1c3Rlck1vbml0b3IxMjM0NTY=
ZGF0YWJhc2VBZG1pbjEyMzQ1Ng==
dXNlckFkbWluMTIzNDU2
YmFja3VwMTIzNDU2
Y2x1c3RlckFkbWluMTIzNDU2
Y2x1c3Rlck1vbml0b3IxMjM0NTY=
ZGF0YWJhc2VBZG1pbjEyMzQ1Ng==
dXNlckFkbWluMTIzNDU2'
+ echo passwords=backup123456 clusterAdmin123456 clusterMonitor123456 databaseAdmin123456 userAdmin123456 backup123456 clusterAdmin123456 clusterMonitor123456 databaseAdmin123456 userAdmin123456 YmFja3VwMTIzNDU2 Y2x1c3RlckFkbWluMTIzNDU2 Y2x1c3Rlck1vbml0b3IxMjM0NTY= ZGF0YWJhc2VBZG1pbjEyMzQ1Ng== dXNlckFkbWluMTIzNDU2 YmFja3VwMTIzNDU2 Y2x1c3RlckFkbWluMTIzNDU2 Y2x1c3Rlck1vbml0b3IxMjM0NTY= ZGF0YWJhc2VBZG1pbjEyMzQ1Ng== dXNlckFkbWluMTIzNDU2
passwords=backup123456 clusterAdmin123456 clusterMonitor123456 databaseAdmin123456 userAdmin123456 backup123456 clusterAdmin123456 clusterMonitor123456 databaseAdmin123456 userAdmin123456 YmFja3VwMTIzNDU2 Y2x1c3RlckFkbWluMTIzNDU2 Y2x1c3Rlck1vbml0b3IxMjM0NTY= ZGF0YWJhc2VBZG1pbjEyMzQ1Ng== dXNlckFkbWluMTIzNDU2 YmFja3VwMTIzNDU2 Y2x1c3RlckFkbWluMTIzNDU2 Y2x1c3Rlck1vbml0b3IxMjM0NTY= ZGF0YWJhc2VBZG1pbjEyMzQ1Ng== dXNlckFkbWluMTIzNDU2
++ awk -F / '{print $2}'
++ kubectl_bin get pods -o name
+++ mktemp
++ local LAST_OUT=/tmp/tmp.NhX8fuQC71
+++ mktemp
++ local LAST_ERR=/tmp/tmp.d0CosYPfg1
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods -o name
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.NhX8fuQC71
++ cat /tmp/tmp.d0CosYPfg1
++ rm /tmp/tmp.NhX8fuQC71 /tmp/tmp.d0CosYPfg1
++ return 0
+ pods='monitoring-0
psmdb-client-6f7f47c4b-gdrlr'
+ echo pods=monitoring-0 psmdb-client-6f7f47c4b-gdrlr
pods=monitoring-0 psmdb-client-6f7f47c4b-gdrlr
+ collect_logs monitoring-2-0-16090
+ local containers
+ local count
+ NS=monitoring-2-0-16090
+ for p in '$pods'
++ kubectl_bin -n monitoring-2-0-16090 get pod monitoring-0 -o 'jsonpath={.spec.containers[*].name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.DDmHbIzuMP
+++ mktemp
++ local LAST_ERR=/tmp/tmp.ywT18o0YWy
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl -n monitoring-2-0-16090 get pod monitoring-0 -o 'jsonpath={.spec.containers[*].name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.DDmHbIzuMP
++ cat /tmp/tmp.ywT18o0YWy
++ rm /tmp/tmp.DDmHbIzuMP /tmp/tmp.ywT18o0YWy
++ return 0
+ containers=monitoring
+ for c in '$containers'
+ [[ monitoring =~ pmm ]]
+ kubectl_bin -n monitoring-2-0-16090 logs monitoring-0 -c monitoring
++ mktemp
+ local LAST_OUT=/tmp/tmp.XobgRbdOsM
++ mktemp
+ local LAST_ERR=/tmp/tmp.ie1ulxc8Jk
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl -n monitoring-2-0-16090 logs monitoring-0 -c monitoring
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.XobgRbdOsM
+ cat /tmp/tmp.ie1ulxc8Jk
+ rm /tmp/tmp.XobgRbdOsM /tmp/tmp.ie1ulxc8Jk
+ return 0
+ echo logs saved in: /tmp/tmp.39yPQ2cQ0t/logs_output-monitoring-0-monitoring.txt
logs saved in: /tmp/tmp.39yPQ2cQ0t/logs_output-monitoring-0-monitoring.txt
+ for pass in '$passwords'
++ grep -c --fixed-strings -- backup123456 /tmp/tmp.39yPQ2cQ0t/logs_output-monitoring-0-monitoring.txt
++ :
+ count=0
+ [[ 0 != 0 ]]
+ for pass in '$passwords'
++ grep -c --fixed-strings -- clusterAdmin123456 /tmp/tmp.39yPQ2cQ0t/logs_output-monitoring-0-monitoring.txt
++ :
+ count=0
+ [[ 0 != 0 ]]
+ for pass in '$passwords'
++ grep -c --fixed-strings -- clusterMonitor123456 /tmp/tmp.39yPQ2cQ0t/logs_output-monitoring-0-monitoring.txt
++ :
+ count=0
+ [[ 0 != 0 ]]
+ for pass in '$passwords'
++ grep -c --fixed-strings -- databaseAdmin123456 /tmp/tmp.39yPQ2cQ0t/logs_output-monitoring-0-monitoring.txt
++ :
+ count=0
+ [[ 0 != 0 ]]
+ for pass in '$passwords'
++ grep -c --fixed-strings -- userAdmin123456 /tmp/tmp.39yPQ2cQ0t/logs_output-monitoring-0-monitoring.txt
++ :
+ count=0
+ [[ 0 != 0 ]]
+ for pass in '$passwords'
++ grep -c --fixed-strings -- backup123456 /tmp/tmp.39yPQ2cQ0t/logs_output-monitoring-0-monitoring.txt
++ :
+ count=0
+ [[ 0 != 0 ]]
+ for pass in '$passwords'
++ grep -c --fixed-strings -- clusterAdmin123456 /tmp/tmp.39yPQ2cQ0t/logs_output-monitoring-0-monitoring.txt
++ :
+ count=0
+ [[ 0 != 0 ]]
+ for pass in '$passwords'
++ grep -c --fixed-strings -- clusterMonitor123456 /tmp/tmp.39yPQ2cQ0t/logs_output-monitoring-0-monitoring.txt
++ :
+ count=0
+ [[ 0 != 0 ]]
+ for pass in '$passwords'
++ grep -c --fixed-strings -- databaseAdmin123456 /tmp/tmp.39yPQ2cQ0t/logs_output-monitoring-0-monitoring.txt
++ :
+ count=0
+ [[ 0 != 0 ]]
+ for pass in '$passwords'
++ grep -c --fixed-strings -- userAdmin123456 /tmp/tmp.39yPQ2cQ0t/logs_output-monitoring-0-monitoring.txt
++ :
+ count=0
+ [[ 0 != 0 ]]
+ for pass in '$passwords'
++ grep -c --fixed-strings -- YmFja3VwMTIzNDU2 /tmp/tmp.39yPQ2cQ0t/logs_output-monitoring-0-monitoring.txt
++ :
+ count=0
+ [[ 0 != 0 ]]
+ for pass in '$passwords'
++ grep -c --fixed-strings -- Y2x1c3RlckFkbWluMTIzNDU2 /tmp/tmp.39yPQ2cQ0t/logs_output-monitoring-0-monitoring.txt
++ :
+ count=0
+ [[ 0 != 0 ]]
+ for pass in '$passwords'
++ grep -c --fixed-strings -- Y2x1c3Rlck1vbml0b3IxMjM0NTY= /tmp/tmp.39yPQ2cQ0t/logs_output-monitoring-0-monitoring.txt
++ :
+ count=0
+ [[ 0 != 0 ]]
+ for pass in '$passwords'
++ grep -c --fixed-strings -- ZGF0YWJhc2VBZG1pbjEyMzQ1Ng== /tmp/tmp.39yPQ2cQ0t/logs_output-monitoring-0-monitoring.txt
++ :
+ count=0
+ [[ 0 != 0 ]]
+ for pass in '$passwords'
++ grep -c --fixed-strings -- dXNlckFkbWluMTIzNDU2 /tmp/tmp.39yPQ2cQ0t/logs_output-monitoring-0-monitoring.txt
++ :
+ count=0
+ [[ 0 != 0 ]]
+ for pass in '$passwords'
++ grep -c --fixed-strings -- YmFja3VwMTIzNDU2 /tmp/tmp.39yPQ2cQ0t/logs_output-monitoring-0-monitoring.txt
++ :
+ count=0
+ [[ 0 != 0 ]]
+ for pass in '$passwords'
++ grep -c --fixed-strings -- Y2x1c3RlckFkbWluMTIzNDU2 /tmp/tmp.39yPQ2cQ0t/logs_output-monitoring-0-monitoring.txt
++ :
+ count=0
+ [[ 0 != 0 ]]
+ for pass in '$passwords'
++ grep -c --fixed-strings -- Y2x1c3Rlck1vbml0b3IxMjM0NTY= /tmp/tmp.39yPQ2cQ0t/logs_output-monitoring-0-monitoring.txt
++ :
+ count=0
+ [[ 0 != 0 ]]
+ for pass in '$passwords'
++ grep -c --fixed-strings -- ZGF0YWJhc2VBZG1pbjEyMzQ1Ng== /tmp/tmp.39yPQ2cQ0t/logs_output-monitoring-0-monitoring.txt
++ :
+ count=0
+ [[ 0 != 0 ]]
+ for pass in '$passwords'
++ grep -c --fixed-strings -- dXNlckFkbWluMTIzNDU2 /tmp/tmp.39yPQ2cQ0t/logs_output-monitoring-0-monitoring.txt
++ :
+ count=0
+ [[ 0 != 0 ]]
+ echo

+ for p in '$pods'
++ kubectl_bin -n monitoring-2-0-16090 get pod psmdb-client-6f7f47c4b-gdrlr -o 'jsonpath={.spec.containers[*].name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.Jjv9QjZ2Sx
+++ mktemp
++ local LAST_ERR=/tmp/tmp.GOmgmrkLq1
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl -n monitoring-2-0-16090 get pod psmdb-client-6f7f47c4b-gdrlr -o 'jsonpath={.spec.containers[*].name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.Jjv9QjZ2Sx
++ cat /tmp/tmp.GOmgmrkLq1
++ rm /tmp/tmp.Jjv9QjZ2Sx /tmp/tmp.GOmgmrkLq1
++ return 0
+ containers=psmdb-client
+ for c in '$containers'
+ [[ psmdb-client =~ pmm ]]
+ kubectl_bin -n monitoring-2-0-16090 logs psmdb-client-6f7f47c4b-gdrlr -c psmdb-client
++ mktemp
+ local LAST_OUT=/tmp/tmp.A8SwRrFbeV
++ mktemp
+ local LAST_ERR=/tmp/tmp.MDXURpcSoZ
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl -n monitoring-2-0-16090 logs psmdb-client-6f7f47c4b-gdrlr -c psmdb-client
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.A8SwRrFbeV
+ cat /tmp/tmp.MDXURpcSoZ
+ rm /tmp/tmp.A8SwRrFbeV /tmp/tmp.MDXURpcSoZ
+ return 0
+ echo logs saved in: /tmp/tmp.39yPQ2cQ0t/logs_output-psmdb-client-6f7f47c4b-gdrlr-psmdb-client.txt
logs saved in: /tmp/tmp.39yPQ2cQ0t/logs_output-psmdb-client-6f7f47c4b-gdrlr-psmdb-client.txt
+ for pass in '$passwords'
++ grep -c --fixed-strings -- backup123456 /tmp/tmp.39yPQ2cQ0t/logs_output-psmdb-client-6f7f47c4b-gdrlr-psmdb-client.txt
++ :
+ count=0
+ [[ 0 != 0 ]]
+ for pass in '$passwords'
++ grep -c --fixed-strings -- clusterAdmin123456 /tmp/tmp.39yPQ2cQ0t/logs_output-psmdb-client-6f7f47c4b-gdrlr-psmdb-client.txt
++ :
+ count=0
+ [[ 0 != 0 ]]
+ for pass in '$passwords'
++ grep -c --fixed-strings -- clusterMonitor123456 /tmp/tmp.39yPQ2cQ0t/logs_output-psmdb-client-6f7f47c4b-gdrlr-psmdb-client.txt
++ :
+ count=0
+ [[ 0 != 0 ]]
+ for pass in '$passwords'
++ grep -c --fixed-strings -- databaseAdmin123456 /tmp/tmp.39yPQ2cQ0t/logs_output-psmdb-client-6f7f47c4b-gdrlr-psmdb-client.txt
++ :
+ count=0
+ [[ 0 != 0 ]]
+ for pass in '$passwords'
++ grep -c --fixed-strings -- userAdmin123456 /tmp/tmp.39yPQ2cQ0t/logs_output-psmdb-client-6f7f47c4b-gdrlr-psmdb-client.txt
++ :
+ count=0
+ [[ 0 != 0 ]]
+ for pass in '$passwords'
++ grep -c --fixed-strings -- backup123456 /tmp/tmp.39yPQ2cQ0t/logs_output-psmdb-client-6f7f47c4b-gdrlr-psmdb-client.txt
++ :
+ count=0
+ [[ 0 != 0 ]]
+ for pass in '$passwords'
++ grep -c --fixed-strings -- clusterAdmin123456 /tmp/tmp.39yPQ2cQ0t/logs_output-psmdb-client-6f7f47c4b-gdrlr-psmdb-client.txt
++ :
+ count=0
+ [[ 0 != 0 ]]
+ for pass in '$passwords'
++ grep -c --fixed-strings -- clusterMonitor123456 /tmp/tmp.39yPQ2cQ0t/logs_output-psmdb-client-6f7f47c4b-gdrlr-psmdb-client.txt
++ :
+ count=0
+ [[ 0 != 0 ]]
+ for pass in '$passwords'
++ grep -c --fixed-strings -- databaseAdmin123456 /tmp/tmp.39yPQ2cQ0t/logs_output-psmdb-client-6f7f47c4b-gdrlr-psmdb-client.txt
++ :
+ count=0
+ [[ 0 != 0 ]]
+ for pass in '$passwords'
++ grep -c --fixed-strings -- userAdmin123456 /tmp/tmp.39yPQ2cQ0t/logs_output-psmdb-client-6f7f47c4b-gdrlr-psmdb-client.txt
++ :
+ count=0
+ [[ 0 != 0 ]]
+ for pass in '$passwords'
++ grep -c --fixed-strings -- YmFja3VwMTIzNDU2 /tmp/tmp.39yPQ2cQ0t/logs_output-psmdb-client-6f7f47c4b-gdrlr-psmdb-client.txt
++ :
+ count=0
+ [[ 0 != 0 ]]
+ for pass in '$passwords'
++ grep -c --fixed-strings -- Y2x1c3RlckFkbWluMTIzNDU2 /tmp/tmp.39yPQ2cQ0t/logs_output-psmdb-client-6f7f47c4b-gdrlr-psmdb-client.txt
++ :
+ count=0
+ [[ 0 != 0 ]]
+ for pass in '$passwords'
++ grep -c --fixed-strings -- Y2x1c3Rlck1vbml0b3IxMjM0NTY= /tmp/tmp.39yPQ2cQ0t/logs_output-psmdb-client-6f7f47c4b-gdrlr-psmdb-client.txt
++ :
+ count=0
+ [[ 0 != 0 ]]
+ for pass in '$passwords'
++ grep -c --fixed-strings -- ZGF0YWJhc2VBZG1pbjEyMzQ1Ng== /tmp/tmp.39yPQ2cQ0t/logs_output-psmdb-client-6f7f47c4b-gdrlr-psmdb-client.txt
++ :
+ count=0
+ [[ 0 != 0 ]]
+ for pass in '$passwords'
++ grep -c --fixed-strings -- dXNlckFkbWluMTIzNDU2 /tmp/tmp.39yPQ2cQ0t/logs_output-psmdb-client-6f7f47c4b-gdrlr-psmdb-client.txt
++ :
+ count=0
+ [[ 0 != 0 ]]
+ for pass in '$passwords'
++ grep -c --fixed-strings -- YmFja3VwMTIzNDU2 /tmp/tmp.39yPQ2cQ0t/logs_output-psmdb-client-6f7f47c4b-gdrlr-psmdb-client.txt
++ :
+ count=0
+ [[ 0 != 0 ]]
+ for pass in '$passwords'
++ grep -c --fixed-strings -- Y2x1c3RlckFkbWluMTIzNDU2 /tmp/tmp.39yPQ2cQ0t/logs_output-psmdb-client-6f7f47c4b-gdrlr-psmdb-client.txt
++ :
+ count=0
+ [[ 0 != 0 ]]
+ for pass in '$passwords'
++ grep -c --fixed-strings -- Y2x1c3Rlck1vbml0b3IxMjM0NTY= /tmp/tmp.39yPQ2cQ0t/logs_output-psmdb-client-6f7f47c4b-gdrlr-psmdb-client.txt
++ :
+ count=0
+ [[ 0 != 0 ]]
+ for pass in '$passwords'
++ grep -c --fixed-strings -- ZGF0YWJhc2VBZG1pbjEyMzQ1Ng== /tmp/tmp.39yPQ2cQ0t/logs_output-psmdb-client-6f7f47c4b-gdrlr-psmdb-client.txt
++ :
+ count=0
+ [[ 0 != 0 ]]
+ for pass in '$passwords'
++ grep -c --fixed-strings -- dXNlckFkbWluMTIzNDU2 /tmp/tmp.39yPQ2cQ0t/logs_output-psmdb-client-6f7f47c4b-gdrlr-psmdb-client.txt
++ :
+ count=0
+ [[ 0 != 0 ]]
+ echo

+ '[' -n psmdb-operator ']'
++ kubectl_bin -n psmdb-operator get pods -o name
+++ mktemp
++ local LAST_OUT=/tmp/tmp.qy6kfPBNsT
+++ mktemp
++ local LAST_ERR=/tmp/tmp.aRmbeze7Zs
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ awk -F / '{print $2}'
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl -n psmdb-operator get pods -o name
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.qy6kfPBNsT
++ cat /tmp/tmp.aRmbeze7Zs
++ rm /tmp/tmp.qy6kfPBNsT /tmp/tmp.aRmbeze7Zs
++ return 0
+ pods=percona-server-mongodb-operator-64975ffb9f-w95gh
+ collect_logs psmdb-operator
+ local containers
+ local count
+ NS=psmdb-operator
+ for p in '$pods'
++ kubectl_bin -n psmdb-operator get pod percona-server-mongodb-operator-64975ffb9f-w95gh -o 'jsonpath={.spec.containers[*].name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.DJaOYk9AaX
+++ mktemp
++ local LAST_ERR=/tmp/tmp.L8UBO3Agrv
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl -n psmdb-operator get pod percona-server-mongodb-operator-64975ffb9f-w95gh -o 'jsonpath={.spec.containers[*].name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.DJaOYk9AaX
++ cat /tmp/tmp.L8UBO3Agrv
++ rm /tmp/tmp.DJaOYk9AaX /tmp/tmp.L8UBO3Agrv
++ return 0
+ containers=percona-server-mongodb-operator
+ for c in '$containers'
+ [[ percona-server-mongodb-operator =~ pmm ]]
+ kubectl_bin -n psmdb-operator logs percona-server-mongodb-operator-64975ffb9f-w95gh -c percona-server-mongodb-operator
++ mktemp
+ local LAST_OUT=/tmp/tmp.6QUQLFZ2oN
++ mktemp
+ local LAST_ERR=/tmp/tmp.r5npfXHHqn
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl -n psmdb-operator logs percona-server-mongodb-operator-64975ffb9f-w95gh -c percona-server-mongodb-operator
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.6QUQLFZ2oN
+ cat /tmp/tmp.r5npfXHHqn
+ rm /tmp/tmp.6QUQLFZ2oN /tmp/tmp.r5npfXHHqn
+ return 0
+ echo logs saved in: /tmp/tmp.39yPQ2cQ0t/logs_output-percona-server-mongodb-operator-64975ffb9f-w95gh-percona-server-mongodb-operator.txt
logs saved in: /tmp/tmp.39yPQ2cQ0t/logs_output-percona-server-mongodb-operator-64975ffb9f-w95gh-percona-server-mongodb-operator.txt
+ for pass in '$passwords'
++ grep -c --fixed-strings -- backup123456 /tmp/tmp.39yPQ2cQ0t/logs_output-percona-server-mongodb-operator-64975ffb9f-w95gh-percona-server-mongodb-operator.txt
++ :
+ count=0
+ [[ 0 != 0 ]]
+ for pass in '$passwords'
++ grep -c --fixed-strings -- clusterAdmin123456 /tmp/tmp.39yPQ2cQ0t/logs_output-percona-server-mongodb-operator-64975ffb9f-w95gh-percona-server-mongodb-operator.txt
++ :
+ count=0
+ [[ 0 != 0 ]]
+ for pass in '$passwords'
++ grep -c --fixed-strings -- clusterMonitor123456 /tmp/tmp.39yPQ2cQ0t/logs_output-percona-server-mongodb-operator-64975ffb9f-w95gh-percona-server-mongodb-operator.txt
++ :
+ count=0
+ [[ 0 != 0 ]]
+ for pass in '$passwords'
++ grep -c --fixed-strings -- databaseAdmin123456 /tmp/tmp.39yPQ2cQ0t/logs_output-percona-server-mongodb-operator-64975ffb9f-w95gh-percona-server-mongodb-operator.txt
++ :
+ count=0
+ [[ 0 != 0 ]]
+ for pass in '$passwords'
++ grep -c --fixed-strings -- userAdmin123456 /tmp/tmp.39yPQ2cQ0t/logs_output-percona-server-mongodb-operator-64975ffb9f-w95gh-percona-server-mongodb-operator.txt
++ :
+ count=0
+ [[ 0 != 0 ]]
+ for pass in '$passwords'
++ grep -c --fixed-strings -- backup123456 /tmp/tmp.39yPQ2cQ0t/logs_output-percona-server-mongodb-operator-64975ffb9f-w95gh-percona-server-mongodb-operator.txt
++ :
+ count=0
+ [[ 0 != 0 ]]
+ for pass in '$passwords'
++ grep -c --fixed-strings -- clusterAdmin123456 /tmp/tmp.39yPQ2cQ0t/logs_output-percona-server-mongodb-operator-64975ffb9f-w95gh-percona-server-mongodb-operator.txt
++ :
+ count=0
+ [[ 0 != 0 ]]
+ for pass in '$passwords'
++ grep -c --fixed-strings -- clusterMonitor123456 /tmp/tmp.39yPQ2cQ0t/logs_output-percona-server-mongodb-operator-64975ffb9f-w95gh-percona-server-mongodb-operator.txt
++ :
+ count=0
+ [[ 0 != 0 ]]
+ for pass in '$passwords'
++ grep -c --fixed-strings -- databaseAdmin123456 /tmp/tmp.39yPQ2cQ0t/logs_output-percona-server-mongodb-operator-64975ffb9f-w95gh-percona-server-mongodb-operator.txt
++ :
+ count=0
+ [[ 0 != 0 ]]
+ for pass in '$passwords'
++ grep -c --fixed-strings -- userAdmin123456 /tmp/tmp.39yPQ2cQ0t/logs_output-percona-server-mongodb-operator-64975ffb9f-w95gh-percona-server-mongodb-operator.txt
++ :
+ count=0
+ [[ 0 != 0 ]]
+ for pass in '$passwords'
++ grep -c --fixed-strings -- YmFja3VwMTIzNDU2 /tmp/tmp.39yPQ2cQ0t/logs_output-percona-server-mongodb-operator-64975ffb9f-w95gh-percona-server-mongodb-operator.txt
++ :
+ count=0
+ [[ 0 != 0 ]]
+ for pass in '$passwords'
++ grep -c --fixed-strings -- Y2x1c3RlckFkbWluMTIzNDU2 /tmp/tmp.39yPQ2cQ0t/logs_output-percona-server-mongodb-operator-64975ffb9f-w95gh-percona-server-mongodb-operator.txt
++ :
+ count=0
+ [[ 0 != 0 ]]
+ for pass in '$passwords'
++ grep -c --fixed-strings -- Y2x1c3Rlck1vbml0b3IxMjM0NTY= /tmp/tmp.39yPQ2cQ0t/logs_output-percona-server-mongodb-operator-64975ffb9f-w95gh-percona-server-mongodb-operator.txt
++ :
+ count=0
+ [[ 0 != 0 ]]
+ for pass in '$passwords'
++ grep -c --fixed-strings -- ZGF0YWJhc2VBZG1pbjEyMzQ1Ng== /tmp/tmp.39yPQ2cQ0t/logs_output-percona-server-mongodb-operator-64975ffb9f-w95gh-percona-server-mongodb-operator.txt
++ :
+ count=0
+ [[ 0 != 0 ]]
+ for pass in '$passwords'
++ grep -c --fixed-strings -- dXNlckFkbWluMTIzNDU2 /tmp/tmp.39yPQ2cQ0t/logs_output-percona-server-mongodb-operator-64975ffb9f-w95gh-percona-server-mongodb-operator.txt
++ :
+ count=0
+ [[ 0 != 0 ]]
+ for pass in '$passwords'
++ grep -c --fixed-strings -- YmFja3VwMTIzNDU2 /tmp/tmp.39yPQ2cQ0t/logs_output-percona-server-mongodb-operator-64975ffb9f-w95gh-percona-server-mongodb-operator.txt
++ :
+ count=0
+ [[ 0 != 0 ]]
+ for pass in '$passwords'
++ grep -c --fixed-strings -- Y2x1c3RlckFkbWluMTIzNDU2 /tmp/tmp.39yPQ2cQ0t/logs_output-percona-server-mongodb-operator-64975ffb9f-w95gh-percona-server-mongodb-operator.txt
++ :
+ count=0
+ [[ 0 != 0 ]]
+ for pass in '$passwords'
++ grep -c --fixed-strings -- Y2x1c3Rlck1vbml0b3IxMjM0NTY= /tmp/tmp.39yPQ2cQ0t/logs_output-percona-server-mongodb-operator-64975ffb9f-w95gh-percona-server-mongodb-operator.txt
++ :
+ count=0
+ [[ 0 != 0 ]]
+ for pass in '$passwords'
++ grep -c --fixed-strings -- ZGF0YWJhc2VBZG1pbjEyMzQ1Ng== /tmp/tmp.39yPQ2cQ0t/logs_output-percona-server-mongodb-operator-64975ffb9f-w95gh-percona-server-mongodb-operator.txt
++ :
+ count=0
+ [[ 0 != 0 ]]
+ for pass in '$passwords'
++ grep -c --fixed-strings -- dXNlckFkbWluMTIzNDU2 /tmp/tmp.39yPQ2cQ0t/logs_output-percona-server-mongodb-operator-64975ffb9f-w95gh-percona-server-mongodb-operator.txt
++ :
+ count=0
+ [[ 0 != 0 ]]
+ echo

+ helm uninstall monitoring
release "monitoring" uninstalled
+ destroy monitoring-2-0-16090
+ local namespace=monitoring-2-0-16090
+ local ignore_logs=true
+ desc 'destroy cluster/operator and all other resources'
+ set +o xtrace


-----------------------------------------------------------------------------------
destroy cluster/operator and all other resources
-----------------------------------------------------------------------------------

+ '[' true == false ']'
+ delete_crd
+ desc 'get and delete old CRDs and RBAC'
+ set +o xtrace


-----------------------------------------------------------------------------------
get and delete old CRDs and RBAC
-----------------------------------------------------------------------------------

+ kubectl_bin delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1596/deploy/crd.yaml --ignore-not-found --wait=false
++ mktemp
+ local LAST_OUT=/tmp/tmp.97VWlHw8V2
++ mktemp
+ local LAST_ERR=/tmp/tmp.CeXQJB6DJe
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1596/deploy/crd.yaml --ignore-not-found --wait=false
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.97VWlHw8V2
customresourcedefinition.apiextensions.k8s.io "perconaservermongodbbackups.psmdb.percona.com" deleted
customresourcedefinition.apiextensions.k8s.io "perconaservermongodbrestores.psmdb.percona.com" deleted
customresourcedefinition.apiextensions.k8s.io "perconaservermongodbs.psmdb.percona.com" deleted
+ cat /tmp/tmp.CeXQJB6DJe
+ rm /tmp/tmp.97VWlHw8V2 /tmp/tmp.CeXQJB6DJe
+ return 0
++ yq eval .metadata.name /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1596/deploy/crd.yaml
++ grep -v '\-\-\-'
+ for crd_name in '$(yq eval '\''.metadata.name'\'' "${src_dir}/deploy/crd.yaml" | grep -v '\''\-\-\-'\'')'
+ kubectl get perconaservermongodbbackups.psmdb.percona.com --all-namespaces -o wide
+ xargs -L 1 sh -xc 'kubectl patch perconaservermongodbbackups.psmdb.percona.com -n $0 $1 --type=merge -p "{\"metadata\":{\"finalizers\":[]}}"'
+ grep -v NAMESPACE
error: the server doesn't have a resource type "perconaservermongodbbackups"
+ kubectl patch perconaservermongodbbackups.psmdb.percona.com -n sh --type=merge -p '{"metadata":{"finalizers":[]}}'
error: the server doesn't have a resource type "perconaservermongodbbackups"
+ :
+ kubectl_bin wait --for=delete crd perconaservermongodbbackups.psmdb.percona.com
++ mktemp
+ local LAST_OUT=/tmp/tmp.lSW2b9Mz6d
++ mktemp
+ local LAST_ERR=/tmp/tmp.rQck6Kqh24
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl wait --for=delete crd perconaservermongodbbackups.psmdb.percona.com
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.lSW2b9Mz6d
+ cat /tmp/tmp.rQck6Kqh24
+ rm /tmp/tmp.lSW2b9Mz6d /tmp/tmp.rQck6Kqh24
+ return 0
+ for crd_name in '$(yq eval '\''.metadata.name'\'' "${src_dir}/deploy/crd.yaml" | grep -v '\''\-\-\-'\'')'
+ grep -v NAMESPACE
+ kubectl get perconaservermongodbrestores.psmdb.percona.com --all-namespaces -o wide
+ xargs -L 1 sh -xc 'kubectl patch perconaservermongodbrestores.psmdb.percona.com -n $0 $1 --type=merge -p "{\"metadata\":{\"finalizers\":[]}}"'
error: the server doesn't have a resource type "perconaservermongodbrestores"
+ kubectl patch perconaservermongodbrestores.psmdb.percona.com -n sh --type=merge -p '{"metadata":{"finalizers":[]}}'
error: the server doesn't have a resource type "perconaservermongodbrestores"
+ :
+ kubectl_bin wait --for=delete crd perconaservermongodbrestores.psmdb.percona.com
++ mktemp
+ local LAST_OUT=/tmp/tmp.q6J4uPuHJR
++ mktemp
+ local LAST_ERR=/tmp/tmp.blZAhnt23U
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl wait --for=delete crd perconaservermongodbrestores.psmdb.percona.com
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.q6J4uPuHJR
+ cat /tmp/tmp.blZAhnt23U
+ rm /tmp/tmp.q6J4uPuHJR /tmp/tmp.blZAhnt23U
+ return 0
+ for crd_name in '$(yq eval '\''.metadata.name'\'' "${src_dir}/deploy/crd.yaml" | grep -v '\''\-\-\-'\'')'
+ kubectl get perconaservermongodbs.psmdb.percona.com --all-namespaces -o wide
+ grep -v NAMESPACE
+ xargs -L 1 sh -xc 'kubectl patch perconaservermongodbs.psmdb.percona.com -n $0 $1 --type=merge -p "{\"metadata\":{\"finalizers\":[]}}"'
error: the server doesn't have a resource type "perconaservermongodbs"
+ kubectl patch perconaservermongodbs.psmdb.percona.com -n sh --type=merge -p '{"metadata":{"finalizers":[]}}'
error: the server doesn't have a resource type "perconaservermongodbs"
+ :
+ kubectl_bin wait --for=delete crd perconaservermongodbs.psmdb.percona.com
++ mktemp
+ local LAST_OUT=/tmp/tmp.RHYSKSKN7f
++ mktemp
+ local LAST_ERR=/tmp/tmp.6JruZpEcXT
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl wait --for=delete crd perconaservermongodbs.psmdb.percona.com
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.RHYSKSKN7f
+ cat /tmp/tmp.6JruZpEcXT
+ rm /tmp/tmp.RHYSKSKN7f /tmp/tmp.6JruZpEcXT
+ return 0
+ local rbac_yaml=rbac.yaml
+ '[' -n psmdb-operator ']'
+ rbac_yaml=cw-rbac.yaml
+ kubectl_bin delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1596/deploy/cw-rbac.yaml --ignore-not-found
++ mktemp
+ local LAST_OUT=/tmp/tmp.pLCWvKuftj
++ mktemp
+ local LAST_ERR=/tmp/tmp.BUtpjiZSKT
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1596/deploy/cw-rbac.yaml --ignore-not-found
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.pLCWvKuftj
clusterrole.rbac.authorization.k8s.io "percona-server-mongodb-operator" deleted
clusterrolebinding.rbac.authorization.k8s.io "service-account-percona-server-mongodb-operator" deleted
+ cat /tmp/tmp.BUtpjiZSKT
+ rm /tmp/tmp.pLCWvKuftj /tmp/tmp.BUtpjiZSKT
+ return 0
+ destroy_cert_manager
+ kubectl_bin delete -f https://github.com/cert-manager/cert-manager/releases/download/v1.15.1/cert-manager.yaml
++ mktemp
+ local LAST_OUT=/tmp/tmp.7qIgq6sE7J
++ mktemp
+ local LAST_ERR=/tmp/tmp.j2KRNvKDPq
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl delete -f https://github.com/cert-manager/cert-manager/releases/download/v1.15.1/cert-manager.yaml
+ exit_status=1
+ set -e
+ '[' 1 '!=' 0 -a -n 1 ']'
+ cat /tmp/tmp.7qIgq6sE7J
namespace "cert-manager" deleted
customresourcedefinition.apiextensions.k8s.io "certificaterequests.cert-manager.io" deleted
customresourcedefinition.apiextensions.k8s.io "certificates.cert-manager.io" deleted
customresourcedefinition.apiextensions.k8s.io "challenges.acme.cert-manager.io" deleted
customresourcedefinition.apiextensions.k8s.io "clusterissuers.cert-manager.io" deleted
customresourcedefinition.apiextensions.k8s.io "issuers.cert-manager.io" deleted
customresourcedefinition.apiextensions.k8s.io "orders.acme.cert-manager.io" deleted
serviceaccount "cert-manager-cainjector" deleted
serviceaccount "cert-manager" deleted
serviceaccount "cert-manager-webhook" deleted
clusterrole.rbac.authorization.k8s.io "cert-manager-cainjector" deleted
clusterrole.rbac.authorization.k8s.io "cert-manager-controller-issuers" deleted
clusterrole.rbac.authorization.k8s.io "cert-manager-controller-clusterissuers" deleted
clusterrole.rbac.authorization.k8s.io "cert-manager-controller-certificates" deleted
clusterrole.rbac.authorization.k8s.io "cert-manager-controller-orders" deleted
clusterrole.rbac.authorization.k8s.io "cert-manager-controller-challenges" deleted
clusterrole.rbac.authorization.k8s.io "cert-manager-controller-ingress-shim" deleted
clusterrole.rbac.authorization.k8s.io "cert-manager-cluster-view" deleted
clusterrole.rbac.authorization.k8s.io "cert-manager-view" deleted
clusterrole.rbac.authorization.k8s.io "cert-manager-edit" deleted
clusterrole.rbac.authorization.k8s.io "cert-manager-controller-approve:cert-manager-io" deleted
clusterrole.rbac.authorization.k8s.io "cert-manager-controller-certificatesigningrequests" deleted
clusterrole.rbac.authorization.k8s.io "cert-manager-webhook:subjectaccessreviews" deleted
clusterrolebinding.rbac.authorization.k8s.io "cert-manager-cainjector" deleted
clusterrolebinding.rbac.authorization.k8s.io "cert-manager-controller-issuers" deleted
clusterrolebinding.rbac.authorization.k8s.io "cert-manager-controller-clusterissuers" deleted
clusterrolebinding.rbac.authorization.k8s.io "cert-manager-controller-certificates" deleted
clusterrolebinding.rbac.authorization.k8s.io "cert-manager-controller-orders" deleted
clusterrolebinding.rbac.authorization.k8s.io "cert-manager-controller-challenges" deleted
clusterrolebinding.rbac.authorization.k8s.io "cert-manager-controller-ingress-shim" deleted
clusterrolebinding.rbac.authorization.k8s.io "cert-manager-controller-approve:cert-manager-io" deleted
clusterrolebinding.rbac.authorization.k8s.io "cert-manager-controller-certificatesigningrequests" deleted
clusterrolebinding.rbac.authorization.k8s.io "cert-manager-webhook:subjectaccessreviews" deleted
role.rbac.authorization.k8s.io "cert-manager-cainjector:leaderelection" deleted
role.rbac.authorization.k8s.io "cert-manager:leaderelection" deleted
role.rbac.authorization.k8s.io "cert-manager-webhook:dynamic-serving" deleted
rolebinding.rbac.authorization.k8s.io "cert-manager-cainjector:leaderelection" deleted
rolebinding.rbac.authorization.k8s.io "cert-manager:leaderelection" deleted
rolebinding.rbac.authorization.k8s.io "cert-manager-webhook:dynamic-serving" deleted
deployment.apps "cert-manager-cainjector" deleted
mutatingwebhookconfiguration.admissionregistration.k8s.io "cert-manager-webhook" deleted
validatingwebhookconfiguration.admissionregistration.k8s.io "cert-manager-webhook" deleted
+ cat /tmp/tmp.j2KRNvKDPq
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.1/cert-manager.yaml": services "cert-manager" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.1/cert-manager.yaml": services "cert-manager-webhook" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.1/cert-manager.yaml": deployments.apps "cert-manager" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.1/cert-manager.yaml": deployments.apps "cert-manager-webhook" not found
+ sleep 0
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl delete -f https://github.com/cert-manager/cert-manager/releases/download/v1.15.1/cert-manager.yaml
+ exit_status=1
+ set -e
+ '[' 1 '!=' 0 -a -n 1 ']'
+ cat /tmp/tmp.7qIgq6sE7J
namespace "cert-manager" deleted
+ cat /tmp/tmp.j2KRNvKDPq
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.1/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "certificaterequests.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.1/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "certificates.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.1/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "challenges.acme.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.1/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "clusterissuers.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.1/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "issuers.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.1/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "orders.acme.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.1/cert-manager.yaml": serviceaccounts "cert-manager-cainjector" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.1/cert-manager.yaml": serviceaccounts "cert-manager" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.1/cert-manager.yaml": serviceaccounts "cert-manager-webhook" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.1/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-cainjector" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.1/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-issuers" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.1/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-clusterissuers" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.1/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-certificates" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.1/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-orders" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.1/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-challenges" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.1/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-ingress-shim" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.1/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-cluster-view" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.1/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-view" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.1/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-edit" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.1/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-approve:cert-manager-io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.1/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-certificatesigningrequests" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.1/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-webhook:subjectaccessreviews" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.1/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-cainjector" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.1/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-issuers" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.1/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-clusterissuers" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.1/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-certificates" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.1/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-orders" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.1/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-challenges" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.1/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-ingress-shim" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.1/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-approve:cert-manager-io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.1/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-certificatesigningrequests" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.1/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-webhook:subjectaccessreviews" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.1/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager-cainjector:leaderelection" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.1/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager:leaderelection" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.1/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager-webhook:dynamic-serving" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.1/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager-cainjector:leaderelection" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.1/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager:leaderelection" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.1/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager-webhook:dynamic-serving" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.1/cert-manager.yaml": services "cert-manager" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.1/cert-manager.yaml": services "cert-manager-webhook" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.1/cert-manager.yaml": deployments.apps "cert-manager-cainjector" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.1/cert-manager.yaml": deployments.apps "cert-manager" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.1/cert-manager.yaml": deployments.apps "cert-manager-webhook" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.1/cert-manager.yaml": mutatingwebhookconfigurations.admissionregistration.k8s.io "cert-manager-webhook" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.1/cert-manager.yaml": validatingwebhookconfigurations.admissionregistration.k8s.io "cert-manager-webhook" not found
+ sleep 4
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl delete -f https://github.com/cert-manager/cert-manager/releases/download/v1.15.1/cert-manager.yaml
+ exit_status=1
+ set -e
+ '[' 1 '!=' 0 -a -n 1 ']'
+ cat /tmp/tmp.7qIgq6sE7J
+ cat /tmp/tmp.j2KRNvKDPq
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.1/cert-manager.yaml": namespaces "cert-manager" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.1/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "certificaterequests.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.1/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "certificates.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.1/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "challenges.acme.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.1/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "clusterissuers.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.1/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "issuers.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.1/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "orders.acme.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.1/cert-manager.yaml": serviceaccounts "cert-manager-cainjector" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.1/cert-manager.yaml": serviceaccounts "cert-manager" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.1/cert-manager.yaml": serviceaccounts "cert-manager-webhook" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.1/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-cainjector" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.1/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-issuers" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.1/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-clusterissuers" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.1/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-certificates" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.1/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-orders" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.1/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-challenges" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.1/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-ingress-shim" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.1/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-cluster-view" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.1/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-view" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.1/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-edit" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.1/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-approve:cert-manager-io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.1/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-certificatesigningrequests" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.1/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-webhook:subjectaccessreviews" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.1/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-cainjector" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.1/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-issuers" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.1/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-clusterissuers" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.1/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-certificates" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.1/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-orders" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.1/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-challenges" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.1/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-ingress-shim" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.1/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-approve:cert-manager-io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.1/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-certificatesigningrequests" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.1/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-webhook:subjectaccessreviews" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.1/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager-cainjector:leaderelection" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.1/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager:leaderelection" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.1/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager-webhook:dynamic-serving" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.1/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager-cainjector:leaderelection" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.1/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager:leaderelection" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.1/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager-webhook:dynamic-serving" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.1/cert-manager.yaml": services "cert-manager" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.1/cert-manager.yaml": services "cert-manager-webhook" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.1/cert-manager.yaml": deployments.apps "cert-manager-cainjector" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.1/cert-manager.yaml": deployments.apps "cert-manager" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.1/cert-manager.yaml": deployments.apps "cert-manager-webhook" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.1/cert-manager.yaml": mutatingwebhookconfigurations.admissionregistration.k8s.io "cert-manager-webhook" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.1/cert-manager.yaml": validatingwebhookconfigurations.admissionregistration.k8s.io "cert-manager-webhook" not found
+ sleep 8
+ cat /tmp/tmp.7qIgq6sE7J
+ cat /tmp/tmp.j2KRNvKDPq
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.1/cert-manager.yaml": namespaces "cert-manager" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.1/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "certificaterequests.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.1/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "certificates.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.1/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "challenges.acme.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.1/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "clusterissuers.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.1/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "issuers.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.1/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "orders.acme.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.1/cert-manager.yaml": serviceaccounts "cert-manager-cainjector" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.1/cert-manager.yaml": serviceaccounts "cert-manager" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.1/cert-manager.yaml": serviceaccounts "cert-manager-webhook" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.1/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-cainjector" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.1/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-issuers" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.1/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-clusterissuers" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.1/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-certificates" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.1/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-orders" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.1/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-challenges" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.1/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-ingress-shim" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.1/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-cluster-view" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.1/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-view" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.1/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-edit" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.1/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-approve:cert-manager-io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.1/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-certificatesigningrequests" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.1/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-webhook:subjectaccessreviews" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.1/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-cainjector" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.1/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-issuers" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.1/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-clusterissuers" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.1/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-certificates" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.1/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-orders" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.1/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-challenges" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.1/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-ingress-shim" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.1/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-approve:cert-manager-io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.1/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-certificatesigningrequests" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.1/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-webhook:subjectaccessreviews" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.1/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager-cainjector:leaderelection" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.1/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager:leaderelection" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.1/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager-webhook:dynamic-serving" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.1/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager-cainjector:leaderelection" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.1/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager:leaderelection" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.1/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager-webhook:dynamic-serving" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.1/cert-manager.yaml": services "cert-manager" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.1/cert-manager.yaml": services "cert-manager-webhook" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.1/cert-manager.yaml": deployments.apps "cert-manager-cainjector" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.1/cert-manager.yaml": deployments.apps "cert-manager" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.1/cert-manager.yaml": deployments.apps "cert-manager-webhook" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.1/cert-manager.yaml": mutatingwebhookconfigurations.admissionregistration.k8s.io "cert-manager-webhook" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.1/cert-manager.yaml": validatingwebhookconfigurations.admissionregistration.k8s.io "cert-manager-webhook" not found
+ rm /tmp/tmp.7qIgq6sE7J /tmp/tmp.j2KRNvKDPq
+ return 1
+ true
+ '[' -n '' ']'
+ '[' -n psmdb-operator ']'
+ kubectl_bin delete --grace-period=0 --force=true namespace monitoring-2-0-16090
+ rm -rf /tmp/tmp.39yPQ2cQ0t
++ mktemp
+ kubectl_bin delete --grace-period=0 --force=true namespace psmdb-operator
++ mktemp
+ local LAST_OUT=/tmp/tmp.DSCwhk1yMZ
++ mktemp
+ local LAST_OUT=/tmp/tmp.9tG2p7qBHa
++ mktemp
+ local LAST_ERR=/tmp/tmp.rD8dJn5vTv
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ local LAST_ERR=/tmp/tmp.pBf7QmlUcw
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl delete --grace-period=0 --force=true namespace monitoring-2-0-16090
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl delete --grace-period=0 --force=true namespace psmdb-operator
+ desc 'test passed'
+ set +o xtrace


-----------------------------------------------------------------------------------
test passed
-----------------------------------------------------------------------------------