Log: /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1584/e2e-tests/logs/monitoring-2-0.log
WARNING: version difference between client (1.30) and server (1.26) exceeds the supported minor version skew of +/-1
WARNING: version difference between client (1.30) and server (1.26) exceeds the supported minor version skew of +/-1
WARNING: version difference between client (1.30) and server (1.26) exceeds the supported minor version skew of +/-1
+ create_infra monitoring-2-0-31670
+ local ns=monitoring-2-0-31670
+ delete_crd
+ desc 'get and delete old CRDs and RBAC'
+ set +o xtrace


-----------------------------------------------------------------------------------
get and delete old CRDs and RBAC
-----------------------------------------------------------------------------------

+ kubectl_bin delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1584/deploy/crd.yaml --ignore-not-found --wait=false
++ mktemp
+ local LAST_OUT=/tmp/tmp.oRn2xkwQXa
++ mktemp
+ local LAST_ERR=/tmp/tmp.YvW9OgYDuq
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1584/deploy/crd.yaml --ignore-not-found --wait=false
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.oRn2xkwQXa
customresourcedefinition.apiextensions.k8s.io "perconaservermongodbbackups.psmdb.percona.com" deleted
customresourcedefinition.apiextensions.k8s.io "perconaservermongodbrestores.psmdb.percona.com" deleted
customresourcedefinition.apiextensions.k8s.io "perconaservermongodbs.psmdb.percona.com" deleted
+ cat /tmp/tmp.YvW9OgYDuq
+ rm /tmp/tmp.oRn2xkwQXa /tmp/tmp.YvW9OgYDuq
+ return 0
++ yq eval .metadata.name /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1584/deploy/crd.yaml
++ grep -v '\-\-\-'
+ for crd_name in '$(yq eval '\''.metadata.name'\'' "${src_dir}/deploy/crd.yaml" | grep -v '\''\-\-\-'\'')'
+ kubectl get perconaservermongodbbackups.psmdb.percona.com --all-namespaces -o wide
+ grep -v NAMESPACE
+ xargs -L 1 sh -xc 'kubectl patch perconaservermongodbbackups.psmdb.percona.com -n $0 $1 --type=merge -p "{\"metadata\":{\"finalizers\":[]}}"'
error: the server doesn't have a resource type "perconaservermongodbbackups"
+ kubectl patch perconaservermongodbbackups.psmdb.percona.com -n sh --type=merge -p '{"metadata":{"finalizers":[]}}'
error: the server doesn't have a resource type "perconaservermongodbbackups"
+ :
+ kubectl_bin wait --for=delete crd perconaservermongodbbackups.psmdb.percona.com
++ mktemp
+ local LAST_OUT=/tmp/tmp.dMzvirPWNL
++ mktemp
+ local LAST_ERR=/tmp/tmp.YgVdvPnftO
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl wait --for=delete crd perconaservermongodbbackups.psmdb.percona.com
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.dMzvirPWNL
+ cat /tmp/tmp.YgVdvPnftO
+ rm /tmp/tmp.dMzvirPWNL /tmp/tmp.YgVdvPnftO
+ return 0
+ for crd_name in '$(yq eval '\''.metadata.name'\'' "${src_dir}/deploy/crd.yaml" | grep -v '\''\-\-\-'\'')'
+ kubectl get perconaservermongodbrestores.psmdb.percona.com --all-namespaces -o wide
+ grep -v NAMESPACE
+ xargs -L 1 sh -xc 'kubectl patch perconaservermongodbrestores.psmdb.percona.com -n $0 $1 --type=merge -p "{\"metadata\":{\"finalizers\":[]}}"'
error: the server doesn't have a resource type "perconaservermongodbrestores"
+ kubectl patch perconaservermongodbrestores.psmdb.percona.com -n sh --type=merge -p '{"metadata":{"finalizers":[]}}'
error: the server doesn't have a resource type "perconaservermongodbrestores"
+ :
+ kubectl_bin wait --for=delete crd perconaservermongodbrestores.psmdb.percona.com
++ mktemp
+ local LAST_OUT=/tmp/tmp.y5dmuCHIXq
++ mktemp
+ local LAST_ERR=/tmp/tmp.4wDSCSIBCr
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl wait --for=delete crd perconaservermongodbrestores.psmdb.percona.com
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.y5dmuCHIXq
+ cat /tmp/tmp.4wDSCSIBCr
+ rm /tmp/tmp.y5dmuCHIXq /tmp/tmp.4wDSCSIBCr
+ return 0
+ for crd_name in '$(yq eval '\''.metadata.name'\'' "${src_dir}/deploy/crd.yaml" | grep -v '\''\-\-\-'\'')'
+ kubectl get perconaservermongodbs.psmdb.percona.com --all-namespaces -o wide
+ grep -v NAMESPACE
+ xargs -L 1 sh -xc 'kubectl patch perconaservermongodbs.psmdb.percona.com -n $0 $1 --type=merge -p "{\"metadata\":{\"finalizers\":[]}}"'
error: the server doesn't have a resource type "perconaservermongodbs"
+ kubectl patch perconaservermongodbs.psmdb.percona.com -n sh --type=merge -p '{"metadata":{"finalizers":[]}}'
error: the server doesn't have a resource type "perconaservermongodbs"
+ :
+ kubectl_bin wait --for=delete crd perconaservermongodbs.psmdb.percona.com
++ mktemp
+ local LAST_OUT=/tmp/tmp.eRJRthfS5Q
++ mktemp
+ local LAST_ERR=/tmp/tmp.GenM8pNI1s
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl wait --for=delete crd perconaservermongodbs.psmdb.percona.com
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.eRJRthfS5Q
+ cat /tmp/tmp.GenM8pNI1s
+ rm /tmp/tmp.eRJRthfS5Q /tmp/tmp.GenM8pNI1s
+ return 0
+ local rbac_yaml=rbac.yaml
+ '[' -n psmdb-operator ']'
+ rbac_yaml=cw-rbac.yaml
+ kubectl_bin delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1584/deploy/cw-rbac.yaml --ignore-not-found
++ mktemp
+ local LAST_OUT=/tmp/tmp.AEUVIWrtwb
++ mktemp
+ local LAST_ERR=/tmp/tmp.uObm1wZftT
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1584/deploy/cw-rbac.yaml --ignore-not-found
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.AEUVIWrtwb
clusterrole.rbac.authorization.k8s.io "percona-server-mongodb-operator" deleted
clusterrolebinding.rbac.authorization.k8s.io "service-account-percona-server-mongodb-operator" deleted
+ cat /tmp/tmp.uObm1wZftT
+ rm /tmp/tmp.AEUVIWrtwb /tmp/tmp.uObm1wZftT
+ return 0
+ check_crd_for_deletion PR-1584-aa9cda44
+ local git_tag=PR-1584-aa9cda44
++ curl -s https://raw.githubusercontent.com/percona/percona-server-mongodb-operator/PR-1584-aa9cda44/deploy/crd.yaml
++ yq eval .metadata.name
++ /usr/bin/sed s/---//g
++ /usr/bin/sed ':a;N;$!ba;s/\n/ /g'
+ for crd_name in '$(curl -s https://raw.githubusercontent.com/percona/percona-server-mongodb-operator/${git_tag}/deploy/crd.yaml | yq eval '\''.metadata.name'\'' | $sed '\''s/---//g'\'' | $sed '\'':a;N;$!ba;s/\n/ /g'\'')'
++ kubectl_bin get crd/null -o 'jsonpath={.status.conditions[-1].type}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.zgPBTLdZ1l
+++ mktemp
++ local LAST_ERR=/tmp/tmp.0IAyOuzWdB
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get crd/null -o 'jsonpath={.status.conditions[-1].type}'
++ exit_status=1
++ set -e
++ '[' 1 '!=' 0 -a -n 1 ']'
++ cat /tmp/tmp.zgPBTLdZ1l
++ cat /tmp/tmp.0IAyOuzWdB
Error from server (NotFound): customresourcedefinitions.apiextensions.k8s.io "null" not found
++ sleep 0
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get crd/null -o 'jsonpath={.status.conditions[-1].type}'
++ exit_status=1
++ set -e
++ '[' 1 '!=' 0 -a -n 1 ']'
++ cat /tmp/tmp.zgPBTLdZ1l
++ cat /tmp/tmp.0IAyOuzWdB
Error from server (NotFound): customresourcedefinitions.apiextensions.k8s.io "null" not found
++ sleep 4
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get crd/null -o 'jsonpath={.status.conditions[-1].type}'
++ exit_status=1
++ set -e
++ '[' 1 '!=' 0 -a -n 1 ']'
++ cat /tmp/tmp.zgPBTLdZ1l
++ cat /tmp/tmp.0IAyOuzWdB
Error from server (NotFound): customresourcedefinitions.apiextensions.k8s.io "null" not found
++ sleep 8
++ cat /tmp/tmp.zgPBTLdZ1l
++ cat /tmp/tmp.0IAyOuzWdB
Error from server (NotFound): customresourcedefinitions.apiextensions.k8s.io "null" not found
++ rm /tmp/tmp.zgPBTLdZ1l /tmp/tmp.0IAyOuzWdB
++ return 1
+ [[ '' == \T\e\r\m\i\n\a\t\i\n\g ]]
+ '[' -n psmdb-operator ']'
+ create_namespace psmdb-operator
+ local namespace=psmdb-operator
+ local skip_clean_namespace=
+ [[ 1 == 1 ]]
+ [[ -z '' ]]
+ destroy_chaos_mesh
++ helm list --all-namespaces --filter chaos-mesh
++ tail -n1
++ sed s/NAMESPACE//
++ awk '-F ' '{print $2}'
+ local chaos_mesh_ns=
+ desc 'destroy chaos-mesh'
+ set +o xtrace


-----------------------------------------------------------------------------------
destroy chaos-mesh
-----------------------------------------------------------------------------------

+ '[' -n '' ']'
++ kubectl get MutatingWebhookConfiguration
++ grep chaos-mesh
++ awk '{print $1}'
+ timeout 30 kubectl delete MutatingWebhookConfiguration
error: resource(s) were provided, but no name was specified
+ :
++ kubectl get ValidatingWebhookConfiguration
++ grep chaos-mesh
++ awk '{print $1}'
+ timeout 30 kubectl delete ValidatingWebhookConfiguration
error: resource(s) were provided, but no name was specified
+ :
++ kubectl get ValidatingWebhookConfiguration
++ grep validate-auth
++ awk '{print $1}'
+ timeout 30 kubectl delete ValidatingWebhookConfiguration
error: resource(s) were provided, but no name was specified
+ :
++ kubectl api-resources
++ grep chaos-mesh
++ awk '{print $1}'
++ kubectl get crd
++ awk '{print $1}'
++ grep chaos-mesh.org
+ timeout 30 kubectl delete crd
error: resource(s) were provided, but no name was specified
+ :
++ kubectl get clusterrolebinding
++ grep chaos-mesh
++ awk '{print $1}'
+ timeout 30 kubectl delete clusterrolebinding
error: resource(s) were provided, but no name was specified
+ :
++ kubectl get clusterrole
++ grep chaos-mesh
++ awk '{print $1}'
+ timeout 30 kubectl delete clusterrole
error: resource(s) were provided, but no name was specified
+ :
+ desc 'cleaned up all old namespaces'
+ set +o xtrace


-----------------------------------------------------------------------------------
cleaned up all old namespaces
-----------------------------------------------------------------------------------

+ kubectl_bin get ns
+ egrep -v '^kube-|^default|Terminating|psmdb-operator|openshift|gke-mcs|^NAME'
+ '[' -n '' ']'
+ desc 'cleaned up old namespaces psmdb-operator'
+ set +o xtrace


-----------------------------------------------------------------------------------
cleaned up old namespaces psmdb-operator
-----------------------------------------------------------------------------------

+ kubectl_bin delete namespace psmdb-operator --ignore-not-found
+ awk '{print$1}'
+ xargs kubectl delete ns
++ mktemp
+ local LAST_OUT=/tmp/tmp.ORSjW8AjFo
++ mktemp
+ local LAST_ERR=/tmp/tmp.RPYhwkymUa
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl delete namespace psmdb-operator --ignore-not-found
++ mktemp
+ local LAST_OUT=/tmp/tmp.zyZqjuu7of
++ mktemp
+ local LAST_ERR=/tmp/tmp.mRu6uIglPN
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl get ns
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.zyZqjuu7of
+ cat /tmp/tmp.mRu6uIglPN
+ rm /tmp/tmp.zyZqjuu7of /tmp/tmp.mRu6uIglPN
+ return 0
namespace "cert-manager" deleted
namespace "monitoring-2-0-21665" deleted
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.ORSjW8AjFo
namespace "psmdb-operator" deleted
+ cat /tmp/tmp.RPYhwkymUa
+ rm /tmp/tmp.ORSjW8AjFo /tmp/tmp.RPYhwkymUa
+ return 0
+ kubectl_bin wait --for=delete namespace psmdb-operator
++ mktemp
+ local LAST_OUT=/tmp/tmp.OcQUxhmJZj
++ mktemp
+ local LAST_ERR=/tmp/tmp.DP4d9shb8T
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl wait --for=delete namespace psmdb-operator
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.OcQUxhmJZj
+ cat /tmp/tmp.DP4d9shb8T
+ rm /tmp/tmp.OcQUxhmJZj /tmp/tmp.DP4d9shb8T
+ return 0
+ desc 'create namespace psmdb-operator'
+ set +o xtrace


-----------------------------------------------------------------------------------
create namespace psmdb-operator
-----------------------------------------------------------------------------------

+ kubectl_bin create namespace psmdb-operator
++ mktemp
+ local LAST_OUT=/tmp/tmp.uZeiCE570r
++ mktemp
+ local LAST_ERR=/tmp/tmp.iI7D1HdAIj
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl create namespace psmdb-operator
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.uZeiCE570r
namespace/psmdb-operator created
+ cat /tmp/tmp.iI7D1HdAIj
+ rm /tmp/tmp.uZeiCE570r /tmp/tmp.iI7D1HdAIj
+ return 0
++ kubectl_bin config current-context
+++ mktemp
++ local LAST_OUT=/tmp/tmp.qCxo6ULaHJ
+++ mktemp
++ local LAST_ERR=/tmp/tmp.Nn08dddfjC
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl config current-context
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.qCxo6ULaHJ
++ cat /tmp/tmp.Nn08dddfjC
++ rm /tmp/tmp.qCxo6ULaHJ /tmp/tmp.Nn08dddfjC
++ return 0
+ kubectl_bin config set-context gke_cloud-dev-112233_us-central1-a_jen-psmdb-1584-aa9cda44-2-cluster1 --namespace=psmdb-operator
++ mktemp
+ local LAST_OUT=/tmp/tmp.MEL5GTpdNc
++ mktemp
+ local LAST_ERR=/tmp/tmp.9M5bwMDZZq
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl config set-context gke_cloud-dev-112233_us-central1-a_jen-psmdb-1584-aa9cda44-2-cluster1 --namespace=psmdb-operator
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.MEL5GTpdNc
Context "gke_cloud-dev-112233_us-central1-a_jen-psmdb-1584-aa9cda44-2-cluster1" modified.
+ cat /tmp/tmp.9M5bwMDZZq
+ rm /tmp/tmp.MEL5GTpdNc /tmp/tmp.9M5bwMDZZq
+ return 0
+ deploy_operator
+ desc 'start PSMDB operator'
+ set +o xtrace


-----------------------------------------------------------------------------------
start PSMDB operator
-----------------------------------------------------------------------------------

+ local cr_file
+ '[' -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1584/e2e-tests/monitoring-2-0/conf/crd.yaml ']'
+ cr_file=/mnt/jenkins/workspace/cloud-psmdb-operator_PR-1584/deploy/crd.yaml
+ kubectl_bin apply --server-side --force-conflicts -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1584/deploy/crd.yaml
++ mktemp
+ local LAST_OUT=/tmp/tmp.PBjjE20eCO
++ mktemp
+ local LAST_ERR=/tmp/tmp.FJNdiyxvix
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl apply --server-side --force-conflicts -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1584/deploy/crd.yaml
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.PBjjE20eCO
customresourcedefinition.apiextensions.k8s.io/perconaservermongodbbackups.psmdb.percona.com serverside-applied
customresourcedefinition.apiextensions.k8s.io/perconaservermongodbrestores.psmdb.percona.com serverside-applied
customresourcedefinition.apiextensions.k8s.io/perconaservermongodbs.psmdb.percona.com serverside-applied
+ cat /tmp/tmp.FJNdiyxvix
+ rm /tmp/tmp.PBjjE20eCO /tmp/tmp.FJNdiyxvix
+ return 0
+ '[' -n psmdb-operator ']'
+ apply_rbac cw-rbac
+ local operator_namespace=psmdb-operator
+ local rbac=cw-rbac
+ cat /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1584/deploy/cw-rbac.yaml
+ sed -e 's^namespace: .*^namespace: psmdb-operator^'
+ kubectl_bin apply -n psmdb-operator -f -
++ mktemp
+ local LAST_OUT=/tmp/tmp.K50blVeT6e
++ mktemp
+ local LAST_ERR=/tmp/tmp.LzE4Tadl9M
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl apply -n psmdb-operator -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.K50blVeT6e
clusterrole.rbac.authorization.k8s.io/percona-server-mongodb-operator created
serviceaccount/percona-server-mongodb-operator created
clusterrolebinding.rbac.authorization.k8s.io/service-account-percona-server-mongodb-operator created
+ cat /tmp/tmp.LzE4Tadl9M
+ rm /tmp/tmp.K50blVeT6e /tmp/tmp.LzE4Tadl9M
+ return 0
+ yq eval '
			(.spec.template.spec.containers[].image = "perconalab/percona-server-mongodb-operator:PR-1584-aa9cda44") |
			((.. | select(.[] == "DISABLE_TELEMETRY")) |= .value="true") |
			((.. | select(.[] == "LOG_LEVEL")) |= .value="DEBUG")' /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1584/deploy/cw-operator.yaml
+ kubectl_bin apply -f -
++ mktemp
+ local LAST_OUT=/tmp/tmp.eNfX7Bh045
++ mktemp
+ local LAST_ERR=/tmp/tmp.BCJT6A35Rv
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl apply -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.eNfX7Bh045
deployment.apps/percona-server-mongodb-operator created
+ cat /tmp/tmp.BCJT6A35Rv
+ rm /tmp/tmp.eNfX7Bh045 /tmp/tmp.BCJT6A35Rv
+ return 0
+ sleep 2
++ get_operator_pod
++ kubectl_bin get pods --selector=name=percona-server-mongodb-operator -o 'jsonpath={.items[].metadata.name}' -n psmdb-operator
+++ mktemp
++ local LAST_OUT=/tmp/tmp.UVZbTu6ilw
+++ mktemp
++ local LAST_ERR=/tmp/tmp.tg0KIg1tsZ
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=percona-server-mongodb-operator -o 'jsonpath={.items[].metadata.name}' -n psmdb-operator
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.UVZbTu6ilw
++ cat /tmp/tmp.tg0KIg1tsZ
++ rm /tmp/tmp.UVZbTu6ilw /tmp/tmp.tg0KIg1tsZ
++ return 0
+ wait_pod percona-server-mongodb-operator-7ff48667b4-j652f
+ local pod=percona-server-mongodb-operator-7ff48667b4-j652f
+ set +o xtrace
waiting for pod/percona-server-mongodb-operator-7ff48667b4-j652f to be ready.OK
+ create_namespace monitoring-2-0-31670
+ local namespace=monitoring-2-0-31670
+ local skip_clean_namespace=
+ [[ 1 == 1 ]]
+ [[ -z '' ]]
+ destroy_chaos_mesh
++ helm list --all-namespaces --filter chaos-mesh
++ tail -n1
++ sed s/NAMESPACE//
++ awk '-F ' '{print $2}'
+ local chaos_mesh_ns=
+ desc 'destroy chaos-mesh'
+ set +o xtrace


-----------------------------------------------------------------------------------
destroy chaos-mesh
-----------------------------------------------------------------------------------

+ '[' -n '' ']'
++ kubectl get MutatingWebhookConfiguration
++ grep chaos-mesh
++ awk '{print $1}'
+ timeout 30 kubectl delete MutatingWebhookConfiguration
error: resource(s) were provided, but no name was specified
+ :
++ kubectl get ValidatingWebhookConfiguration
++ grep chaos-mesh
++ awk '{print $1}'
+ timeout 30 kubectl delete ValidatingWebhookConfiguration
error: resource(s) were provided, but no name was specified
+ :
++ kubectl get ValidatingWebhookConfiguration
++ grep validate-auth
++ awk '{print $1}'
+ timeout 30 kubectl delete ValidatingWebhookConfiguration
error: resource(s) were provided, but no name was specified
+ :
++ kubectl api-resources
++ grep chaos-mesh
++ awk '{print $1}'
++ kubectl get crd
++ grep chaos-mesh.org
++ awk '{print $1}'
+ timeout 30 kubectl delete crd
error: resource(s) were provided, but no name was specified
+ :
++ kubectl get clusterrolebinding
++ grep chaos-mesh
++ awk '{print $1}'
+ timeout 30 kubectl delete clusterrolebinding
error: resource(s) were provided, but no name was specified
+ :
++ kubectl get clusterrole
++ grep chaos-mesh
++ awk '{print $1}'
+ timeout 30 kubectl delete clusterrole
error: resource(s) were provided, but no name was specified
+ :
+ desc 'cleaned up all old namespaces'
+ set +o xtrace


-----------------------------------------------------------------------------------
cleaned up all old namespaces
-----------------------------------------------------------------------------------

+ kubectl_bin get ns
+ awk '{print$1}'
+ '[' -n '' ']'
+ desc 'cleaned up old namespaces monitoring-2-0-31670'
+ xargs kubectl delete ns
+ set +o xtrace


-----------------------------------------------------------------------------------
cleaned up old namespaces monitoring-2-0-31670
-----------------------------------------------------------------------------------

+ kubectl_bin delete namespace monitoring-2-0-31670 --ignore-not-found
++ mktemp
+ egrep -v '^kube-|^default|Terminating|psmdb-operator|openshift|gke-mcs|^NAME'
++ mktemp
+ local LAST_OUT=/tmp/tmp.ZX2rwb08bw
++ mktemp
+ local LAST_OUT=/tmp/tmp.9N6yRtNsKs
++ mktemp
+ local LAST_ERR=/tmp/tmp.QVA3poQ54t
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ local LAST_ERR=/tmp/tmp.DfboFVePaH
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl delete namespace monitoring-2-0-31670 --ignore-not-found
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl get ns
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.9N6yRtNsKs
+ cat /tmp/tmp.DfboFVePaH
+ rm /tmp/tmp.9N6yRtNsKs /tmp/tmp.DfboFVePaH
+ return 0
error: resource(s) were provided, but no name was specified
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.ZX2rwb08bw
+ cat /tmp/tmp.QVA3poQ54t
+ rm /tmp/tmp.ZX2rwb08bw /tmp/tmp.QVA3poQ54t
+ return 0
+ kubectl_bin wait --for=delete namespace monitoring-2-0-31670
++ mktemp
+ local LAST_OUT=/tmp/tmp.lkPZDmeuDH
++ mktemp
+ local LAST_ERR=/tmp/tmp.OAPpsQQkhS
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl wait --for=delete namespace monitoring-2-0-31670
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.lkPZDmeuDH
+ cat /tmp/tmp.OAPpsQQkhS
+ rm /tmp/tmp.lkPZDmeuDH /tmp/tmp.OAPpsQQkhS
+ return 0
+ desc 'create namespace monitoring-2-0-31670'
+ set +o xtrace


-----------------------------------------------------------------------------------
create namespace monitoring-2-0-31670
-----------------------------------------------------------------------------------

+ kubectl_bin create namespace monitoring-2-0-31670
++ mktemp
+ local LAST_OUT=/tmp/tmp.Wo6VMJHUoa
++ mktemp
+ local LAST_ERR=/tmp/tmp.fFdUOxI69U
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl create namespace monitoring-2-0-31670
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.Wo6VMJHUoa
namespace/monitoring-2-0-31670 created
+ cat /tmp/tmp.fFdUOxI69U
+ rm /tmp/tmp.Wo6VMJHUoa /tmp/tmp.fFdUOxI69U
+ return 0
++ kubectl_bin config current-context
+++ mktemp
++ local LAST_OUT=/tmp/tmp.P3E4fbmggL
+++ mktemp
++ local LAST_ERR=/tmp/tmp.YSS1VvUIE6
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl config current-context
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.P3E4fbmggL
++ cat /tmp/tmp.YSS1VvUIE6
++ rm /tmp/tmp.P3E4fbmggL /tmp/tmp.YSS1VvUIE6
++ return 0
+ kubectl_bin config set-context gke_cloud-dev-112233_us-central1-a_jen-psmdb-1584-aa9cda44-2-cluster1 --namespace=monitoring-2-0-31670
++ mktemp
+ local LAST_OUT=/tmp/tmp.HWFgdtkf1v
++ mktemp
+ local LAST_ERR=/tmp/tmp.Bs9oUX2y74
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl config set-context gke_cloud-dev-112233_us-central1-a_jen-psmdb-1584-aa9cda44-2-cluster1 --namespace=monitoring-2-0-31670
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.HWFgdtkf1v
Context "gke_cloud-dev-112233_us-central1-a_jen-psmdb-1584-aa9cda44-2-cluster1" modified.
+ cat /tmp/tmp.Bs9oUX2y74
+ rm /tmp/tmp.HWFgdtkf1v /tmp/tmp.Bs9oUX2y74
+ return 0
+ deploy_cert_manager
+ desc 'deploy cert manager'
+ set +o xtrace


-----------------------------------------------------------------------------------
deploy cert manager
-----------------------------------------------------------------------------------

+ kubectl_bin create namespace cert-manager
++ mktemp
+ local LAST_OUT=/tmp/tmp.NuKOgSZyvo
++ mktemp
+ local LAST_ERR=/tmp/tmp.qKl9Z0anLM
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl create namespace cert-manager
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.NuKOgSZyvo
namespace/cert-manager created
+ cat /tmp/tmp.qKl9Z0anLM
+ rm /tmp/tmp.NuKOgSZyvo /tmp/tmp.qKl9Z0anLM
+ return 0
+ kubectl_bin label namespace cert-manager certmanager.k8s.io/disable-validation=true
++ mktemp
+ local LAST_OUT=/tmp/tmp.KP5ohmPz54
++ mktemp
+ local LAST_ERR=/tmp/tmp.hmBJlHwLcb
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl label namespace cert-manager certmanager.k8s.io/disable-validation=true
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.KP5ohmPz54
namespace/cert-manager labeled
+ cat /tmp/tmp.hmBJlHwLcb
+ rm /tmp/tmp.KP5ohmPz54 /tmp/tmp.hmBJlHwLcb
+ return 0
+ kubectl_bin apply -f https://github.com/cert-manager/cert-manager/releases/download/v1.14.5/cert-manager.yaml --validate=false
++ mktemp
+ local LAST_OUT=/tmp/tmp.ucszjyakY9
++ mktemp
+ local LAST_ERR=/tmp/tmp.CyiIxf5Nnw
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl apply -f https://github.com/cert-manager/cert-manager/releases/download/v1.14.5/cert-manager.yaml --validate=false
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.ucszjyakY9
namespace/cert-manager configured
customresourcedefinition.apiextensions.k8s.io/certificaterequests.cert-manager.io unchanged
customresourcedefinition.apiextensions.k8s.io/certificates.cert-manager.io unchanged
customresourcedefinition.apiextensions.k8s.io/challenges.acme.cert-manager.io unchanged
customresourcedefinition.apiextensions.k8s.io/clusterissuers.cert-manager.io unchanged
customresourcedefinition.apiextensions.k8s.io/issuers.cert-manager.io unchanged
customresourcedefinition.apiextensions.k8s.io/orders.acme.cert-manager.io unchanged
serviceaccount/cert-manager-cainjector created
serviceaccount/cert-manager created
serviceaccount/cert-manager-webhook created
clusterrole.rbac.authorization.k8s.io/cert-manager-cainjector unchanged
clusterrole.rbac.authorization.k8s.io/cert-manager-controller-issuers unchanged
clusterrole.rbac.authorization.k8s.io/cert-manager-controller-clusterissuers unchanged
clusterrole.rbac.authorization.k8s.io/cert-manager-controller-certificates unchanged
clusterrole.rbac.authorization.k8s.io/cert-manager-controller-orders unchanged
clusterrole.rbac.authorization.k8s.io/cert-manager-controller-challenges unchanged
clusterrole.rbac.authorization.k8s.io/cert-manager-controller-ingress-shim unchanged
clusterrole.rbac.authorization.k8s.io/cert-manager-cluster-view unchanged
clusterrole.rbac.authorization.k8s.io/cert-manager-view unchanged
clusterrole.rbac.authorization.k8s.io/cert-manager-edit unchanged
clusterrole.rbac.authorization.k8s.io/cert-manager-controller-approve:cert-manager-io unchanged
clusterrole.rbac.authorization.k8s.io/cert-manager-controller-certificatesigningrequests unchanged
clusterrole.rbac.authorization.k8s.io/cert-manager-webhook:subjectaccessreviews unchanged
clusterrolebinding.rbac.authorization.k8s.io/cert-manager-cainjector unchanged
clusterrolebinding.rbac.authorization.k8s.io/cert-manager-controller-issuers unchanged
clusterrolebinding.rbac.authorization.k8s.io/cert-manager-controller-clusterissuers unchanged
clusterrolebinding.rbac.authorization.k8s.io/cert-manager-controller-certificates unchanged
clusterrolebinding.rbac.authorization.k8s.io/cert-manager-controller-orders unchanged
clusterrolebinding.rbac.authorization.k8s.io/cert-manager-controller-challenges unchanged
clusterrolebinding.rbac.authorization.k8s.io/cert-manager-controller-ingress-shim unchanged
clusterrolebinding.rbac.authorization.k8s.io/cert-manager-controller-approve:cert-manager-io unchanged
clusterrolebinding.rbac.authorization.k8s.io/cert-manager-controller-certificatesigningrequests unchanged
clusterrolebinding.rbac.authorization.k8s.io/cert-manager-webhook:subjectaccessreviews configured
role.rbac.authorization.k8s.io/cert-manager-cainjector:leaderelection unchanged
role.rbac.authorization.k8s.io/cert-manager:leaderelection unchanged
role.rbac.authorization.k8s.io/cert-manager-webhook:dynamic-serving created
rolebinding.rbac.authorization.k8s.io/cert-manager-cainjector:leaderelection unchanged
rolebinding.rbac.authorization.k8s.io/cert-manager:leaderelection configured
rolebinding.rbac.authorization.k8s.io/cert-manager-webhook:dynamic-serving created
service/cert-manager created
service/cert-manager-webhook created
deployment.apps/cert-manager-cainjector created
deployment.apps/cert-manager created
deployment.apps/cert-manager-webhook created
mutatingwebhookconfiguration.admissionregistration.k8s.io/cert-manager-webhook configured
validatingwebhookconfiguration.admissionregistration.k8s.io/cert-manager-webhook configured
+ cat /tmp/tmp.CyiIxf5Nnw
Warning: resource namespaces/cert-manager is missing the kubectl.kubernetes.io/last-applied-configuration annotation which is required by kubectl apply. kubectl apply should only be used on resources created declaratively by either kubectl create --save-config or kubectl apply. The missing annotation will be patched automatically.
+ rm /tmp/tmp.ucszjyakY9 /tmp/tmp.CyiIxf5Nnw
+ return 0
+ kubectl_bin -n cert-manager wait pod -l app.kubernetes.io/instance=cert-manager --for=condition=ready
++ mktemp
+ local LAST_OUT=/tmp/tmp.Z1RUr20TlM
++ mktemp
+ local LAST_ERR=/tmp/tmp.sJ3jKpMgHZ
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl -n cert-manager wait pod -l app.kubernetes.io/instance=cert-manager --for=condition=ready
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.Z1RUr20TlM
pod/cert-manager-5658d944df-cthcp condition met
pod/cert-manager-cainjector-cb99ff845-mzl6p condition met
pod/cert-manager-webhook-7fd74b8dc7-kg4sk condition met
+ cat /tmp/tmp.sJ3jKpMgHZ
+ rm /tmp/tmp.Z1RUr20TlM /tmp/tmp.sJ3jKpMgHZ
+ return 0
+ sleep 120
+ desc 'install PMM Server'
+ set +o xtrace


-----------------------------------------------------------------------------------
install PMM Server
-----------------------------------------------------------------------------------

+ deploy_pmm_server
+ helm uninstall monitoring
Error: uninstall: Release not loaded: monitoring: release: not found
+ :
+ helm repo remove stable
"stable" has been removed from your repositories
+ helm repo add stable https://charts.helm.sh/stable
"stable" has been added to your repositories
+ [[ -n '' ]]
+ retry 10 60 helm install monitoring --set imageTag=dev-latest --set imageRepo=perconalab/pmm-server https://percona-charts.storage.googleapis.com/pmm-server-9.9.9.tgz
+ local max=10
+ local delay=60
+ shift 2
+ local n=1
+ helm install monitoring --set imageTag=dev-latest --set imageRepo=perconalab/pmm-server https://percona-charts.storage.googleapis.com/pmm-server-9.9.9.tgz
NAME: monitoring
LAST DEPLOYED: Sun Jul  7 10:00:23 2024
NAMESPACE: monitoring-2-0-31670
STATUS: deployed
REVISION: 1
TEST SUITE: None
NOTES:
PMM server can be accessed via HTTPS (port 443) on the following DNS name from within your cluster:

endpoint: https://monitoring-service.monitoring-2-0-31670.svc.cluster.local:443
login:    admin
password: admin
+ sleep 20
+ kubectl_bin exec monitoring-0 -- bash -c 'ls -l /proc/*/exe 2>/dev/null| grep postgres >/dev/null'
++ mktemp
+ local LAST_OUT=/tmp/tmp.a1HPENswTC
++ mktemp
+ local LAST_ERR=/tmp/tmp.PF4J0hb7Iv
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl exec monitoring-0 -- bash -c 'ls -l /proc/*/exe 2>/dev/null| grep postgres >/dev/null'
+ exit_status=1
+ set -e
+ '[' 1 '!=' 0 -a -n 1 ']'
+ cat /tmp/tmp.a1HPENswTC
+ cat /tmp/tmp.PF4J0hb7Iv
error: unable to upgrade connection: container not found ("monitoring")
+ sleep 0
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl exec monitoring-0 -- bash -c 'ls -l /proc/*/exe 2>/dev/null| grep postgres >/dev/null'
+ exit_status=1
+ set -e
+ '[' 1 '!=' 0 -a -n 1 ']'
+ cat /tmp/tmp.a1HPENswTC
+ cat /tmp/tmp.PF4J0hb7Iv
error: unable to upgrade connection: container not found ("monitoring")
+ sleep 4
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl exec monitoring-0 -- bash -c 'ls -l /proc/*/exe 2>/dev/null| grep postgres >/dev/null'
+ exit_status=1
+ set -e
+ '[' 1 '!=' 0 -a -n 1 ']'
+ cat /tmp/tmp.a1HPENswTC
+ cat /tmp/tmp.PF4J0hb7Iv
error: unable to upgrade connection: container not found ("monitoring")
+ sleep 8
+ cat /tmp/tmp.a1HPENswTC
+ cat /tmp/tmp.PF4J0hb7Iv
error: unable to upgrade connection: container not found ("monitoring")
+ rm /tmp/tmp.a1HPENswTC /tmp/tmp.PF4J0hb7Iv
+ return 1
+ echo 'Retry 0'
Retry 0
+ sleep 5
+ let retry+=1
+ '[' 1 -ge 20 ']'
+ kubectl_bin exec monitoring-0 -- bash -c 'ls -l /proc/*/exe 2>/dev/null| grep postgres >/dev/null'
++ mktemp
+ local LAST_OUT=/tmp/tmp.llPqZAQe2b
++ mktemp
+ local LAST_ERR=/tmp/tmp.84cuvUOvdm
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl exec monitoring-0 -- bash -c 'ls -l /proc/*/exe 2>/dev/null| grep postgres >/dev/null'
+ exit_status=1
+ set -e
+ '[' 1 '!=' 0 -a -n 1 ']'
+ cat /tmp/tmp.llPqZAQe2b
+ cat /tmp/tmp.84cuvUOvdm
error: unable to upgrade connection: container not found ("monitoring")
+ sleep 0
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl exec monitoring-0 -- bash -c 'ls -l /proc/*/exe 2>/dev/null| grep postgres >/dev/null'
+ exit_status=1
+ set -e
+ '[' 1 '!=' 0 -a -n 1 ']'
+ cat /tmp/tmp.llPqZAQe2b
+ cat /tmp/tmp.84cuvUOvdm
error: unable to upgrade connection: container not found ("monitoring")
+ sleep 4
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl exec monitoring-0 -- bash -c 'ls -l /proc/*/exe 2>/dev/null| grep postgres >/dev/null'
+ exit_status=1
+ set -e
+ '[' 1 '!=' 0 -a -n 1 ']'
+ cat /tmp/tmp.llPqZAQe2b
+ cat /tmp/tmp.84cuvUOvdm
error: unable to upgrade connection: container not found ("monitoring")
+ sleep 8
+ cat /tmp/tmp.llPqZAQe2b
+ cat /tmp/tmp.84cuvUOvdm
error: unable to upgrade connection: container not found ("monitoring")
+ rm /tmp/tmp.llPqZAQe2b /tmp/tmp.84cuvUOvdm
+ return 1
+ echo 'Retry 1'
Retry 1
+ sleep 5
+ let retry+=1
+ '[' 2 -ge 20 ']'
+ kubectl_bin exec monitoring-0 -- bash -c 'ls -l /proc/*/exe 2>/dev/null| grep postgres >/dev/null'
++ mktemp
+ local LAST_OUT=/tmp/tmp.GkIDRi7tTc
++ mktemp
+ local LAST_ERR=/tmp/tmp.xitldrTnvr
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl exec monitoring-0 -- bash -c 'ls -l /proc/*/exe 2>/dev/null| grep postgres >/dev/null'
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.GkIDRi7tTc
+ cat /tmp/tmp.xitldrTnvr
+ rm /tmp/tmp.GkIDRi7tTc /tmp/tmp.xitldrTnvr
+ return 0
+ cluster=monitoring
+ desc 'create secrets and start client'
+ set +o xtrace


-----------------------------------------------------------------------------------
create secrets and start client
-----------------------------------------------------------------------------------

+ kubectl_bin apply -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1584/e2e-tests/conf/secrets.yml -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1584/e2e-tests/monitoring-2-0/conf/secrets.yml
++ mktemp
+ local LAST_OUT=/tmp/tmp.QRIkzlpugU
++ mktemp
+ local LAST_ERR=/tmp/tmp.hi0DvBAAoA
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl apply -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1584/e2e-tests/conf/secrets.yml -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1584/e2e-tests/monitoring-2-0/conf/secrets.yml
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.QRIkzlpugU
secret/some-users created
secret/some-users unchanged
+ cat /tmp/tmp.hi0DvBAAoA
+ rm /tmp/tmp.QRIkzlpugU /tmp/tmp.hi0DvBAAoA
+ return 0
+ yq '.spec.template.spec.volumes[0].secret.secretName="monitoring-ssl"' /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1584/e2e-tests/conf/client_with_tls.yml
+ kubectl_bin apply -f -
++ mktemp
+ local LAST_OUT=/tmp/tmp.d1CTGBjsjf
++ mktemp
+ local LAST_ERR=/tmp/tmp.cPCEVvat5B
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl apply -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.d1CTGBjsjf
deployment.apps/psmdb-client created
+ cat /tmp/tmp.cPCEVvat5B
+ rm /tmp/tmp.d1CTGBjsjf /tmp/tmp.cPCEVvat5B
+ return 0
+ sleep 90
+ desc 'create first PSMDB cluster monitoring'
+ set +o xtrace


-----------------------------------------------------------------------------------
create first PSMDB cluster monitoring
-----------------------------------------------------------------------------------

+ apply_cluster /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1584/e2e-tests/monitoring-2-0/conf/monitoring-rs0.yml
+ '[' -z '' ']'
+ kubectl_bin apply -f -
+ cat_config /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1584/e2e-tests/monitoring-2-0/conf/monitoring-rs0.yml
+ cat /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1584/e2e-tests/monitoring-2-0/conf/monitoring-rs0.yml
+ yq eval '(.spec | select(.image == null)).image = "perconalab/percona-server-mongodb-operator:main-mongod7.0"'
++ mktemp
+ yq eval '(.spec | select(has("pmm"))).pmm.image = "perconalab/pmm-client:dev-latest"'
+ yq eval '(.spec | select(has("initImage"))).initImage = "perconalab/percona-server-mongodb-operator:PR-1584-aa9cda44"'
+ yq eval '(.spec | select(has("backup"))).backup.image = "perconalab/percona-server-mongodb-operator:main-backup"'
+ yq eval '.spec.upgradeOptions.apply="Never"'
+ local LAST_OUT=/tmp/tmp.PAamhNbUq9
++ mktemp
+ local LAST_ERR=/tmp/tmp.PtPc93vpYX
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl apply -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.PAamhNbUq9
perconaservermongodb.psmdb.percona.com/monitoring created
+ cat /tmp/tmp.PtPc93vpYX
+ rm /tmp/tmp.PAamhNbUq9 /tmp/tmp.PtPc93vpYX
+ return 0
+ wait_for_running monitoring-rs0 3
+ local name=monitoring-rs0
+ let last_pod=2
+ local check_cluster_readyness=true
+ set_debug
+ [[ 1 == 1 ]]
+ set -o xtrace
+ local rs_name=rs0
+ local cluster_name=monitoring
++ seq 0 2
+ for i in '$(seq 0 $last_pod)'
+ [[ 0 -eq 2 ]]
+ wait_pod monitoring-rs0-0
+ local pod=monitoring-rs0-0
+ set +o xtrace
waiting for pod/monitoring-rs0-0 to be ready...............OK
+ for i in '$(seq 0 $last_pod)'
+ [[ 1 -eq 2 ]]
+ wait_pod monitoring-rs0-1
+ local pod=monitoring-rs0-1
+ set +o xtrace
waiting for pod/monitoring-rs0-1 to be ready...........OK
+ for i in '$(seq 0 $last_pod)'
+ [[ 2 -eq 2 ]]
++ kubectl_bin get psmdb monitoring -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].arbiter.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.NTKlWspDnL
+++ mktemp
++ local LAST_ERR=/tmp/tmp.Ymq3NGxqQ7
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb monitoring -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].arbiter.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.NTKlWspDnL
++ cat /tmp/tmp.Ymq3NGxqQ7
++ rm /tmp/tmp.NTKlWspDnL /tmp/tmp.Ymq3NGxqQ7
++ return 0
+ [[ '' == \t\r\u\e ]]
+ wait_pod monitoring-rs0-2
+ local pod=monitoring-rs0-2
+ set +o xtrace
waiting for pod/monitoring-rs0-2 to be ready..............OK
++ kubectl_bin get psmdb monitoring -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].non_voting.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.GfOMGEgDqo
+++ mktemp
++ local LAST_ERR=/tmp/tmp.H8dkIwpTcf
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb monitoring -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].non_voting.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.GfOMGEgDqo
++ cat /tmp/tmp.H8dkIwpTcf
++ rm /tmp/tmp.GfOMGEgDqo /tmp/tmp.H8dkIwpTcf
++ return 0
+ [[ '' == \t\r\u\e ]]
+ sleep 10
+ [[ true == \t\r\u\e ]]
+ set +x
Waiting for cluster readyness.......................
+ desc 'check if pmm-client container is not enabled'
+ set +o xtrace


-----------------------------------------------------------------------------------
check if pmm-client container is not enabled
-----------------------------------------------------------------------------------

+ compare_kubectl statefulset/monitoring-rs0 -no-pmm
+ local resource=statefulset/monitoring-rs0
+ local postfix=-no-pmm
+ local expected_result=/mnt/jenkins/workspace/cloud-psmdb-operator_PR-1584/e2e-tests/monitoring-2-0/compare/statefulset_monitoring-rs0-no-pmm.yml
+ local new_result=/tmp/tmp.2rzkpQHbaz/statefulset_monitoring-rs0.yml
+ '[' -n '' -a -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1584/e2e-tests/monitoring-2-0/compare/statefulset_monitoring-rs0-no-pmm-oc.yml ']'
+ kubectl_bin get -o yaml statefulset/monitoring-rs0
++ mktemp
+ yq eval '
			del(.metadata.ownerReferences[].apiVersion) |
			del(.metadata.managedFields) |
			del(.. | select(has("creationTimestamp")).creationTimestamp) |
			del(.. | select(has("namespace")).namespace) |
			del(.. | select(has("uid")).uid) |
			del(.metadata.resourceVersion) |
			del(.spec.template.spec.containers[].env[] | select(.name == "NAMESPACE")) |
			del(.metadata.selfLink) |
			del(.metadata.annotations."cloud.google.com/neg") |
			del(.metadata.annotations."kubectl.kubernetes.io/last-applied-configuration") |
			del(.. | select(has("image")).image) |
			del(.. | select(has("clusterIP")).clusterIP) |
			del(.. | select(has("clusterIPs")).clusterIPs) |
			del(.. | select(has("dataSource")).dataSource) |
			del(.. | select(has("procMount")).procMount) |
			del(.. | select(has("storageClassName")).storageClassName) |
			del(.. | select(has("finalizers")).finalizers) |
			del(.. | select(has("kubernetes.io/pvc-protection"))."kubernetes.io/pvc-protection") |
			del(.. | select(has("volumeName")).volumeName) |
			del(.. | select(has("volume.beta.kubernetes.io/storage-provisioner"))."volume.beta.kubernetes.io/storage-provisioner") |
			del(.. | select(has("volume.kubernetes.io/storage-provisioner"))."volume.kubernetes.io/storage-provisioner") |
			del(.spec.volumeMode) |
			del(.. | select(has("volume.kubernetes.io/selected-node"))."volume.kubernetes.io/selected-node") |
			del(.. | select(has("percona.com/last-config-hash"))."percona.com/last-config-hash") |
			del(.. | select(has("percona.com/configuration-hash"))."percona.com/configuration-hash") |
			del(.. | select(has("percona.com/ssl-hash"))."percona.com/ssl-hash") |
			del(.. | select(has("percona.com/ssl-internal-hash"))."percona.com/ssl-internal-hash") |
			del(.spec.volumeClaimTemplates[].spec.volumeMode | select(. == "Filesystem")) |
			del(.. | select(has("healthCheckNodePort")).healthCheckNodePort) |
			del(.. | select(has("nodePort")).nodePort) |
			del(.status) |
			(.. | select(tag == "!!str")) |= sub("monitoring-2-0-31670", "NAME_SPACE") |
			del(.spec.volumeClaimTemplates[].apiVersion) |
			del(.spec.volumeClaimTemplates[].kind) |
			del(.spec.ipFamilies) |
			del(.spec.ipFamilyPolicy) |
			(.. | select(. == "extensions/v1beta1")) = "apps/v1" |
			(.. | select(. == "batch/v1beta1")) = "batch/v1" ' -
+ local LAST_OUT=/tmp/tmp.yxMyvD56TZ
++ mktemp
+ local LAST_ERR=/tmp/tmp.au4UGt40qC
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl get -o yaml statefulset/monitoring-rs0
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.yxMyvD56TZ
+ cat /tmp/tmp.au4UGt40qC
+ rm /tmp/tmp.yxMyvD56TZ /tmp/tmp.au4UGt40qC
+ return 0
+ yq -i eval 'del(.spec.persistentVolumeClaimRetentionPolicy)' /tmp/tmp.2rzkpQHbaz/statefulset_monitoring-rs0.yml
+ version_gt 1.22
++ echo '1.26 >= 1.22'
++ bc -l
+ '[' 1 -eq 1 ']'
+ return 0
+ yq -i eval 'del(.spec.internalTrafficPolicy)' /tmp/tmp.2rzkpQHbaz/statefulset_monitoring-rs0.yml
+ yq -i eval 'del(.spec.allocateLoadBalancerNodePorts)' /tmp/tmp.2rzkpQHbaz/statefulset_monitoring-rs0.yml
+ [[ /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1584/e2e-tests/monitoring-2-0/compare/statefulset_monitoring-rs0-no-pmm.yml == */cronjob* ]]
+ diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1584/e2e-tests/monitoring-2-0/compare/statefulset_monitoring-rs0-no-pmm.yml /tmp/tmp.2rzkpQHbaz/statefulset_monitoring-rs0.yml
+ sleep 10
+ run_mongos 'db.createUser({user:"myApp",pwd:"myPass",roles:[{db:"myApp",role:"readWrite"}]})' userAdmin:userAdmin123456@monitoring-mongos.monitoring-2-0-31670 '' '' '--tlsCertificateKeyFile /tmp/tls.pem --tlsCAFile /etc/mongodb-ssl/ca.crt --tls'
+ local 'command=db.createUser({user:"myApp",pwd:"myPass",roles:[{db:"myApp",role:"readWrite"}]})'
+ local uri=userAdmin:userAdmin123456@monitoring-mongos.monitoring-2-0-31670
+ local driver=mongodb
+ local suffix=.svc.cluster.local
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.8E2cPjiz5u
+++ mktemp
++ local LAST_ERR=/tmp/tmp.Rq71aKKb7v
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.8E2cPjiz5u
++ cat /tmp/tmp.Rq71aKKb7v
++ rm /tmp/tmp.8E2cPjiz5u /tmp/tmp.Rq71aKKb7v
++ return 0
+ local client_container=psmdb-client-6cd48df8b6-f6x7p
+ local 'mongo_flag=--tlsCertificateKeyFile /tmp/tls.pem --tlsCAFile /etc/mongodb-ssl/ca.crt --tls'
+ kubectl_bin exec psmdb-client-6cd48df8b6-f6x7p -- bash -c 'printf '\''db.createUser({user:"myApp",pwd:"myPass",roles:[{db:"myApp",role:"readWrite"}]})\n'\'' | mongo mongodb://userAdmin:userAdmin123456@monitoring-mongos.monitoring-2-0-31670.svc.cluster.local/admin --tlsCertificateKeyFile /tmp/tls.pem --tlsCAFile /etc/mongodb-ssl/ca.crt --tls'
++ mktemp
+ local LAST_OUT=/tmp/tmp.4IL0mUrPho
++ mktemp
+ local LAST_ERR=/tmp/tmp.HaPxVbzxKg
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl exec psmdb-client-6cd48df8b6-f6x7p -- bash -c 'printf '\''db.createUser({user:"myApp",pwd:"myPass",roles:[{db:"myApp",role:"readWrite"}]})\n'\'' | mongo mongodb://userAdmin:userAdmin123456@monitoring-mongos.monitoring-2-0-31670.svc.cluster.local/admin --tlsCertificateKeyFile /tmp/tls.pem --tlsCAFile /etc/mongodb-ssl/ca.crt --tls'
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.4IL0mUrPho
Percona Server for MongoDB shell version v4.4.29-28
connecting to: mongodb://monitoring-mongos.monitoring-2-0-31670.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb
{"t":{"$date":"2024-07-07T10:05:38.169Z"},"s":"I",  "c":"NETWORK",  "id":5490002, "ctx":"thread1","msg":"Started a new thread for the timer service"}
Implicit session: session { "id" : UUID("7829e5ba-01d7-4458-9160-bcec24997a91") }
Percona Server for MongoDB server version: v7.0.11-6
WARNING: shell and server versions do not match
Successfully added user: {
	"user" : "myApp",
	"roles" : [
		{
			"db" : "myApp",
			"role" : "readWrite"
		}
	]
}
bye
+ cat /tmp/tmp.HaPxVbzxKg
+ rm /tmp/tmp.4IL0mUrPho /tmp/tmp.HaPxVbzxKg
+ return 0
+ run_mongos 'sh.enableSharding("myApp")' clusterAdmin:clusterAdmin123456@monitoring-mongos.monitoring-2-0-31670 '' '' '--tlsCertificateKeyFile /tmp/tls.pem --tlsCAFile /etc/mongodb-ssl/ca.crt --tls'
+ local 'command=sh.enableSharding("myApp")'
+ local uri=clusterAdmin:clusterAdmin123456@monitoring-mongos.monitoring-2-0-31670
+ local driver=mongodb
+ local suffix=.svc.cluster.local
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.lvhqGkLZZ4
+++ mktemp
++ local LAST_ERR=/tmp/tmp.1HoqfJB7ve
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.lvhqGkLZZ4
++ cat /tmp/tmp.1HoqfJB7ve
++ rm /tmp/tmp.lvhqGkLZZ4 /tmp/tmp.1HoqfJB7ve
++ return 0
+ local client_container=psmdb-client-6cd48df8b6-f6x7p
+ local 'mongo_flag=--tlsCertificateKeyFile /tmp/tls.pem --tlsCAFile /etc/mongodb-ssl/ca.crt --tls'
+ kubectl_bin exec psmdb-client-6cd48df8b6-f6x7p -- bash -c 'printf '\''sh.enableSharding("myApp")\n'\'' | mongo mongodb://clusterAdmin:clusterAdmin123456@monitoring-mongos.monitoring-2-0-31670.svc.cluster.local/admin --tlsCertificateKeyFile /tmp/tls.pem --tlsCAFile /etc/mongodb-ssl/ca.crt --tls'
++ mktemp
+ local LAST_OUT=/tmp/tmp.3vJVDcrXqG
++ mktemp
+ local LAST_ERR=/tmp/tmp.04LTnZlRy9
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl exec psmdb-client-6cd48df8b6-f6x7p -- bash -c 'printf '\''sh.enableSharding("myApp")\n'\'' | mongo mongodb://clusterAdmin:clusterAdmin123456@monitoring-mongos.monitoring-2-0-31670.svc.cluster.local/admin --tlsCertificateKeyFile /tmp/tls.pem --tlsCAFile /etc/mongodb-ssl/ca.crt --tls'
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.3vJVDcrXqG
Percona Server for MongoDB shell version v4.4.29-28
connecting to: mongodb://monitoring-mongos.monitoring-2-0-31670.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb
{"t":{"$date":"2024-07-07T10:05:41.466Z"},"s":"I",  "c":"NETWORK",  "id":5490002, "ctx":"thread1","msg":"Started a new thread for the timer service"}
Implicit session: session { "id" : UUID("a1c55888-c01a-4d34-b8c7-b1db7e76b956") }
Percona Server for MongoDB server version: v7.0.11-6
WARNING: shell and server versions do not match
{
	"ok" : 1,
	"$clusterTime" : {
		"clusterTime" : Timestamp(1720346741, 8),
		"signature" : {
			"hash" : BinData(0,"5315QaggECCKlzqJ5SKgOI27kU0="),
			"keyId" : NumberLong("7388832629597929495")
		}
	},
	"operationTime" : Timestamp(1720346741, 2)
}
bye
+ cat /tmp/tmp.04LTnZlRy9
+ rm /tmp/tmp.3vJVDcrXqG /tmp/tmp.04LTnZlRy9
+ return 0
+ insert_data_mongos 100500 myApp '--tlsCertificateKeyFile /tmp/tls.pem --tlsCAFile /etc/mongodb-ssl/ca.crt --tls'
+ local data=100500
+ local db_name=myApp
+ local 'flags=--tlsCertificateKeyFile /tmp/tls.pem --tlsCAFile /etc/mongodb-ssl/ca.crt --tls'
+ run_mongos 'use myApp\n db.test.insert({ x: 100500 })' myApp:myPass@monitoring-mongos.monitoring-2-0-31670 '' '' '--tlsCertificateKeyFile /tmp/tls.pem --tlsCAFile /etc/mongodb-ssl/ca.crt --tls'
+ local 'command=use myApp\n db.test.insert({ x: 100500 })'
+ local uri=myApp:myPass@monitoring-mongos.monitoring-2-0-31670
+ local driver=mongodb
+ local suffix=.svc.cluster.local
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.kIuRMNqFp2
+++ mktemp
++ local LAST_ERR=/tmp/tmp.iZr6mtgoaT
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.kIuRMNqFp2
++ cat /tmp/tmp.iZr6mtgoaT
++ rm /tmp/tmp.kIuRMNqFp2 /tmp/tmp.iZr6mtgoaT
++ return 0
+ local client_container=psmdb-client-6cd48df8b6-f6x7p
+ local 'mongo_flag=--tlsCertificateKeyFile /tmp/tls.pem --tlsCAFile /etc/mongodb-ssl/ca.crt --tls'
+ kubectl_bin exec psmdb-client-6cd48df8b6-f6x7p -- bash -c 'printf '\''use myApp\n db.test.insert({ x: 100500 })\n'\'' | mongo mongodb://myApp:myPass@monitoring-mongos.monitoring-2-0-31670.svc.cluster.local/admin --tlsCertificateKeyFile /tmp/tls.pem --tlsCAFile /etc/mongodb-ssl/ca.crt --tls'
++ mktemp
+ local LAST_OUT=/tmp/tmp.V5KFtFGhdI
++ mktemp
+ local LAST_ERR=/tmp/tmp.Xh5SQPR91m
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl exec psmdb-client-6cd48df8b6-f6x7p -- bash -c 'printf '\''use myApp\n db.test.insert({ x: 100500 })\n'\'' | mongo mongodb://myApp:myPass@monitoring-mongos.monitoring-2-0-31670.svc.cluster.local/admin --tlsCertificateKeyFile /tmp/tls.pem --tlsCAFile /etc/mongodb-ssl/ca.crt --tls'
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.V5KFtFGhdI
Percona Server for MongoDB shell version v4.4.29-28
connecting to: mongodb://monitoring-mongos.monitoring-2-0-31670.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb
{"t":{"$date":"2024-07-07T10:05:44.524Z"},"s":"I",  "c":"NETWORK",  "id":5490002, "ctx":"thread1","msg":"Started a new thread for the timer service"}
Implicit session: session { "id" : UUID("2ece12cb-a9fc-4c6d-830b-3255aa764b1b") }
Percona Server for MongoDB server version: v7.0.11-6
WARNING: shell and server versions do not match
switched to db myApp
WriteResult({ "nInserted" : 1 })
bye
+ cat /tmp/tmp.Xh5SQPR91m
+ rm /tmp/tmp.V5KFtFGhdI /tmp/tmp.Xh5SQPR91m
+ return 0
+ insert_data_mongos 100600 myApp '--tlsCertificateKeyFile /tmp/tls.pem --tlsCAFile /etc/mongodb-ssl/ca.crt --tls'
+ local data=100600
+ local db_name=myApp
+ local 'flags=--tlsCertificateKeyFile /tmp/tls.pem --tlsCAFile /etc/mongodb-ssl/ca.crt --tls'
+ run_mongos 'use myApp\n db.test.insert({ x: 100600 })' myApp:myPass@monitoring-mongos.monitoring-2-0-31670 '' '' '--tlsCertificateKeyFile /tmp/tls.pem --tlsCAFile /etc/mongodb-ssl/ca.crt --tls'
+ local 'command=use myApp\n db.test.insert({ x: 100600 })'
+ local uri=myApp:myPass@monitoring-mongos.monitoring-2-0-31670
+ local driver=mongodb
+ local suffix=.svc.cluster.local
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.09u5rTlEG6
+++ mktemp
++ local LAST_ERR=/tmp/tmp.4Fzp5muuua
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.09u5rTlEG6
++ cat /tmp/tmp.4Fzp5muuua
++ rm /tmp/tmp.09u5rTlEG6 /tmp/tmp.4Fzp5muuua
++ return 0
+ local client_container=psmdb-client-6cd48df8b6-f6x7p
+ local 'mongo_flag=--tlsCertificateKeyFile /tmp/tls.pem --tlsCAFile /etc/mongodb-ssl/ca.crt --tls'
+ kubectl_bin exec psmdb-client-6cd48df8b6-f6x7p -- bash -c 'printf '\''use myApp\n db.test.insert({ x: 100600 })\n'\'' | mongo mongodb://myApp:myPass@monitoring-mongos.monitoring-2-0-31670.svc.cluster.local/admin --tlsCertificateKeyFile /tmp/tls.pem --tlsCAFile /etc/mongodb-ssl/ca.crt --tls'
++ mktemp
+ local LAST_OUT=/tmp/tmp.hmc1AQCamL
++ mktemp
+ local LAST_ERR=/tmp/tmp.SFcoymRluc
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl exec psmdb-client-6cd48df8b6-f6x7p -- bash -c 'printf '\''use myApp\n db.test.insert({ x: 100600 })\n'\'' | mongo mongodb://myApp:myPass@monitoring-mongos.monitoring-2-0-31670.svc.cluster.local/admin --tlsCertificateKeyFile /tmp/tls.pem --tlsCAFile /etc/mongodb-ssl/ca.crt --tls'
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.hmc1AQCamL
Percona Server for MongoDB shell version v4.4.29-28
connecting to: mongodb://monitoring-mongos.monitoring-2-0-31670.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb
{"t":{"$date":"2024-07-07T10:05:47.107Z"},"s":"I",  "c":"NETWORK",  "id":5490002, "ctx":"thread1","msg":"Started a new thread for the timer service"}
Implicit session: session { "id" : UUID("b7f70ba6-c160-4e37-9603-841386dbb96e") }
Percona Server for MongoDB server version: v7.0.11-6
WARNING: shell and server versions do not match
switched to db myApp
WriteResult({ "nInserted" : 1 })
bye
+ cat /tmp/tmp.SFcoymRluc
+ rm /tmp/tmp.hmc1AQCamL /tmp/tmp.SFcoymRluc
+ return 0
+ insert_data_mongos 100700 myApp '--tlsCertificateKeyFile /tmp/tls.pem --tlsCAFile /etc/mongodb-ssl/ca.crt --tls'
+ local data=100700
+ local db_name=myApp
+ local 'flags=--tlsCertificateKeyFile /tmp/tls.pem --tlsCAFile /etc/mongodb-ssl/ca.crt --tls'
+ run_mongos 'use myApp\n db.test.insert({ x: 100700 })' myApp:myPass@monitoring-mongos.monitoring-2-0-31670 '' '' '--tlsCertificateKeyFile /tmp/tls.pem --tlsCAFile /etc/mongodb-ssl/ca.crt --tls'
+ local 'command=use myApp\n db.test.insert({ x: 100700 })'
+ local uri=myApp:myPass@monitoring-mongos.monitoring-2-0-31670
+ local driver=mongodb
+ local suffix=.svc.cluster.local
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.T1m1QesvNO
+++ mktemp
++ local LAST_ERR=/tmp/tmp.TkDgmjTmMs
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.T1m1QesvNO
++ cat /tmp/tmp.TkDgmjTmMs
++ rm /tmp/tmp.T1m1QesvNO /tmp/tmp.TkDgmjTmMs
++ return 0
+ local client_container=psmdb-client-6cd48df8b6-f6x7p
+ local 'mongo_flag=--tlsCertificateKeyFile /tmp/tls.pem --tlsCAFile /etc/mongodb-ssl/ca.crt --tls'
+ kubectl_bin exec psmdb-client-6cd48df8b6-f6x7p -- bash -c 'printf '\''use myApp\n db.test.insert({ x: 100700 })\n'\'' | mongo mongodb://myApp:myPass@monitoring-mongos.monitoring-2-0-31670.svc.cluster.local/admin --tlsCertificateKeyFile /tmp/tls.pem --tlsCAFile /etc/mongodb-ssl/ca.crt --tls'
++ mktemp
+ local LAST_OUT=/tmp/tmp.QfQkbVpQSx
++ mktemp
+ local LAST_ERR=/tmp/tmp.kfeulEJJbG
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl exec psmdb-client-6cd48df8b6-f6x7p -- bash -c 'printf '\''use myApp\n db.test.insert({ x: 100700 })\n'\'' | mongo mongodb://myApp:myPass@monitoring-mongos.monitoring-2-0-31670.svc.cluster.local/admin --tlsCertificateKeyFile /tmp/tls.pem --tlsCAFile /etc/mongodb-ssl/ca.crt --tls'
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.QfQkbVpQSx
Percona Server for MongoDB shell version v4.4.29-28
connecting to: mongodb://monitoring-mongos.monitoring-2-0-31670.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb
{"t":{"$date":"2024-07-07T10:05:50.343Z"},"s":"I",  "c":"NETWORK",  "id":5490002, "ctx":"thread1","msg":"Started a new thread for the timer service"}
Implicit session: session { "id" : UUID("61e5f986-4855-4188-8c06-55bf78f934ff") }
Percona Server for MongoDB server version: v7.0.11-6
WARNING: shell and server versions do not match
switched to db myApp
WriteResult({ "nInserted" : 1 })
bye
+ cat /tmp/tmp.kfeulEJJbG
+ rm /tmp/tmp.QfQkbVpQSx /tmp/tmp.kfeulEJJbG
+ return 0
+ desc 'add PMM_SERVER_API_KEY for secret some-users'
+ set +o xtrace


-----------------------------------------------------------------------------------
add PMM_SERVER_API_KEY for secret some-users
-----------------------------------------------------------------------------------

++ jq .key
+++ get_service_endpoint monitoring-service
+++ local service=monitoring-service
++++ kubectl_bin get service/monitoring-service -o json
++++ jq '.status.loadBalancer.ingress[].hostname'
++++ sed -e 's/^"//; s/"$//;'
+++++ mktemp
++++ local LAST_OUT=/tmp/tmp.OlHbLfT93V
+++++ mktemp
++++ local LAST_ERR=/tmp/tmp.7ssZi1q070
++++ local exit_status=0
++++ local timeout=4
+++++ seq 0 2
++++ for i in '$(seq 0 2)'
++++ set +e
++++ kubectl get service/monitoring-service -o json
++++ exit_status=0
++++ set -e
++++ '[' 0 '!=' 0 -a -n 1 ']'
++++ break
++++ cat /tmp/tmp.OlHbLfT93V
++++ cat /tmp/tmp.7ssZi1q070
++++ rm /tmp/tmp.OlHbLfT93V /tmp/tmp.7ssZi1q070
++++ return 0
+++ local hostname=null
+++ '[' -n null -a null '!=' null ']'
++++ kubectl_bin get service/monitoring-service -o json
++++ jq '.status.loadBalancer.ingress[].ip'
++++ sed -e 's/^"//; s/"$//;'
+++++ mktemp
++++ local LAST_OUT=/tmp/tmp.tnatYKip5X
+++++ mktemp
++++ local LAST_ERR=/tmp/tmp.Xp8mPkeW6I
++++ local exit_status=0
++++ local timeout=4
+++++ seq 0 2
++++ for i in '$(seq 0 2)'
++++ set +e
++++ kubectl get service/monitoring-service -o json
++++ exit_status=0
++++ set -e
++++ '[' 0 '!=' 0 -a -n 1 ']'
++++ break
++++ cat /tmp/tmp.tnatYKip5X
++++ cat /tmp/tmp.Xp8mPkeW6I
++++ rm /tmp/tmp.tnatYKip5X /tmp/tmp.Xp8mPkeW6I
++++ return 0
+++ local ip=34.29.4.83
+++ '[' -n 34.29.4.83 -a 34.29.4.83 '!=' null ']'
+++ echo 34.29.4.83
+++ return
++ curl --insecure -X POST -H 'Content-Type: application/json' -d '{"name":"operator", "role": "Admin"}' https://admin:admin@34.29.4.83/graph/api/auth/keys
  % Total    % Received % Xferd  Average Speed   Time    Time     Time  Current
                                 Dload  Upload   Total   Spent    Left  Speed

  0     0    0     0    0     0      0      0 --:--:-- --:--:-- --:--:--     0
100   155  100   119  100    36    246     74 --:--:-- --:--:-- --:--:--   322
+ API_KEY='"eyJrIjoiTjBxNURvZDZYbjBsRlRubTExdzNURmhBZUswTFNDYzgiLCJuIjoib3BlcmF0b3IiLCJpZCI6MX0="'
+ kubectl_bin patch secret some-users --type merge --patch '{"stringData": {"PMM_SERVER_API_KEY": "eyJrIjoiTjBxNURvZDZYbjBsRlRubTExdzNURmhBZUswTFNDYzgiLCJuIjoib3BlcmF0b3IiLCJpZCI6MX0="}}'
++ mktemp
+ local LAST_OUT=/tmp/tmp.5WvhaStWC2
++ mktemp
+ local LAST_ERR=/tmp/tmp.pPvMEIF67q
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl patch secret some-users --type merge --patch '{"stringData": {"PMM_SERVER_API_KEY": "eyJrIjoiTjBxNURvZDZYbjBsRlRubTExdzNURmhBZUswTFNDYzgiLCJuIjoib3BlcmF0b3IiLCJpZCI6MX0="}}'
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.5WvhaStWC2
secret/some-users patched
+ cat /tmp/tmp.pPvMEIF67q
+ rm /tmp/tmp.5WvhaStWC2 /tmp/tmp.pPvMEIF67q
+ return 0
+ desc 'check if all 3 Pods started'
+ set +o xtrace


-----------------------------------------------------------------------------------
check if all 3 Pods started
-----------------------------------------------------------------------------------

+ wait_for_running monitoring-rs0 3
+ local name=monitoring-rs0
+ let last_pod=2
+ local check_cluster_readyness=true
+ set_debug
+ [[ 1 == 1 ]]
+ set -o xtrace
+ local rs_name=rs0
+ local cluster_name=monitoring
++ seq 0 2
+ for i in '$(seq 0 $last_pod)'
+ [[ 0 -eq 2 ]]
+ wait_pod monitoring-rs0-0
+ local pod=monitoring-rs0-0
+ set +o xtrace
waiting for pod/monitoring-rs0-0 to be ready.OK
+ for i in '$(seq 0 $last_pod)'
+ [[ 1 -eq 2 ]]
+ wait_pod monitoring-rs0-1
+ local pod=monitoring-rs0-1
+ set +o xtrace
waiting for pod/monitoring-rs0-1 to be ready.OK
+ for i in '$(seq 0 $last_pod)'
+ [[ 2 -eq 2 ]]
++ kubectl_bin get psmdb monitoring -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].arbiter.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.iF3BDjEqPU
+++ mktemp
++ local LAST_ERR=/tmp/tmp.iXnFNEWggK
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb monitoring -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].arbiter.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.iF3BDjEqPU
++ cat /tmp/tmp.iXnFNEWggK
++ rm /tmp/tmp.iF3BDjEqPU /tmp/tmp.iXnFNEWggK
++ return 0
+ [[ '' == \t\r\u\e ]]
+ wait_pod monitoring-rs0-2
+ local pod=monitoring-rs0-2
+ set +o xtrace
waiting for pod/monitoring-rs0-2 to be ready.OK
++ kubectl_bin get psmdb monitoring -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].non_voting.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.KAhvfeFoCL
+++ mktemp
++ local LAST_ERR=/tmp/tmp.ZidiBnMcLY
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb monitoring -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].non_voting.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.KAhvfeFoCL
++ cat /tmp/tmp.ZidiBnMcLY
++ rm /tmp/tmp.KAhvfeFoCL /tmp/tmp.ZidiBnMcLY
++ return 0
+ [[ '' == \t\r\u\e ]]
+ sleep 10
+ [[ true == \t\r\u\e ]]
+ set +x
Waiting for cluster readyness...............................................................................................
+ sleep 90
+ desc 'check if pmm-client container enabled'
+ set +o xtrace


-----------------------------------------------------------------------------------
check if pmm-client container enabled
-----------------------------------------------------------------------------------

+ compare_kubectl statefulset/monitoring-rs0
+ local resource=statefulset/monitoring-rs0
+ local postfix=
+ local expected_result=/mnt/jenkins/workspace/cloud-psmdb-operator_PR-1584/e2e-tests/monitoring-2-0/compare/statefulset_monitoring-rs0.yml
+ local new_result=/tmp/tmp.2rzkpQHbaz/statefulset_monitoring-rs0.yml
+ '[' -n '' -a -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1584/e2e-tests/monitoring-2-0/compare/statefulset_monitoring-rs0-oc.yml ']'
+ kubectl_bin get -o yaml statefulset/monitoring-rs0
+ yq eval '
			del(.metadata.ownerReferences[].apiVersion) |
			del(.metadata.managedFields) |
			del(.. | select(has("creationTimestamp")).creationTimestamp) |
			del(.. | select(has("namespace")).namespace) |
			del(.. | select(has("uid")).uid) |
			del(.metadata.resourceVersion) |
			del(.spec.template.spec.containers[].env[] | select(.name == "NAMESPACE")) |
			del(.metadata.selfLink) |
			del(.metadata.annotations."cloud.google.com/neg") |
			del(.metadata.annotations."kubectl.kubernetes.io/last-applied-configuration") |
			del(.. | select(has("image")).image) |
			del(.. | select(has("clusterIP")).clusterIP) |
			del(.. | select(has("clusterIPs")).clusterIPs) |
			del(.. | select(has("dataSource")).dataSource) |
			del(.. | select(has("procMount")).procMount) |
			del(.. | select(has("storageClassName")).storageClassName) |
			del(.. | select(has("finalizers")).finalizers) |
			del(.. | select(has("kubernetes.io/pvc-protection"))."kubernetes.io/pvc-protection") |
			del(.. | select(has("volumeName")).volumeName) |
			del(.. | select(has("volume.beta.kubernetes.io/storage-provisioner"))."volume.beta.kubernetes.io/storage-provisioner") |
			del(.. | select(has("volume.kubernetes.io/storage-provisioner"))."volume.kubernetes.io/storage-provisioner") |
			del(.spec.volumeMode) |
			del(.. | select(has("volume.kubernetes.io/selected-node"))."volume.kubernetes.io/selected-node") |
			del(.. | select(has("percona.com/last-config-hash"))."percona.com/last-config-hash") |
			del(.. | select(has("percona.com/configuration-hash"))."percona.com/configuration-hash") |
			del(.. | select(has("percona.com/ssl-hash"))."percona.com/ssl-hash") |
			del(.. | select(has("percona.com/ssl-internal-hash"))."percona.com/ssl-internal-hash") |
			del(.spec.volumeClaimTemplates[].spec.volumeMode | select(. == "Filesystem")) |
			del(.. | select(has("healthCheckNodePort")).healthCheckNodePort) |
			del(.. | select(has("nodePort")).nodePort) |
			del(.status) |
			(.. | select(tag == "!!str")) |= sub("monitoring-2-0-31670", "NAME_SPACE") |
			del(.spec.volumeClaimTemplates[].apiVersion) |
			del(.spec.volumeClaimTemplates[].kind) |
			del(.spec.ipFamilies) |
			del(.spec.ipFamilyPolicy) |
			(.. | select(. == "extensions/v1beta1")) = "apps/v1" |
			(.. | select(. == "batch/v1beta1")) = "batch/v1" ' -
++ mktemp
+ local LAST_OUT=/tmp/tmp.Uv5C6gjzJW
++ mktemp
+ local LAST_ERR=/tmp/tmp.Dr0l9vWQL0
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl get -o yaml statefulset/monitoring-rs0
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.Uv5C6gjzJW
+ cat /tmp/tmp.Dr0l9vWQL0
+ rm /tmp/tmp.Uv5C6gjzJW /tmp/tmp.Dr0l9vWQL0
+ return 0
+ yq -i eval 'del(.spec.persistentVolumeClaimRetentionPolicy)' /tmp/tmp.2rzkpQHbaz/statefulset_monitoring-rs0.yml
+ version_gt 1.22
++ echo '1.26 >= 1.22'
++ bc -l
+ '[' 1 -eq 1 ']'
+ return 0
+ yq -i eval 'del(.spec.internalTrafficPolicy)' /tmp/tmp.2rzkpQHbaz/statefulset_monitoring-rs0.yml
+ yq -i eval 'del(.spec.allocateLoadBalancerNodePorts)' /tmp/tmp.2rzkpQHbaz/statefulset_monitoring-rs0.yml
+ [[ /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1584/e2e-tests/monitoring-2-0/compare/statefulset_monitoring-rs0.yml == */cronjob* ]]
+ diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1584/e2e-tests/monitoring-2-0/compare/statefulset_monitoring-rs0.yml /tmp/tmp.2rzkpQHbaz/statefulset_monitoring-rs0.yml
+ compare_kubectl service/monitoring-rs0
+ local resource=service/monitoring-rs0
+ local postfix=
+ local expected_result=/mnt/jenkins/workspace/cloud-psmdb-operator_PR-1584/e2e-tests/monitoring-2-0/compare/service_monitoring-rs0.yml
+ local new_result=/tmp/tmp.2rzkpQHbaz/service_monitoring-rs0.yml
+ '[' -n '' -a -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1584/e2e-tests/monitoring-2-0/compare/service_monitoring-rs0-oc.yml ']'
+ kubectl_bin get -o yaml service/monitoring-rs0
++ mktemp
+ yq eval '
			del(.metadata.ownerReferences[].apiVersion) |
			del(.metadata.managedFields) |
			del(.. | select(has("creationTimestamp")).creationTimestamp) |
			del(.. | select(has("namespace")).namespace) |
			del(.. | select(has("uid")).uid) |
			del(.metadata.resourceVersion) |
			del(.spec.template.spec.containers[].env[] | select(.name == "NAMESPACE")) |
			del(.metadata.selfLink) |
			del(.metadata.annotations."cloud.google.com/neg") |
			del(.metadata.annotations."kubectl.kubernetes.io/last-applied-configuration") |
			del(.. | select(has("image")).image) |
			del(.. | select(has("clusterIP")).clusterIP) |
			del(.. | select(has("clusterIPs")).clusterIPs) |
			del(.. | select(has("dataSource")).dataSource) |
			del(.. | select(has("procMount")).procMount) |
			del(.. | select(has("storageClassName")).storageClassName) |
			del(.. | select(has("finalizers")).finalizers) |
			del(.. | select(has("kubernetes.io/pvc-protection"))."kubernetes.io/pvc-protection") |
			del(.. | select(has("volumeName")).volumeName) |
			del(.. | select(has("volume.beta.kubernetes.io/storage-provisioner"))."volume.beta.kubernetes.io/storage-provisioner") |
			del(.. | select(has("volume.kubernetes.io/storage-provisioner"))."volume.kubernetes.io/storage-provisioner") |
			del(.spec.volumeMode) |
			del(.. | select(has("volume.kubernetes.io/selected-node"))."volume.kubernetes.io/selected-node") |
			del(.. | select(has("percona.com/last-config-hash"))."percona.com/last-config-hash") |
			del(.. | select(has("percona.com/configuration-hash"))."percona.com/configuration-hash") |
			del(.. | select(has("percona.com/ssl-hash"))."percona.com/ssl-hash") |
			del(.. | select(has("percona.com/ssl-internal-hash"))."percona.com/ssl-internal-hash") |
			del(.spec.volumeClaimTemplates[].spec.volumeMode | select(. == "Filesystem")) |
			del(.. | select(has("healthCheckNodePort")).healthCheckNodePort) |
			del(.. | select(has("nodePort")).nodePort) |
			del(.status) |
			(.. | select(tag == "!!str")) |= sub("monitoring-2-0-31670", "NAME_SPACE") |
			del(.spec.volumeClaimTemplates[].apiVersion) |
			del(.spec.volumeClaimTemplates[].kind) |
			del(.spec.ipFamilies) |
			del(.spec.ipFamilyPolicy) |
			(.. | select(. == "extensions/v1beta1")) = "apps/v1" |
			(.. | select(. == "batch/v1beta1")) = "batch/v1" ' -
+ local LAST_OUT=/tmp/tmp.k5DDbWOmI3
++ mktemp
+ local LAST_ERR=/tmp/tmp.KKkbCmdKyu
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl get -o yaml service/monitoring-rs0
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.k5DDbWOmI3
+ cat /tmp/tmp.KKkbCmdKyu
+ rm /tmp/tmp.k5DDbWOmI3 /tmp/tmp.KKkbCmdKyu
+ return 0
+ yq -i eval 'del(.spec.persistentVolumeClaimRetentionPolicy)' /tmp/tmp.2rzkpQHbaz/service_monitoring-rs0.yml
+ version_gt 1.22
++ echo '1.26 >= 1.22'
++ bc -l
+ '[' 1 -eq 1 ']'
+ return 0
+ yq -i eval 'del(.spec.internalTrafficPolicy)' /tmp/tmp.2rzkpQHbaz/service_monitoring-rs0.yml
+ yq -i eval 'del(.spec.allocateLoadBalancerNodePorts)' /tmp/tmp.2rzkpQHbaz/service_monitoring-rs0.yml
+ [[ /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1584/e2e-tests/monitoring-2-0/compare/service_monitoring-rs0.yml == */cronjob* ]]
+ diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1584/e2e-tests/monitoring-2-0/compare/service_monitoring-rs0.yml /tmp/tmp.2rzkpQHbaz/service_monitoring-rs0.yml
+ compare_kubectl service/monitoring-mongos
+ local resource=service/monitoring-mongos
+ local postfix=
+ local expected_result=/mnt/jenkins/workspace/cloud-psmdb-operator_PR-1584/e2e-tests/monitoring-2-0/compare/service_monitoring-mongos.yml
+ local new_result=/tmp/tmp.2rzkpQHbaz/service_monitoring-mongos.yml
+ '[' -n '' -a -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1584/e2e-tests/monitoring-2-0/compare/service_monitoring-mongos-oc.yml ']'
+ kubectl_bin get -o yaml service/monitoring-mongos
+ yq eval '
			del(.metadata.ownerReferences[].apiVersion) |
			del(.metadata.managedFields) |
			del(.. | select(has("creationTimestamp")).creationTimestamp) |
			del(.. | select(has("namespace")).namespace) |
			del(.. | select(has("uid")).uid) |
			del(.metadata.resourceVersion) |
			del(.spec.template.spec.containers[].env[] | select(.name == "NAMESPACE")) |
			del(.metadata.selfLink) |
			del(.metadata.annotations."cloud.google.com/neg") |
			del(.metadata.annotations."kubectl.kubernetes.io/last-applied-configuration") |
			del(.. | select(has("image")).image) |
			del(.. | select(has("clusterIP")).clusterIP) |
			del(.. | select(has("clusterIPs")).clusterIPs) |
			del(.. | select(has("dataSource")).dataSource) |
			del(.. | select(has("procMount")).procMount) |
			del(.. | select(has("storageClassName")).storageClassName) |
			del(.. | select(has("finalizers")).finalizers) |
			del(.. | select(has("kubernetes.io/pvc-protection"))."kubernetes.io/pvc-protection") |
			del(.. | select(has("volumeName")).volumeName) |
			del(.. | select(has("volume.beta.kubernetes.io/storage-provisioner"))."volume.beta.kubernetes.io/storage-provisioner") |
			del(.. | select(has("volume.kubernetes.io/storage-provisioner"))."volume.kubernetes.io/storage-provisioner") |
			del(.spec.volumeMode) |
			del(.. | select(has("volume.kubernetes.io/selected-node"))."volume.kubernetes.io/selected-node") |
			del(.. | select(has("percona.com/last-config-hash"))."percona.com/last-config-hash") |
			del(.. | select(has("percona.com/configuration-hash"))."percona.com/configuration-hash") |
			del(.. | select(has("percona.com/ssl-hash"))."percona.com/ssl-hash") |
			del(.. | select(has("percona.com/ssl-internal-hash"))."percona.com/ssl-internal-hash") |
			del(.spec.volumeClaimTemplates[].spec.volumeMode | select(. == "Filesystem")) |
			del(.. | select(has("healthCheckNodePort")).healthCheckNodePort) |
			del(.. | select(has("nodePort")).nodePort) |
			del(.status) |
			(.. | select(tag == "!!str")) |= sub("monitoring-2-0-31670", "NAME_SPACE") |
			del(.spec.volumeClaimTemplates[].apiVersion) |
			del(.spec.volumeClaimTemplates[].kind) |
			del(.spec.ipFamilies) |
			del(.spec.ipFamilyPolicy) |
			(.. | select(. == "extensions/v1beta1")) = "apps/v1" |
			(.. | select(. == "batch/v1beta1")) = "batch/v1" ' -
++ mktemp
+ local LAST_OUT=/tmp/tmp.bWUSAtuf4O
++ mktemp
+ local LAST_ERR=/tmp/tmp.hd05Tpzxuj
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl get -o yaml service/monitoring-mongos
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.bWUSAtuf4O
+ cat /tmp/tmp.hd05Tpzxuj
+ rm /tmp/tmp.bWUSAtuf4O /tmp/tmp.hd05Tpzxuj
+ return 0
+ yq -i eval 'del(.spec.persistentVolumeClaimRetentionPolicy)' /tmp/tmp.2rzkpQHbaz/service_monitoring-mongos.yml
+ version_gt 1.22
++ echo '1.26 >= 1.22'
++ bc -l
+ '[' 1 -eq 1 ']'
+ return 0
+ yq -i eval 'del(.spec.internalTrafficPolicy)' /tmp/tmp.2rzkpQHbaz/service_monitoring-mongos.yml
+ yq -i eval 'del(.spec.allocateLoadBalancerNodePorts)' /tmp/tmp.2rzkpQHbaz/service_monitoring-mongos.yml
+ [[ /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1584/e2e-tests/monitoring-2-0/compare/service_monitoring-mongos.yml == */cronjob* ]]
+ diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1584/e2e-tests/monitoring-2-0/compare/service_monitoring-mongos.yml /tmp/tmp.2rzkpQHbaz/service_monitoring-mongos.yml
+ compare_kubectl statefulset/monitoring-cfg
+ local resource=statefulset/monitoring-cfg
+ local postfix=
+ local expected_result=/mnt/jenkins/workspace/cloud-psmdb-operator_PR-1584/e2e-tests/monitoring-2-0/compare/statefulset_monitoring-cfg.yml
+ local new_result=/tmp/tmp.2rzkpQHbaz/statefulset_monitoring-cfg.yml
+ '[' -n '' -a -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1584/e2e-tests/monitoring-2-0/compare/statefulset_monitoring-cfg-oc.yml ']'
+ kubectl_bin get -o yaml statefulset/monitoring-cfg
++ mktemp
+ yq eval '
			del(.metadata.ownerReferences[].apiVersion) |
			del(.metadata.managedFields) |
			del(.. | select(has("creationTimestamp")).creationTimestamp) |
			del(.. | select(has("namespace")).namespace) |
			del(.. | select(has("uid")).uid) |
			del(.metadata.resourceVersion) |
			del(.spec.template.spec.containers[].env[] | select(.name == "NAMESPACE")) |
			del(.metadata.selfLink) |
			del(.metadata.annotations."cloud.google.com/neg") |
			del(.metadata.annotations."kubectl.kubernetes.io/last-applied-configuration") |
			del(.. | select(has("image")).image) |
			del(.. | select(has("clusterIP")).clusterIP) |
			del(.. | select(has("clusterIPs")).clusterIPs) |
			del(.. | select(has("dataSource")).dataSource) |
			del(.. | select(has("procMount")).procMount) |
			del(.. | select(has("storageClassName")).storageClassName) |
			del(.. | select(has("finalizers")).finalizers) |
			del(.. | select(has("kubernetes.io/pvc-protection"))."kubernetes.io/pvc-protection") |
			del(.. | select(has("volumeName")).volumeName) |
			del(.. | select(has("volume.beta.kubernetes.io/storage-provisioner"))."volume.beta.kubernetes.io/storage-provisioner") |
			del(.. | select(has("volume.kubernetes.io/storage-provisioner"))."volume.kubernetes.io/storage-provisioner") |
			del(.spec.volumeMode) |
			del(.. | select(has("volume.kubernetes.io/selected-node"))."volume.kubernetes.io/selected-node") |
			del(.. | select(has("percona.com/last-config-hash"))."percona.com/last-config-hash") |
			del(.. | select(has("percona.com/configuration-hash"))."percona.com/configuration-hash") |
			del(.. | select(has("percona.com/ssl-hash"))."percona.com/ssl-hash") |
			del(.. | select(has("percona.com/ssl-internal-hash"))."percona.com/ssl-internal-hash") |
			del(.spec.volumeClaimTemplates[].spec.volumeMode | select(. == "Filesystem")) |
			del(.. | select(has("healthCheckNodePort")).healthCheckNodePort) |
			del(.. | select(has("nodePort")).nodePort) |
			del(.status) |
			(.. | select(tag == "!!str")) |= sub("monitoring-2-0-31670", "NAME_SPACE") |
			del(.spec.volumeClaimTemplates[].apiVersion) |
			del(.spec.volumeClaimTemplates[].kind) |
			del(.spec.ipFamilies) |
			del(.spec.ipFamilyPolicy) |
			(.. | select(. == "extensions/v1beta1")) = "apps/v1" |
			(.. | select(. == "batch/v1beta1")) = "batch/v1" ' -
+ local LAST_OUT=/tmp/tmp.5HlbsZdlea
++ mktemp
+ local LAST_ERR=/tmp/tmp.25hmhsgj90
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl get -o yaml statefulset/monitoring-cfg
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.5HlbsZdlea
+ cat /tmp/tmp.25hmhsgj90
+ rm /tmp/tmp.5HlbsZdlea /tmp/tmp.25hmhsgj90
+ return 0
+ yq -i eval 'del(.spec.persistentVolumeClaimRetentionPolicy)' /tmp/tmp.2rzkpQHbaz/statefulset_monitoring-cfg.yml
+ version_gt 1.22
++ echo '1.26 >= 1.22'
++ bc -l
+ '[' 1 -eq 1 ']'
+ return 0
+ yq -i eval 'del(.spec.internalTrafficPolicy)' /tmp/tmp.2rzkpQHbaz/statefulset_monitoring-cfg.yml
+ yq -i eval 'del(.spec.allocateLoadBalancerNodePorts)' /tmp/tmp.2rzkpQHbaz/statefulset_monitoring-cfg.yml
+ [[ /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1584/e2e-tests/monitoring-2-0/compare/statefulset_monitoring-cfg.yml == */cronjob* ]]
+ diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1584/e2e-tests/monitoring-2-0/compare/statefulset_monitoring-cfg.yml /tmp/tmp.2rzkpQHbaz/statefulset_monitoring-cfg.yml
+ compare_kubectl statefulset/monitoring-mongos
+ local resource=statefulset/monitoring-mongos
+ local postfix=
+ local expected_result=/mnt/jenkins/workspace/cloud-psmdb-operator_PR-1584/e2e-tests/monitoring-2-0/compare/statefulset_monitoring-mongos.yml
+ local new_result=/tmp/tmp.2rzkpQHbaz/statefulset_monitoring-mongos.yml
+ '[' -n '' -a -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1584/e2e-tests/monitoring-2-0/compare/statefulset_monitoring-mongos-oc.yml ']'
+ kubectl_bin get -o yaml statefulset/monitoring-mongos
+ yq eval '
			del(.metadata.ownerReferences[].apiVersion) |
			del(.metadata.managedFields) |
			del(.. | select(has("creationTimestamp")).creationTimestamp) |
			del(.. | select(has("namespace")).namespace) |
			del(.. | select(has("uid")).uid) |
			del(.metadata.resourceVersion) |
			del(.spec.template.spec.containers[].env[] | select(.name == "NAMESPACE")) |
			del(.metadata.selfLink) |
			del(.metadata.annotations."cloud.google.com/neg") |
			del(.metadata.annotations."kubectl.kubernetes.io/last-applied-configuration") |
			del(.. | select(has("image")).image) |
			del(.. | select(has("clusterIP")).clusterIP) |
			del(.. | select(has("clusterIPs")).clusterIPs) |
			del(.. | select(has("dataSource")).dataSource) |
			del(.. | select(has("procMount")).procMount) |
			del(.. | select(has("storageClassName")).storageClassName) |
			del(.. | select(has("finalizers")).finalizers) |
			del(.. | select(has("kubernetes.io/pvc-protection"))."kubernetes.io/pvc-protection") |
			del(.. | select(has("volumeName")).volumeName) |
			del(.. | select(has("volume.beta.kubernetes.io/storage-provisioner"))."volume.beta.kubernetes.io/storage-provisioner") |
			del(.. | select(has("volume.kubernetes.io/storage-provisioner"))."volume.kubernetes.io/storage-provisioner") |
			del(.spec.volumeMode) |
			del(.. | select(has("volume.kubernetes.io/selected-node"))."volume.kubernetes.io/selected-node") |
			del(.. | select(has("percona.com/last-config-hash"))."percona.com/last-config-hash") |
			del(.. | select(has("percona.com/configuration-hash"))."percona.com/configuration-hash") |
			del(.. | select(has("percona.com/ssl-hash"))."percona.com/ssl-hash") |
			del(.. | select(has("percona.com/ssl-internal-hash"))."percona.com/ssl-internal-hash") |
			del(.spec.volumeClaimTemplates[].spec.volumeMode | select(. == "Filesystem")) |
			del(.. | select(has("healthCheckNodePort")).healthCheckNodePort) |
			del(.. | select(has("nodePort")).nodePort) |
			del(.status) |
			(.. | select(tag == "!!str")) |= sub("monitoring-2-0-31670", "NAME_SPACE") |
			del(.spec.volumeClaimTemplates[].apiVersion) |
			del(.spec.volumeClaimTemplates[].kind) |
			del(.spec.ipFamilies) |
			del(.spec.ipFamilyPolicy) |
			(.. | select(. == "extensions/v1beta1")) = "apps/v1" |
			(.. | select(. == "batch/v1beta1")) = "batch/v1" ' -
++ mktemp
+ local LAST_OUT=/tmp/tmp.kWSX5YPgBL
++ mktemp
+ local LAST_ERR=/tmp/tmp.iAAxxyJgC2
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl get -o yaml statefulset/monitoring-mongos
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.kWSX5YPgBL
+ cat /tmp/tmp.iAAxxyJgC2
+ rm /tmp/tmp.kWSX5YPgBL /tmp/tmp.iAAxxyJgC2
+ return 0
+ yq -i eval 'del(.spec.persistentVolumeClaimRetentionPolicy)' /tmp/tmp.2rzkpQHbaz/statefulset_monitoring-mongos.yml
+ version_gt 1.22
++ echo '1.26 >= 1.22'
++ bc -l
+ '[' 1 -eq 1 ']'
+ return 0
+ yq -i eval 'del(.spec.internalTrafficPolicy)' /tmp/tmp.2rzkpQHbaz/statefulset_monitoring-mongos.yml
+ yq -i eval 'del(.spec.allocateLoadBalancerNodePorts)' /tmp/tmp.2rzkpQHbaz/statefulset_monitoring-mongos.yml
+ [[ /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1584/e2e-tests/monitoring-2-0/compare/statefulset_monitoring-mongos.yml == */cronjob* ]]
+ diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1584/e2e-tests/monitoring-2-0/compare/statefulset_monitoring-mongos.yml /tmp/tmp.2rzkpQHbaz/statefulset_monitoring-mongos.yml
+ desc 'check mongod metrics'
+ set +o xtrace


-----------------------------------------------------------------------------------
check mongod metrics
-----------------------------------------------------------------------------------

+ get_metric_values node_boot_time_seconds monitoring-2-0-31670-monitoring-rs0-1 admin:admin
+ local metric=node_boot_time_seconds
+ local instance=monitoring-2-0-31670-monitoring-rs0-1
+ local user_pass=admin:admin
++ /usr/bin/date -u +%s -d '-1 minute'
+ local start=1720346979
++ /usr/bin/date -u +%s
+ local end=1720347039
++ get_service_endpoint monitoring-service
++ local service=monitoring-service
+++ kubectl_bin get service/monitoring-service -o json
+++ jq '.status.loadBalancer.ingress[].hostname'
++++ mktemp
+++ sed -e 's/^"//; s/"$//;'
+++ local LAST_OUT=/tmp/tmp.4sel4zfUu2
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.fOGlb3cLJV
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in '$(seq 0 2)'
+++ set +e
+++ kubectl get service/monitoring-service -o json
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.4sel4zfUu2
+++ cat /tmp/tmp.fOGlb3cLJV
+++ rm /tmp/tmp.4sel4zfUu2 /tmp/tmp.fOGlb3cLJV
+++ return 0
++ local hostname=null
++ '[' -n null -a null '!=' null ']'
+++ kubectl_bin get service/monitoring-service -o json
+++ jq '.status.loadBalancer.ingress[].ip'
+++ sed -e 's/^"//; s/"$//;'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.qB2JALIK34
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.ZsZjGi4HmT
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in '$(seq 0 2)'
+++ set +e
+++ kubectl get service/monitoring-service -o json
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.qB2JALIK34
+++ cat /tmp/tmp.ZsZjGi4HmT
+++ rm /tmp/tmp.qB2JALIK34 /tmp/tmp.ZsZjGi4HmT
+++ return 0
++ local ip=34.29.4.83
++ '[' -n 34.29.4.83 -a 34.29.4.83 '!=' null ']'
++ echo 34.29.4.83
++ return
+ local endpoint=34.29.4.83
+ curl -s -k 'https://admin:admin@34.29.4.83/graph/api/datasources/proxy/1/api/v1/query_range?query=min%28node_boot_time_seconds%7Bnode_name%3D%7E%22monitoring-2-0-31670-monitoring-rs0-1%22%7d%20or%20node_boot_time_seconds%7Bnode_name%3D%7E%22monitoring-2-0-31670-monitoring-rs0-1%22%7D%29&start=1720346979&end=1720347039&step=60'
+ jq '.data.result[0].values[][1]'
+ grep '^"[0-9]'
"1720345101"
"1720345101"
+ get_metric_values mongodb_connections monitoring-2-0-31670-monitoring-rs0-1 admin:admin
+ local metric=mongodb_connections
+ local instance=monitoring-2-0-31670-monitoring-rs0-1
+ local user_pass=admin:admin
++ /usr/bin/date -u +%s -d '-1 minute'
+ local start=1720346981
++ /usr/bin/date -u +%s
+ local end=1720347041
++ get_service_endpoint monitoring-service
++ local service=monitoring-service
+++ kubectl_bin get service/monitoring-service -o json
+++ jq '.status.loadBalancer.ingress[].hostname'
+++ sed -e 's/^"//; s/"$//;'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.IyZs4dLr3C
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.YwmR7vVCCn
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in '$(seq 0 2)'
+++ set +e
+++ kubectl get service/monitoring-service -o json
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.IyZs4dLr3C
+++ cat /tmp/tmp.YwmR7vVCCn
+++ rm /tmp/tmp.IyZs4dLr3C /tmp/tmp.YwmR7vVCCn
+++ return 0
++ local hostname=null
++ '[' -n null -a null '!=' null ']'
+++ kubectl_bin get service/monitoring-service -o json
+++ jq '.status.loadBalancer.ingress[].ip'
+++ sed -e 's/^"//; s/"$//;'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.6MlSLzyMVt
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.tefT0wsCTS
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in '$(seq 0 2)'
+++ set +e
+++ kubectl get service/monitoring-service -o json
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.6MlSLzyMVt
+++ cat /tmp/tmp.tefT0wsCTS
+++ rm /tmp/tmp.6MlSLzyMVt /tmp/tmp.tefT0wsCTS
+++ return 0
++ local ip=34.29.4.83
++ '[' -n 34.29.4.83 -a 34.29.4.83 '!=' null ']'
++ echo 34.29.4.83
++ return
+ local endpoint=34.29.4.83
+ jq '.data.result[0].values[][1]'
+ grep '^"[0-9]'
+ curl -s -k 'https://admin:admin@34.29.4.83/graph/api/datasources/proxy/1/api/v1/query_range?query=min%28mongodb_connections%7Bnode_name%3D%7E%22monitoring-2-0-31670-monitoring-rs0-1%22%7d%20or%20mongodb_connections%7Bnode_name%3D%7E%22monitoring-2-0-31670-monitoring-rs0-1%22%7D%29&start=1720346981&end=1720347041&step=60'
"0"
"0"
+ desc 'check mongo config  metrics'
+ set +o xtrace


-----------------------------------------------------------------------------------
check mongo config  metrics
-----------------------------------------------------------------------------------

+ get_metric_values node_boot_time_seconds monitoring-2-0-31670-monitoring-cfg-1 admin:admin
+ local metric=node_boot_time_seconds
+ local instance=monitoring-2-0-31670-monitoring-cfg-1
+ local user_pass=admin:admin
++ /usr/bin/date -u +%s -d '-1 minute'
+ local start=1720346982
++ /usr/bin/date -u +%s
+ local end=1720347042
++ get_service_endpoint monitoring-service
++ local service=monitoring-service
+++ kubectl_bin get service/monitoring-service -o json
+++ jq '.status.loadBalancer.ingress[].hostname'
+++ sed -e 's/^"//; s/"$//;'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.1DT2v9GqFY
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.FKEi3KmPoP
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in '$(seq 0 2)'
+++ set +e
+++ kubectl get service/monitoring-service -o json
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.1DT2v9GqFY
+++ cat /tmp/tmp.FKEi3KmPoP
+++ rm /tmp/tmp.1DT2v9GqFY /tmp/tmp.FKEi3KmPoP
+++ return 0
++ local hostname=null
++ '[' -n null -a null '!=' null ']'
+++ kubectl_bin get service/monitoring-service -o json
+++ jq '.status.loadBalancer.ingress[].ip'
++++ mktemp
+++ sed -e 's/^"//; s/"$//;'
+++ local LAST_OUT=/tmp/tmp.jYSrzZuTT6
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.RMML4LIe2z
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in '$(seq 0 2)'
+++ set +e
+++ kubectl get service/monitoring-service -o json
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.jYSrzZuTT6
+++ cat /tmp/tmp.RMML4LIe2z
+++ rm /tmp/tmp.jYSrzZuTT6 /tmp/tmp.RMML4LIe2z
+++ return 0
++ local ip=34.29.4.83
++ '[' -n 34.29.4.83 -a 34.29.4.83 '!=' null ']'
++ echo 34.29.4.83
++ return
+ local endpoint=34.29.4.83
+ jq '.data.result[0].values[][1]'
+ grep '^"[0-9]'
+ curl -s -k 'https://admin:admin@34.29.4.83/graph/api/datasources/proxy/1/api/v1/query_range?query=min%28node_boot_time_seconds%7Bnode_name%3D%7E%22monitoring-2-0-31670-monitoring-cfg-1%22%7d%20or%20node_boot_time_seconds%7Bnode_name%3D%7E%22monitoring-2-0-31670-monitoring-cfg-1%22%7D%29&start=1720346982&end=1720347042&step=60'
"1720345101"
"1720345101"
+ get_metric_values mongodb_connections monitoring-2-0-31670-monitoring-cfg-1 admin:admin
+ local metric=mongodb_connections
+ local instance=monitoring-2-0-31670-monitoring-cfg-1
+ local user_pass=admin:admin
++ /usr/bin/date -u +%s -d '-1 minute'
+ local start=1720346983
++ /usr/bin/date -u +%s
+ local end=1720347043
++ get_service_endpoint monitoring-service
++ local service=monitoring-service
+++ kubectl_bin get service/monitoring-service -o json
+++ jq '.status.loadBalancer.ingress[].hostname'
+++ sed -e 's/^"//; s/"$//;'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.xgCn5sQjAP
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.X7mDsE1DRZ
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in '$(seq 0 2)'
+++ set +e
+++ kubectl get service/monitoring-service -o json
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.xgCn5sQjAP
+++ cat /tmp/tmp.X7mDsE1DRZ
+++ rm /tmp/tmp.xgCn5sQjAP /tmp/tmp.X7mDsE1DRZ
+++ return 0
++ local hostname=null
++ '[' -n null -a null '!=' null ']'
+++ kubectl_bin get service/monitoring-service -o json
+++ jq '.status.loadBalancer.ingress[].ip'
+++ sed -e 's/^"//; s/"$//;'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.JujBCc3h6b
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.2dEm8dDy7H
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in '$(seq 0 2)'
+++ set +e
+++ kubectl get service/monitoring-service -o json
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.JujBCc3h6b
+++ cat /tmp/tmp.2dEm8dDy7H
+++ rm /tmp/tmp.JujBCc3h6b /tmp/tmp.2dEm8dDy7H
+++ return 0
++ local ip=34.29.4.83
++ '[' -n 34.29.4.83 -a 34.29.4.83 '!=' null ']'
++ echo 34.29.4.83
++ return
+ local endpoint=34.29.4.83
+ curl -s -k 'https://admin:admin@34.29.4.83/graph/api/datasources/proxy/1/api/v1/query_range?query=min%28mongodb_connections%7Bnode_name%3D%7E%22monitoring-2-0-31670-monitoring-cfg-1%22%7d%20or%20mongodb_connections%7Bnode_name%3D%7E%22monitoring-2-0-31670-monitoring-cfg-1%22%7D%29&start=1720346983&end=1720347043&step=60'
+ grep '^"[0-9]'
+ jq '.data.result[0].values[][1]'
"0"
"0"
+ desc 'check mongos metrics'
+ set +o xtrace


-----------------------------------------------------------------------------------
check mongos metrics
-----------------------------------------------------------------------------------

++ kubectl get pod -l app.kubernetes.io/component=mongos -o 'jsonpath={.items[0].metadata.name}'
+ MONGOS_POD_NAME=monitoring-mongos-0
+ get_metric_values node_boot_time_seconds monitoring-2-0-31670-monitoring-mongos-0 admin:admin
+ local metric=node_boot_time_seconds
+ local instance=monitoring-2-0-31670-monitoring-mongos-0
+ local user_pass=admin:admin
++ /usr/bin/date -u +%s -d '-1 minute'
+ local start=1720346986
++ /usr/bin/date -u +%s
+ local end=1720347046
++ get_service_endpoint monitoring-service
++ local service=monitoring-service
+++ kubectl_bin get service/monitoring-service -o json
+++ jq '.status.loadBalancer.ingress[].hostname'
+++ sed -e 's/^"//; s/"$//;'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.kQvzljyTqQ
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.g2djbSVXk5
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in '$(seq 0 2)'
+++ set +e
+++ kubectl get service/monitoring-service -o json
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.kQvzljyTqQ
+++ cat /tmp/tmp.g2djbSVXk5
+++ rm /tmp/tmp.kQvzljyTqQ /tmp/tmp.g2djbSVXk5
+++ return 0
++ local hostname=null
++ '[' -n null -a null '!=' null ']'
+++ kubectl_bin get service/monitoring-service -o json
+++ sed -e 's/^"//; s/"$//;'
+++ jq '.status.loadBalancer.ingress[].ip'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.HxsfELEalc
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.QpjfxQG2O7
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in '$(seq 0 2)'
+++ set +e
+++ kubectl get service/monitoring-service -o json
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.HxsfELEalc
+++ cat /tmp/tmp.QpjfxQG2O7
+++ rm /tmp/tmp.HxsfELEalc /tmp/tmp.QpjfxQG2O7
+++ return 0
++ local ip=34.29.4.83
++ '[' -n 34.29.4.83 -a 34.29.4.83 '!=' null ']'
++ echo 34.29.4.83
++ return
+ local endpoint=34.29.4.83
+ curl -s -k 'https://admin:admin@34.29.4.83/graph/api/datasources/proxy/1/api/v1/query_range?query=min%28node_boot_time_seconds%7Bnode_name%3D%7E%22monitoring-2-0-31670-monitoring-mongos-0%22%7d%20or%20node_boot_time_seconds%7Bnode_name%3D%7E%22monitoring-2-0-31670-monitoring-mongos-0%22%7D%29&start=1720346986&end=1720347046&step=60'
+ grep '^"[0-9]'
+ jq '.data.result[0].values[][1]'
"1720344950"
"1720344950"
+ sleep 90
+ desc 'check QAN data'
+ set +o xtrace


-----------------------------------------------------------------------------------
check QAN data
-----------------------------------------------------------------------------------

+ get_qan_values mongodb dev-mongod admin:admin
+ local service_type=mongodb
+ local environment=dev-mongod
+ local user_pass=admin:admin
+ local start
+ local end
+ local endpoint
++ /usr/bin/date -u -d '-12 hour' +%Y-%m-%dT%H:%M:%S%:z
+ start=2024-07-06T22:12:17+00:00
++ /usr/bin/date -u +%Y-%m-%dT%H:%M:%S%:z
+ end=2024-07-07T10:12:17+00:00
++ get_service_endpoint monitoring-service
++ local service=monitoring-service
+++ kubectl_bin get service/monitoring-service -o json
+++ jq '.status.loadBalancer.ingress[].hostname'
+++ sed -e 's/^"//; s/"$//;'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.DfONLPM1KB
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.rkOegBsP4J
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in '$(seq 0 2)'
+++ set +e
+++ kubectl get service/monitoring-service -o json
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.DfONLPM1KB
+++ cat /tmp/tmp.rkOegBsP4J
+++ rm /tmp/tmp.DfONLPM1KB /tmp/tmp.rkOegBsP4J
+++ return 0
++ local hostname=null
++ '[' -n null -a null '!=' null ']'
+++ kubectl_bin get service/monitoring-service -o json
+++ sed -e 's/^"//; s/"$//;'
++++ mktemp
+++ jq '.status.loadBalancer.ingress[].ip'
+++ local LAST_OUT=/tmp/tmp.eo4ALL3sJS
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.OnU12m3L54
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in '$(seq 0 2)'
+++ set +e
+++ kubectl get service/monitoring-service -o json
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.eo4ALL3sJS
+++ cat /tmp/tmp.OnU12m3L54
+++ rm /tmp/tmp.eo4ALL3sJS /tmp/tmp.OnU12m3L54
+++ return 0
++ local ip=34.29.4.83
++ '[' -n 34.29.4.83 -a 34.29.4.83 '!=' null ']'
++ echo 34.29.4.83
++ return
+ endpoint=34.29.4.83
+ cat
+ local response
++ curl -s -k -XPOST -d @payload.json https://admin:admin@34.29.4.83/v0/qan/GetReport
++ jq '.rows[].fingerprint'
+ response='"TOTAL"
"DBSTATS clusterTime,dbStats,hash,id,keyId,lsid,mode,scale,signature"
"FIND system.version _id"
"FIND oplog.rs"'
+ rm -f payload.json
+ [[ "TOTAL"
"DBSTATS clusterTime,dbStats,hash,id,keyId,lsid,mode,scale,signature"
"FIND system.version _id"
"FIND oplog.rs" == \n\u\l\l ]]
+ get_qan_values mongodb dev-mongos admin:admin
+ local service_type=mongodb
+ local environment=dev-mongos
+ local user_pass=admin:admin
+ local start
+ local end
+ local endpoint
++ /usr/bin/date -u -d '-12 hour' +%Y-%m-%dT%H:%M:%S%:z
+ start=2024-07-06T22:12:20+00:00
++ /usr/bin/date -u +%Y-%m-%dT%H:%M:%S%:z
+ end=2024-07-07T10:12:20+00:00
++ get_service_endpoint monitoring-service
++ local service=monitoring-service
+++ kubectl_bin get service/monitoring-service -o json
+++ jq '.status.loadBalancer.ingress[].hostname'
+++ sed -e 's/^"//; s/"$//;'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.Ywa23jcZSi
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.MVjiI1Kqcj
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in '$(seq 0 2)'
+++ set +e
+++ kubectl get service/monitoring-service -o json
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.Ywa23jcZSi
+++ cat /tmp/tmp.MVjiI1Kqcj
+++ rm /tmp/tmp.Ywa23jcZSi /tmp/tmp.MVjiI1Kqcj
+++ return 0
++ local hostname=null
++ '[' -n null -a null '!=' null ']'
+++ kubectl_bin get service/monitoring-service -o json
+++ jq '.status.loadBalancer.ingress[].ip'
+++ sed -e 's/^"//; s/"$//;'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.yzM36eS5ci
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.mVMUzyVJYM
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in '$(seq 0 2)'
+++ set +e
+++ kubectl get service/monitoring-service -o json
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.yzM36eS5ci
+++ cat /tmp/tmp.mVMUzyVJYM
+++ rm /tmp/tmp.yzM36eS5ci /tmp/tmp.mVMUzyVJYM
+++ return 0
++ local ip=34.29.4.83
++ '[' -n 34.29.4.83 -a 34.29.4.83 '!=' null ']'
++ echo 34.29.4.83
++ return
+ endpoint=34.29.4.83
+ cat
+ local response
++ curl -s -k -XPOST -d @payload.json https://admin:admin@34.29.4.83/v0/qan/GetReport
++ jq '.rows[].fingerprint'
+ response='"TOTAL"
"DBSTATS application,architecture,client,clusterTime,db,dbStats,driver,hash,host,id,keyId,level,lsid,mayBypassWriteBlocking,mongos,name,os,platform,provenance,readConcern,role,scale,signature,type,uid,user,version"'
+ rm -f payload.json
+ [[ "TOTAL"
"DBSTATS application,architecture,client,clusterTime,db,dbStats,driver,hash,host,id,keyId,level,lsid,mayBypassWriteBlocking,mongos,name,os,platform,provenance,readConcern,role,scale,signature,type,uid,user,version" == \n\u\l\l ]]
+ nodeList=($(get_node_id_from_pmm))
++ get_node_id_from_pmm
++ nodeList=()
++ local -a nodeList
+++ kubectl_bin get pods --no-headers -l app.kubernetes.io/name=percona-server-mongodb --output=custom-columns=NAME:.metadata.name
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.gFWjbjQV7g
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.VELkd0C9NU
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in '$(seq 0 2)'
+++ set +e
+++ kubectl get pods --no-headers -l app.kubernetes.io/name=percona-server-mongodb --output=custom-columns=NAME:.metadata.name
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.gFWjbjQV7g
+++ cat /tmp/tmp.VELkd0C9NU
+++ rm /tmp/tmp.gFWjbjQV7g /tmp/tmp.VELkd0C9NU
+++ return 0
++ for instance in '$(kubectl_bin get pods --no-headers -l app.kubernetes.io/name=percona-server-mongodb --output=custom-columns='\''NAME:.metadata.name'\'')'
++ nodeList+=($(kubectl_bin exec -n "$namespace" $instance -c pmm-client -- pmm-admin status --json | jq -r '.pmm_agent_status.node_id'))
+++ kubectl_bin exec -n monitoring-2-0-31670 monitoring-cfg-0 -c pmm-client -- pmm-admin status --json
+++ jq -r .pmm_agent_status.node_id
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.iRatZYm39L
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.oiptBOOTim
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in '$(seq 0 2)'
+++ set +e
+++ kubectl exec -n monitoring-2-0-31670 monitoring-cfg-0 -c pmm-client -- pmm-admin status --json
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.iRatZYm39L
+++ cat /tmp/tmp.oiptBOOTim
+++ rm /tmp/tmp.iRatZYm39L /tmp/tmp.oiptBOOTim
+++ return 0
++ for instance in '$(kubectl_bin get pods --no-headers -l app.kubernetes.io/name=percona-server-mongodb --output=custom-columns='\''NAME:.metadata.name'\'')'
++ nodeList+=($(kubectl_bin exec -n "$namespace" $instance -c pmm-client -- pmm-admin status --json | jq -r '.pmm_agent_status.node_id'))
+++ kubectl_bin exec -n monitoring-2-0-31670 monitoring-cfg-1 -c pmm-client -- pmm-admin status --json
+++ jq -r .pmm_agent_status.node_id
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.YKGmyBh4cy
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.WP83z0IPjq
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in '$(seq 0 2)'
+++ set +e
+++ kubectl exec -n monitoring-2-0-31670 monitoring-cfg-1 -c pmm-client -- pmm-admin status --json
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.YKGmyBh4cy
+++ cat /tmp/tmp.WP83z0IPjq
+++ rm /tmp/tmp.YKGmyBh4cy /tmp/tmp.WP83z0IPjq
+++ return 0
++ for instance in '$(kubectl_bin get pods --no-headers -l app.kubernetes.io/name=percona-server-mongodb --output=custom-columns='\''NAME:.metadata.name'\'')'
++ nodeList+=($(kubectl_bin exec -n "$namespace" $instance -c pmm-client -- pmm-admin status --json | jq -r '.pmm_agent_status.node_id'))
+++ kubectl_bin exec -n monitoring-2-0-31670 monitoring-cfg-2 -c pmm-client -- pmm-admin status --json
+++ jq -r .pmm_agent_status.node_id
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.SAyxS4ZqMz
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.ux4Kcl07yc
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in '$(seq 0 2)'
+++ set +e
+++ kubectl exec -n monitoring-2-0-31670 monitoring-cfg-2 -c pmm-client -- pmm-admin status --json
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.SAyxS4ZqMz
+++ cat /tmp/tmp.ux4Kcl07yc
+++ rm /tmp/tmp.SAyxS4ZqMz /tmp/tmp.ux4Kcl07yc
+++ return 0
++ for instance in '$(kubectl_bin get pods --no-headers -l app.kubernetes.io/name=percona-server-mongodb --output=custom-columns='\''NAME:.metadata.name'\'')'
++ nodeList+=($(kubectl_bin exec -n "$namespace" $instance -c pmm-client -- pmm-admin status --json | jq -r '.pmm_agent_status.node_id'))
+++ kubectl_bin exec -n monitoring-2-0-31670 monitoring-mongos-0 -c pmm-client -- pmm-admin status --json
+++ jq -r .pmm_agent_status.node_id
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.vaXPKBM9Rm
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.mPatRULAS6
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in '$(seq 0 2)'
+++ set +e
+++ kubectl exec -n monitoring-2-0-31670 monitoring-mongos-0 -c pmm-client -- pmm-admin status --json
+++ exit_status=1
+++ set -e
+++ '[' 1 '!=' 0 -a -n 1 ']'
+++ cat /tmp/tmp.vaXPKBM9Rm
+++ cat /tmp/tmp.mPatRULAS6
error: unable to upgrade connection: container not found ("pmm-client")
+++ sleep 0
+++ for i in '$(seq 0 2)'
+++ set +e
+++ kubectl exec -n monitoring-2-0-31670 monitoring-mongos-0 -c pmm-client -- pmm-admin status --json
+++ exit_status=1
+++ set -e
+++ '[' 1 '!=' 0 -a -n 1 ']'
+++ cat /tmp/tmp.vaXPKBM9Rm
+++ cat /tmp/tmp.mPatRULAS6
error: unable to upgrade connection: container not found ("pmm-client")
+++ sleep 4
+++ for i in '$(seq 0 2)'
+++ set +e
+++ kubectl exec -n monitoring-2-0-31670 monitoring-mongos-0 -c pmm-client -- pmm-admin status --json
+++ exit_status=1
+++ set -e
+++ '[' 1 '!=' 0 -a -n 1 ']'
+++ cat /tmp/tmp.vaXPKBM9Rm
+++ cat /tmp/tmp.mPatRULAS6
error: unable to upgrade connection: container not found ("pmm-client")
+++ sleep 8
+++ cat /tmp/tmp.vaXPKBM9Rm
+++ cat /tmp/tmp.mPatRULAS6
error: unable to upgrade connection: container not found ("pmm-client")
+++ rm /tmp/tmp.vaXPKBM9Rm /tmp/tmp.mPatRULAS6
+++ return 1
++ for instance in '$(kubectl_bin get pods --no-headers -l app.kubernetes.io/name=percona-server-mongodb --output=custom-columns='\''NAME:.metadata.name'\'')'
++ nodeList+=($(kubectl_bin exec -n "$namespace" $instance -c pmm-client -- pmm-admin status --json | jq -r '.pmm_agent_status.node_id'))
+++ kubectl_bin exec -n monitoring-2-0-31670 monitoring-mongos-1 -c pmm-client -- pmm-admin status --json
+++ jq -r .pmm_agent_status.node_id
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.Qac3XhZPCV
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.7I0IE9sr91
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in '$(seq 0 2)'
+++ set +e
+++ kubectl exec -n monitoring-2-0-31670 monitoring-mongos-1 -c pmm-client -- pmm-admin status --json
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.Qac3XhZPCV
+++ cat /tmp/tmp.7I0IE9sr91
+++ rm /tmp/tmp.Qac3XhZPCV /tmp/tmp.7I0IE9sr91
+++ return 0
++ for instance in '$(kubectl_bin get pods --no-headers -l app.kubernetes.io/name=percona-server-mongodb --output=custom-columns='\''NAME:.metadata.name'\'')'
++ nodeList+=($(kubectl_bin exec -n "$namespace" $instance -c pmm-client -- pmm-admin status --json | jq -r '.pmm_agent_status.node_id'))
+++ kubectl_bin exec -n monitoring-2-0-31670 monitoring-mongos-2 -c pmm-client -- pmm-admin status --json
+++ jq -r .pmm_agent_status.node_id
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.kK2i1FONQ3
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.QqNWPbFFTe
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in '$(seq 0 2)'
+++ set +e
+++ kubectl exec -n monitoring-2-0-31670 monitoring-mongos-2 -c pmm-client -- pmm-admin status --json
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.kK2i1FONQ3
+++ cat /tmp/tmp.QqNWPbFFTe
+++ rm /tmp/tmp.kK2i1FONQ3 /tmp/tmp.QqNWPbFFTe
+++ return 0
++ for instance in '$(kubectl_bin get pods --no-headers -l app.kubernetes.io/name=percona-server-mongodb --output=custom-columns='\''NAME:.metadata.name'\'')'
++ nodeList+=($(kubectl_bin exec -n "$namespace" $instance -c pmm-client -- pmm-admin status --json | jq -r '.pmm_agent_status.node_id'))
+++ kubectl_bin exec -n monitoring-2-0-31670 monitoring-rs0-0 -c pmm-client -- pmm-admin status --json
+++ jq -r .pmm_agent_status.node_id
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.4fBXWO3unw
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.4a4AekwfUc
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in '$(seq 0 2)'
+++ set +e
+++ kubectl exec -n monitoring-2-0-31670 monitoring-rs0-0 -c pmm-client -- pmm-admin status --json
+++ exit_status=1
+++ set -e
+++ '[' 1 '!=' 0 -a -n 1 ']'
+++ cat /tmp/tmp.4fBXWO3unw
+++ cat /tmp/tmp.4a4AekwfUc
error: unable to upgrade connection: container not found ("pmm-client")
+++ sleep 0
+++ for i in '$(seq 0 2)'
+++ set +e
+++ kubectl exec -n monitoring-2-0-31670 monitoring-rs0-0 -c pmm-client -- pmm-admin status --json
+++ exit_status=1
+++ set -e
+++ '[' 1 '!=' 0 -a -n 1 ']'
+++ cat /tmp/tmp.4fBXWO3unw
+++ cat /tmp/tmp.4a4AekwfUc
error: unable to upgrade connection: container not found ("pmm-client")
+++ sleep 4
+++ for i in '$(seq 0 2)'
+++ set +e
+++ kubectl exec -n monitoring-2-0-31670 monitoring-rs0-0 -c pmm-client -- pmm-admin status --json
+++ exit_status=1
+++ set -e
+++ '[' 1 '!=' 0 -a -n 1 ']'
+++ cat /tmp/tmp.4fBXWO3unw
+++ cat /tmp/tmp.4a4AekwfUc
error: unable to upgrade connection: container not found ("pmm-client")
+++ sleep 8
+++ cat /tmp/tmp.4fBXWO3unw
+++ cat /tmp/tmp.4a4AekwfUc
error: unable to upgrade connection: container not found ("pmm-client")
+++ rm /tmp/tmp.4fBXWO3unw /tmp/tmp.4a4AekwfUc
+++ return 1
++ for instance in '$(kubectl_bin get pods --no-headers -l app.kubernetes.io/name=percona-server-mongodb --output=custom-columns='\''NAME:.metadata.name'\'')'
++ nodeList+=($(kubectl_bin exec -n "$namespace" $instance -c pmm-client -- pmm-admin status --json | jq -r '.pmm_agent_status.node_id'))
+++ kubectl_bin exec -n monitoring-2-0-31670 monitoring-rs0-1 -c pmm-client -- pmm-admin status --json
+++ jq -r .pmm_agent_status.node_id
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.8yNfFcysBn
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.3FoSu9MP3F
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in '$(seq 0 2)'
+++ set +e
+++ kubectl exec -n monitoring-2-0-31670 monitoring-rs0-1 -c pmm-client -- pmm-admin status --json
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.8yNfFcysBn
+++ cat /tmp/tmp.3FoSu9MP3F
+++ rm /tmp/tmp.8yNfFcysBn /tmp/tmp.3FoSu9MP3F
+++ return 0
++ for instance in '$(kubectl_bin get pods --no-headers -l app.kubernetes.io/name=percona-server-mongodb --output=custom-columns='\''NAME:.metadata.name'\'')'
++ nodeList+=($(kubectl_bin exec -n "$namespace" $instance -c pmm-client -- pmm-admin status --json | jq -r '.pmm_agent_status.node_id'))
+++ kubectl_bin exec -n monitoring-2-0-31670 monitoring-rs0-2 -c pmm-client -- pmm-admin status --json
+++ jq -r .pmm_agent_status.node_id
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.nP3WkGs0vZ
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.9VV6HCNAva
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in '$(seq 0 2)'
+++ set +e
+++ kubectl exec -n monitoring-2-0-31670 monitoring-rs0-2 -c pmm-client -- pmm-admin status --json
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.nP3WkGs0vZ
+++ cat /tmp/tmp.9VV6HCNAva
+++ rm /tmp/tmp.nP3WkGs0vZ /tmp/tmp.9VV6HCNAva
+++ return 0
++ echo /node_id/51f35ae2-bdb4-4d66-a273-0fa655cce07e /node_id/e4d1e4a2-a57b-4cb5-8e9e-7ef9abae0bec /node_id/e5bfc490-c328-4b20-ac8d-b90982c0c551 /node_id/9135e58f-5ef2-4263-9284-06b5784d32c4 /node_id/183bb166-4262-4338-b215-e6f12647dfb3 /node_id/f2a4a15c-f4e4-4f16-bbd2-1c43a6653952 /node_id/18235048-c559-4cc3-a045-76317a1f73fc
+ nodeList_from_pmm=($(does_node_id_exists "${nodeList[@]}"))
++ does_node_id_exists /node_id/51f35ae2-bdb4-4d66-a273-0fa655cce07e /node_id/e4d1e4a2-a57b-4cb5-8e9e-7ef9abae0bec /node_id/e5bfc490-c328-4b20-ac8d-b90982c0c551 /node_id/9135e58f-5ef2-4263-9284-06b5784d32c4 /node_id/183bb166-4262-4338-b215-e6f12647dfb3 /node_id/f2a4a15c-f4e4-4f16-bbd2-1c43a6653952 /node_id/18235048-c559-4cc3-a045-76317a1f73fc
++ nodeList=("$@")
++ local -a nodeList
++ nodeList_from_pmm=()
++ local -a nodeList_from_pmm
++ for node_id in '"${nodeList[@]}"'
++ nodeList_from_pmm+=($(kubectl_bin exec -n "${namespace}" monitoring-0 -- pmm-admin --server-url=https://admin:admin@$(get_service_ip monitoring-service)/ --server-insecure-tls inventory list nodes --node-type=CONTAINER_NODE | grep $node_id | awk '{print $4}'))
+++ grep /node_id/51f35ae2-bdb4-4d66-a273-0fa655cce07e
+++ awk '{print $4}'
++++ get_service_ip monitoring-service
++++ local service=monitoring-service
++++ local server_type=rs0
+++++ kubectl_bin get psmdb/monitoring-service -o 'jsonpath={.spec.replsets[].expose.enabled}'
++++++ mktemp
+++++ local LAST_OUT=/tmp/tmp.TzzHMUKx3d
++++++ mktemp
+++++ local LAST_ERR=/tmp/tmp.1Fo9vxBMVx
+++++ local exit_status=0
+++++ local timeout=4
++++++ seq 0 2
+++++ for i in '$(seq 0 2)'
+++++ set +e
+++++ kubectl get psmdb/monitoring-service -o 'jsonpath={.spec.replsets[].expose.enabled}'
+++++ exit_status=1
+++++ set -e
+++++ '[' 1 '!=' 0 -a -n 1 ']'
+++++ cat /tmp/tmp.TzzHMUKx3d
+++++ cat /tmp/tmp.1Fo9vxBMVx
Error from server (NotFound): perconaservermongodbs.psmdb.percona.com "monitoring-service" not found
+++++ sleep 0
+++++ for i in '$(seq 0 2)'
+++++ set +e
+++++ kubectl get psmdb/monitoring-service -o 'jsonpath={.spec.replsets[].expose.enabled}'
+++++ exit_status=1
+++++ set -e
+++++ '[' 1 '!=' 0 -a -n 1 ']'
+++++ cat /tmp/tmp.TzzHMUKx3d
+++++ cat /tmp/tmp.1Fo9vxBMVx
Error from server (NotFound): perconaservermongodbs.psmdb.percona.com "monitoring-service" not found
+++++ sleep 4
+++++ for i in '$(seq 0 2)'
+++++ set +e
+++++ kubectl get psmdb/monitoring-service -o 'jsonpath={.spec.replsets[].expose.enabled}'
+++++ exit_status=1
+++++ set -e
+++++ '[' 1 '!=' 0 -a -n 1 ']'
+++++ cat /tmp/tmp.TzzHMUKx3d
+++++ cat /tmp/tmp.1Fo9vxBMVx
Error from server (NotFound): perconaservermongodbs.psmdb.percona.com "monitoring-service" not found
+++++ sleep 8
+++++ cat /tmp/tmp.TzzHMUKx3d
+++++ cat /tmp/tmp.1Fo9vxBMVx
Error from server (NotFound): perconaservermongodbs.psmdb.percona.com "monitoring-service" not found
+++++ rm /tmp/tmp.TzzHMUKx3d /tmp/tmp.1Fo9vxBMVx
+++++ return 1
++++ '[' '' '!=' true ']'
++++ echo -n monitoring-service.monitoring-service-rs0
++++ return
+++ kubectl_bin exec -n monitoring-2-0-31670 monitoring-0 -- pmm-admin --server-url=https://admin:admin@monitoring-service.monitoring-service-rs0/ --server-insecure-tls inventory list nodes --node-type=CONTAINER_NODE
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.wRuBKL3qSw
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.vNcTczphwH
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in '$(seq 0 2)'
+++ set +e
+++ kubectl exec -n monitoring-2-0-31670 monitoring-0 -- pmm-admin --server-url=https://admin:admin@monitoring-service.monitoring-service-rs0/ --server-insecure-tls inventory list nodes --node-type=CONTAINER_NODE
+++ exit_status=1
+++ set -e
+++ '[' 1 '!=' 0 -a -n 1 ']'
+++ cat /tmp/tmp.wRuBKL3qSw
+++ cat /tmp/tmp.vNcTczphwH
command terminated with exit code 1
+++ sleep 0
+++ for i in '$(seq 0 2)'
+++ set +e
+++ kubectl exec -n monitoring-2-0-31670 monitoring-0 -- pmm-admin --server-url=https://admin:admin@monitoring-service.monitoring-service-rs0/ --server-insecure-tls inventory list nodes --node-type=CONTAINER_NODE
+++ exit_status=1
+++ set -e
+++ '[' 1 '!=' 0 -a -n 1 ']'
+++ cat /tmp/tmp.wRuBKL3qSw
+++ cat /tmp/tmp.vNcTczphwH
command terminated with exit code 1
+++ sleep 4
+++ for i in '$(seq 0 2)'
+++ set +e
+++ kubectl exec -n monitoring-2-0-31670 monitoring-0 -- pmm-admin --server-url=https://admin:admin@monitoring-service.monitoring-service-rs0/ --server-insecure-tls inventory list nodes --node-type=CONTAINER_NODE
+++ exit_status=1
+++ set -e
+++ '[' 1 '!=' 0 -a -n 1 ']'
+++ cat /tmp/tmp.wRuBKL3qSw
+++ cat /tmp/tmp.vNcTczphwH
command terminated with exit code 1
+++ sleep 8
+++ cat /tmp/tmp.wRuBKL3qSw
+++ cat /tmp/tmp.vNcTczphwH
command terminated with exit code 1
+++ rm /tmp/tmp.wRuBKL3qSw /tmp/tmp.vNcTczphwH
+++ return 1
++ for node_id in '"${nodeList[@]}"'
++ nodeList_from_pmm+=($(kubectl_bin exec -n "${namespace}" monitoring-0 -- pmm-admin --server-url=https://admin:admin@$(get_service_ip monitoring-service)/ --server-insecure-tls inventory list nodes --node-type=CONTAINER_NODE | grep $node_id | awk '{print $4}'))
+++ grep /node_id/e4d1e4a2-a57b-4cb5-8e9e-7ef9abae0bec
+++ awk '{print $4}'
++++ get_service_ip monitoring-service
++++ local service=monitoring-service
++++ local server_type=rs0
+++++ kubectl_bin get psmdb/monitoring-service -o 'jsonpath={.spec.replsets[].expose.enabled}'
++++++ mktemp
+++++ local LAST_OUT=/tmp/tmp.WWl4qF1J74
++++++ mktemp
+++++ local LAST_ERR=/tmp/tmp.7gQjXElJ9j
+++++ local exit_status=0
+++++ local timeout=4
++++++ seq 0 2
+++++ for i in '$(seq 0 2)'
+++++ set +e
+++++ kubectl get psmdb/monitoring-service -o 'jsonpath={.spec.replsets[].expose.enabled}'
+++++ exit_status=1
+++++ set -e
+++++ '[' 1 '!=' 0 -a -n 1 ']'
+++++ cat /tmp/tmp.WWl4qF1J74
+++++ cat /tmp/tmp.7gQjXElJ9j
Error from server (NotFound): perconaservermongodbs.psmdb.percona.com "monitoring-service" not found
+++++ sleep 0
+++++ for i in '$(seq 0 2)'
+++++ set +e
+++++ kubectl get psmdb/monitoring-service -o 'jsonpath={.spec.replsets[].expose.enabled}'
+++++ exit_status=1
+++++ set -e
+++++ '[' 1 '!=' 0 -a -n 1 ']'
+++++ cat /tmp/tmp.WWl4qF1J74
+++++ cat /tmp/tmp.7gQjXElJ9j
Error from server (NotFound): perconaservermongodbs.psmdb.percona.com "monitoring-service" not found
+++++ sleep 4
+++++ for i in '$(seq 0 2)'
+++++ set +e
+++++ kubectl get psmdb/monitoring-service -o 'jsonpath={.spec.replsets[].expose.enabled}'
+++++ exit_status=1
+++++ set -e
+++++ '[' 1 '!=' 0 -a -n 1 ']'
+++++ cat /tmp/tmp.WWl4qF1J74
+++++ cat /tmp/tmp.7gQjXElJ9j
Error from server (NotFound): perconaservermongodbs.psmdb.percona.com "monitoring-service" not found
+++++ sleep 8
+++++ cat /tmp/tmp.WWl4qF1J74
+++++ cat /tmp/tmp.7gQjXElJ9j
Error from server (NotFound): perconaservermongodbs.psmdb.percona.com "monitoring-service" not found
+++++ rm /tmp/tmp.WWl4qF1J74 /tmp/tmp.7gQjXElJ9j
+++++ return 1
++++ '[' '' '!=' true ']'
++++ echo -n monitoring-service.monitoring-service-rs0
++++ return
+++ kubectl_bin exec -n monitoring-2-0-31670 monitoring-0 -- pmm-admin --server-url=https://admin:admin@monitoring-service.monitoring-service-rs0/ --server-insecure-tls inventory list nodes --node-type=CONTAINER_NODE
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.NKpg0ZoSrm
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.kHZO60pJYa
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in '$(seq 0 2)'
+++ set +e
+++ kubectl exec -n monitoring-2-0-31670 monitoring-0 -- pmm-admin --server-url=https://admin:admin@monitoring-service.monitoring-service-rs0/ --server-insecure-tls inventory list nodes --node-type=CONTAINER_NODE
+++ exit_status=1
+++ set -e
+++ '[' 1 '!=' 0 -a -n 1 ']'
+++ cat /tmp/tmp.NKpg0ZoSrm
+++ cat /tmp/tmp.kHZO60pJYa
command terminated with exit code 1
+++ sleep 0
+++ for i in '$(seq 0 2)'
+++ set +e
+++ kubectl exec -n monitoring-2-0-31670 monitoring-0 -- pmm-admin --server-url=https://admin:admin@monitoring-service.monitoring-service-rs0/ --server-insecure-tls inventory list nodes --node-type=CONTAINER_NODE
+++ exit_status=1
+++ set -e
+++ '[' 1 '!=' 0 -a -n 1 ']'
+++ cat /tmp/tmp.NKpg0ZoSrm
+++ cat /tmp/tmp.kHZO60pJYa
command terminated with exit code 1
+++ sleep 4
+++ for i in '$(seq 0 2)'
+++ set +e
+++ kubectl exec -n monitoring-2-0-31670 monitoring-0 -- pmm-admin --server-url=https://admin:admin@monitoring-service.monitoring-service-rs0/ --server-insecure-tls inventory list nodes --node-type=CONTAINER_NODE
+++ exit_status=1
+++ set -e
+++ '[' 1 '!=' 0 -a -n 1 ']'
+++ cat /tmp/tmp.NKpg0ZoSrm
+++ cat /tmp/tmp.kHZO60pJYa
command terminated with exit code 1
+++ sleep 8
+++ cat /tmp/tmp.NKpg0ZoSrm
+++ cat /tmp/tmp.kHZO60pJYa
command terminated with exit code 1
+++ rm /tmp/tmp.NKpg0ZoSrm /tmp/tmp.kHZO60pJYa
+++ return 1
++ for node_id in '"${nodeList[@]}"'
++ nodeList_from_pmm+=($(kubectl_bin exec -n "${namespace}" monitoring-0 -- pmm-admin --server-url=https://admin:admin@$(get_service_ip monitoring-service)/ --server-insecure-tls inventory list nodes --node-type=CONTAINER_NODE | grep $node_id | awk '{print $4}'))
+++ grep /node_id/e5bfc490-c328-4b20-ac8d-b90982c0c551
++++ get_service_ip monitoring-service
+++ awk '{print $4}'
++++ local service=monitoring-service
++++ local server_type=rs0
+++++ kubectl_bin get psmdb/monitoring-service -o 'jsonpath={.spec.replsets[].expose.enabled}'
++++++ mktemp
+++++ local LAST_OUT=/tmp/tmp.ZhnmhdxQFm
++++++ mktemp
+++++ local LAST_ERR=/tmp/tmp.XAKeAOJ1mg
+++++ local exit_status=0
+++++ local timeout=4
++++++ seq 0 2
+++++ for i in '$(seq 0 2)'
+++++ set +e
+++++ kubectl get psmdb/monitoring-service -o 'jsonpath={.spec.replsets[].expose.enabled}'
+++++ exit_status=1
+++++ set -e
+++++ '[' 1 '!=' 0 -a -n 1 ']'
+++++ cat /tmp/tmp.ZhnmhdxQFm
+++++ cat /tmp/tmp.XAKeAOJ1mg
Error from server (NotFound): perconaservermongodbs.psmdb.percona.com "monitoring-service" not found
+++++ sleep 0
+++++ for i in '$(seq 0 2)'
+++++ set +e
+++++ kubectl get psmdb/monitoring-service -o 'jsonpath={.spec.replsets[].expose.enabled}'
+++++ exit_status=1
+++++ set -e
+++++ '[' 1 '!=' 0 -a -n 1 ']'
+++++ cat /tmp/tmp.ZhnmhdxQFm
+++++ cat /tmp/tmp.XAKeAOJ1mg
Error from server (NotFound): perconaservermongodbs.psmdb.percona.com "monitoring-service" not found
+++++ sleep 4
+++++ for i in '$(seq 0 2)'
+++++ set +e
+++++ kubectl get psmdb/monitoring-service -o 'jsonpath={.spec.replsets[].expose.enabled}'
+++++ exit_status=1
+++++ set -e
+++++ '[' 1 '!=' 0 -a -n 1 ']'
+++++ cat /tmp/tmp.ZhnmhdxQFm
+++++ cat /tmp/tmp.XAKeAOJ1mg
Error from server (NotFound): perconaservermongodbs.psmdb.percona.com "monitoring-service" not found
+++++ sleep 8
+++++ cat /tmp/tmp.ZhnmhdxQFm
+++++ cat /tmp/tmp.XAKeAOJ1mg
Error from server (NotFound): perconaservermongodbs.psmdb.percona.com "monitoring-service" not found
+++++ rm /tmp/tmp.ZhnmhdxQFm /tmp/tmp.XAKeAOJ1mg
+++++ return 1
++++ '[' '' '!=' true ']'
++++ echo -n monitoring-service.monitoring-service-rs0
++++ return
+++ kubectl_bin exec -n monitoring-2-0-31670 monitoring-0 -- pmm-admin --server-url=https://admin:admin@monitoring-service.monitoring-service-rs0/ --server-insecure-tls inventory list nodes --node-type=CONTAINER_NODE
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.hKJ8OkGnEo
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.GM1k9nDZwC
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in '$(seq 0 2)'
+++ set +e
+++ kubectl exec -n monitoring-2-0-31670 monitoring-0 -- pmm-admin --server-url=https://admin:admin@monitoring-service.monitoring-service-rs0/ --server-insecure-tls inventory list nodes --node-type=CONTAINER_NODE
+++ exit_status=1
+++ set -e
+++ '[' 1 '!=' 0 -a -n 1 ']'
+++ cat /tmp/tmp.hKJ8OkGnEo
+++ cat /tmp/tmp.GM1k9nDZwC
command terminated with exit code 1
+++ sleep 0
+++ for i in '$(seq 0 2)'
+++ set +e
+++ kubectl exec -n monitoring-2-0-31670 monitoring-0 -- pmm-admin --server-url=https://admin:admin@monitoring-service.monitoring-service-rs0/ --server-insecure-tls inventory list nodes --node-type=CONTAINER_NODE
+++ exit_status=1
+++ set -e
+++ '[' 1 '!=' 0 -a -n 1 ']'
+++ cat /tmp/tmp.hKJ8OkGnEo
+++ cat /tmp/tmp.GM1k9nDZwC
command terminated with exit code 1
+++ sleep 4
+++ for i in '$(seq 0 2)'
+++ set +e
+++ kubectl exec -n monitoring-2-0-31670 monitoring-0 -- pmm-admin --server-url=https://admin:admin@monitoring-service.monitoring-service-rs0/ --server-insecure-tls inventory list nodes --node-type=CONTAINER_NODE
+++ exit_status=1
+++ set -e
+++ '[' 1 '!=' 0 -a -n 1 ']'
+++ cat /tmp/tmp.hKJ8OkGnEo
+++ cat /tmp/tmp.GM1k9nDZwC
command terminated with exit code 1
+++ sleep 8
+++ cat /tmp/tmp.hKJ8OkGnEo
+++ cat /tmp/tmp.GM1k9nDZwC
command terminated with exit code 1
+++ rm /tmp/tmp.hKJ8OkGnEo /tmp/tmp.GM1k9nDZwC
+++ return 1
++ for node_id in '"${nodeList[@]}"'
++ nodeList_from_pmm+=($(kubectl_bin exec -n "${namespace}" monitoring-0 -- pmm-admin --server-url=https://admin:admin@$(get_service_ip monitoring-service)/ --server-insecure-tls inventory list nodes --node-type=CONTAINER_NODE | grep $node_id | awk '{print $4}'))
+++ grep /node_id/9135e58f-5ef2-4263-9284-06b5784d32c4
+++ awk '{print $4}'
++++ get_service_ip monitoring-service
++++ local service=monitoring-service
++++ local server_type=rs0
+++++ kubectl_bin get psmdb/monitoring-service -o 'jsonpath={.spec.replsets[].expose.enabled}'
++++++ mktemp
+++++ local LAST_OUT=/tmp/tmp.qprcjdrBkt
++++++ mktemp
+++++ local LAST_ERR=/tmp/tmp.LWOw6NshdU
+++++ local exit_status=0
+++++ local timeout=4
++++++ seq 0 2
+++++ for i in '$(seq 0 2)'
+++++ set +e
+++++ kubectl get psmdb/monitoring-service -o 'jsonpath={.spec.replsets[].expose.enabled}'
+++++ exit_status=1
+++++ set -e
+++++ '[' 1 '!=' 0 -a -n 1 ']'
+++++ cat /tmp/tmp.qprcjdrBkt
+++++ cat /tmp/tmp.LWOw6NshdU
Error from server (NotFound): perconaservermongodbs.psmdb.percona.com "monitoring-service" not found
+++++ sleep 0
+++++ for i in '$(seq 0 2)'
+++++ set +e
+++++ kubectl get psmdb/monitoring-service -o 'jsonpath={.spec.replsets[].expose.enabled}'
+++++ exit_status=1
+++++ set -e
+++++ '[' 1 '!=' 0 -a -n 1 ']'
+++++ cat /tmp/tmp.qprcjdrBkt
+++++ cat /tmp/tmp.LWOw6NshdU
Error from server (NotFound): perconaservermongodbs.psmdb.percona.com "monitoring-service" not found
+++++ sleep 4
+++++ for i in '$(seq 0 2)'
+++++ set +e
+++++ kubectl get psmdb/monitoring-service -o 'jsonpath={.spec.replsets[].expose.enabled}'
+++++ exit_status=1
+++++ set -e
+++++ '[' 1 '!=' 0 -a -n 1 ']'
+++++ cat /tmp/tmp.qprcjdrBkt
+++++ cat /tmp/tmp.LWOw6NshdU
Error from server (NotFound): perconaservermongodbs.psmdb.percona.com "monitoring-service" not found
+++++ sleep 8
+++++ cat /tmp/tmp.qprcjdrBkt
+++++ cat /tmp/tmp.LWOw6NshdU
Error from server (NotFound): perconaservermongodbs.psmdb.percona.com "monitoring-service" not found
+++++ rm /tmp/tmp.qprcjdrBkt /tmp/tmp.LWOw6NshdU
+++++ return 1
++++ '[' '' '!=' true ']'
++++ echo -n monitoring-service.monitoring-service-rs0
++++ return
+++ kubectl_bin exec -n monitoring-2-0-31670 monitoring-0 -- pmm-admin --server-url=https://admin:admin@monitoring-service.monitoring-service-rs0/ --server-insecure-tls inventory list nodes --node-type=CONTAINER_NODE
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.447iW0i2SQ
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.iAQulRNOLI
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in '$(seq 0 2)'
+++ set +e
+++ kubectl exec -n monitoring-2-0-31670 monitoring-0 -- pmm-admin --server-url=https://admin:admin@monitoring-service.monitoring-service-rs0/ --server-insecure-tls inventory list nodes --node-type=CONTAINER_NODE
+++ exit_status=1
+++ set -e
+++ '[' 1 '!=' 0 -a -n 1 ']'
+++ cat /tmp/tmp.447iW0i2SQ
+++ cat /tmp/tmp.iAQulRNOLI
command terminated with exit code 1
+++ sleep 0
+++ for i in '$(seq 0 2)'
+++ set +e
+++ kubectl exec -n monitoring-2-0-31670 monitoring-0 -- pmm-admin --server-url=https://admin:admin@monitoring-service.monitoring-service-rs0/ --server-insecure-tls inventory list nodes --node-type=CONTAINER_NODE
+++ exit_status=1
+++ set -e
+++ '[' 1 '!=' 0 -a -n 1 ']'
+++ cat /tmp/tmp.447iW0i2SQ
+++ cat /tmp/tmp.iAQulRNOLI
command terminated with exit code 1
+++ sleep 4
+++ for i in '$(seq 0 2)'
+++ set +e
+++ kubectl exec -n monitoring-2-0-31670 monitoring-0 -- pmm-admin --server-url=https://admin:admin@monitoring-service.monitoring-service-rs0/ --server-insecure-tls inventory list nodes --node-type=CONTAINER_NODE
+++ exit_status=1
+++ set -e
+++ '[' 1 '!=' 0 -a -n 1 ']'
+++ cat /tmp/tmp.447iW0i2SQ
+++ cat /tmp/tmp.iAQulRNOLI
command terminated with exit code 1
+++ sleep 8
+++ cat /tmp/tmp.447iW0i2SQ
+++ cat /tmp/tmp.iAQulRNOLI
command terminated with exit code 1
+++ rm /tmp/tmp.447iW0i2SQ /tmp/tmp.iAQulRNOLI
+++ return 1
++ for node_id in '"${nodeList[@]}"'
++ nodeList_from_pmm+=($(kubectl_bin exec -n "${namespace}" monitoring-0 -- pmm-admin --server-url=https://admin:admin@$(get_service_ip monitoring-service)/ --server-insecure-tls inventory list nodes --node-type=CONTAINER_NODE | grep $node_id | awk '{print $4}'))
+++ awk '{print $4}'
+++ grep /node_id/183bb166-4262-4338-b215-e6f12647dfb3
++++ get_service_ip monitoring-service
++++ local service=monitoring-service
++++ local server_type=rs0
+++++ kubectl_bin get psmdb/monitoring-service -o 'jsonpath={.spec.replsets[].expose.enabled}'
++++++ mktemp
+++++ local LAST_OUT=/tmp/tmp.mv5A3KHnaT
++++++ mktemp
+++++ local LAST_ERR=/tmp/tmp.Y1LEV5NlXx
+++++ local exit_status=0
+++++ local timeout=4
++++++ seq 0 2
+++++ for i in '$(seq 0 2)'
+++++ set +e
+++++ kubectl get psmdb/monitoring-service -o 'jsonpath={.spec.replsets[].expose.enabled}'
+++++ exit_status=1
+++++ set -e
+++++ '[' 1 '!=' 0 -a -n 1 ']'
+++++ cat /tmp/tmp.mv5A3KHnaT
+++++ cat /tmp/tmp.Y1LEV5NlXx
Error from server (NotFound): perconaservermongodbs.psmdb.percona.com "monitoring-service" not found
+++++ sleep 0
+++++ for i in '$(seq 0 2)'
+++++ set +e
+++++ kubectl get psmdb/monitoring-service -o 'jsonpath={.spec.replsets[].expose.enabled}'
+++++ exit_status=1
+++++ set -e
+++++ '[' 1 '!=' 0 -a -n 1 ']'
+++++ cat /tmp/tmp.mv5A3KHnaT
+++++ cat /tmp/tmp.Y1LEV5NlXx
Error from server (NotFound): perconaservermongodbs.psmdb.percona.com "monitoring-service" not found
+++++ sleep 4
+++++ for i in '$(seq 0 2)'
+++++ set +e
+++++ kubectl get psmdb/monitoring-service -o 'jsonpath={.spec.replsets[].expose.enabled}'
+++++ exit_status=1
+++++ set -e
+++++ '[' 1 '!=' 0 -a -n 1 ']'
+++++ cat /tmp/tmp.mv5A3KHnaT
+++++ cat /tmp/tmp.Y1LEV5NlXx
Error from server (NotFound): perconaservermongodbs.psmdb.percona.com "monitoring-service" not found
+++++ sleep 8
+++++ cat /tmp/tmp.mv5A3KHnaT
+++++ cat /tmp/tmp.Y1LEV5NlXx
Error from server (NotFound): perconaservermongodbs.psmdb.percona.com "monitoring-service" not found
+++++ rm /tmp/tmp.mv5A3KHnaT /tmp/tmp.Y1LEV5NlXx
+++++ return 1
++++ '[' '' '!=' true ']'
++++ echo -n monitoring-service.monitoring-service-rs0
++++ return
+++ kubectl_bin exec -n monitoring-2-0-31670 monitoring-0 -- pmm-admin --server-url=https://admin:admin@monitoring-service.monitoring-service-rs0/ --server-insecure-tls inventory list nodes --node-type=CONTAINER_NODE
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.8BGGyksnSm
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.yYoFn1N9vZ
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in '$(seq 0 2)'
+++ set +e
+++ kubectl exec -n monitoring-2-0-31670 monitoring-0 -- pmm-admin --server-url=https://admin:admin@monitoring-service.monitoring-service-rs0/ --server-insecure-tls inventory list nodes --node-type=CONTAINER_NODE
+++ exit_status=1
+++ set -e
+++ '[' 1 '!=' 0 -a -n 1 ']'
+++ cat /tmp/tmp.8BGGyksnSm
+++ cat /tmp/tmp.yYoFn1N9vZ
command terminated with exit code 1
+++ sleep 0
+++ for i in '$(seq 0 2)'
+++ set +e
+++ kubectl exec -n monitoring-2-0-31670 monitoring-0 -- pmm-admin --server-url=https://admin:admin@monitoring-service.monitoring-service-rs0/ --server-insecure-tls inventory list nodes --node-type=CONTAINER_NODE
+++ exit_status=1
+++ set -e
+++ '[' 1 '!=' 0 -a -n 1 ']'
+++ cat /tmp/tmp.8BGGyksnSm
+++ cat /tmp/tmp.yYoFn1N9vZ
command terminated with exit code 1
+++ sleep 4
+++ for i in '$(seq 0 2)'
+++ set +e
+++ kubectl exec -n monitoring-2-0-31670 monitoring-0 -- pmm-admin --server-url=https://admin:admin@monitoring-service.monitoring-service-rs0/ --server-insecure-tls inventory list nodes --node-type=CONTAINER_NODE
+++ exit_status=1
+++ set -e
+++ '[' 1 '!=' 0 -a -n 1 ']'
+++ cat /tmp/tmp.8BGGyksnSm
+++ cat /tmp/tmp.yYoFn1N9vZ
command terminated with exit code 1
+++ sleep 8
+++ cat /tmp/tmp.8BGGyksnSm
+++ cat /tmp/tmp.yYoFn1N9vZ
command terminated with exit code 1
+++ rm /tmp/tmp.8BGGyksnSm /tmp/tmp.yYoFn1N9vZ
+++ return 1
++ for node_id in '"${nodeList[@]}"'
++ nodeList_from_pmm+=($(kubectl_bin exec -n "${namespace}" monitoring-0 -- pmm-admin --server-url=https://admin:admin@$(get_service_ip monitoring-service)/ --server-insecure-tls inventory list nodes --node-type=CONTAINER_NODE | grep $node_id | awk '{print $4}'))
+++ grep /node_id/f2a4a15c-f4e4-4f16-bbd2-1c43a6653952
+++ awk '{print $4}'
++++ get_service_ip monitoring-service
++++ local service=monitoring-service
++++ local server_type=rs0
+++++ kubectl_bin get psmdb/monitoring-service -o 'jsonpath={.spec.replsets[].expose.enabled}'
++++++ mktemp
+++++ local LAST_OUT=/tmp/tmp.28snRu5PUd
++++++ mktemp
+++++ local LAST_ERR=/tmp/tmp.qqyFjwz1Aq
+++++ local exit_status=0
+++++ local timeout=4
++++++ seq 0 2
+++++ for i in '$(seq 0 2)'
+++++ set +e
+++++ kubectl get psmdb/monitoring-service -o 'jsonpath={.spec.replsets[].expose.enabled}'
+++++ exit_status=1
+++++ set -e
+++++ '[' 1 '!=' 0 -a -n 1 ']'
+++++ cat /tmp/tmp.28snRu5PUd
+++++ cat /tmp/tmp.qqyFjwz1Aq
Error from server (NotFound): perconaservermongodbs.psmdb.percona.com "monitoring-service" not found
+++++ sleep 0
+++++ for i in '$(seq 0 2)'
+++++ set +e
+++++ kubectl get psmdb/monitoring-service -o 'jsonpath={.spec.replsets[].expose.enabled}'
+++++ exit_status=1
+++++ set -e
+++++ '[' 1 '!=' 0 -a -n 1 ']'
+++++ cat /tmp/tmp.28snRu5PUd
+++++ cat /tmp/tmp.qqyFjwz1Aq
Error from server (NotFound): perconaservermongodbs.psmdb.percona.com "monitoring-service" not found
+++++ sleep 4
+++++ for i in '$(seq 0 2)'
+++++ set +e
+++++ kubectl get psmdb/monitoring-service -o 'jsonpath={.spec.replsets[].expose.enabled}'
+++++ exit_status=1
+++++ set -e
+++++ '[' 1 '!=' 0 -a -n 1 ']'
+++++ cat /tmp/tmp.28snRu5PUd
+++++ cat /tmp/tmp.qqyFjwz1Aq
Error from server (NotFound): perconaservermongodbs.psmdb.percona.com "monitoring-service" not found
+++++ sleep 8
+++++ cat /tmp/tmp.28snRu5PUd
+++++ cat /tmp/tmp.qqyFjwz1Aq
Error from server (NotFound): perconaservermongodbs.psmdb.percona.com "monitoring-service" not found
+++++ rm /tmp/tmp.28snRu5PUd /tmp/tmp.qqyFjwz1Aq
+++++ return 1
++++ '[' '' '!=' true ']'
++++ echo -n monitoring-service.monitoring-service-rs0
++++ return
+++ kubectl_bin exec -n monitoring-2-0-31670 monitoring-0 -- pmm-admin --server-url=https://admin:admin@monitoring-service.monitoring-service-rs0/ --server-insecure-tls inventory list nodes --node-type=CONTAINER_NODE
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.urADKxsRe4
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.IJHYcKgM0t
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in '$(seq 0 2)'
+++ set +e
+++ kubectl exec -n monitoring-2-0-31670 monitoring-0 -- pmm-admin --server-url=https://admin:admin@monitoring-service.monitoring-service-rs0/ --server-insecure-tls inventory list nodes --node-type=CONTAINER_NODE
+++ exit_status=1
+++ set -e
+++ '[' 1 '!=' 0 -a -n 1 ']'
+++ cat /tmp/tmp.urADKxsRe4
+++ cat /tmp/tmp.IJHYcKgM0t
command terminated with exit code 1
+++ sleep 0
+++ for i in '$(seq 0 2)'
+++ set +e
+++ kubectl exec -n monitoring-2-0-31670 monitoring-0 -- pmm-admin --server-url=https://admin:admin@monitoring-service.monitoring-service-rs0/ --server-insecure-tls inventory list nodes --node-type=CONTAINER_NODE
+++ exit_status=1
+++ set -e
+++ '[' 1 '!=' 0 -a -n 1 ']'
+++ cat /tmp/tmp.urADKxsRe4
+++ cat /tmp/tmp.IJHYcKgM0t
command terminated with exit code 1
+++ sleep 4
+++ for i in '$(seq 0 2)'
+++ set +e
+++ kubectl exec -n monitoring-2-0-31670 monitoring-0 -- pmm-admin --server-url=https://admin:admin@monitoring-service.monitoring-service-rs0/ --server-insecure-tls inventory list nodes --node-type=CONTAINER_NODE
+++ exit_status=1
+++ set -e
+++ '[' 1 '!=' 0 -a -n 1 ']'
+++ cat /tmp/tmp.urADKxsRe4
+++ cat /tmp/tmp.IJHYcKgM0t
command terminated with exit code 1
+++ sleep 8
+++ cat /tmp/tmp.urADKxsRe4
+++ cat /tmp/tmp.IJHYcKgM0t
command terminated with exit code 1
+++ rm /tmp/tmp.urADKxsRe4 /tmp/tmp.IJHYcKgM0t
+++ return 1
++ for node_id in '"${nodeList[@]}"'
++ nodeList_from_pmm+=($(kubectl_bin exec -n "${namespace}" monitoring-0 -- pmm-admin --server-url=https://admin:admin@$(get_service_ip monitoring-service)/ --server-insecure-tls inventory list nodes --node-type=CONTAINER_NODE | grep $node_id | awk '{print $4}'))
+++ grep /node_id/18235048-c559-4cc3-a045-76317a1f73fc
+++ awk '{print $4}'
++++ get_service_ip monitoring-service
++++ local service=monitoring-service
++++ local server_type=rs0
+++++ kubectl_bin get psmdb/monitoring-service -o 'jsonpath={.spec.replsets[].expose.enabled}'
++++++ mktemp
+++++ local LAST_OUT=/tmp/tmp.vohst7p5TW
++++++ mktemp
+++++ local LAST_ERR=/tmp/tmp.U7LKYHppQ7
+++++ local exit_status=0
+++++ local timeout=4
++++++ seq 0 2
+++++ for i in '$(seq 0 2)'
+++++ set +e
+++++ kubectl get psmdb/monitoring-service -o 'jsonpath={.spec.replsets[].expose.enabled}'
+++++ exit_status=1
+++++ set -e
+++++ '[' 1 '!=' 0 -a -n 1 ']'
+++++ cat /tmp/tmp.vohst7p5TW
+++++ cat /tmp/tmp.U7LKYHppQ7
Error from server (NotFound): perconaservermongodbs.psmdb.percona.com "monitoring-service" not found
+++++ sleep 0
+++++ for i in '$(seq 0 2)'
+++++ set +e
+++++ kubectl get psmdb/monitoring-service -o 'jsonpath={.spec.replsets[].expose.enabled}'
+++++ exit_status=1
+++++ set -e
+++++ '[' 1 '!=' 0 -a -n 1 ']'
+++++ cat /tmp/tmp.vohst7p5TW
+++++ cat /tmp/tmp.U7LKYHppQ7
Error from server (NotFound): perconaservermongodbs.psmdb.percona.com "monitoring-service" not found
+++++ sleep 4
+++++ for i in '$(seq 0 2)'
+++++ set +e
+++++ kubectl get psmdb/monitoring-service -o 'jsonpath={.spec.replsets[].expose.enabled}'
+++++ exit_status=1
+++++ set -e
+++++ '[' 1 '!=' 0 -a -n 1 ']'
+++++ cat /tmp/tmp.vohst7p5TW
+++++ cat /tmp/tmp.U7LKYHppQ7
Error from server (NotFound): perconaservermongodbs.psmdb.percona.com "monitoring-service" not found
+++++ sleep 8
+++++ cat /tmp/tmp.vohst7p5TW
+++++ cat /tmp/tmp.U7LKYHppQ7
Error from server (NotFound): perconaservermongodbs.psmdb.percona.com "monitoring-service" not found
+++++ rm /tmp/tmp.vohst7p5TW /tmp/tmp.U7LKYHppQ7
+++++ return 1
++++ '[' '' '!=' true ']'
++++ echo -n monitoring-service.monitoring-service-rs0
++++ return
+++ kubectl_bin exec -n monitoring-2-0-31670 monitoring-0 -- pmm-admin --server-url=https://admin:admin@monitoring-service.monitoring-service-rs0/ --server-insecure-tls inventory list nodes --node-type=CONTAINER_NODE
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.wkNUFKuzgR
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.rwm2W3uLIM
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in '$(seq 0 2)'
+++ set +e
+++ kubectl exec -n monitoring-2-0-31670 monitoring-0 -- pmm-admin --server-url=https://admin:admin@monitoring-service.monitoring-service-rs0/ --server-insecure-tls inventory list nodes --node-type=CONTAINER_NODE
+++ exit_status=1
+++ set -e
+++ '[' 1 '!=' 0 -a -n 1 ']'
+++ cat /tmp/tmp.wkNUFKuzgR
+++ cat /tmp/tmp.rwm2W3uLIM
command terminated with exit code 1
+++ sleep 0
+++ for i in '$(seq 0 2)'
+++ set +e
+++ kubectl exec -n monitoring-2-0-31670 monitoring-0 -- pmm-admin --server-url=https://admin:admin@monitoring-service.monitoring-service-rs0/ --server-insecure-tls inventory list nodes --node-type=CONTAINER_NODE
+++ exit_status=1
+++ set -e
+++ '[' 1 '!=' 0 -a -n 1 ']'
+++ cat /tmp/tmp.wkNUFKuzgR
+++ cat /tmp/tmp.rwm2W3uLIM
command terminated with exit code 1
+++ sleep 4
+++ for i in '$(seq 0 2)'
+++ set +e
+++ kubectl exec -n monitoring-2-0-31670 monitoring-0 -- pmm-admin --server-url=https://admin:admin@monitoring-service.monitoring-service-rs0/ --server-insecure-tls inventory list nodes --node-type=CONTAINER_NODE
+++ exit_status=1
+++ set -e
+++ '[' 1 '!=' 0 -a -n 1 ']'
+++ cat /tmp/tmp.wkNUFKuzgR
+++ cat /tmp/tmp.rwm2W3uLIM
command terminated with exit code 1
+++ sleep 8
+++ cat /tmp/tmp.wkNUFKuzgR
+++ cat /tmp/tmp.rwm2W3uLIM
command terminated with exit code 1
+++ rm /tmp/tmp.wkNUFKuzgR /tmp/tmp.rwm2W3uLIM
+++ return 1
++ echo
+ kubectl_bin patch psmdb monitoring --type json '-p=[{"op":"add","path":"/spec/pause","value":true}]'
++ mktemp
+ local LAST_OUT=/tmp/tmp.pUDPJsHmLm
++ mktemp
+ local LAST_ERR=/tmp/tmp.kth8AnfdVM
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl patch psmdb monitoring --type json '-p=[{"op":"add","path":"/spec/pause","value":true}]'
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.pUDPJsHmLm
perconaservermongodb.psmdb.percona.com/monitoring patched
+ cat /tmp/tmp.kth8AnfdVM
+ rm /tmp/tmp.pUDPJsHmLm /tmp/tmp.kth8AnfdVM
+ return 0
+ wait_for_delete pod/monitoring-mongos-0
+ local res=pod/monitoring-mongos-0
+ local wait_time=60
+ set +o xtrace
pod/monitoring-mongos-0 - ..................................Error from server (NotFound): pods "monitoring-mongos-0" not found
Error from server (NotFound): pods "monitoring-mongos-0" not found
Error from server (NotFound): pods "monitoring-mongos-0" not found
Error from server (NotFound): pods "monitoring-mongos-0" not found
+ wait_for_delete pod/monitoring-rs0-0
+ local res=pod/monitoring-rs0-0
+ local wait_time=60
+ set +o xtrace
pod/monitoring-rs0-0 - .......Error from server (NotFound): pods "monitoring-rs0-0" not found
Error from server (NotFound): pods "monitoring-rs0-0" not found
Error from server (NotFound): pods "monitoring-rs0-0" not found
Error from server (NotFound): pods "monitoring-rs0-0" not found
+ wait_for_delete pod/monitoring-cfg-0
+ local res=pod/monitoring-cfg-0
+ local wait_time=60
+ set +o xtrace
pod/monitoring-cfg-0 - ....Error from server (NotFound): pods "monitoring-cfg-0" not found
Error from server (NotFound): pods "monitoring-cfg-0" not found
Error from server (NotFound): pods "monitoring-cfg-0" not found
Error from server (NotFound): pods "monitoring-cfg-0" not found
+ desc 'check if services are not deleted'
+ set +o xtrace


-----------------------------------------------------------------------------------
check if services are not deleted
-----------------------------------------------------------------------------------

+ kubectl_bin get svc monitoring-rs0
++ mktemp
+ local LAST_OUT=/tmp/tmp.pCbNqZ9HNw
++ mktemp
+ local LAST_ERR=/tmp/tmp.AioIzfcEc1
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl get svc monitoring-rs0
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.pCbNqZ9HNw
NAME             TYPE        CLUSTER-IP   EXTERNAL-IP   PORT(S)     AGE
monitoring-rs0   ClusterIP   None         <none>        27017/TCP   16m
+ cat /tmp/tmp.AioIzfcEc1
+ rm /tmp/tmp.pCbNqZ9HNw /tmp/tmp.AioIzfcEc1
+ return 0
+ kubectl_bin get svc monitoring-cfg
++ mktemp
+ local LAST_OUT=/tmp/tmp.Zx3ACjxmjm
++ mktemp
+ local LAST_ERR=/tmp/tmp.M09D5u9k3i
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl get svc monitoring-cfg
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.Zx3ACjxmjm
NAME             TYPE        CLUSTER-IP   EXTERNAL-IP   PORT(S)     AGE
monitoring-cfg   ClusterIP   None         <none>        27017/TCP   16m
+ cat /tmp/tmp.M09D5u9k3i
+ rm /tmp/tmp.Zx3ACjxmjm /tmp/tmp.M09D5u9k3i
+ return 0
+ kubectl_bin get svc monitoring-mongos
++ mktemp
+ local LAST_OUT=/tmp/tmp.C1hYJrky07
++ mktemp
+ local LAST_ERR=/tmp/tmp.1qwgR0D84t
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl get svc monitoring-mongos
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.C1hYJrky07
NAME                TYPE        CLUSTER-IP      EXTERNAL-IP   PORT(S)     AGE
monitoring-mongos   ClusterIP   10.249.148.33   <none>        27017/TCP   15m
+ cat /tmp/tmp.1qwgR0D84t
+ rm /tmp/tmp.C1hYJrky07 /tmp/tmp.1qwgR0D84t
+ return 0
+ does_node_id_exists_in_pmm=($(does_node_id_exists "${nodeList[@]}"))
++ does_node_id_exists /node_id/51f35ae2-bdb4-4d66-a273-0fa655cce07e /node_id/e4d1e4a2-a57b-4cb5-8e9e-7ef9abae0bec /node_id/e5bfc490-c328-4b20-ac8d-b90982c0c551 /node_id/9135e58f-5ef2-4263-9284-06b5784d32c4 /node_id/183bb166-4262-4338-b215-e6f12647dfb3 /node_id/f2a4a15c-f4e4-4f16-bbd2-1c43a6653952 /node_id/18235048-c559-4cc3-a045-76317a1f73fc
++ nodeList=("$@")
++ local -a nodeList
++ nodeList_from_pmm=()
++ local -a nodeList_from_pmm
++ for node_id in '"${nodeList[@]}"'
++ nodeList_from_pmm+=($(kubectl_bin exec -n "${namespace}" monitoring-0 -- pmm-admin --server-url=https://admin:admin@$(get_service_ip monitoring-service)/ --server-insecure-tls inventory list nodes --node-type=CONTAINER_NODE | grep $node_id | awk '{print $4}'))
+++ grep /node_id/51f35ae2-bdb4-4d66-a273-0fa655cce07e
+++ awk '{print $4}'
++++ get_service_ip monitoring-service
++++ local service=monitoring-service
++++ local server_type=rs0
+++++ kubectl_bin get psmdb/monitoring-service -o 'jsonpath={.spec.replsets[].expose.enabled}'
++++++ mktemp
+++++ local LAST_OUT=/tmp/tmp.Cia7HT2J3O
++++++ mktemp
+++++ local LAST_ERR=/tmp/tmp.JMfiSXkODN
+++++ local exit_status=0
+++++ local timeout=4
++++++ seq 0 2
+++++ for i in '$(seq 0 2)'
+++++ set +e
+++++ kubectl get psmdb/monitoring-service -o 'jsonpath={.spec.replsets[].expose.enabled}'
+++++ exit_status=1
+++++ set -e
+++++ '[' 1 '!=' 0 -a -n 1 ']'
+++++ cat /tmp/tmp.Cia7HT2J3O
+++++ cat /tmp/tmp.JMfiSXkODN
Error from server (NotFound): perconaservermongodbs.psmdb.percona.com "monitoring-service" not found
+++++ sleep 0
+++++ for i in '$(seq 0 2)'
+++++ set +e
+++++ kubectl get psmdb/monitoring-service -o 'jsonpath={.spec.replsets[].expose.enabled}'
+++++ exit_status=1
+++++ set -e
+++++ '[' 1 '!=' 0 -a -n 1 ']'
+++++ cat /tmp/tmp.Cia7HT2J3O
+++++ cat /tmp/tmp.JMfiSXkODN
Error from server (NotFound): perconaservermongodbs.psmdb.percona.com "monitoring-service" not found
+++++ sleep 4
+++++ for i in '$(seq 0 2)'
+++++ set +e
+++++ kubectl get psmdb/monitoring-service -o 'jsonpath={.spec.replsets[].expose.enabled}'
+++++ exit_status=1
+++++ set -e
+++++ '[' 1 '!=' 0 -a -n 1 ']'
+++++ cat /tmp/tmp.Cia7HT2J3O
+++++ cat /tmp/tmp.JMfiSXkODN
Error from server (NotFound): perconaservermongodbs.psmdb.percona.com "monitoring-service" not found
+++++ sleep 8
+++++ cat /tmp/tmp.Cia7HT2J3O
+++++ cat /tmp/tmp.JMfiSXkODN
Error from server (NotFound): perconaservermongodbs.psmdb.percona.com "monitoring-service" not found
+++++ rm /tmp/tmp.Cia7HT2J3O /tmp/tmp.JMfiSXkODN
+++++ return 1
++++ '[' '' '!=' true ']'
++++ echo -n monitoring-service.monitoring-service-rs0
++++ return
+++ kubectl_bin exec -n monitoring-2-0-31670 monitoring-0 -- pmm-admin --server-url=https://admin:admin@monitoring-service.monitoring-service-rs0/ --server-insecure-tls inventory list nodes --node-type=CONTAINER_NODE
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.P8Lf0o66Wi
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.Xp05tQOmBc
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in '$(seq 0 2)'
+++ set +e
+++ kubectl exec -n monitoring-2-0-31670 monitoring-0 -- pmm-admin --server-url=https://admin:admin@monitoring-service.monitoring-service-rs0/ --server-insecure-tls inventory list nodes --node-type=CONTAINER_NODE
+++ exit_status=1
+++ set -e
+++ '[' 1 '!=' 0 -a -n 1 ']'
+++ cat /tmp/tmp.P8Lf0o66Wi
+++ cat /tmp/tmp.Xp05tQOmBc
command terminated with exit code 1
+++ sleep 0
+++ for i in '$(seq 0 2)'
+++ set +e
+++ kubectl exec -n monitoring-2-0-31670 monitoring-0 -- pmm-admin --server-url=https://admin:admin@monitoring-service.monitoring-service-rs0/ --server-insecure-tls inventory list nodes --node-type=CONTAINER_NODE
+++ exit_status=1
+++ set -e
+++ '[' 1 '!=' 0 -a -n 1 ']'
+++ cat /tmp/tmp.P8Lf0o66Wi
+++ cat /tmp/tmp.Xp05tQOmBc
command terminated with exit code 1
+++ sleep 4
+++ for i in '$(seq 0 2)'
+++ set +e
+++ kubectl exec -n monitoring-2-0-31670 monitoring-0 -- pmm-admin --server-url=https://admin:admin@monitoring-service.monitoring-service-rs0/ --server-insecure-tls inventory list nodes --node-type=CONTAINER_NODE
+++ exit_status=1
+++ set -e
+++ '[' 1 '!=' 0 -a -n 1 ']'
+++ cat /tmp/tmp.P8Lf0o66Wi
+++ cat /tmp/tmp.Xp05tQOmBc
command terminated with exit code 1
+++ sleep 8
+++ cat /tmp/tmp.P8Lf0o66Wi
+++ cat /tmp/tmp.Xp05tQOmBc
command terminated with exit code 1
+++ rm /tmp/tmp.P8Lf0o66Wi /tmp/tmp.Xp05tQOmBc
+++ return 1
++ for node_id in '"${nodeList[@]}"'
++ nodeList_from_pmm+=($(kubectl_bin exec -n "${namespace}" monitoring-0 -- pmm-admin --server-url=https://admin:admin@$(get_service_ip monitoring-service)/ --server-insecure-tls inventory list nodes --node-type=CONTAINER_NODE | grep $node_id | awk '{print $4}'))
+++ grep /node_id/e4d1e4a2-a57b-4cb5-8e9e-7ef9abae0bec
+++ awk '{print $4}'
++++ get_service_ip monitoring-service
++++ local service=monitoring-service
++++ local server_type=rs0
+++++ kubectl_bin get psmdb/monitoring-service -o 'jsonpath={.spec.replsets[].expose.enabled}'
++++++ mktemp
+++++ local LAST_OUT=/tmp/tmp.fBfgGbAoEc
++++++ mktemp
+++++ local LAST_ERR=/tmp/tmp.1vVEvMsc9m
+++++ local exit_status=0
+++++ local timeout=4
++++++ seq 0 2
+++++ for i in '$(seq 0 2)'
+++++ set +e
+++++ kubectl get psmdb/monitoring-service -o 'jsonpath={.spec.replsets[].expose.enabled}'
+++++ exit_status=1
+++++ set -e
+++++ '[' 1 '!=' 0 -a -n 1 ']'
+++++ cat /tmp/tmp.fBfgGbAoEc
+++++ cat /tmp/tmp.1vVEvMsc9m
Error from server (NotFound): perconaservermongodbs.psmdb.percona.com "monitoring-service" not found
+++++ sleep 0
+++++ for i in '$(seq 0 2)'
+++++ set +e
+++++ kubectl get psmdb/monitoring-service -o 'jsonpath={.spec.replsets[].expose.enabled}'
+++++ exit_status=1
+++++ set -e
+++++ '[' 1 '!=' 0 -a -n 1 ']'
+++++ cat /tmp/tmp.fBfgGbAoEc
+++++ cat /tmp/tmp.1vVEvMsc9m
Error from server (NotFound): perconaservermongodbs.psmdb.percona.com "monitoring-service" not found
+++++ sleep 4
+++++ for i in '$(seq 0 2)'
+++++ set +e
+++++ kubectl get psmdb/monitoring-service -o 'jsonpath={.spec.replsets[].expose.enabled}'
+++++ exit_status=1
+++++ set -e
+++++ '[' 1 '!=' 0 -a -n 1 ']'
+++++ cat /tmp/tmp.fBfgGbAoEc
+++++ cat /tmp/tmp.1vVEvMsc9m
Error from server (NotFound): perconaservermongodbs.psmdb.percona.com "monitoring-service" not found
+++++ sleep 8
+++++ cat /tmp/tmp.fBfgGbAoEc
+++++ cat /tmp/tmp.1vVEvMsc9m
Error from server (NotFound): perconaservermongodbs.psmdb.percona.com "monitoring-service" not found
+++++ rm /tmp/tmp.fBfgGbAoEc /tmp/tmp.1vVEvMsc9m
+++++ return 1
++++ '[' '' '!=' true ']'
++++ echo -n monitoring-service.monitoring-service-rs0
++++ return
+++ kubectl_bin exec -n monitoring-2-0-31670 monitoring-0 -- pmm-admin --server-url=https://admin:admin@monitoring-service.monitoring-service-rs0/ --server-insecure-tls inventory list nodes --node-type=CONTAINER_NODE
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.DLyom0PBu1
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.K3wgnXX98n
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in '$(seq 0 2)'
+++ set +e
+++ kubectl exec -n monitoring-2-0-31670 monitoring-0 -- pmm-admin --server-url=https://admin:admin@monitoring-service.monitoring-service-rs0/ --server-insecure-tls inventory list nodes --node-type=CONTAINER_NODE
+++ exit_status=1
+++ set -e
+++ '[' 1 '!=' 0 -a -n 1 ']'
+++ cat /tmp/tmp.DLyom0PBu1
+++ cat /tmp/tmp.K3wgnXX98n
command terminated with exit code 1
+++ sleep 0
+++ for i in '$(seq 0 2)'
+++ set +e
+++ kubectl exec -n monitoring-2-0-31670 monitoring-0 -- pmm-admin --server-url=https://admin:admin@monitoring-service.monitoring-service-rs0/ --server-insecure-tls inventory list nodes --node-type=CONTAINER_NODE
+++ exit_status=1
+++ set -e
+++ '[' 1 '!=' 0 -a -n 1 ']'
+++ cat /tmp/tmp.DLyom0PBu1
+++ cat /tmp/tmp.K3wgnXX98n
command terminated with exit code 1
+++ sleep 4
+++ for i in '$(seq 0 2)'
+++ set +e
+++ kubectl exec -n monitoring-2-0-31670 monitoring-0 -- pmm-admin --server-url=https://admin:admin@monitoring-service.monitoring-service-rs0/ --server-insecure-tls inventory list nodes --node-type=CONTAINER_NODE
+++ exit_status=1
+++ set -e
+++ '[' 1 '!=' 0 -a -n 1 ']'
+++ cat /tmp/tmp.DLyom0PBu1
+++ cat /tmp/tmp.K3wgnXX98n
command terminated with exit code 1
+++ sleep 8
+++ cat /tmp/tmp.DLyom0PBu1
+++ cat /tmp/tmp.K3wgnXX98n
command terminated with exit code 1
+++ rm /tmp/tmp.DLyom0PBu1 /tmp/tmp.K3wgnXX98n
+++ return 1
++ for node_id in '"${nodeList[@]}"'
++ nodeList_from_pmm+=($(kubectl_bin exec -n "${namespace}" monitoring-0 -- pmm-admin --server-url=https://admin:admin@$(get_service_ip monitoring-service)/ --server-insecure-tls inventory list nodes --node-type=CONTAINER_NODE | grep $node_id | awk '{print $4}'))
+++ grep /node_id/e5bfc490-c328-4b20-ac8d-b90982c0c551
+++ awk '{print $4}'
++++ get_service_ip monitoring-service
++++ local service=monitoring-service
++++ local server_type=rs0
+++++ kubectl_bin get psmdb/monitoring-service -o 'jsonpath={.spec.replsets[].expose.enabled}'
++++++ mktemp
+++++ local LAST_OUT=/tmp/tmp.T4k1EiJZbc
++++++ mktemp
+++++ local LAST_ERR=/tmp/tmp.yOSLdYj0Zd
+++++ local exit_status=0
+++++ local timeout=4
++++++ seq 0 2
+++++ for i in '$(seq 0 2)'
+++++ set +e
+++++ kubectl get psmdb/monitoring-service -o 'jsonpath={.spec.replsets[].expose.enabled}'
+++++ exit_status=1
+++++ set -e
+++++ '[' 1 '!=' 0 -a -n 1 ']'
+++++ cat /tmp/tmp.T4k1EiJZbc
+++++ cat /tmp/tmp.yOSLdYj0Zd
Error from server (NotFound): perconaservermongodbs.psmdb.percona.com "monitoring-service" not found
+++++ sleep 0
+++++ for i in '$(seq 0 2)'
+++++ set +e
+++++ kubectl get psmdb/monitoring-service -o 'jsonpath={.spec.replsets[].expose.enabled}'
+++++ exit_status=1
+++++ set -e
+++++ '[' 1 '!=' 0 -a -n 1 ']'
+++++ cat /tmp/tmp.T4k1EiJZbc
+++++ cat /tmp/tmp.yOSLdYj0Zd
Error from server (NotFound): perconaservermongodbs.psmdb.percona.com "monitoring-service" not found
+++++ sleep 4
+++++ for i in '$(seq 0 2)'
+++++ set +e
+++++ kubectl get psmdb/monitoring-service -o 'jsonpath={.spec.replsets[].expose.enabled}'
+++++ exit_status=1
+++++ set -e
+++++ '[' 1 '!=' 0 -a -n 1 ']'
+++++ cat /tmp/tmp.T4k1EiJZbc
+++++ cat /tmp/tmp.yOSLdYj0Zd
Error from server (NotFound): perconaservermongodbs.psmdb.percona.com "monitoring-service" not found
+++++ sleep 8
+++++ cat /tmp/tmp.T4k1EiJZbc
+++++ cat /tmp/tmp.yOSLdYj0Zd
Error from server (NotFound): perconaservermongodbs.psmdb.percona.com "monitoring-service" not found
+++++ rm /tmp/tmp.T4k1EiJZbc /tmp/tmp.yOSLdYj0Zd
+++++ return 1
++++ '[' '' '!=' true ']'
++++ echo -n monitoring-service.monitoring-service-rs0
++++ return
+++ kubectl_bin exec -n monitoring-2-0-31670 monitoring-0 -- pmm-admin --server-url=https://admin:admin@monitoring-service.monitoring-service-rs0/ --server-insecure-tls inventory list nodes --node-type=CONTAINER_NODE
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.wGNjDUZ21J
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.TJKws7z1xW
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in '$(seq 0 2)'
+++ set +e
+++ kubectl exec -n monitoring-2-0-31670 monitoring-0 -- pmm-admin --server-url=https://admin:admin@monitoring-service.monitoring-service-rs0/ --server-insecure-tls inventory list nodes --node-type=CONTAINER_NODE
+++ exit_status=1
+++ set -e
+++ '[' 1 '!=' 0 -a -n 1 ']'
+++ cat /tmp/tmp.wGNjDUZ21J
+++ cat /tmp/tmp.TJKws7z1xW
command terminated with exit code 1
+++ sleep 0
+++ for i in '$(seq 0 2)'
+++ set +e
+++ kubectl exec -n monitoring-2-0-31670 monitoring-0 -- pmm-admin --server-url=https://admin:admin@monitoring-service.monitoring-service-rs0/ --server-insecure-tls inventory list nodes --node-type=CONTAINER_NODE
+++ exit_status=1
+++ set -e
+++ '[' 1 '!=' 0 -a -n 1 ']'
+++ cat /tmp/tmp.wGNjDUZ21J
+++ cat /tmp/tmp.TJKws7z1xW
command terminated with exit code 1
+++ sleep 4
+++ for i in '$(seq 0 2)'
+++ set +e
+++ kubectl exec -n monitoring-2-0-31670 monitoring-0 -- pmm-admin --server-url=https://admin:admin@monitoring-service.monitoring-service-rs0/ --server-insecure-tls inventory list nodes --node-type=CONTAINER_NODE
+++ exit_status=1
+++ set -e
+++ '[' 1 '!=' 0 -a -n 1 ']'
+++ cat /tmp/tmp.wGNjDUZ21J
+++ cat /tmp/tmp.TJKws7z1xW
command terminated with exit code 1
+++ sleep 8
+++ cat /tmp/tmp.wGNjDUZ21J
+++ cat /tmp/tmp.TJKws7z1xW
command terminated with exit code 1
+++ rm /tmp/tmp.wGNjDUZ21J /tmp/tmp.TJKws7z1xW
+++ return 1
++ for node_id in '"${nodeList[@]}"'
++ nodeList_from_pmm+=($(kubectl_bin exec -n "${namespace}" monitoring-0 -- pmm-admin --server-url=https://admin:admin@$(get_service_ip monitoring-service)/ --server-insecure-tls inventory list nodes --node-type=CONTAINER_NODE | grep $node_id | awk '{print $4}'))
+++ grep /node_id/9135e58f-5ef2-4263-9284-06b5784d32c4
+++ awk '{print $4}'
++++ get_service_ip monitoring-service
++++ local service=monitoring-service
++++ local server_type=rs0
+++++ kubectl_bin get psmdb/monitoring-service -o 'jsonpath={.spec.replsets[].expose.enabled}'
++++++ mktemp
+++++ local LAST_OUT=/tmp/tmp.7dsMCc46FS
++++++ mktemp
+++++ local LAST_ERR=/tmp/tmp.qS9FSmlKt9
+++++ local exit_status=0
+++++ local timeout=4
++++++ seq 0 2
+++++ for i in '$(seq 0 2)'
+++++ set +e
+++++ kubectl get psmdb/monitoring-service -o 'jsonpath={.spec.replsets[].expose.enabled}'
+++++ exit_status=1
+++++ set -e
+++++ '[' 1 '!=' 0 -a -n 1 ']'
+++++ cat /tmp/tmp.7dsMCc46FS
+++++ cat /tmp/tmp.qS9FSmlKt9
Error from server (NotFound): perconaservermongodbs.psmdb.percona.com "monitoring-service" not found
+++++ sleep 0
+++++ for i in '$(seq 0 2)'
+++++ set +e
+++++ kubectl get psmdb/monitoring-service -o 'jsonpath={.spec.replsets[].expose.enabled}'
+++++ exit_status=1
+++++ set -e
+++++ '[' 1 '!=' 0 -a -n 1 ']'
+++++ cat /tmp/tmp.7dsMCc46FS
+++++ cat /tmp/tmp.qS9FSmlKt9
Error from server (NotFound): perconaservermongodbs.psmdb.percona.com "monitoring-service" not found
+++++ sleep 4
+++++ for i in '$(seq 0 2)'
+++++ set +e
+++++ kubectl get psmdb/monitoring-service -o 'jsonpath={.spec.replsets[].expose.enabled}'
+++++ exit_status=1
+++++ set -e
+++++ '[' 1 '!=' 0 -a -n 1 ']'
+++++ cat /tmp/tmp.7dsMCc46FS
+++++ cat /tmp/tmp.qS9FSmlKt9
Error from server (NotFound): perconaservermongodbs.psmdb.percona.com "monitoring-service" not found
+++++ sleep 8
+++++ cat /tmp/tmp.7dsMCc46FS
+++++ cat /tmp/tmp.qS9FSmlKt9
Error from server (NotFound): perconaservermongodbs.psmdb.percona.com "monitoring-service" not found
+++++ rm /tmp/tmp.7dsMCc46FS /tmp/tmp.qS9FSmlKt9
+++++ return 1
++++ '[' '' '!=' true ']'
++++ echo -n monitoring-service.monitoring-service-rs0
++++ return
+++ kubectl_bin exec -n monitoring-2-0-31670 monitoring-0 -- pmm-admin --server-url=https://admin:admin@monitoring-service.monitoring-service-rs0/ --server-insecure-tls inventory list nodes --node-type=CONTAINER_NODE
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.15CiSKWS0j
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.I4WXQ0yflA
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in '$(seq 0 2)'
+++ set +e
+++ kubectl exec -n monitoring-2-0-31670 monitoring-0 -- pmm-admin --server-url=https://admin:admin@monitoring-service.monitoring-service-rs0/ --server-insecure-tls inventory list nodes --node-type=CONTAINER_NODE
+++ exit_status=1
+++ set -e
+++ '[' 1 '!=' 0 -a -n 1 ']'
+++ cat /tmp/tmp.15CiSKWS0j
+++ cat /tmp/tmp.I4WXQ0yflA
command terminated with exit code 1
+++ sleep 0
+++ for i in '$(seq 0 2)'
+++ set +e
+++ kubectl exec -n monitoring-2-0-31670 monitoring-0 -- pmm-admin --server-url=https://admin:admin@monitoring-service.monitoring-service-rs0/ --server-insecure-tls inventory list nodes --node-type=CONTAINER_NODE
+++ exit_status=1
+++ set -e
+++ '[' 1 '!=' 0 -a -n 1 ']'
+++ cat /tmp/tmp.15CiSKWS0j
+++ cat /tmp/tmp.I4WXQ0yflA
command terminated with exit code 1
+++ sleep 4
+++ for i in '$(seq 0 2)'
+++ set +e
+++ kubectl exec -n monitoring-2-0-31670 monitoring-0 -- pmm-admin --server-url=https://admin:admin@monitoring-service.monitoring-service-rs0/ --server-insecure-tls inventory list nodes --node-type=CONTAINER_NODE
+++ exit_status=1
+++ set -e
+++ '[' 1 '!=' 0 -a -n 1 ']'
+++ cat /tmp/tmp.15CiSKWS0j
+++ cat /tmp/tmp.I4WXQ0yflA
command terminated with exit code 1
+++ sleep 8
+++ cat /tmp/tmp.15CiSKWS0j
+++ cat /tmp/tmp.I4WXQ0yflA
command terminated with exit code 1
+++ rm /tmp/tmp.15CiSKWS0j /tmp/tmp.I4WXQ0yflA
+++ return 1
++ for node_id in '"${nodeList[@]}"'
++ nodeList_from_pmm+=($(kubectl_bin exec -n "${namespace}" monitoring-0 -- pmm-admin --server-url=https://admin:admin@$(get_service_ip monitoring-service)/ --server-insecure-tls inventory list nodes --node-type=CONTAINER_NODE | grep $node_id | awk '{print $4}'))
+++ grep /node_id/183bb166-4262-4338-b215-e6f12647dfb3
+++ awk '{print $4}'
++++ get_service_ip monitoring-service
++++ local service=monitoring-service
++++ local server_type=rs0
+++++ kubectl_bin get psmdb/monitoring-service -o 'jsonpath={.spec.replsets[].expose.enabled}'
++++++ mktemp
+++++ local LAST_OUT=/tmp/tmp.PqY3k9t2Dn
++++++ mktemp
+++++ local LAST_ERR=/tmp/tmp.R53nY9j3ql
+++++ local exit_status=0
+++++ local timeout=4
++++++ seq 0 2
+++++ for i in '$(seq 0 2)'
+++++ set +e
+++++ kubectl get psmdb/monitoring-service -o 'jsonpath={.spec.replsets[].expose.enabled}'
+++++ exit_status=1
+++++ set -e
+++++ '[' 1 '!=' 0 -a -n 1 ']'
+++++ cat /tmp/tmp.PqY3k9t2Dn
+++++ cat /tmp/tmp.R53nY9j3ql
Error from server (NotFound): perconaservermongodbs.psmdb.percona.com "monitoring-service" not found
+++++ sleep 0
+++++ for i in '$(seq 0 2)'
+++++ set +e
+++++ kubectl get psmdb/monitoring-service -o 'jsonpath={.spec.replsets[].expose.enabled}'
+++++ exit_status=1
+++++ set -e
+++++ '[' 1 '!=' 0 -a -n 1 ']'
+++++ cat /tmp/tmp.PqY3k9t2Dn
+++++ cat /tmp/tmp.R53nY9j3ql
Error from server (NotFound): perconaservermongodbs.psmdb.percona.com "monitoring-service" not found
+++++ sleep 4
+++++ for i in '$(seq 0 2)'
+++++ set +e
+++++ kubectl get psmdb/monitoring-service -o 'jsonpath={.spec.replsets[].expose.enabled}'
+++++ exit_status=1
+++++ set -e
+++++ '[' 1 '!=' 0 -a -n 1 ']'
+++++ cat /tmp/tmp.PqY3k9t2Dn
+++++ cat /tmp/tmp.R53nY9j3ql
Error from server (NotFound): perconaservermongodbs.psmdb.percona.com "monitoring-service" not found
+++++ sleep 8
+++++ cat /tmp/tmp.PqY3k9t2Dn
+++++ cat /tmp/tmp.R53nY9j3ql
Error from server (NotFound): perconaservermongodbs.psmdb.percona.com "monitoring-service" not found
+++++ rm /tmp/tmp.PqY3k9t2Dn /tmp/tmp.R53nY9j3ql
+++++ return 1
++++ '[' '' '!=' true ']'
++++ echo -n monitoring-service.monitoring-service-rs0
++++ return
+++ kubectl_bin exec -n monitoring-2-0-31670 monitoring-0 -- pmm-admin --server-url=https://admin:admin@monitoring-service.monitoring-service-rs0/ --server-insecure-tls inventory list nodes --node-type=CONTAINER_NODE
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.tswJWQj9jD
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.K1ofVLwumU
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in '$(seq 0 2)'
+++ set +e
+++ kubectl exec -n monitoring-2-0-31670 monitoring-0 -- pmm-admin --server-url=https://admin:admin@monitoring-service.monitoring-service-rs0/ --server-insecure-tls inventory list nodes --node-type=CONTAINER_NODE
+++ exit_status=1
+++ set -e
+++ '[' 1 '!=' 0 -a -n 1 ']'
+++ cat /tmp/tmp.tswJWQj9jD
+++ cat /tmp/tmp.K1ofVLwumU
command terminated with exit code 1
+++ sleep 0
+++ for i in '$(seq 0 2)'
+++ set +e
+++ kubectl exec -n monitoring-2-0-31670 monitoring-0 -- pmm-admin --server-url=https://admin:admin@monitoring-service.monitoring-service-rs0/ --server-insecure-tls inventory list nodes --node-type=CONTAINER_NODE
+++ exit_status=1
+++ set -e
+++ '[' 1 '!=' 0 -a -n 1 ']'
+++ cat /tmp/tmp.tswJWQj9jD
+++ cat /tmp/tmp.K1ofVLwumU
command terminated with exit code 1
+++ sleep 4
+++ for i in '$(seq 0 2)'
+++ set +e
+++ kubectl exec -n monitoring-2-0-31670 monitoring-0 -- pmm-admin --server-url=https://admin:admin@monitoring-service.monitoring-service-rs0/ --server-insecure-tls inventory list nodes --node-type=CONTAINER_NODE
+++ exit_status=1
+++ set -e
+++ '[' 1 '!=' 0 -a -n 1 ']'
+++ cat /tmp/tmp.tswJWQj9jD
+++ cat /tmp/tmp.K1ofVLwumU
command terminated with exit code 1
+++ sleep 8
+++ cat /tmp/tmp.tswJWQj9jD
+++ cat /tmp/tmp.K1ofVLwumU
command terminated with exit code 1
+++ rm /tmp/tmp.tswJWQj9jD /tmp/tmp.K1ofVLwumU
+++ return 1
++ for node_id in '"${nodeList[@]}"'
++ nodeList_from_pmm+=($(kubectl_bin exec -n "${namespace}" monitoring-0 -- pmm-admin --server-url=https://admin:admin@$(get_service_ip monitoring-service)/ --server-insecure-tls inventory list nodes --node-type=CONTAINER_NODE | grep $node_id | awk '{print $4}'))
+++ grep /node_id/f2a4a15c-f4e4-4f16-bbd2-1c43a6653952
+++ awk '{print $4}'
++++ get_service_ip monitoring-service
++++ local service=monitoring-service
++++ local server_type=rs0
+++++ kubectl_bin get psmdb/monitoring-service -o 'jsonpath={.spec.replsets[].expose.enabled}'
++++++ mktemp
+++++ local LAST_OUT=/tmp/tmp.kIPOqKmuvs
++++++ mktemp
+++++ local LAST_ERR=/tmp/tmp.UvHGPpE9bz
+++++ local exit_status=0
+++++ local timeout=4
++++++ seq 0 2
+++++ for i in '$(seq 0 2)'
+++++ set +e
+++++ kubectl get psmdb/monitoring-service -o 'jsonpath={.spec.replsets[].expose.enabled}'
+++++ exit_status=1
+++++ set -e
+++++ '[' 1 '!=' 0 -a -n 1 ']'
+++++ cat /tmp/tmp.kIPOqKmuvs
+++++ cat /tmp/tmp.UvHGPpE9bz
Error from server (NotFound): perconaservermongodbs.psmdb.percona.com "monitoring-service" not found
+++++ sleep 0
+++++ for i in '$(seq 0 2)'
+++++ set +e
+++++ kubectl get psmdb/monitoring-service -o 'jsonpath={.spec.replsets[].expose.enabled}'
+++++ exit_status=1
+++++ set -e
+++++ '[' 1 '!=' 0 -a -n 1 ']'
+++++ cat /tmp/tmp.kIPOqKmuvs
+++++ cat /tmp/tmp.UvHGPpE9bz
Error from server (NotFound): perconaservermongodbs.psmdb.percona.com "monitoring-service" not found
+++++ sleep 4
+++++ for i in '$(seq 0 2)'
+++++ set +e
+++++ kubectl get psmdb/monitoring-service -o 'jsonpath={.spec.replsets[].expose.enabled}'
+++++ exit_status=1
+++++ set -e
+++++ '[' 1 '!=' 0 -a -n 1 ']'
+++++ cat /tmp/tmp.kIPOqKmuvs
+++++ cat /tmp/tmp.UvHGPpE9bz
Error from server (NotFound): perconaservermongodbs.psmdb.percona.com "monitoring-service" not found
+++++ sleep 8
+++++ cat /tmp/tmp.kIPOqKmuvs
+++++ cat /tmp/tmp.UvHGPpE9bz
Error from server (NotFound): perconaservermongodbs.psmdb.percona.com "monitoring-service" not found
+++++ rm /tmp/tmp.kIPOqKmuvs /tmp/tmp.UvHGPpE9bz
+++++ return 1
++++ '[' '' '!=' true ']'
++++ echo -n monitoring-service.monitoring-service-rs0
++++ return
+++ kubectl_bin exec -n monitoring-2-0-31670 monitoring-0 -- pmm-admin --server-url=https://admin:admin@monitoring-service.monitoring-service-rs0/ --server-insecure-tls inventory list nodes --node-type=CONTAINER_NODE
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.HBH3ob0TSr
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.x3Rie1qYAy
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in '$(seq 0 2)'
+++ set +e
+++ kubectl exec -n monitoring-2-0-31670 monitoring-0 -- pmm-admin --server-url=https://admin:admin@monitoring-service.monitoring-service-rs0/ --server-insecure-tls inventory list nodes --node-type=CONTAINER_NODE
+++ exit_status=1
+++ set -e
+++ '[' 1 '!=' 0 -a -n 1 ']'
+++ cat /tmp/tmp.HBH3ob0TSr
+++ cat /tmp/tmp.x3Rie1qYAy
command terminated with exit code 1
+++ sleep 0
+++ for i in '$(seq 0 2)'
+++ set +e
+++ kubectl exec -n monitoring-2-0-31670 monitoring-0 -- pmm-admin --server-url=https://admin:admin@monitoring-service.monitoring-service-rs0/ --server-insecure-tls inventory list nodes --node-type=CONTAINER_NODE
+++ exit_status=1
+++ set -e
+++ '[' 1 '!=' 0 -a -n 1 ']'
+++ cat /tmp/tmp.HBH3ob0TSr
+++ cat /tmp/tmp.x3Rie1qYAy
command terminated with exit code 1
+++ sleep 4
+++ for i in '$(seq 0 2)'
+++ set +e
+++ kubectl exec -n monitoring-2-0-31670 monitoring-0 -- pmm-admin --server-url=https://admin:admin@monitoring-service.monitoring-service-rs0/ --server-insecure-tls inventory list nodes --node-type=CONTAINER_NODE
+++ exit_status=1
+++ set -e
+++ '[' 1 '!=' 0 -a -n 1 ']'
+++ cat /tmp/tmp.HBH3ob0TSr
+++ cat /tmp/tmp.x3Rie1qYAy
command terminated with exit code 1
+++ sleep 8
+++ cat /tmp/tmp.HBH3ob0TSr
+++ cat /tmp/tmp.x3Rie1qYAy
command terminated with exit code 1
+++ rm /tmp/tmp.HBH3ob0TSr /tmp/tmp.x3Rie1qYAy
+++ return 1
++ for node_id in '"${nodeList[@]}"'
++ nodeList_from_pmm+=($(kubectl_bin exec -n "${namespace}" monitoring-0 -- pmm-admin --server-url=https://admin:admin@$(get_service_ip monitoring-service)/ --server-insecure-tls inventory list nodes --node-type=CONTAINER_NODE | grep $node_id | awk '{print $4}'))
+++ grep /node_id/18235048-c559-4cc3-a045-76317a1f73fc
++++ get_service_ip monitoring-service
+++ awk '{print $4}'
++++ local service=monitoring-service
++++ local server_type=rs0
+++++ kubectl_bin get psmdb/monitoring-service -o 'jsonpath={.spec.replsets[].expose.enabled}'
++++++ mktemp
+++++ local LAST_OUT=/tmp/tmp.qpIKjOAvpd
++++++ mktemp
+++++ local LAST_ERR=/tmp/tmp.eay6pEFd3D
+++++ local exit_status=0
+++++ local timeout=4
++++++ seq 0 2
+++++ for i in '$(seq 0 2)'
+++++ set +e
+++++ kubectl get psmdb/monitoring-service -o 'jsonpath={.spec.replsets[].expose.enabled}'
+++++ exit_status=1
+++++ set -e
+++++ '[' 1 '!=' 0 -a -n 1 ']'
+++++ cat /tmp/tmp.qpIKjOAvpd
+++++ cat /tmp/tmp.eay6pEFd3D
Error from server (NotFound): perconaservermongodbs.psmdb.percona.com "monitoring-service" not found
+++++ sleep 0
+++++ for i in '$(seq 0 2)'
+++++ set +e
+++++ kubectl get psmdb/monitoring-service -o 'jsonpath={.spec.replsets[].expose.enabled}'
+++++ exit_status=1
+++++ set -e
+++++ '[' 1 '!=' 0 -a -n 1 ']'
+++++ cat /tmp/tmp.qpIKjOAvpd
+++++ cat /tmp/tmp.eay6pEFd3D
Error from server (NotFound): perconaservermongodbs.psmdb.percona.com "monitoring-service" not found
+++++ sleep 4
+++++ for i in '$(seq 0 2)'
+++++ set +e
+++++ kubectl get psmdb/monitoring-service -o 'jsonpath={.spec.replsets[].expose.enabled}'
+++++ exit_status=1
+++++ set -e
+++++ '[' 1 '!=' 0 -a -n 1 ']'
+++++ cat /tmp/tmp.qpIKjOAvpd
+++++ cat /tmp/tmp.eay6pEFd3D
Error from server (NotFound): perconaservermongodbs.psmdb.percona.com "monitoring-service" not found
+++++ sleep 8
+++++ cat /tmp/tmp.qpIKjOAvpd
+++++ cat /tmp/tmp.eay6pEFd3D
Error from server (NotFound): perconaservermongodbs.psmdb.percona.com "monitoring-service" not found
+++++ rm /tmp/tmp.qpIKjOAvpd /tmp/tmp.eay6pEFd3D
+++++ return 1
++++ '[' '' '!=' true ']'
++++ echo -n monitoring-service.monitoring-service-rs0
++++ return
+++ kubectl_bin exec -n monitoring-2-0-31670 monitoring-0 -- pmm-admin --server-url=https://admin:admin@monitoring-service.monitoring-service-rs0/ --server-insecure-tls inventory list nodes --node-type=CONTAINER_NODE
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.SNG55mpQhr
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.CFYE3ZEt4z
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in '$(seq 0 2)'
+++ set +e
+++ kubectl exec -n monitoring-2-0-31670 monitoring-0 -- pmm-admin --server-url=https://admin:admin@monitoring-service.monitoring-service-rs0/ --server-insecure-tls inventory list nodes --node-type=CONTAINER_NODE
+++ exit_status=1
+++ set -e
+++ '[' 1 '!=' 0 -a -n 1 ']'
+++ cat /tmp/tmp.SNG55mpQhr
+++ cat /tmp/tmp.CFYE3ZEt4z
command terminated with exit code 1
+++ sleep 0
+++ for i in '$(seq 0 2)'
+++ set +e
+++ kubectl exec -n monitoring-2-0-31670 monitoring-0 -- pmm-admin --server-url=https://admin:admin@monitoring-service.monitoring-service-rs0/ --server-insecure-tls inventory list nodes --node-type=CONTAINER_NODE
+++ exit_status=1
+++ set -e
+++ '[' 1 '!=' 0 -a -n 1 ']'
+++ cat /tmp/tmp.SNG55mpQhr
+++ cat /tmp/tmp.CFYE3ZEt4z
command terminated with exit code 1
+++ sleep 4
+++ for i in '$(seq 0 2)'
+++ set +e
+++ kubectl exec -n monitoring-2-0-31670 monitoring-0 -- pmm-admin --server-url=https://admin:admin@monitoring-service.monitoring-service-rs0/ --server-insecure-tls inventory list nodes --node-type=CONTAINER_NODE
+++ exit_status=1
+++ set -e
+++ '[' 1 '!=' 0 -a -n 1 ']'
+++ cat /tmp/tmp.SNG55mpQhr
+++ cat /tmp/tmp.CFYE3ZEt4z
command terminated with exit code 1
+++ sleep 8
+++ cat /tmp/tmp.SNG55mpQhr
+++ cat /tmp/tmp.CFYE3ZEt4z
command terminated with exit code 1
+++ rm /tmp/tmp.SNG55mpQhr /tmp/tmp.CFYE3ZEt4z
+++ return 1
++ echo
+ [[ -n '' ]]
++ kubectl_bin logs monitoring-rs0-0 pmm-client
++ grep -c 'cannot auto discover databases and collections'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.kUCPShyItw
+++ mktemp
++ local LAST_ERR=/tmp/tmp.PYON9rmbqE
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl logs monitoring-rs0-0 pmm-client
++ exit_status=1
++ set -e
++ '[' 1 '!=' 0 -a -n 1 ']'
++ cat /tmp/tmp.kUCPShyItw
++ cat /tmp/tmp.PYON9rmbqE
error: error from server (NotFound): pods "monitoring-rs0-0" not found in namespace "monitoring-2-0-31670"
++ sleep 0
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl logs monitoring-rs0-0 pmm-client
++ exit_status=1
++ set -e
++ '[' 1 '!=' 0 -a -n 1 ']'
++ cat /tmp/tmp.kUCPShyItw
++ cat /tmp/tmp.PYON9rmbqE
error: error from server (NotFound): pods "monitoring-rs0-0" not found in namespace "monitoring-2-0-31670"
++ sleep 4
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl logs monitoring-rs0-0 pmm-client
++ exit_status=1
++ set -e
++ '[' 1 '!=' 0 -a -n 1 ']'
++ cat /tmp/tmp.kUCPShyItw
++ cat /tmp/tmp.PYON9rmbqE
error: error from server (NotFound): pods "monitoring-rs0-0" not found in namespace "monitoring-2-0-31670"
++ sleep 8
++ cat /tmp/tmp.kUCPShyItw
++ cat /tmp/tmp.PYON9rmbqE
error: error from server (NotFound): pods "monitoring-rs0-0" not found in namespace "monitoring-2-0-31670"
++ rm /tmp/tmp.kUCPShyItw /tmp/tmp.PYON9rmbqE
++ return 1
+ [[ 0 != 0 ]]
+ desc 'check for passwords leak'
+ set +o xtrace


-----------------------------------------------------------------------------------
check for passwords leak
-----------------------------------------------------------------------------------

+ check_passwords_leak
+ local secrets
+ local passwords
+ local pods
++ kubectl_bin get secrets -o json
++ jq -r '.items[].data | to_entries | .[] | select(.key | (contains("_PASSWORD"))) | .value'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.nniOEGqBXi
+++ mktemp
++ local LAST_ERR=/tmp/tmp.xQuxD00EJl
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get secrets -o json
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.nniOEGqBXi
++ cat /tmp/tmp.xQuxD00EJl
++ rm /tmp/tmp.nniOEGqBXi /tmp/tmp.xQuxD00EJl
++ return 0
+ secrets='YmFja3VwMTIzNDU2
Y2x1c3RlckFkbWluMTIzNDU2
Y2x1c3Rlck1vbml0b3IxMjM0NTY=
ZGF0YWJhc2VBZG1pbjEyMzQ1Ng==
dXNlckFkbWluMTIzNDU2
YmFja3VwMTIzNDU2
Y2x1c3RlckFkbWluMTIzNDU2
Y2x1c3Rlck1vbml0b3IxMjM0NTY=
ZGF0YWJhc2VBZG1pbjEyMzQ1Ng==
dXNlckFkbWluMTIzNDU2'
+ echo secrets=YmFja3VwMTIzNDU2 Y2x1c3RlckFkbWluMTIzNDU2 Y2x1c3Rlck1vbml0b3IxMjM0NTY= ZGF0YWJhc2VBZG1pbjEyMzQ1Ng== dXNlckFkbWluMTIzNDU2 YmFja3VwMTIzNDU2 Y2x1c3RlckFkbWluMTIzNDU2 Y2x1c3Rlck1vbml0b3IxMjM0NTY= ZGF0YWJhc2VBZG1pbjEyMzQ1Ng== dXNlckFkbWluMTIzNDU2
secrets=YmFja3VwMTIzNDU2 Y2x1c3RlckFkbWluMTIzNDU2 Y2x1c3Rlck1vbml0b3IxMjM0NTY= ZGF0YWJhc2VBZG1pbjEyMzQ1Ng== dXNlckFkbWluMTIzNDU2 YmFja3VwMTIzNDU2 Y2x1c3RlckFkbWluMTIzNDU2 Y2x1c3Rlck1vbml0b3IxMjM0NTY= ZGF0YWJhc2VBZG1pbjEyMzQ1Ng== dXNlckFkbWluMTIzNDU2
++ for i in '$secrets'
++ base64 -d
++ echo
++ for i in '$secrets'
++ base64 -d
++ echo
++ for i in '$secrets'
++ base64 -d
++ echo
++ for i in '$secrets'
++ base64 -d
++ echo
++ for i in '$secrets'
++ base64 -d
++ echo
++ for i in '$secrets'
++ base64 -d
++ echo
++ for i in '$secrets'
++ base64 -d
++ echo
++ for i in '$secrets'
++ base64 -d
++ echo
++ for i in '$secrets'
++ base64 -d
++ echo
++ for i in '$secrets'
++ base64 -d
++ echo
+ passwords='backup123456
clusterAdmin123456
clusterMonitor123456
databaseAdmin123456
userAdmin123456
backup123456
clusterAdmin123456
clusterMonitor123456
databaseAdmin123456
userAdmin123456 YmFja3VwMTIzNDU2
Y2x1c3RlckFkbWluMTIzNDU2
Y2x1c3Rlck1vbml0b3IxMjM0NTY=
ZGF0YWJhc2VBZG1pbjEyMzQ1Ng==
dXNlckFkbWluMTIzNDU2
YmFja3VwMTIzNDU2
Y2x1c3RlckFkbWluMTIzNDU2
Y2x1c3Rlck1vbml0b3IxMjM0NTY=
ZGF0YWJhc2VBZG1pbjEyMzQ1Ng==
dXNlckFkbWluMTIzNDU2'
+ echo passwords=backup123456 clusterAdmin123456 clusterMonitor123456 databaseAdmin123456 userAdmin123456 backup123456 clusterAdmin123456 clusterMonitor123456 databaseAdmin123456 userAdmin123456 YmFja3VwMTIzNDU2 Y2x1c3RlckFkbWluMTIzNDU2 Y2x1c3Rlck1vbml0b3IxMjM0NTY= ZGF0YWJhc2VBZG1pbjEyMzQ1Ng== dXNlckFkbWluMTIzNDU2 YmFja3VwMTIzNDU2 Y2x1c3RlckFkbWluMTIzNDU2 Y2x1c3Rlck1vbml0b3IxMjM0NTY= ZGF0YWJhc2VBZG1pbjEyMzQ1Ng== dXNlckFkbWluMTIzNDU2
passwords=backup123456 clusterAdmin123456 clusterMonitor123456 databaseAdmin123456 userAdmin123456 backup123456 clusterAdmin123456 clusterMonitor123456 databaseAdmin123456 userAdmin123456 YmFja3VwMTIzNDU2 Y2x1c3RlckFkbWluMTIzNDU2 Y2x1c3Rlck1vbml0b3IxMjM0NTY= ZGF0YWJhc2VBZG1pbjEyMzQ1Ng== dXNlckFkbWluMTIzNDU2 YmFja3VwMTIzNDU2 Y2x1c3RlckFkbWluMTIzNDU2 Y2x1c3Rlck1vbml0b3IxMjM0NTY= ZGF0YWJhc2VBZG1pbjEyMzQ1Ng== dXNlckFkbWluMTIzNDU2
++ kubectl_bin get pods -o name
++ awk -F / '{print $2}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.R9vb19ldWv
+++ mktemp
++ local LAST_ERR=/tmp/tmp.dQiXI6wpmM
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods -o name
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.R9vb19ldWv
++ cat /tmp/tmp.dQiXI6wpmM
++ rm /tmp/tmp.R9vb19ldWv /tmp/tmp.dQiXI6wpmM
++ return 0
+ pods='monitoring-0
psmdb-client-6cd48df8b6-t28bz'
+ echo pods=monitoring-0 psmdb-client-6cd48df8b6-t28bz
pods=monitoring-0 psmdb-client-6cd48df8b6-t28bz
+ collect_logs monitoring-2-0-31670
+ local containers
+ local count
+ NS=monitoring-2-0-31670
+ for p in '$pods'
++ kubectl_bin -n monitoring-2-0-31670 get pod monitoring-0 -o 'jsonpath={.spec.containers[*].name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.QadatZoDKH
+++ mktemp
++ local LAST_ERR=/tmp/tmp.RsjcbXkQIH
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl -n monitoring-2-0-31670 get pod monitoring-0 -o 'jsonpath={.spec.containers[*].name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.QadatZoDKH
++ cat /tmp/tmp.RsjcbXkQIH
++ rm /tmp/tmp.QadatZoDKH /tmp/tmp.RsjcbXkQIH
++ return 0
+ containers=monitoring
+ for c in '$containers'
+ [[ monitoring =~ pmm ]]
+ kubectl_bin -n monitoring-2-0-31670 logs monitoring-0 -c monitoring
++ mktemp
+ local LAST_OUT=/tmp/tmp.5REa1swYXm
++ mktemp
+ local LAST_ERR=/tmp/tmp.56FsIrRvfP
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl -n monitoring-2-0-31670 logs monitoring-0 -c monitoring
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.5REa1swYXm
+ cat /tmp/tmp.56FsIrRvfP
+ rm /tmp/tmp.5REa1swYXm /tmp/tmp.56FsIrRvfP
+ return 0
+ echo logs saved in: /tmp/tmp.2rzkpQHbaz/logs_output-monitoring-0-monitoring.txt
logs saved in: /tmp/tmp.2rzkpQHbaz/logs_output-monitoring-0-monitoring.txt
+ for pass in '$passwords'
++ grep -c --fixed-strings -- backup123456 /tmp/tmp.2rzkpQHbaz/logs_output-monitoring-0-monitoring.txt
++ :
+ count=0
+ [[ 0 != 0 ]]
+ for pass in '$passwords'
++ grep -c --fixed-strings -- clusterAdmin123456 /tmp/tmp.2rzkpQHbaz/logs_output-monitoring-0-monitoring.txt
++ :
+ count=0
+ [[ 0 != 0 ]]
+ for pass in '$passwords'
++ grep -c --fixed-strings -- clusterMonitor123456 /tmp/tmp.2rzkpQHbaz/logs_output-monitoring-0-monitoring.txt
++ :
+ count=0
+ [[ 0 != 0 ]]
+ for pass in '$passwords'
++ grep -c --fixed-strings -- databaseAdmin123456 /tmp/tmp.2rzkpQHbaz/logs_output-monitoring-0-monitoring.txt
++ :
+ count=0
+ [[ 0 != 0 ]]
+ for pass in '$passwords'
++ grep -c --fixed-strings -- userAdmin123456 /tmp/tmp.2rzkpQHbaz/logs_output-monitoring-0-monitoring.txt
++ :
+ count=0
+ [[ 0 != 0 ]]
+ for pass in '$passwords'
++ grep -c --fixed-strings -- backup123456 /tmp/tmp.2rzkpQHbaz/logs_output-monitoring-0-monitoring.txt
++ :
+ count=0
+ [[ 0 != 0 ]]
+ for pass in '$passwords'
++ grep -c --fixed-strings -- clusterAdmin123456 /tmp/tmp.2rzkpQHbaz/logs_output-monitoring-0-monitoring.txt
++ :
+ count=0
+ [[ 0 != 0 ]]
+ for pass in '$passwords'
++ grep -c --fixed-strings -- clusterMonitor123456 /tmp/tmp.2rzkpQHbaz/logs_output-monitoring-0-monitoring.txt
++ :
+ count=0
+ [[ 0 != 0 ]]
+ for pass in '$passwords'
++ grep -c --fixed-strings -- databaseAdmin123456 /tmp/tmp.2rzkpQHbaz/logs_output-monitoring-0-monitoring.txt
++ :
+ count=0
+ [[ 0 != 0 ]]
+ for pass in '$passwords'
++ grep -c --fixed-strings -- userAdmin123456 /tmp/tmp.2rzkpQHbaz/logs_output-monitoring-0-monitoring.txt
++ :
+ count=0
+ [[ 0 != 0 ]]
+ for pass in '$passwords'
++ grep -c --fixed-strings -- YmFja3VwMTIzNDU2 /tmp/tmp.2rzkpQHbaz/logs_output-monitoring-0-monitoring.txt
++ :
+ count=0
+ [[ 0 != 0 ]]
+ for pass in '$passwords'
++ grep -c --fixed-strings -- Y2x1c3RlckFkbWluMTIzNDU2 /tmp/tmp.2rzkpQHbaz/logs_output-monitoring-0-monitoring.txt
++ :
+ count=0
+ [[ 0 != 0 ]]
+ for pass in '$passwords'
++ grep -c --fixed-strings -- Y2x1c3Rlck1vbml0b3IxMjM0NTY= /tmp/tmp.2rzkpQHbaz/logs_output-monitoring-0-monitoring.txt
++ :
+ count=0
+ [[ 0 != 0 ]]
+ for pass in '$passwords'
++ grep -c --fixed-strings -- ZGF0YWJhc2VBZG1pbjEyMzQ1Ng== /tmp/tmp.2rzkpQHbaz/logs_output-monitoring-0-monitoring.txt
++ :
+ count=0
+ [[ 0 != 0 ]]
+ for pass in '$passwords'
++ grep -c --fixed-strings -- dXNlckFkbWluMTIzNDU2 /tmp/tmp.2rzkpQHbaz/logs_output-monitoring-0-monitoring.txt
++ :
+ count=0
+ [[ 0 != 0 ]]
+ for pass in '$passwords'
++ grep -c --fixed-strings -- YmFja3VwMTIzNDU2 /tmp/tmp.2rzkpQHbaz/logs_output-monitoring-0-monitoring.txt
++ :
+ count=0
+ [[ 0 != 0 ]]
+ for pass in '$passwords'
++ grep -c --fixed-strings -- Y2x1c3RlckFkbWluMTIzNDU2 /tmp/tmp.2rzkpQHbaz/logs_output-monitoring-0-monitoring.txt
++ :
+ count=0
+ [[ 0 != 0 ]]
+ for pass in '$passwords'
++ grep -c --fixed-strings -- Y2x1c3Rlck1vbml0b3IxMjM0NTY= /tmp/tmp.2rzkpQHbaz/logs_output-monitoring-0-monitoring.txt
++ :
+ count=0
+ [[ 0 != 0 ]]
+ for pass in '$passwords'
++ grep -c --fixed-strings -- ZGF0YWJhc2VBZG1pbjEyMzQ1Ng== /tmp/tmp.2rzkpQHbaz/logs_output-monitoring-0-monitoring.txt
++ :
+ count=0
+ [[ 0 != 0 ]]
+ for pass in '$passwords'
++ grep -c --fixed-strings -- dXNlckFkbWluMTIzNDU2 /tmp/tmp.2rzkpQHbaz/logs_output-monitoring-0-monitoring.txt
++ :
+ count=0
+ [[ 0 != 0 ]]
+ echo

+ for p in '$pods'
++ kubectl_bin -n monitoring-2-0-31670 get pod psmdb-client-6cd48df8b6-t28bz -o 'jsonpath={.spec.containers[*].name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.QjekSlopaw
+++ mktemp
++ local LAST_ERR=/tmp/tmp.4kUD7TrYRk
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl -n monitoring-2-0-31670 get pod psmdb-client-6cd48df8b6-t28bz -o 'jsonpath={.spec.containers[*].name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.QjekSlopaw
++ cat /tmp/tmp.4kUD7TrYRk
++ rm /tmp/tmp.QjekSlopaw /tmp/tmp.4kUD7TrYRk
++ return 0
+ containers=psmdb-client
+ for c in '$containers'
+ [[ psmdb-client =~ pmm ]]
+ kubectl_bin -n monitoring-2-0-31670 logs psmdb-client-6cd48df8b6-t28bz -c psmdb-client
++ mktemp
+ local LAST_OUT=/tmp/tmp.EcwpNKqhIF
++ mktemp
+ local LAST_ERR=/tmp/tmp.pt6cODsa2J
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl -n monitoring-2-0-31670 logs psmdb-client-6cd48df8b6-t28bz -c psmdb-client
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.EcwpNKqhIF
+ cat /tmp/tmp.pt6cODsa2J
+ rm /tmp/tmp.EcwpNKqhIF /tmp/tmp.pt6cODsa2J
+ return 0
+ echo logs saved in: /tmp/tmp.2rzkpQHbaz/logs_output-psmdb-client-6cd48df8b6-t28bz-psmdb-client.txt
logs saved in: /tmp/tmp.2rzkpQHbaz/logs_output-psmdb-client-6cd48df8b6-t28bz-psmdb-client.txt
+ for pass in '$passwords'
++ grep -c --fixed-strings -- backup123456 /tmp/tmp.2rzkpQHbaz/logs_output-psmdb-client-6cd48df8b6-t28bz-psmdb-client.txt
++ :
+ count=0
+ [[ 0 != 0 ]]
+ for pass in '$passwords'
++ grep -c --fixed-strings -- clusterAdmin123456 /tmp/tmp.2rzkpQHbaz/logs_output-psmdb-client-6cd48df8b6-t28bz-psmdb-client.txt
++ :
+ count=0
+ [[ 0 != 0 ]]
+ for pass in '$passwords'
++ grep -c --fixed-strings -- clusterMonitor123456 /tmp/tmp.2rzkpQHbaz/logs_output-psmdb-client-6cd48df8b6-t28bz-psmdb-client.txt
++ :
+ count=0
+ [[ 0 != 0 ]]
+ for pass in '$passwords'
++ grep -c --fixed-strings -- databaseAdmin123456 /tmp/tmp.2rzkpQHbaz/logs_output-psmdb-client-6cd48df8b6-t28bz-psmdb-client.txt
++ :
+ count=0
+ [[ 0 != 0 ]]
+ for pass in '$passwords'
++ grep -c --fixed-strings -- userAdmin123456 /tmp/tmp.2rzkpQHbaz/logs_output-psmdb-client-6cd48df8b6-t28bz-psmdb-client.txt
++ :
+ count=0
+ [[ 0 != 0 ]]
+ for pass in '$passwords'
++ grep -c --fixed-strings -- backup123456 /tmp/tmp.2rzkpQHbaz/logs_output-psmdb-client-6cd48df8b6-t28bz-psmdb-client.txt
++ :
+ count=0
+ [[ 0 != 0 ]]
+ for pass in '$passwords'
++ grep -c --fixed-strings -- clusterAdmin123456 /tmp/tmp.2rzkpQHbaz/logs_output-psmdb-client-6cd48df8b6-t28bz-psmdb-client.txt
++ :
+ count=0
+ [[ 0 != 0 ]]
+ for pass in '$passwords'
++ grep -c --fixed-strings -- clusterMonitor123456 /tmp/tmp.2rzkpQHbaz/logs_output-psmdb-client-6cd48df8b6-t28bz-psmdb-client.txt
++ :
+ count=0
+ [[ 0 != 0 ]]
+ for pass in '$passwords'
++ grep -c --fixed-strings -- databaseAdmin123456 /tmp/tmp.2rzkpQHbaz/logs_output-psmdb-client-6cd48df8b6-t28bz-psmdb-client.txt
++ :
+ count=0
+ [[ 0 != 0 ]]
+ for pass in '$passwords'
++ grep -c --fixed-strings -- userAdmin123456 /tmp/tmp.2rzkpQHbaz/logs_output-psmdb-client-6cd48df8b6-t28bz-psmdb-client.txt
++ :
+ count=0
+ [[ 0 != 0 ]]
+ for pass in '$passwords'
++ grep -c --fixed-strings -- YmFja3VwMTIzNDU2 /tmp/tmp.2rzkpQHbaz/logs_output-psmdb-client-6cd48df8b6-t28bz-psmdb-client.txt
++ :
+ count=0
+ [[ 0 != 0 ]]
+ for pass in '$passwords'
++ grep -c --fixed-strings -- Y2x1c3RlckFkbWluMTIzNDU2 /tmp/tmp.2rzkpQHbaz/logs_output-psmdb-client-6cd48df8b6-t28bz-psmdb-client.txt
++ :
+ count=0
+ [[ 0 != 0 ]]
+ for pass in '$passwords'
++ grep -c --fixed-strings -- Y2x1c3Rlck1vbml0b3IxMjM0NTY= /tmp/tmp.2rzkpQHbaz/logs_output-psmdb-client-6cd48df8b6-t28bz-psmdb-client.txt
++ :
+ count=0
+ [[ 0 != 0 ]]
+ for pass in '$passwords'
++ grep -c --fixed-strings -- ZGF0YWJhc2VBZG1pbjEyMzQ1Ng== /tmp/tmp.2rzkpQHbaz/logs_output-psmdb-client-6cd48df8b6-t28bz-psmdb-client.txt
++ :
+ count=0
+ [[ 0 != 0 ]]
+ for pass in '$passwords'
++ grep -c --fixed-strings -- dXNlckFkbWluMTIzNDU2 /tmp/tmp.2rzkpQHbaz/logs_output-psmdb-client-6cd48df8b6-t28bz-psmdb-client.txt
++ :
+ count=0
+ [[ 0 != 0 ]]
+ for pass in '$passwords'
++ grep -c --fixed-strings -- YmFja3VwMTIzNDU2 /tmp/tmp.2rzkpQHbaz/logs_output-psmdb-client-6cd48df8b6-t28bz-psmdb-client.txt
++ :
+ count=0
+ [[ 0 != 0 ]]
+ for pass in '$passwords'
++ grep -c --fixed-strings -- Y2x1c3RlckFkbWluMTIzNDU2 /tmp/tmp.2rzkpQHbaz/logs_output-psmdb-client-6cd48df8b6-t28bz-psmdb-client.txt
++ :
+ count=0
+ [[ 0 != 0 ]]
+ for pass in '$passwords'
++ grep -c --fixed-strings -- Y2x1c3Rlck1vbml0b3IxMjM0NTY= /tmp/tmp.2rzkpQHbaz/logs_output-psmdb-client-6cd48df8b6-t28bz-psmdb-client.txt
++ :
+ count=0
+ [[ 0 != 0 ]]
+ for pass in '$passwords'
++ grep -c --fixed-strings -- ZGF0YWJhc2VBZG1pbjEyMzQ1Ng== /tmp/tmp.2rzkpQHbaz/logs_output-psmdb-client-6cd48df8b6-t28bz-psmdb-client.txt
++ :
+ count=0
+ [[ 0 != 0 ]]
+ for pass in '$passwords'
++ grep -c --fixed-strings -- dXNlckFkbWluMTIzNDU2 /tmp/tmp.2rzkpQHbaz/logs_output-psmdb-client-6cd48df8b6-t28bz-psmdb-client.txt
++ :
+ count=0
+ [[ 0 != 0 ]]
+ echo

+ '[' -n psmdb-operator ']'
++ kubectl_bin -n psmdb-operator get pods -o name
++ awk -F / '{print $2}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.ftdPvWCWrT
+++ mktemp
++ local LAST_ERR=/tmp/tmp.vco58O1eoq
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl -n psmdb-operator get pods -o name
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.ftdPvWCWrT
++ cat /tmp/tmp.vco58O1eoq
++ rm /tmp/tmp.ftdPvWCWrT /tmp/tmp.vco58O1eoq
++ return 0
+ pods=percona-server-mongodb-operator-7ff48667b4-j652f
+ collect_logs psmdb-operator
+ local containers
+ local count
+ NS=psmdb-operator
+ for p in '$pods'
++ kubectl_bin -n psmdb-operator get pod percona-server-mongodb-operator-7ff48667b4-j652f -o 'jsonpath={.spec.containers[*].name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.KVicbu1Hr2
+++ mktemp
++ local LAST_ERR=/tmp/tmp.rcVxobiYQV
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl -n psmdb-operator get pod percona-server-mongodb-operator-7ff48667b4-j652f -o 'jsonpath={.spec.containers[*].name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.KVicbu1Hr2
++ cat /tmp/tmp.rcVxobiYQV
++ rm /tmp/tmp.KVicbu1Hr2 /tmp/tmp.rcVxobiYQV
++ return 0
+ containers=percona-server-mongodb-operator
+ for c in '$containers'
+ [[ percona-server-mongodb-operator =~ pmm ]]
+ kubectl_bin -n psmdb-operator logs percona-server-mongodb-operator-7ff48667b4-j652f -c percona-server-mongodb-operator
++ mktemp
+ local LAST_OUT=/tmp/tmp.tDYmuqG1og
++ mktemp
+ local LAST_ERR=/tmp/tmp.75lOAl1w1m
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl -n psmdb-operator logs percona-server-mongodb-operator-7ff48667b4-j652f -c percona-server-mongodb-operator
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.tDYmuqG1og
+ cat /tmp/tmp.75lOAl1w1m
+ rm /tmp/tmp.tDYmuqG1og /tmp/tmp.75lOAl1w1m
+ return 0
+ echo logs saved in: /tmp/tmp.2rzkpQHbaz/logs_output-percona-server-mongodb-operator-7ff48667b4-j652f-percona-server-mongodb-operator.txt
logs saved in: /tmp/tmp.2rzkpQHbaz/logs_output-percona-server-mongodb-operator-7ff48667b4-j652f-percona-server-mongodb-operator.txt
+ for pass in '$passwords'
++ grep -c --fixed-strings -- backup123456 /tmp/tmp.2rzkpQHbaz/logs_output-percona-server-mongodb-operator-7ff48667b4-j652f-percona-server-mongodb-operator.txt
++ :
+ count=0
+ [[ 0 != 0 ]]
+ for pass in '$passwords'
++ grep -c --fixed-strings -- clusterAdmin123456 /tmp/tmp.2rzkpQHbaz/logs_output-percona-server-mongodb-operator-7ff48667b4-j652f-percona-server-mongodb-operator.txt
++ :
+ count=0
+ [[ 0 != 0 ]]
+ for pass in '$passwords'
++ grep -c --fixed-strings -- clusterMonitor123456 /tmp/tmp.2rzkpQHbaz/logs_output-percona-server-mongodb-operator-7ff48667b4-j652f-percona-server-mongodb-operator.txt
++ :
+ count=0
+ [[ 0 != 0 ]]
+ for pass in '$passwords'
++ grep -c --fixed-strings -- databaseAdmin123456 /tmp/tmp.2rzkpQHbaz/logs_output-percona-server-mongodb-operator-7ff48667b4-j652f-percona-server-mongodb-operator.txt
++ :
+ count=0
+ [[ 0 != 0 ]]
+ for pass in '$passwords'
++ grep -c --fixed-strings -- userAdmin123456 /tmp/tmp.2rzkpQHbaz/logs_output-percona-server-mongodb-operator-7ff48667b4-j652f-percona-server-mongodb-operator.txt
++ :
+ count=0
+ [[ 0 != 0 ]]
+ for pass in '$passwords'
++ grep -c --fixed-strings -- backup123456 /tmp/tmp.2rzkpQHbaz/logs_output-percona-server-mongodb-operator-7ff48667b4-j652f-percona-server-mongodb-operator.txt
++ :
+ count=0
+ [[ 0 != 0 ]]
+ for pass in '$passwords'
++ grep -c --fixed-strings -- clusterAdmin123456 /tmp/tmp.2rzkpQHbaz/logs_output-percona-server-mongodb-operator-7ff48667b4-j652f-percona-server-mongodb-operator.txt
++ :
+ count=0
+ [[ 0 != 0 ]]
+ for pass in '$passwords'
++ grep -c --fixed-strings -- clusterMonitor123456 /tmp/tmp.2rzkpQHbaz/logs_output-percona-server-mongodb-operator-7ff48667b4-j652f-percona-server-mongodb-operator.txt
++ :
+ count=0
+ [[ 0 != 0 ]]
+ for pass in '$passwords'
++ grep -c --fixed-strings -- databaseAdmin123456 /tmp/tmp.2rzkpQHbaz/logs_output-percona-server-mongodb-operator-7ff48667b4-j652f-percona-server-mongodb-operator.txt
++ :
+ count=0
+ [[ 0 != 0 ]]
+ for pass in '$passwords'
++ grep -c --fixed-strings -- userAdmin123456 /tmp/tmp.2rzkpQHbaz/logs_output-percona-server-mongodb-operator-7ff48667b4-j652f-percona-server-mongodb-operator.txt
++ :
+ count=0
+ [[ 0 != 0 ]]
+ for pass in '$passwords'
++ grep -c --fixed-strings -- YmFja3VwMTIzNDU2 /tmp/tmp.2rzkpQHbaz/logs_output-percona-server-mongodb-operator-7ff48667b4-j652f-percona-server-mongodb-operator.txt
++ :
+ count=0
+ [[ 0 != 0 ]]
+ for pass in '$passwords'
++ grep -c --fixed-strings -- Y2x1c3RlckFkbWluMTIzNDU2 /tmp/tmp.2rzkpQHbaz/logs_output-percona-server-mongodb-operator-7ff48667b4-j652f-percona-server-mongodb-operator.txt
++ :
+ count=0
+ [[ 0 != 0 ]]
+ for pass in '$passwords'
++ grep -c --fixed-strings -- Y2x1c3Rlck1vbml0b3IxMjM0NTY= /tmp/tmp.2rzkpQHbaz/logs_output-percona-server-mongodb-operator-7ff48667b4-j652f-percona-server-mongodb-operator.txt
++ :
+ count=0
+ [[ 0 != 0 ]]
+ for pass in '$passwords'
++ grep -c --fixed-strings -- ZGF0YWJhc2VBZG1pbjEyMzQ1Ng== /tmp/tmp.2rzkpQHbaz/logs_output-percona-server-mongodb-operator-7ff48667b4-j652f-percona-server-mongodb-operator.txt
++ :
+ count=0
+ [[ 0 != 0 ]]
+ for pass in '$passwords'
++ grep -c --fixed-strings -- dXNlckFkbWluMTIzNDU2 /tmp/tmp.2rzkpQHbaz/logs_output-percona-server-mongodb-operator-7ff48667b4-j652f-percona-server-mongodb-operator.txt
++ :
+ count=0
+ [[ 0 != 0 ]]
+ for pass in '$passwords'
++ grep -c --fixed-strings -- YmFja3VwMTIzNDU2 /tmp/tmp.2rzkpQHbaz/logs_output-percona-server-mongodb-operator-7ff48667b4-j652f-percona-server-mongodb-operator.txt
++ :
+ count=0
+ [[ 0 != 0 ]]
+ for pass in '$passwords'
++ grep -c --fixed-strings -- Y2x1c3RlckFkbWluMTIzNDU2 /tmp/tmp.2rzkpQHbaz/logs_output-percona-server-mongodb-operator-7ff48667b4-j652f-percona-server-mongodb-operator.txt
++ :
+ count=0
+ [[ 0 != 0 ]]
+ for pass in '$passwords'
++ grep -c --fixed-strings -- Y2x1c3Rlck1vbml0b3IxMjM0NTY= /tmp/tmp.2rzkpQHbaz/logs_output-percona-server-mongodb-operator-7ff48667b4-j652f-percona-server-mongodb-operator.txt
++ :
+ count=0
+ [[ 0 != 0 ]]
+ for pass in '$passwords'
++ grep -c --fixed-strings -- ZGF0YWJhc2VBZG1pbjEyMzQ1Ng== /tmp/tmp.2rzkpQHbaz/logs_output-percona-server-mongodb-operator-7ff48667b4-j652f-percona-server-mongodb-operator.txt
++ :
+ count=0
+ [[ 0 != 0 ]]
+ for pass in '$passwords'
++ grep -c --fixed-strings -- dXNlckFkbWluMTIzNDU2 /tmp/tmp.2rzkpQHbaz/logs_output-percona-server-mongodb-operator-7ff48667b4-j652f-percona-server-mongodb-operator.txt
++ :
+ count=0
+ [[ 0 != 0 ]]
+ echo

+ helm uninstall monitoring
release "monitoring" uninstalled
+ destroy monitoring-2-0-31670
+ local namespace=monitoring-2-0-31670
+ local ignore_logs=true
+ desc 'destroy cluster/operator and all other resources'
+ set +o xtrace


-----------------------------------------------------------------------------------
destroy cluster/operator and all other resources
-----------------------------------------------------------------------------------

+ '[' true == false ']'
+ delete_crd
+ desc 'get and delete old CRDs and RBAC'
+ set +o xtrace


-----------------------------------------------------------------------------------
get and delete old CRDs and RBAC
-----------------------------------------------------------------------------------

+ kubectl_bin delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1584/deploy/crd.yaml --ignore-not-found --wait=false
++ mktemp
+ local LAST_OUT=/tmp/tmp.OuSM02MX6k
++ mktemp
+ local LAST_ERR=/tmp/tmp.qcTsdKqB4d
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1584/deploy/crd.yaml --ignore-not-found --wait=false
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.OuSM02MX6k
customresourcedefinition.apiextensions.k8s.io "perconaservermongodbbackups.psmdb.percona.com" deleted
customresourcedefinition.apiextensions.k8s.io "perconaservermongodbrestores.psmdb.percona.com" deleted
customresourcedefinition.apiextensions.k8s.io "perconaservermongodbs.psmdb.percona.com" deleted
+ cat /tmp/tmp.qcTsdKqB4d
+ rm /tmp/tmp.OuSM02MX6k /tmp/tmp.qcTsdKqB4d
+ return 0
++ yq eval .metadata.name /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1584/deploy/crd.yaml
++ grep -v '\-\-\-'
+ for crd_name in '$(yq eval '\''.metadata.name'\'' "${src_dir}/deploy/crd.yaml" | grep -v '\''\-\-\-'\'')'
+ grep -v NAMESPACE
+ kubectl get perconaservermongodbbackups.psmdb.percona.com --all-namespaces -o wide
+ xargs -L 1 sh -xc 'kubectl patch perconaservermongodbbackups.psmdb.percona.com -n $0 $1 --type=merge -p "{\"metadata\":{\"finalizers\":[]}}"'
error: the server doesn't have a resource type "perconaservermongodbbackups"
+ kubectl patch perconaservermongodbbackups.psmdb.percona.com -n sh --type=merge -p '{"metadata":{"finalizers":[]}}'
error: the server doesn't have a resource type "perconaservermongodbbackups"
+ :
+ kubectl_bin wait --for=delete crd perconaservermongodbbackups.psmdb.percona.com
++ mktemp
+ local LAST_OUT=/tmp/tmp.hTKsxZGNjs
++ mktemp
+ local LAST_ERR=/tmp/tmp.OJSQsTVMNB
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl wait --for=delete crd perconaservermongodbbackups.psmdb.percona.com
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.hTKsxZGNjs
+ cat /tmp/tmp.OJSQsTVMNB
+ rm /tmp/tmp.hTKsxZGNjs /tmp/tmp.OJSQsTVMNB
+ return 0
+ for crd_name in '$(yq eval '\''.metadata.name'\'' "${src_dir}/deploy/crd.yaml" | grep -v '\''\-\-\-'\'')'
+ kubectl get perconaservermongodbrestores.psmdb.percona.com --all-namespaces -o wide
+ grep -v NAMESPACE
+ xargs -L 1 sh -xc 'kubectl patch perconaservermongodbrestores.psmdb.percona.com -n $0 $1 --type=merge -p "{\"metadata\":{\"finalizers\":[]}}"'
error: the server doesn't have a resource type "perconaservermongodbrestores"
+ kubectl patch perconaservermongodbrestores.psmdb.percona.com -n sh --type=merge -p '{"metadata":{"finalizers":[]}}'
error: the server doesn't have a resource type "perconaservermongodbrestores"
+ :
+ kubectl_bin wait --for=delete crd perconaservermongodbrestores.psmdb.percona.com
++ mktemp
+ local LAST_OUT=/tmp/tmp.ejW2NTs0kU
++ mktemp
+ local LAST_ERR=/tmp/tmp.lofQhSycui
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl wait --for=delete crd perconaservermongodbrestores.psmdb.percona.com
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.ejW2NTs0kU
+ cat /tmp/tmp.lofQhSycui
+ rm /tmp/tmp.ejW2NTs0kU /tmp/tmp.lofQhSycui
+ return 0
+ for crd_name in '$(yq eval '\''.metadata.name'\'' "${src_dir}/deploy/crd.yaml" | grep -v '\''\-\-\-'\'')'
+ kubectl get perconaservermongodbs.psmdb.percona.com --all-namespaces -o wide
+ grep -v NAMESPACE
+ xargs -L 1 sh -xc 'kubectl patch perconaservermongodbs.psmdb.percona.com -n $0 $1 --type=merge -p "{\"metadata\":{\"finalizers\":[]}}"'
error: the server doesn't have a resource type "perconaservermongodbs"
+ kubectl patch perconaservermongodbs.psmdb.percona.com -n sh --type=merge -p '{"metadata":{"finalizers":[]}}'
error: the server doesn't have a resource type "perconaservermongodbs"
+ :
+ kubectl_bin wait --for=delete crd perconaservermongodbs.psmdb.percona.com
++ mktemp
+ local LAST_OUT=/tmp/tmp.I1tzBYfbAE
++ mktemp
+ local LAST_ERR=/tmp/tmp.VgTTsozONb
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl wait --for=delete crd perconaservermongodbs.psmdb.percona.com
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.I1tzBYfbAE
+ cat /tmp/tmp.VgTTsozONb
+ rm /tmp/tmp.I1tzBYfbAE /tmp/tmp.VgTTsozONb
+ return 0
+ local rbac_yaml=rbac.yaml
+ '[' -n psmdb-operator ']'
+ rbac_yaml=cw-rbac.yaml
+ kubectl_bin delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1584/deploy/cw-rbac.yaml --ignore-not-found
++ mktemp
+ local LAST_OUT=/tmp/tmp.w9ASOpfjZA
++ mktemp
+ local LAST_ERR=/tmp/tmp.lmIhujNl3W
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1584/deploy/cw-rbac.yaml --ignore-not-found
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.w9ASOpfjZA
clusterrole.rbac.authorization.k8s.io "percona-server-mongodb-operator" deleted
clusterrolebinding.rbac.authorization.k8s.io "service-account-percona-server-mongodb-operator" deleted
+ cat /tmp/tmp.lmIhujNl3W
+ rm /tmp/tmp.w9ASOpfjZA /tmp/tmp.lmIhujNl3W
+ return 0
+ destroy_cert_manager
+ kubectl_bin delete -f https://github.com/cert-manager/cert-manager/releases/download/v1.14.5/cert-manager.yaml
++ mktemp
+ local LAST_OUT=/tmp/tmp.nvdydPEXNZ
++ mktemp
+ local LAST_ERR=/tmp/tmp.Ok7khvjAKU
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl delete -f https://github.com/cert-manager/cert-manager/releases/download/v1.14.5/cert-manager.yaml
+ exit_status=1
+ set -e
+ '[' 1 '!=' 0 -a -n 1 ']'
+ cat /tmp/tmp.nvdydPEXNZ
namespace "cert-manager" deleted
customresourcedefinition.apiextensions.k8s.io "certificaterequests.cert-manager.io" deleted
customresourcedefinition.apiextensions.k8s.io "certificates.cert-manager.io" deleted
customresourcedefinition.apiextensions.k8s.io "challenges.acme.cert-manager.io" deleted
customresourcedefinition.apiextensions.k8s.io "clusterissuers.cert-manager.io" deleted
customresourcedefinition.apiextensions.k8s.io "issuers.cert-manager.io" deleted
customresourcedefinition.apiextensions.k8s.io "orders.acme.cert-manager.io" deleted
serviceaccount "cert-manager-cainjector" deleted
serviceaccount "cert-manager" deleted
serviceaccount "cert-manager-webhook" deleted
clusterrole.rbac.authorization.k8s.io "cert-manager-cainjector" deleted
clusterrole.rbac.authorization.k8s.io "cert-manager-controller-issuers" deleted
clusterrole.rbac.authorization.k8s.io "cert-manager-controller-clusterissuers" deleted
clusterrole.rbac.authorization.k8s.io "cert-manager-controller-certificates" deleted
clusterrole.rbac.authorization.k8s.io "cert-manager-controller-orders" deleted
clusterrole.rbac.authorization.k8s.io "cert-manager-controller-challenges" deleted
clusterrole.rbac.authorization.k8s.io "cert-manager-controller-ingress-shim" deleted
clusterrole.rbac.authorization.k8s.io "cert-manager-cluster-view" deleted
clusterrole.rbac.authorization.k8s.io "cert-manager-view" deleted
clusterrole.rbac.authorization.k8s.io "cert-manager-edit" deleted
clusterrole.rbac.authorization.k8s.io "cert-manager-controller-approve:cert-manager-io" deleted
clusterrole.rbac.authorization.k8s.io "cert-manager-controller-certificatesigningrequests" deleted
clusterrole.rbac.authorization.k8s.io "cert-manager-webhook:subjectaccessreviews" deleted
clusterrolebinding.rbac.authorization.k8s.io "cert-manager-cainjector" deleted
clusterrolebinding.rbac.authorization.k8s.io "cert-manager-controller-issuers" deleted
clusterrolebinding.rbac.authorization.k8s.io "cert-manager-controller-clusterissuers" deleted
clusterrolebinding.rbac.authorization.k8s.io "cert-manager-controller-certificates" deleted
clusterrolebinding.rbac.authorization.k8s.io "cert-manager-controller-orders" deleted
clusterrolebinding.rbac.authorization.k8s.io "cert-manager-controller-challenges" deleted
clusterrolebinding.rbac.authorization.k8s.io "cert-manager-controller-ingress-shim" deleted
clusterrolebinding.rbac.authorization.k8s.io "cert-manager-controller-approve:cert-manager-io" deleted
clusterrolebinding.rbac.authorization.k8s.io "cert-manager-controller-certificatesigningrequests" deleted
clusterrolebinding.rbac.authorization.k8s.io "cert-manager-webhook:subjectaccessreviews" deleted
role.rbac.authorization.k8s.io "cert-manager-cainjector:leaderelection" deleted
role.rbac.authorization.k8s.io "cert-manager:leaderelection" deleted
rolebinding.rbac.authorization.k8s.io "cert-manager-cainjector:leaderelection" deleted
rolebinding.rbac.authorization.k8s.io "cert-manager:leaderelection" deleted
mutatingwebhookconfiguration.admissionregistration.k8s.io "cert-manager-webhook" deleted
validatingwebhookconfiguration.admissionregistration.k8s.io "cert-manager-webhook" deleted
+ cat /tmp/tmp.Ok7khvjAKU
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.14.5/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager-webhook:dynamic-serving" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.14.5/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager-webhook:dynamic-serving" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.14.5/cert-manager.yaml": services "cert-manager" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.14.5/cert-manager.yaml": services "cert-manager-webhook" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.14.5/cert-manager.yaml": deployments.apps "cert-manager-cainjector" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.14.5/cert-manager.yaml": deployments.apps "cert-manager" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.14.5/cert-manager.yaml": deployments.apps "cert-manager-webhook" not found
+ sleep 0
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl delete -f https://github.com/cert-manager/cert-manager/releases/download/v1.14.5/cert-manager.yaml
+ exit_status=1
+ set -e
+ '[' 1 '!=' 0 -a -n 1 ']'
+ cat /tmp/tmp.nvdydPEXNZ
namespace "cert-manager" deleted
+ cat /tmp/tmp.Ok7khvjAKU
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.14.5/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "certificaterequests.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.14.5/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "certificates.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.14.5/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "challenges.acme.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.14.5/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "clusterissuers.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.14.5/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "issuers.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.14.5/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "orders.acme.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.14.5/cert-manager.yaml": serviceaccounts "cert-manager-cainjector" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.14.5/cert-manager.yaml": serviceaccounts "cert-manager" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.14.5/cert-manager.yaml": serviceaccounts "cert-manager-webhook" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.14.5/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-cainjector" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.14.5/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-issuers" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.14.5/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-clusterissuers" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.14.5/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-certificates" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.14.5/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-orders" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.14.5/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-challenges" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.14.5/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-ingress-shim" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.14.5/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-cluster-view" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.14.5/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-view" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.14.5/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-edit" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.14.5/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-approve:cert-manager-io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.14.5/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-certificatesigningrequests" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.14.5/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-webhook:subjectaccessreviews" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.14.5/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-cainjector" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.14.5/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-issuers" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.14.5/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-clusterissuers" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.14.5/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-certificates" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.14.5/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-orders" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.14.5/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-challenges" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.14.5/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-ingress-shim" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.14.5/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-approve:cert-manager-io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.14.5/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-certificatesigningrequests" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.14.5/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-webhook:subjectaccessreviews" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.14.5/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager-cainjector:leaderelection" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.14.5/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager:leaderelection" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.14.5/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager-webhook:dynamic-serving" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.14.5/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager-cainjector:leaderelection" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.14.5/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager:leaderelection" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.14.5/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager-webhook:dynamic-serving" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.14.5/cert-manager.yaml": services "cert-manager" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.14.5/cert-manager.yaml": services "cert-manager-webhook" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.14.5/cert-manager.yaml": deployments.apps "cert-manager-cainjector" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.14.5/cert-manager.yaml": deployments.apps "cert-manager" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.14.5/cert-manager.yaml": deployments.apps "cert-manager-webhook" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.14.5/cert-manager.yaml": mutatingwebhookconfigurations.admissionregistration.k8s.io "cert-manager-webhook" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.14.5/cert-manager.yaml": validatingwebhookconfigurations.admissionregistration.k8s.io "cert-manager-webhook" not found
+ sleep 4
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl delete -f https://github.com/cert-manager/cert-manager/releases/download/v1.14.5/cert-manager.yaml
+ exit_status=1
+ set -e
+ '[' 1 '!=' 0 -a -n 1 ']'
+ cat /tmp/tmp.nvdydPEXNZ
+ cat /tmp/tmp.Ok7khvjAKU
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.14.5/cert-manager.yaml": namespaces "cert-manager" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.14.5/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "certificaterequests.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.14.5/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "certificates.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.14.5/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "challenges.acme.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.14.5/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "clusterissuers.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.14.5/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "issuers.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.14.5/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "orders.acme.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.14.5/cert-manager.yaml": serviceaccounts "cert-manager-cainjector" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.14.5/cert-manager.yaml": serviceaccounts "cert-manager" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.14.5/cert-manager.yaml": serviceaccounts "cert-manager-webhook" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.14.5/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-cainjector" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.14.5/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-issuers" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.14.5/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-clusterissuers" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.14.5/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-certificates" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.14.5/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-orders" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.14.5/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-challenges" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.14.5/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-ingress-shim" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.14.5/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-cluster-view" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.14.5/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-view" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.14.5/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-edit" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.14.5/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-approve:cert-manager-io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.14.5/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-certificatesigningrequests" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.14.5/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-webhook:subjectaccessreviews" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.14.5/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-cainjector" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.14.5/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-issuers" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.14.5/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-clusterissuers" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.14.5/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-certificates" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.14.5/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-orders" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.14.5/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-challenges" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.14.5/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-ingress-shim" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.14.5/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-approve:cert-manager-io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.14.5/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-certificatesigningrequests" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.14.5/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-webhook:subjectaccessreviews" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.14.5/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager-cainjector:leaderelection" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.14.5/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager:leaderelection" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.14.5/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager-webhook:dynamic-serving" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.14.5/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager-cainjector:leaderelection" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.14.5/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager:leaderelection" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.14.5/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager-webhook:dynamic-serving" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.14.5/cert-manager.yaml": services "cert-manager" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.14.5/cert-manager.yaml": services "cert-manager-webhook" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.14.5/cert-manager.yaml": deployments.apps "cert-manager-cainjector" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.14.5/cert-manager.yaml": deployments.apps "cert-manager" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.14.5/cert-manager.yaml": deployments.apps "cert-manager-webhook" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.14.5/cert-manager.yaml": mutatingwebhookconfigurations.admissionregistration.k8s.io "cert-manager-webhook" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.14.5/cert-manager.yaml": validatingwebhookconfigurations.admissionregistration.k8s.io "cert-manager-webhook" not found
+ sleep 8
+ cat /tmp/tmp.nvdydPEXNZ
+ cat /tmp/tmp.Ok7khvjAKU
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.14.5/cert-manager.yaml": namespaces "cert-manager" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.14.5/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "certificaterequests.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.14.5/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "certificates.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.14.5/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "challenges.acme.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.14.5/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "clusterissuers.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.14.5/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "issuers.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.14.5/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "orders.acme.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.14.5/cert-manager.yaml": serviceaccounts "cert-manager-cainjector" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.14.5/cert-manager.yaml": serviceaccounts "cert-manager" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.14.5/cert-manager.yaml": serviceaccounts "cert-manager-webhook" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.14.5/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-cainjector" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.14.5/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-issuers" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.14.5/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-clusterissuers" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.14.5/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-certificates" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.14.5/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-orders" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.14.5/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-challenges" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.14.5/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-ingress-shim" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.14.5/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-cluster-view" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.14.5/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-view" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.14.5/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-edit" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.14.5/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-approve:cert-manager-io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.14.5/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-certificatesigningrequests" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.14.5/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-webhook:subjectaccessreviews" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.14.5/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-cainjector" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.14.5/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-issuers" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.14.5/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-clusterissuers" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.14.5/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-certificates" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.14.5/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-orders" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.14.5/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-challenges" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.14.5/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-ingress-shim" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.14.5/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-approve:cert-manager-io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.14.5/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-certificatesigningrequests" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.14.5/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-webhook:subjectaccessreviews" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.14.5/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager-cainjector:leaderelection" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.14.5/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager:leaderelection" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.14.5/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager-webhook:dynamic-serving" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.14.5/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager-cainjector:leaderelection" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.14.5/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager:leaderelection" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.14.5/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager-webhook:dynamic-serving" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.14.5/cert-manager.yaml": services "cert-manager" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.14.5/cert-manager.yaml": services "cert-manager-webhook" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.14.5/cert-manager.yaml": deployments.apps "cert-manager-cainjector" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.14.5/cert-manager.yaml": deployments.apps "cert-manager" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.14.5/cert-manager.yaml": deployments.apps "cert-manager-webhook" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.14.5/cert-manager.yaml": mutatingwebhookconfigurations.admissionregistration.k8s.io "cert-manager-webhook" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.14.5/cert-manager.yaml": validatingwebhookconfigurations.admissionregistration.k8s.io "cert-manager-webhook" not found
+ rm /tmp/tmp.nvdydPEXNZ /tmp/tmp.Ok7khvjAKU
+ return 1
+ true
+ '[' -n '' ']'
+ '[' -n psmdb-operator ']'
+ kubectl_bin delete --grace-period=0 --force=true namespace monitoring-2-0-31670
+ rm -rf /tmp/tmp.2rzkpQHbaz
+ kubectl_bin delete --grace-period=0 --force=true namespace psmdb-operator
++ mktemp
++ mktemp
+ local LAST_OUT=/tmp/tmp.iHdaKd95FC
+ desc 'test passed'
+ set +o xtrace
+ local LAST_OUT=/tmp/tmp.xRmfPh32LP
++ mktemp


-----------------------------------------------------------------------------------
test passed
-----------------------------------------------------------------------------------

++ mktemp
+ local LAST_ERR=/tmp/tmp.gMK3VN5ZBU
+ local exit_status=0
+ local timeout=4
+ local LAST_ERR=/tmp/tmp.p1Y6o0a4bq
+ local exit_status=0
+ local timeout=4
++ seq 0 2
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl delete --grace-period=0 --force=true namespace monitoring-2-0-31670
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl delete --grace-period=0 --force=true namespace psmdb-operator