Log: /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1554/e2e-tests/logs/service-per-pod.log
WARNING: version difference between client (1.30) and server (1.26) exceeds the supported minor version skew of +/-1
WARNING: version difference between client (1.30) and server (1.26) exceeds the supported minor version skew of +/-1
WARNING: version difference between client (1.30) and server (1.26) exceeds the supported minor version skew of +/-1
+ main
+ create_infra service-per-pod-27980
+ local ns=service-per-pod-27980
+ delete_crd
+ desc 'get and delete old CRDs and RBAC'
+ set +o xtrace


-----------------------------------------------------------------------------------
get and delete old CRDs and RBAC
-----------------------------------------------------------------------------------

+ kubectl_bin delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1554/deploy/crd.yaml --ignore-not-found --wait=false
++ mktemp
+ local LAST_OUT=/tmp/tmp.JdGzX0yi4L
++ mktemp
+ local LAST_ERR=/tmp/tmp.D0wPWWkusB
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1554/deploy/crd.yaml --ignore-not-found --wait=false
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.JdGzX0yi4L
customresourcedefinition.apiextensions.k8s.io "perconaservermongodbbackups.psmdb.percona.com" deleted
customresourcedefinition.apiextensions.k8s.io "perconaservermongodbrestores.psmdb.percona.com" deleted
customresourcedefinition.apiextensions.k8s.io "perconaservermongodbs.psmdb.percona.com" deleted
+ cat /tmp/tmp.D0wPWWkusB
+ rm /tmp/tmp.JdGzX0yi4L /tmp/tmp.D0wPWWkusB
+ return 0
++ yq eval .metadata.name /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1554/deploy/crd.yaml
++ grep -v '\-\-\-'
+ for crd_name in '$(yq eval '\''.metadata.name'\'' "${src_dir}/deploy/crd.yaml" | grep -v '\''\-\-\-'\'')'
+ kubectl get perconaservermongodbbackups.psmdb.percona.com --all-namespaces -o wide
+ xargs -L 1 sh -xc 'kubectl patch perconaservermongodbbackups.psmdb.percona.com -n $0 $1 --type=merge -p "{\"metadata\":{\"finalizers\":[]}}"'
+ grep -v NAMESPACE
error: the server doesn't have a resource type "perconaservermongodbbackups"
+ kubectl patch perconaservermongodbbackups.psmdb.percona.com -n sh --type=merge -p '{"metadata":{"finalizers":[]}}'
error: the server doesn't have a resource type "perconaservermongodbbackups"
+ :
+ kubectl_bin wait --for=delete crd perconaservermongodbbackups.psmdb.percona.com
++ mktemp
+ local LAST_OUT=/tmp/tmp.Q40irmG3bX
++ mktemp
+ local LAST_ERR=/tmp/tmp.KIPRsQ6El0
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl wait --for=delete crd perconaservermongodbbackups.psmdb.percona.com
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.Q40irmG3bX
+ cat /tmp/tmp.KIPRsQ6El0
+ rm /tmp/tmp.Q40irmG3bX /tmp/tmp.KIPRsQ6El0
+ return 0
+ for crd_name in '$(yq eval '\''.metadata.name'\'' "${src_dir}/deploy/crd.yaml" | grep -v '\''\-\-\-'\'')'
+ kubectl get perconaservermongodbrestores.psmdb.percona.com --all-namespaces -o wide
+ grep -v NAMESPACE
+ xargs -L 1 sh -xc 'kubectl patch perconaservermongodbrestores.psmdb.percona.com -n $0 $1 --type=merge -p "{\"metadata\":{\"finalizers\":[]}}"'
error: the server doesn't have a resource type "perconaservermongodbrestores"
+ kubectl patch perconaservermongodbrestores.psmdb.percona.com -n sh --type=merge -p '{"metadata":{"finalizers":[]}}'
error: the server doesn't have a resource type "perconaservermongodbrestores"
+ :
+ kubectl_bin wait --for=delete crd perconaservermongodbrestores.psmdb.percona.com
++ mktemp
+ local LAST_OUT=/tmp/tmp.8qMdY7TFo0
++ mktemp
+ local LAST_ERR=/tmp/tmp.8S4ENBpJsx
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl wait --for=delete crd perconaservermongodbrestores.psmdb.percona.com
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.8qMdY7TFo0
+ cat /tmp/tmp.8S4ENBpJsx
+ rm /tmp/tmp.8qMdY7TFo0 /tmp/tmp.8S4ENBpJsx
+ return 0
+ for crd_name in '$(yq eval '\''.metadata.name'\'' "${src_dir}/deploy/crd.yaml" | grep -v '\''\-\-\-'\'')'
+ kubectl get perconaservermongodbs.psmdb.percona.com --all-namespaces -o wide
+ grep -v NAMESPACE
+ xargs -L 1 sh -xc 'kubectl patch perconaservermongodbs.psmdb.percona.com -n $0 $1 --type=merge -p "{\"metadata\":{\"finalizers\":[]}}"'
error: the server doesn't have a resource type "perconaservermongodbs"
+ kubectl patch perconaservermongodbs.psmdb.percona.com -n sh --type=merge -p '{"metadata":{"finalizers":[]}}'
error: the server doesn't have a resource type "perconaservermongodbs"
+ :
+ kubectl_bin wait --for=delete crd perconaservermongodbs.psmdb.percona.com
++ mktemp
+ local LAST_OUT=/tmp/tmp.Fzqe9EBRP8
++ mktemp
+ local LAST_ERR=/tmp/tmp.waqtObhSw6
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl wait --for=delete crd perconaservermongodbs.psmdb.percona.com
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.Fzqe9EBRP8
+ cat /tmp/tmp.waqtObhSw6
+ rm /tmp/tmp.Fzqe9EBRP8 /tmp/tmp.waqtObhSw6
+ return 0
+ local rbac_yaml=rbac.yaml
+ '[' -n psmdb-operator ']'
+ rbac_yaml=cw-rbac.yaml
+ kubectl_bin delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1554/deploy/cw-rbac.yaml --ignore-not-found
++ mktemp
+ local LAST_OUT=/tmp/tmp.fKnrib5QaR
++ mktemp
+ local LAST_ERR=/tmp/tmp.jwqpnTu7JW
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1554/deploy/cw-rbac.yaml --ignore-not-found
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.fKnrib5QaR
clusterrole.rbac.authorization.k8s.io "percona-server-mongodb-operator" deleted
clusterrolebinding.rbac.authorization.k8s.io "service-account-percona-server-mongodb-operator" deleted
+ cat /tmp/tmp.jwqpnTu7JW
+ rm /tmp/tmp.fKnrib5QaR /tmp/tmp.jwqpnTu7JW
+ return 0
+ check_crd_for_deletion PR-1554-4f44f8bb
+ local git_tag=PR-1554-4f44f8bb
++ curl -s https://raw.githubusercontent.com/percona/percona-server-mongodb-operator/PR-1554-4f44f8bb/deploy/crd.yaml
++ yq eval .metadata.name
++ /usr/bin/sed s/---//g
++ /usr/bin/sed ':a;N;$!ba;s/\n/ /g'
+ for crd_name in '$(curl -s https://raw.githubusercontent.com/percona/percona-server-mongodb-operator/${git_tag}/deploy/crd.yaml | yq eval '\''.metadata.name'\'' | $sed '\''s/---//g'\'' | $sed '\'':a;N;$!ba;s/\n/ /g'\'')'
++ kubectl_bin get crd/null -o 'jsonpath={.status.conditions[-1].type}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.d27FPNQd1I
+++ mktemp
++ local LAST_ERR=/tmp/tmp.OgXJmqBxoB
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get crd/null -o 'jsonpath={.status.conditions[-1].type}'
++ exit_status=1
++ set -e
++ '[' 1 '!=' 0 -a -n 1 ']'
++ cat /tmp/tmp.d27FPNQd1I
++ cat /tmp/tmp.OgXJmqBxoB
Error from server (NotFound): customresourcedefinitions.apiextensions.k8s.io "null" not found
++ sleep 0
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get crd/null -o 'jsonpath={.status.conditions[-1].type}'
++ exit_status=1
++ set -e
++ '[' 1 '!=' 0 -a -n 1 ']'
++ cat /tmp/tmp.d27FPNQd1I
++ cat /tmp/tmp.OgXJmqBxoB
Error from server (NotFound): customresourcedefinitions.apiextensions.k8s.io "null" not found
++ sleep 4
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get crd/null -o 'jsonpath={.status.conditions[-1].type}'
++ exit_status=1
++ set -e
++ '[' 1 '!=' 0 -a -n 1 ']'
++ cat /tmp/tmp.d27FPNQd1I
++ cat /tmp/tmp.OgXJmqBxoB
Error from server (NotFound): customresourcedefinitions.apiextensions.k8s.io "null" not found
++ sleep 8
++ cat /tmp/tmp.d27FPNQd1I
++ cat /tmp/tmp.OgXJmqBxoB
Error from server (NotFound): customresourcedefinitions.apiextensions.k8s.io "null" not found
++ rm /tmp/tmp.d27FPNQd1I /tmp/tmp.OgXJmqBxoB
++ return 1
+ [[ '' == \T\e\r\m\i\n\a\t\i\n\g ]]
+ '[' -n psmdb-operator ']'
+ create_namespace psmdb-operator
+ local namespace=psmdb-operator
+ local skip_clean_namespace=
+ [[ 1 == 1 ]]
+ [[ -z '' ]]
+ destroy_chaos_mesh
++ helm list --all-namespaces --filter chaos-mesh
++ tail -n1
++ awk '-F ' '{print $2}'
++ sed s/NAMESPACE//
+ local chaos_mesh_ns=
+ desc 'destroy chaos-mesh'
+ set +o xtrace


-----------------------------------------------------------------------------------
destroy chaos-mesh
-----------------------------------------------------------------------------------

+ '[' -n '' ']'
++ kubectl get MutatingWebhookConfiguration
++ awk '{print $1}'
++ grep chaos-mesh
+ timeout 30 kubectl delete MutatingWebhookConfiguration
error: resource(s) were provided, but no name was specified
+ :
++ kubectl get ValidatingWebhookConfiguration
++ grep chaos-mesh
++ awk '{print $1}'
+ timeout 30 kubectl delete ValidatingWebhookConfiguration
error: resource(s) were provided, but no name was specified
+ :
++ kubectl get ValidatingWebhookConfiguration
++ grep validate-auth
++ awk '{print $1}'
+ timeout 30 kubectl delete ValidatingWebhookConfiguration
error: resource(s) were provided, but no name was specified
+ :
++ kubectl api-resources
++ grep chaos-mesh
++ awk '{print $1}'
++ kubectl get crd
++ grep chaos-mesh.org
++ awk '{print $1}'
+ timeout 30 kubectl delete crd
error: resource(s) were provided, but no name was specified
+ :
++ kubectl get clusterrolebinding
++ awk '{print $1}'
++ grep chaos-mesh
+ timeout 30 kubectl delete clusterrolebinding
error: resource(s) were provided, but no name was specified
+ :
++ kubectl get clusterrole
++ grep chaos-mesh
++ awk '{print $1}'
+ timeout 30 kubectl delete clusterrole
error: resource(s) were provided, but no name was specified
+ :
+ desc 'cleaned up all old namespaces'
+ set +o xtrace


-----------------------------------------------------------------------------------
cleaned up all old namespaces
-----------------------------------------------------------------------------------

+ '[' -n '' ']'
+ desc 'cleaned up old namespaces psmdb-operator'
+ set +o xtrace


-----------------------------------------------------------------------------------
cleaned up old namespaces psmdb-operator
-----------------------------------------------------------------------------------

+ kubectl_bin delete namespace psmdb-operator --ignore-not-found
++ mktemp
+ kubectl_bin get ns
+ xargs kubectl delete ns
++ mktemp
+ egrep -v '^kube-|^default|Terminating|psmdb-operator|openshift|gke-mcs|^NAME'
+ awk '{print$1}'
+ local LAST_OUT=/tmp/tmp.yS5uP4KfBc
+ local LAST_OUT=/tmp/tmp.CgOCZcR2di
++ mktemp
++ mktemp
+ local LAST_ERR=/tmp/tmp.YbCyyyYDtt
+ local exit_status=0
+ local timeout=4
+ local LAST_ERR=/tmp/tmp.a1YyKM6G9u
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl delete namespace psmdb-operator --ignore-not-found
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl get ns
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.CgOCZcR2di
+ cat /tmp/tmp.YbCyyyYDtt
+ rm /tmp/tmp.CgOCZcR2di /tmp/tmp.YbCyyyYDtt
+ return 0
namespace "cert-manager" deleted
namespace "service-per-pod-351" deleted
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.yS5uP4KfBc
namespace "psmdb-operator" deleted
+ cat /tmp/tmp.a1YyKM6G9u
+ rm /tmp/tmp.yS5uP4KfBc /tmp/tmp.a1YyKM6G9u
+ return 0
+ kubectl_bin wait --for=delete namespace psmdb-operator
++ mktemp
+ local LAST_OUT=/tmp/tmp.1iAuh41fUP
++ mktemp
+ local LAST_ERR=/tmp/tmp.2CHwDnJyVA
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl wait --for=delete namespace psmdb-operator
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.1iAuh41fUP
+ cat /tmp/tmp.2CHwDnJyVA
+ rm /tmp/tmp.1iAuh41fUP /tmp/tmp.2CHwDnJyVA
+ return 0
+ desc 'create namespace psmdb-operator'
+ set +o xtrace


-----------------------------------------------------------------------------------
create namespace psmdb-operator
-----------------------------------------------------------------------------------

+ kubectl_bin create namespace psmdb-operator
++ mktemp
+ local LAST_OUT=/tmp/tmp.xm91LI3pvs
++ mktemp
+ local LAST_ERR=/tmp/tmp.pXyu7ZHkHG
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl create namespace psmdb-operator
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.xm91LI3pvs
namespace/psmdb-operator created
+ cat /tmp/tmp.pXyu7ZHkHG
+ rm /tmp/tmp.xm91LI3pvs /tmp/tmp.pXyu7ZHkHG
+ return 0
++ kubectl_bin config current-context
+++ mktemp
++ local LAST_OUT=/tmp/tmp.DRYbyUFyc0
+++ mktemp
++ local LAST_ERR=/tmp/tmp.NexEkD3CgY
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl config current-context
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.DRYbyUFyc0
++ cat /tmp/tmp.NexEkD3CgY
++ rm /tmp/tmp.DRYbyUFyc0 /tmp/tmp.NexEkD3CgY
++ return 0
+ kubectl_bin config set-context gke_cloud-dev-112233_us-central1-a_jen-psmdb-1554-4f44f8bb-2-cluster8 --namespace=psmdb-operator
++ mktemp
+ local LAST_OUT=/tmp/tmp.LCwCZhfo3M
++ mktemp
+ local LAST_ERR=/tmp/tmp.XvH7IXkwC3
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl config set-context gke_cloud-dev-112233_us-central1-a_jen-psmdb-1554-4f44f8bb-2-cluster8 --namespace=psmdb-operator
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.LCwCZhfo3M
Context "gke_cloud-dev-112233_us-central1-a_jen-psmdb-1554-4f44f8bb-2-cluster8" modified.
+ cat /tmp/tmp.XvH7IXkwC3
+ rm /tmp/tmp.LCwCZhfo3M /tmp/tmp.XvH7IXkwC3
+ return 0
+ deploy_operator
+ desc 'start PSMDB operator'
+ set +o xtrace


-----------------------------------------------------------------------------------
start PSMDB operator
-----------------------------------------------------------------------------------

+ local cr_file
+ '[' -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1554/e2e-tests/service-per-pod/conf/crd.yaml ']'
+ cr_file=/mnt/jenkins/workspace/cloud-psmdb-operator_PR-1554/deploy/crd.yaml
+ kubectl_bin apply --server-side --force-conflicts -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1554/deploy/crd.yaml
++ mktemp
+ local LAST_OUT=/tmp/tmp.jpg4gkerO1
++ mktemp
+ local LAST_ERR=/tmp/tmp.1J79D4dEvV
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl apply --server-side --force-conflicts -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1554/deploy/crd.yaml
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.jpg4gkerO1
customresourcedefinition.apiextensions.k8s.io/perconaservermongodbbackups.psmdb.percona.com serverside-applied
customresourcedefinition.apiextensions.k8s.io/perconaservermongodbrestores.psmdb.percona.com serverside-applied
customresourcedefinition.apiextensions.k8s.io/perconaservermongodbs.psmdb.percona.com serverside-applied
+ cat /tmp/tmp.1J79D4dEvV
+ rm /tmp/tmp.jpg4gkerO1 /tmp/tmp.1J79D4dEvV
+ return 0
+ '[' -n psmdb-operator ']'
+ apply_rbac cw-rbac
+ local operator_namespace=psmdb-operator
+ local rbac=cw-rbac
+ sed -e 's^namespace: .*^namespace: psmdb-operator^'
+ cat /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1554/deploy/cw-rbac.yaml
+ kubectl_bin apply -n psmdb-operator -f -
++ mktemp
+ local LAST_OUT=/tmp/tmp.BcGlZjwB9s
++ mktemp
+ local LAST_ERR=/tmp/tmp.k20yT91u0q
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl apply -n psmdb-operator -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.BcGlZjwB9s
clusterrole.rbac.authorization.k8s.io/percona-server-mongodb-operator created
serviceaccount/percona-server-mongodb-operator created
clusterrolebinding.rbac.authorization.k8s.io/service-account-percona-server-mongodb-operator created
+ cat /tmp/tmp.k20yT91u0q
+ rm /tmp/tmp.BcGlZjwB9s /tmp/tmp.k20yT91u0q
+ return 0
+ kubectl_bin apply -f -
+ yq eval '
			(.spec.template.spec.containers[].image = "perconalab/percona-server-mongodb-operator:PR-1554-4f44f8bb") |
			((.. | select(.[] == "DISABLE_TELEMETRY")) |= .value="true") |
			((.. | select(.[] == "LOG_LEVEL")) |= .value="DEBUG")' /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1554/deploy/cw-operator.yaml
++ mktemp
+ local LAST_OUT=/tmp/tmp.PjAKSAkGyC
++ mktemp
+ local LAST_ERR=/tmp/tmp.099sFYxgZr
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl apply -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.PjAKSAkGyC
deployment.apps/percona-server-mongodb-operator created
+ cat /tmp/tmp.099sFYxgZr
+ rm /tmp/tmp.PjAKSAkGyC /tmp/tmp.099sFYxgZr
+ return 0
+ sleep 2
++ get_operator_pod
++ kubectl_bin get pods --selector=name=percona-server-mongodb-operator -o 'jsonpath={.items[].metadata.name}' -n psmdb-operator
+++ mktemp
++ local LAST_OUT=/tmp/tmp.g6QJw1uAcH
+++ mktemp
++ local LAST_ERR=/tmp/tmp.Hf7mRBdmNX
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=percona-server-mongodb-operator -o 'jsonpath={.items[].metadata.name}' -n psmdb-operator
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.g6QJw1uAcH
++ cat /tmp/tmp.Hf7mRBdmNX
++ rm /tmp/tmp.g6QJw1uAcH /tmp/tmp.Hf7mRBdmNX
++ return 0
+ wait_pod percona-server-mongodb-operator-7464d9b8bb-bfsn4
+ local pod=percona-server-mongodb-operator-7464d9b8bb-bfsn4
+ set +o xtrace
waiting for pod/percona-server-mongodb-operator-7464d9b8bb-bfsn4 to be ready.OK
+ create_namespace service-per-pod-27980
+ local namespace=service-per-pod-27980
+ local skip_clean_namespace=
+ [[ 1 == 1 ]]
+ [[ -z '' ]]
+ destroy_chaos_mesh
++ helm list --all-namespaces --filter chaos-mesh
++ tail -n1
++ sed s/NAMESPACE//
++ awk '-F ' '{print $2}'
+ local chaos_mesh_ns=
+ desc 'destroy chaos-mesh'
+ set +o xtrace


-----------------------------------------------------------------------------------
destroy chaos-mesh
-----------------------------------------------------------------------------------

+ '[' -n '' ']'
++ kubectl get MutatingWebhookConfiguration
++ awk '{print $1}'
++ grep chaos-mesh
+ timeout 30 kubectl delete MutatingWebhookConfiguration
error: resource(s) were provided, but no name was specified
+ :
++ kubectl get ValidatingWebhookConfiguration
++ grep chaos-mesh
++ awk '{print $1}'
+ timeout 30 kubectl delete ValidatingWebhookConfiguration
error: resource(s) were provided, but no name was specified
+ :
++ kubectl get ValidatingWebhookConfiguration
++ grep validate-auth
++ awk '{print $1}'
+ timeout 30 kubectl delete ValidatingWebhookConfiguration
error: resource(s) were provided, but no name was specified
+ :
++ kubectl api-resources
++ awk '{print $1}'
++ grep chaos-mesh
++ kubectl get crd
++ grep chaos-mesh.org
++ awk '{print $1}'
+ timeout 30 kubectl delete crd
error: resource(s) were provided, but no name was specified
+ :
++ kubectl get clusterrolebinding
++ grep chaos-mesh
++ awk '{print $1}'
+ timeout 30 kubectl delete clusterrolebinding
error: resource(s) were provided, but no name was specified
+ :
++ awk '{print $1}'
++ grep chaos-mesh
++ kubectl get clusterrole
+ timeout 30 kubectl delete clusterrole
error: resource(s) were provided, but no name was specified
+ :
+ desc 'cleaned up all old namespaces'
+ set +o xtrace


-----------------------------------------------------------------------------------
cleaned up all old namespaces
-----------------------------------------------------------------------------------

+ kubectl_bin get ns
+ awk '{print$1}'
+ '[' -n '' ']'
+ desc 'cleaned up old namespaces service-per-pod-27980'
+ xargs kubectl delete ns
+ set +o xtrace


-----------------------------------------------------------------------------------
cleaned up old namespaces service-per-pod-27980
-----------------------------------------------------------------------------------

+ kubectl_bin delete namespace service-per-pod-27980 --ignore-not-found
++ mktemp
+ egrep -v '^kube-|^default|Terminating|psmdb-operator|openshift|gke-mcs|^NAME'
+ local LAST_OUT=/tmp/tmp.ua7Vdc0Kgd
++ mktemp
+ local LAST_ERR=/tmp/tmp.JIL3dtxGYc
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl delete namespace service-per-pod-27980 --ignore-not-found
++ mktemp
+ local LAST_OUT=/tmp/tmp.vuxRFCv3q8
++ mktemp
+ local LAST_ERR=/tmp/tmp.wHpNnvGk2U
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl get ns
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.vuxRFCv3q8
+ cat /tmp/tmp.wHpNnvGk2U
+ rm /tmp/tmp.vuxRFCv3q8 /tmp/tmp.wHpNnvGk2U
+ return 0
error: resource(s) were provided, but no name was specified
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.ua7Vdc0Kgd
+ cat /tmp/tmp.JIL3dtxGYc
+ rm /tmp/tmp.ua7Vdc0Kgd /tmp/tmp.JIL3dtxGYc
+ return 0
+ kubectl_bin wait --for=delete namespace service-per-pod-27980
++ mktemp
+ local LAST_OUT=/tmp/tmp.spflbpBdG4
++ mktemp
+ local LAST_ERR=/tmp/tmp.wSxZhfcy6q
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl wait --for=delete namespace service-per-pod-27980
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.spflbpBdG4
+ cat /tmp/tmp.wSxZhfcy6q
+ rm /tmp/tmp.spflbpBdG4 /tmp/tmp.wSxZhfcy6q
+ return 0
+ desc 'create namespace service-per-pod-27980'
+ set +o xtrace


-----------------------------------------------------------------------------------
create namespace service-per-pod-27980
-----------------------------------------------------------------------------------

+ kubectl_bin create namespace service-per-pod-27980
++ mktemp
+ local LAST_OUT=/tmp/tmp.7OrgLJH7wV
++ mktemp
+ local LAST_ERR=/tmp/tmp.pO1K27UfeI
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl create namespace service-per-pod-27980
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.7OrgLJH7wV
namespace/service-per-pod-27980 created
+ cat /tmp/tmp.pO1K27UfeI
+ rm /tmp/tmp.7OrgLJH7wV /tmp/tmp.pO1K27UfeI
+ return 0
++ kubectl_bin config current-context
+++ mktemp
++ local LAST_OUT=/tmp/tmp.X5mvm6tfVV
+++ mktemp
++ local LAST_ERR=/tmp/tmp.pokpKLDHf9
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl config current-context
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.X5mvm6tfVV
++ cat /tmp/tmp.pokpKLDHf9
++ rm /tmp/tmp.X5mvm6tfVV /tmp/tmp.pokpKLDHf9
++ return 0
+ kubectl_bin config set-context gke_cloud-dev-112233_us-central1-a_jen-psmdb-1554-4f44f8bb-2-cluster8 --namespace=service-per-pod-27980
++ mktemp
+ local LAST_OUT=/tmp/tmp.Kasn6WDLRw
++ mktemp
+ local LAST_ERR=/tmp/tmp.er86EqMkZ3
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl config set-context gke_cloud-dev-112233_us-central1-a_jen-psmdb-1554-4f44f8bb-2-cluster8 --namespace=service-per-pod-27980
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.Kasn6WDLRw
Context "gke_cloud-dev-112233_us-central1-a_jen-psmdb-1554-4f44f8bb-2-cluster8" modified.
+ cat /tmp/tmp.er86EqMkZ3
+ rm /tmp/tmp.Kasn6WDLRw /tmp/tmp.er86EqMkZ3
+ return 0
+ deploy_cert_manager
+ desc 'deploy cert manager'
+ set +o xtrace


-----------------------------------------------------------------------------------
deploy cert manager
-----------------------------------------------------------------------------------

+ kubectl_bin create namespace cert-manager
++ mktemp
+ local LAST_OUT=/tmp/tmp.TkQyGLoCYD
++ mktemp
+ local LAST_ERR=/tmp/tmp.KWUD0vlKEo
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl create namespace cert-manager
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.TkQyGLoCYD
namespace/cert-manager created
+ cat /tmp/tmp.KWUD0vlKEo
+ rm /tmp/tmp.TkQyGLoCYD /tmp/tmp.KWUD0vlKEo
+ return 0
+ kubectl_bin label namespace cert-manager certmanager.k8s.io/disable-validation=true
++ mktemp
+ local LAST_OUT=/tmp/tmp.3D5vh89dCS
++ mktemp
+ local LAST_ERR=/tmp/tmp.ZznjNoDHcr
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl label namespace cert-manager certmanager.k8s.io/disable-validation=true
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.3D5vh89dCS
namespace/cert-manager labeled
+ cat /tmp/tmp.ZznjNoDHcr
+ rm /tmp/tmp.3D5vh89dCS /tmp/tmp.ZznjNoDHcr
+ return 0
+ kubectl_bin apply -f https://github.com/cert-manager/cert-manager/releases/download/v1.14.5/cert-manager.yaml --validate=false
++ mktemp
+ local LAST_OUT=/tmp/tmp.NsQRw341Xt
++ mktemp
+ local LAST_ERR=/tmp/tmp.EdBDaX2Vlm
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl apply -f https://github.com/cert-manager/cert-manager/releases/download/v1.14.5/cert-manager.yaml --validate=false
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.NsQRw341Xt
namespace/cert-manager configured
customresourcedefinition.apiextensions.k8s.io/certificaterequests.cert-manager.io unchanged
customresourcedefinition.apiextensions.k8s.io/certificates.cert-manager.io unchanged
customresourcedefinition.apiextensions.k8s.io/challenges.acme.cert-manager.io unchanged
customresourcedefinition.apiextensions.k8s.io/clusterissuers.cert-manager.io unchanged
customresourcedefinition.apiextensions.k8s.io/issuers.cert-manager.io unchanged
customresourcedefinition.apiextensions.k8s.io/orders.acme.cert-manager.io unchanged
serviceaccount/cert-manager-cainjector created
serviceaccount/cert-manager created
serviceaccount/cert-manager-webhook created
clusterrole.rbac.authorization.k8s.io/cert-manager-cainjector unchanged
clusterrole.rbac.authorization.k8s.io/cert-manager-controller-issuers unchanged
clusterrole.rbac.authorization.k8s.io/cert-manager-controller-clusterissuers unchanged
clusterrole.rbac.authorization.k8s.io/cert-manager-controller-certificates unchanged
clusterrole.rbac.authorization.k8s.io/cert-manager-controller-orders unchanged
clusterrole.rbac.authorization.k8s.io/cert-manager-controller-challenges unchanged
clusterrole.rbac.authorization.k8s.io/cert-manager-controller-ingress-shim unchanged
clusterrole.rbac.authorization.k8s.io/cert-manager-cluster-view unchanged
clusterrole.rbac.authorization.k8s.io/cert-manager-view unchanged
clusterrole.rbac.authorization.k8s.io/cert-manager-edit unchanged
clusterrole.rbac.authorization.k8s.io/cert-manager-controller-approve:cert-manager-io unchanged
clusterrole.rbac.authorization.k8s.io/cert-manager-controller-certificatesigningrequests unchanged
clusterrole.rbac.authorization.k8s.io/cert-manager-webhook:subjectaccessreviews unchanged
clusterrolebinding.rbac.authorization.k8s.io/cert-manager-cainjector unchanged
clusterrolebinding.rbac.authorization.k8s.io/cert-manager-controller-issuers unchanged
clusterrolebinding.rbac.authorization.k8s.io/cert-manager-controller-clusterissuers unchanged
clusterrolebinding.rbac.authorization.k8s.io/cert-manager-controller-certificates unchanged
clusterrolebinding.rbac.authorization.k8s.io/cert-manager-controller-orders unchanged
clusterrolebinding.rbac.authorization.k8s.io/cert-manager-controller-challenges unchanged
clusterrolebinding.rbac.authorization.k8s.io/cert-manager-controller-ingress-shim unchanged
clusterrolebinding.rbac.authorization.k8s.io/cert-manager-controller-approve:cert-manager-io unchanged
clusterrolebinding.rbac.authorization.k8s.io/cert-manager-controller-certificatesigningrequests unchanged
clusterrolebinding.rbac.authorization.k8s.io/cert-manager-webhook:subjectaccessreviews configured
role.rbac.authorization.k8s.io/cert-manager-cainjector:leaderelection unchanged
role.rbac.authorization.k8s.io/cert-manager:leaderelection unchanged
role.rbac.authorization.k8s.io/cert-manager-webhook:dynamic-serving created
rolebinding.rbac.authorization.k8s.io/cert-manager-cainjector:leaderelection unchanged
rolebinding.rbac.authorization.k8s.io/cert-manager:leaderelection configured
rolebinding.rbac.authorization.k8s.io/cert-manager-webhook:dynamic-serving created
service/cert-manager created
service/cert-manager-webhook created
deployment.apps/cert-manager-cainjector created
deployment.apps/cert-manager created
deployment.apps/cert-manager-webhook created
mutatingwebhookconfiguration.admissionregistration.k8s.io/cert-manager-webhook configured
validatingwebhookconfiguration.admissionregistration.k8s.io/cert-manager-webhook configured
+ cat /tmp/tmp.EdBDaX2Vlm
Warning: resource namespaces/cert-manager is missing the kubectl.kubernetes.io/last-applied-configuration annotation which is required by kubectl apply. kubectl apply should only be used on resources created declaratively by either kubectl create --save-config or kubectl apply. The missing annotation will be patched automatically.
+ rm /tmp/tmp.NsQRw341Xt /tmp/tmp.EdBDaX2Vlm
+ return 0
+ kubectl_bin -n cert-manager wait pod -l app.kubernetes.io/instance=cert-manager --for=condition=ready
++ mktemp
+ local LAST_OUT=/tmp/tmp.4MHv2VybPP
++ mktemp
+ local LAST_ERR=/tmp/tmp.4iW800B3vW
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl -n cert-manager wait pod -l app.kubernetes.io/instance=cert-manager --for=condition=ready
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.4MHv2VybPP
pod/cert-manager-5658d944df-6hwp5 condition met
pod/cert-manager-cainjector-cb99ff845-pkvvd condition met
pod/cert-manager-webhook-7fd74b8dc7-d22ms condition met
+ cat /tmp/tmp.4iW800B3vW
+ rm /tmp/tmp.4MHv2VybPP /tmp/tmp.4iW800B3vW
+ return 0
+ sleep 120
+ desc 'create secrets and start client'
+ set +o xtrace


-----------------------------------------------------------------------------------
create secrets and start client
-----------------------------------------------------------------------------------

+ kubectl_bin apply -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1554/e2e-tests/conf/client.yml -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1554/e2e-tests/conf/secrets.yml
++ mktemp
+ local LAST_OUT=/tmp/tmp.XhFyx8ms4u
++ mktemp
+ local LAST_ERR=/tmp/tmp.fvHJyotUqq
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl apply -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1554/e2e-tests/conf/client.yml -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1554/e2e-tests/conf/secrets.yml
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.XhFyx8ms4u
deployment.apps/psmdb-client created
secret/some-users created
+ cat /tmp/tmp.fvHJyotUqq
+ rm /tmp/tmp.XhFyx8ms4u /tmp/tmp.fvHJyotUqq
+ return 0
+ desc 'check ClusterIP'
+ set +o xtrace


-----------------------------------------------------------------------------------
check ClusterIP
-----------------------------------------------------------------------------------

+ check_cr_config cluster-ip-rs0
+ local cluster=cluster-ip-rs0
+ desc 'create first PSMDB cluster cluster-ip-rs0'
+ set +o xtrace


-----------------------------------------------------------------------------------
create first PSMDB cluster cluster-ip-rs0
-----------------------------------------------------------------------------------

+ apply_cluster /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1554/e2e-tests/service-per-pod/conf/cluster-ip-rs0.yml
+ '[' -z '' ']'
+ cat_config /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1554/e2e-tests/service-per-pod/conf/cluster-ip-rs0.yml
+ kubectl_bin apply -f -
+ cat /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1554/e2e-tests/service-per-pod/conf/cluster-ip-rs0.yml
+ yq eval '(.spec | select(has("pmm"))).pmm.image = "perconalab/pmm-client:dev-latest"'
+ yq eval '(.spec | select(has("initImage"))).initImage = "perconalab/percona-server-mongodb-operator:PR-1554-4f44f8bb"'
+ yq eval '(.spec | select(has("backup"))).backup.image = "perconalab/percona-server-mongodb-operator:main-backup"'
+ yq eval '(.spec | select(.image == null)).image = "perconalab/percona-server-mongodb-operator:main-mongod7.0"'
++ mktemp
+ yq eval '.spec.upgradeOptions.apply="Never"'
+ local LAST_OUT=/tmp/tmp.gVONfK8OQh
++ mktemp
+ local LAST_ERR=/tmp/tmp.ASukiRz9RP
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl apply -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.gVONfK8OQh
perconaservermongodb.psmdb.percona.com/cluster-ip created
+ cat /tmp/tmp.ASukiRz9RP
+ rm /tmp/tmp.gVONfK8OQh /tmp/tmp.ASukiRz9RP
+ return 0
+ desc 'check if all 3 Pods started'
+ set +o xtrace


-----------------------------------------------------------------------------------
check if all 3 Pods started
-----------------------------------------------------------------------------------

+ wait_for_running cluster-ip-rs0 3 false
+ local name=cluster-ip-rs0
+ let last_pod=2
+ local check_cluster_readyness=false
+ set_debug
+ [[ 1 == 1 ]]
+ set -o xtrace
+ local rs_name=rs0
+ local cluster_name=cluster-ip
++ seq 0 2
+ for i in '$(seq 0 $last_pod)'
+ [[ 0 -eq 2 ]]
+ wait_pod cluster-ip-rs0-0
+ local pod=cluster-ip-rs0-0
+ set +o xtrace
waiting for pod/cluster-ip-rs0-0 to be ready........OK
+ for i in '$(seq 0 $last_pod)'
+ [[ 1 -eq 2 ]]
+ wait_pod cluster-ip-rs0-1
+ local pod=cluster-ip-rs0-1
+ set +o xtrace
waiting for pod/cluster-ip-rs0-1 to be ready.......OK
+ for i in '$(seq 0 $last_pod)'
+ [[ 2 -eq 2 ]]
++ kubectl_bin get psmdb cluster-ip -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].arbiter.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.4P25WqOjIM
+++ mktemp
++ local LAST_ERR=/tmp/tmp.iSExjJwXTP
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb cluster-ip -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].arbiter.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.4P25WqOjIM
++ cat /tmp/tmp.iSExjJwXTP
++ rm /tmp/tmp.4P25WqOjIM /tmp/tmp.iSExjJwXTP
++ return 0
+ [[ '' == \t\r\u\e ]]
+ wait_pod cluster-ip-rs0-2
+ local pod=cluster-ip-rs0-2
+ set +o xtrace
waiting for pod/cluster-ip-rs0-2 to be ready.....OK
++ kubectl_bin get psmdb cluster-ip -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].non_voting.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.bHXmilVWpI
+++ mktemp
++ local LAST_ERR=/tmp/tmp.SbcUAs4KbT
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb cluster-ip -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].non_voting.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.bHXmilVWpI
++ cat /tmp/tmp.SbcUAs4KbT
++ rm /tmp/tmp.bHXmilVWpI /tmp/tmp.SbcUAs4KbT
++ return 0
+ [[ '' == \t\r\u\e ]]
+ sleep 10
+ [[ false == \t\r\u\e ]]
+ desc 'check if service and statefulset created with expected config'
+ set +o xtrace


-----------------------------------------------------------------------------------
check if service and statefulset created with expected config
-----------------------------------------------------------------------------------

+ compare_kubectl statefulset/cluster-ip-rs0
+ local resource=statefulset/cluster-ip-rs0
+ local postfix=
+ local expected_result=/mnt/jenkins/workspace/cloud-psmdb-operator_PR-1554/e2e-tests/service-per-pod/compare/statefulset_cluster-ip-rs0.yml
+ local new_result=/tmp/tmp.8ZWelD8e7G/statefulset_cluster-ip-rs0.yml
+ '[' -n '' -a -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1554/e2e-tests/service-per-pod/compare/statefulset_cluster-ip-rs0-oc.yml ']'
+ kubectl_bin get -o yaml statefulset/cluster-ip-rs0
++ mktemp
+ yq eval '
			del(.metadata.ownerReferences[].apiVersion) |
			del(.metadata.managedFields) |
			del(.. | select(has("creationTimestamp")).creationTimestamp) |
			del(.. | select(has("namespace")).namespace) |
			del(.. | select(has("uid")).uid) |
			del(.metadata.resourceVersion) |
			del(.spec.template.spec.containers[].env[] | select(.name == "NAMESPACE")) |
			del(.metadata.selfLink) |
			del(.metadata.annotations."cloud.google.com/neg") |
			del(.. | select(has("image")).image) |
			del(.. | select(has("clusterIP")).clusterIP) |
			del(.. | select(has("clusterIPs")).clusterIPs) |
			del(.. | select(has("dataSource")).dataSource) |
			del(.. | select(has("procMount")).procMount) |
			del(.. | select(has("storageClassName")).storageClassName) |
			del(.. | select(has("finalizers")).finalizers) |
			del(.. | select(has("kubernetes.io/pvc-protection"))."kubernetes.io/pvc-protection") |
			del(.. | select(has("volumeName")).volumeName) |
			del(.. | select(has("volume.beta.kubernetes.io/storage-provisioner"))."volume.beta.kubernetes.io/storage-provisioner") |
			del(.. | select(has("volume.kubernetes.io/storage-provisioner"))."volume.kubernetes.io/storage-provisioner") |
			del(.spec.volumeMode) |
			del(.. | select(has("volume.kubernetes.io/selected-node"))."volume.kubernetes.io/selected-node") |
			del(.. | select(has("percona.com/last-config-hash"))."percona.com/last-config-hash") |
			del(.. | select(has("percona.com/configuration-hash"))."percona.com/configuration-hash") |
			del(.. | select(has("percona.com/ssl-hash"))."percona.com/ssl-hash") |
			del(.. | select(has("percona.com/ssl-internal-hash"))."percona.com/ssl-internal-hash") |
			del(.spec.volumeClaimTemplates[].spec.volumeMode | select(. == "Filesystem")) |
			del(.. | select(has("healthCheckNodePort")).healthCheckNodePort) |
			del(.. | select(has("nodePort")).nodePort) |
			del(.status) |
			(.. | select(tag == "!!str")) |= sub("service-per-pod-27980", "NAME_SPACE") |
			del(.spec.volumeClaimTemplates[].apiVersion) |
			del(.spec.volumeClaimTemplates[].kind) |
			del(.spec.ipFamilies) |
			del(.spec.ipFamilyPolicy) |
			(.. | select(. == "extensions/v1beta1")) = "apps/v1" |
			(.. | select(. == "batch/v1beta1")) = "batch/v1" ' -
+ local LAST_OUT=/tmp/tmp.cLM1nxGX09
++ mktemp
+ local LAST_ERR=/tmp/tmp.NMNqpocjWO
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl get -o yaml statefulset/cluster-ip-rs0
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.cLM1nxGX09
+ cat /tmp/tmp.NMNqpocjWO
+ rm /tmp/tmp.cLM1nxGX09 /tmp/tmp.NMNqpocjWO
+ return 0
+ yq -i eval 'del(.spec.persistentVolumeClaimRetentionPolicy)' /tmp/tmp.8ZWelD8e7G/statefulset_cluster-ip-rs0.yml
+ version_gt 1.22
++ echo '1.26 >= 1.22'
++ bc -l
+ '[' 1 -eq 1 ']'
+ return 0
+ yq -i eval 'del(.spec.internalTrafficPolicy)' /tmp/tmp.8ZWelD8e7G/statefulset_cluster-ip-rs0.yml
+ yq -i eval 'del(.spec.allocateLoadBalancerNodePorts)' /tmp/tmp.8ZWelD8e7G/statefulset_cluster-ip-rs0.yml
+ [[ /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1554/e2e-tests/service-per-pod/compare/statefulset_cluster-ip-rs0.yml == */cronjob* ]]
+ diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1554/e2e-tests/service-per-pod/compare/statefulset_cluster-ip-rs0.yml /tmp/tmp.8ZWelD8e7G/statefulset_cluster-ip-rs0.yml
+ compare_kubectl service/cluster-ip-rs0-0
+ local resource=service/cluster-ip-rs0-0
+ local postfix=
+ local expected_result=/mnt/jenkins/workspace/cloud-psmdb-operator_PR-1554/e2e-tests/service-per-pod/compare/service_cluster-ip-rs0-0.yml
+ local new_result=/tmp/tmp.8ZWelD8e7G/service_cluster-ip-rs0-0.yml
+ '[' -n '' -a -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1554/e2e-tests/service-per-pod/compare/service_cluster-ip-rs0-0-oc.yml ']'
+ kubectl_bin get -o yaml service/cluster-ip-rs0-0
+ yq eval '
			del(.metadata.ownerReferences[].apiVersion) |
			del(.metadata.managedFields) |
			del(.. | select(has("creationTimestamp")).creationTimestamp) |
			del(.. | select(has("namespace")).namespace) |
			del(.. | select(has("uid")).uid) |
			del(.metadata.resourceVersion) |
			del(.spec.template.spec.containers[].env[] | select(.name == "NAMESPACE")) |
			del(.metadata.selfLink) |
			del(.metadata.annotations."cloud.google.com/neg") |
			del(.. | select(has("image")).image) |
			del(.. | select(has("clusterIP")).clusterIP) |
			del(.. | select(has("clusterIPs")).clusterIPs) |
			del(.. | select(has("dataSource")).dataSource) |
			del(.. | select(has("procMount")).procMount) |
			del(.. | select(has("storageClassName")).storageClassName) |
			del(.. | select(has("finalizers")).finalizers) |
			del(.. | select(has("kubernetes.io/pvc-protection"))."kubernetes.io/pvc-protection") |
			del(.. | select(has("volumeName")).volumeName) |
			del(.. | select(has("volume.beta.kubernetes.io/storage-provisioner"))."volume.beta.kubernetes.io/storage-provisioner") |
			del(.. | select(has("volume.kubernetes.io/storage-provisioner"))."volume.kubernetes.io/storage-provisioner") |
			del(.spec.volumeMode) |
			del(.. | select(has("volume.kubernetes.io/selected-node"))."volume.kubernetes.io/selected-node") |
			del(.. | select(has("percona.com/last-config-hash"))."percona.com/last-config-hash") |
			del(.. | select(has("percona.com/configuration-hash"))."percona.com/configuration-hash") |
			del(.. | select(has("percona.com/ssl-hash"))."percona.com/ssl-hash") |
			del(.. | select(has("percona.com/ssl-internal-hash"))."percona.com/ssl-internal-hash") |
			del(.spec.volumeClaimTemplates[].spec.volumeMode | select(. == "Filesystem")) |
			del(.. | select(has("healthCheckNodePort")).healthCheckNodePort) |
			del(.. | select(has("nodePort")).nodePort) |
			del(.status) |
			(.. | select(tag == "!!str")) |= sub("service-per-pod-27980", "NAME_SPACE") |
			del(.spec.volumeClaimTemplates[].apiVersion) |
			del(.spec.volumeClaimTemplates[].kind) |
			del(.spec.ipFamilies) |
			del(.spec.ipFamilyPolicy) |
			(.. | select(. == "extensions/v1beta1")) = "apps/v1" |
			(.. | select(. == "batch/v1beta1")) = "batch/v1" ' -
++ mktemp
+ local LAST_OUT=/tmp/tmp.yrinFq6sZe
++ mktemp
+ local LAST_ERR=/tmp/tmp.tJsnGzD0W4
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl get -o yaml service/cluster-ip-rs0-0
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.yrinFq6sZe
+ cat /tmp/tmp.tJsnGzD0W4
+ rm /tmp/tmp.yrinFq6sZe /tmp/tmp.tJsnGzD0W4
+ return 0
+ yq -i eval 'del(.spec.persistentVolumeClaimRetentionPolicy)' /tmp/tmp.8ZWelD8e7G/service_cluster-ip-rs0-0.yml
+ version_gt 1.22
++ echo '1.26 >= 1.22'
++ bc -l
+ '[' 1 -eq 1 ']'
+ return 0
+ yq -i eval 'del(.spec.internalTrafficPolicy)' /tmp/tmp.8ZWelD8e7G/service_cluster-ip-rs0-0.yml
+ yq -i eval 'del(.spec.allocateLoadBalancerNodePorts)' /tmp/tmp.8ZWelD8e7G/service_cluster-ip-rs0-0.yml
+ [[ /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1554/e2e-tests/service-per-pod/compare/service_cluster-ip-rs0-0.yml == */cronjob* ]]
+ diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1554/e2e-tests/service-per-pod/compare/service_cluster-ip-rs0-0.yml /tmp/tmp.8ZWelD8e7G/service_cluster-ip-rs0-0.yml
++ get_service_ip cluster-ip-rs0-0
++ local service=cluster-ip-rs0-0
++ local server_type=rs0
+++ kubectl_bin get psmdb/cluster-ip -o 'jsonpath={.spec.replsets[].expose.enabled}'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.FefJE3ADR3
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.RGy0hiqZPB
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in '$(seq 0 2)'
+++ set +e
+++ kubectl get psmdb/cluster-ip -o 'jsonpath={.spec.replsets[].expose.enabled}'
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.FefJE3ADR3
+++ cat /tmp/tmp.RGy0hiqZPB
+++ rm /tmp/tmp.FefJE3ADR3 /tmp/tmp.RGy0hiqZPB
+++ return 0
++ '[' true '!=' true ']'
++ kubectl_bin get service/cluster-ip-rs0-0 -o 'jsonpath={.spec.type}'
++ grep -q NotFound
+++ kubectl_bin get service/cluster-ip-rs0-0 -o 'jsonpath={.spec.type}'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.83Xy8PehZ5
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.AUtNvOo8qz
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in '$(seq 0 2)'
+++ set +e
+++ kubectl get service/cluster-ip-rs0-0 -o 'jsonpath={.spec.type}'
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.83Xy8PehZ5
+++ cat /tmp/tmp.AUtNvOo8qz
+++ rm /tmp/tmp.83Xy8PehZ5 /tmp/tmp.AUtNvOo8qz
+++ return 0
++ service_type=ClusterIP
++ '[' ClusterIP = ClusterIP ']'
++ kubectl_bin get service/cluster-ip-rs0-0 -o 'jsonpath={.spec.clusterIP}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.5rcEbNAsQg
+++ mktemp
++ local LAST_ERR=/tmp/tmp.eyea7PYkgU
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get service/cluster-ip-rs0-0 -o 'jsonpath={.spec.clusterIP}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.5rcEbNAsQg
++ cat /tmp/tmp.eyea7PYkgU
++ rm /tmp/tmp.5rcEbNAsQg /tmp/tmp.eyea7PYkgU
++ return 0
++ return
++ get_service_ip cluster-ip-rs0-1
++ local service=cluster-ip-rs0-1
++ local server_type=rs0
+++ kubectl_bin get psmdb/cluster-ip -o 'jsonpath={.spec.replsets[].expose.enabled}'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.ooqjRonstg
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.lhe1pXdUtX
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in '$(seq 0 2)'
+++ set +e
+++ kubectl get psmdb/cluster-ip -o 'jsonpath={.spec.replsets[].expose.enabled}'
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.ooqjRonstg
+++ cat /tmp/tmp.lhe1pXdUtX
+++ rm /tmp/tmp.ooqjRonstg /tmp/tmp.lhe1pXdUtX
+++ return 0
++ '[' true '!=' true ']'
++ kubectl_bin get service/cluster-ip-rs0-1 -o 'jsonpath={.spec.type}'
++ grep -q NotFound
+++ kubectl_bin get service/cluster-ip-rs0-1 -o 'jsonpath={.spec.type}'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.LWWxLsyXR5
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.5latfixVcP
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in '$(seq 0 2)'
+++ set +e
+++ kubectl get service/cluster-ip-rs0-1 -o 'jsonpath={.spec.type}'
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.LWWxLsyXR5
+++ cat /tmp/tmp.5latfixVcP
+++ rm /tmp/tmp.LWWxLsyXR5 /tmp/tmp.5latfixVcP
+++ return 0
++ service_type=ClusterIP
++ '[' ClusterIP = ClusterIP ']'
++ kubectl_bin get service/cluster-ip-rs0-1 -o 'jsonpath={.spec.clusterIP}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.vFLgL91PCx
+++ mktemp
++ local LAST_ERR=/tmp/tmp.jk0X3EBLdF
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get service/cluster-ip-rs0-1 -o 'jsonpath={.spec.clusterIP}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.vFLgL91PCx
++ cat /tmp/tmp.jk0X3EBLdF
++ rm /tmp/tmp.vFLgL91PCx /tmp/tmp.jk0X3EBLdF
++ return 0
++ return
++ get_service_ip cluster-ip-rs0-2
++ local service=cluster-ip-rs0-2
++ local server_type=rs0
+++ kubectl_bin get psmdb/cluster-ip -o 'jsonpath={.spec.replsets[].expose.enabled}'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.RDLbkRFRCI
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.06RzLWIeoo
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in '$(seq 0 2)'
+++ set +e
+++ kubectl get psmdb/cluster-ip -o 'jsonpath={.spec.replsets[].expose.enabled}'
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.RDLbkRFRCI
+++ cat /tmp/tmp.06RzLWIeoo
+++ rm /tmp/tmp.RDLbkRFRCI /tmp/tmp.06RzLWIeoo
+++ return 0
++ '[' true '!=' true ']'
++ kubectl_bin get service/cluster-ip-rs0-2 -o 'jsonpath={.spec.type}'
++ grep -q NotFound
+++ kubectl_bin get service/cluster-ip-rs0-2 -o 'jsonpath={.spec.type}'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.Jz73w4p7Mx
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.VTF7oyPSbE
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in '$(seq 0 2)'
+++ set +e
+++ kubectl get service/cluster-ip-rs0-2 -o 'jsonpath={.spec.type}'
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.Jz73w4p7Mx
+++ cat /tmp/tmp.VTF7oyPSbE
+++ rm /tmp/tmp.Jz73w4p7Mx /tmp/tmp.VTF7oyPSbE
+++ return 0
++ service_type=ClusterIP
++ '[' ClusterIP = ClusterIP ']'
++ kubectl_bin get service/cluster-ip-rs0-2 -o 'jsonpath={.spec.clusterIP}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.tXuYFm6UyN
+++ mktemp
++ local LAST_ERR=/tmp/tmp.JtdrxE48s2
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get service/cluster-ip-rs0-2 -o 'jsonpath={.spec.clusterIP}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.tXuYFm6UyN
++ cat /tmp/tmp.JtdrxE48s2
++ rm /tmp/tmp.tXuYFm6UyN /tmp/tmp.JtdrxE48s2
++ return 0
++ return
+ local URI=10.101.167.224,10.101.174.200,10.101.164.208
+ sleep 30
+ desc 'create user myApp'
+ set +o xtrace


-----------------------------------------------------------------------------------
create user myApp
-----------------------------------------------------------------------------------

+ run_mongo 'db.createUser({user:"myApp",pwd:"myPass",roles:[{db:"myApp",role:"readWrite"}]})' userAdmin:userAdmin123456@10.101.167.224,10.101.174.200,10.101.164.208 mongodb :27017
+ local 'command=db.createUser({user:"myApp",pwd:"myPass",roles:[{db:"myApp",role:"readWrite"}]})'
+ local uri=userAdmin:userAdmin123456@10.101.167.224,10.101.174.200,10.101.164.208
+ local driver=mongodb
+ local suffix=:27017
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.tPULU6uqEz
+++ mktemp
++ local LAST_ERR=/tmp/tmp.08ZL4eEndP
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.tPULU6uqEz
++ cat /tmp/tmp.08ZL4eEndP
++ rm /tmp/tmp.tPULU6uqEz /tmp/tmp.08ZL4eEndP
++ return 0
+ local client_container=psmdb-client-7469665986-gbbng
+ local mongo_flag=
+ [[ userAdmin:userAdmin123456@10.101.167.224,10.101.174.200,10.101.164.208 == *cfg* ]]
+ replica_set=rs0
+ kubectl_bin exec psmdb-client-7469665986-gbbng -- bash -c 'printf '\''db.createUser({user:"myApp",pwd:"myPass",roles:[{db:"myApp",role:"readWrite"}]})\n'\'' | mongo mongodb://userAdmin:userAdmin123456@10.101.167.224,10.101.174.200,10.101.164.208:27017/admin?ssl=false\&replicaSet=rs0 '
++ mktemp
+ local LAST_OUT=/tmp/tmp.95lHNflX75
++ mktemp
+ local LAST_ERR=/tmp/tmp.wSOBybBYL5
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl exec psmdb-client-7469665986-gbbng -- bash -c 'printf '\''db.createUser({user:"myApp",pwd:"myPass",roles:[{db:"myApp",role:"readWrite"}]})\n'\'' | mongo mongodb://userAdmin:userAdmin123456@10.101.167.224,10.101.174.200,10.101.164.208:27017/admin?ssl=false\&replicaSet=rs0 '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.95lHNflX75
Percona Server for MongoDB shell version v4.4.29-28
connecting to: mongodb://10.101.167.224:27017,10.101.174.200:27017,10.101.164.208:27017/admin?compressors=disabled&gssapiServiceName=mongodb&replicaSet=rs0&ssl=false
Implicit session: session { "id" : UUID("1370f5d8-869a-4ce0-a3fa-18c272180438") }
Percona Server for MongoDB server version: v7.0.11-6
WARNING: shell and server versions do not match
Successfully added user: {
	"user" : "myApp",
	"roles" : [
		{
			"db" : "myApp",
			"role" : "readWrite"
		}
	]
}
bye
+ cat /tmp/tmp.wSOBybBYL5
+ rm /tmp/tmp.95lHNflX75 /tmp/tmp.wSOBybBYL5
+ return 0
+ sleep 10
+ desc 'write data, read from all'
+ set +o xtrace


-----------------------------------------------------------------------------------
write data, read from all
-----------------------------------------------------------------------------------

+ run_mongo 'use myApp\n db.test.insert({ x: 100500 })' myApp:myPass@10.101.167.224,10.101.174.200,10.101.164.208 mongodb :27017
+ local 'command=use myApp\n db.test.insert({ x: 100500 })'
+ local uri=myApp:myPass@10.101.167.224,10.101.174.200,10.101.164.208
+ local driver=mongodb
+ local suffix=:27017
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.marJgtFvPO
+++ mktemp
++ local LAST_ERR=/tmp/tmp.1ipXuBV4Gn
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.marJgtFvPO
++ cat /tmp/tmp.1ipXuBV4Gn
++ rm /tmp/tmp.marJgtFvPO /tmp/tmp.1ipXuBV4Gn
++ return 0
+ local client_container=psmdb-client-7469665986-gbbng
+ local mongo_flag=
+ [[ myApp:myPass@10.101.167.224,10.101.174.200,10.101.164.208 == *cfg* ]]
+ replica_set=rs0
+ kubectl_bin exec psmdb-client-7469665986-gbbng -- bash -c 'printf '\''use myApp\n db.test.insert({ x: 100500 })\n'\'' | mongo mongodb://myApp:myPass@10.101.167.224,10.101.174.200,10.101.164.208:27017/admin?ssl=false\&replicaSet=rs0 '
++ mktemp
+ local LAST_OUT=/tmp/tmp.se6T8FQOZw
++ mktemp
+ local LAST_ERR=/tmp/tmp.29H9KZuXcn
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl exec psmdb-client-7469665986-gbbng -- bash -c 'printf '\''use myApp\n db.test.insert({ x: 100500 })\n'\'' | mongo mongodb://myApp:myPass@10.101.167.224,10.101.174.200,10.101.164.208:27017/admin?ssl=false\&replicaSet=rs0 '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.se6T8FQOZw
Percona Server for MongoDB shell version v4.4.29-28
connecting to: mongodb://10.101.167.224:27017,10.101.174.200:27017,10.101.164.208:27017/admin?compressors=disabled&gssapiServiceName=mongodb&replicaSet=rs0&ssl=false
Implicit session: session { "id" : UUID("096ad818-4621-4e07-8d7f-51c51a278944") }
Percona Server for MongoDB server version: v7.0.11-6
WARNING: shell and server versions do not match
switched to db myApp
WriteResult({ "nInserted" : 1 })
bye
+ cat /tmp/tmp.29H9KZuXcn
+ rm /tmp/tmp.se6T8FQOZw /tmp/tmp.29H9KZuXcn
+ return 0
+ sleep 10
++ get_service_ip cluster-ip-rs0-0
++ local service=cluster-ip-rs0-0
++ local server_type=rs0
+++ kubectl_bin get psmdb/cluster-ip -o 'jsonpath={.spec.replsets[].expose.enabled}'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.el8e6ZOJPf
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.FDe0TwUq2X
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in '$(seq 0 2)'
+++ set +e
+++ kubectl get psmdb/cluster-ip -o 'jsonpath={.spec.replsets[].expose.enabled}'
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.el8e6ZOJPf
+++ cat /tmp/tmp.FDe0TwUq2X
+++ rm /tmp/tmp.el8e6ZOJPf /tmp/tmp.FDe0TwUq2X
+++ return 0
++ '[' true '!=' true ']'
++ kubectl_bin get service/cluster-ip-rs0-0 -o 'jsonpath={.spec.type}'
++ grep -q NotFound
+++ kubectl_bin get service/cluster-ip-rs0-0 -o 'jsonpath={.spec.type}'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.Vw7pMvY73T
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.bhMGUUKTrd
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in '$(seq 0 2)'
+++ set +e
+++ kubectl get service/cluster-ip-rs0-0 -o 'jsonpath={.spec.type}'
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.Vw7pMvY73T
+++ cat /tmp/tmp.bhMGUUKTrd
+++ rm /tmp/tmp.Vw7pMvY73T /tmp/tmp.bhMGUUKTrd
+++ return 0
++ service_type=ClusterIP
++ '[' ClusterIP = ClusterIP ']'
++ kubectl_bin get service/cluster-ip-rs0-0 -o 'jsonpath={.spec.clusterIP}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.4KARR0QKrL
+++ mktemp
++ local LAST_ERR=/tmp/tmp.lp4OznXUJK
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get service/cluster-ip-rs0-0 -o 'jsonpath={.spec.clusterIP}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.4KARR0QKrL
++ cat /tmp/tmp.lp4OznXUJK
++ rm /tmp/tmp.4KARR0QKrL /tmp/tmp.lp4OznXUJK
++ return 0
++ return
+ compare_mongo_cmd find myApp:myPass@10.101.167.224 '' :27017
+ local command=find
+ local uri=myApp:myPass@10.101.167.224
+ local postfix=
+ local suffix=:27017
+ local database=myApp
+ local collection=test
+ run_mongo 'use myApp\n db.test.find()' myApp:myPass@10.101.167.224 mongodb :27017
+ local 'command=use myApp\n db.test.find()'
+ local uri=myApp:myPass@10.101.167.224
+ egrep -v 'I NETWORK|W NETWORK|F NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:'
+ local driver=mongodb
+ local suffix=:27017
+ /usr/bin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.jawV2pj9Le
+++ mktemp
++ local LAST_ERR=/tmp/tmp.Dk1ea2p8Mu
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.jawV2pj9Le
++ cat /tmp/tmp.Dk1ea2p8Mu
++ rm /tmp/tmp.jawV2pj9Le /tmp/tmp.Dk1ea2p8Mu
++ return 0
+ local client_container=psmdb-client-7469665986-gbbng
+ local mongo_flag=
+ [[ myApp:myPass@10.101.167.224 == *cfg* ]]
+ replica_set=rs0
+ kubectl_bin exec psmdb-client-7469665986-gbbng -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@10.101.167.224:27017/admin?ssl=false\&replicaSet=rs0 '
++ mktemp
+ local LAST_OUT=/tmp/tmp.rtDw0iY3ri
++ mktemp
+ local LAST_ERR=/tmp/tmp.tTLe7P0moT
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl exec psmdb-client-7469665986-gbbng -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@10.101.167.224:27017/admin?ssl=false\&replicaSet=rs0 '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.rtDw0iY3ri
+ cat /tmp/tmp.tTLe7P0moT
+ rm /tmp/tmp.rtDw0iY3ri /tmp/tmp.tTLe7P0moT
+ return 0
+ diff /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1554/e2e-tests/service-per-pod/compare/find.json /tmp/tmp.8ZWelD8e7G/find
++ get_service_ip cluster-ip-rs0-1
++ local service=cluster-ip-rs0-1
++ local server_type=rs0
+++ kubectl_bin get psmdb/cluster-ip -o 'jsonpath={.spec.replsets[].expose.enabled}'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.A3thTJ4jzO
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.oLOWwLU6eS
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in '$(seq 0 2)'
+++ set +e
+++ kubectl get psmdb/cluster-ip -o 'jsonpath={.spec.replsets[].expose.enabled}'
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.A3thTJ4jzO
+++ cat /tmp/tmp.oLOWwLU6eS
+++ rm /tmp/tmp.A3thTJ4jzO /tmp/tmp.oLOWwLU6eS
+++ return 0
++ '[' true '!=' true ']'
++ grep -q NotFound
++ kubectl_bin get service/cluster-ip-rs0-1 -o 'jsonpath={.spec.type}'
+++ kubectl_bin get service/cluster-ip-rs0-1 -o 'jsonpath={.spec.type}'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.u0PcTLqqsa
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.QQbOhlI2Fn
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in '$(seq 0 2)'
+++ set +e
+++ kubectl get service/cluster-ip-rs0-1 -o 'jsonpath={.spec.type}'
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.u0PcTLqqsa
+++ cat /tmp/tmp.QQbOhlI2Fn
+++ rm /tmp/tmp.u0PcTLqqsa /tmp/tmp.QQbOhlI2Fn
+++ return 0
++ service_type=ClusterIP
++ '[' ClusterIP = ClusterIP ']'
++ kubectl_bin get service/cluster-ip-rs0-1 -o 'jsonpath={.spec.clusterIP}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.1G8kwdVvIr
+++ mktemp
++ local LAST_ERR=/tmp/tmp.iqqsppBOPm
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get service/cluster-ip-rs0-1 -o 'jsonpath={.spec.clusterIP}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.1G8kwdVvIr
++ cat /tmp/tmp.iqqsppBOPm
++ rm /tmp/tmp.1G8kwdVvIr /tmp/tmp.iqqsppBOPm
++ return 0
++ return
+ compare_mongo_cmd find myApp:myPass@10.101.174.200 '' :27017
+ local command=find
+ local uri=myApp:myPass@10.101.174.200
+ local postfix=
+ local suffix=:27017
+ local database=myApp
+ local collection=test
+ run_mongo 'use myApp\n db.test.find()' myApp:myPass@10.101.174.200 mongodb :27017
+ local 'command=use myApp\n db.test.find()'
+ local uri=myApp:myPass@10.101.174.200
+ local driver=mongodb
+ local suffix=:27017
+ /usr/bin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
+ egrep -v 'I NETWORK|W NETWORK|F NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:'
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.PpfYnyMMNb
+++ mktemp
++ local LAST_ERR=/tmp/tmp.SaK5zQKqRQ
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.PpfYnyMMNb
++ cat /tmp/tmp.SaK5zQKqRQ
++ rm /tmp/tmp.PpfYnyMMNb /tmp/tmp.SaK5zQKqRQ
++ return 0
+ local client_container=psmdb-client-7469665986-gbbng
+ local mongo_flag=
+ [[ myApp:myPass@10.101.174.200 == *cfg* ]]
+ replica_set=rs0
+ kubectl_bin exec psmdb-client-7469665986-gbbng -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@10.101.174.200:27017/admin?ssl=false\&replicaSet=rs0 '
++ mktemp
+ local LAST_OUT=/tmp/tmp.XtTc0tGc4C
++ mktemp
+ local LAST_ERR=/tmp/tmp.HXEfTYujdw
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl exec psmdb-client-7469665986-gbbng -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@10.101.174.200:27017/admin?ssl=false\&replicaSet=rs0 '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.XtTc0tGc4C
+ cat /tmp/tmp.HXEfTYujdw
+ rm /tmp/tmp.XtTc0tGc4C /tmp/tmp.HXEfTYujdw
+ return 0
+ diff /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1554/e2e-tests/service-per-pod/compare/find.json /tmp/tmp.8ZWelD8e7G/find
++ get_service_ip cluster-ip-rs0-2
++ local service=cluster-ip-rs0-2
++ local server_type=rs0
+++ kubectl_bin get psmdb/cluster-ip -o 'jsonpath={.spec.replsets[].expose.enabled}'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.eswFY7G97n
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.HCfGALRLkZ
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in '$(seq 0 2)'
+++ set +e
+++ kubectl get psmdb/cluster-ip -o 'jsonpath={.spec.replsets[].expose.enabled}'
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.eswFY7G97n
+++ cat /tmp/tmp.HCfGALRLkZ
+++ rm /tmp/tmp.eswFY7G97n /tmp/tmp.HCfGALRLkZ
+++ return 0
++ '[' true '!=' true ']'
++ kubectl_bin get service/cluster-ip-rs0-2 -o 'jsonpath={.spec.type}'
++ grep -q NotFound
+++ kubectl_bin get service/cluster-ip-rs0-2 -o 'jsonpath={.spec.type}'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.PSfZbKfK3I
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.73mbwUter4
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in '$(seq 0 2)'
+++ set +e
+++ kubectl get service/cluster-ip-rs0-2 -o 'jsonpath={.spec.type}'
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.PSfZbKfK3I
+++ cat /tmp/tmp.73mbwUter4
+++ rm /tmp/tmp.PSfZbKfK3I /tmp/tmp.73mbwUter4
+++ return 0
++ service_type=ClusterIP
++ '[' ClusterIP = ClusterIP ']'
++ kubectl_bin get service/cluster-ip-rs0-2 -o 'jsonpath={.spec.clusterIP}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.BWQqhz5SBW
+++ mktemp
++ local LAST_ERR=/tmp/tmp.X1dzLrbn4g
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get service/cluster-ip-rs0-2 -o 'jsonpath={.spec.clusterIP}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.BWQqhz5SBW
++ cat /tmp/tmp.X1dzLrbn4g
++ rm /tmp/tmp.BWQqhz5SBW /tmp/tmp.X1dzLrbn4g
++ return 0
++ return
+ compare_mongo_cmd find myApp:myPass@10.101.164.208 '' :27017
+ local command=find
+ local uri=myApp:myPass@10.101.164.208
+ local postfix=
+ local suffix=:27017
+ local database=myApp
+ local collection=test
+ run_mongo 'use myApp\n db.test.find()' myApp:myPass@10.101.164.208 mongodb :27017
+ local 'command=use myApp\n db.test.find()'
+ local uri=myApp:myPass@10.101.164.208
+ local driver=mongodb
+ local suffix=:27017
+ egrep -v 'I NETWORK|W NETWORK|F NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:'
+ /usr/bin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.V1DsuU1rHG
+++ mktemp
++ local LAST_ERR=/tmp/tmp.6dtk6ymACT
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.V1DsuU1rHG
++ cat /tmp/tmp.6dtk6ymACT
++ rm /tmp/tmp.V1DsuU1rHG /tmp/tmp.6dtk6ymACT
++ return 0
+ local client_container=psmdb-client-7469665986-gbbng
+ local mongo_flag=
+ [[ myApp:myPass@10.101.164.208 == *cfg* ]]
+ replica_set=rs0
+ kubectl_bin exec psmdb-client-7469665986-gbbng -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@10.101.164.208:27017/admin?ssl=false\&replicaSet=rs0 '
++ mktemp
+ local LAST_OUT=/tmp/tmp.LJV1imu328
++ mktemp
+ local LAST_ERR=/tmp/tmp.FPgmL021Z7
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl exec psmdb-client-7469665986-gbbng -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@10.101.164.208:27017/admin?ssl=false\&replicaSet=rs0 '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.LJV1imu328
+ cat /tmp/tmp.FPgmL021Z7
+ rm /tmp/tmp.LJV1imu328 /tmp/tmp.FPgmL021Z7
+ return 0
+ diff /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1554/e2e-tests/service-per-pod/compare/find.json /tmp/tmp.8ZWelD8e7G/find
+ desc 'add service-per-pod lebel and annotation'
+ set +o xtrace


-----------------------------------------------------------------------------------
add service-per-pod lebel and annotation
-----------------------------------------------------------------------------------

+ [[ cluster-ip-rs0 == \n\o\d\e\-\p\o\r\t\-\r\s\0 ]]
+ desc 'delete PSMDB cluster cluster-ip-rs0'
+ set +o xtrace


-----------------------------------------------------------------------------------
delete PSMDB cluster cluster-ip-rs0
-----------------------------------------------------------------------------------

+ kubectl_bin delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1554/e2e-tests/service-per-pod/conf/cluster-ip-rs0.yml
++ mktemp
+ local LAST_OUT=/tmp/tmp.YcxM0AdIiQ
++ mktemp
+ local LAST_ERR=/tmp/tmp.D7dwhNFfbe
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1554/e2e-tests/service-per-pod/conf/cluster-ip-rs0.yml
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.YcxM0AdIiQ
perconaservermongodb.psmdb.percona.com "cluster-ip" deleted
+ cat /tmp/tmp.D7dwhNFfbe
+ rm /tmp/tmp.YcxM0AdIiQ /tmp/tmp.D7dwhNFfbe
+ return 0
+ desc 'check LoadBalancer'
+ set +o xtrace


-----------------------------------------------------------------------------------
check LoadBalancer
-----------------------------------------------------------------------------------

+ check_cr_config local-balancer-rs0
+ local cluster=local-balancer-rs0
+ desc 'create first PSMDB cluster local-balancer-rs0'
+ set +o xtrace


-----------------------------------------------------------------------------------
create first PSMDB cluster local-balancer-rs0
-----------------------------------------------------------------------------------

+ apply_cluster /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1554/e2e-tests/service-per-pod/conf/local-balancer-rs0.yml
+ '[' -z '' ']'
+ cat_config /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1554/e2e-tests/service-per-pod/conf/local-balancer-rs0.yml
+ kubectl_bin apply -f -
+ cat /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1554/e2e-tests/service-per-pod/conf/local-balancer-rs0.yml
+ yq eval '(.spec | select(.image == null)).image = "perconalab/percona-server-mongodb-operator:main-mongod7.0"'
++ mktemp
+ yq eval '(.spec | select(has("initImage"))).initImage = "perconalab/percona-server-mongodb-operator:PR-1554-4f44f8bb"'
+ yq eval '.spec.upgradeOptions.apply="Never"'
+ yq eval '(.spec | select(has("pmm"))).pmm.image = "perconalab/pmm-client:dev-latest"'
+ local LAST_OUT=/tmp/tmp.HuidbzH60S
++ mktemp
+ local LAST_ERR=/tmp/tmp.TM3UFqmYfx
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl apply -f -
+ yq eval '(.spec | select(has("backup"))).backup.image = "perconalab/percona-server-mongodb-operator:main-backup"'
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.HuidbzH60S
perconaservermongodb.psmdb.percona.com/local-balancer created
+ cat /tmp/tmp.TM3UFqmYfx
+ rm /tmp/tmp.HuidbzH60S /tmp/tmp.TM3UFqmYfx
+ return 0
+ desc 'check if all 3 Pods started'
+ set +o xtrace


-----------------------------------------------------------------------------------
check if all 3 Pods started
-----------------------------------------------------------------------------------

+ wait_for_running local-balancer-rs0 3 false
+ local name=local-balancer-rs0
+ let last_pod=2
+ local check_cluster_readyness=false
+ set_debug
+ [[ 1 == 1 ]]
+ set -o xtrace
+ local rs_name=rs0
+ local cluster_name=local-balancer
++ seq 0 2
+ for i in '$(seq 0 $last_pod)'
+ [[ 0 -eq 2 ]]
+ wait_pod local-balancer-rs0-0
+ local pod=local-balancer-rs0-0
+ set +o xtrace
waiting for pod/local-balancer-rs0-0 to be ready.......OK
+ for i in '$(seq 0 $last_pod)'
+ [[ 1 -eq 2 ]]
+ wait_pod local-balancer-rs0-1
+ local pod=local-balancer-rs0-1
+ set +o xtrace
waiting for pod/local-balancer-rs0-1 to be ready......OK
+ for i in '$(seq 0 $last_pod)'
+ [[ 2 -eq 2 ]]
++ kubectl_bin get psmdb local-balancer -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].arbiter.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.q5jOMllHIZ
+++ mktemp
++ local LAST_ERR=/tmp/tmp.3LZ1Q3ShdF
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb local-balancer -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].arbiter.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.q5jOMllHIZ
++ cat /tmp/tmp.3LZ1Q3ShdF
++ rm /tmp/tmp.q5jOMllHIZ /tmp/tmp.3LZ1Q3ShdF
++ return 0
+ [[ '' == \t\r\u\e ]]
+ wait_pod local-balancer-rs0-2
+ local pod=local-balancer-rs0-2
+ set +o xtrace
waiting for pod/local-balancer-rs0-2 to be ready......OK
++ kubectl_bin get psmdb local-balancer -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].non_voting.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.HAOM3wgUIK
+++ mktemp
++ local LAST_ERR=/tmp/tmp.40j7tEPyJH
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb local-balancer -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].non_voting.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.HAOM3wgUIK
++ cat /tmp/tmp.40j7tEPyJH
++ rm /tmp/tmp.HAOM3wgUIK /tmp/tmp.40j7tEPyJH
++ return 0
+ [[ '' == \t\r\u\e ]]
+ sleep 10
+ [[ false == \t\r\u\e ]]
+ desc 'check if service and statefulset created with expected config'
+ set +o xtrace


-----------------------------------------------------------------------------------
check if service and statefulset created with expected config
-----------------------------------------------------------------------------------

+ compare_kubectl statefulset/local-balancer-rs0
+ local resource=statefulset/local-balancer-rs0
+ local postfix=
+ local expected_result=/mnt/jenkins/workspace/cloud-psmdb-operator_PR-1554/e2e-tests/service-per-pod/compare/statefulset_local-balancer-rs0.yml
+ local new_result=/tmp/tmp.8ZWelD8e7G/statefulset_local-balancer-rs0.yml
+ '[' -n '' -a -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1554/e2e-tests/service-per-pod/compare/statefulset_local-balancer-rs0-oc.yml ']'
+ kubectl_bin get -o yaml statefulset/local-balancer-rs0
+ yq eval '
			del(.metadata.ownerReferences[].apiVersion) |
			del(.metadata.managedFields) |
			del(.. | select(has("creationTimestamp")).creationTimestamp) |
			del(.. | select(has("namespace")).namespace) |
			del(.. | select(has("uid")).uid) |
			del(.metadata.resourceVersion) |
			del(.spec.template.spec.containers[].env[] | select(.name == "NAMESPACE")) |
			del(.metadata.selfLink) |
			del(.metadata.annotations."cloud.google.com/neg") |
			del(.. | select(has("image")).image) |
			del(.. | select(has("clusterIP")).clusterIP) |
			del(.. | select(has("clusterIPs")).clusterIPs) |
			del(.. | select(has("dataSource")).dataSource) |
			del(.. | select(has("procMount")).procMount) |
			del(.. | select(has("storageClassName")).storageClassName) |
			del(.. | select(has("finalizers")).finalizers) |
			del(.. | select(has("kubernetes.io/pvc-protection"))."kubernetes.io/pvc-protection") |
			del(.. | select(has("volumeName")).volumeName) |
			del(.. | select(has("volume.beta.kubernetes.io/storage-provisioner"))."volume.beta.kubernetes.io/storage-provisioner") |
			del(.. | select(has("volume.kubernetes.io/storage-provisioner"))."volume.kubernetes.io/storage-provisioner") |
			del(.spec.volumeMode) |
			del(.. | select(has("volume.kubernetes.io/selected-node"))."volume.kubernetes.io/selected-node") |
			del(.. | select(has("percona.com/last-config-hash"))."percona.com/last-config-hash") |
			del(.. | select(has("percona.com/configuration-hash"))."percona.com/configuration-hash") |
			del(.. | select(has("percona.com/ssl-hash"))."percona.com/ssl-hash") |
			del(.. | select(has("percona.com/ssl-internal-hash"))."percona.com/ssl-internal-hash") |
			del(.spec.volumeClaimTemplates[].spec.volumeMode | select(. == "Filesystem")) |
			del(.. | select(has("healthCheckNodePort")).healthCheckNodePort) |
			del(.. | select(has("nodePort")).nodePort) |
			del(.status) |
			(.. | select(tag == "!!str")) |= sub("service-per-pod-27980", "NAME_SPACE") |
			del(.spec.volumeClaimTemplates[].apiVersion) |
			del(.spec.volumeClaimTemplates[].kind) |
			del(.spec.ipFamilies) |
			del(.spec.ipFamilyPolicy) |
			(.. | select(. == "extensions/v1beta1")) = "apps/v1" |
			(.. | select(. == "batch/v1beta1")) = "batch/v1" ' -
++ mktemp
+ local LAST_OUT=/tmp/tmp.Rqxb9fOtEc
++ mktemp
+ local LAST_ERR=/tmp/tmp.Ot7llSCGxc
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl get -o yaml statefulset/local-balancer-rs0
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.Rqxb9fOtEc
+ cat /tmp/tmp.Ot7llSCGxc
+ rm /tmp/tmp.Rqxb9fOtEc /tmp/tmp.Ot7llSCGxc
+ return 0
+ yq -i eval 'del(.spec.persistentVolumeClaimRetentionPolicy)' /tmp/tmp.8ZWelD8e7G/statefulset_local-balancer-rs0.yml
+ version_gt 1.22
++ echo '1.26 >= 1.22'
++ bc -l
+ '[' 1 -eq 1 ']'
+ return 0
+ yq -i eval 'del(.spec.internalTrafficPolicy)' /tmp/tmp.8ZWelD8e7G/statefulset_local-balancer-rs0.yml
+ yq -i eval 'del(.spec.allocateLoadBalancerNodePorts)' /tmp/tmp.8ZWelD8e7G/statefulset_local-balancer-rs0.yml
+ [[ /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1554/e2e-tests/service-per-pod/compare/statefulset_local-balancer-rs0.yml == */cronjob* ]]
+ diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1554/e2e-tests/service-per-pod/compare/statefulset_local-balancer-rs0.yml /tmp/tmp.8ZWelD8e7G/statefulset_local-balancer-rs0.yml
+ compare_kubectl service/local-balancer-rs0-0
+ local resource=service/local-balancer-rs0-0
+ local postfix=
+ local expected_result=/mnt/jenkins/workspace/cloud-psmdb-operator_PR-1554/e2e-tests/service-per-pod/compare/service_local-balancer-rs0-0.yml
+ local new_result=/tmp/tmp.8ZWelD8e7G/service_local-balancer-rs0-0.yml
+ '[' -n '' -a -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1554/e2e-tests/service-per-pod/compare/service_local-balancer-rs0-0-oc.yml ']'
+ kubectl_bin get -o yaml service/local-balancer-rs0-0
+ yq eval '
			del(.metadata.ownerReferences[].apiVersion) |
			del(.metadata.managedFields) |
			del(.. | select(has("creationTimestamp")).creationTimestamp) |
			del(.. | select(has("namespace")).namespace) |
			del(.. | select(has("uid")).uid) |
			del(.metadata.resourceVersion) |
			del(.spec.template.spec.containers[].env[] | select(.name == "NAMESPACE")) |
			del(.metadata.selfLink) |
			del(.metadata.annotations."cloud.google.com/neg") |
			del(.. | select(has("image")).image) |
			del(.. | select(has("clusterIP")).clusterIP) |
			del(.. | select(has("clusterIPs")).clusterIPs) |
			del(.. | select(has("dataSource")).dataSource) |
			del(.. | select(has("procMount")).procMount) |
			del(.. | select(has("storageClassName")).storageClassName) |
			del(.. | select(has("finalizers")).finalizers) |
			del(.. | select(has("kubernetes.io/pvc-protection"))."kubernetes.io/pvc-protection") |
			del(.. | select(has("volumeName")).volumeName) |
			del(.. | select(has("volume.beta.kubernetes.io/storage-provisioner"))."volume.beta.kubernetes.io/storage-provisioner") |
			del(.. | select(has("volume.kubernetes.io/storage-provisioner"))."volume.kubernetes.io/storage-provisioner") |
			del(.spec.volumeMode) |
			del(.. | select(has("volume.kubernetes.io/selected-node"))."volume.kubernetes.io/selected-node") |
			del(.. | select(has("percona.com/last-config-hash"))."percona.com/last-config-hash") |
			del(.. | select(has("percona.com/configuration-hash"))."percona.com/configuration-hash") |
			del(.. | select(has("percona.com/ssl-hash"))."percona.com/ssl-hash") |
			del(.. | select(has("percona.com/ssl-internal-hash"))."percona.com/ssl-internal-hash") |
			del(.spec.volumeClaimTemplates[].spec.volumeMode | select(. == "Filesystem")) |
			del(.. | select(has("healthCheckNodePort")).healthCheckNodePort) |
			del(.. | select(has("nodePort")).nodePort) |
			del(.status) |
			(.. | select(tag == "!!str")) |= sub("service-per-pod-27980", "NAME_SPACE") |
			del(.spec.volumeClaimTemplates[].apiVersion) |
			del(.spec.volumeClaimTemplates[].kind) |
			del(.spec.ipFamilies) |
			del(.spec.ipFamilyPolicy) |
			(.. | select(. == "extensions/v1beta1")) = "apps/v1" |
++ mktemp
			(.. | select(. == "batch/v1beta1")) = "batch/v1" ' -
+ local LAST_OUT=/tmp/tmp.yKmdZ1VfmO
++ mktemp
+ local LAST_ERR=/tmp/tmp.wG27hmvYQZ
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl get -o yaml service/local-balancer-rs0-0
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.yKmdZ1VfmO
+ cat /tmp/tmp.wG27hmvYQZ
+ rm /tmp/tmp.yKmdZ1VfmO /tmp/tmp.wG27hmvYQZ
+ return 0
+ yq -i eval 'del(.spec.persistentVolumeClaimRetentionPolicy)' /tmp/tmp.8ZWelD8e7G/service_local-balancer-rs0-0.yml
+ version_gt 1.22
++ echo '1.26 >= 1.22'
++ bc -l
+ '[' 1 -eq 1 ']'
+ return 0
+ yq -i eval 'del(.spec.internalTrafficPolicy)' /tmp/tmp.8ZWelD8e7G/service_local-balancer-rs0-0.yml
+ yq -i eval 'del(.spec.allocateLoadBalancerNodePorts)' /tmp/tmp.8ZWelD8e7G/service_local-balancer-rs0-0.yml
+ [[ /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1554/e2e-tests/service-per-pod/compare/service_local-balancer-rs0-0.yml == */cronjob* ]]
+ diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1554/e2e-tests/service-per-pod/compare/service_local-balancer-rs0-0.yml /tmp/tmp.8ZWelD8e7G/service_local-balancer-rs0-0.yml
++ get_service_ip local-balancer-rs0-0
++ local service=local-balancer-rs0-0
++ local server_type=rs0
+++ kubectl_bin get psmdb/local-balancer -o 'jsonpath={.spec.replsets[].expose.enabled}'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.5tSh226BQZ
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.bfpxjTaODa
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in '$(seq 0 2)'
+++ set +e
+++ kubectl get psmdb/local-balancer -o 'jsonpath={.spec.replsets[].expose.enabled}'
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.5tSh226BQZ
+++ cat /tmp/tmp.bfpxjTaODa
+++ rm /tmp/tmp.5tSh226BQZ /tmp/tmp.bfpxjTaODa
+++ return 0
++ '[' true '!=' true ']'
++ kubectl_bin get service/local-balancer-rs0-0 -o 'jsonpath={.spec.type}'
++ grep -q NotFound
+++ kubectl_bin get service/local-balancer-rs0-0 -o 'jsonpath={.spec.type}'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.dYMdnxP1oO
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.sB2Y5Yqxwy
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in '$(seq 0 2)'
+++ set +e
+++ kubectl get service/local-balancer-rs0-0 -o 'jsonpath={.spec.type}'
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.dYMdnxP1oO
+++ cat /tmp/tmp.sB2Y5Yqxwy
+++ rm /tmp/tmp.dYMdnxP1oO /tmp/tmp.sB2Y5Yqxwy
+++ return 0
++ service_type=LoadBalancer
++ '[' LoadBalancer = ClusterIP ']'
++ '[' LoadBalancer = NodePort ']'
++ kubectl_bin get service/local-balancer-rs0-0 -o 'jsonpath={.status.loadBalancer.ingress[]}'
++ egrep -q 'hostname|ip'
++ kubectl_bin get service/local-balancer-rs0-0 -o 'jsonpath={.status.loadBalancer.ingress[].ip}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.c1rHhFYp49
+++ mktemp
++ local LAST_ERR=/tmp/tmp.yYHFRSTziC
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get service/local-balancer-rs0-0 -o 'jsonpath={.status.loadBalancer.ingress[].ip}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.c1rHhFYp49
++ cat /tmp/tmp.yYHFRSTziC
++ rm /tmp/tmp.c1rHhFYp49 /tmp/tmp.yYHFRSTziC
++ return 0
++ kubectl_bin get service/local-balancer-rs0-0 -o 'jsonpath={.status.loadBalancer.ingress[].hostname}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.YXnFhzqpjB
+++ mktemp
++ local LAST_ERR=/tmp/tmp.REHtWy5IKB
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get service/local-balancer-rs0-0 -o 'jsonpath={.status.loadBalancer.ingress[].hostname}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.YXnFhzqpjB
++ cat /tmp/tmp.REHtWy5IKB
++ rm /tmp/tmp.YXnFhzqpjB /tmp/tmp.REHtWy5IKB
++ return 0
++ get_service_ip local-balancer-rs0-1
++ local service=local-balancer-rs0-1
++ local server_type=rs0
+++ kubectl_bin get psmdb/local-balancer -o 'jsonpath={.spec.replsets[].expose.enabled}'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.dyMIopqdkg
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.wmHZCWKdKK
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in '$(seq 0 2)'
+++ set +e
+++ kubectl get psmdb/local-balancer -o 'jsonpath={.spec.replsets[].expose.enabled}'
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.dyMIopqdkg
+++ cat /tmp/tmp.wmHZCWKdKK
+++ rm /tmp/tmp.dyMIopqdkg /tmp/tmp.wmHZCWKdKK
+++ return 0
++ '[' true '!=' true ']'
++ grep -q NotFound
++ kubectl_bin get service/local-balancer-rs0-1 -o 'jsonpath={.spec.type}'
+++ kubectl_bin get service/local-balancer-rs0-1 -o 'jsonpath={.spec.type}'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.cqH1GVmtbs
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.WGPijhDCcp
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in '$(seq 0 2)'
+++ set +e
+++ kubectl get service/local-balancer-rs0-1 -o 'jsonpath={.spec.type}'
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.cqH1GVmtbs
+++ cat /tmp/tmp.WGPijhDCcp
+++ rm /tmp/tmp.cqH1GVmtbs /tmp/tmp.WGPijhDCcp
+++ return 0
++ service_type=LoadBalancer
++ '[' LoadBalancer = ClusterIP ']'
++ '[' LoadBalancer = NodePort ']'
++ kubectl_bin get service/local-balancer-rs0-1 -o 'jsonpath={.status.loadBalancer.ingress[]}'
++ egrep -q 'hostname|ip'
++ kubectl_bin get service/local-balancer-rs0-1 -o 'jsonpath={.status.loadBalancer.ingress[].ip}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.4bMNvKTBpx
+++ mktemp
++ local LAST_ERR=/tmp/tmp.lHJxbjMxjJ
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get service/local-balancer-rs0-1 -o 'jsonpath={.status.loadBalancer.ingress[].ip}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.4bMNvKTBpx
++ cat /tmp/tmp.lHJxbjMxjJ
++ rm /tmp/tmp.4bMNvKTBpx /tmp/tmp.lHJxbjMxjJ
++ return 0
++ kubectl_bin get service/local-balancer-rs0-1 -o 'jsonpath={.status.loadBalancer.ingress[].hostname}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.ntgJAoIqSO
+++ mktemp
++ local LAST_ERR=/tmp/tmp.IOKNAv50uo
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get service/local-balancer-rs0-1 -o 'jsonpath={.status.loadBalancer.ingress[].hostname}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.ntgJAoIqSO
++ cat /tmp/tmp.IOKNAv50uo
++ rm /tmp/tmp.ntgJAoIqSO /tmp/tmp.IOKNAv50uo
++ return 0
++ get_service_ip local-balancer-rs0-2
++ local service=local-balancer-rs0-2
++ local server_type=rs0
+++ kubectl_bin get psmdb/local-balancer -o 'jsonpath={.spec.replsets[].expose.enabled}'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.Q9s3dJfxWx
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.zZdyCqVwZ5
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in '$(seq 0 2)'
+++ set +e
+++ kubectl get psmdb/local-balancer -o 'jsonpath={.spec.replsets[].expose.enabled}'
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.Q9s3dJfxWx
+++ cat /tmp/tmp.zZdyCqVwZ5
+++ rm /tmp/tmp.Q9s3dJfxWx /tmp/tmp.zZdyCqVwZ5
+++ return 0
++ '[' true '!=' true ']'
++ kubectl_bin get service/local-balancer-rs0-2 -o 'jsonpath={.spec.type}'
++ grep -q NotFound
+++ kubectl_bin get service/local-balancer-rs0-2 -o 'jsonpath={.spec.type}'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.EY4kQBx0mA
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.IKgbBEvZlE
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in '$(seq 0 2)'
+++ set +e
+++ kubectl get service/local-balancer-rs0-2 -o 'jsonpath={.spec.type}'
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.EY4kQBx0mA
+++ cat /tmp/tmp.IKgbBEvZlE
+++ rm /tmp/tmp.EY4kQBx0mA /tmp/tmp.IKgbBEvZlE
+++ return 0
++ service_type=LoadBalancer
++ '[' LoadBalancer = ClusterIP ']'
++ '[' LoadBalancer = NodePort ']'
++ kubectl_bin get service/local-balancer-rs0-2 -o 'jsonpath={.status.loadBalancer.ingress[]}'
++ egrep -q 'hostname|ip'
++ sleep 1
++ kubectl_bin get service/local-balancer-rs0-2 -o 'jsonpath={.status.loadBalancer.ingress[]}'
++ egrep -q 'hostname|ip'
++ sleep 1
++ kubectl_bin get service/local-balancer-rs0-2 -o 'jsonpath={.status.loadBalancer.ingress[]}'
++ egrep -q 'hostname|ip'
++ sleep 1
++ kubectl_bin get service/local-balancer-rs0-2 -o 'jsonpath={.status.loadBalancer.ingress[]}'
++ egrep -q 'hostname|ip'
++ sleep 1
++ kubectl_bin get service/local-balancer-rs0-2 -o 'jsonpath={.status.loadBalancer.ingress[]}'
++ egrep -q 'hostname|ip'
++ sleep 1
++ kubectl_bin get service/local-balancer-rs0-2 -o 'jsonpath={.status.loadBalancer.ingress[]}'
++ egrep -q 'hostname|ip'
++ sleep 1
++ kubectl_bin get service/local-balancer-rs0-2 -o 'jsonpath={.status.loadBalancer.ingress[]}'
++ egrep -q 'hostname|ip'
++ sleep 1
++ kubectl_bin get service/local-balancer-rs0-2 -o 'jsonpath={.status.loadBalancer.ingress[]}'
++ egrep -q 'hostname|ip'
++ sleep 1
++ egrep -q 'hostname|ip'
++ kubectl_bin get service/local-balancer-rs0-2 -o 'jsonpath={.status.loadBalancer.ingress[]}'
++ sleep 1
++ kubectl_bin get service/local-balancer-rs0-2 -o 'jsonpath={.status.loadBalancer.ingress[]}'
++ egrep -q 'hostname|ip'
++ sleep 1
++ kubectl_bin get service/local-balancer-rs0-2 -o 'jsonpath={.status.loadBalancer.ingress[]}'
++ egrep -q 'hostname|ip'
++ sleep 1
++ kubectl_bin get service/local-balancer-rs0-2 -o 'jsonpath={.status.loadBalancer.ingress[]}'
++ egrep -q 'hostname|ip'
++ sleep 1
++ kubectl_bin get service/local-balancer-rs0-2 -o 'jsonpath={.status.loadBalancer.ingress[]}'
++ egrep -q 'hostname|ip'
++ kubectl_bin get service/local-balancer-rs0-2 -o 'jsonpath={.status.loadBalancer.ingress[].ip}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.oFPn5qhhvA
+++ mktemp
++ local LAST_ERR=/tmp/tmp.sjFS45wfaa
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get service/local-balancer-rs0-2 -o 'jsonpath={.status.loadBalancer.ingress[].ip}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.oFPn5qhhvA
++ cat /tmp/tmp.sjFS45wfaa
++ rm /tmp/tmp.oFPn5qhhvA /tmp/tmp.sjFS45wfaa
++ return 0
++ kubectl_bin get service/local-balancer-rs0-2 -o 'jsonpath={.status.loadBalancer.ingress[].hostname}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.tOf5GwyLQ1
+++ mktemp
++ local LAST_ERR=/tmp/tmp.KawlbKqdBJ
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get service/local-balancer-rs0-2 -o 'jsonpath={.status.loadBalancer.ingress[].hostname}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.tOf5GwyLQ1
++ cat /tmp/tmp.KawlbKqdBJ
++ rm /tmp/tmp.tOf5GwyLQ1 /tmp/tmp.KawlbKqdBJ
++ return 0
+ local URI=34.29.52.68,35.202.14.178,34.41.193.65
+ sleep 30
+ desc 'create user myApp'
+ set +o xtrace


-----------------------------------------------------------------------------------
create user myApp
-----------------------------------------------------------------------------------

+ run_mongo 'db.createUser({user:"myApp",pwd:"myPass",roles:[{db:"myApp",role:"readWrite"}]})' userAdmin:userAdmin123456@34.29.52.68,35.202.14.178,34.41.193.65 mongodb :27017
+ local 'command=db.createUser({user:"myApp",pwd:"myPass",roles:[{db:"myApp",role:"readWrite"}]})'
+ local uri=userAdmin:userAdmin123456@34.29.52.68,35.202.14.178,34.41.193.65
+ local driver=mongodb
+ local suffix=:27017
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.50LTFVe5O3
+++ mktemp
++ local LAST_ERR=/tmp/tmp.Xq79hq3D8v
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.50LTFVe5O3
++ cat /tmp/tmp.Xq79hq3D8v
++ rm /tmp/tmp.50LTFVe5O3 /tmp/tmp.Xq79hq3D8v
++ return 0
+ local client_container=psmdb-client-7469665986-gbbng
+ local mongo_flag=
+ [[ userAdmin:userAdmin123456@34.29.52.68,35.202.14.178,34.41.193.65 == *cfg* ]]
+ replica_set=rs0
+ kubectl_bin exec psmdb-client-7469665986-gbbng -- bash -c 'printf '\''db.createUser({user:"myApp",pwd:"myPass",roles:[{db:"myApp",role:"readWrite"}]})\n'\'' | mongo mongodb://userAdmin:userAdmin123456@34.29.52.68,35.202.14.178,34.41.193.65:27017/admin?ssl=false\&replicaSet=rs0 '
++ mktemp
+ local LAST_OUT=/tmp/tmp.6YTXX8zWzy
++ mktemp
+ local LAST_ERR=/tmp/tmp.KwCa1Bd7lK
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl exec psmdb-client-7469665986-gbbng -- bash -c 'printf '\''db.createUser({user:"myApp",pwd:"myPass",roles:[{db:"myApp",role:"readWrite"}]})\n'\'' | mongo mongodb://userAdmin:userAdmin123456@34.29.52.68,35.202.14.178,34.41.193.65:27017/admin?ssl=false\&replicaSet=rs0 '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.6YTXX8zWzy
Percona Server for MongoDB shell version v4.4.29-28
connecting to: mongodb://34.29.52.68:27017,35.202.14.178:27017,34.41.193.65:27017/admin?compressors=disabled&gssapiServiceName=mongodb&replicaSet=rs0&ssl=false
Implicit session: session { "id" : UUID("5c1d353c-f91a-4ae7-91ea-f9b6b35bcdc6") }
Percona Server for MongoDB server version: v7.0.11-6
WARNING: shell and server versions do not match
Successfully added user: {
	"user" : "myApp",
	"roles" : [
		{
			"db" : "myApp",
			"role" : "readWrite"
		}
	]
}
bye
+ cat /tmp/tmp.KwCa1Bd7lK
+ rm /tmp/tmp.6YTXX8zWzy /tmp/tmp.KwCa1Bd7lK
+ return 0
+ sleep 10
+ desc 'write data, read from all'
+ set +o xtrace


-----------------------------------------------------------------------------------
write data, read from all
-----------------------------------------------------------------------------------

+ run_mongo 'use myApp\n db.test.insert({ x: 100500 })' myApp:myPass@34.29.52.68,35.202.14.178,34.41.193.65 mongodb :27017
+ local 'command=use myApp\n db.test.insert({ x: 100500 })'
+ local uri=myApp:myPass@34.29.52.68,35.202.14.178,34.41.193.65
+ local driver=mongodb
+ local suffix=:27017
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.sassPDctGJ
+++ mktemp
++ local LAST_ERR=/tmp/tmp.A0YvTQqou0
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.sassPDctGJ
++ cat /tmp/tmp.A0YvTQqou0
++ rm /tmp/tmp.sassPDctGJ /tmp/tmp.A0YvTQqou0
++ return 0
+ local client_container=psmdb-client-7469665986-gbbng
+ local mongo_flag=
+ [[ myApp:myPass@34.29.52.68,35.202.14.178,34.41.193.65 == *cfg* ]]
+ replica_set=rs0
+ kubectl_bin exec psmdb-client-7469665986-gbbng -- bash -c 'printf '\''use myApp\n db.test.insert({ x: 100500 })\n'\'' | mongo mongodb://myApp:myPass@34.29.52.68,35.202.14.178,34.41.193.65:27017/admin?ssl=false\&replicaSet=rs0 '
++ mktemp
+ local LAST_OUT=/tmp/tmp.eZy5UeCTfQ
++ mktemp
+ local LAST_ERR=/tmp/tmp.eFxfrhXEOc
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl exec psmdb-client-7469665986-gbbng -- bash -c 'printf '\''use myApp\n db.test.insert({ x: 100500 })\n'\'' | mongo mongodb://myApp:myPass@34.29.52.68,35.202.14.178,34.41.193.65:27017/admin?ssl=false\&replicaSet=rs0 '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.eZy5UeCTfQ
Percona Server for MongoDB shell version v4.4.29-28
connecting to: mongodb://34.29.52.68:27017,35.202.14.178:27017,34.41.193.65:27017/admin?compressors=disabled&gssapiServiceName=mongodb&replicaSet=rs0&ssl=false
Implicit session: session { "id" : UUID("f771655d-eae3-40d2-8f33-4e83e260ca8e") }
Percona Server for MongoDB server version: v7.0.11-6
WARNING: shell and server versions do not match
switched to db myApp
WriteResult({ "nInserted" : 1 })
bye
+ cat /tmp/tmp.eFxfrhXEOc
+ rm /tmp/tmp.eZy5UeCTfQ /tmp/tmp.eFxfrhXEOc
+ return 0
+ sleep 10
++ get_service_ip local-balancer-rs0-0
++ local service=local-balancer-rs0-0
++ local server_type=rs0
+++ kubectl_bin get psmdb/local-balancer -o 'jsonpath={.spec.replsets[].expose.enabled}'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.hrpGxsoWOp
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.iP48AQzOFI
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in '$(seq 0 2)'
+++ set +e
+++ kubectl get psmdb/local-balancer -o 'jsonpath={.spec.replsets[].expose.enabled}'
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.hrpGxsoWOp
+++ cat /tmp/tmp.iP48AQzOFI
+++ rm /tmp/tmp.hrpGxsoWOp /tmp/tmp.iP48AQzOFI
+++ return 0
++ '[' true '!=' true ']'
++ kubectl_bin get service/local-balancer-rs0-0 -o 'jsonpath={.spec.type}'
++ grep -q NotFound
+++ kubectl_bin get service/local-balancer-rs0-0 -o 'jsonpath={.spec.type}'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.bejP6MPZna
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.X46MH6C9KS
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in '$(seq 0 2)'
+++ set +e
+++ kubectl get service/local-balancer-rs0-0 -o 'jsonpath={.spec.type}'
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.bejP6MPZna
+++ cat /tmp/tmp.X46MH6C9KS
+++ rm /tmp/tmp.bejP6MPZna /tmp/tmp.X46MH6C9KS
+++ return 0
++ service_type=LoadBalancer
++ '[' LoadBalancer = ClusterIP ']'
++ '[' LoadBalancer = NodePort ']'
++ kubectl_bin get service/local-balancer-rs0-0 -o 'jsonpath={.status.loadBalancer.ingress[]}'
++ egrep -q 'hostname|ip'
++ kubectl_bin get service/local-balancer-rs0-0 -o 'jsonpath={.status.loadBalancer.ingress[].ip}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.vBH3LtscQ8
+++ mktemp
++ local LAST_ERR=/tmp/tmp.cCciPPpZdP
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get service/local-balancer-rs0-0 -o 'jsonpath={.status.loadBalancer.ingress[].ip}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.vBH3LtscQ8
++ cat /tmp/tmp.cCciPPpZdP
++ rm /tmp/tmp.vBH3LtscQ8 /tmp/tmp.cCciPPpZdP
++ return 0
++ kubectl_bin get service/local-balancer-rs0-0 -o 'jsonpath={.status.loadBalancer.ingress[].hostname}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.Dbup9aLdVm
+++ mktemp
++ local LAST_ERR=/tmp/tmp.fBDdfGGIjj
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get service/local-balancer-rs0-0 -o 'jsonpath={.status.loadBalancer.ingress[].hostname}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.Dbup9aLdVm
++ cat /tmp/tmp.fBDdfGGIjj
++ rm /tmp/tmp.Dbup9aLdVm /tmp/tmp.fBDdfGGIjj
++ return 0
+ compare_mongo_cmd find myApp:myPass@34.29.52.68 '' :27017
+ local command=find
+ local uri=myApp:myPass@34.29.52.68
+ local postfix=
+ local suffix=:27017
+ local database=myApp
+ local collection=test
+ egrep -v 'I NETWORK|W NETWORK|F NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:'
+ /usr/bin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
+ run_mongo 'use myApp\n db.test.find()' myApp:myPass@34.29.52.68 mongodb :27017
+ local 'command=use myApp\n db.test.find()'
+ local uri=myApp:myPass@34.29.52.68
+ local driver=mongodb
+ local suffix=:27017
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.i4ZGMptHBU
+++ mktemp
++ local LAST_ERR=/tmp/tmp.JIyzCXh3ks
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.i4ZGMptHBU
++ cat /tmp/tmp.JIyzCXh3ks
++ rm /tmp/tmp.i4ZGMptHBU /tmp/tmp.JIyzCXh3ks
++ return 0
+ local client_container=psmdb-client-7469665986-gbbng
+ local mongo_flag=
+ [[ myApp:myPass@34.29.52.68 == *cfg* ]]
+ replica_set=rs0
+ kubectl_bin exec psmdb-client-7469665986-gbbng -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@34.29.52.68:27017/admin?ssl=false\&replicaSet=rs0 '
++ mktemp
+ local LAST_OUT=/tmp/tmp.6MpzORk1Y3
++ mktemp
+ local LAST_ERR=/tmp/tmp.KxLfyDEbVo
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl exec psmdb-client-7469665986-gbbng -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@34.29.52.68:27017/admin?ssl=false\&replicaSet=rs0 '
+ exit_status=1
+ set -e
+ '[' 1 '!=' 0 -a -n 1 ']'
+ cat /tmp/tmp.6MpzORk1Y3
+ cat /tmp/tmp.KxLfyDEbVo
command terminated with exit code 1
+ sleep 0
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl exec psmdb-client-7469665986-gbbng -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@34.29.52.68:27017/admin?ssl=false\&replicaSet=rs0 '
/mnt/jenkins/workspace/cloud-psmdb-operator_PR-1554/e2e-tests/service-per-pod/../functions: line 1103: 18711 Terminated              kubectl "$@" > "$LAST_OUT" 2> "$LAST_ERR"
+ exit_status=143
+ set -e
+ '[' 143 '!=' 0 -a -n 1 ']'
+ cat /tmp/tmp.6MpzORk1Y3
+ cat /tmp/tmp.KxLfyDEbVo
+ sleep 4
+ diff /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1554/e2e-tests/service-per-pod/compare/find.json /tmp/tmp.8ZWelD8e7G/find
1,3d0
< switched to db myApp
< { "_id" : , "x" : 100500 }
< bye