Log: /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1536/e2e-tests/logs/cross-site-sharded.log
WARNING: version difference between client (1.30) and server (1.26) exceeds the supported minor version skew of +/-1
WARNING: version difference between client (1.30) and server (1.26) exceeds the supported minor version skew of +/-1
WARNING: version difference between client (1.30) and server (1.26) exceeds the supported minor version skew of +/-1
++ get_mongod_ver_from_image perconalab/percona-server-mongodb-operator:main-mongod7.0
++ local image=perconalab/percona-server-mongodb-operator:main-mongod7.0
+++ /usr/bin/sed -r 's/^.*db version v(([0-9]+\.){2}[0-9]+-[0-9]+).*$/\1/g'
+++ run_simple_cli_inside_image perconalab/percona-server-mongodb-operator:main-mongod7.0 'mongod --version'
+++ local image=perconalab/percona-server-mongodb-operator:main-mongod7.0
+++ local 'cli=mongod --version'
+++ local pod_name=15318
+++ kubectl_bin -n default run 15318 --image=perconalab/percona-server-mongodb-operator:main-mongod7.0 --restart=Never --command -- sleep infinity
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.suz025HJN4
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.pJ1zsomjG4
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in '$(seq 0 2)'
+++ set +e
+++ kubectl -n default run 15318 --image=perconalab/percona-server-mongodb-operator:main-mongod7.0 --restart=Never --command -- sleep infinity
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.suz025HJN4
+++ cat /tmp/tmp.pJ1zsomjG4
+++ rm /tmp/tmp.suz025HJN4 /tmp/tmp.pJ1zsomjG4
+++ return 0
+++ kubectl_bin -n default wait --for=condition=Ready pod/15318
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.tsjC8OzvkW
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.bEwVvPDfOg
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in '$(seq 0 2)'
+++ set +e
+++ kubectl -n default wait --for=condition=Ready pod/15318
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.tsjC8OzvkW
+++ cat /tmp/tmp.bEwVvPDfOg
+++ rm /tmp/tmp.tsjC8OzvkW /tmp/tmp.bEwVvPDfOg
+++ return 0
++++ kubectl_bin -n default exec 15318 -- mongod --version
+++++ mktemp
++++ local LAST_OUT=/tmp/tmp.RQgthSL48r
+++++ mktemp
++++ local LAST_ERR=/tmp/tmp.U3aii8Ppry
++++ local exit_status=0
++++ local timeout=4
+++++ seq 0 2
++++ for i in '$(seq 0 2)'
++++ set +e
++++ kubectl -n default exec 15318 -- mongod --version
++++ exit_status=0
++++ set -e
++++ '[' 0 '!=' 0 -a -n 1 ']'
++++ break
++++ cat /tmp/tmp.RQgthSL48r
++++ cat /tmp/tmp.U3aii8Ppry
++++ rm /tmp/tmp.RQgthSL48r /tmp/tmp.U3aii8Ppry
++++ return 0
+++ local 'output=db version v7.0.8-5
Build Info: {
    "version": "7.0.8-5",
    "gitVersion": "197ff4d49589e8b5a444b7ebc6f4d33b6f00105e",
    "openSSLVersion": "OpenSSL 1.1.1k  FIPS 25 Mar 2021",
    "modules": [],
    "proFeatures": [],
    "allocator": "tcmalloc",
    "environment": {
        "distarch": "x86_64",
        "target_arch": "x86_64"
    }
}'
+++ kubectl_bin -n default delete pod/15318 --grace-period=0 --force
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.BlQDAEvQhq
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.WAC00N0fhh
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in '$(seq 0 2)'
+++ set +e
+++ kubectl -n default delete pod/15318 --grace-period=0 --force
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.BlQDAEvQhq
+++ cat /tmp/tmp.WAC00N0fhh
Warning: Immediate deletion does not wait for confirmation that the running resource has been terminated. The resource may continue to run on the cluster indefinitely.
+++ rm /tmp/tmp.BlQDAEvQhq /tmp/tmp.WAC00N0fhh
+++ return 0
+++ echo db version v7.0.8-5 Build Info: '{' '"version":' '"7.0.8-5",' '"gitVersion":' '"197ff4d49589e8b5a444b7ebc6f4d33b6f00105e",' '"openSSLVersion":' '"OpenSSL' 1.1.1k FIPS 25 Mar '2021",' '"modules":' '[],' '"proFeatures":' '[],' '"allocator":' '"tcmalloc",' '"environment":' '{' '"distarch":' '"x86_64",' '"target_arch":' '"x86_64"' '}' '}'
++ version_info=7.0.8-5
++ [[ ! 7.0.8-5 =~ ^([0-9]+\.){2}[0-9]+-[0-9]+$ ]]
++ echo 7.0.8-5
+ FULL_VER=7.0.8-5
+ MONGO_VER=7.0
+ unset OPERATOR_NS
+ main_cluster=cross-site-sharded-main
+ replica_cluster=cross-site-sharded-replica
+ desc 'create main cluster'
+ set +o xtrace


-----------------------------------------------------------------------------------
create main cluster
-----------------------------------------------------------------------------------

+ create_infra cross-site-sharded-9438
+ local ns=cross-site-sharded-9438
+ delete_crd
+ desc 'get and delete old CRDs and RBAC'
+ set +o xtrace


-----------------------------------------------------------------------------------
get and delete old CRDs and RBAC
-----------------------------------------------------------------------------------

+ kubectl_bin delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1536/deploy/crd.yaml --ignore-not-found --wait=false
++ mktemp
+ local LAST_OUT=/tmp/tmp.8UUXoKZKU5
++ mktemp
+ local LAST_ERR=/tmp/tmp.HaPUyE0ce0
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1536/deploy/crd.yaml --ignore-not-found --wait=false
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.8UUXoKZKU5
customresourcedefinition.apiextensions.k8s.io "perconaservermongodbbackups.psmdb.percona.com" deleted
customresourcedefinition.apiextensions.k8s.io "perconaservermongodbrestores.psmdb.percona.com" deleted
customresourcedefinition.apiextensions.k8s.io "perconaservermongodbs.psmdb.percona.com" deleted
+ cat /tmp/tmp.HaPUyE0ce0
+ rm /tmp/tmp.8UUXoKZKU5 /tmp/tmp.HaPUyE0ce0
+ return 0
++ yq eval .metadata.name /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1536/deploy/crd.yaml
++ grep -v '\-\-\-'
+ for crd_name in '$(yq eval '\''.metadata.name'\'' "${src_dir}/deploy/crd.yaml" | grep -v '\''\-\-\-'\'')'
+ kubectl get perconaservermongodbbackups.psmdb.percona.com --all-namespaces -o wide
+ grep -v NAMESPACE
+ xargs -L 1 sh -xc 'kubectl patch perconaservermongodbbackups.psmdb.percona.com -n $0 $1 --type=merge -p "{\"metadata\":{\"finalizers\":[]}}"'
error: the server doesn't have a resource type "perconaservermongodbbackups"
+ kubectl patch perconaservermongodbbackups.psmdb.percona.com -n sh --type=merge -p '{"metadata":{"finalizers":[]}}'
error: the server doesn't have a resource type "perconaservermongodbbackups"
+ :
+ kubectl_bin wait --for=delete crd perconaservermongodbbackups.psmdb.percona.com
++ mktemp
+ local LAST_OUT=/tmp/tmp.sBl6kQxfLJ
++ mktemp
+ local LAST_ERR=/tmp/tmp.P1fvbn15zB
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl wait --for=delete crd perconaservermongodbbackups.psmdb.percona.com
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.sBl6kQxfLJ
+ cat /tmp/tmp.P1fvbn15zB
+ rm /tmp/tmp.sBl6kQxfLJ /tmp/tmp.P1fvbn15zB
+ return 0
+ for crd_name in '$(yq eval '\''.metadata.name'\'' "${src_dir}/deploy/crd.yaml" | grep -v '\''\-\-\-'\'')'
+ xargs -L 1 sh -xc 'kubectl patch perconaservermongodbrestores.psmdb.percona.com -n $0 $1 --type=merge -p "{\"metadata\":{\"finalizers\":[]}}"'
+ grep -v NAMESPACE
+ kubectl get perconaservermongodbrestores.psmdb.percona.com --all-namespaces -o wide
error: the server doesn't have a resource type "perconaservermongodbrestores"
+ kubectl patch perconaservermongodbrestores.psmdb.percona.com -n sh --type=merge -p '{"metadata":{"finalizers":[]}}'
error: the server doesn't have a resource type "perconaservermongodbrestores"
+ :
+ kubectl_bin wait --for=delete crd perconaservermongodbrestores.psmdb.percona.com
++ mktemp
+ local LAST_OUT=/tmp/tmp.vBVWfg6398
++ mktemp
+ local LAST_ERR=/tmp/tmp.W2fSjLXI6h
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl wait --for=delete crd perconaservermongodbrestores.psmdb.percona.com
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.vBVWfg6398
+ cat /tmp/tmp.W2fSjLXI6h
+ rm /tmp/tmp.vBVWfg6398 /tmp/tmp.W2fSjLXI6h
+ return 0
+ for crd_name in '$(yq eval '\''.metadata.name'\'' "${src_dir}/deploy/crd.yaml" | grep -v '\''\-\-\-'\'')'
+ kubectl get perconaservermongodbs.psmdb.percona.com --all-namespaces -o wide
+ grep -v NAMESPACE
+ xargs -L 1 sh -xc 'kubectl patch perconaservermongodbs.psmdb.percona.com -n $0 $1 --type=merge -p "{\"metadata\":{\"finalizers\":[]}}"'
error: the server doesn't have a resource type "perconaservermongodbs"
+ kubectl patch perconaservermongodbs.psmdb.percona.com -n sh --type=merge -p '{"metadata":{"finalizers":[]}}'
error: the server doesn't have a resource type "perconaservermongodbs"
+ :
+ kubectl_bin wait --for=delete crd perconaservermongodbs.psmdb.percona.com
++ mktemp
+ local LAST_OUT=/tmp/tmp.lo1SWNdDr7
++ mktemp
+ local LAST_ERR=/tmp/tmp.qJqM5z2O5R
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl wait --for=delete crd perconaservermongodbs.psmdb.percona.com
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.lo1SWNdDr7
+ cat /tmp/tmp.qJqM5z2O5R
+ rm /tmp/tmp.lo1SWNdDr7 /tmp/tmp.qJqM5z2O5R
+ return 0
+ local rbac_yaml=rbac.yaml
+ '[' -n '' ']'
+ kubectl_bin delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1536/deploy/rbac.yaml --ignore-not-found
++ mktemp
+ local LAST_OUT=/tmp/tmp.4viy92xkgL
++ mktemp
+ local LAST_ERR=/tmp/tmp.Psh2GdKYTY
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1536/deploy/rbac.yaml --ignore-not-found
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.4viy92xkgL
role.rbac.authorization.k8s.io "percona-server-mongodb-operator" deleted
serviceaccount "percona-server-mongodb-operator" deleted
rolebinding.rbac.authorization.k8s.io "service-account-percona-server-mongodb-operator" deleted
+ cat /tmp/tmp.Psh2GdKYTY
+ rm /tmp/tmp.4viy92xkgL /tmp/tmp.Psh2GdKYTY
+ return 0
+ check_crd_for_deletion PR-1536-4cc529f8
+ local git_tag=PR-1536-4cc529f8
++ /usr/bin/sed s/---//g
++ /usr/bin/sed ':a;N;$!ba;s/\n/ /g'
++ curl -s https://raw.githubusercontent.com/percona/percona-server-mongodb-operator/PR-1536-4cc529f8/deploy/crd.yaml
++ yq eval .metadata.name
+ for crd_name in '$(curl -s https://raw.githubusercontent.com/percona/percona-server-mongodb-operator/${git_tag}/deploy/crd.yaml | yq eval '\''.metadata.name'\'' | $sed '\''s/---//g'\'' | $sed '\'':a;N;$!ba;s/\n/ /g'\'')'
++ kubectl_bin get crd/null -o 'jsonpath={.status.conditions[-1].type}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.jjjcu1wnN4
+++ mktemp
++ local LAST_ERR=/tmp/tmp.TEX639kZEe
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get crd/null -o 'jsonpath={.status.conditions[-1].type}'
++ exit_status=1
++ set -e
++ '[' 1 '!=' 0 -a -n 1 ']'
++ cat /tmp/tmp.jjjcu1wnN4
++ cat /tmp/tmp.TEX639kZEe
Error from server (NotFound): customresourcedefinitions.apiextensions.k8s.io "null" not found
++ sleep 0
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get crd/null -o 'jsonpath={.status.conditions[-1].type}'
++ exit_status=1
++ set -e
++ '[' 1 '!=' 0 -a -n 1 ']'
++ cat /tmp/tmp.jjjcu1wnN4
++ cat /tmp/tmp.TEX639kZEe
Error from server (NotFound): customresourcedefinitions.apiextensions.k8s.io "null" not found
++ sleep 4
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get crd/null -o 'jsonpath={.status.conditions[-1].type}'
++ exit_status=1
++ set -e
++ '[' 1 '!=' 0 -a -n 1 ']'
++ cat /tmp/tmp.jjjcu1wnN4
++ cat /tmp/tmp.TEX639kZEe
Error from server (NotFound): customresourcedefinitions.apiextensions.k8s.io "null" not found
++ sleep 8
++ cat /tmp/tmp.jjjcu1wnN4
++ cat /tmp/tmp.TEX639kZEe
Error from server (NotFound): customresourcedefinitions.apiextensions.k8s.io "null" not found
++ rm /tmp/tmp.jjjcu1wnN4 /tmp/tmp.TEX639kZEe
++ return 1
+ [[ '' == \T\e\r\m\i\n\a\t\i\n\g ]]
+ '[' -n '' ']'
+ create_namespace cross-site-sharded-9438
+ local namespace=cross-site-sharded-9438
+ local skip_clean_namespace=
+ [[ 1 == 1 ]]
+ [[ -z '' ]]
+ destroy_chaos_mesh
++ helm list --all-namespaces --filter chaos-mesh
++ sed s/NAMESPACE//
++ tail -n1
++ awk '-F ' '{print $2}'
+ local chaos_mesh_ns=
+ desc 'destroy chaos-mesh'
+ set +o xtrace


-----------------------------------------------------------------------------------
destroy chaos-mesh
-----------------------------------------------------------------------------------

+ '[' -n '' ']'
++ kubectl get MutatingWebhookConfiguration
++ grep chaos-mesh
++ awk '{print $1}'
+ timeout 30 kubectl delete MutatingWebhookConfiguration
error: resource(s) were provided, but no name was specified
+ :
++ kubectl get ValidatingWebhookConfiguration
++ grep chaos-mesh
++ awk '{print $1}'
+ timeout 30 kubectl delete ValidatingWebhookConfiguration
error: resource(s) were provided, but no name was specified
+ :
++ kubectl get ValidatingWebhookConfiguration
++ awk '{print $1}'
++ grep validate-auth
+ timeout 30 kubectl delete ValidatingWebhookConfiguration
error: resource(s) were provided, but no name was specified
+ :
++ kubectl api-resources
++ grep chaos-mesh
++ awk '{print $1}'
++ kubectl get crd
++ awk '{print $1}'
++ grep chaos-mesh.org
+ timeout 30 kubectl delete crd
error: resource(s) were provided, but no name was specified
+ :
++ kubectl get clusterrolebinding
++ awk '{print $1}'
++ grep chaos-mesh
+ timeout 30 kubectl delete clusterrolebinding
error: resource(s) were provided, but no name was specified
+ :
++ awk '{print $1}'
++ kubectl get clusterrole
++ grep chaos-mesh
+ timeout 30 kubectl delete clusterrole
error: resource(s) were provided, but no name was specified
+ :
+ desc 'cleaned up all old namespaces'
+ set +o xtrace


-----------------------------------------------------------------------------------
cleaned up all old namespaces
-----------------------------------------------------------------------------------

+ kubectl_bin get ns
+ egrep -v '^kube-|^default|Terminating|psmdb-operator|openshift|gke-mcs|^NAME'
+ awk '{print$1}'
+ '[' -n '' ']'
+ xargs kubectl delete ns
+ desc 'cleaned up old namespaces cross-site-sharded-9438'
+ set +o xtrace


-----------------------------------------------------------------------------------
cleaned up old namespaces cross-site-sharded-9438
-----------------------------------------------------------------------------------

+ kubectl_bin delete namespace cross-site-sharded-9438 --ignore-not-found
++ mktemp
++ mktemp
+ local LAST_OUT=/tmp/tmp.BtiFNAEqP8
+ local LAST_OUT=/tmp/tmp.MHGeoSUzoR
++ mktemp
++ mktemp
+ local LAST_ERR=/tmp/tmp.ScAojxnZ6n
+ local exit_status=0
+ local timeout=4
+ local LAST_ERR=/tmp/tmp.Dp8RoUaoss
+ local exit_status=0
+ local timeout=4
++ seq 0 2
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl delete namespace cross-site-sharded-9438 --ignore-not-found
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl get ns
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.MHGeoSUzoR
+ cat /tmp/tmp.Dp8RoUaoss
+ rm /tmp/tmp.MHGeoSUzoR /tmp/tmp.Dp8RoUaoss
+ return 0
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.BtiFNAEqP8
+ cat /tmp/tmp.ScAojxnZ6n
+ rm /tmp/tmp.BtiFNAEqP8 /tmp/tmp.ScAojxnZ6n
+ return 0
+ kubectl_bin wait --for=delete namespace cross-site-sharded-9438
++ mktemp
+ local LAST_OUT=/tmp/tmp.Y6kqq2kQrD
++ mktemp
+ local LAST_ERR=/tmp/tmp.D9POCGTbAA
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl wait --for=delete namespace cross-site-sharded-9438
namespace "cross-site-sharded-6750" deleted
namespace "cross-site-sharded-replica-4758" deleted
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.Y6kqq2kQrD
+ cat /tmp/tmp.D9POCGTbAA
+ rm /tmp/tmp.Y6kqq2kQrD /tmp/tmp.D9POCGTbAA
+ return 0
+ desc 'create namespace cross-site-sharded-9438'
+ set +o xtrace


-----------------------------------------------------------------------------------
create namespace cross-site-sharded-9438
-----------------------------------------------------------------------------------

+ kubectl_bin create namespace cross-site-sharded-9438
++ mktemp
+ local LAST_OUT=/tmp/tmp.5XDuODfzc9
++ mktemp
+ local LAST_ERR=/tmp/tmp.n30SSbRRui
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl create namespace cross-site-sharded-9438
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.5XDuODfzc9
namespace/cross-site-sharded-9438 created
+ cat /tmp/tmp.n30SSbRRui
+ rm /tmp/tmp.5XDuODfzc9 /tmp/tmp.n30SSbRRui
+ return 0
++ kubectl_bin config current-context
+++ mktemp
++ local LAST_OUT=/tmp/tmp.AFecUSSU4b
+++ mktemp
++ local LAST_ERR=/tmp/tmp.33IUnQ7FVA
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl config current-context
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.AFecUSSU4b
++ cat /tmp/tmp.33IUnQ7FVA
++ rm /tmp/tmp.AFecUSSU4b /tmp/tmp.33IUnQ7FVA
++ return 0
+ kubectl_bin config set-context gke_cloud-dev-112233_us-central1-a_jen-psmdb-1536-4cc529f8-10-cluster4 --namespace=cross-site-sharded-9438
++ mktemp
+ local LAST_OUT=/tmp/tmp.3W0ekYRnRD
++ mktemp
+ local LAST_ERR=/tmp/tmp.5unZYPR2ek
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl config set-context gke_cloud-dev-112233_us-central1-a_jen-psmdb-1536-4cc529f8-10-cluster4 --namespace=cross-site-sharded-9438
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.3W0ekYRnRD
Context "gke_cloud-dev-112233_us-central1-a_jen-psmdb-1536-4cc529f8-10-cluster4" modified.
+ cat /tmp/tmp.5unZYPR2ek
+ rm /tmp/tmp.3W0ekYRnRD /tmp/tmp.5unZYPR2ek
+ return 0
+ deploy_operator
+ desc 'start PSMDB operator'
+ set +o xtrace


-----------------------------------------------------------------------------------
start PSMDB operator
-----------------------------------------------------------------------------------

+ local cr_file
+ '[' -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1536/e2e-tests/cross-site-sharded/conf/crd.yaml ']'
+ cr_file=/mnt/jenkins/workspace/cloud-psmdb-operator_PR-1536/deploy/crd.yaml
+ kubectl_bin apply --server-side --force-conflicts -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1536/deploy/crd.yaml
++ mktemp
+ local LAST_OUT=/tmp/tmp.8qbO3AeNRG
++ mktemp
+ local LAST_ERR=/tmp/tmp.CzXrxkVMeb
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl apply --server-side --force-conflicts -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1536/deploy/crd.yaml
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.8qbO3AeNRG
customresourcedefinition.apiextensions.k8s.io/perconaservermongodbbackups.psmdb.percona.com serverside-applied
customresourcedefinition.apiextensions.k8s.io/perconaservermongodbrestores.psmdb.percona.com serverside-applied
customresourcedefinition.apiextensions.k8s.io/perconaservermongodbs.psmdb.percona.com serverside-applied
+ cat /tmp/tmp.CzXrxkVMeb
+ rm /tmp/tmp.8qbO3AeNRG /tmp/tmp.CzXrxkVMeb
+ return 0
+ '[' -n '' ']'
+ apply_rbac rbac
+ local operator_namespace=psmdb-operator
+ local rbac=rbac
+ sed -e 's^namespace: .*^namespace: psmdb-operator^'
+ cat /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1536/deploy/rbac.yaml
+ kubectl_bin apply -f -
++ mktemp
+ local LAST_OUT=/tmp/tmp.RELoNtr1fh
++ mktemp
+ local LAST_ERR=/tmp/tmp.rmkD6v1p9A
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl apply -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.RELoNtr1fh
role.rbac.authorization.k8s.io/percona-server-mongodb-operator created
serviceaccount/percona-server-mongodb-operator created
rolebinding.rbac.authorization.k8s.io/service-account-percona-server-mongodb-operator created
+ cat /tmp/tmp.rmkD6v1p9A
+ rm /tmp/tmp.RELoNtr1fh /tmp/tmp.rmkD6v1p9A
+ return 0
+ kubectl_bin apply -f -
+ yq eval '
			(.spec.template.spec.containers[].image = "perconalab/percona-server-mongodb-operator:PR-1536-4cc529f8") |
			((.. | select(.[] == "DISABLE_TELEMETRY")) |= .value="true") |
			((.. | select(.[] == "LOG_LEVEL")) |= .value="DEBUG")' /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1536/deploy/operator.yaml
++ mktemp
+ local LAST_OUT=/tmp/tmp.uV8O3TSZVE
++ mktemp
+ local LAST_ERR=/tmp/tmp.78n1C9pvbH
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl apply -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.uV8O3TSZVE
deployment.apps/percona-server-mongodb-operator created
+ cat /tmp/tmp.78n1C9pvbH
+ rm /tmp/tmp.uV8O3TSZVE /tmp/tmp.78n1C9pvbH
+ return 0
+ sleep 2
++ get_operator_pod
++ kubectl_bin get pods --selector=name=percona-server-mongodb-operator -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.FuDOFrIbql
+++ mktemp
++ local LAST_ERR=/tmp/tmp.MDCqPzqH7j
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=percona-server-mongodb-operator -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.FuDOFrIbql
++ cat /tmp/tmp.MDCqPzqH7j
++ rm /tmp/tmp.FuDOFrIbql /tmp/tmp.MDCqPzqH7j
++ return 0
+ wait_pod percona-server-mongodb-operator-765b885b75-w4dth
+ local pod=percona-server-mongodb-operator-765b885b75-w4dth
+ set +o xtrace
waiting for pod/percona-server-mongodb-operator-765b885b75-w4dth to be ready.OK
+ desc 'create secrets and start client'
+ set +o xtrace


-----------------------------------------------------------------------------------
create secrets and start client
-----------------------------------------------------------------------------------

+ kubectl_bin apply -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1536/e2e-tests/conf/client.yml -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1536/e2e-tests/cross-site-sharded/conf/secrets.yml
++ mktemp
+ local LAST_OUT=/tmp/tmp.mJzKHnCLso
++ mktemp
+ local LAST_ERR=/tmp/tmp.cia4UWsqgE
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl apply -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1536/e2e-tests/conf/client.yml -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1536/e2e-tests/cross-site-sharded/conf/secrets.yml
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.mJzKHnCLso
deployment.apps/psmdb-client created
secret/cross-site-sharded-main-secrets created
secret/cross-site-sharded-main-ssl created
secret/cross-site-sharded-main-ssl-internal created
+ cat /tmp/tmp.cia4UWsqgE
+ rm /tmp/tmp.mJzKHnCLso /tmp/tmp.cia4UWsqgE
+ return 0
+ desc 'create main PSMDB cluster cross-site-sharded-main.'
+ set +o xtrace


-----------------------------------------------------------------------------------
create main PSMDB cluster cross-site-sharded-main.
-----------------------------------------------------------------------------------

+ apply_cluster /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1536/e2e-tests/cross-site-sharded/conf/cross-site-sharded-main.yml
+ '[' -z '' ']'
+ cat_config /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1536/e2e-tests/cross-site-sharded/conf/cross-site-sharded-main.yml
+ kubectl_bin apply -f -
+ yq eval '(.spec | select(has("backup"))).backup.image = "perconalab/percona-server-mongodb-operator:main-backup"'
+ yq eval '.spec.upgradeOptions.apply="Never"'
+ cat /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1536/e2e-tests/cross-site-sharded/conf/cross-site-sharded-main.yml
+ yq eval '(.spec | select(.image == null)).image = "perconalab/percona-server-mongodb-operator:main-mongod7.0"'
+ yq eval '(.spec | select(has("pmm"))).pmm.image = "perconalab/pmm-client:dev-latest"'
+ yq eval '(.spec | select(has("initImage"))).initImage = "perconalab/percona-server-mongodb-operator:PR-1536-4cc529f8"'
++ mktemp
+ local LAST_OUT=/tmp/tmp.uUeoiLapo5
++ mktemp
+ local LAST_ERR=/tmp/tmp.Wr9Hnmk6Cu
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl apply -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.uUeoiLapo5
perconaservermongodb.psmdb.percona.com/cross-site-sharded-main created
+ cat /tmp/tmp.Wr9Hnmk6Cu
+ rm /tmp/tmp.uUeoiLapo5 /tmp/tmp.Wr9Hnmk6Cu
+ return 0
+ desc 'check if all 3 Pods started'
+ set +o xtrace


-----------------------------------------------------------------------------------
check if all 3 Pods started
-----------------------------------------------------------------------------------

+ wait_for_running cross-site-sharded-main-rs0 3
+ local name=cross-site-sharded-main-rs0
+ let last_pod=2
+ local check_cluster_readyness=true
+ set_debug
+ [[ 1 == 1 ]]
+ set -o xtrace
+ local rs_name=rs0
+ local cluster_name=cross-site-sharded-main
++ seq 0 2
+ for i in '$(seq 0 $last_pod)'
+ [[ 0 -eq 2 ]]
+ wait_pod cross-site-sharded-main-rs0-0
+ local pod=cross-site-sharded-main-rs0-0
+ set +o xtrace
waiting for pod/cross-site-sharded-main-rs0-0 to be ready..............OK
+ for i in '$(seq 0 $last_pod)'
+ [[ 1 -eq 2 ]]
+ wait_pod cross-site-sharded-main-rs0-1
+ local pod=cross-site-sharded-main-rs0-1
+ set +o xtrace
waiting for pod/cross-site-sharded-main-rs0-1 to be ready.............OK
+ for i in '$(seq 0 $last_pod)'
+ [[ 2 -eq 2 ]]
++ kubectl_bin get psmdb cross-site-sharded-main -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].arbiter.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.U0DIWAK90y
+++ mktemp
++ local LAST_ERR=/tmp/tmp.AaF82aCLg6
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb cross-site-sharded-main -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].arbiter.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.U0DIWAK90y
++ cat /tmp/tmp.AaF82aCLg6
++ rm /tmp/tmp.U0DIWAK90y /tmp/tmp.AaF82aCLg6
++ return 0
+ [[ false == \t\r\u\e ]]
+ wait_pod cross-site-sharded-main-rs0-2
+ local pod=cross-site-sharded-main-rs0-2
+ set +o xtrace
waiting for pod/cross-site-sharded-main-rs0-2 to be ready..............OK
++ kubectl_bin get psmdb cross-site-sharded-main -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].non_voting.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.BO55CjsHT3
+++ mktemp
++ local LAST_ERR=/tmp/tmp.p1918YZn2V
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb cross-site-sharded-main -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].non_voting.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.BO55CjsHT3
++ cat /tmp/tmp.p1918YZn2V
++ rm /tmp/tmp.BO55CjsHT3 /tmp/tmp.p1918YZn2V
++ return 0
+ [[ '' == \t\r\u\e ]]
+ sleep 10
+ [[ true == \t\r\u\e ]]
+ set +x
Waiting for cluster readyness............................
+ wait_for_running cross-site-sharded-main-cfg 3 false
+ local name=cross-site-sharded-main-cfg
+ let last_pod=2
+ local check_cluster_readyness=false
+ set_debug
+ [[ 1 == 1 ]]
+ set -o xtrace
+ local rs_name=cfg
+ local cluster_name=cross-site-sharded-main
++ seq 0 2
+ for i in '$(seq 0 $last_pod)'
+ [[ 0 -eq 2 ]]
+ wait_pod cross-site-sharded-main-cfg-0
+ local pod=cross-site-sharded-main-cfg-0
+ set +o xtrace
waiting for pod/cross-site-sharded-main-cfg-0 to be ready.OK
+ for i in '$(seq 0 $last_pod)'
+ [[ 1 -eq 2 ]]
+ wait_pod cross-site-sharded-main-cfg-1
+ local pod=cross-site-sharded-main-cfg-1
+ set +o xtrace
waiting for pod/cross-site-sharded-main-cfg-1 to be ready.OK
+ for i in '$(seq 0 $last_pod)'
+ [[ 2 -eq 2 ]]
++ kubectl_bin get psmdb cross-site-sharded-main -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].arbiter.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.W0JEu3XAvk
+++ mktemp
++ local LAST_ERR=/tmp/tmp.4s8dnDi1od
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb cross-site-sharded-main -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].arbiter.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.W0JEu3XAvk
++ cat /tmp/tmp.4s8dnDi1od
++ rm /tmp/tmp.W0JEu3XAvk /tmp/tmp.4s8dnDi1od
++ return 0
+ [[ '' == \t\r\u\e ]]
+ wait_pod cross-site-sharded-main-cfg-2
+ local pod=cross-site-sharded-main-cfg-2
+ set +o xtrace
waiting for pod/cross-site-sharded-main-cfg-2 to be ready.OK
++ kubectl_bin get psmdb cross-site-sharded-main -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].non_voting.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.KjFUxh9sDg
+++ mktemp
++ local LAST_ERR=/tmp/tmp.Umbywbw2QF
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb cross-site-sharded-main -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].non_voting.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.KjFUxh9sDg
++ cat /tmp/tmp.Umbywbw2QF
++ rm /tmp/tmp.KjFUxh9sDg /tmp/tmp.Umbywbw2QF
++ return 0
+ [[ '' == \t\r\u\e ]]
+ sleep 10
+ [[ false == \t\r\u\e ]]
+ desc 'create user'
+ set +o xtrace


-----------------------------------------------------------------------------------
create user
-----------------------------------------------------------------------------------

+ run_mongos 'db.createUser({user:"user",pwd:"pass",roles:[{db:"app",role:"readWrite"}]})' userAdmin:userAdmin123456@cross-site-sharded-main-mongos.cross-site-sharded-9438
+ local 'command=db.createUser({user:"user",pwd:"pass",roles:[{db:"app",role:"readWrite"}]})'
+ local uri=userAdmin:userAdmin123456@cross-site-sharded-main-mongos.cross-site-sharded-9438
+ local driver=mongodb
+ local suffix=.svc.cluster.local
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.2FbEQA5HMo
+++ mktemp
++ local LAST_ERR=/tmp/tmp.bnFXYXlWBs
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.2FbEQA5HMo
++ cat /tmp/tmp.bnFXYXlWBs
++ rm /tmp/tmp.2FbEQA5HMo /tmp/tmp.bnFXYXlWBs
++ return 0
+ local client_container=psmdb-client-5dc94d5b48-wl5cq
+ local mongo_flag=
+ kubectl_bin exec psmdb-client-5dc94d5b48-wl5cq -- bash -c 'printf '\''db.createUser({user:"user",pwd:"pass",roles:[{db:"app",role:"readWrite"}]})\n'\'' | mongo mongodb://userAdmin:userAdmin123456@cross-site-sharded-main-mongos.cross-site-sharded-9438.svc.cluster.local/admin '
++ mktemp
+ local LAST_OUT=/tmp/tmp.WHF9Pv4m0z
++ mktemp
+ local LAST_ERR=/tmp/tmp.6XIMgvNpbA
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl exec psmdb-client-5dc94d5b48-wl5cq -- bash -c 'printf '\''db.createUser({user:"user",pwd:"pass",roles:[{db:"app",role:"readWrite"}]})\n'\'' | mongo mongodb://userAdmin:userAdmin123456@cross-site-sharded-main-mongos.cross-site-sharded-9438.svc.cluster.local/admin '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.WHF9Pv4m0z
Percona Server for MongoDB shell version v4.4.29-28
connecting to: mongodb://cross-site-sharded-main-mongos.cross-site-sharded-9438.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb
Implicit session: session { "id" : UUID("24234bb4-38fd-4d74-87c2-28ac115227b3") }
Percona Server for MongoDB server version: v7.0.8-5
WARNING: shell and server versions do not match
Successfully added user: {
	"user" : "user",
	"roles" : [
		{
			"db" : "app",
			"role" : "readWrite"
		}
	]
}
bye
+ cat /tmp/tmp.6XIMgvNpbA
+ rm /tmp/tmp.WHF9Pv4m0z /tmp/tmp.6XIMgvNpbA
+ return 0
+ sleep 2
+ desc 'set chunk size to 2 MB'
+ set +o xtrace


-----------------------------------------------------------------------------------
set chunk size to 2 MB
-----------------------------------------------------------------------------------

+ run_mongos 'use config\n db.settings.save( { _id:"chunksize", value: 2 } )' clusterAdmin:clusterAdmin123456@cross-site-sharded-main-mongos.cross-site-sharded-9438
+ local 'command=use config\n db.settings.save( { _id:"chunksize", value: 2 } )'
+ local uri=clusterAdmin:clusterAdmin123456@cross-site-sharded-main-mongos.cross-site-sharded-9438
+ local driver=mongodb
+ local suffix=.svc.cluster.local
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.eqlmfTF6ag
+++ mktemp
++ local LAST_ERR=/tmp/tmp.lszukjwX23
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.eqlmfTF6ag
++ cat /tmp/tmp.lszukjwX23
++ rm /tmp/tmp.eqlmfTF6ag /tmp/tmp.lszukjwX23
++ return 0
+ local client_container=psmdb-client-5dc94d5b48-wl5cq
+ local mongo_flag=
+ kubectl_bin exec psmdb-client-5dc94d5b48-wl5cq -- bash -c 'printf '\''use config\n db.settings.save( { _id:"chunksize", value: 2 } )\n'\'' | mongo mongodb://clusterAdmin:clusterAdmin123456@cross-site-sharded-main-mongos.cross-site-sharded-9438.svc.cluster.local/admin '
++ mktemp
+ local LAST_OUT=/tmp/tmp.7TjGWPUIsM
++ mktemp
+ local LAST_ERR=/tmp/tmp.GRq4BQ2h1Q
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl exec psmdb-client-5dc94d5b48-wl5cq -- bash -c 'printf '\''use config\n db.settings.save( { _id:"chunksize", value: 2 } )\n'\'' | mongo mongodb://clusterAdmin:clusterAdmin123456@cross-site-sharded-main-mongos.cross-site-sharded-9438.svc.cluster.local/admin '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.7TjGWPUIsM
Percona Server for MongoDB shell version v4.4.29-28
connecting to: mongodb://cross-site-sharded-main-mongos.cross-site-sharded-9438.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb
Implicit session: session { "id" : UUID("455c84cb-f2cd-4a57-937c-fed46d691c40") }
Percona Server for MongoDB server version: v7.0.8-5
WARNING: shell and server versions do not match
switched to db config
WriteResult({ "nMatched" : 0, "nUpserted" : 1, "nModified" : 0, "_id" : "chunksize" })
bye
+ cat /tmp/tmp.GRq4BQ2h1Q
+ rm /tmp/tmp.7TjGWPUIsM /tmp/tmp.GRq4BQ2h1Q
+ return 0
+ sleep 2
+ desc 'write data'
+ set +o xtrace


-----------------------------------------------------------------------------------
write data
-----------------------------------------------------------------------------------

+ run_script_mongos /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1536/e2e-tests/cross-site-sharded/data.js user:pass@cross-site-sharded-main-mongos.cross-site-sharded-9438
+ local script=/mnt/jenkins/workspace/cloud-psmdb-operator_PR-1536/e2e-tests/cross-site-sharded/data.js
+ local uri=user:pass@cross-site-sharded-main-mongos.cross-site-sharded-9438
+ local driver=mongodb
+ local suffix=.svc.cluster.local
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.yo2Ls5dMpM
+++ mktemp
++ local LAST_ERR=/tmp/tmp.xHeiKjqUgn
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.yo2Ls5dMpM
++ cat /tmp/tmp.xHeiKjqUgn
++ rm /tmp/tmp.yo2Ls5dMpM /tmp/tmp.xHeiKjqUgn
++ return 0
+ local client_container=psmdb-client-5dc94d5b48-wl5cq
+ local mongo_flag=
++ basename /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1536/e2e-tests/cross-site-sharded/data.js
+ name=data.js
+ kubectl_bin cp /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1536/e2e-tests/cross-site-sharded/data.js cross-site-sharded-9438/psmdb-client-5dc94d5b48-wl5cq:/tmp
++ mktemp
+ local LAST_OUT=/tmp/tmp.2VmLW9JFPS
++ mktemp
+ local LAST_ERR=/tmp/tmp.AiIQed6Ks2
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl cp /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1536/e2e-tests/cross-site-sharded/data.js cross-site-sharded-9438/psmdb-client-5dc94d5b48-wl5cq:/tmp
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.2VmLW9JFPS
+ cat /tmp/tmp.AiIQed6Ks2
+ rm /tmp/tmp.2VmLW9JFPS /tmp/tmp.AiIQed6Ks2
+ return 0
+ kubectl_bin exec psmdb-client-5dc94d5b48-wl5cq -- bash -c 'mongo mongodb://user:pass@cross-site-sharded-main-mongos.cross-site-sharded-9438.svc.cluster.local/admin  /tmp/data.js'
++ mktemp
+ local LAST_OUT=/tmp/tmp.Gj42xFJZIs
++ mktemp
+ local LAST_ERR=/tmp/tmp.gATlH6NgZb
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl exec psmdb-client-5dc94d5b48-wl5cq -- bash -c 'mongo mongodb://user:pass@cross-site-sharded-main-mongos.cross-site-sharded-9438.svc.cluster.local/admin  /tmp/data.js'
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.Gj42xFJZIs
Percona Server for MongoDB shell version v4.4.29-28
connecting to: mongodb://cross-site-sharded-main-mongos.cross-site-sharded-9438.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb
Implicit session: session { "id" : UUID("26b7efcd-303d-4651-8797-a847769f1dcd") }
Percona Server for MongoDB server version: v7.0.8-5
WARNING: shell and server versions do not match
+ cat /tmp/tmp.gATlH6NgZb
+ rm /tmp/tmp.Gj42xFJZIs /tmp/tmp.gATlH6NgZb
+ return 0
+ desc 'shard collection'
+ set +o xtrace


-----------------------------------------------------------------------------------
shard collection
-----------------------------------------------------------------------------------

+ run_mongos 'sh.enableSharding("app")' clusterAdmin:clusterAdmin123456@cross-site-sharded-main-mongos.cross-site-sharded-9438
+ local 'command=sh.enableSharding("app")'
+ local uri=clusterAdmin:clusterAdmin123456@cross-site-sharded-main-mongos.cross-site-sharded-9438
+ local driver=mongodb
+ local suffix=.svc.cluster.local
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.rX6VKo3ef0
+++ mktemp
++ local LAST_ERR=/tmp/tmp.Cac1y25Bx9
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.rX6VKo3ef0
++ cat /tmp/tmp.Cac1y25Bx9
++ rm /tmp/tmp.rX6VKo3ef0 /tmp/tmp.Cac1y25Bx9
++ return 0
+ local client_container=psmdb-client-5dc94d5b48-wl5cq
+ local mongo_flag=
+ kubectl_bin exec psmdb-client-5dc94d5b48-wl5cq -- bash -c 'printf '\''sh.enableSharding("app")\n'\'' | mongo mongodb://clusterAdmin:clusterAdmin123456@cross-site-sharded-main-mongos.cross-site-sharded-9438.svc.cluster.local/admin '
++ mktemp
+ local LAST_OUT=/tmp/tmp.ytIODianKH
++ mktemp
+ local LAST_ERR=/tmp/tmp.l4NNJsRTZW
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl exec psmdb-client-5dc94d5b48-wl5cq -- bash -c 'printf '\''sh.enableSharding("app")\n'\'' | mongo mongodb://clusterAdmin:clusterAdmin123456@cross-site-sharded-main-mongos.cross-site-sharded-9438.svc.cluster.local/admin '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.ytIODianKH
Percona Server for MongoDB shell version v4.4.29-28
connecting to: mongodb://cross-site-sharded-main-mongos.cross-site-sharded-9438.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb
Implicit session: session { "id" : UUID("813f535f-591b-4bae-8a23-c950659c627f") }
Percona Server for MongoDB server version: v7.0.8-5
WARNING: shell and server versions do not match
{
	"ok" : 1,
	"$clusterTime" : {
		"clusterTime" : Timestamp(1714660393, 1),
		"signature" : {
			"hash" : BinData(0,"2bWhjFkvd62noiy9PNgD2wencmw="),
			"keyId" : NumberLong("7364409817760268309")
		}
	},
	"operationTime" : Timestamp(1714660393, 1)
}
bye
+ cat /tmp/tmp.l4NNJsRTZW
+ rm /tmp/tmp.ytIODianKH /tmp/tmp.l4NNJsRTZW
+ return 0
+ sleep 2
+ run_mongos 'sh.shardCollection("app.city", { _id: 1 } )' clusterAdmin:clusterAdmin123456@cross-site-sharded-main-mongos.cross-site-sharded-9438
+ local 'command=sh.shardCollection("app.city", { _id: 1 } )'
+ local uri=clusterAdmin:clusterAdmin123456@cross-site-sharded-main-mongos.cross-site-sharded-9438
+ local driver=mongodb
+ local suffix=.svc.cluster.local
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.DfiqMc8DwC
+++ mktemp
++ local LAST_ERR=/tmp/tmp.T5rtJAIMZ7
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.DfiqMc8DwC
++ cat /tmp/tmp.T5rtJAIMZ7
++ rm /tmp/tmp.DfiqMc8DwC /tmp/tmp.T5rtJAIMZ7
++ return 0
+ local client_container=psmdb-client-5dc94d5b48-wl5cq
+ local mongo_flag=
+ kubectl_bin exec psmdb-client-5dc94d5b48-wl5cq -- bash -c 'printf '\''sh.shardCollection("app.city", { _id: 1 } )\n'\'' | mongo mongodb://clusterAdmin:clusterAdmin123456@cross-site-sharded-main-mongos.cross-site-sharded-9438.svc.cluster.local/admin '
++ mktemp
+ local LAST_OUT=/tmp/tmp.qyHazzJTBm
++ mktemp
+ local LAST_ERR=/tmp/tmp.PKeEv1093N
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl exec psmdb-client-5dc94d5b48-wl5cq -- bash -c 'printf '\''sh.shardCollection("app.city", { _id: 1 } )\n'\'' | mongo mongodb://clusterAdmin:clusterAdmin123456@cross-site-sharded-main-mongos.cross-site-sharded-9438.svc.cluster.local/admin '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.qyHazzJTBm
Percona Server for MongoDB shell version v4.4.29-28
connecting to: mongodb://cross-site-sharded-main-mongos.cross-site-sharded-9438.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb
Implicit session: session { "id" : UUID("f2db8618-e1a4-4604-8508-a7c067b52e72") }
Percona Server for MongoDB server version: v7.0.8-5
WARNING: shell and server versions do not match
{
	"collectionsharded" : "app.city",
	"ok" : 1,
	"$clusterTime" : {
		"clusterTime" : Timestamp(1714660400, 30),
		"signature" : {
			"hash" : BinData(0,"4mlnzmbDzXoZoaybKTxpEsFDMP0="),
			"keyId" : NumberLong("7364409817760268309")
		}
	},
	"operationTime" : Timestamp(1714660400, 30)
}
bye
+ cat /tmp/tmp.PKeEv1093N
+ rm /tmp/tmp.qyHazzJTBm /tmp/tmp.PKeEv1093N
+ return 0
+ sleep 120
+ desc 'Check chunks'
+ set +o xtrace


-----------------------------------------------------------------------------------
Check chunks
-----------------------------------------------------------------------------------

+ chunks_param1=ns
+ chunks_param2='"app.city"'
+ [[ 7.0 != \4\.\4 ]]
+ chunks_param1=uuid
++ run_mongos 'use app\n db.getCollectionInfos({ "name": "city" })[0].info.uuid' user:pass@cross-site-sharded-main-mongos.cross-site-sharded-9438
++ local 'command=use app\n db.getCollectionInfos({ "name": "city" })[0].info.uuid'
++ local uri=user:pass@cross-site-sharded-main-mongos.cross-site-sharded-9438
++ local driver=mongodb
++ local suffix=.svc.cluster.local
++ grep -v 'switched to db app'
++ grep 'switched to db app' -A 1
+++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.ZLPJK4QLf3
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.HLYSvZbLbq
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in '$(seq 0 2)'
+++ set +e
+++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.ZLPJK4QLf3
+++ cat /tmp/tmp.HLYSvZbLbq
+++ rm /tmp/tmp.ZLPJK4QLf3 /tmp/tmp.HLYSvZbLbq
+++ return 0
++ local client_container=psmdb-client-5dc94d5b48-wl5cq
++ local mongo_flag=
++ kubectl_bin exec psmdb-client-5dc94d5b48-wl5cq -- bash -c 'printf '\''use app\n db.getCollectionInfos({ "name": "city" })[0].info.uuid\n'\'' | mongo mongodb://user:pass@cross-site-sharded-main-mongos.cross-site-sharded-9438.svc.cluster.local/admin '
+++ mktemp
++ local LAST_OUT=/tmp/tmp.KV9SeRQ5Xn
+++ mktemp
++ local LAST_ERR=/tmp/tmp.8yvRCcaRhW
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl exec psmdb-client-5dc94d5b48-wl5cq -- bash -c 'printf '\''use app\n db.getCollectionInfos({ "name": "city" })[0].info.uuid\n'\'' | mongo mongodb://user:pass@cross-site-sharded-main-mongos.cross-site-sharded-9438.svc.cluster.local/admin '
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.KV9SeRQ5Xn
++ cat /tmp/tmp.8yvRCcaRhW
++ rm /tmp/tmp.KV9SeRQ5Xn /tmp/tmp.8yvRCcaRhW
++ return 0
+ chunks_param2='UUID("8e9f7a0a-3e41-4765-9795-ad37a4fd03bf")'
+ shards=0
+ for i in '"rs0"' '"rs1"'
++ run_mongos 'use config\n db.chunks.count({"uuid": UUID("8e9f7a0a-3e41-4765-9795-ad37a4fd03bf"), "shard": "rs0"})' clusterAdmin:clusterAdmin123456@cross-site-sharded-main-mongos.cross-site-sharded-9438
++ grep -v 'switched to db config'
++ local 'command=use config\n db.chunks.count({"uuid": UUID("8e9f7a0a-3e41-4765-9795-ad37a4fd03bf"), "shard": "rs0"})'
++ local uri=clusterAdmin:clusterAdmin123456@cross-site-sharded-main-mongos.cross-site-sharded-9438
++ local driver=mongodb
++ local suffix=.svc.cluster.local
+++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.z3Bix4zPau
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.915JjH98Y9
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in '$(seq 0 2)'
+++ set +e
+++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ grep 'switched to db config' -A 1
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.z3Bix4zPau
+++ cat /tmp/tmp.915JjH98Y9
+++ rm /tmp/tmp.z3Bix4zPau /tmp/tmp.915JjH98Y9
+++ return 0
++ local client_container=psmdb-client-5dc94d5b48-wl5cq
++ local mongo_flag=
++ kubectl_bin exec psmdb-client-5dc94d5b48-wl5cq -- bash -c 'printf '\''use config\n db.chunks.count({"uuid": UUID("8e9f7a0a-3e41-4765-9795-ad37a4fd03bf"), "shard": "rs0"})\n'\'' | mongo mongodb://clusterAdmin:clusterAdmin123456@cross-site-sharded-main-mongos.cross-site-sharded-9438.svc.cluster.local/admin '
+++ mktemp
++ local LAST_OUT=/tmp/tmp.sxOqIu6mxM
+++ mktemp
++ local LAST_ERR=/tmp/tmp.PMVt9fUVXW
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl exec psmdb-client-5dc94d5b48-wl5cq -- bash -c 'printf '\''use config\n db.chunks.count({"uuid": UUID("8e9f7a0a-3e41-4765-9795-ad37a4fd03bf"), "shard": "rs0"})\n'\'' | mongo mongodb://clusterAdmin:clusterAdmin123456@cross-site-sharded-main-mongos.cross-site-sharded-9438.svc.cluster.local/admin '
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.sxOqIu6mxM
++ cat /tmp/tmp.PMVt9fUVXW
++ rm /tmp/tmp.sxOqIu6mxM /tmp/tmp.PMVt9fUVXW
++ return 0
+ out=3
+ desc 'rs0 has 3 chunks'
+ set +o xtrace


-----------------------------------------------------------------------------------
rs0 has 3 chunks
-----------------------------------------------------------------------------------

+ [[ 3 -ne 0 ]]
+ (( shards = shards + 1 ))
+ for i in '"rs0"' '"rs1"'
++ grep 'switched to db config' -A 1
++ grep -v 'switched to db config'
++ run_mongos 'use config\n db.chunks.count({"uuid": UUID("8e9f7a0a-3e41-4765-9795-ad37a4fd03bf"), "shard": "rs1"})' clusterAdmin:clusterAdmin123456@cross-site-sharded-main-mongos.cross-site-sharded-9438
++ local 'command=use config\n db.chunks.count({"uuid": UUID("8e9f7a0a-3e41-4765-9795-ad37a4fd03bf"), "shard": "rs1"})'
++ local uri=clusterAdmin:clusterAdmin123456@cross-site-sharded-main-mongos.cross-site-sharded-9438
++ local driver=mongodb
++ local suffix=.svc.cluster.local
+++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.AdyMInelPA
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.oJ8zdfnPnS
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in '$(seq 0 2)'
+++ set +e
+++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.AdyMInelPA
+++ cat /tmp/tmp.oJ8zdfnPnS
+++ rm /tmp/tmp.AdyMInelPA /tmp/tmp.oJ8zdfnPnS
+++ return 0
++ local client_container=psmdb-client-5dc94d5b48-wl5cq
++ local mongo_flag=
++ kubectl_bin exec psmdb-client-5dc94d5b48-wl5cq -- bash -c 'printf '\''use config\n db.chunks.count({"uuid": UUID("8e9f7a0a-3e41-4765-9795-ad37a4fd03bf"), "shard": "rs1"})\n'\'' | mongo mongodb://clusterAdmin:clusterAdmin123456@cross-site-sharded-main-mongos.cross-site-sharded-9438.svc.cluster.local/admin '
+++ mktemp
++ local LAST_OUT=/tmp/tmp.WlcXOmfyRA
+++ mktemp
++ local LAST_ERR=/tmp/tmp.kDejzZqHU9
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl exec psmdb-client-5dc94d5b48-wl5cq -- bash -c 'printf '\''use config\n db.chunks.count({"uuid": UUID("8e9f7a0a-3e41-4765-9795-ad37a4fd03bf"), "shard": "rs1"})\n'\'' | mongo mongodb://clusterAdmin:clusterAdmin123456@cross-site-sharded-main-mongos.cross-site-sharded-9438.svc.cluster.local/admin '
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.WlcXOmfyRA
++ cat /tmp/tmp.kDejzZqHU9
++ rm /tmp/tmp.WlcXOmfyRA /tmp/tmp.kDejzZqHU9
++ return 0
+ out=1
+ desc 'rs1 has 1 chunks'
+ set +o xtrace


-----------------------------------------------------------------------------------
rs1 has 1 chunks
-----------------------------------------------------------------------------------

+ [[ 1 -ne 0 ]]
+ (( shards = shards + 1 ))
+ [[ 2 -lt 2 ]]
+ desc 'create replica cluster'
+ set +o xtrace


-----------------------------------------------------------------------------------
create replica cluster
-----------------------------------------------------------------------------------

+ create_namespace cross-site-sharded-replica-31492 0
+ local namespace=cross-site-sharded-replica-31492
+ local skip_clean_namespace=0
+ [[ 1 == 1 ]]
+ [[ -z 0 ]]
+ '[' -n '' ']'
+ desc 'cleaned up old namespaces cross-site-sharded-replica-31492'
+ set +o xtrace


-----------------------------------------------------------------------------------
cleaned up old namespaces cross-site-sharded-replica-31492
-----------------------------------------------------------------------------------

+ kubectl_bin delete namespace cross-site-sharded-replica-31492 --ignore-not-found
++ mktemp
+ local LAST_OUT=/tmp/tmp.gSwS1uobbf
++ mktemp
+ local LAST_ERR=/tmp/tmp.aWHcUZ1VrG
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl delete namespace cross-site-sharded-replica-31492 --ignore-not-found
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.gSwS1uobbf
+ cat /tmp/tmp.aWHcUZ1VrG
+ rm /tmp/tmp.gSwS1uobbf /tmp/tmp.aWHcUZ1VrG
+ return 0
+ kubectl_bin wait --for=delete namespace cross-site-sharded-replica-31492
++ mktemp
+ local LAST_OUT=/tmp/tmp.S52peCghgN
++ mktemp
+ local LAST_ERR=/tmp/tmp.RHgbJnIEwK
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl wait --for=delete namespace cross-site-sharded-replica-31492
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.S52peCghgN
+ cat /tmp/tmp.RHgbJnIEwK
+ rm /tmp/tmp.S52peCghgN /tmp/tmp.RHgbJnIEwK
+ return 0
+ desc 'create namespace cross-site-sharded-replica-31492'
+ set +o xtrace


-----------------------------------------------------------------------------------
create namespace cross-site-sharded-replica-31492
-----------------------------------------------------------------------------------

+ kubectl_bin create namespace cross-site-sharded-replica-31492
++ mktemp
+ local LAST_OUT=/tmp/tmp.dmbiWKNuMX
++ mktemp
+ local LAST_ERR=/tmp/tmp.vFnA7MDRFB
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl create namespace cross-site-sharded-replica-31492
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.dmbiWKNuMX
namespace/cross-site-sharded-replica-31492 created
+ cat /tmp/tmp.vFnA7MDRFB
+ rm /tmp/tmp.dmbiWKNuMX /tmp/tmp.vFnA7MDRFB
+ return 0
++ kubectl_bin config current-context
+++ mktemp
++ local LAST_OUT=/tmp/tmp.K6h3S8iRnT
+++ mktemp
++ local LAST_ERR=/tmp/tmp.XcEdw8KCsJ
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl config current-context
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.K6h3S8iRnT
++ cat /tmp/tmp.XcEdw8KCsJ
++ rm /tmp/tmp.K6h3S8iRnT /tmp/tmp.XcEdw8KCsJ
++ return 0
+ kubectl_bin config set-context gke_cloud-dev-112233_us-central1-a_jen-psmdb-1536-4cc529f8-10-cluster4 --namespace=cross-site-sharded-replica-31492
++ mktemp
+ local LAST_OUT=/tmp/tmp.l9aVG817AG
++ mktemp
+ local LAST_ERR=/tmp/tmp.8TkspuCkKx
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl config set-context gke_cloud-dev-112233_us-central1-a_jen-psmdb-1536-4cc529f8-10-cluster4 --namespace=cross-site-sharded-replica-31492
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.l9aVG817AG
Context "gke_cloud-dev-112233_us-central1-a_jen-psmdb-1536-4cc529f8-10-cluster4" modified.
+ cat /tmp/tmp.8TkspuCkKx
+ rm /tmp/tmp.l9aVG817AG /tmp/tmp.8TkspuCkKx
+ return 0
+ deploy_operator
+ desc 'start PSMDB operator'
+ set +o xtrace


-----------------------------------------------------------------------------------
start PSMDB operator
-----------------------------------------------------------------------------------

+ local cr_file
+ '[' -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1536/e2e-tests/cross-site-sharded/conf/crd.yaml ']'
+ cr_file=/mnt/jenkins/workspace/cloud-psmdb-operator_PR-1536/deploy/crd.yaml
+ kubectl_bin apply --server-side --force-conflicts -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1536/deploy/crd.yaml
++ mktemp
+ local LAST_OUT=/tmp/tmp.1A1AZzVqVi
++ mktemp
+ local LAST_ERR=/tmp/tmp.uw1wWEmsVh
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl apply --server-side --force-conflicts -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1536/deploy/crd.yaml
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.1A1AZzVqVi
customresourcedefinition.apiextensions.k8s.io/perconaservermongodbbackups.psmdb.percona.com serverside-applied
customresourcedefinition.apiextensions.k8s.io/perconaservermongodbrestores.psmdb.percona.com serverside-applied
customresourcedefinition.apiextensions.k8s.io/perconaservermongodbs.psmdb.percona.com serverside-applied
+ cat /tmp/tmp.uw1wWEmsVh
+ rm /tmp/tmp.1A1AZzVqVi /tmp/tmp.uw1wWEmsVh
+ return 0
+ '[' -n '' ']'
+ apply_rbac rbac
+ local operator_namespace=psmdb-operator
+ local rbac=rbac
+ cat /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1536/deploy/rbac.yaml
+ sed -e 's^namespace: .*^namespace: psmdb-operator^'
+ kubectl_bin apply -f -
++ mktemp
+ local LAST_OUT=/tmp/tmp.vUlLP8ZYfl
++ mktemp
+ local LAST_ERR=/tmp/tmp.N6UCLBWcCy
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl apply -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.vUlLP8ZYfl
role.rbac.authorization.k8s.io/percona-server-mongodb-operator created
serviceaccount/percona-server-mongodb-operator created
rolebinding.rbac.authorization.k8s.io/service-account-percona-server-mongodb-operator created
+ cat /tmp/tmp.N6UCLBWcCy
+ rm /tmp/tmp.vUlLP8ZYfl /tmp/tmp.N6UCLBWcCy
+ return 0
+ yq eval '
			(.spec.template.spec.containers[].image = "perconalab/percona-server-mongodb-operator:PR-1536-4cc529f8") |
			((.. | select(.[] == "DISABLE_TELEMETRY")) |= .value="true") |
			((.. | select(.[] == "LOG_LEVEL")) |= .value="DEBUG")' /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1536/deploy/operator.yaml
+ kubectl_bin apply -f -
++ mktemp
+ local LAST_OUT=/tmp/tmp.p9OvvVoN5B
++ mktemp
+ local LAST_ERR=/tmp/tmp.gEC4CGitaU
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl apply -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.p9OvvVoN5B
deployment.apps/percona-server-mongodb-operator created
+ cat /tmp/tmp.gEC4CGitaU
+ rm /tmp/tmp.p9OvvVoN5B /tmp/tmp.gEC4CGitaU
+ return 0
+ sleep 2
++ get_operator_pod
++ kubectl_bin get pods --selector=name=percona-server-mongodb-operator -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.TC669NdWpy
+++ mktemp
++ local LAST_ERR=/tmp/tmp.M1g1QKl1hY
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=percona-server-mongodb-operator -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.TC669NdWpy
++ cat /tmp/tmp.M1g1QKl1hY
++ rm /tmp/tmp.TC669NdWpy /tmp/tmp.M1g1QKl1hY
++ return 0
+ wait_pod percona-server-mongodb-operator-765b885b75-tnwzm
+ local pod=percona-server-mongodb-operator-765b885b75-tnwzm
+ set +o xtrace
waiting for pod/percona-server-mongodb-operator-765b885b75-tnwzm to be ready.OK
+ desc 'start client'
+ set +o xtrace


-----------------------------------------------------------------------------------
start client
-----------------------------------------------------------------------------------

+ kubectl_bin apply -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1536/e2e-tests/conf/client.yml
++ mktemp
+ local LAST_OUT=/tmp/tmp.FpdAfQQLDm
++ mktemp
+ local LAST_ERR=/tmp/tmp.aiLKZqxUFS
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl apply -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1536/e2e-tests/conf/client.yml
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.FpdAfQQLDm
deployment.apps/psmdb-client created
+ cat /tmp/tmp.aiLKZqxUFS
+ rm /tmp/tmp.FpdAfQQLDm /tmp/tmp.aiLKZqxUFS
+ return 0
+ desc 'copy secrets from main to replica namespace and create all of them'
+ set +o xtrace


-----------------------------------------------------------------------------------
copy secrets from main to replica namespace and create all of them
-----------------------------------------------------------------------------------

+ yq eval '
		del(.metadata) |
		(.metadata.name = "cross-site-sharded-replica-secrets")' -
+ kubectl get secret cross-site-sharded-main-secrets -o yaml -n cross-site-sharded-9438
+ kubectl_bin apply -f -
++ mktemp
+ local LAST_OUT=/tmp/tmp.E01MR8lYmC
++ mktemp
+ local LAST_ERR=/tmp/tmp.QL0cY2ltwH
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl apply -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.E01MR8lYmC
secret/cross-site-sharded-replica-secrets created
+ cat /tmp/tmp.QL0cY2ltwH
+ rm /tmp/tmp.E01MR8lYmC /tmp/tmp.QL0cY2ltwH
+ return 0
+ kubectl_bin get secret cross-site-sharded-main-ssl-internal -o yaml -n cross-site-sharded-9438
+ yq eval '
		del(.metadata) |
		del(.status) |
		(.metadata.name = "cross-site-sharded-replica-ssl-internal")' -
+ kubectl_bin apply -f -
++ mktemp
++ mktemp
+ local LAST_OUT=/tmp/tmp.PmN5zyOzM6
++ mktemp
+ local LAST_OUT=/tmp/tmp.p500jlxbNw
++ mktemp
+ local LAST_ERR=/tmp/tmp.qIeeNetTXy
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ local LAST_ERR=/tmp/tmp.vLq16J6xFA
+ local exit_status=0
+ local timeout=4
+ for i in '$(seq 0 2)'
+ set +e
++ seq 0 2
+ kubectl get secret cross-site-sharded-main-ssl-internal -o yaml -n cross-site-sharded-9438
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl apply -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.PmN5zyOzM6
+ cat /tmp/tmp.qIeeNetTXy
+ rm /tmp/tmp.PmN5zyOzM6 /tmp/tmp.qIeeNetTXy
+ return 0
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.p500jlxbNw
secret/cross-site-sharded-replica-ssl-internal created
+ cat /tmp/tmp.vLq16J6xFA
+ rm /tmp/tmp.p500jlxbNw /tmp/tmp.vLq16J6xFA
+ return 0
+ yq eval '
		del(.metadata) |
		del(.status) |
+ kubectl_bin apply -f -
		(.metadata.name = "cross-site-sharded-replica-ssl")' -
++ mktemp
+ local LAST_OUT=/tmp/tmp.fRkpB9thha
++ mktemp
+ local LAST_ERR=/tmp/tmp.WT9PSU0C7B
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl apply -f -
+ kubectl_bin get secret cross-site-sharded-main-ssl -o yaml -n cross-site-sharded-9438
++ mktemp
+ local LAST_OUT=/tmp/tmp.vefr31VeCF
++ mktemp
+ local LAST_ERR=/tmp/tmp.nrauyyCF9Z
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl get secret cross-site-sharded-main-ssl -o yaml -n cross-site-sharded-9438
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.vefr31VeCF
+ cat /tmp/tmp.nrauyyCF9Z
+ rm /tmp/tmp.vefr31VeCF /tmp/tmp.nrauyyCF9Z
+ return 0
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.fRkpB9thha
secret/cross-site-sharded-replica-ssl created
+ cat /tmp/tmp.WT9PSU0C7B
+ rm /tmp/tmp.fRkpB9thha /tmp/tmp.WT9PSU0C7B
+ return 0
+ sleep 30
+ desc 'create replica PSMDB cluster '
+ set +o xtrace


-----------------------------------------------------------------------------------
create replica PSMDB cluster 
-----------------------------------------------------------------------------------

+ apply_cluster /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1536/e2e-tests/cross-site-sharded/conf/cross-site-sharded-replica.yml
+ '[' -z '' ']'
+ cat_config /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1536/e2e-tests/cross-site-sharded/conf/cross-site-sharded-replica.yml
+ kubectl_bin apply -f -
+ cat /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1536/e2e-tests/cross-site-sharded/conf/cross-site-sharded-replica.yml
++ mktemp
+ yq eval '(.spec | select(.image == null)).image = "perconalab/percona-server-mongodb-operator:main-mongod7.0"'
+ yq eval '(.spec | select(has("pmm"))).pmm.image = "perconalab/pmm-client:dev-latest"'
+ yq eval '.spec.upgradeOptions.apply="Never"'
+ local LAST_OUT=/tmp/tmp.WEZCwk3WfU
+ yq eval '(.spec | select(has("backup"))).backup.image = "perconalab/percona-server-mongodb-operator:main-backup"'
++ mktemp
+ yq eval '(.spec | select(has("initImage"))).initImage = "perconalab/percona-server-mongodb-operator:PR-1536-4cc529f8"'
+ local LAST_ERR=/tmp/tmp.Rz93bJWx1d
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl apply -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.WEZCwk3WfU
perconaservermongodb.psmdb.percona.com/cross-site-sharded-replica created
+ cat /tmp/tmp.Rz93bJWx1d
+ rm /tmp/tmp.WEZCwk3WfU /tmp/tmp.Rz93bJWx1d
+ return 0
+ wait_for_running cross-site-sharded-replica-rs0 3 false
+ local name=cross-site-sharded-replica-rs0
+ let last_pod=2
+ local check_cluster_readyness=false
+ set_debug
+ [[ 1 == 1 ]]
+ set -o xtrace
+ local rs_name=rs0
+ local cluster_name=cross-site-sharded-replica
++ seq 0 2
+ for i in '$(seq 0 $last_pod)'
+ [[ 0 -eq 2 ]]
+ wait_pod cross-site-sharded-replica-rs0-0
+ local pod=cross-site-sharded-replica-rs0-0
+ set +o xtrace
waiting for pod/cross-site-sharded-replica-rs0-0 to be ready...........OK
+ for i in '$(seq 0 $last_pod)'
+ [[ 1 -eq 2 ]]
+ wait_pod cross-site-sharded-replica-rs0-1
+ local pod=cross-site-sharded-replica-rs0-1
+ set +o xtrace
waiting for pod/cross-site-sharded-replica-rs0-1 to be ready............OK
+ for i in '$(seq 0 $last_pod)'
+ [[ 2 -eq 2 ]]
++ kubectl_bin get psmdb cross-site-sharded-replica -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].arbiter.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.4HbyT62TaZ
+++ mktemp
++ local LAST_ERR=/tmp/tmp.uSr25O8yNL
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb cross-site-sharded-replica -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].arbiter.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.4HbyT62TaZ
++ cat /tmp/tmp.uSr25O8yNL
++ rm /tmp/tmp.4HbyT62TaZ /tmp/tmp.uSr25O8yNL
++ return 0
+ [[ false == \t\r\u\e ]]
+ wait_pod cross-site-sharded-replica-rs0-2
+ local pod=cross-site-sharded-replica-rs0-2
+ set +o xtrace
waiting for pod/cross-site-sharded-replica-rs0-2 to be ready...............OK
++ kubectl_bin get psmdb cross-site-sharded-replica -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].non_voting.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.N8A6iXgTKQ
+++ mktemp
++ local LAST_ERR=/tmp/tmp.HjTJInRP1l
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb cross-site-sharded-replica -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].non_voting.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.N8A6iXgTKQ
++ cat /tmp/tmp.HjTJInRP1l
++ rm /tmp/tmp.N8A6iXgTKQ /tmp/tmp.HjTJInRP1l
++ return 0
+ [[ '' == \t\r\u\e ]]
+ sleep 10
+ [[ false == \t\r\u\e ]]
+ wait_for_running cross-site-sharded-replica-rs1 3 false
+ local name=cross-site-sharded-replica-rs1
+ let last_pod=2
+ local check_cluster_readyness=false
+ set_debug
+ [[ 1 == 1 ]]
+ set -o xtrace
+ local rs_name=rs1
+ local cluster_name=cross-site-sharded-replica
++ seq 0 2
+ for i in '$(seq 0 $last_pod)'
+ [[ 0 -eq 2 ]]
+ wait_pod cross-site-sharded-replica-rs1-0
+ local pod=cross-site-sharded-replica-rs1-0
+ set +o xtrace
waiting for pod/cross-site-sharded-replica-rs1-0 to be ready.OK
+ for i in '$(seq 0 $last_pod)'
+ [[ 1 -eq 2 ]]
+ wait_pod cross-site-sharded-replica-rs1-1
+ local pod=cross-site-sharded-replica-rs1-1
+ set +o xtrace
waiting for pod/cross-site-sharded-replica-rs1-1 to be ready.OK
+ for i in '$(seq 0 $last_pod)'
+ [[ 2 -eq 2 ]]
++ kubectl_bin get psmdb cross-site-sharded-replica -o 'jsonpath={.spec.replsets[?(@.name=="rs1")].arbiter.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.C6BEtfl3kw
+++ mktemp
++ local LAST_ERR=/tmp/tmp.bfX979QKFA
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb cross-site-sharded-replica -o 'jsonpath={.spec.replsets[?(@.name=="rs1")].arbiter.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.C6BEtfl3kw
++ cat /tmp/tmp.bfX979QKFA
++ rm /tmp/tmp.C6BEtfl3kw /tmp/tmp.bfX979QKFA
++ return 0
+ [[ false == \t\r\u\e ]]
+ wait_pod cross-site-sharded-replica-rs1-2
+ local pod=cross-site-sharded-replica-rs1-2
+ set +o xtrace
waiting for pod/cross-site-sharded-replica-rs1-2 to be ready.OK
++ kubectl_bin get psmdb cross-site-sharded-replica -o 'jsonpath={.spec.replsets[?(@.name=="rs1")].non_voting.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.UMVqIEhxfw
+++ mktemp
++ local LAST_ERR=/tmp/tmp.OCKJLBna6U
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb cross-site-sharded-replica -o 'jsonpath={.spec.replsets[?(@.name=="rs1")].non_voting.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.UMVqIEhxfw
++ cat /tmp/tmp.OCKJLBna6U
++ rm /tmp/tmp.UMVqIEhxfw /tmp/tmp.OCKJLBna6U
++ return 0
+ [[ '' == \t\r\u\e ]]
+ sleep 10
+ [[ false == \t\r\u\e ]]
+ wait_for_running cross-site-sharded-replica-cfg 3 false
+ local name=cross-site-sharded-replica-cfg
+ let last_pod=2
+ local check_cluster_readyness=false
+ set_debug
+ [[ 1 == 1 ]]
+ set -o xtrace
+ local rs_name=cfg
+ local cluster_name=cross-site-sharded-replica
++ seq 0 2
+ for i in '$(seq 0 $last_pod)'
+ [[ 0 -eq 2 ]]
+ wait_pod cross-site-sharded-replica-cfg-0
+ local pod=cross-site-sharded-replica-cfg-0
+ set +o xtrace
waiting for pod/cross-site-sharded-replica-cfg-0 to be ready.OK
+ for i in '$(seq 0 $last_pod)'
+ [[ 1 -eq 2 ]]
+ wait_pod cross-site-sharded-replica-cfg-1
+ local pod=cross-site-sharded-replica-cfg-1
+ set +o xtrace
waiting for pod/cross-site-sharded-replica-cfg-1 to be ready.OK
+ for i in '$(seq 0 $last_pod)'
+ [[ 2 -eq 2 ]]
++ kubectl_bin get psmdb cross-site-sharded-replica -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].arbiter.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.Jf6OaSIfG3
+++ mktemp
++ local LAST_ERR=/tmp/tmp.ZXfJsjltS7
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb cross-site-sharded-replica -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].arbiter.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.Jf6OaSIfG3
++ cat /tmp/tmp.ZXfJsjltS7
++ rm /tmp/tmp.Jf6OaSIfG3 /tmp/tmp.ZXfJsjltS7
++ return 0
+ [[ '' == \t\r\u\e ]]
+ wait_pod cross-site-sharded-replica-cfg-2
+ local pod=cross-site-sharded-replica-cfg-2
+ set +o xtrace
waiting for pod/cross-site-sharded-replica-cfg-2 to be ready.OK
++ kubectl_bin get psmdb cross-site-sharded-replica -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].non_voting.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.vgYvL5JQhq
+++ mktemp
++ local LAST_ERR=/tmp/tmp.BialO1zEie
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb cross-site-sharded-replica -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].non_voting.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.vgYvL5JQhq
++ cat /tmp/tmp.BialO1zEie
++ rm /tmp/tmp.vgYvL5JQhq /tmp/tmp.BialO1zEie
++ return 0
+ [[ '' == \t\r\u\e ]]
+ sleep 10
+ [[ false == \t\r\u\e ]]
++ get_service_ip cross-site-sharded-replica-cfg-0 cfg
++ local service=cross-site-sharded-replica-cfg-0
++ local server_type=cfg
+++ kubectl_bin get psmdb/cross-site-sharded-replica -o 'jsonpath={.spec.replsets[].expose.enabled}'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.A7DygCZ70E
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.VhE39hgtJ2
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in '$(seq 0 2)'
+++ set +e
+++ kubectl get psmdb/cross-site-sharded-replica -o 'jsonpath={.spec.replsets[].expose.enabled}'
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.A7DygCZ70E
+++ cat /tmp/tmp.VhE39hgtJ2
+++ rm /tmp/tmp.A7DygCZ70E /tmp/tmp.VhE39hgtJ2
+++ return 0
++ '[' true '!=' true ']'
++ kubectl_bin get service/cross-site-sharded-replica-cfg-0 -o 'jsonpath={.spec.type}'
++ grep -q NotFound
+++ kubectl_bin get service/cross-site-sharded-replica-cfg-0 -o 'jsonpath={.spec.type}'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.OL8B7EI3F7
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.ndCMBUcfHi
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in '$(seq 0 2)'
+++ set +e
+++ kubectl get service/cross-site-sharded-replica-cfg-0 -o 'jsonpath={.spec.type}'
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.OL8B7EI3F7
+++ cat /tmp/tmp.ndCMBUcfHi
+++ rm /tmp/tmp.OL8B7EI3F7 /tmp/tmp.ndCMBUcfHi
+++ return 0
++ service_type=ClusterIP
++ '[' ClusterIP = ClusterIP ']'
++ kubectl_bin get service/cross-site-sharded-replica-cfg-0 -o 'jsonpath={.spec.clusterIP}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.yQOERJbvKe
+++ mktemp
++ local LAST_ERR=/tmp/tmp.GPPOQ71Fz4
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get service/cross-site-sharded-replica-cfg-0 -o 'jsonpath={.spec.clusterIP}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.yQOERJbvKe
++ cat /tmp/tmp.GPPOQ71Fz4
++ rm /tmp/tmp.yQOERJbvKe /tmp/tmp.GPPOQ71Fz4
++ return 0
++ return
+ replica_cfg_0_endpoint=10.184.115.45
++ get_service_ip cross-site-sharded-replica-cfg-1 cfg
++ local service=cross-site-sharded-replica-cfg-1
++ local server_type=cfg
+++ kubectl_bin get psmdb/cross-site-sharded-replica -o 'jsonpath={.spec.replsets[].expose.enabled}'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.b5hpuA3rZp
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.Qou7scpyMv
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in '$(seq 0 2)'
+++ set +e
+++ kubectl get psmdb/cross-site-sharded-replica -o 'jsonpath={.spec.replsets[].expose.enabled}'
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.b5hpuA3rZp
+++ cat /tmp/tmp.Qou7scpyMv
+++ rm /tmp/tmp.b5hpuA3rZp /tmp/tmp.Qou7scpyMv
+++ return 0
++ '[' true '!=' true ']'
++ grep -q NotFound
++ kubectl_bin get service/cross-site-sharded-replica-cfg-1 -o 'jsonpath={.spec.type}'
+++ kubectl_bin get service/cross-site-sharded-replica-cfg-1 -o 'jsonpath={.spec.type}'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.b97cvEhBVL
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.N4ATCoaNi5
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in '$(seq 0 2)'
+++ set +e
+++ kubectl get service/cross-site-sharded-replica-cfg-1 -o 'jsonpath={.spec.type}'
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.b97cvEhBVL
+++ cat /tmp/tmp.N4ATCoaNi5
+++ rm /tmp/tmp.b97cvEhBVL /tmp/tmp.N4ATCoaNi5
+++ return 0
++ service_type=ClusterIP
++ '[' ClusterIP = ClusterIP ']'
++ kubectl_bin get service/cross-site-sharded-replica-cfg-1 -o 'jsonpath={.spec.clusterIP}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.iuf3nfObuX
+++ mktemp
++ local LAST_ERR=/tmp/tmp.QapVWKSBlu
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get service/cross-site-sharded-replica-cfg-1 -o 'jsonpath={.spec.clusterIP}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.iuf3nfObuX
++ cat /tmp/tmp.QapVWKSBlu
++ rm /tmp/tmp.iuf3nfObuX /tmp/tmp.QapVWKSBlu
++ return 0
++ return
+ replica_cfg_1_endpoint=10.184.122.111
++ get_service_ip cross-site-sharded-replica-cfg-2 cfg
++ local service=cross-site-sharded-replica-cfg-2
++ local server_type=cfg
+++ kubectl_bin get psmdb/cross-site-sharded-replica -o 'jsonpath={.spec.replsets[].expose.enabled}'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.iOfveEWzqV
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.MZjMDZRG1T
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in '$(seq 0 2)'
+++ set +e
+++ kubectl get psmdb/cross-site-sharded-replica -o 'jsonpath={.spec.replsets[].expose.enabled}'
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.iOfveEWzqV
+++ cat /tmp/tmp.MZjMDZRG1T
+++ rm /tmp/tmp.iOfveEWzqV /tmp/tmp.MZjMDZRG1T
+++ return 0
++ '[' true '!=' true ']'
++ kubectl_bin get service/cross-site-sharded-replica-cfg-2 -o 'jsonpath={.spec.type}'
++ grep -q NotFound
+++ kubectl_bin get service/cross-site-sharded-replica-cfg-2 -o 'jsonpath={.spec.type}'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.PZcw081zh0
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.F1ubeQOwDJ
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in '$(seq 0 2)'
+++ set +e
+++ kubectl get service/cross-site-sharded-replica-cfg-2 -o 'jsonpath={.spec.type}'
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.PZcw081zh0
+++ cat /tmp/tmp.F1ubeQOwDJ
+++ rm /tmp/tmp.PZcw081zh0 /tmp/tmp.F1ubeQOwDJ
+++ return 0
++ service_type=ClusterIP
++ '[' ClusterIP = ClusterIP ']'
++ kubectl_bin get service/cross-site-sharded-replica-cfg-2 -o 'jsonpath={.spec.clusterIP}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.mcGag1co8s
+++ mktemp
++ local LAST_ERR=/tmp/tmp.9AiNjXxzmS
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get service/cross-site-sharded-replica-cfg-2 -o 'jsonpath={.spec.clusterIP}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.mcGag1co8s
++ cat /tmp/tmp.9AiNjXxzmS
++ rm /tmp/tmp.mcGag1co8s /tmp/tmp.9AiNjXxzmS
++ return 0
++ return
+ replica_cfg_2_endpoint=10.184.114.91
++ get_service_ip cross-site-sharded-replica-rs0-0
++ local service=cross-site-sharded-replica-rs0-0
++ local server_type=rs0
+++ kubectl_bin get psmdb/cross-site-sharded-replica -o 'jsonpath={.spec.replsets[].expose.enabled}'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.D3ausu9WC7
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.VS6ZUKa7b6
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in '$(seq 0 2)'
+++ set +e
+++ kubectl get psmdb/cross-site-sharded-replica -o 'jsonpath={.spec.replsets[].expose.enabled}'
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.D3ausu9WC7
+++ cat /tmp/tmp.VS6ZUKa7b6
+++ rm /tmp/tmp.D3ausu9WC7 /tmp/tmp.VS6ZUKa7b6
+++ return 0
++ '[' true '!=' true ']'
++ grep -q NotFound
++ kubectl_bin get service/cross-site-sharded-replica-rs0-0 -o 'jsonpath={.spec.type}'
+++ kubectl_bin get service/cross-site-sharded-replica-rs0-0 -o 'jsonpath={.spec.type}'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.XwfhnjkpkP
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.ogyE02LCXY
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in '$(seq 0 2)'
+++ set +e
+++ kubectl get service/cross-site-sharded-replica-rs0-0 -o 'jsonpath={.spec.type}'
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.XwfhnjkpkP
+++ cat /tmp/tmp.ogyE02LCXY
+++ rm /tmp/tmp.XwfhnjkpkP /tmp/tmp.ogyE02LCXY
+++ return 0
++ service_type=ClusterIP
++ '[' ClusterIP = ClusterIP ']'
++ kubectl_bin get service/cross-site-sharded-replica-rs0-0 -o 'jsonpath={.spec.clusterIP}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.rUxxONWM0X
+++ mktemp
++ local LAST_ERR=/tmp/tmp.bQgkdeH1oO
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get service/cross-site-sharded-replica-rs0-0 -o 'jsonpath={.spec.clusterIP}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.rUxxONWM0X
++ cat /tmp/tmp.bQgkdeH1oO
++ rm /tmp/tmp.rUxxONWM0X /tmp/tmp.bQgkdeH1oO
++ return 0
++ return
+ replica_rs0_0_endpoint=10.184.120.80
++ get_service_ip cross-site-sharded-replica-rs0-1
++ local service=cross-site-sharded-replica-rs0-1
++ local server_type=rs0
+++ kubectl_bin get psmdb/cross-site-sharded-replica -o 'jsonpath={.spec.replsets[].expose.enabled}'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.CPdwpxIgYz
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.LyN8jIFL3P
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in '$(seq 0 2)'
+++ set +e
+++ kubectl get psmdb/cross-site-sharded-replica -o 'jsonpath={.spec.replsets[].expose.enabled}'
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.CPdwpxIgYz
+++ cat /tmp/tmp.LyN8jIFL3P
+++ rm /tmp/tmp.CPdwpxIgYz /tmp/tmp.LyN8jIFL3P
+++ return 0
++ '[' true '!=' true ']'
++ kubectl_bin get service/cross-site-sharded-replica-rs0-1 -o 'jsonpath={.spec.type}'
++ grep -q NotFound
+++ kubectl_bin get service/cross-site-sharded-replica-rs0-1 -o 'jsonpath={.spec.type}'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.tfmThpdW53
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.DaMn9ZrqFx
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in '$(seq 0 2)'
+++ set +e
+++ kubectl get service/cross-site-sharded-replica-rs0-1 -o 'jsonpath={.spec.type}'
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.tfmThpdW53
+++ cat /tmp/tmp.DaMn9ZrqFx
+++ rm /tmp/tmp.tfmThpdW53 /tmp/tmp.DaMn9ZrqFx
+++ return 0
++ service_type=ClusterIP
++ '[' ClusterIP = ClusterIP ']'
++ kubectl_bin get service/cross-site-sharded-replica-rs0-1 -o 'jsonpath={.spec.clusterIP}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.3y0mvPu089
+++ mktemp
++ local LAST_ERR=/tmp/tmp.kpLSVrvT9R
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get service/cross-site-sharded-replica-rs0-1 -o 'jsonpath={.spec.clusterIP}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.3y0mvPu089
++ cat /tmp/tmp.kpLSVrvT9R
++ rm /tmp/tmp.3y0mvPu089 /tmp/tmp.kpLSVrvT9R
++ return 0
++ return
+ replica_rs0_1_endpoint=10.184.122.33
++ get_service_ip cross-site-sharded-replica-rs0-2
++ local service=cross-site-sharded-replica-rs0-2
++ local server_type=rs0
+++ kubectl_bin get psmdb/cross-site-sharded-replica -o 'jsonpath={.spec.replsets[].expose.enabled}'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.7FUvlUTS39
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.B7tFCVN9i6
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in '$(seq 0 2)'
+++ set +e
+++ kubectl get psmdb/cross-site-sharded-replica -o 'jsonpath={.spec.replsets[].expose.enabled}'
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.7FUvlUTS39
+++ cat /tmp/tmp.B7tFCVN9i6
+++ rm /tmp/tmp.7FUvlUTS39 /tmp/tmp.B7tFCVN9i6
+++ return 0
++ '[' true '!=' true ']'
++ kubectl_bin get service/cross-site-sharded-replica-rs0-2 -o 'jsonpath={.spec.type}'
++ grep -q NotFound
+++ kubectl_bin get service/cross-site-sharded-replica-rs0-2 -o 'jsonpath={.spec.type}'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.VrDF9h3Avr
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.e8KR3ElOOe
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in '$(seq 0 2)'
+++ set +e
+++ kubectl get service/cross-site-sharded-replica-rs0-2 -o 'jsonpath={.spec.type}'
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.VrDF9h3Avr
+++ cat /tmp/tmp.e8KR3ElOOe
+++ rm /tmp/tmp.VrDF9h3Avr /tmp/tmp.e8KR3ElOOe
+++ return 0
++ service_type=ClusterIP
++ '[' ClusterIP = ClusterIP ']'
++ kubectl_bin get service/cross-site-sharded-replica-rs0-2 -o 'jsonpath={.spec.clusterIP}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.FUlWjC5rr8
+++ mktemp
++ local LAST_ERR=/tmp/tmp.RB9iU6wIkU
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get service/cross-site-sharded-replica-rs0-2 -o 'jsonpath={.spec.clusterIP}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.FUlWjC5rr8
++ cat /tmp/tmp.RB9iU6wIkU
++ rm /tmp/tmp.FUlWjC5rr8 /tmp/tmp.RB9iU6wIkU
++ return 0
++ return
+ replica_rs0_2_endpoint=10.184.121.71
++ get_service_ip cross-site-sharded-replica-rs1-0 rs1
++ local service=cross-site-sharded-replica-rs1-0
++ local server_type=rs1
+++ kubectl_bin get psmdb/cross-site-sharded-replica -o 'jsonpath={.spec.replsets[].expose.enabled}'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.JldKIeKa2a
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.yfi1fAKQPp
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in '$(seq 0 2)'
+++ set +e
+++ kubectl get psmdb/cross-site-sharded-replica -o 'jsonpath={.spec.replsets[].expose.enabled}'
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.JldKIeKa2a
+++ cat /tmp/tmp.yfi1fAKQPp
+++ rm /tmp/tmp.JldKIeKa2a /tmp/tmp.yfi1fAKQPp
+++ return 0
++ '[' true '!=' true ']'
++ grep -q NotFound
++ kubectl_bin get service/cross-site-sharded-replica-rs1-0 -o 'jsonpath={.spec.type}'
+++ kubectl_bin get service/cross-site-sharded-replica-rs1-0 -o 'jsonpath={.spec.type}'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.fkv9NA0RDw
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.g7dZxZODwX
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in '$(seq 0 2)'
+++ set +e
+++ kubectl get service/cross-site-sharded-replica-rs1-0 -o 'jsonpath={.spec.type}'
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.fkv9NA0RDw
+++ cat /tmp/tmp.g7dZxZODwX
+++ rm /tmp/tmp.fkv9NA0RDw /tmp/tmp.g7dZxZODwX
+++ return 0
++ service_type=ClusterIP
++ '[' ClusterIP = ClusterIP ']'
++ kubectl_bin get service/cross-site-sharded-replica-rs1-0 -o 'jsonpath={.spec.clusterIP}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.wMmGZBboiU
+++ mktemp
++ local LAST_ERR=/tmp/tmp.lANpshfAhO
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get service/cross-site-sharded-replica-rs1-0 -o 'jsonpath={.spec.clusterIP}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.wMmGZBboiU
++ cat /tmp/tmp.lANpshfAhO
++ rm /tmp/tmp.wMmGZBboiU /tmp/tmp.lANpshfAhO
++ return 0
++ return
+ replica_rs1_0_endpoint=10.184.115.170
++ get_service_ip cross-site-sharded-replica-rs1-1 rs1
++ local service=cross-site-sharded-replica-rs1-1
++ local server_type=rs1
+++ kubectl_bin get psmdb/cross-site-sharded-replica -o 'jsonpath={.spec.replsets[].expose.enabled}'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.EFvlRJdYhM
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.1w7zUOkBao
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in '$(seq 0 2)'
+++ set +e
+++ kubectl get psmdb/cross-site-sharded-replica -o 'jsonpath={.spec.replsets[].expose.enabled}'
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.EFvlRJdYhM
+++ cat /tmp/tmp.1w7zUOkBao
+++ rm /tmp/tmp.EFvlRJdYhM /tmp/tmp.1w7zUOkBao
+++ return 0
++ '[' true '!=' true ']'
++ kubectl_bin get service/cross-site-sharded-replica-rs1-1 -o 'jsonpath={.spec.type}'
++ grep -q NotFound
+++ kubectl_bin get service/cross-site-sharded-replica-rs1-1 -o 'jsonpath={.spec.type}'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.NQxLRpBaEs
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.nRkMbvbsgG
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in '$(seq 0 2)'
+++ set +e
+++ kubectl get service/cross-site-sharded-replica-rs1-1 -o 'jsonpath={.spec.type}'
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.NQxLRpBaEs
+++ cat /tmp/tmp.nRkMbvbsgG
+++ rm /tmp/tmp.NQxLRpBaEs /tmp/tmp.nRkMbvbsgG
+++ return 0
++ service_type=ClusterIP
++ '[' ClusterIP = ClusterIP ']'
++ kubectl_bin get service/cross-site-sharded-replica-rs1-1 -o 'jsonpath={.spec.clusterIP}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.oBIujwGucy
+++ mktemp
++ local LAST_ERR=/tmp/tmp.5GGAy9AlpC
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get service/cross-site-sharded-replica-rs1-1 -o 'jsonpath={.spec.clusterIP}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.oBIujwGucy
++ cat /tmp/tmp.5GGAy9AlpC
++ rm /tmp/tmp.oBIujwGucy /tmp/tmp.5GGAy9AlpC
++ return 0
++ return
+ replica_rs1_1_endpoint=10.184.123.240
++ get_service_ip cross-site-sharded-replica-rs1-2 rs1
++ local service=cross-site-sharded-replica-rs1-2
++ local server_type=rs1
+++ kubectl_bin get psmdb/cross-site-sharded-replica -o 'jsonpath={.spec.replsets[].expose.enabled}'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.3xNXOITtdk
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.TXAN4hCqAs
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in '$(seq 0 2)'
+++ set +e
+++ kubectl get psmdb/cross-site-sharded-replica -o 'jsonpath={.spec.replsets[].expose.enabled}'
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.3xNXOITtdk
+++ cat /tmp/tmp.TXAN4hCqAs
+++ rm /tmp/tmp.3xNXOITtdk /tmp/tmp.TXAN4hCqAs
+++ return 0
++ '[' true '!=' true ']'
++ kubectl_bin get service/cross-site-sharded-replica-rs1-2 -o 'jsonpath={.spec.type}'
++ grep -q NotFound
+++ kubectl_bin get service/cross-site-sharded-replica-rs1-2 -o 'jsonpath={.spec.type}'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.OzL4OYs4Wd
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.zamruh3ymT
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in '$(seq 0 2)'
+++ set +e
+++ kubectl get service/cross-site-sharded-replica-rs1-2 -o 'jsonpath={.spec.type}'
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.OzL4OYs4Wd
+++ cat /tmp/tmp.zamruh3ymT
+++ rm /tmp/tmp.OzL4OYs4Wd /tmp/tmp.zamruh3ymT
+++ return 0
++ service_type=ClusterIP
++ '[' ClusterIP = ClusterIP ']'
++ kubectl_bin get service/cross-site-sharded-replica-rs1-2 -o 'jsonpath={.spec.clusterIP}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.PvgZ8D5mmC
+++ mktemp
++ local LAST_ERR=/tmp/tmp.Qs1OcOrb8m
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get service/cross-site-sharded-replica-rs1-2 -o 'jsonpath={.spec.clusterIP}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.PvgZ8D5mmC
++ cat /tmp/tmp.Qs1OcOrb8m
++ rm /tmp/tmp.PvgZ8D5mmC /tmp/tmp.Qs1OcOrb8m
++ return 0
++ return
+ replica_rs1_2_endpoint=10.184.114.107
++ kubectl_bin config current-context
+++ mktemp
++ local LAST_OUT=/tmp/tmp.bL7cZaiRt7
+++ mktemp
++ local LAST_ERR=/tmp/tmp.lKzdxwdBL3
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl config current-context
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.bL7cZaiRt7
++ cat /tmp/tmp.lKzdxwdBL3
++ rm /tmp/tmp.bL7cZaiRt7 /tmp/tmp.lKzdxwdBL3
++ return 0
+ kubectl_bin config set-context gke_cloud-dev-112233_us-central1-a_jen-psmdb-1536-4cc529f8-10-cluster4 --namespace=cross-site-sharded-9438
++ mktemp
+ local LAST_OUT=/tmp/tmp.BoIUrbl0Cz
++ mktemp
+ local LAST_ERR=/tmp/tmp.3njbyeyKYb
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl config set-context gke_cloud-dev-112233_us-central1-a_jen-psmdb-1536-4cc529f8-10-cluster4 --namespace=cross-site-sharded-9438
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.BoIUrbl0Cz
Context "gke_cloud-dev-112233_us-central1-a_jen-psmdb-1536-4cc529f8-10-cluster4" modified.
+ cat /tmp/tmp.3njbyeyKYb
+ rm /tmp/tmp.BoIUrbl0Cz /tmp/tmp.3njbyeyKYb
+ return 0
+ kubectl_bin patch psmdb cross-site-sharded-main --type=merge --patch '{
			"spec": {"replsets":[
			  {"affinity":{"antiAffinityTopologyKey": "none"},"arbiter":{"affinity":{"antiAffinityTopologyKey": "none"},"enabled":false,"size":1},"expose":{"enabled":true,"exposeType":"ClusterIp"},"externalNodes":[{"host":"10.184.120.80","priority":0,"votes":0},{"host":"10.184.122.33","port":27017,"priority":1,"votes":1},{"host":"10.184.121.71", "port":27017,"priority":1,"votes":1}],"name":"rs0","nonvoting":{"affinity":{"antiAffinityTopologyKey":"none"},"enabled":false,"podDisruptionBudget":{"maxUnavailable":1},"resources":{"limits":{"cpu":"300m","memory":"0.5G"},"requests":{"cpu":"300m","memory":"0.5G"}},"size":3,"volumeSpec":{"persistentVolumeClaim":{"resources":{"requests":{"storage":"1Gi"}}}}},"podDisruptionBudget":{"maxUnavailable":1},"resources":{"limits":{"cpu":"300m","memory":"0.5G"},"requests":{"cpu":"300m","memory":"0.5G"}},"size":3,"volumeSpec":{"persistentVolumeClaim":{"resources":{"requests":{"storage":"3Gi"}}}}},
			  {"affinity":{"antiAffinityTopologyKey": "none"},"arbiter":{"affinity":{"antiAffinityTopologyKey": "none"},"enabled":false,"size":1},"expose":{"enabled":true,"exposeType":"ClusterIp"},"externalNodes":[{"host":"10.184.115.170","priority":0,"votes":0},{"host":"10.184.123.240","port":27017,"priority":1,"votes":1},{"host":"10.184.114.107", "port":27017,"priority":1,"votes":1}],"name":"rs1","nonvoting":{"affinity":{"antiAffinityTopologyKey":"none"},"enabled":false,"podDisruptionBudget":{"maxUnavailable":1},"resources":{"limits":{"cpu":"300m","memory":"0.5G"},"requests":{"cpu":"300m","memory":"0.5G"}},"size":3,"volumeSpec":{"persistentVolumeClaim":{"resources":{"requests":{"storage":"1Gi"}}}}},"podDisruptionBudget":{"maxUnavailable":1},"resources":{"limits":{"cpu":"300m","memory":"0.5G"},"requests":{"cpu":"300m","memory":"0.5G"}},"size":3,"volumeSpec":{"persistentVolumeClaim":{"resources":{"requests":{"storage":"3Gi"}}}}}
			  ],
			  "sharding":{"configsvrReplSet":{ "externalNodes": [{"host":"10.184.115.45","priority":1,"votes":1 },{"host":"10.184.122.111", "priority":1,"votes":1},{"host":"10.184.114.91","priority":0,"votes":0}]}}
			  }
		}'
++ mktemp
+ local LAST_OUT=/tmp/tmp.1uzL0P6Dij
++ mktemp
+ local LAST_ERR=/tmp/tmp.AD8W4oF8X1
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl patch psmdb cross-site-sharded-main --type=merge --patch '{
			"spec": {"replsets":[
			  {"affinity":{"antiAffinityTopologyKey": "none"},"arbiter":{"affinity":{"antiAffinityTopologyKey": "none"},"enabled":false,"size":1},"expose":{"enabled":true,"exposeType":"ClusterIp"},"externalNodes":[{"host":"10.184.120.80","priority":0,"votes":0},{"host":"10.184.122.33","port":27017,"priority":1,"votes":1},{"host":"10.184.121.71", "port":27017,"priority":1,"votes":1}],"name":"rs0","nonvoting":{"affinity":{"antiAffinityTopologyKey":"none"},"enabled":false,"podDisruptionBudget":{"maxUnavailable":1},"resources":{"limits":{"cpu":"300m","memory":"0.5G"},"requests":{"cpu":"300m","memory":"0.5G"}},"size":3,"volumeSpec":{"persistentVolumeClaim":{"resources":{"requests":{"storage":"1Gi"}}}}},"podDisruptionBudget":{"maxUnavailable":1},"resources":{"limits":{"cpu":"300m","memory":"0.5G"},"requests":{"cpu":"300m","memory":"0.5G"}},"size":3,"volumeSpec":{"persistentVolumeClaim":{"resources":{"requests":{"storage":"3Gi"}}}}},
			  {"affinity":{"antiAffinityTopologyKey": "none"},"arbiter":{"affinity":{"antiAffinityTopologyKey": "none"},"enabled":false,"size":1},"expose":{"enabled":true,"exposeType":"ClusterIp"},"externalNodes":[{"host":"10.184.115.170","priority":0,"votes":0},{"host":"10.184.123.240","port":27017,"priority":1,"votes":1},{"host":"10.184.114.107", "port":27017,"priority":1,"votes":1}],"name":"rs1","nonvoting":{"affinity":{"antiAffinityTopologyKey":"none"},"enabled":false,"podDisruptionBudget":{"maxUnavailable":1},"resources":{"limits":{"cpu":"300m","memory":"0.5G"},"requests":{"cpu":"300m","memory":"0.5G"}},"size":3,"volumeSpec":{"persistentVolumeClaim":{"resources":{"requests":{"storage":"1Gi"}}}}},"podDisruptionBudget":{"maxUnavailable":1},"resources":{"limits":{"cpu":"300m","memory":"0.5G"},"requests":{"cpu":"300m","memory":"0.5G"}},"size":3,"volumeSpec":{"persistentVolumeClaim":{"resources":{"requests":{"storage":"3Gi"}}}}}
			  ],
			  "sharding":{"configsvrReplSet":{ "externalNodes": [{"host":"10.184.115.45","priority":1,"votes":1 },{"host":"10.184.122.111", "priority":1,"votes":1},{"host":"10.184.114.91","priority":0,"votes":0}]}}
			  }
		}'
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.1uzL0P6Dij
perconaservermongodb.psmdb.percona.com/cross-site-sharded-main patched
+ cat /tmp/tmp.AD8W4oF8X1
+ rm /tmp/tmp.1uzL0P6Dij /tmp/tmp.AD8W4oF8X1
+ return 0
+ wait_for_members 10.184.115.45 cfg
+ local endpoint=10.184.115.45
+ local rsName=cfg
+ local nodes_amount=0
+ [[ 0 == 6 ]]
++ run_mongos 'rs.conf().members.length' clusterAdmin:clusterAdmin123456@10.184.115.45 mongodb :27017
++ local 'command=rs.conf().members.length'
++ local uri=clusterAdmin:clusterAdmin123456@10.184.115.45
++ local driver=mongodb
++ local suffix=:27017
++ egrep -v 'I NETWORK|W NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|bye'
++ /usr/bin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
+++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.xolaZjba7e
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.izwlgjafE7
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in '$(seq 0 2)'
+++ set +e
+++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.xolaZjba7e
+++ cat /tmp/tmp.izwlgjafE7
+++ rm /tmp/tmp.xolaZjba7e /tmp/tmp.izwlgjafE7
+++ return 0
++ local client_container=psmdb-client-5dc94d5b48-wl5cq
++ local mongo_flag=
++ kubectl_bin exec psmdb-client-5dc94d5b48-wl5cq -- bash -c 'printf '\''rs.conf().members.length\n'\'' | mongo mongodb://clusterAdmin:clusterAdmin123456@10.184.115.45:27017/admin '
+++ mktemp
++ local LAST_OUT=/tmp/tmp.y9go6ymVkC
+++ mktemp
++ local LAST_ERR=/tmp/tmp.ZoOVUG3zjA
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl exec psmdb-client-5dc94d5b48-wl5cq -- bash -c 'printf '\''rs.conf().members.length\n'\'' | mongo mongodb://clusterAdmin:clusterAdmin123456@10.184.115.45:27017/admin '
++ exit_status=1
++ set -e
++ '[' 1 '!=' 0 -a -n 1 ']'
++ cat /tmp/tmp.y9go6ymVkC
++ cat /tmp/tmp.ZoOVUG3zjA
command terminated with exit code 1
++ sleep 0
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl exec psmdb-client-5dc94d5b48-wl5cq -- bash -c 'printf '\''rs.conf().members.length\n'\'' | mongo mongodb://clusterAdmin:clusterAdmin123456@10.184.115.45:27017/admin '
++ exit_status=1
++ set -e
++ '[' 1 '!=' 0 -a -n 1 ']'
++ cat /tmp/tmp.y9go6ymVkC
++ cat /tmp/tmp.ZoOVUG3zjA
command terminated with exit code 1
++ sleep 4
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl exec psmdb-client-5dc94d5b48-wl5cq -- bash -c 'printf '\''rs.conf().members.length\n'\'' | mongo mongodb://clusterAdmin:clusterAdmin123456@10.184.115.45:27017/admin '
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.y9go6ymVkC
++ cat /tmp/tmp.ZoOVUG3zjA
++ rm /tmp/tmp.y9go6ymVkC /tmp/tmp.ZoOVUG3zjA
++ return 0
+ nodes_amount='Error: Authentication failed. :
connect@src/mongo/shell/mongo.js:374:17
@(connect):2:6
exception: connect failed
exiting with code 1
Error: Authentication failed. :
connect@src/mongo/shell/mongo.js:374:17
@(connect):2:6
exception: connect failed
exiting with code 1
5'
+ echo 'waiting for all members to be configured in cfg'
waiting for all members to be configured in cfg
+ let retry+=1
+ '[' 1 -ge 15 ']'
+ echo -n .
.+ sleep 10
+ [[ Error: Authentication failed. :
connect@src/mongo/shell/mongo.js:374:17
@(connect):2:6
exception: connect failed
exiting with code 1
Error: Authentication failed. :
connect@src/mongo/shell/mongo.js:374:17
@(connect):2:6
exception: connect failed
exiting with code 1
5 == 6 ]]
++ run_mongos 'rs.conf().members.length' clusterAdmin:clusterAdmin123456@10.184.115.45 mongodb :27017
++ local 'command=rs.conf().members.length'
++ local uri=clusterAdmin:clusterAdmin123456@10.184.115.45
++ local driver=mongodb
++ local suffix=:27017
++ egrep -v 'I NETWORK|W NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|bye'
++ /usr/bin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
+++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.eMGnO2El2W
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.bIoN9B7ICE
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in '$(seq 0 2)'
+++ set +e
+++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.eMGnO2El2W
+++ cat /tmp/tmp.bIoN9B7ICE
+++ rm /tmp/tmp.eMGnO2El2W /tmp/tmp.bIoN9B7ICE
+++ return 0
++ local client_container=psmdb-client-5dc94d5b48-wl5cq
++ local mongo_flag=
++ kubectl_bin exec psmdb-client-5dc94d5b48-wl5cq -- bash -c 'printf '\''rs.conf().members.length\n'\'' | mongo mongodb://clusterAdmin:clusterAdmin123456@10.184.115.45:27017/admin '
+++ mktemp
++ local LAST_OUT=/tmp/tmp.iFm8bsITSH
+++ mktemp
++ local LAST_ERR=/tmp/tmp.52tCxZgl6b
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl exec psmdb-client-5dc94d5b48-wl5cq -- bash -c 'printf '\''rs.conf().members.length\n'\'' | mongo mongodb://clusterAdmin:clusterAdmin123456@10.184.115.45:27017/admin '
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.iFm8bsITSH
++ cat /tmp/tmp.52tCxZgl6b
++ rm /tmp/tmp.iFm8bsITSH /tmp/tmp.52tCxZgl6b
++ return 0
+ nodes_amount=6
+ echo 'waiting for all members to be configured in cfg'
waiting for all members to be configured in cfg
+ let retry+=1
+ '[' 2 -ge 15 ']'
+ echo -n .
.+ sleep 10
+ [[ 6 == 6 ]]
+ wait_for_members 10.184.120.80 rs0
+ local endpoint=10.184.120.80
+ local rsName=rs0
+ local nodes_amount=0
+ [[ 0 == 6 ]]
++ run_mongos 'rs.conf().members.length' clusterAdmin:clusterAdmin123456@10.184.120.80 mongodb :27017
++ local 'command=rs.conf().members.length'
++ local uri=clusterAdmin:clusterAdmin123456@10.184.120.80
++ local driver=mongodb
++ egrep -v 'I NETWORK|W NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|bye'
++ local suffix=:27017
++ /usr/bin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
+++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.WfjaZd8XZE
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.sbuxe5HPFp
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in '$(seq 0 2)'
+++ set +e
+++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.WfjaZd8XZE
+++ cat /tmp/tmp.sbuxe5HPFp
+++ rm /tmp/tmp.WfjaZd8XZE /tmp/tmp.sbuxe5HPFp
+++ return 0
++ local client_container=psmdb-client-5dc94d5b48-wl5cq
++ local mongo_flag=
++ kubectl_bin exec psmdb-client-5dc94d5b48-wl5cq -- bash -c 'printf '\''rs.conf().members.length\n'\'' | mongo mongodb://clusterAdmin:clusterAdmin123456@10.184.120.80:27017/admin '
+++ mktemp
++ local LAST_OUT=/tmp/tmp.ZS60kM62He
+++ mktemp
++ local LAST_ERR=/tmp/tmp.wiQwEwypcB
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl exec psmdb-client-5dc94d5b48-wl5cq -- bash -c 'printf '\''rs.conf().members.length\n'\'' | mongo mongodb://clusterAdmin:clusterAdmin123456@10.184.120.80:27017/admin '
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.ZS60kM62He
++ cat /tmp/tmp.wiQwEwypcB
++ rm /tmp/tmp.ZS60kM62He /tmp/tmp.wiQwEwypcB
++ return 0
+ nodes_amount=6
+ echo 'waiting for all members to be configured in rs0'
waiting for all members to be configured in rs0
+ let retry+=1
+ '[' 3 -ge 15 ']'
+ echo -n .
.+ sleep 10
+ [[ 6 == 6 ]]
+ wait_for_members 10.184.115.170 rs1
+ local endpoint=10.184.115.170
+ local rsName=rs1
+ local nodes_amount=0
+ [[ 0 == 6 ]]
++ /usr/bin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
++ egrep -v 'I NETWORK|W NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|bye'
++ run_mongos 'rs.conf().members.length' clusterAdmin:clusterAdmin123456@10.184.115.170 mongodb :27017
++ local 'command=rs.conf().members.length'
++ local uri=clusterAdmin:clusterAdmin123456@10.184.115.170
++ local driver=mongodb
++ local suffix=:27017
+++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.8NZszAfVeK
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.zokpVVx5B3
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in '$(seq 0 2)'
+++ set +e
+++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.8NZszAfVeK
+++ cat /tmp/tmp.zokpVVx5B3
+++ rm /tmp/tmp.8NZszAfVeK /tmp/tmp.zokpVVx5B3
+++ return 0
++ local client_container=psmdb-client-5dc94d5b48-wl5cq
++ local mongo_flag=
++ kubectl_bin exec psmdb-client-5dc94d5b48-wl5cq -- bash -c 'printf '\''rs.conf().members.length\n'\'' | mongo mongodb://clusterAdmin:clusterAdmin123456@10.184.115.170:27017/admin '
+++ mktemp
++ local LAST_OUT=/tmp/tmp.7Vykz3mCxg
+++ mktemp
++ local LAST_ERR=/tmp/tmp.KTtU0RmJjW
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl exec psmdb-client-5dc94d5b48-wl5cq -- bash -c 'printf '\''rs.conf().members.length\n'\'' | mongo mongodb://clusterAdmin:clusterAdmin123456@10.184.115.170:27017/admin '
++ exit_status=1
++ set -e
++ '[' 1 '!=' 0 -a -n 1 ']'
++ cat /tmp/tmp.7Vykz3mCxg
++ cat /tmp/tmp.KTtU0RmJjW
command terminated with exit code 1
++ sleep 0
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl exec psmdb-client-5dc94d5b48-wl5cq -- bash -c 'printf '\''rs.conf().members.length\n'\'' | mongo mongodb://clusterAdmin:clusterAdmin123456@10.184.115.170:27017/admin '
++ exit_status=1
++ set -e
++ '[' 1 '!=' 0 -a -n 1 ']'
++ cat /tmp/tmp.7Vykz3mCxg
++ cat /tmp/tmp.KTtU0RmJjW
command terminated with exit code 1
++ sleep 4
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl exec psmdb-client-5dc94d5b48-wl5cq -- bash -c 'printf '\''rs.conf().members.length\n'\'' | mongo mongodb://clusterAdmin:clusterAdmin123456@10.184.115.170:27017/admin '
++ exit_status=1
++ set -e
++ '[' 1 '!=' 0 -a -n 1 ']'
++ cat /tmp/tmp.7Vykz3mCxg
++ cat /tmp/tmp.KTtU0RmJjW
command terminated with exit code 1
++ sleep 8
++ cat /tmp/tmp.7Vykz3mCxg
++ cat /tmp/tmp.KTtU0RmJjW
command terminated with exit code 1
++ rm /tmp/tmp.7Vykz3mCxg /tmp/tmp.KTtU0RmJjW
++ return 1
+ nodes_amount='Error: The server is in quiesce mode and will shut down :
connect@src/mongo/shell/mongo.js:374:17
@(connect):2:6
exception: connect failed
exiting with code 1
Error: The server is in quiesce mode and will shut down :
connect@src/mongo/shell/mongo.js:374:17
@(connect):2:6
exception: connect failed
exiting with code 1
Error: The server is in quiesce mode and will shut down :
connect@src/mongo/shell/mongo.js:374:17
@(connect):2:6
exception: connect failed
exiting with code 1
Error: The server is in quiesce mode and will shut down :
connect@src/mongo/shell/mongo.js:374:17
@(connect):2:6
exception: connect failed
exiting with code 1'
+ echo 'waiting for all members to be configured in rs1'
waiting for all members to be configured in rs1
+ let retry+=1
+ '[' 4 -ge 15 ']'
+ echo -n .
.+ sleep 10
+ [[ Error: The server is in quiesce mode and will shut down :
connect@src/mongo/shell/mongo.js:374:17
@(connect):2:6
exception: connect failed
exiting with code 1
Error: The server is in quiesce mode and will shut down :
connect@src/mongo/shell/mongo.js:374:17
@(connect):2:6
exception: connect failed
exiting with code 1
Error: The server is in quiesce mode and will shut down :
connect@src/mongo/shell/mongo.js:374:17
@(connect):2:6
exception: connect failed
exiting with code 1
Error: The server is in quiesce mode and will shut down :
connect@src/mongo/shell/mongo.js:374:17
@(connect):2:6
exception: connect failed
exiting with code 1 == 6 ]]
++ run_mongos 'rs.conf().members.length' clusterAdmin:clusterAdmin123456@10.184.115.170 mongodb :27017
++ local 'command=rs.conf().members.length'
++ local uri=clusterAdmin:clusterAdmin123456@10.184.115.170
++ local driver=mongodb
++ local suffix=:27017
++ /usr/bin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
++ egrep -v 'I NETWORK|W NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|bye'
+++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.0oyJ7u6juv
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.R2TezeuKm7
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in '$(seq 0 2)'
+++ set +e
+++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.0oyJ7u6juv
+++ cat /tmp/tmp.R2TezeuKm7
+++ rm /tmp/tmp.0oyJ7u6juv /tmp/tmp.R2TezeuKm7
+++ return 0
++ local client_container=psmdb-client-5dc94d5b48-wl5cq
++ local mongo_flag=
++ kubectl_bin exec psmdb-client-5dc94d5b48-wl5cq -- bash -c 'printf '\''rs.conf().members.length\n'\'' | mongo mongodb://clusterAdmin:clusterAdmin123456@10.184.115.170:27017/admin '
+++ mktemp
++ local LAST_OUT=/tmp/tmp.LvwLzdeyeI
+++ mktemp
++ local LAST_ERR=/tmp/tmp.YCM9zaRi7i
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl exec psmdb-client-5dc94d5b48-wl5cq -- bash -c 'printf '\''rs.conf().members.length\n'\'' | mongo mongodb://clusterAdmin:clusterAdmin123456@10.184.115.170:27017/admin '
++ exit_status=1
++ set -e
++ '[' 1 '!=' 0 -a -n 1 ']'
++ cat /tmp/tmp.LvwLzdeyeI
++ cat /tmp/tmp.YCM9zaRi7i
command terminated with exit code 1
++ sleep 0
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl exec psmdb-client-5dc94d5b48-wl5cq -- bash -c 'printf '\''rs.conf().members.length\n'\'' | mongo mongodb://clusterAdmin:clusterAdmin123456@10.184.115.170:27017/admin '
++ exit_status=1
++ set -e
++ '[' 1 '!=' 0 -a -n 1 ']'
++ cat /tmp/tmp.LvwLzdeyeI
++ cat /tmp/tmp.YCM9zaRi7i
command terminated with exit code 1
++ sleep 4
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl exec psmdb-client-5dc94d5b48-wl5cq -- bash -c 'printf '\''rs.conf().members.length\n'\'' | mongo mongodb://clusterAdmin:clusterAdmin123456@10.184.115.170:27017/admin '
++ exit_status=1
++ set -e
++ '[' 1 '!=' 0 -a -n 1 ']'
++ cat /tmp/tmp.LvwLzdeyeI
++ cat /tmp/tmp.YCM9zaRi7i
command terminated with exit code 1
++ sleep 8
++ cat /tmp/tmp.LvwLzdeyeI
++ cat /tmp/tmp.YCM9zaRi7i
command terminated with exit code 1
++ rm /tmp/tmp.LvwLzdeyeI /tmp/tmp.YCM9zaRi7i
++ return 1
+ nodes_amount='Error: couldn'\''t connect to server 10.184.115.170:27017, connection attempt failed: SocketException: Error connecting to 10.184.115.170:27017 :: caused by :: Connection refused :
connect@src/mongo/shell/mongo.js:374:17
@(connect):2:6
exception: connect failed
exiting with code 1
Error: couldn'\''t connect to server 10.184.115.170:27017, connection attempt failed: SocketException: Error connecting to 10.184.115.170:27017 :: caused by :: Connection refused :
connect@src/mongo/shell/mongo.js:374:17
@(connect):2:6
exception: connect failed
exiting with code 1
Error: couldn'\''t connect to server 10.184.115.170:27017, connection attempt failed: SocketException: Error connecting to 10.184.115.170:27017 :: caused by :: Connection refused :
connect@src/mongo/shell/mongo.js:374:17
@(connect):2:6
exception: connect failed
exiting with code 1
Error: couldn'\''t connect to server 10.184.115.170:27017, connection attempt failed: SocketException: Error connecting to 10.184.115.170:27017 :: caused by :: Connection refused :
connect@src/mongo/shell/mongo.js:374:17
@(connect):2:6
exception: connect failed
exiting with code 1'
+ echo 'waiting for all members to be configured in rs1'
waiting for all members to be configured in rs1
+ let retry+=1
+ '[' 5 -ge 15 ']'
+ echo -n .
.+ sleep 10
+ [[ Error: couldn't connect to server 10.184.115.170:27017, connection attempt failed: SocketException: Error connecting to 10.184.115.170:27017 :: caused by :: Connection refused :
connect@src/mongo/shell/mongo.js:374:17
@(connect):2:6
exception: connect failed
exiting with code 1
Error: couldn't connect to server 10.184.115.170:27017, connection attempt failed: SocketException: Error connecting to 10.184.115.170:27017 :: caused by :: Connection refused :
connect@src/mongo/shell/mongo.js:374:17
@(connect):2:6
exception: connect failed
exiting with code 1
Error: couldn't connect to server 10.184.115.170:27017, connection attempt failed: SocketException: Error connecting to 10.184.115.170:27017 :: caused by :: Connection refused :
connect@src/mongo/shell/mongo.js:374:17
@(connect):2:6
exception: connect failed
exiting with code 1
Error: couldn't connect to server 10.184.115.170:27017, connection attempt failed: SocketException: Error connecting to 10.184.115.170:27017 :: caused by :: Connection refused :
connect@src/mongo/shell/mongo.js:374:17
@(connect):2:6
exception: connect failed
exiting with code 1 == 6 ]]
++ run_mongos 'rs.conf().members.length' clusterAdmin:clusterAdmin123456@10.184.115.170 mongodb :27017
++ local 'command=rs.conf().members.length'
++ local uri=clusterAdmin:clusterAdmin123456@10.184.115.170
++ local driver=mongodb
++ local suffix=:27017
++ egrep -v 'I NETWORK|W NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|bye'
++ /usr/bin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
+++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.1WIPitRhPx
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.2Mxq8nbLDS
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in '$(seq 0 2)'
+++ set +e
+++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.1WIPitRhPx
+++ cat /tmp/tmp.2Mxq8nbLDS
+++ rm /tmp/tmp.1WIPitRhPx /tmp/tmp.2Mxq8nbLDS
+++ return 0
++ local client_container=psmdb-client-5dc94d5b48-wl5cq
++ local mongo_flag=
++ kubectl_bin exec psmdb-client-5dc94d5b48-wl5cq -- bash -c 'printf '\''rs.conf().members.length\n'\'' | mongo mongodb://clusterAdmin:clusterAdmin123456@10.184.115.170:27017/admin '
+++ mktemp
++ local LAST_OUT=/tmp/tmp.nhmKATDpnq
+++ mktemp
++ local LAST_ERR=/tmp/tmp.Q5ECAq0hFX
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl exec psmdb-client-5dc94d5b48-wl5cq -- bash -c 'printf '\''rs.conf().members.length\n'\'' | mongo mongodb://clusterAdmin:clusterAdmin123456@10.184.115.170:27017/admin '
++ exit_status=1
++ set -e
++ '[' 1 '!=' 0 -a -n 1 ']'
++ cat /tmp/tmp.nhmKATDpnq
++ cat /tmp/tmp.Q5ECAq0hFX
command terminated with exit code 1
++ sleep 0
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl exec psmdb-client-5dc94d5b48-wl5cq -- bash -c 'printf '\''rs.conf().members.length\n'\'' | mongo mongodb://clusterAdmin:clusterAdmin123456@10.184.115.170:27017/admin '
++ exit_status=1
++ set -e
++ '[' 1 '!=' 0 -a -n 1 ']'
++ cat /tmp/tmp.nhmKATDpnq
++ cat /tmp/tmp.Q5ECAq0hFX
command terminated with exit code 1
++ sleep 4
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl exec psmdb-client-5dc94d5b48-wl5cq -- bash -c 'printf '\''rs.conf().members.length\n'\'' | mongo mongodb://clusterAdmin:clusterAdmin123456@10.184.115.170:27017/admin '
++ exit_status=1
++ set -e
++ '[' 1 '!=' 0 -a -n 1 ']'
++ cat /tmp/tmp.nhmKATDpnq
++ cat /tmp/tmp.Q5ECAq0hFX
command terminated with exit code 1
++ sleep 8
++ cat /tmp/tmp.nhmKATDpnq
++ cat /tmp/tmp.Q5ECAq0hFX
command terminated with exit code 1
++ rm /tmp/tmp.nhmKATDpnq /tmp/tmp.Q5ECAq0hFX
++ return 1
+ nodes_amount='Error: couldn'\''t connect to server 10.184.115.170:27017, connection attempt failed: SocketException: Error connecting to 10.184.115.170:27017 :: caused by :: Connection refused :
connect@src/mongo/shell/mongo.js:374:17
@(connect):2:6
exception: connect failed
exiting with code 1
Error: couldn'\''t connect to server 10.184.115.170:27017, connection attempt failed: SocketException: Error connecting to 10.184.115.170:27017 :: caused by :: Connection refused :
connect@src/mongo/shell/mongo.js:374:17
@(connect):2:6
exception: connect failed
exiting with code 1
Error: couldn'\''t connect to server 10.184.115.170:27017, connection attempt failed: SocketException: Error connecting to 10.184.115.170:27017 :: caused by :: Connection refused :
connect@src/mongo/shell/mongo.js:374:17
@(connect):2:6
exception: connect failed
exiting with code 1
Error: couldn'\''t connect to server 10.184.115.170:27017, connection attempt failed: SocketException: Error connecting to 10.184.115.170:27017 :: caused by :: Connection refused :
connect@src/mongo/shell/mongo.js:374:17
@(connect):2:6
exception: connect failed
exiting with code 1'
+ echo 'waiting for all members to be configured in rs1'
waiting for all members to be configured in rs1
+ let retry+=1
+ '[' 6 -ge 15 ']'
+ echo -n .
.+ sleep 10
+ [[ Error: couldn't connect to server 10.184.115.170:27017, connection attempt failed: SocketException: Error connecting to 10.184.115.170:27017 :: caused by :: Connection refused :
connect@src/mongo/shell/mongo.js:374:17
@(connect):2:6
exception: connect failed
exiting with code 1
Error: couldn't connect to server 10.184.115.170:27017, connection attempt failed: SocketException: Error connecting to 10.184.115.170:27017 :: caused by :: Connection refused :
connect@src/mongo/shell/mongo.js:374:17
@(connect):2:6
exception: connect failed
exiting with code 1
Error: couldn't connect to server 10.184.115.170:27017, connection attempt failed: SocketException: Error connecting to 10.184.115.170:27017 :: caused by :: Connection refused :
connect@src/mongo/shell/mongo.js:374:17
@(connect):2:6
exception: connect failed
exiting with code 1
Error: couldn't connect to server 10.184.115.170:27017, connection attempt failed: SocketException: Error connecting to 10.184.115.170:27017 :: caused by :: Connection refused :
connect@src/mongo/shell/mongo.js:374:17
@(connect):2:6
exception: connect failed
exiting with code 1 == 6 ]]
++ run_mongos 'rs.conf().members.length' clusterAdmin:clusterAdmin123456@10.184.115.170 mongodb :27017
++ egrep -v 'I NETWORK|W NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|bye'
++ local 'command=rs.conf().members.length'
++ local uri=clusterAdmin:clusterAdmin123456@10.184.115.170
++ local driver=mongodb
++ local suffix=:27017
++ /usr/bin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
+++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.18DdCllT0z
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.LpXXeq2aN6
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in '$(seq 0 2)'
+++ set +e
+++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.18DdCllT0z
+++ cat /tmp/tmp.LpXXeq2aN6
+++ rm /tmp/tmp.18DdCllT0z /tmp/tmp.LpXXeq2aN6
+++ return 0
++ local client_container=psmdb-client-5dc94d5b48-wl5cq
++ local mongo_flag=
++ kubectl_bin exec psmdb-client-5dc94d5b48-wl5cq -- bash -c 'printf '\''rs.conf().members.length\n'\'' | mongo mongodb://clusterAdmin:clusterAdmin123456@10.184.115.170:27017/admin '
+++ mktemp
++ local LAST_OUT=/tmp/tmp.XLWHd1XkhW
+++ mktemp
++ local LAST_ERR=/tmp/tmp.f3Wpnsqfcd
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl exec psmdb-client-5dc94d5b48-wl5cq -- bash -c 'printf '\''rs.conf().members.length\n'\'' | mongo mongodb://clusterAdmin:clusterAdmin123456@10.184.115.170:27017/admin '
++ exit_status=1
++ set -e
++ '[' 1 '!=' 0 -a -n 1 ']'
++ cat /tmp/tmp.XLWHd1XkhW
++ cat /tmp/tmp.f3Wpnsqfcd
command terminated with exit code 1
++ sleep 0
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl exec psmdb-client-5dc94d5b48-wl5cq -- bash -c 'printf '\''rs.conf().members.length\n'\'' | mongo mongodb://clusterAdmin:clusterAdmin123456@10.184.115.170:27017/admin '
++ exit_status=1
++ set -e
++ '[' 1 '!=' 0 -a -n 1 ']'
++ cat /tmp/tmp.XLWHd1XkhW
++ cat /tmp/tmp.f3Wpnsqfcd
command terminated with exit code 1
++ sleep 4
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl exec psmdb-client-5dc94d5b48-wl5cq -- bash -c 'printf '\''rs.conf().members.length\n'\'' | mongo mongodb://clusterAdmin:clusterAdmin123456@10.184.115.170:27017/admin '
++ exit_status=1
++ set -e
++ '[' 1 '!=' 0 -a -n 1 ']'
++ cat /tmp/tmp.XLWHd1XkhW
++ cat /tmp/tmp.f3Wpnsqfcd
command terminated with exit code 1
++ sleep 8
++ cat /tmp/tmp.XLWHd1XkhW
++ cat /tmp/tmp.f3Wpnsqfcd
command terminated with exit code 1
++ rm /tmp/tmp.XLWHd1XkhW /tmp/tmp.f3Wpnsqfcd
++ return 1
+ nodes_amount='Error: couldn'\''t connect to server 10.184.115.170:27017, connection attempt failed: SocketException: Error connecting to 10.184.115.170:27017 :: caused by :: Connection refused :
connect@src/mongo/shell/mongo.js:374:17
@(connect):2:6
exception: connect failed
exiting with code 1
Error: couldn'\''t connect to server 10.184.115.170:27017, connection attempt failed: SocketException: Error connecting to 10.184.115.170:27017 :: caused by :: Connection refused :
connect@src/mongo/shell/mongo.js:374:17
@(connect):2:6
exception: connect failed
exiting with code 1
Error: couldn'\''t connect to server 10.184.115.170:27017, connection attempt failed: SocketException: Error connecting to 10.184.115.170:27017 :: caused by :: Connection refused :
connect@src/mongo/shell/mongo.js:374:17
@(connect):2:6
exception: connect failed
exiting with code 1
Error: couldn'\''t connect to server 10.184.115.170:27017, connection attempt failed: SocketException: Error connecting to 10.184.115.170:27017 :: caused by :: Connection refused :
connect@src/mongo/shell/mongo.js:374:17
@(connect):2:6
exception: connect failed
exiting with code 1'
+ echo 'waiting for all members to be configured in rs1'
waiting for all members to be configured in rs1
+ let retry+=1
+ '[' 7 -ge 15 ']'
+ echo -n .
.+ sleep 10
+ [[ Error: couldn't connect to server 10.184.115.170:27017, connection attempt failed: SocketException: Error connecting to 10.184.115.170:27017 :: caused by :: Connection refused :
connect@src/mongo/shell/mongo.js:374:17
@(connect):2:6
exception: connect failed
exiting with code 1
Error: couldn't connect to server 10.184.115.170:27017, connection attempt failed: SocketException: Error connecting to 10.184.115.170:27017 :: caused by :: Connection refused :
connect@src/mongo/shell/mongo.js:374:17
@(connect):2:6
exception: connect failed
exiting with code 1
Error: couldn't connect to server 10.184.115.170:27017, connection attempt failed: SocketException: Error connecting to 10.184.115.170:27017 :: caused by :: Connection refused :
connect@src/mongo/shell/mongo.js:374:17
@(connect):2:6
exception: connect failed
exiting with code 1
Error: couldn't connect to server 10.184.115.170:27017, connection attempt failed: SocketException: Error connecting to 10.184.115.170:27017 :: caused by :: Connection refused :
connect@src/mongo/shell/mongo.js:374:17
@(connect):2:6
exception: connect failed
exiting with code 1 == 6 ]]
++ run_mongos 'rs.conf().members.length' clusterAdmin:clusterAdmin123456@10.184.115.170 mongodb :27017
++ local 'command=rs.conf().members.length'
++ egrep -v 'I NETWORK|W NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|bye'
++ local uri=clusterAdmin:clusterAdmin123456@10.184.115.170
++ local driver=mongodb
++ local suffix=:27017
++ /usr/bin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
+++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.gDfTQ8YCuz
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.AeuVoaXywB
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in '$(seq 0 2)'
+++ set +e
+++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.gDfTQ8YCuz
+++ cat /tmp/tmp.AeuVoaXywB
+++ rm /tmp/tmp.gDfTQ8YCuz /tmp/tmp.AeuVoaXywB
+++ return 0
++ local client_container=psmdb-client-5dc94d5b48-wl5cq
++ local mongo_flag=
++ kubectl_bin exec psmdb-client-5dc94d5b48-wl5cq -- bash -c 'printf '\''rs.conf().members.length\n'\'' | mongo mongodb://clusterAdmin:clusterAdmin123456@10.184.115.170:27017/admin '
+++ mktemp
++ local LAST_OUT=/tmp/tmp.O7ax19BkeC
+++ mktemp
++ local LAST_ERR=/tmp/tmp.uiQE1yLvr5
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl exec psmdb-client-5dc94d5b48-wl5cq -- bash -c 'printf '\''rs.conf().members.length\n'\'' | mongo mongodb://clusterAdmin:clusterAdmin123456@10.184.115.170:27017/admin '
++ exit_status=1
++ set -e
++ '[' 1 '!=' 0 -a -n 1 ']'
++ cat /tmp/tmp.O7ax19BkeC
++ cat /tmp/tmp.uiQE1yLvr5
command terminated with exit code 1
++ sleep 0
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl exec psmdb-client-5dc94d5b48-wl5cq -- bash -c 'printf '\''rs.conf().members.length\n'\'' | mongo mongodb://clusterAdmin:clusterAdmin123456@10.184.115.170:27017/admin '
++ exit_status=1
++ set -e
++ '[' 1 '!=' 0 -a -n 1 ']'
++ cat /tmp/tmp.O7ax19BkeC
++ cat /tmp/tmp.uiQE1yLvr5
command terminated with exit code 1
++ sleep 4
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl exec psmdb-client-5dc94d5b48-wl5cq -- bash -c 'printf '\''rs.conf().members.length\n'\'' | mongo mongodb://clusterAdmin:clusterAdmin123456@10.184.115.170:27017/admin '
++ exit_status=1
++ set -e
++ '[' 1 '!=' 0 -a -n 1 ']'
++ cat /tmp/tmp.O7ax19BkeC
++ cat /tmp/tmp.uiQE1yLvr5
command terminated with exit code 1
++ sleep 8
++ cat /tmp/tmp.O7ax19BkeC
++ cat /tmp/tmp.uiQE1yLvr5
command terminated with exit code 1
++ rm /tmp/tmp.O7ax19BkeC /tmp/tmp.uiQE1yLvr5
++ return 1
+ nodes_amount='Error: couldn'\''t connect to server 10.184.115.170:27017, connection attempt failed: SocketException: Error connecting to 10.184.115.170:27017 :: caused by :: Connection refused :
connect@src/mongo/shell/mongo.js:374:17
@(connect):2:6
exception: connect failed
exiting with code 1
Error: couldn'\''t connect to server 10.184.115.170:27017, connection attempt failed: SocketException: Error connecting to 10.184.115.170:27017 :: caused by :: Connection refused :
connect@src/mongo/shell/mongo.js:374:17
@(connect):2:6
exception: connect failed
exiting with code 1
Error: couldn'\''t connect to server 10.184.115.170:27017, connection attempt failed: SocketException: Error connecting to 10.184.115.170:27017 :: caused by :: Connection refused :
connect@src/mongo/shell/mongo.js:374:17
@(connect):2:6
exception: connect failed
exiting with code 1
Error: couldn'\''t connect to server 10.184.115.170:27017, connection attempt failed: SocketException: Error connecting to 10.184.115.170:27017 :: caused by :: Connection refused :
connect@src/mongo/shell/mongo.js:374:17
@(connect):2:6
exception: connect failed
exiting with code 1'
+ echo 'waiting for all members to be configured in rs1'
waiting for all members to be configured in rs1
+ let retry+=1
+ '[' 8 -ge 15 ']'
+ echo -n .
.+ sleep 10
+ [[ Error: couldn't connect to server 10.184.115.170:27017, connection attempt failed: SocketException: Error connecting to 10.184.115.170:27017 :: caused by :: Connection refused :
connect@src/mongo/shell/mongo.js:374:17
@(connect):2:6
exception: connect failed
exiting with code 1
Error: couldn't connect to server 10.184.115.170:27017, connection attempt failed: SocketException: Error connecting to 10.184.115.170:27017 :: caused by :: Connection refused :
connect@src/mongo/shell/mongo.js:374:17
@(connect):2:6
exception: connect failed
exiting with code 1
Error: couldn't connect to server 10.184.115.170:27017, connection attempt failed: SocketException: Error connecting to 10.184.115.170:27017 :: caused by :: Connection refused :
connect@src/mongo/shell/mongo.js:374:17
@(connect):2:6
exception: connect failed
exiting with code 1
Error: couldn't connect to server 10.184.115.170:27017, connection attempt failed: SocketException: Error connecting to 10.184.115.170:27017 :: caused by :: Connection refused :
connect@src/mongo/shell/mongo.js:374:17
@(connect):2:6
exception: connect failed
exiting with code 1 == 6 ]]
++ run_mongos 'rs.conf().members.length' clusterAdmin:clusterAdmin123456@10.184.115.170 mongodb :27017
++ local 'command=rs.conf().members.length'
++ local uri=clusterAdmin:clusterAdmin123456@10.184.115.170
++ local driver=mongodb
++ local suffix=:27017
++ egrep -v 'I NETWORK|W NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|bye'
++ /usr/bin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
+++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.aJAwoKd2bx
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.VE0toAmsqv
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in '$(seq 0 2)'
+++ set +e
+++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.aJAwoKd2bx
+++ cat /tmp/tmp.VE0toAmsqv
+++ rm /tmp/tmp.aJAwoKd2bx /tmp/tmp.VE0toAmsqv
+++ return 0
++ local client_container=psmdb-client-5dc94d5b48-wl5cq
++ local mongo_flag=
++ kubectl_bin exec psmdb-client-5dc94d5b48-wl5cq -- bash -c 'printf '\''rs.conf().members.length\n'\'' | mongo mongodb://clusterAdmin:clusterAdmin123456@10.184.115.170:27017/admin '
+++ mktemp
++ local LAST_OUT=/tmp/tmp.dbhqnn4iyd
+++ mktemp
++ local LAST_ERR=/tmp/tmp.iTdRurZeoF
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl exec psmdb-client-5dc94d5b48-wl5cq -- bash -c 'printf '\''rs.conf().members.length\n'\'' | mongo mongodb://clusterAdmin:clusterAdmin123456@10.184.115.170:27017/admin '
++ exit_status=1
++ set -e
++ '[' 1 '!=' 0 -a -n 1 ']'
++ cat /tmp/tmp.dbhqnn4iyd
++ cat /tmp/tmp.iTdRurZeoF
command terminated with exit code 1
++ sleep 0
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl exec psmdb-client-5dc94d5b48-wl5cq -- bash -c 'printf '\''rs.conf().members.length\n'\'' | mongo mongodb://clusterAdmin:clusterAdmin123456@10.184.115.170:27017/admin '
++ exit_status=1
++ set -e
++ '[' 1 '!=' 0 -a -n 1 ']'
++ cat /tmp/tmp.dbhqnn4iyd
++ cat /tmp/tmp.iTdRurZeoF
command terminated with exit code 1
++ sleep 4
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl exec psmdb-client-5dc94d5b48-wl5cq -- bash -c 'printf '\''rs.conf().members.length\n'\'' | mongo mongodb://clusterAdmin:clusterAdmin123456@10.184.115.170:27017/admin '
++ exit_status=1
++ set -e
++ '[' 1 '!=' 0 -a -n 1 ']'
++ cat /tmp/tmp.dbhqnn4iyd
++ cat /tmp/tmp.iTdRurZeoF
command terminated with exit code 1
++ sleep 8
++ cat /tmp/tmp.dbhqnn4iyd
++ cat /tmp/tmp.iTdRurZeoF
command terminated with exit code 1
++ rm /tmp/tmp.dbhqnn4iyd /tmp/tmp.iTdRurZeoF
++ return 1
+ nodes_amount='Error: couldn'\''t connect to server 10.184.115.170:27017, connection attempt failed: SocketException: Error connecting to 10.184.115.170:27017 :: caused by :: Connection refused :
connect@src/mongo/shell/mongo.js:374:17
@(connect):2:6
exception: connect failed
exiting with code 1
Error: couldn'\''t connect to server 10.184.115.170:27017, connection attempt failed: SocketException: Error connecting to 10.184.115.170:27017 :: caused by :: Connection refused :
connect@src/mongo/shell/mongo.js:374:17
@(connect):2:6
exception: connect failed
exiting with code 1
Error: couldn'\''t connect to server 10.184.115.170:27017, connection attempt failed: SocketException: Error connecting to 10.184.115.170:27017 :: caused by :: Connection refused :
connect@src/mongo/shell/mongo.js:374:17
@(connect):2:6
exception: connect failed
exiting with code 1
Error: couldn'\''t connect to server 10.184.115.170:27017, connection attempt failed: SocketException: Error connecting to 10.184.115.170:27017 :: caused by :: Connection refused :
connect@src/mongo/shell/mongo.js:374:17
@(connect):2:6
exception: connect failed
exiting with code 1'
+ echo 'waiting for all members to be configured in rs1'
waiting for all members to be configured in rs1
+ let retry+=1
+ '[' 9 -ge 15 ']'
+ echo -n .
.+ sleep 10
+ [[ Error: couldn't connect to server 10.184.115.170:27017, connection attempt failed: SocketException: Error connecting to 10.184.115.170:27017 :: caused by :: Connection refused :
connect@src/mongo/shell/mongo.js:374:17
@(connect):2:6
exception: connect failed
exiting with code 1
Error: couldn't connect to server 10.184.115.170:27017, connection attempt failed: SocketException: Error connecting to 10.184.115.170:27017 :: caused by :: Connection refused :
connect@src/mongo/shell/mongo.js:374:17
@(connect):2:6
exception: connect failed
exiting with code 1
Error: couldn't connect to server 10.184.115.170:27017, connection attempt failed: SocketException: Error connecting to 10.184.115.170:27017 :: caused by :: Connection refused :
connect@src/mongo/shell/mongo.js:374:17
@(connect):2:6
exception: connect failed
exiting with code 1
Error: couldn't connect to server 10.184.115.170:27017, connection attempt failed: SocketException: Error connecting to 10.184.115.170:27017 :: caused by :: Connection refused :
connect@src/mongo/shell/mongo.js:374:17
@(connect):2:6
exception: connect failed
exiting with code 1 == 6 ]]
++ run_mongos 'rs.conf().members.length' clusterAdmin:clusterAdmin123456@10.184.115.170 mongodb :27017
++ local 'command=rs.conf().members.length'
++ local uri=clusterAdmin:clusterAdmin123456@10.184.115.170
++ local driver=mongodb
++ local suffix=:27017
++ /usr/bin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
+++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ egrep -v 'I NETWORK|W NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|bye'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.FK7yDSoOrh
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.Qko8b0XxSI
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in '$(seq 0 2)'
+++ set +e
+++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.FK7yDSoOrh
+++ cat /tmp/tmp.Qko8b0XxSI
+++ rm /tmp/tmp.FK7yDSoOrh /tmp/tmp.Qko8b0XxSI
+++ return 0
++ local client_container=psmdb-client-5dc94d5b48-wl5cq
++ local mongo_flag=
++ kubectl_bin exec psmdb-client-5dc94d5b48-wl5cq -- bash -c 'printf '\''rs.conf().members.length\n'\'' | mongo mongodb://clusterAdmin:clusterAdmin123456@10.184.115.170:27017/admin '
+++ mktemp
++ local LAST_OUT=/tmp/tmp.WkCoommziA
+++ mktemp
++ local LAST_ERR=/tmp/tmp.h6RiMw49v3
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl exec psmdb-client-5dc94d5b48-wl5cq -- bash -c 'printf '\''rs.conf().members.length\n'\'' | mongo mongodb://clusterAdmin:clusterAdmin123456@10.184.115.170:27017/admin '
++ exit_status=1
++ set -e
++ '[' 1 '!=' 0 -a -n 1 ']'
++ cat /tmp/tmp.WkCoommziA
++ cat /tmp/tmp.h6RiMw49v3
command terminated with exit code 1
++ sleep 0
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl exec psmdb-client-5dc94d5b48-wl5cq -- bash -c 'printf '\''rs.conf().members.length\n'\'' | mongo mongodb://clusterAdmin:clusterAdmin123456@10.184.115.170:27017/admin '
++ exit_status=1
++ set -e
++ '[' 1 '!=' 0 -a -n 1 ']'
++ cat /tmp/tmp.WkCoommziA
++ cat /tmp/tmp.h6RiMw49v3
command terminated with exit code 1
++ sleep 4
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl exec psmdb-client-5dc94d5b48-wl5cq -- bash -c 'printf '\''rs.conf().members.length\n'\'' | mongo mongodb://clusterAdmin:clusterAdmin123456@10.184.115.170:27017/admin '
++ exit_status=1
++ set -e
++ '[' 1 '!=' 0 -a -n 1 ']'
++ cat /tmp/tmp.WkCoommziA
++ cat /tmp/tmp.h6RiMw49v3
command terminated with exit code 1
++ sleep 8
++ cat /tmp/tmp.WkCoommziA
++ cat /tmp/tmp.h6RiMw49v3
command terminated with exit code 1
++ rm /tmp/tmp.WkCoommziA /tmp/tmp.h6RiMw49v3
++ return 1
+ nodes_amount='Error: couldn'\''t connect to server 10.184.115.170:27017, connection attempt failed: SocketException: Error connecting to 10.184.115.170:27017 :: caused by :: Connection refused :
connect@src/mongo/shell/mongo.js:374:17
@(connect):2:6
exception: connect failed
exiting with code 1
Error: couldn'\''t connect to server 10.184.115.170:27017, connection attempt failed: SocketException: Error connecting to 10.184.115.170:27017 :: caused by :: Connection refused :
connect@src/mongo/shell/mongo.js:374:17
@(connect):2:6
exception: connect failed
exiting with code 1
Error: couldn'\''t connect to server 10.184.115.170:27017, connection attempt failed: SocketException: Error connecting to 10.184.115.170:27017 :: caused by :: Connection refused :
connect@src/mongo/shell/mongo.js:374:17
@(connect):2:6
exception: connect failed
exiting with code 1
Error: couldn'\''t connect to server 10.184.115.170:27017, connection attempt failed: SocketException: Error connecting to 10.184.115.170:27017 :: caused by :: Connection refused :
connect@src/mongo/shell/mongo.js:374:17
@(connect):2:6
exception: connect failed
exiting with code 1'
+ echo 'waiting for all members to be configured in rs1'
waiting for all members to be configured in rs1
+ let retry+=1
+ '[' 10 -ge 15 ']'
+ echo -n .
.+ sleep 10
+ [[ Error: couldn't connect to server 10.184.115.170:27017, connection attempt failed: SocketException: Error connecting to 10.184.115.170:27017 :: caused by :: Connection refused :
connect@src/mongo/shell/mongo.js:374:17
@(connect):2:6
exception: connect failed
exiting with code 1
Error: couldn't connect to server 10.184.115.170:27017, connection attempt failed: SocketException: Error connecting to 10.184.115.170:27017 :: caused by :: Connection refused :
connect@src/mongo/shell/mongo.js:374:17
@(connect):2:6
exception: connect failed
exiting with code 1
Error: couldn't connect to server 10.184.115.170:27017, connection attempt failed: SocketException: Error connecting to 10.184.115.170:27017 :: caused by :: Connection refused :
connect@src/mongo/shell/mongo.js:374:17
@(connect):2:6
exception: connect failed
exiting with code 1
Error: couldn't connect to server 10.184.115.170:27017, connection attempt failed: SocketException: Error connecting to 10.184.115.170:27017 :: caused by :: Connection refused :
connect@src/mongo/shell/mongo.js:374:17
@(connect):2:6
exception: connect failed
exiting with code 1 == 6 ]]
++ /usr/bin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
++ egrep -v 'I NETWORK|W NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|bye'
++ run_mongos 'rs.conf().members.length' clusterAdmin:clusterAdmin123456@10.184.115.170 mongodb :27017
++ local 'command=rs.conf().members.length'
++ local uri=clusterAdmin:clusterAdmin123456@10.184.115.170
++ local driver=mongodb
++ local suffix=:27017
+++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.yx89vYNw2a
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.U18bdVCQo5
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in '$(seq 0 2)'
+++ set +e
+++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.yx89vYNw2a
+++ cat /tmp/tmp.U18bdVCQo5
+++ rm /tmp/tmp.yx89vYNw2a /tmp/tmp.U18bdVCQo5
+++ return 0
++ local client_container=psmdb-client-5dc94d5b48-wl5cq
++ local mongo_flag=
++ kubectl_bin exec psmdb-client-5dc94d5b48-wl5cq -- bash -c 'printf '\''rs.conf().members.length\n'\'' | mongo mongodb://clusterAdmin:clusterAdmin123456@10.184.115.170:27017/admin '
+++ mktemp
++ local LAST_OUT=/tmp/tmp.RdAIVDnlZi
+++ mktemp
++ local LAST_ERR=/tmp/tmp.WK3R4abldp
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl exec psmdb-client-5dc94d5b48-wl5cq -- bash -c 'printf '\''rs.conf().members.length\n'\'' | mongo mongodb://clusterAdmin:clusterAdmin123456@10.184.115.170:27017/admin '
++ exit_status=1
++ set -e
++ '[' 1 '!=' 0 -a -n 1 ']'
++ cat /tmp/tmp.RdAIVDnlZi
++ cat /tmp/tmp.WK3R4abldp
command terminated with exit code 1
++ sleep 0
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl exec psmdb-client-5dc94d5b48-wl5cq -- bash -c 'printf '\''rs.conf().members.length\n'\'' | mongo mongodb://clusterAdmin:clusterAdmin123456@10.184.115.170:27017/admin '
++ exit_status=1
++ set -e
++ '[' 1 '!=' 0 -a -n 1 ']'
++ cat /tmp/tmp.RdAIVDnlZi
++ cat /tmp/tmp.WK3R4abldp
command terminated with exit code 1
++ sleep 4
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl exec psmdb-client-5dc94d5b48-wl5cq -- bash -c 'printf '\''rs.conf().members.length\n'\'' | mongo mongodb://clusterAdmin:clusterAdmin123456@10.184.115.170:27017/admin '
++ exit_status=1
++ set -e
++ '[' 1 '!=' 0 -a -n 1 ']'
++ cat /tmp/tmp.RdAIVDnlZi
++ cat /tmp/tmp.WK3R4abldp
command terminated with exit code 1
++ sleep 8
++ cat /tmp/tmp.RdAIVDnlZi
++ cat /tmp/tmp.WK3R4abldp
command terminated with exit code 1
++ rm /tmp/tmp.RdAIVDnlZi /tmp/tmp.WK3R4abldp
++ return 1
+ nodes_amount='Error: couldn'\''t connect to server 10.184.115.170:27017, connection attempt failed: SocketException: Error connecting to 10.184.115.170:27017 :: caused by :: Connection refused :
connect@src/mongo/shell/mongo.js:374:17
@(connect):2:6
exception: connect failed
exiting with code 1
Error: couldn'\''t connect to server 10.184.115.170:27017, connection attempt failed: SocketException: Error connecting to 10.184.115.170:27017 :: caused by :: Connection refused :
connect@src/mongo/shell/mongo.js:374:17
@(connect):2:6
exception: connect failed
exiting with code 1
Error: couldn'\''t connect to server 10.184.115.170:27017, connection attempt failed: SocketException: Error connecting to 10.184.115.170:27017 :: caused by :: Connection refused :
connect@src/mongo/shell/mongo.js:374:17
@(connect):2:6
exception: connect failed
exiting with code 1
Error: couldn'\''t connect to server 10.184.115.170:27017, connection attempt failed: SocketException: Error connecting to 10.184.115.170:27017 :: caused by :: Connection refused :
connect@src/mongo/shell/mongo.js:374:17
@(connect):2:6
exception: connect failed
exiting with code 1'
+ echo 'waiting for all members to be configured in rs1'
waiting for all members to be configured in rs1
+ let retry+=1
+ '[' 11 -ge 15 ']'
+ echo -n .
.+ sleep 10
+ [[ Error: couldn't connect to server 10.184.115.170:27017, connection attempt failed: SocketException: Error connecting to 10.184.115.170:27017 :: caused by :: Connection refused :
connect@src/mongo/shell/mongo.js:374:17
@(connect):2:6
exception: connect failed
exiting with code 1
Error: couldn't connect to server 10.184.115.170:27017, connection attempt failed: SocketException: Error connecting to 10.184.115.170:27017 :: caused by :: Connection refused :
connect@src/mongo/shell/mongo.js:374:17
@(connect):2:6
exception: connect failed
exiting with code 1
Error: couldn't connect to server 10.184.115.170:27017, connection attempt failed: SocketException: Error connecting to 10.184.115.170:27017 :: caused by :: Connection refused :
connect@src/mongo/shell/mongo.js:374:17
@(connect):2:6
exception: connect failed
exiting with code 1
Error: couldn't connect to server 10.184.115.170:27017, connection attempt failed: SocketException: Error connecting to 10.184.115.170:27017 :: caused by :: Connection refused :
connect@src/mongo/shell/mongo.js:374:17
@(connect):2:6
exception: connect failed
exiting with code 1 == 6 ]]
++ run_mongos 'rs.conf().members.length' clusterAdmin:clusterAdmin123456@10.184.115.170 mongodb :27017
++ local 'command=rs.conf().members.length'
++ local uri=clusterAdmin:clusterAdmin123456@10.184.115.170
++ local driver=mongodb
++ local suffix=:27017
++ /usr/bin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
+++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.HA4QZ6Jv2p
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.FkhdCHtBRa
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
++ egrep -v 'I NETWORK|W NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|bye'
+++ for i in '$(seq 0 2)'
+++ set +e
+++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.HA4QZ6Jv2p
+++ cat /tmp/tmp.FkhdCHtBRa
+++ rm /tmp/tmp.HA4QZ6Jv2p /tmp/tmp.FkhdCHtBRa
+++ return 0
++ local client_container=psmdb-client-5dc94d5b48-wl5cq
++ local mongo_flag=
++ kubectl_bin exec psmdb-client-5dc94d5b48-wl5cq -- bash -c 'printf '\''rs.conf().members.length\n'\'' | mongo mongodb://clusterAdmin:clusterAdmin123456@10.184.115.170:27017/admin '
+++ mktemp
++ local LAST_OUT=/tmp/tmp.STB7BoS6Zp
+++ mktemp
++ local LAST_ERR=/tmp/tmp.AZm8cWxpSq
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl exec psmdb-client-5dc94d5b48-wl5cq -- bash -c 'printf '\''rs.conf().members.length\n'\'' | mongo mongodb://clusterAdmin:clusterAdmin123456@10.184.115.170:27017/admin '
++ exit_status=1
++ set -e
++ '[' 1 '!=' 0 -a -n 1 ']'
++ cat /tmp/tmp.STB7BoS6Zp
++ cat /tmp/tmp.AZm8cWxpSq
command terminated with exit code 1
++ sleep 0
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl exec psmdb-client-5dc94d5b48-wl5cq -- bash -c 'printf '\''rs.conf().members.length\n'\'' | mongo mongodb://clusterAdmin:clusterAdmin123456@10.184.115.170:27017/admin '
++ exit_status=1
++ set -e
++ '[' 1 '!=' 0 -a -n 1 ']'
++ cat /tmp/tmp.STB7BoS6Zp
++ cat /tmp/tmp.AZm8cWxpSq
command terminated with exit code 1
++ sleep 4
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl exec psmdb-client-5dc94d5b48-wl5cq -- bash -c 'printf '\''rs.conf().members.length\n'\'' | mongo mongodb://clusterAdmin:clusterAdmin123456@10.184.115.170:27017/admin '
++ exit_status=1
++ set -e
++ '[' 1 '!=' 0 -a -n 1 ']'
++ cat /tmp/tmp.STB7BoS6Zp
++ cat /tmp/tmp.AZm8cWxpSq
command terminated with exit code 1
++ sleep 8
++ cat /tmp/tmp.STB7BoS6Zp
++ cat /tmp/tmp.AZm8cWxpSq
command terminated with exit code 1
++ rm /tmp/tmp.STB7BoS6Zp /tmp/tmp.AZm8cWxpSq
++ return 1
+ nodes_amount='Error: couldn'\''t connect to server 10.184.115.170:27017, connection attempt failed: SocketException: Error connecting to 10.184.115.170:27017 :: caused by :: Connection refused :
connect@src/mongo/shell/mongo.js:374:17
@(connect):2:6
exception: connect failed
exiting with code 1
Error: couldn'\''t connect to server 10.184.115.170:27017, connection attempt failed: SocketException: Error connecting to 10.184.115.170:27017 :: caused by :: Connection refused :
connect@src/mongo/shell/mongo.js:374:17
@(connect):2:6
exception: connect failed
exiting with code 1
Error: couldn'\''t connect to server 10.184.115.170:27017, connection attempt failed: SocketException: Error connecting to 10.184.115.170:27017 :: caused by :: Connection refused :
connect@src/mongo/shell/mongo.js:374:17
@(connect):2:6
exception: connect failed
exiting with code 1
Error: couldn'\''t connect to server 10.184.115.170:27017, connection attempt failed: SocketException: Error connecting to 10.184.115.170:27017 :: caused by :: Connection refused :
connect@src/mongo/shell/mongo.js:374:17
@(connect):2:6
exception: connect failed
exiting with code 1'
+ echo 'waiting for all members to be configured in rs1'
waiting for all members to be configured in rs1
+ let retry+=1
+ '[' 12 -ge 15 ']'
+ echo -n .
.+ sleep 10
+ [[ Error: couldn't connect to server 10.184.115.170:27017, connection attempt failed: SocketException: Error connecting to 10.184.115.170:27017 :: caused by :: Connection refused :
connect@src/mongo/shell/mongo.js:374:17
@(connect):2:6
exception: connect failed
exiting with code 1
Error: couldn't connect to server 10.184.115.170:27017, connection attempt failed: SocketException: Error connecting to 10.184.115.170:27017 :: caused by :: Connection refused :
connect@src/mongo/shell/mongo.js:374:17
@(connect):2:6
exception: connect failed
exiting with code 1
Error: couldn't connect to server 10.184.115.170:27017, connection attempt failed: SocketException: Error connecting to 10.184.115.170:27017 :: caused by :: Connection refused :
connect@src/mongo/shell/mongo.js:374:17
@(connect):2:6
exception: connect failed
exiting with code 1
Error: couldn't connect to server 10.184.115.170:27017, connection attempt failed: SocketException: Error connecting to 10.184.115.170:27017 :: caused by :: Connection refused :
connect@src/mongo/shell/mongo.js:374:17
@(connect):2:6
exception: connect failed
exiting with code 1 == 6 ]]
++ run_mongos 'rs.conf().members.length' clusterAdmin:clusterAdmin123456@10.184.115.170 mongodb :27017
++ local 'command=rs.conf().members.length'
++ local uri=clusterAdmin:clusterAdmin123456@10.184.115.170
++ local driver=mongodb
++ local suffix=:27017
++ egrep -v 'I NETWORK|W NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|bye'
++ /usr/bin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
+++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.juBWHdS28e
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.WOl2DUicTS
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in '$(seq 0 2)'
+++ set +e
+++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.juBWHdS28e
+++ cat /tmp/tmp.WOl2DUicTS
+++ rm /tmp/tmp.juBWHdS28e /tmp/tmp.WOl2DUicTS
+++ return 0
++ local client_container=psmdb-client-5dc94d5b48-wl5cq
++ local mongo_flag=
++ kubectl_bin exec psmdb-client-5dc94d5b48-wl5cq -- bash -c 'printf '\''rs.conf().members.length\n'\'' | mongo mongodb://clusterAdmin:clusterAdmin123456@10.184.115.170:27017/admin '
+++ mktemp
++ local LAST_OUT=/tmp/tmp.N1oi7EXbqJ
+++ mktemp
++ local LAST_ERR=/tmp/tmp.pkR5Gb1DmP
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl exec psmdb-client-5dc94d5b48-wl5cq -- bash -c 'printf '\''rs.conf().members.length\n'\'' | mongo mongodb://clusterAdmin:clusterAdmin123456@10.184.115.170:27017/admin '
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.N1oi7EXbqJ
++ cat /tmp/tmp.pkR5Gb1DmP
++ rm /tmp/tmp.N1oi7EXbqJ /tmp/tmp.pkR5Gb1DmP
++ return 0
+ nodes_amount=6
+ echo 'waiting for all members to be configured in rs1'
waiting for all members to be configured in rs1
+ let retry+=1
+ '[' 13 -ge 15 ']'
+ echo -n .
.+ sleep 10
+ [[ 6 == 6 ]]
++ kubectl_bin config current-context
+++ mktemp
++ local LAST_OUT=/tmp/tmp.Slm68g2wo3
+++ mktemp
++ local LAST_ERR=/tmp/tmp.IZVkciV0qG
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl config current-context
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.Slm68g2wo3
++ cat /tmp/tmp.IZVkciV0qG
++ rm /tmp/tmp.Slm68g2wo3 /tmp/tmp.IZVkciV0qG
++ return 0
+ kubectl_bin config set-context gke_cloud-dev-112233_us-central1-a_jen-psmdb-1536-4cc529f8-10-cluster4 --namespace=cross-site-sharded-replica-31492
++ mktemp
+ local LAST_OUT=/tmp/tmp.PGf5D2Alm7
++ mktemp
+ local LAST_ERR=/tmp/tmp.dB9NUtjliF
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl config set-context gke_cloud-dev-112233_us-central1-a_jen-psmdb-1536-4cc529f8-10-cluster4 --namespace=cross-site-sharded-replica-31492
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.PGf5D2Alm7
Context "gke_cloud-dev-112233_us-central1-a_jen-psmdb-1536-4cc529f8-10-cluster4" modified.
+ cat /tmp/tmp.dB9NUtjliF
+ rm /tmp/tmp.PGf5D2Alm7 /tmp/tmp.dB9NUtjliF
+ return 0
+ desc 'check if all 3 Pods started'
+ set +o xtrace


-----------------------------------------------------------------------------------
check if all 3 Pods started
-----------------------------------------------------------------------------------

+ wait_for_running cross-site-sharded-replica-rs0 3
+ local name=cross-site-sharded-replica-rs0
+ let last_pod=2
+ local check_cluster_readyness=true
+ set_debug
+ [[ 1 == 1 ]]
+ set -o xtrace
+ local rs_name=rs0
+ local cluster_name=cross-site-sharded-replica
++ seq 0 2
+ for i in '$(seq 0 $last_pod)'
+ [[ 0 -eq 2 ]]
+ wait_pod cross-site-sharded-replica-rs0-0
+ local pod=cross-site-sharded-replica-rs0-0
+ set +o xtrace
waiting for pod/cross-site-sharded-replica-rs0-0 to be ready.OK
+ for i in '$(seq 0 $last_pod)'
+ [[ 1 -eq 2 ]]
+ wait_pod cross-site-sharded-replica-rs0-1
+ local pod=cross-site-sharded-replica-rs0-1
+ set +o xtrace
waiting for pod/cross-site-sharded-replica-rs0-1 to be ready.OK
+ for i in '$(seq 0 $last_pod)'
+ [[ 2 -eq 2 ]]
++ kubectl_bin get psmdb cross-site-sharded-replica -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].arbiter.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.GbgqHWCBe3
+++ mktemp
++ local LAST_ERR=/tmp/tmp.6rA3KbyQyF
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb cross-site-sharded-replica -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].arbiter.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.GbgqHWCBe3
++ cat /tmp/tmp.6rA3KbyQyF
++ rm /tmp/tmp.GbgqHWCBe3 /tmp/tmp.6rA3KbyQyF
++ return 0
+ [[ false == \t\r\u\e ]]
+ wait_pod cross-site-sharded-replica-rs0-2
+ local pod=cross-site-sharded-replica-rs0-2
+ set +o xtrace
waiting for pod/cross-site-sharded-replica-rs0-2 to be ready.OK
++ kubectl_bin get psmdb cross-site-sharded-replica -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].non_voting.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.t4QrQ6kpzJ
+++ mktemp
++ local LAST_ERR=/tmp/tmp.CCcys0XTlk
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb cross-site-sharded-replica -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].non_voting.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.t4QrQ6kpzJ
++ cat /tmp/tmp.CCcys0XTlk
++ rm /tmp/tmp.t4QrQ6kpzJ /tmp/tmp.CCcys0XTlk
++ return 0
+ [[ '' == \t\r\u\e ]]
+ sleep 10
+ [[ true == \t\r\u\e ]]
+ set +x
Waiting for cluster readyness
+ wait_for_running cross-site-sharded-replica-cfg 3 false
+ local name=cross-site-sharded-replica-cfg
+ let last_pod=2
+ local check_cluster_readyness=false
+ set_debug
+ [[ 1 == 1 ]]
+ set -o xtrace
+ local rs_name=cfg
+ local cluster_name=cross-site-sharded-replica
++ seq 0 2
+ for i in '$(seq 0 $last_pod)'
+ [[ 0 -eq 2 ]]
+ wait_pod cross-site-sharded-replica-cfg-0
+ local pod=cross-site-sharded-replica-cfg-0
+ set +o xtrace
waiting for pod/cross-site-sharded-replica-cfg-0 to be ready.OK
+ for i in '$(seq 0 $last_pod)'
+ [[ 1 -eq 2 ]]
+ wait_pod cross-site-sharded-replica-cfg-1
+ local pod=cross-site-sharded-replica-cfg-1
+ set +o xtrace
waiting for pod/cross-site-sharded-replica-cfg-1 to be ready.OK
+ for i in '$(seq 0 $last_pod)'
+ [[ 2 -eq 2 ]]
++ kubectl_bin get psmdb cross-site-sharded-replica -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].arbiter.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.xPOc8HJHLk
+++ mktemp
++ local LAST_ERR=/tmp/tmp.ddoaRhTVhD
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb cross-site-sharded-replica -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].arbiter.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.xPOc8HJHLk
++ cat /tmp/tmp.ddoaRhTVhD
++ rm /tmp/tmp.xPOc8HJHLk /tmp/tmp.ddoaRhTVhD
++ return 0
+ [[ '' == \t\r\u\e ]]
+ wait_pod cross-site-sharded-replica-cfg-2
+ local pod=cross-site-sharded-replica-cfg-2
+ set +o xtrace
waiting for pod/cross-site-sharded-replica-cfg-2 to be ready.OK
++ kubectl_bin get psmdb cross-site-sharded-replica -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].non_voting.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.JZNfihIte4
+++ mktemp
++ local LAST_ERR=/tmp/tmp.lIDgqS4NWt
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb cross-site-sharded-replica -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].non_voting.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.JZNfihIte4
++ cat /tmp/tmp.lIDgqS4NWt
++ rm /tmp/tmp.JZNfihIte4 /tmp/tmp.lIDgqS4NWt
++ return 0
+ [[ '' == \t\r\u\e ]]
+ sleep 10
+ [[ false == \t\r\u\e ]]
+ desc 'create user'
+ set +o xtrace


-----------------------------------------------------------------------------------
create user
-----------------------------------------------------------------------------------

+ run_mongos 'db.createUser({user:"myApp",pwd:"myPass",roles:[{db:"myApp",role:"readWrite"}]})' userAdmin:userAdmin123456@cross-site-sharded-main-mongos.cross-site-sharded-9438
+ local 'command=db.createUser({user:"myApp",pwd:"myPass",roles:[{db:"myApp",role:"readWrite"}]})'
+ local uri=userAdmin:userAdmin123456@cross-site-sharded-main-mongos.cross-site-sharded-9438
+ local driver=mongodb
+ local suffix=.svc.cluster.local
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.1CIq5LCzfR
+++ mktemp
++ local LAST_ERR=/tmp/tmp.o47ZMuLo95
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.1CIq5LCzfR
++ cat /tmp/tmp.o47ZMuLo95
++ rm /tmp/tmp.1CIq5LCzfR /tmp/tmp.o47ZMuLo95
++ return 0
+ local client_container=psmdb-client-5dc94d5b48-fjk4f
+ local mongo_flag=
+ kubectl_bin exec psmdb-client-5dc94d5b48-fjk4f -- bash -c 'printf '\''db.createUser({user:"myApp",pwd:"myPass",roles:[{db:"myApp",role:"readWrite"}]})\n'\'' | mongo mongodb://userAdmin:userAdmin123456@cross-site-sharded-main-mongos.cross-site-sharded-9438.svc.cluster.local/admin '
++ mktemp
+ local LAST_OUT=/tmp/tmp.G4C9buYhqX
++ mktemp
+ local LAST_ERR=/tmp/tmp.Or6mnboz7c
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl exec psmdb-client-5dc94d5b48-fjk4f -- bash -c 'printf '\''db.createUser({user:"myApp",pwd:"myPass",roles:[{db:"myApp",role:"readWrite"}]})\n'\'' | mongo mongodb://userAdmin:userAdmin123456@cross-site-sharded-main-mongos.cross-site-sharded-9438.svc.cluster.local/admin '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.G4C9buYhqX
Percona Server for MongoDB shell version v4.4.29-28
connecting to: mongodb://cross-site-sharded-main-mongos.cross-site-sharded-9438.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb
Implicit session: session { "id" : UUID("48f3a03c-d03a-4e46-b723-8d8ea3284926") }
Percona Server for MongoDB server version: v7.0.8-5
WARNING: shell and server versions do not match
Successfully added user: {
	"user" : "myApp",
	"roles" : [
		{
			"db" : "myApp",
			"role" : "readWrite"
		}
	]
}
bye
+ cat /tmp/tmp.Or6mnboz7c
+ rm /tmp/tmp.G4C9buYhqX /tmp/tmp.Or6mnboz7c
+ return 0
+ sleep 2
+ desc 'write data, read from all'
+ set +o xtrace


-----------------------------------------------------------------------------------
write data, read from all
-----------------------------------------------------------------------------------

+ run_mongos 'use myApp\n db.test.insert({ x: 100500 })' myApp:myPass@cross-site-sharded-main-mongos.cross-site-sharded-9438
+ local 'command=use myApp\n db.test.insert({ x: 100500 })'
+ local uri=myApp:myPass@cross-site-sharded-main-mongos.cross-site-sharded-9438
+ local driver=mongodb
+ local suffix=.svc.cluster.local
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.QFAzHjILy2
+++ mktemp
++ local LAST_ERR=/tmp/tmp.Rkr0NtLXN1
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.QFAzHjILy2
++ cat /tmp/tmp.Rkr0NtLXN1
++ rm /tmp/tmp.QFAzHjILy2 /tmp/tmp.Rkr0NtLXN1
++ return 0
+ local client_container=psmdb-client-5dc94d5b48-fjk4f
+ local mongo_flag=
+ kubectl_bin exec psmdb-client-5dc94d5b48-fjk4f -- bash -c 'printf '\''use myApp\n db.test.insert({ x: 100500 })\n'\'' | mongo mongodb://myApp:myPass@cross-site-sharded-main-mongos.cross-site-sharded-9438.svc.cluster.local/admin '
++ mktemp
+ local LAST_OUT=/tmp/tmp.QeMg92RAZ0
++ mktemp
+ local LAST_ERR=/tmp/tmp.yUYEl75PwM
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl exec psmdb-client-5dc94d5b48-fjk4f -- bash -c 'printf '\''use myApp\n db.test.insert({ x: 100500 })\n'\'' | mongo mongodb://myApp:myPass@cross-site-sharded-main-mongos.cross-site-sharded-9438.svc.cluster.local/admin '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.QeMg92RAZ0
Percona Server for MongoDB shell version v4.4.29-28
connecting to: mongodb://cross-site-sharded-main-mongos.cross-site-sharded-9438.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb
Implicit session: session { "id" : UUID("a4d881b0-33dc-4823-9bad-4116c032d06b") }
Percona Server for MongoDB server version: v7.0.8-5
WARNING: shell and server versions do not match
switched to db myApp
WriteResult({ "nInserted" : 1 })
bye
+ cat /tmp/tmp.yUYEl75PwM
+ rm /tmp/tmp.QeMg92RAZ0 /tmp/tmp.yUYEl75PwM
+ return 0
+ minikube_sleep
+ sleep_time=10
+ [[ '' == 1 ]]
+ desc 'Compare data'
+ set +o xtrace


-----------------------------------------------------------------------------------
Compare data
-----------------------------------------------------------------------------------

+ compare_mongos_cmd find myApp:myPass@cross-site-sharded-main-mongos.cross-site-sharded-9438
+ local command=find
+ local uri=myApp:myPass@cross-site-sharded-main-mongos.cross-site-sharded-9438
+ local postfix=
+ local suffix=
+ local database=myApp
+ local collection=test
+ run_mongos 'use myApp\n db.test.find()' myApp:myPass@cross-site-sharded-main-mongos.cross-site-sharded-9438 mongodb ''
+ local 'command=use myApp\n db.test.find()'
+ local uri=myApp:myPass@cross-site-sharded-main-mongos.cross-site-sharded-9438
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ egrep -v 'I NETWORK|W NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:'
+ /usr/bin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.RAgHAGfKSl
+++ mktemp
++ local LAST_ERR=/tmp/tmp.ZzlsM7rmCq
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.RAgHAGfKSl
++ cat /tmp/tmp.ZzlsM7rmCq
++ rm /tmp/tmp.RAgHAGfKSl /tmp/tmp.ZzlsM7rmCq
++ return 0
+ local client_container=psmdb-client-5dc94d5b48-fjk4f
+ local mongo_flag=
+ kubectl_bin exec psmdb-client-5dc94d5b48-fjk4f -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@cross-site-sharded-main-mongos.cross-site-sharded-9438.svc.cluster.local/admin '
++ mktemp
+ local LAST_OUT=/tmp/tmp.2DlMQu48Vv
++ mktemp
+ local LAST_ERR=/tmp/tmp.oqd2qPv9PI
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl exec psmdb-client-5dc94d5b48-fjk4f -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@cross-site-sharded-main-mongos.cross-site-sharded-9438.svc.cluster.local/admin '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.2DlMQu48Vv
+ cat /tmp/tmp.oqd2qPv9PI
+ rm /tmp/tmp.2DlMQu48Vv /tmp/tmp.oqd2qPv9PI
+ return 0
+ diff /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1536/e2e-tests/cross-site-sharded/compare/find.json /tmp/tmp.a26yFxZivT/find
+ desc 'test failover'
+ set +o xtrace


-----------------------------------------------------------------------------------
test failover
-----------------------------------------------------------------------------------

++ kubectl_bin config current-context
+++ mktemp
++ local LAST_OUT=/tmp/tmp.BDAMXiSpK0
+++ mktemp
++ local LAST_ERR=/tmp/tmp.nokRKy6Mqq
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl config current-context
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.BDAMXiSpK0
++ cat /tmp/tmp.nokRKy6Mqq
++ rm /tmp/tmp.BDAMXiSpK0 /tmp/tmp.nokRKy6Mqq
++ return 0
+ kubectl_bin config set-context gke_cloud-dev-112233_us-central1-a_jen-psmdb-1536-4cc529f8-10-cluster4 --namespace=cross-site-sharded-9438
++ mktemp
+ local LAST_OUT=/tmp/tmp.HPZM9St5sQ
++ mktemp
+ local LAST_ERR=/tmp/tmp.tRNoNGqMW1
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl config set-context gke_cloud-dev-112233_us-central1-a_jen-psmdb-1536-4cc529f8-10-cluster4 --namespace=cross-site-sharded-9438
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.HPZM9St5sQ
Context "gke_cloud-dev-112233_us-central1-a_jen-psmdb-1536-4cc529f8-10-cluster4" modified.
+ cat /tmp/tmp.tRNoNGqMW1
+ rm /tmp/tmp.HPZM9St5sQ /tmp/tmp.tRNoNGqMW1
+ return 0
+ kubectl_bin delete psmdb cross-site-sharded-main
++ mktemp
+ local LAST_OUT=/tmp/tmp.bPfgx0wP95
++ mktemp
+ local LAST_ERR=/tmp/tmp.Xzzt3cKsqT
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl delete psmdb cross-site-sharded-main
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.bPfgx0wP95
perconaservermongodb.psmdb.percona.com "cross-site-sharded-main" deleted
+ cat /tmp/tmp.Xzzt3cKsqT
+ rm /tmp/tmp.bPfgx0wP95 /tmp/tmp.Xzzt3cKsqT
+ return 0
+ desc 'run disaster recovery script for replset: cfg'
+ set +o xtrace


-----------------------------------------------------------------------------------
run disaster recovery script for replset: cfg
-----------------------------------------------------------------------------------

+ run_script_mongos /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1536/e2e-tests/cross-site-sharded/disaster_recovery.js clusterAdmin:clusterAdmin123456@10.184.115.45 mongodb :27017
+ local script=/mnt/jenkins/workspace/cloud-psmdb-operator_PR-1536/e2e-tests/cross-site-sharded/disaster_recovery.js
+ local uri=clusterAdmin:clusterAdmin123456@10.184.115.45
+ local driver=mongodb
+ local suffix=:27017
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.7I3UN4GJ7E
+++ mktemp
++ local LAST_ERR=/tmp/tmp.p1IzlCo73V
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.7I3UN4GJ7E
++ cat /tmp/tmp.p1IzlCo73V
++ rm /tmp/tmp.7I3UN4GJ7E /tmp/tmp.p1IzlCo73V
++ return 0
+ local client_container=psmdb-client-5dc94d5b48-wl5cq
+ local mongo_flag=
++ basename /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1536/e2e-tests/cross-site-sharded/disaster_recovery.js
+ name=disaster_recovery.js
+ kubectl_bin cp /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1536/e2e-tests/cross-site-sharded/disaster_recovery.js cross-site-sharded-9438/psmdb-client-5dc94d5b48-wl5cq:/tmp
++ mktemp
+ local LAST_OUT=/tmp/tmp.vCpTBeV5WQ
++ mktemp
+ local LAST_ERR=/tmp/tmp.NeOLKhopQS
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl cp /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1536/e2e-tests/cross-site-sharded/disaster_recovery.js cross-site-sharded-9438/psmdb-client-5dc94d5b48-wl5cq:/tmp
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.vCpTBeV5WQ
+ cat /tmp/tmp.NeOLKhopQS
+ rm /tmp/tmp.vCpTBeV5WQ /tmp/tmp.NeOLKhopQS
+ return 0
+ kubectl_bin exec psmdb-client-5dc94d5b48-wl5cq -- bash -c 'mongo mongodb://clusterAdmin:clusterAdmin123456@10.184.115.45:27017/admin  /tmp/disaster_recovery.js'
++ mktemp
+ local LAST_OUT=/tmp/tmp.iPwGUC0UIK
++ mktemp
+ local LAST_ERR=/tmp/tmp.wtunXeyHOL
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl exec psmdb-client-5dc94d5b48-wl5cq -- bash -c 'mongo mongodb://clusterAdmin:clusterAdmin123456@10.184.115.45:27017/admin  /tmp/disaster_recovery.js'
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.iPwGUC0UIK
Percona Server for MongoDB shell version v4.4.29-28
connecting to: mongodb://10.184.115.45:27017/admin?compressors=disabled&gssapiServiceName=mongodb
Implicit session: session { "id" : UUID("65b1888b-9e2f-457e-a403-4ab174caba55") }
Percona Server for MongoDB server version: v7.0.8-5
WARNING: shell and server versions do not match
+ cat /tmp/tmp.wtunXeyHOL
+ rm /tmp/tmp.iPwGUC0UIK /tmp/tmp.wtunXeyHOL
+ return 0
+ desc 'run disaster recovery script for replset: rs0'
+ set +o xtrace


-----------------------------------------------------------------------------------
run disaster recovery script for replset: rs0
-----------------------------------------------------------------------------------

+ run_script_mongos /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1536/e2e-tests/cross-site-sharded/disaster_recovery.js clusterAdmin:clusterAdmin123456@10.184.120.80 mongodb :27017
+ local script=/mnt/jenkins/workspace/cloud-psmdb-operator_PR-1536/e2e-tests/cross-site-sharded/disaster_recovery.js
+ local uri=clusterAdmin:clusterAdmin123456@10.184.120.80
+ local driver=mongodb
+ local suffix=:27017
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.cYIjMQYY23
+++ mktemp
++ local LAST_ERR=/tmp/tmp.sNEZkL4LOr
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.cYIjMQYY23
++ cat /tmp/tmp.sNEZkL4LOr
++ rm /tmp/tmp.cYIjMQYY23 /tmp/tmp.sNEZkL4LOr
++ return 0
+ local client_container=psmdb-client-5dc94d5b48-wl5cq
+ local mongo_flag=
++ basename /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1536/e2e-tests/cross-site-sharded/disaster_recovery.js
+ name=disaster_recovery.js
+ kubectl_bin cp /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1536/e2e-tests/cross-site-sharded/disaster_recovery.js cross-site-sharded-9438/psmdb-client-5dc94d5b48-wl5cq:/tmp
++ mktemp
+ local LAST_OUT=/tmp/tmp.dXVqjXpWHv
++ mktemp
+ local LAST_ERR=/tmp/tmp.FOxRQN8gqp
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl cp /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1536/e2e-tests/cross-site-sharded/disaster_recovery.js cross-site-sharded-9438/psmdb-client-5dc94d5b48-wl5cq:/tmp
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.dXVqjXpWHv
+ cat /tmp/tmp.FOxRQN8gqp
+ rm /tmp/tmp.dXVqjXpWHv /tmp/tmp.FOxRQN8gqp
+ return 0
+ kubectl_bin exec psmdb-client-5dc94d5b48-wl5cq -- bash -c 'mongo mongodb://clusterAdmin:clusterAdmin123456@10.184.120.80:27017/admin  /tmp/disaster_recovery.js'
++ mktemp
+ local LAST_OUT=/tmp/tmp.rYq72onT6T
++ mktemp
+ local LAST_ERR=/tmp/tmp.QsRnWt7FPS
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl exec psmdb-client-5dc94d5b48-wl5cq -- bash -c 'mongo mongodb://clusterAdmin:clusterAdmin123456@10.184.120.80:27017/admin  /tmp/disaster_recovery.js'
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.rYq72onT6T
Percona Server for MongoDB shell version v4.4.29-28
connecting to: mongodb://10.184.120.80:27017/admin?compressors=disabled&gssapiServiceName=mongodb
Implicit session: session { "id" : UUID("a04535a4-5d43-42c2-8366-8bcf2b55783f") }
Percona Server for MongoDB server version: v7.0.8-5
WARNING: shell and server versions do not match
+ cat /tmp/tmp.QsRnWt7FPS
+ rm /tmp/tmp.rYq72onT6T /tmp/tmp.QsRnWt7FPS
+ return 0
+ desc 'run disaster recovery script for replset: rs1'
+ set +o xtrace


-----------------------------------------------------------------------------------
run disaster recovery script for replset: rs1
-----------------------------------------------------------------------------------

+ run_script_mongos /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1536/e2e-tests/cross-site-sharded/disaster_recovery.js clusterAdmin:clusterAdmin123456@10.184.115.170 mongodb :27017
+ local script=/mnt/jenkins/workspace/cloud-psmdb-operator_PR-1536/e2e-tests/cross-site-sharded/disaster_recovery.js
+ local uri=clusterAdmin:clusterAdmin123456@10.184.115.170
+ local driver=mongodb
+ local suffix=:27017
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.DAbeTPAGj9
+++ mktemp
++ local LAST_ERR=/tmp/tmp.Hv0KKldezD
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.DAbeTPAGj9
++ cat /tmp/tmp.Hv0KKldezD
++ rm /tmp/tmp.DAbeTPAGj9 /tmp/tmp.Hv0KKldezD
++ return 0
+ local client_container=psmdb-client-5dc94d5b48-wl5cq
+ local mongo_flag=
++ basename /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1536/e2e-tests/cross-site-sharded/disaster_recovery.js
+ name=disaster_recovery.js
+ kubectl_bin cp /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1536/e2e-tests/cross-site-sharded/disaster_recovery.js cross-site-sharded-9438/psmdb-client-5dc94d5b48-wl5cq:/tmp
++ mktemp
+ local LAST_OUT=/tmp/tmp.l1kph3lBSs
++ mktemp
+ local LAST_ERR=/tmp/tmp.UX8zuOZrJa
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl cp /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1536/e2e-tests/cross-site-sharded/disaster_recovery.js cross-site-sharded-9438/psmdb-client-5dc94d5b48-wl5cq:/tmp
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.l1kph3lBSs
+ cat /tmp/tmp.UX8zuOZrJa
+ rm /tmp/tmp.l1kph3lBSs /tmp/tmp.UX8zuOZrJa
+ return 0
+ kubectl_bin exec psmdb-client-5dc94d5b48-wl5cq -- bash -c 'mongo mongodb://clusterAdmin:clusterAdmin123456@10.184.115.170:27017/admin  /tmp/disaster_recovery.js'
++ mktemp
+ local LAST_OUT=/tmp/tmp.lhigBdbjJM
++ mktemp
+ local LAST_ERR=/tmp/tmp.lnwEdZIBO6
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl exec psmdb-client-5dc94d5b48-wl5cq -- bash -c 'mongo mongodb://clusterAdmin:clusterAdmin123456@10.184.115.170:27017/admin  /tmp/disaster_recovery.js'
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.lhigBdbjJM
Percona Server for MongoDB shell version v4.4.29-28
connecting to: mongodb://10.184.115.170:27017/admin?compressors=disabled&gssapiServiceName=mongodb
Implicit session: session { "id" : UUID("7f6857dc-5bae-433f-b7ed-7297f52b4f81") }
Percona Server for MongoDB server version: v7.0.8-5
WARNING: shell and server versions do not match
+ cat /tmp/tmp.lnwEdZIBO6
+ rm /tmp/tmp.lhigBdbjJM /tmp/tmp.lnwEdZIBO6
+ return 0
+ desc 'make replica cluster managed'
+ set +o xtrace


-----------------------------------------------------------------------------------
make replica cluster managed
-----------------------------------------------------------------------------------

++ kubectl_bin config current-context
+++ mktemp
++ local LAST_OUT=/tmp/tmp.XpbpKYVl90
+++ mktemp
++ local LAST_ERR=/tmp/tmp.UBQCWPSDfT
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl config current-context
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.XpbpKYVl90
++ cat /tmp/tmp.UBQCWPSDfT
++ rm /tmp/tmp.XpbpKYVl90 /tmp/tmp.UBQCWPSDfT
++ return 0
+ kubectl_bin config set-context gke_cloud-dev-112233_us-central1-a_jen-psmdb-1536-4cc529f8-10-cluster4 --namespace=cross-site-sharded-replica-31492
++ mktemp
+ local LAST_OUT=/tmp/tmp.cHsVmUaXJW
++ mktemp
+ local LAST_ERR=/tmp/tmp.fA8GmfpnTU
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl config set-context gke_cloud-dev-112233_us-central1-a_jen-psmdb-1536-4cc529f8-10-cluster4 --namespace=cross-site-sharded-replica-31492
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.cHsVmUaXJW
Context "gke_cloud-dev-112233_us-central1-a_jen-psmdb-1536-4cc529f8-10-cluster4" modified.
+ cat /tmp/tmp.fA8GmfpnTU
+ rm /tmp/tmp.cHsVmUaXJW /tmp/tmp.fA8GmfpnTU
+ return 0
+ kubectl_bin patch psmdb cross-site-sharded-replica --type=merge --patch '{"spec":{"unmanaged": false}}'
++ mktemp
+ local LAST_OUT=/tmp/tmp.dBaBi2Rm97
++ mktemp
+ local LAST_ERR=/tmp/tmp.NkZQmEYDbL
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl patch psmdb cross-site-sharded-replica --type=merge --patch '{"spec":{"unmanaged": false}}'
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.dBaBi2Rm97
perconaservermongodb.psmdb.percona.com/cross-site-sharded-replica patched
+ cat /tmp/tmp.NkZQmEYDbL
+ rm /tmp/tmp.dBaBi2Rm97 /tmp/tmp.NkZQmEYDbL
+ return 0
+ wait_for_running cross-site-sharded-replica-rs0 3
+ local name=cross-site-sharded-replica-rs0
+ let last_pod=2
+ local check_cluster_readyness=true
+ set_debug
+ [[ 1 == 1 ]]
+ set -o xtrace
+ local rs_name=rs0
+ local cluster_name=cross-site-sharded-replica
++ seq 0 2
+ for i in '$(seq 0 $last_pod)'
+ [[ 0 -eq 2 ]]
+ wait_pod cross-site-sharded-replica-rs0-0
+ local pod=cross-site-sharded-replica-rs0-0
+ set +o xtrace
waiting for pod/cross-site-sharded-replica-rs0-0 to be ready.OK
+ for i in '$(seq 0 $last_pod)'
+ [[ 1 -eq 2 ]]
+ wait_pod cross-site-sharded-replica-rs0-1
+ local pod=cross-site-sharded-replica-rs0-1
+ set +o xtrace
waiting for pod/cross-site-sharded-replica-rs0-1 to be ready.OK
+ for i in '$(seq 0 $last_pod)'
+ [[ 2 -eq 2 ]]
++ kubectl_bin get psmdb cross-site-sharded-replica -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].arbiter.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.ejy21ZfKjc
+++ mktemp
++ local LAST_ERR=/tmp/tmp.HAUzrbl8Fv
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb cross-site-sharded-replica -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].arbiter.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.ejy21ZfKjc
++ cat /tmp/tmp.HAUzrbl8Fv
++ rm /tmp/tmp.ejy21ZfKjc /tmp/tmp.HAUzrbl8Fv
++ return 0
+ [[ false == \t\r\u\e ]]
+ wait_pod cross-site-sharded-replica-rs0-2
+ local pod=cross-site-sharded-replica-rs0-2
+ set +o xtrace
waiting for pod/cross-site-sharded-replica-rs0-2 to be ready.OK
++ kubectl_bin get psmdb cross-site-sharded-replica -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].non_voting.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.vN7IALuCHT
+++ mktemp
++ local LAST_ERR=/tmp/tmp.0w925etPgv
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb cross-site-sharded-replica -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].non_voting.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.vN7IALuCHT
++ cat /tmp/tmp.0w925etPgv
++ rm /tmp/tmp.vN7IALuCHT /tmp/tmp.0w925etPgv
++ return 0
+ [[ '' == \t\r\u\e ]]
+ sleep 10
+ [[ true == \t\r\u\e ]]
+ set +x
Waiting for cluster readyness
+ wait_for_running cross-site-sharded-replica-cfg 3
+ local name=cross-site-sharded-replica-cfg
+ let last_pod=2
+ local check_cluster_readyness=true
+ set_debug
+ [[ 1 == 1 ]]
+ set -o xtrace
+ local rs_name=cfg
+ local cluster_name=cross-site-sharded-replica
++ seq 0 2
+ for i in '$(seq 0 $last_pod)'
+ [[ 0 -eq 2 ]]
+ wait_pod cross-site-sharded-replica-cfg-0
+ local pod=cross-site-sharded-replica-cfg-0
+ set +o xtrace
waiting for pod/cross-site-sharded-replica-cfg-0 to be ready.OK
+ for i in '$(seq 0 $last_pod)'
+ [[ 1 -eq 2 ]]
+ wait_pod cross-site-sharded-replica-cfg-1
+ local pod=cross-site-sharded-replica-cfg-1
+ set +o xtrace
waiting for pod/cross-site-sharded-replica-cfg-1 to be ready.OK
+ for i in '$(seq 0 $last_pod)'
+ [[ 2 -eq 2 ]]
++ kubectl_bin get psmdb cross-site-sharded-replica -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].arbiter.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.KuZ1srJ0Lu
+++ mktemp
++ local LAST_ERR=/tmp/tmp.QQe2YBj8xd
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb cross-site-sharded-replica -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].arbiter.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.KuZ1srJ0Lu
++ cat /tmp/tmp.QQe2YBj8xd
++ rm /tmp/tmp.KuZ1srJ0Lu /tmp/tmp.QQe2YBj8xd
++ return 0
+ [[ '' == \t\r\u\e ]]
+ wait_pod cross-site-sharded-replica-cfg-2
+ local pod=cross-site-sharded-replica-cfg-2
+ set +o xtrace
waiting for pod/cross-site-sharded-replica-cfg-2 to be ready.OK
++ kubectl_bin get psmdb cross-site-sharded-replica -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].non_voting.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.yoDFrd4PrJ
+++ mktemp
++ local LAST_ERR=/tmp/tmp.NRwMAiT9Wk
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb cross-site-sharded-replica -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].non_voting.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.yoDFrd4PrJ
++ cat /tmp/tmp.NRwMAiT9Wk
++ rm /tmp/tmp.yoDFrd4PrJ /tmp/tmp.NRwMAiT9Wk
++ return 0
+ [[ '' == \t\r\u\e ]]
+ sleep 10
+ [[ true == \t\r\u\e ]]
+ set +x
Waiting for cluster readyness
+ desc 'check failover status'
+ set +o xtrace


-----------------------------------------------------------------------------------
check failover status
-----------------------------------------------------------------------------------

+ compare_mongos_cmd find myApp:myPass@cross-site-sharded-replica-mongos.cross-site-sharded-replica-31492
+ local command=find
+ local uri=myApp:myPass@cross-site-sharded-replica-mongos.cross-site-sharded-replica-31492
+ local postfix=
+ local suffix=
+ local database=myApp
+ local collection=test
+ egrep -v 'I NETWORK|W NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:'
+ run_mongos 'use myApp\n db.test.find()' myApp:myPass@cross-site-sharded-replica-mongos.cross-site-sharded-replica-31492 mongodb ''
+ local 'command=use myApp\n db.test.find()'
+ local uri=myApp:myPass@cross-site-sharded-replica-mongos.cross-site-sharded-replica-31492
+ local driver=mongodb
+ local suffix=.svc.cluster.local
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
+ /usr/bin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
++ local LAST_OUT=/tmp/tmp.UDKRhEzCpP
+++ mktemp
++ local LAST_ERR=/tmp/tmp.0ab8bENjvx
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.UDKRhEzCpP
++ cat /tmp/tmp.0ab8bENjvx
++ rm /tmp/tmp.UDKRhEzCpP /tmp/tmp.0ab8bENjvx
++ return 0
+ local client_container=psmdb-client-5dc94d5b48-fjk4f
+ local mongo_flag=
+ kubectl_bin exec psmdb-client-5dc94d5b48-fjk4f -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@cross-site-sharded-replica-mongos.cross-site-sharded-replica-31492.svc.cluster.local/admin '
++ mktemp
+ local LAST_OUT=/tmp/tmp.fWtrjY333T
++ mktemp
+ local LAST_ERR=/tmp/tmp.MnXYxrJ7Fr
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl exec psmdb-client-5dc94d5b48-fjk4f -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@cross-site-sharded-replica-mongos.cross-site-sharded-replica-31492.svc.cluster.local/admin '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.fWtrjY333T
+ cat /tmp/tmp.MnXYxrJ7Fr
+ rm /tmp/tmp.fWtrjY333T /tmp/tmp.MnXYxrJ7Fr
+ return 0
+ diff /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1536/e2e-tests/cross-site-sharded/compare/find.json /tmp/tmp.a26yFxZivT/find
+ desc 'Failover check finished successfully'
+ set +o xtrace


-----------------------------------------------------------------------------------
Failover check finished successfully
-----------------------------------------------------------------------------------

+ wait_cluster_consistency cross-site-sharded-replica
+ local cluster_name=cross-site-sharded-replica
+ local wait_time=32
+ retry=0
+ sleep 7
+ echo -n 'waiting for cluster readyness'
waiting for cluster readyness++ kubectl_bin get psmdb cross-site-sharded-replica -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.HITl86Hc1K
+++ mktemp
++ local LAST_ERR=/tmp/tmp.e07XU9ax1v
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb cross-site-sharded-replica -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.HITl86Hc1K
++ cat /tmp/tmp.e07XU9ax1v
++ rm /tmp/tmp.HITl86Hc1K /tmp/tmp.e07XU9ax1v
++ return 0
+ [[ ready == \r\e\a\d\y ]]
+ destroy cross-site-sharded-9438 true
+ local namespace=cross-site-sharded-9438
+ local ignore_logs=true
+ desc 'destroy cluster/operator and all other resources'
+ set +o xtrace


-----------------------------------------------------------------------------------
destroy cluster/operator and all other resources
-----------------------------------------------------------------------------------

+ '[' true == false ']'
+ delete_crd
+ desc 'get and delete old CRDs and RBAC'
+ set +o xtrace


-----------------------------------------------------------------------------------
get and delete old CRDs and RBAC
-----------------------------------------------------------------------------------

+ kubectl_bin delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1536/deploy/crd.yaml --ignore-not-found --wait=false
++ mktemp
+ local LAST_OUT=/tmp/tmp.EfP64fzXte
++ mktemp
+ local LAST_ERR=/tmp/tmp.IRfYNjTxRG
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1536/deploy/crd.yaml --ignore-not-found --wait=false
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.EfP64fzXte
customresourcedefinition.apiextensions.k8s.io "perconaservermongodbbackups.psmdb.percona.com" deleted
customresourcedefinition.apiextensions.k8s.io "perconaservermongodbrestores.psmdb.percona.com" deleted
customresourcedefinition.apiextensions.k8s.io "perconaservermongodbs.psmdb.percona.com" deleted
+ cat /tmp/tmp.IRfYNjTxRG
+ rm /tmp/tmp.EfP64fzXte /tmp/tmp.IRfYNjTxRG
+ return 0
++ yq eval .metadata.name /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1536/deploy/crd.yaml
++ grep -v '\-\-\-'
+ for crd_name in '$(yq eval '\''.metadata.name'\'' "${src_dir}/deploy/crd.yaml" | grep -v '\''\-\-\-'\'')'
+ kubectl get perconaservermongodbbackups.psmdb.percona.com --all-namespaces -o wide
+ grep -v NAMESPACE
+ xargs -L 1 sh -xc 'kubectl patch perconaservermongodbbackups.psmdb.percona.com -n $0 $1 --type=merge -p "{\"metadata\":{\"finalizers\":[]}}"'
error: the server doesn't have a resource type "perconaservermongodbbackups"
+ kubectl patch perconaservermongodbbackups.psmdb.percona.com -n sh --type=merge -p '{"metadata":{"finalizers":[]}}'
error: the server doesn't have a resource type "perconaservermongodbbackups"
+ :
+ kubectl_bin wait --for=delete crd perconaservermongodbbackups.psmdb.percona.com
++ mktemp
+ local LAST_OUT=/tmp/tmp.tzwYO2uOxv
++ mktemp
+ local LAST_ERR=/tmp/tmp.zrF6S9cRgw
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl wait --for=delete crd perconaservermongodbbackups.psmdb.percona.com
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.tzwYO2uOxv
+ cat /tmp/tmp.zrF6S9cRgw
+ rm /tmp/tmp.tzwYO2uOxv /tmp/tmp.zrF6S9cRgw
+ return 0
+ for crd_name in '$(yq eval '\''.metadata.name'\'' "${src_dir}/deploy/crd.yaml" | grep -v '\''\-\-\-'\'')'
+ kubectl get perconaservermongodbrestores.psmdb.percona.com --all-namespaces -o wide
+ grep -v NAMESPACE
+ xargs -L 1 sh -xc 'kubectl patch perconaservermongodbrestores.psmdb.percona.com -n $0 $1 --type=merge -p "{\"metadata\":{\"finalizers\":[]}}"'
error: the server doesn't have a resource type "perconaservermongodbrestores"
+ kubectl patch perconaservermongodbrestores.psmdb.percona.com -n sh --type=merge -p '{"metadata":{"finalizers":[]}}'
error: the server doesn't have a resource type "perconaservermongodbrestores"
+ :
+ kubectl_bin wait --for=delete crd perconaservermongodbrestores.psmdb.percona.com
++ mktemp
+ local LAST_OUT=/tmp/tmp.bc2Bz1CBXP
++ mktemp
+ local LAST_ERR=/tmp/tmp.oqYNy9xK75
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl wait --for=delete crd perconaservermongodbrestores.psmdb.percona.com
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.bc2Bz1CBXP
+ cat /tmp/tmp.oqYNy9xK75
+ rm /tmp/tmp.bc2Bz1CBXP /tmp/tmp.oqYNy9xK75
+ return 0
+ for crd_name in '$(yq eval '\''.metadata.name'\'' "${src_dir}/deploy/crd.yaml" | grep -v '\''\-\-\-'\'')'
+ grep -v NAMESPACE
+ kubectl get perconaservermongodbs.psmdb.percona.com --all-namespaces -o wide
+ xargs -L 1 sh -xc 'kubectl patch perconaservermongodbs.psmdb.percona.com -n $0 $1 --type=merge -p "{\"metadata\":{\"finalizers\":[]}}"'
error: the server doesn't have a resource type "perconaservermongodbs"
+ kubectl patch perconaservermongodbs.psmdb.percona.com -n sh --type=merge -p '{"metadata":{"finalizers":[]}}'
error: the server doesn't have a resource type "perconaservermongodbs"
+ :
+ kubectl_bin wait --for=delete crd perconaservermongodbs.psmdb.percona.com
++ mktemp
+ local LAST_OUT=/tmp/tmp.gBtLm21S1K
++ mktemp
+ local LAST_ERR=/tmp/tmp.2HwYTxXR2O
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl wait --for=delete crd perconaservermongodbs.psmdb.percona.com
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.gBtLm21S1K
+ cat /tmp/tmp.2HwYTxXR2O
+ rm /tmp/tmp.gBtLm21S1K /tmp/tmp.2HwYTxXR2O
+ return 0
+ local rbac_yaml=rbac.yaml
+ '[' -n '' ']'
+ kubectl_bin delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1536/deploy/rbac.yaml --ignore-not-found
++ mktemp
+ local LAST_OUT=/tmp/tmp.xj93qC4At8
++ mktemp
+ local LAST_ERR=/tmp/tmp.6IfJV98v8D
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1536/deploy/rbac.yaml --ignore-not-found
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.xj93qC4At8
role.rbac.authorization.k8s.io "percona-server-mongodb-operator" deleted
serviceaccount "percona-server-mongodb-operator" deleted
rolebinding.rbac.authorization.k8s.io "service-account-percona-server-mongodb-operator" deleted
+ cat /tmp/tmp.6IfJV98v8D
+ rm /tmp/tmp.xj93qC4At8 /tmp/tmp.6IfJV98v8D
+ return 0
+ kubectl_bin delete -f https://github.com/cert-manager/cert-manager/releases/download/v1.12.4/cert-manager.yaml
+ :
+ '[' -n '' ']'
+ '[' -n '' ']'
+ rm -rf /tmp/tmp.a26yFxZivT
+ kubectl_bin delete --grace-period=0 --force=true namespace cross-site-sharded-9438
++ mktemp
+ destroy cross-site-sharded-replica-31492 true
+ local namespace=cross-site-sharded-replica-31492
+ local ignore_logs=true
+ desc 'destroy cluster/operator and all other resources'
+ set +o xtrace


-----------------------------------------------------------------------------------
+ local LAST_OUT=/tmp/tmp.1V7gS4rDgt
destroy cluster/operator and all other resources
-----------------------------------------------------------------------------------

+ '[' true == false ']'
+ delete_crd
+ desc 'get and delete old CRDs and RBAC'
+ set +o xtrace


-----------------------------------------------------------------------------------
get and delete old CRDs and RBAC
-----------------------------------------------------------------------------------

+ kubectl_bin delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1536/deploy/crd.yaml --ignore-not-found --wait=false
++ mktemp
++ mktemp
+ local LAST_ERR=/tmp/tmp.znMqPRmAXq
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ local LAST_OUT=/tmp/tmp.YvWkM5Mzkr
++ mktemp
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl delete --grace-period=0 --force=true namespace cross-site-sharded-9438
+ local LAST_ERR=/tmp/tmp.7OATY0KD0t
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1536/deploy/crd.yaml --ignore-not-found --wait=false
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.YvWkM5Mzkr
+ cat /tmp/tmp.7OATY0KD0t
+ rm /tmp/tmp.YvWkM5Mzkr /tmp/tmp.7OATY0KD0t
+ return 0
++ yq eval .metadata.name /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1536/deploy/crd.yaml
++ grep -v '\-\-\-'
+ for crd_name in '$(yq eval '\''.metadata.name'\'' "${src_dir}/deploy/crd.yaml" | grep -v '\''\-\-\-'\'')'
+ grep -v NAMESPACE
+ kubectl get perconaservermongodbbackups.psmdb.percona.com --all-namespaces -o wide
+ xargs -L 1 sh -xc 'kubectl patch perconaservermongodbbackups.psmdb.percona.com -n $0 $1 --type=merge -p "{\"metadata\":{\"finalizers\":[]}}"'
error: the server doesn't have a resource type "perconaservermongodbbackups"
+ kubectl patch perconaservermongodbbackups.psmdb.percona.com -n sh --type=merge -p '{"metadata":{"finalizers":[]}}'
error: the server doesn't have a resource type "perconaservermongodbbackups"
+ :
+ kubectl_bin wait --for=delete crd perconaservermongodbbackups.psmdb.percona.com
++ mktemp
+ local LAST_OUT=/tmp/tmp.MFNJYaVlL2
++ mktemp
+ local LAST_ERR=/tmp/tmp.48HNM8KHYC
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl wait --for=delete crd perconaservermongodbbackups.psmdb.percona.com
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.MFNJYaVlL2
+ cat /tmp/tmp.48HNM8KHYC
+ rm /tmp/tmp.MFNJYaVlL2 /tmp/tmp.48HNM8KHYC
+ return 0
+ for crd_name in '$(yq eval '\''.metadata.name'\'' "${src_dir}/deploy/crd.yaml" | grep -v '\''\-\-\-'\'')'
+ kubectl get perconaservermongodbrestores.psmdb.percona.com --all-namespaces -o wide
+ grep -v NAMESPACE
+ xargs -L 1 sh -xc 'kubectl patch perconaservermongodbrestores.psmdb.percona.com -n $0 $1 --type=merge -p "{\"metadata\":{\"finalizers\":[]}}"'
error: the server doesn't have a resource type "perconaservermongodbrestores"
+ kubectl patch perconaservermongodbrestores.psmdb.percona.com -n sh --type=merge -p '{"metadata":{"finalizers":[]}}'
error: the server doesn't have a resource type "perconaservermongodbrestores"
+ :
+ kubectl_bin wait --for=delete crd perconaservermongodbrestores.psmdb.percona.com
++ mktemp
+ local LAST_OUT=/tmp/tmp.YelF7J7w7d
++ mktemp
+ local LAST_ERR=/tmp/tmp.9cjj40Z5kO
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl wait --for=delete crd perconaservermongodbrestores.psmdb.percona.com
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.YelF7J7w7d
+ cat /tmp/tmp.9cjj40Z5kO
+ rm /tmp/tmp.YelF7J7w7d /tmp/tmp.9cjj40Z5kO
+ return 0
+ for crd_name in '$(yq eval '\''.metadata.name'\'' "${src_dir}/deploy/crd.yaml" | grep -v '\''\-\-\-'\'')'
+ kubectl get perconaservermongodbs.psmdb.percona.com --all-namespaces -o wide
+ grep -v NAMESPACE
+ xargs -L 1 sh -xc 'kubectl patch perconaservermongodbs.psmdb.percona.com -n $0 $1 --type=merge -p "{\"metadata\":{\"finalizers\":[]}}"'
error: the server doesn't have a resource type "perconaservermongodbs"
+ kubectl patch perconaservermongodbs.psmdb.percona.com -n sh --type=merge -p '{"metadata":{"finalizers":[]}}'
error: the server doesn't have a resource type "perconaservermongodbs"
+ :
+ kubectl_bin wait --for=delete crd perconaservermongodbs.psmdb.percona.com
++ mktemp
+ local LAST_OUT=/tmp/tmp.aqX96UUWx8
++ mktemp
+ local LAST_ERR=/tmp/tmp.uj3o7yPeXE
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl wait --for=delete crd perconaservermongodbs.psmdb.percona.com
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.aqX96UUWx8
+ cat /tmp/tmp.uj3o7yPeXE
+ rm /tmp/tmp.aqX96UUWx8 /tmp/tmp.uj3o7yPeXE
+ return 0
+ local rbac_yaml=rbac.yaml
+ '[' -n '' ']'
+ kubectl_bin delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1536/deploy/rbac.yaml --ignore-not-found
++ mktemp
+ local LAST_OUT=/tmp/tmp.wH0noVFtOA
++ mktemp
+ local LAST_ERR=/tmp/tmp.RqdzczgExv
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1536/deploy/rbac.yaml --ignore-not-found
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.wH0noVFtOA
+ cat /tmp/tmp.RqdzczgExv
+ rm /tmp/tmp.wH0noVFtOA /tmp/tmp.RqdzczgExv
+ return 0
+ kubectl_bin delete -f https://github.com/cert-manager/cert-manager/releases/download/v1.12.4/cert-manager.yaml
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.1V7gS4rDgt
namespace "cross-site-sharded-9438" force deleted
+ cat /tmp/tmp.znMqPRmAXq
Warning: Immediate deletion does not wait for confirmation that the running resource has been terminated. The resource may continue to run on the cluster indefinitely.
+ rm /tmp/tmp.1V7gS4rDgt /tmp/tmp.znMqPRmAXq
+ return 0
+ :
+ '[' -n '' ']'
+ '[' -n '' ']'
+ rm -rf /tmp/tmp.a26yFxZivT
+ kubectl_bin delete --grace-period=0 --force=true namespace cross-site-sharded-replica-31492
+ desc 'test passed'
+ set +o xtrace


-----------------------------------------------------------------------------------
test passed
-----------------------------------------------------------------------------------

++ mktemp
+ local LAST_OUT=/tmp/tmp.1e1OXOULxH
++ mktemp
+ local LAST_ERR=/tmp/tmp.d9VQXKqRo7
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl delete --grace-period=0 --force=true namespace cross-site-sharded-replica-31492