Log: /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2283/e2e-tests/logs/cross-site-sharded.log Warning: version difference between client (1.35) and server (1.32) exceeds the supported minor version skew of +/-1 Warning: version difference between client (1.35) and server (1.32) exceeds the supported minor version skew of +/-1 Warning: version difference between client (1.35) and server (1.32) exceeds the supported minor version skew of +/-1 ++ get_mongod_ver_from_image docker.io/perconalab/percona-server-mongodb-operator:main-mongod8.0 ++ local image=docker.io/perconalab/percona-server-mongodb-operator:main-mongod8.0 +++ run_simple_cli_inside_image docker.io/perconalab/percona-server-mongodb-operator:main-mongod8.0 'mongod --version' +++ local image=docker.io/perconalab/percona-server-mongodb-operator:main-mongod8.0 +++ local 'cli=mongod --version' +++ local pod_name=22406 +++ kubectl_bin -n default run 22406 --image=docker.io/perconalab/percona-server-mongodb-operator:main-mongod8.0 --restart=Never --command -- sleep infinity +++ /usr/sbin/sed -r 's/^.*db version v(([0-9]+\.){2}[0-9]+-[0-9]+).*$/\1/g' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.2O5lOcXa5f ++++ mktemp +++ local LAST_ERR=/tmp/tmp.Px0y2xKaQK +++ local exit_status=0 +++ local timeout=4 ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl -n default run 22406 --image=docker.io/perconalab/percona-server-mongodb-operator:main-mongod8.0 --restart=Never --command -- sleep infinity +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 -a -n 1 ']' +++ break +++ cat /tmp/tmp.2O5lOcXa5f +++ cat /tmp/tmp.Px0y2xKaQK +++ rm /tmp/tmp.2O5lOcXa5f /tmp/tmp.Px0y2xKaQK +++ return 0 +++ kubectl_bin -n default wait --for=condition=Ready pod/22406 ++++ mktemp +++ local LAST_OUT=/tmp/tmp.eK3PkgopMj ++++ mktemp +++ local LAST_ERR=/tmp/tmp.MCtYcHBgPg +++ local exit_status=0 +++ local timeout=4 ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl -n default wait --for=condition=Ready pod/22406 +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 -a -n 1 ']' +++ break +++ cat /tmp/tmp.eK3PkgopMj +++ cat /tmp/tmp.MCtYcHBgPg +++ rm /tmp/tmp.eK3PkgopMj /tmp/tmp.MCtYcHBgPg +++ return 0 ++++ kubectl_bin -n default exec 22406 -- bash -c 'mongod --version 2>&1' +++++ mktemp ++++ local LAST_OUT=/tmp/tmp.CJFCs9Din0 +++++ mktemp ++++ local LAST_ERR=/tmp/tmp.QIKiXVPFgj ++++ local exit_status=0 ++++ local timeout=4 +++++ seq 0 2 ++++ for i in $(seq 0 2) ++++ set +e ++++ kubectl -n default exec 22406 -- bash -c 'mongod --version 2>&1' ++++ exit_status=0 ++++ set -e ++++ '[' 0 '!=' 0 -a -n 1 ']' ++++ break ++++ cat /tmp/tmp.CJFCs9Din0 ++++ cat /tmp/tmp.QIKiXVPFgj ++++ rm /tmp/tmp.CJFCs9Din0 /tmp/tmp.QIKiXVPFgj ++++ return 0 +++ local $'output=db version v8.0.20-8\nBuild Info: {\n "version": "8.0.20-8",\n "gitVersion": "5f8e208f2ce1ec988e9830397069fad1147bc257",\n "openSSLVersion": "OpenSSL 3.5.1 1 Jul 2025",\n "modules": [],\n "perconaFeatures": [\n "MemoryEngine",\n "HotBackup",\n "BackupCursorAggregationStage",\n "BackupCursorExtendAggregationStage",\n "AWSIAM",\n "Kerberos",\n "LDAP",\n "OIDC",\n "TDE",\n "FIPSMode",\n "FCBIS",\n "Auditing",\n "ProfilingRateLimit",\n "LogRedaction",\n "ngram"\n ],\n "allocator": "tcmalloc-google",\n "environment": {\n "distarch": "x86_64",\n "target_arch": "x86_64"\n }\n}' +++ kubectl_bin -n default delete pod/22406 --grace-period=0 --force ++++ mktemp +++ local LAST_OUT=/tmp/tmp.K09a9FAW2S ++++ mktemp +++ local LAST_ERR=/tmp/tmp.V8GYSYhKrO +++ local exit_status=0 +++ local timeout=4 ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl -n default delete pod/22406 --grace-period=0 --force +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 -a -n 1 ']' +++ break +++ cat /tmp/tmp.K09a9FAW2S +++ cat /tmp/tmp.V8GYSYhKrO Warning: Immediate deletion does not wait for confirmation that the running resource has been terminated. The resource may continue to run on the cluster indefinitely. +++ rm /tmp/tmp.K09a9FAW2S /tmp/tmp.V8GYSYhKrO +++ return 0 +++ echo db version v8.0.20-8 Build Info: '{' '"version":' '"8.0.20-8",' '"gitVersion":' '"5f8e208f2ce1ec988e9830397069fad1147bc257",' '"openSSLVersion":' '"OpenSSL' 3.5.1 1 Jul '2025",' '"modules":' '[],' '"perconaFeatures":' '[' '"MemoryEngine",' '"HotBackup",' '"BackupCursorAggregationStage",' '"BackupCursorExtendAggregationStage",' '"AWSIAM",' '"Kerberos",' '"LDAP",' '"OIDC",' '"TDE",' '"FIPSMode",' '"FCBIS",' '"Auditing",' '"ProfilingRateLimit",' '"LogRedaction",' '"ngram"' '],' '"allocator":' '"tcmalloc-google",' '"environment":' '{' '"distarch":' '"x86_64",' '"target_arch":' '"x86_64"' '}' '}' ++ version_info=8.0.20-8 ++ [[ ! 8.0.20-8 =~ ^([0-9]+\.){2}[0-9]+-[0-9]+$ ]] ++ echo 8.0.20-8 + FULL_VER=8.0.20-8 + MONGO_VER=8.0 + unset OPERATOR_NS + main_cluster=cross-site-sharded-main + replica_cluster=cross-site-sharded-replica + desc 'create main cluster' + set +o xtrace ----------------------------------------------------------------------------------- create main cluster ----------------------------------------------------------------------------------- + create_infra cross-site-sharded-2618 + local ns=cross-site-sharded-2618 + [[ 1 == 1 ]] + delete_crd + desc 'get and delete old CRDs and RBAC' + set +o xtrace ----------------------------------------------------------------------------------- get and delete old CRDs and RBAC ----------------------------------------------------------------------------------- + kubectl_bin delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2283/deploy/crd.yaml --ignore-not-found --wait=false ++ mktemp + local LAST_OUT=/tmp/tmp.6aTfNO3f6q ++ mktemp + local LAST_ERR=/tmp/tmp.yeLVaRTQu7 + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2283/deploy/crd.yaml --ignore-not-found --wait=false + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.6aTfNO3f6q customresourcedefinition.apiextensions.k8s.io "perconaservermongodbbackups.psmdb.percona.com" deleted customresourcedefinition.apiextensions.k8s.io "perconaservermongodbrestores.psmdb.percona.com" deleted customresourcedefinition.apiextensions.k8s.io "perconaservermongodbs.psmdb.percona.com" deleted + cat /tmp/tmp.yeLVaRTQu7 + rm /tmp/tmp.6aTfNO3f6q /tmp/tmp.yeLVaRTQu7 + return 0 ++ yq eval .metadata.name /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2283/deploy/crd.yaml ++ grep -v '\-\-\-' grep: warning: stray \ before - grep: warning: stray \ before - + for crd_name in $(yq eval '.metadata.name' "${src_dir}/deploy/crd.yaml" | grep -v '\-\-\-') + kubectl get perconaservermongodbbackups.psmdb.percona.com --all-namespaces -o wide + grep -v NAMESPACE + xargs -L 1 sh -xc 'kubectl patch perconaservermongodbbackups.psmdb.percona.com -n $0 $1 --type=merge -p "{\"metadata\":{\"finalizers\":[]}}"' error: the server doesn't have a resource type "perconaservermongodbbackups" + kubectl patch perconaservermongodbbackups.psmdb.percona.com -n sh --type=merge -p '{"metadata":{"finalizers":[]}}' error: the server doesn't have a resource type "perconaservermongodbbackups" + : + kubectl_bin wait --for=delete crd perconaservermongodbbackups.psmdb.percona.com ++ mktemp + local LAST_OUT=/tmp/tmp.TMxn5s6axj ++ mktemp + local LAST_ERR=/tmp/tmp.oM2Y7YvrsB + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl wait --for=delete crd perconaservermongodbbackups.psmdb.percona.com + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.TMxn5s6axj + cat /tmp/tmp.oM2Y7YvrsB + rm /tmp/tmp.TMxn5s6axj /tmp/tmp.oM2Y7YvrsB + return 0 + for crd_name in $(yq eval '.metadata.name' "${src_dir}/deploy/crd.yaml" | grep -v '\-\-\-') + grep -v NAMESPACE + kubectl get perconaservermongodbrestores.psmdb.percona.com --all-namespaces -o wide + xargs -L 1 sh -xc 'kubectl patch perconaservermongodbrestores.psmdb.percona.com -n $0 $1 --type=merge -p "{\"metadata\":{\"finalizers\":[]}}"' error: the server doesn't have a resource type "perconaservermongodbrestores" + kubectl patch perconaservermongodbrestores.psmdb.percona.com -n sh --type=merge -p '{"metadata":{"finalizers":[]}}' error: the server doesn't have a resource type "perconaservermongodbrestores" + : + kubectl_bin wait --for=delete crd perconaservermongodbrestores.psmdb.percona.com ++ mktemp + local LAST_OUT=/tmp/tmp.ekiYbQTnhx ++ mktemp + local LAST_ERR=/tmp/tmp.48LQi4HA1S + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl wait --for=delete crd perconaservermongodbrestores.psmdb.percona.com + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.ekiYbQTnhx + cat /tmp/tmp.48LQi4HA1S + rm /tmp/tmp.ekiYbQTnhx /tmp/tmp.48LQi4HA1S + return 0 + for crd_name in $(yq eval '.metadata.name' "${src_dir}/deploy/crd.yaml" | grep -v '\-\-\-') + kubectl get perconaservermongodbs.psmdb.percona.com --all-namespaces -o wide + grep -v NAMESPACE + xargs -L 1 sh -xc 'kubectl patch perconaservermongodbs.psmdb.percona.com -n $0 $1 --type=merge -p "{\"metadata\":{\"finalizers\":[]}}"' error: the server doesn't have a resource type "perconaservermongodbs" + kubectl patch perconaservermongodbs.psmdb.percona.com -n sh --type=merge -p '{"metadata":{"finalizers":[]}}' error: the server doesn't have a resource type "perconaservermongodbs" + : + kubectl_bin wait --for=delete crd perconaservermongodbs.psmdb.percona.com ++ mktemp + local LAST_OUT=/tmp/tmp.CEtUs20iHc ++ mktemp + local LAST_ERR=/tmp/tmp.DpOsu6MlUB + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl wait --for=delete crd perconaservermongodbs.psmdb.percona.com + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.CEtUs20iHc + cat /tmp/tmp.DpOsu6MlUB + rm /tmp/tmp.CEtUs20iHc /tmp/tmp.DpOsu6MlUB + return 0 + local rbac_yaml=rbac.yaml + '[' -n '' ']' + kubectl_bin delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2283/deploy/rbac.yaml --ignore-not-found ++ mktemp + local LAST_OUT=/tmp/tmp.CZO6TE4q1U ++ mktemp + local LAST_ERR=/tmp/tmp.d9bhaxg5Ui + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2283/deploy/rbac.yaml --ignore-not-found + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.CZO6TE4q1U role.rbac.authorization.k8s.io "percona-server-mongodb-operator" deleted from cross-site-sharded-6517 namespace serviceaccount "percona-server-mongodb-operator" deleted from cross-site-sharded-6517 namespace rolebinding.rbac.authorization.k8s.io "service-account-percona-server-mongodb-operator" deleted from cross-site-sharded-6517 namespace + cat /tmp/tmp.d9bhaxg5Ui + rm /tmp/tmp.CZO6TE4q1U /tmp/tmp.d9bhaxg5Ui + return 0 + check_crd_for_deletion PR-2283-73ca3795 + local git_tag=PR-2283-73ca3795 ++ curl -s https://raw.githubusercontent.com/percona/percona-server-mongodb-operator/PR-2283-73ca3795/deploy/crd.yaml ++ yq eval .metadata.name ++ /usr/sbin/sed s/---//g ++ /usr/sbin/sed ':a;N;$!ba;s/\n/ /g' + for crd_name in $(curl -s https://raw.githubusercontent.com/percona/percona-server-mongodb-operator/${git_tag}/deploy/crd.yaml | yq eval '.metadata.name' | $sed 's/---//g' | $sed ':a;N;$!ba;s/\n/ /g') ++ kubectl_bin get crd/null -o 'jsonpath={.status.conditions[-1].type}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.yGKNB5sgrK +++ mktemp ++ local LAST_ERR=/tmp/tmp.ngdQzt2riD ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get crd/null -o 'jsonpath={.status.conditions[-1].type}' ++ exit_status=1 ++ set -e ++ '[' 1 '!=' 0 -a -n 1 ']' ++ cat /tmp/tmp.yGKNB5sgrK ++ cat /tmp/tmp.ngdQzt2riD Error from server (NotFound): customresourcedefinitions.apiextensions.k8s.io "null" not found ++ sleep 0 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get crd/null -o 'jsonpath={.status.conditions[-1].type}' ++ exit_status=1 ++ set -e ++ '[' 1 '!=' 0 -a -n 1 ']' ++ cat /tmp/tmp.yGKNB5sgrK ++ cat /tmp/tmp.ngdQzt2riD Error from server (NotFound): customresourcedefinitions.apiextensions.k8s.io "null" not found ++ sleep 4 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get crd/null -o 'jsonpath={.status.conditions[-1].type}' ++ exit_status=1 ++ set -e ++ '[' 1 '!=' 0 -a -n 1 ']' ++ cat /tmp/tmp.yGKNB5sgrK ++ cat /tmp/tmp.ngdQzt2riD Error from server (NotFound): customresourcedefinitions.apiextensions.k8s.io "null" not found ++ sleep 8 ++ cat /tmp/tmp.yGKNB5sgrK ++ cat /tmp/tmp.ngdQzt2riD Error from server (NotFound): customresourcedefinitions.apiextensions.k8s.io "null" not found ++ rm /tmp/tmp.yGKNB5sgrK /tmp/tmp.ngdQzt2riD ++ return 1 + [[ '' == Terminating ]] + '[' -n '' ']' + create_namespace cross-site-sharded-2618 + local namespace=cross-site-sharded-2618 + local skip_clean_namespace= + [[ 1 == 1 ]] + [[ -z '' ]] + destroy_chaos_mesh ++ helm list --all-namespaces --filter chaos-mesh ++ tail -n1 ++ awk '-F ' '{print $2}' ++ sed s/NAMESPACE// + local chaos_mesh_ns= + desc 'destroy chaos-mesh' + set +o xtrace ----------------------------------------------------------------------------------- destroy chaos-mesh ----------------------------------------------------------------------------------- + '[' -n '' ']' ++ kubectl get MutatingWebhookConfiguration ++ grep chaos-mesh ++ awk '{print $1}' + timeout 30 kubectl delete MutatingWebhookConfiguration error: resource(s) were provided, but no name was specified + : ++ kubectl get ValidatingWebhookConfiguration ++ grep chaos-mesh ++ awk '{print $1}' + timeout 30 kubectl delete ValidatingWebhookConfiguration error: resource(s) were provided, but no name was specified + : ++ kubectl get ValidatingWebhookConfiguration ++ grep validate-auth ++ awk '{print $1}' + timeout 30 kubectl delete ValidatingWebhookConfiguration error: resource(s) were provided, but no name was specified + : ++ kubectl api-resources ++ grep chaos-mesh ++ awk '{print $1}' ++ kubectl get crd ++ awk '{print $1}' ++ grep chaos-mesh.org + timeout 30 kubectl delete crd error: resource(s) were provided, but no name was specified + : ++ kubectl get clusterrolebinding ++ awk '{print $1}' ++ grep chaos-mesh + timeout 30 kubectl delete clusterrolebinding error: resource(s) were provided, but no name was specified + : ++ kubectl get clusterrole ++ grep chaos-mesh ++ awk '{print $1}' + timeout 30 kubectl delete clusterrole error: resource(s) were provided, but no name was specified + : + desc 'cleaned up all old namespaces' + set +o xtrace ----------------------------------------------------------------------------------- cleaned up all old namespaces ----------------------------------------------------------------------------------- + kubectl_bin get ns + grep -E -v '^kube-|^default|Terminating|psmdb-operator|openshift|^gke-|^gmp-|^NAME' + awk '{print$1}' + '[' -n '' ']' + desc 'cleaned up old namespaces cross-site-sharded-2618' + set +o xtrace ----------------------------------------------------------------------------------- cleaned up old namespaces cross-site-sharded-2618 ----------------------------------------------------------------------------------- + kubectl_bin delete namespace cross-site-sharded-2618 --ignore-not-found ++ mktemp + xargs kubectl delete ns ++ mktemp + local LAST_OUT=/tmp/tmp.BYJOiWhRDD ++ mktemp + local LAST_OUT=/tmp/tmp.0Fb6TUTXIs ++ mktemp + local LAST_ERR=/tmp/tmp.LhDHYsPJBe + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl get ns + local LAST_ERR=/tmp/tmp.z4uH6VzV8b + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl delete namespace cross-site-sharded-2618 --ignore-not-found + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.BYJOiWhRDD + cat /tmp/tmp.LhDHYsPJBe + rm /tmp/tmp.BYJOiWhRDD /tmp/tmp.LhDHYsPJBe + return 0 + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.0Fb6TUTXIs + cat /tmp/tmp.z4uH6VzV8b + rm /tmp/tmp.0Fb6TUTXIs /tmp/tmp.z4uH6VzV8b + return 0 + kubectl_bin wait --for=delete namespace cross-site-sharded-2618 ++ mktemp + local LAST_OUT=/tmp/tmp.dFk5J5NQAj ++ mktemp + local LAST_ERR=/tmp/tmp.Ky4Qgfhg8Q + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl wait --for=delete namespace cross-site-sharded-2618 namespace "cross-site-sharded-6517" deleted + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.dFk5J5NQAj + cat /tmp/tmp.Ky4Qgfhg8Q + rm /tmp/tmp.dFk5J5NQAj /tmp/tmp.Ky4Qgfhg8Q + return 0 + desc 'create namespace cross-site-sharded-2618' + set +o xtrace ----------------------------------------------------------------------------------- create namespace cross-site-sharded-2618 ----------------------------------------------------------------------------------- + kubectl_bin create namespace cross-site-sharded-2618 ++ mktemp + local LAST_OUT=/tmp/tmp.osgFg5w3sX ++ mktemp + local LAST_ERR=/tmp/tmp.RGbhjmST8f + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl create namespace cross-site-sharded-2618 + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.osgFg5w3sX namespace/cross-site-sharded-2618 created + cat /tmp/tmp.RGbhjmST8f + rm /tmp/tmp.osgFg5w3sX /tmp/tmp.RGbhjmST8f + return 0 + set_kube_ctx cross-site-sharded-2618 + local namespace=cross-site-sharded-2618 ++ kubectl_bin config current-context +++ mktemp ++ local LAST_OUT=/tmp/tmp.6M2xDcR2EP +++ mktemp ++ local LAST_ERR=/tmp/tmp.4ggmLEpJm7 ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl config current-context ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.6M2xDcR2EP ++ cat /tmp/tmp.4ggmLEpJm7 ++ rm /tmp/tmp.6M2xDcR2EP /tmp/tmp.4ggmLEpJm7 ++ return 0 + kubectl_bin config set-context gke_cloud-dev-112233_us-central1-a_jen-psmdb-2283-73ca3795-11-cluster3 --namespace=cross-site-sharded-2618 ++ mktemp + local LAST_OUT=/tmp/tmp.GlQZ7LCqCX ++ mktemp + local LAST_ERR=/tmp/tmp.cWDb2EnG6J + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl config set-context gke_cloud-dev-112233_us-central1-a_jen-psmdb-2283-73ca3795-11-cluster3 --namespace=cross-site-sharded-2618 + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.GlQZ7LCqCX Context "gke_cloud-dev-112233_us-central1-a_jen-psmdb-2283-73ca3795-11-cluster3" modified. + cat /tmp/tmp.cWDb2EnG6J + rm /tmp/tmp.GlQZ7LCqCX /tmp/tmp.cWDb2EnG6J + return 0 + deploy_operator + desc 'start PSMDB operator: docker.io/perconalab/percona-server-mongodb-operator:PR-2283-73ca3795' + set +o xtrace ----------------------------------------------------------------------------------- start PSMDB operator: docker.io/perconalab/percona-server-mongodb-operator:PR-2283-73ca3795 ----------------------------------------------------------------------------------- + local cr_file + '[' -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2283/e2e-tests/cross-site-sharded/conf/crd.yaml ']' + cr_file=/mnt/jenkins/workspace/cloud-psmdb-operator_PR-2283/deploy/crd.yaml + kubectl_bin apply --server-side --force-conflicts -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2283/deploy/crd.yaml ++ mktemp + local LAST_OUT=/tmp/tmp.WsjXY0tL0b ++ mktemp + local LAST_ERR=/tmp/tmp.dhd748hvMb + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl apply --server-side --force-conflicts -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2283/deploy/crd.yaml + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.WsjXY0tL0b customresourcedefinition.apiextensions.k8s.io/perconaservermongodbbackups.psmdb.percona.com serverside-applied customresourcedefinition.apiextensions.k8s.io/perconaservermongodbrestores.psmdb.percona.com serverside-applied customresourcedefinition.apiextensions.k8s.io/perconaservermongodbs.psmdb.percona.com serverside-applied + cat /tmp/tmp.dhd748hvMb + rm /tmp/tmp.WsjXY0tL0b /tmp/tmp.dhd748hvMb + return 0 + '[' -n '' ']' + apply_rbac rbac + local operator_namespace=psmdb-operator + local rbac=rbac + cat /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2283/deploy/rbac.yaml + sed -e 's^namespace: .*^namespace: psmdb-operator^' + kubectl_bin apply -f - ++ mktemp + local LAST_OUT=/tmp/tmp.4EXH6wO1eq ++ mktemp + local LAST_ERR=/tmp/tmp.kpmeyOIqCP + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.4EXH6wO1eq role.rbac.authorization.k8s.io/percona-server-mongodb-operator created serviceaccount/percona-server-mongodb-operator created rolebinding.rbac.authorization.k8s.io/service-account-percona-server-mongodb-operator created + cat /tmp/tmp.kpmeyOIqCP + rm /tmp/tmp.4EXH6wO1eq /tmp/tmp.kpmeyOIqCP + return 0 + kubectl_bin apply -f - + yq eval $'\n\t\t\t(.spec.template.spec.containers[].image = "docker.io/perconalab/percona-server-mongodb-operator:PR-2283-73ca3795") |\n\t\t\t((.. | select(.[] == "DISABLE_TELEMETRY")) |= .value="true") |\n\t\t\t((.. | select(.[] == "LOG_LEVEL")) |= .value="DEBUG")' /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2283/deploy/operator.yaml ++ mktemp + local LAST_OUT=/tmp/tmp.IIAATOZWIY ++ mktemp + local LAST_ERR=/tmp/tmp.eIkhNdCbgl + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.IIAATOZWIY deployment.apps/percona-server-mongodb-operator created + cat /tmp/tmp.eIkhNdCbgl + rm /tmp/tmp.IIAATOZWIY /tmp/tmp.eIkhNdCbgl + return 0 + sleep 20 ++ get_operator_pod ++ kubectl_bin get pods --selector=name=percona-server-mongodb-operator -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.QBIutO2AKv +++ mktemp ++ local LAST_ERR=/tmp/tmp.SAx6LKawes ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pods --selector=name=percona-server-mongodb-operator -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.QBIutO2AKv ++ cat /tmp/tmp.SAx6LKawes ++ rm /tmp/tmp.QBIutO2AKv /tmp/tmp.SAx6LKawes ++ return 0 + wait_operator_pod percona-server-mongodb-operator-6d756c7f8f-pgcp5 + local pod=percona-server-mongodb-operator-6d756c7f8f-pgcp5 + set +o xtrace waiting for pod/percona-server-mongodb-operator-6d756c7f8f-pgcp5 to be ready.OK + echo 'Print operator info from log' Print operator info from log ++ get_operator_pod + grep 'Manager starting up' ++ kubectl_bin get pods --selector=name=percona-server-mongodb-operator -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.qqwlGXoBur +++ mktemp ++ local LAST_ERR=/tmp/tmp.gRe3Stkdbh ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pods --selector=name=percona-server-mongodb-operator -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.qqwlGXoBur ++ cat /tmp/tmp.gRe3Stkdbh ++ rm /tmp/tmp.qqwlGXoBur /tmp/tmp.gRe3Stkdbh ++ return 0 + kubectl_bin logs percona-server-mongodb-operator-6d756c7f8f-pgcp5 ++ mktemp + local LAST_OUT=/tmp/tmp.QRwnQI0kVX ++ mktemp + local LAST_ERR=/tmp/tmp.7xcEsvK0CY + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl logs percona-server-mongodb-operator-6d756c7f8f-pgcp5 + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.QRwnQI0kVX + cat /tmp/tmp.7xcEsvK0CY + rm /tmp/tmp.QRwnQI0kVX /tmp/tmp.7xcEsvK0CY + return 0 2026-04-09T16:18:16.656Z INFO setup Manager starting up {"gitCommit": "73ca37956356b4f3579a3ef1dce0c4105fda2da2", "gitBranch": "PR-2283-73ca3795", "buildTime": "", "goVersion": "go1.25.9", "os": "linux", "arch": "amd64"} + desc 'create secrets and start client' + set +o xtrace ----------------------------------------------------------------------------------- create secrets and start client ----------------------------------------------------------------------------------- + kubectl_bin apply -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2283/e2e-tests/conf/client.yml -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2283/e2e-tests/cross-site-sharded/conf/secrets.yml ++ mktemp + local LAST_OUT=/tmp/tmp.tDY8dMs1iM ++ mktemp + local LAST_ERR=/tmp/tmp.goTNVMQGDv + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl apply -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2283/e2e-tests/conf/client.yml -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2283/e2e-tests/cross-site-sharded/conf/secrets.yml + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.tDY8dMs1iM deployment.apps/psmdb-client created secret/cross-site-sharded-main-secrets created secret/cross-site-sharded-main-ssl created secret/cross-site-sharded-main-ssl-internal created + cat /tmp/tmp.goTNVMQGDv + rm /tmp/tmp.tDY8dMs1iM /tmp/tmp.goTNVMQGDv + return 0 + desc 'create main PSMDB cluster cross-site-sharded-main.' + set +o xtrace ----------------------------------------------------------------------------------- create main PSMDB cluster cross-site-sharded-main. ----------------------------------------------------------------------------------- + apply_cluster /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2283/e2e-tests/cross-site-sharded/conf/cross-site-sharded-main.yml + '[' -z '' ']' + cat_config /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2283/e2e-tests/cross-site-sharded/conf/cross-site-sharded-main.yml + kubectl_bin apply -f - + cat /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2283/e2e-tests/cross-site-sharded/conf/cross-site-sharded-main.yml + yq eval '(.spec | select(.image == null)).image = "docker.io/perconalab/percona-server-mongodb-operator:main-mongod8.0"' ++ mktemp + yq eval '(.spec | select(has("pmm"))).pmm.image = "docker.io/percona/pmm-client:2.44.1-1"' + yq eval '(.spec | select(has("initImage"))).initImage = "docker.io/perconalab/percona-server-mongodb-operator:PR-2283-73ca3795"' + /usr/sbin/sed -e s/NAME_SPACE/cross-site-sharded-2618/g + yq eval '(.spec | select(has("backup"))).backup.image = "docker.io/perconalab/percona-server-mongodb-operator:main-backup"' + local LAST_OUT=/tmp/tmp.YCtfC8Yq7l + yq eval '.spec.upgradeOptions.apply="Never"' ++ mktemp + local LAST_ERR=/tmp/tmp.v2gWX68odS + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.YCtfC8Yq7l perconaservermongodb.psmdb.percona.com/cross-site-sharded-main created + cat /tmp/tmp.v2gWX68odS + rm /tmp/tmp.YCtfC8Yq7l /tmp/tmp.v2gWX68odS + return 0 + desc 'check if all 3 Pods started' + set +o xtrace ----------------------------------------------------------------------------------- check if all 3 Pods started ----------------------------------------------------------------------------------- + wait_for_running cross-site-sharded-main-rs0 3 + local name=cross-site-sharded-main-rs0 + let last_pod=2 + local check_cluster_readyness=true + set_debug + [[ 1 == 1 ]] + set -o xtrace + local rs_name=rs0 + local cluster_name=cross-site-sharded-main ++ seq 0 2 + for i in $(seq 0 $last_pod) + [[ 0 -eq 2 ]] + wait_pod cross-site-sharded-main-rs0-0 + local pod=cross-site-sharded-main-rs0-0 + set +o xtrace waiting for pod/cross-site-sharded-main-rs0-0 to be ready.............OK + for i in $(seq 0 $last_pod) + [[ 1 -eq 2 ]] + wait_pod cross-site-sharded-main-rs0-1 + local pod=cross-site-sharded-main-rs0-1 + set +o xtrace waiting for pod/cross-site-sharded-main-rs0-1 to be ready.............OK + for i in $(seq 0 $last_pod) + [[ 2 -eq 2 ]] ++ kubectl_bin get psmdb cross-site-sharded-main -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].arbiter.enabled}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.IXuxS5nRK9 +++ mktemp ++ local LAST_ERR=/tmp/tmp.eyyNg9ZEh1 ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb cross-site-sharded-main -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].arbiter.enabled}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.IXuxS5nRK9 ++ cat /tmp/tmp.eyyNg9ZEh1 ++ rm /tmp/tmp.IXuxS5nRK9 /tmp/tmp.eyyNg9ZEh1 ++ return 0 + [[ false == true ]] + wait_pod cross-site-sharded-main-rs0-2 + local pod=cross-site-sharded-main-rs0-2 + set +o xtrace waiting for pod/cross-site-sharded-main-rs0-2 to be ready............OK ++ kubectl_bin get psmdb cross-site-sharded-main -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].nonvoting.enabled}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.ydYzkA8xGR +++ mktemp ++ local LAST_ERR=/tmp/tmp.Ve7UB8kQCh ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb cross-site-sharded-main -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].nonvoting.enabled}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.ydYzkA8xGR ++ cat /tmp/tmp.Ve7UB8kQCh ++ rm /tmp/tmp.ydYzkA8xGR /tmp/tmp.Ve7UB8kQCh ++ return 0 + [[ false == true ]] ++ kubectl_bin get psmdb cross-site-sharded-main -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].hidden.enabled}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.Ak1GqdiLBs +++ mktemp ++ local LAST_ERR=/tmp/tmp.6KmvayVLau ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb cross-site-sharded-main -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].hidden.enabled}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.Ak1GqdiLBs ++ cat /tmp/tmp.6KmvayVLau ++ rm /tmp/tmp.Ak1GqdiLBs /tmp/tmp.6KmvayVLau ++ return 0 + [[ '' == true ]] + sleep 10 + [[ true == true ]] + set +x Waiting for cluster readyness................................................... + wait_for_running cross-site-sharded-main-cfg 3 false + local name=cross-site-sharded-main-cfg + let last_pod=2 + local check_cluster_readyness=false + set_debug + [[ 1 == 1 ]] + set -o xtrace + local rs_name=cfg + local cluster_name=cross-site-sharded-main ++ seq 0 2 + for i in $(seq 0 $last_pod) + [[ 0 -eq 2 ]] + wait_pod cross-site-sharded-main-cfg-0 + local pod=cross-site-sharded-main-cfg-0 + set +o xtrace waiting for pod/cross-site-sharded-main-cfg-0 to be ready.OK + for i in $(seq 0 $last_pod) + [[ 1 -eq 2 ]] + wait_pod cross-site-sharded-main-cfg-1 + local pod=cross-site-sharded-main-cfg-1 + set +o xtrace waiting for pod/cross-site-sharded-main-cfg-1 to be ready.OK + for i in $(seq 0 $last_pod) + [[ 2 -eq 2 ]] ++ kubectl_bin get psmdb cross-site-sharded-main -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].arbiter.enabled}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.BM9kygPheb +++ mktemp ++ local LAST_ERR=/tmp/tmp.uao1WDYhhH ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb cross-site-sharded-main -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].arbiter.enabled}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.BM9kygPheb ++ cat /tmp/tmp.uao1WDYhhH ++ rm /tmp/tmp.BM9kygPheb /tmp/tmp.uao1WDYhhH ++ return 0 + [[ '' == true ]] + wait_pod cross-site-sharded-main-cfg-2 + local pod=cross-site-sharded-main-cfg-2 + set +o xtrace waiting for pod/cross-site-sharded-main-cfg-2 to be ready.OK ++ kubectl_bin get psmdb cross-site-sharded-main -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].nonvoting.enabled}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.MU3ITy4Uty +++ mktemp ++ local LAST_ERR=/tmp/tmp.pnRi0M0X60 ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb cross-site-sharded-main -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].nonvoting.enabled}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.MU3ITy4Uty ++ cat /tmp/tmp.pnRi0M0X60 ++ rm /tmp/tmp.MU3ITy4Uty /tmp/tmp.pnRi0M0X60 ++ return 0 + [[ '' == true ]] ++ kubectl_bin get psmdb cross-site-sharded-main -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].hidden.enabled}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.l12jqoepYo +++ mktemp ++ local LAST_ERR=/tmp/tmp.aQq8WF4sAB ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb cross-site-sharded-main -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].hidden.enabled}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.l12jqoepYo ++ cat /tmp/tmp.aQq8WF4sAB ++ rm /tmp/tmp.l12jqoepYo /tmp/tmp.aQq8WF4sAB ++ return 0 + [[ '' == true ]] + sleep 10 + [[ false == true ]] + desc 'create user' + set +o xtrace ----------------------------------------------------------------------------------- create user ----------------------------------------------------------------------------------- + run_mongos 'db.createUser({user:"user",pwd:"pass",roles:[{db:"app",role:"readWrite"}]})' userAdmin:userAdmin123456@cross-site-sharded-main-mongos.cross-site-sharded-2618 + local 'command=db.createUser({user:"user",pwd:"pass",roles:[{db:"app",role:"readWrite"}]})' + local uri=userAdmin:userAdmin123456@cross-site-sharded-main-mongos.cross-site-sharded-2618 + local driver=mongodb + local suffix=.svc.cluster.local + local mongo_flag= + local port=27017 + local mongo_bin=mongo ++ echo .svc.cluster.local ++ awk -F: '{print $2}' + suffix_port= + [[ -z '' ]] + suffix=.svc.cluster.local:27017 ++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.IRz0X74Djx +++ mktemp ++ local LAST_ERR=/tmp/tmp.c12K7KSwGm ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.IRz0X74Djx ++ cat /tmp/tmp.c12K7KSwGm ++ rm /tmp/tmp.IRz0X74Djx /tmp/tmp.c12K7KSwGm ++ return 0 + local client_container=psmdb-client-bb8b97679-xl6jr + kubectl_bin exec psmdb-client-bb8b97679-xl6jr -- bash -c 'printf '\''db.createUser({user:"user",pwd:"pass",roles:[{db:"app",role:"readWrite"}]})\n'\'' | mongo mongodb://userAdmin:userAdmin123456@cross-site-sharded-main-mongos.cross-site-sharded-2618.svc.cluster.local:27017/admin ' ++ mktemp + local LAST_OUT=/tmp/tmp.O0QC89hDtp ++ mktemp + local LAST_ERR=/tmp/tmp.XtL1EKkZN8 + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl exec psmdb-client-bb8b97679-xl6jr -- bash -c 'printf '\''db.createUser({user:"user",pwd:"pass",roles:[{db:"app",role:"readWrite"}]})\n'\'' | mongo mongodb://userAdmin:userAdmin123456@cross-site-sharded-main-mongos.cross-site-sharded-2618.svc.cluster.local:27017/admin ' + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.O0QC89hDtp Percona Server for MongoDB shell version v4.4.29-28 connecting to: mongodb://cross-site-sharded-main-mongos.cross-site-sharded-2618.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb Implicit session: session { "id" : UUID("aa64a62e-a6c5-407a-9094-4a0abbec6dc3") } Percona Server for MongoDB server version: v8.0.20-8 WARNING: shell and server versions do not match Successfully added user: { "user" : "user", "roles" : [ { "db" : "app", "role" : "readWrite" } ] } bye + cat /tmp/tmp.XtL1EKkZN8 + rm /tmp/tmp.O0QC89hDtp /tmp/tmp.XtL1EKkZN8 + return 0 + sleep 2 + desc 'set chunk size to 2 MB' + set +o xtrace ----------------------------------------------------------------------------------- set chunk size to 2 MB ----------------------------------------------------------------------------------- + run_mongos 'use config\n db.settings.save( { _id:"chunksize", value: 2 } )' clusterAdmin:clusterAdmin123456@cross-site-sharded-main-mongos.cross-site-sharded-2618 + local 'command=use config\n db.settings.save( { _id:"chunksize", value: 2 } )' + local uri=clusterAdmin:clusterAdmin123456@cross-site-sharded-main-mongos.cross-site-sharded-2618 + local driver=mongodb + local suffix=.svc.cluster.local + local mongo_flag= + local port=27017 + local mongo_bin=mongo ++ echo .svc.cluster.local ++ awk -F: '{print $2}' + suffix_port= + [[ -z '' ]] + suffix=.svc.cluster.local:27017 ++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.e9ORxv3cQ4 +++ mktemp ++ local LAST_ERR=/tmp/tmp.doWpZ4YXPP ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.e9ORxv3cQ4 ++ cat /tmp/tmp.doWpZ4YXPP ++ rm /tmp/tmp.e9ORxv3cQ4 /tmp/tmp.doWpZ4YXPP ++ return 0 + local client_container=psmdb-client-bb8b97679-xl6jr + kubectl_bin exec psmdb-client-bb8b97679-xl6jr -- bash -c 'printf '\''use config\n db.settings.save( { _id:"chunksize", value: 2 } )\n'\'' | mongo mongodb://clusterAdmin:clusterAdmin123456@cross-site-sharded-main-mongos.cross-site-sharded-2618.svc.cluster.local:27017/admin ' ++ mktemp + local LAST_OUT=/tmp/tmp.PmB1BfzmIf ++ mktemp + local LAST_ERR=/tmp/tmp.fVygLosmju + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl exec psmdb-client-bb8b97679-xl6jr -- bash -c 'printf '\''use config\n db.settings.save( { _id:"chunksize", value: 2 } )\n'\'' | mongo mongodb://clusterAdmin:clusterAdmin123456@cross-site-sharded-main-mongos.cross-site-sharded-2618.svc.cluster.local:27017/admin ' + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.PmB1BfzmIf Percona Server for MongoDB shell version v4.4.29-28 connecting to: mongodb://cross-site-sharded-main-mongos.cross-site-sharded-2618.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb Implicit session: session { "id" : UUID("ad22a924-b42b-4279-9be0-fb44a7d8a200") } Percona Server for MongoDB server version: v8.0.20-8 WARNING: shell and server versions do not match switched to db config WriteResult({ "nMatched" : 0, "nUpserted" : 1, "nModified" : 0, "_id" : "chunksize" }) bye + cat /tmp/tmp.fVygLosmju + rm /tmp/tmp.PmB1BfzmIf /tmp/tmp.fVygLosmju + return 0 + sleep 2 + desc 'write data' + set +o xtrace ----------------------------------------------------------------------------------- write data ----------------------------------------------------------------------------------- + run_script_mongos /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2283/e2e-tests/cross-site-sharded/data.js user:pass@cross-site-sharded-main-mongos.cross-site-sharded-2618 + local script=/mnt/jenkins/workspace/cloud-psmdb-operator_PR-2283/e2e-tests/cross-site-sharded/data.js + local uri=user:pass@cross-site-sharded-main-mongos.cross-site-sharded-2618 + local driver=mongodb + local suffix=.svc.cluster.local + local mongo_flag= + local mongo_bin=mongo ++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.OvQ04xcmLT +++ mktemp ++ local LAST_ERR=/tmp/tmp.ChEDWl4UOJ ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.OvQ04xcmLT ++ cat /tmp/tmp.ChEDWl4UOJ ++ rm /tmp/tmp.OvQ04xcmLT /tmp/tmp.ChEDWl4UOJ ++ return 0 + local client_container=psmdb-client-bb8b97679-xl6jr ++ basename /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2283/e2e-tests/cross-site-sharded/data.js + name=data.js + kubectl_bin cp /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2283/e2e-tests/cross-site-sharded/data.js cross-site-sharded-2618/psmdb-client-bb8b97679-xl6jr:/tmp ++ mktemp + local LAST_OUT=/tmp/tmp.sjLUZOBKvJ ++ mktemp + local LAST_ERR=/tmp/tmp.G58mgE9bCy + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl cp /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2283/e2e-tests/cross-site-sharded/data.js cross-site-sharded-2618/psmdb-client-bb8b97679-xl6jr:/tmp + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.sjLUZOBKvJ + cat /tmp/tmp.G58mgE9bCy + rm /tmp/tmp.sjLUZOBKvJ /tmp/tmp.G58mgE9bCy + return 0 + kubectl_bin exec psmdb-client-bb8b97679-xl6jr -- bash -c 'mongo mongodb://user:pass@cross-site-sharded-main-mongos.cross-site-sharded-2618.svc.cluster.local/admin /tmp/data.js' ++ mktemp + local LAST_OUT=/tmp/tmp.rbtOL4At6p ++ mktemp + local LAST_ERR=/tmp/tmp.dd8lBqLtck + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl exec psmdb-client-bb8b97679-xl6jr -- bash -c 'mongo mongodb://user:pass@cross-site-sharded-main-mongos.cross-site-sharded-2618.svc.cluster.local/admin /tmp/data.js' + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.rbtOL4At6p Percona Server for MongoDB shell version v4.4.29-28 connecting to: mongodb://cross-site-sharded-main-mongos.cross-site-sharded-2618.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb Implicit session: session { "id" : UUID("358656dd-e567-4619-8ae6-de75535f488d") } Percona Server for MongoDB server version: v8.0.20-8 WARNING: shell and server versions do not match + cat /tmp/tmp.dd8lBqLtck + rm /tmp/tmp.rbtOL4At6p /tmp/tmp.dd8lBqLtck + return 0 + desc 'shard collection' + set +o xtrace ----------------------------------------------------------------------------------- shard collection ----------------------------------------------------------------------------------- + run_mongos 'sh.enableSharding("app")' clusterAdmin:clusterAdmin123456@cross-site-sharded-main-mongos.cross-site-sharded-2618 + local 'command=sh.enableSharding("app")' + local uri=clusterAdmin:clusterAdmin123456@cross-site-sharded-main-mongos.cross-site-sharded-2618 + local driver=mongodb + local suffix=.svc.cluster.local + local mongo_flag= + local port=27017 + local mongo_bin=mongo ++ echo .svc.cluster.local ++ awk -F: '{print $2}' + suffix_port= + [[ -z '' ]] + suffix=.svc.cluster.local:27017 ++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.0jl5jegjmE +++ mktemp ++ local LAST_ERR=/tmp/tmp.BTIURf41LB ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.0jl5jegjmE ++ cat /tmp/tmp.BTIURf41LB ++ rm /tmp/tmp.0jl5jegjmE /tmp/tmp.BTIURf41LB ++ return 0 + local client_container=psmdb-client-bb8b97679-xl6jr + kubectl_bin exec psmdb-client-bb8b97679-xl6jr -- bash -c 'printf '\''sh.enableSharding("app")\n'\'' | mongo mongodb://clusterAdmin:clusterAdmin123456@cross-site-sharded-main-mongos.cross-site-sharded-2618.svc.cluster.local:27017/admin ' ++ mktemp + local LAST_OUT=/tmp/tmp.gDB0nOXsj4 ++ mktemp + local LAST_ERR=/tmp/tmp.qtljVRSLSd + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl exec psmdb-client-bb8b97679-xl6jr -- bash -c 'printf '\''sh.enableSharding("app")\n'\'' | mongo mongodb://clusterAdmin:clusterAdmin123456@cross-site-sharded-main-mongos.cross-site-sharded-2618.svc.cluster.local:27017/admin ' + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.gDB0nOXsj4 Percona Server for MongoDB shell version v4.4.29-28 connecting to: mongodb://cross-site-sharded-main-mongos.cross-site-sharded-2618.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb Implicit session: session { "id" : UUID("fd0e35d1-cd17-4b24-9431-b5c119e41a5e") } Percona Server for MongoDB server version: v8.0.20-8 WARNING: shell and server versions do not match { "ok" : 1, "$clusterTime" : { "clusterTime" : Timestamp(1775751789, 1), "signature" : { "hash" : BinData(0,"diVFDaCRdbG+ADoWFfCFlSA8AT0="), "keyId" : NumberLong("7626795322697580550") } }, "operationTime" : Timestamp(1775751789, 1) } bye + cat /tmp/tmp.qtljVRSLSd + rm /tmp/tmp.gDB0nOXsj4 /tmp/tmp.qtljVRSLSd + return 0 + sleep 2 + run_mongos 'sh.shardCollection("app.city", { _id: 1 } )' clusterAdmin:clusterAdmin123456@cross-site-sharded-main-mongos.cross-site-sharded-2618 + local 'command=sh.shardCollection("app.city", { _id: 1 } )' + local uri=clusterAdmin:clusterAdmin123456@cross-site-sharded-main-mongos.cross-site-sharded-2618 + local driver=mongodb + local suffix=.svc.cluster.local + local mongo_flag= + local port=27017 + local mongo_bin=mongo ++ echo .svc.cluster.local ++ awk -F: '{print $2}' + suffix_port= + [[ -z '' ]] + suffix=.svc.cluster.local:27017 ++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.5dm2Qd1MNB +++ mktemp ++ local LAST_ERR=/tmp/tmp.SjbMxsw797 ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.5dm2Qd1MNB ++ cat /tmp/tmp.SjbMxsw797 ++ rm /tmp/tmp.5dm2Qd1MNB /tmp/tmp.SjbMxsw797 ++ return 0 + local client_container=psmdb-client-bb8b97679-xl6jr + kubectl_bin exec psmdb-client-bb8b97679-xl6jr -- bash -c 'printf '\''sh.shardCollection("app.city", { _id: 1 } )\n'\'' | mongo mongodb://clusterAdmin:clusterAdmin123456@cross-site-sharded-main-mongos.cross-site-sharded-2618.svc.cluster.local:27017/admin ' ++ mktemp + local LAST_OUT=/tmp/tmp.9rtZ6eGg8U ++ mktemp + local LAST_ERR=/tmp/tmp.hpJj5JMCGf + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl exec psmdb-client-bb8b97679-xl6jr -- bash -c 'printf '\''sh.shardCollection("app.city", { _id: 1 } )\n'\'' | mongo mongodb://clusterAdmin:clusterAdmin123456@cross-site-sharded-main-mongos.cross-site-sharded-2618.svc.cluster.local:27017/admin ' + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.9rtZ6eGg8U Percona Server for MongoDB shell version v4.4.29-28 connecting to: mongodb://cross-site-sharded-main-mongos.cross-site-sharded-2618.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb Implicit session: session { "id" : UUID("062c70ac-e337-4bd7-b874-e017cb6fabe3") } Percona Server for MongoDB server version: v8.0.20-8 WARNING: shell and server versions do not match { "collectionsharded" : "app.city", "ok" : 1, "$clusterTime" : { "clusterTime" : Timestamp(1775751795, 18), "signature" : { "hash" : BinData(0,"qurpzVs8y76t5DwwCVjd5QJJJnU="), "keyId" : NumberLong("7626795322697580550") } }, "operationTime" : Timestamp(1775751795, 17) } bye + cat /tmp/tmp.hpJj5JMCGf + rm /tmp/tmp.9rtZ6eGg8U /tmp/tmp.hpJj5JMCGf + return 0 + sleep 120 + desc 'Check chunks' + set +o xtrace ----------------------------------------------------------------------------------- Check chunks ----------------------------------------------------------------------------------- + chunks_param1=ns + chunks_param2='"app.city"' + [[ 8.0 != 4\.4 ]] + chunks_param1=uuid ++ run_mongos 'use app\n db.getCollectionInfos({ "name": "city" })[0].info.uuid' user:pass@cross-site-sharded-main-mongos.cross-site-sharded-2618 ++ local 'command=use app\n db.getCollectionInfos({ "name": "city" })[0].info.uuid' ++ local uri=user:pass@cross-site-sharded-main-mongos.cross-site-sharded-2618 ++ local driver=mongodb ++ local suffix=.svc.cluster.local ++ local mongo_flag= ++ local port=27017 ++ local mongo_bin=mongo ++ grep 'switched to db app' -A 1 ++ grep -v 'switched to db app' +++ echo .svc.cluster.local +++ awk -F: '{print $2}' ++ suffix_port= ++ [[ -z '' ]] ++ suffix=.svc.cluster.local:27017 +++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.B06VXtktsv ++++ mktemp +++ local LAST_ERR=/tmp/tmp.waBPbx5DmK +++ local exit_status=0 +++ local timeout=4 ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 -a -n 1 ']' +++ break +++ cat /tmp/tmp.B06VXtktsv +++ cat /tmp/tmp.waBPbx5DmK +++ rm /tmp/tmp.B06VXtktsv /tmp/tmp.waBPbx5DmK +++ return 0 ++ local client_container=psmdb-client-bb8b97679-xl6jr ++ kubectl_bin exec psmdb-client-bb8b97679-xl6jr -- bash -c 'printf '\''use app\n db.getCollectionInfos({ "name": "city" })[0].info.uuid\n'\'' | mongo mongodb://user:pass@cross-site-sharded-main-mongos.cross-site-sharded-2618.svc.cluster.local:27017/admin ' +++ mktemp ++ local LAST_OUT=/tmp/tmp.pMna8DxPjg +++ mktemp ++ local LAST_ERR=/tmp/tmp.6ydOjD3LQo ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl exec psmdb-client-bb8b97679-xl6jr -- bash -c 'printf '\''use app\n db.getCollectionInfos({ "name": "city" })[0].info.uuid\n'\'' | mongo mongodb://user:pass@cross-site-sharded-main-mongos.cross-site-sharded-2618.svc.cluster.local:27017/admin ' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.pMna8DxPjg ++ cat /tmp/tmp.6ydOjD3LQo ++ rm /tmp/tmp.pMna8DxPjg /tmp/tmp.6ydOjD3LQo ++ return 0 + chunks_param2='UUID("7dc94cfa-bc1f-4813-a84d-eff4d96a90f5")' + shards=0 + for i in "rs0" "rs1" ++ run_mongos 'use config\n db.chunks.count({"uuid": UUID("7dc94cfa-bc1f-4813-a84d-eff4d96a90f5"), "shard": "rs0"})' clusterAdmin:clusterAdmin123456@cross-site-sharded-main-mongos.cross-site-sharded-2618 ++ local 'command=use config\n db.chunks.count({"uuid": UUID("7dc94cfa-bc1f-4813-a84d-eff4d96a90f5"), "shard": "rs0"})' ++ local uri=clusterAdmin:clusterAdmin123456@cross-site-sharded-main-mongos.cross-site-sharded-2618 ++ local driver=mongodb ++ local suffix=.svc.cluster.local ++ local mongo_flag= ++ local port=27017 ++ local mongo_bin=mongo ++ grep -v 'switched to db config' ++ grep 'switched to db config' -A 1 +++ echo .svc.cluster.local +++ awk -F: '{print $2}' ++ suffix_port= ++ [[ -z '' ]] ++ suffix=.svc.cluster.local:27017 +++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.hV2fvZG4gz ++++ mktemp +++ local LAST_ERR=/tmp/tmp.Oi6WKHVNwB +++ local exit_status=0 +++ local timeout=4 ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 -a -n 1 ']' +++ break +++ cat /tmp/tmp.hV2fvZG4gz +++ cat /tmp/tmp.Oi6WKHVNwB +++ rm /tmp/tmp.hV2fvZG4gz /tmp/tmp.Oi6WKHVNwB +++ return 0 ++ local client_container=psmdb-client-bb8b97679-xl6jr ++ kubectl_bin exec psmdb-client-bb8b97679-xl6jr -- bash -c 'printf '\''use config\n db.chunks.count({"uuid": UUID("7dc94cfa-bc1f-4813-a84d-eff4d96a90f5"), "shard": "rs0"})\n'\'' | mongo mongodb://clusterAdmin:clusterAdmin123456@cross-site-sharded-main-mongos.cross-site-sharded-2618.svc.cluster.local:27017/admin ' +++ mktemp ++ local LAST_OUT=/tmp/tmp.VX8SaT1d5D +++ mktemp ++ local LAST_ERR=/tmp/tmp.nen4KXGcqk ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl exec psmdb-client-bb8b97679-xl6jr -- bash -c 'printf '\''use config\n db.chunks.count({"uuid": UUID("7dc94cfa-bc1f-4813-a84d-eff4d96a90f5"), "shard": "rs0"})\n'\'' | mongo mongodb://clusterAdmin:clusterAdmin123456@cross-site-sharded-main-mongos.cross-site-sharded-2618.svc.cluster.local:27017/admin ' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.VX8SaT1d5D ++ cat /tmp/tmp.nen4KXGcqk ++ rm /tmp/tmp.VX8SaT1d5D /tmp/tmp.nen4KXGcqk ++ return 0 + out=1 + desc 'rs0 has 1 chunks' + set +o xtrace ----------------------------------------------------------------------------------- rs0 has 1 chunks ----------------------------------------------------------------------------------- + [[ 1 -ne 0 ]] + (( shards = shards + 1 )) + for i in "rs0" "rs1" ++ run_mongos 'use config\n db.chunks.count({"uuid": UUID("7dc94cfa-bc1f-4813-a84d-eff4d96a90f5"), "shard": "rs1"})' clusterAdmin:clusterAdmin123456@cross-site-sharded-main-mongos.cross-site-sharded-2618 ++ local 'command=use config\n db.chunks.count({"uuid": UUID("7dc94cfa-bc1f-4813-a84d-eff4d96a90f5"), "shard": "rs1"})' ++ local uri=clusterAdmin:clusterAdmin123456@cross-site-sharded-main-mongos.cross-site-sharded-2618 ++ local driver=mongodb ++ local suffix=.svc.cluster.local ++ local mongo_flag= ++ local port=27017 ++ local mongo_bin=mongo ++ grep 'switched to db config' -A 1 ++ grep -v 'switched to db config' +++ echo .svc.cluster.local +++ awk -F: '{print $2}' ++ suffix_port= ++ [[ -z '' ]] ++ suffix=.svc.cluster.local:27017 +++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.B4ht8xMJbk ++++ mktemp +++ local LAST_ERR=/tmp/tmp.ApYTamtAhp +++ local exit_status=0 +++ local timeout=4 ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 -a -n 1 ']' +++ break +++ cat /tmp/tmp.B4ht8xMJbk +++ cat /tmp/tmp.ApYTamtAhp +++ rm /tmp/tmp.B4ht8xMJbk /tmp/tmp.ApYTamtAhp +++ return 0 ++ local client_container=psmdb-client-bb8b97679-xl6jr ++ kubectl_bin exec psmdb-client-bb8b97679-xl6jr -- bash -c 'printf '\''use config\n db.chunks.count({"uuid": UUID("7dc94cfa-bc1f-4813-a84d-eff4d96a90f5"), "shard": "rs1"})\n'\'' | mongo mongodb://clusterAdmin:clusterAdmin123456@cross-site-sharded-main-mongos.cross-site-sharded-2618.svc.cluster.local:27017/admin ' +++ mktemp ++ local LAST_OUT=/tmp/tmp.6pYls5IS1Y +++ mktemp ++ local LAST_ERR=/tmp/tmp.WSB5jb9A7z ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl exec psmdb-client-bb8b97679-xl6jr -- bash -c 'printf '\''use config\n db.chunks.count({"uuid": UUID("7dc94cfa-bc1f-4813-a84d-eff4d96a90f5"), "shard": "rs1"})\n'\'' | mongo mongodb://clusterAdmin:clusterAdmin123456@cross-site-sharded-main-mongos.cross-site-sharded-2618.svc.cluster.local:27017/admin ' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.6pYls5IS1Y ++ cat /tmp/tmp.WSB5jb9A7z ++ rm /tmp/tmp.6pYls5IS1Y /tmp/tmp.WSB5jb9A7z ++ return 0 + out=3 + desc 'rs1 has 3 chunks' + set +o xtrace ----------------------------------------------------------------------------------- rs1 has 3 chunks ----------------------------------------------------------------------------------- + [[ 3 -ne 0 ]] + (( shards = shards + 1 )) + [[ 2 -lt 2 ]] + desc 'create replica cluster' + set +o xtrace ----------------------------------------------------------------------------------- create replica cluster ----------------------------------------------------------------------------------- + create_namespace cross-site-sharded-replica-15236 0 + local namespace=cross-site-sharded-replica-15236 + local skip_clean_namespace=0 + [[ 1 == 1 ]] + [[ -z 0 ]] + '[' -n '' ']' + desc 'cleaned up old namespaces cross-site-sharded-replica-15236' + set +o xtrace ----------------------------------------------------------------------------------- cleaned up old namespaces cross-site-sharded-replica-15236 ----------------------------------------------------------------------------------- + kubectl_bin delete namespace cross-site-sharded-replica-15236 --ignore-not-found ++ mktemp + local LAST_OUT=/tmp/tmp.SNpxYSLlZh ++ mktemp + local LAST_ERR=/tmp/tmp.oDTwI9sfXR + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl delete namespace cross-site-sharded-replica-15236 --ignore-not-found + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.SNpxYSLlZh + cat /tmp/tmp.oDTwI9sfXR + rm /tmp/tmp.SNpxYSLlZh /tmp/tmp.oDTwI9sfXR + return 0 + kubectl_bin wait --for=delete namespace cross-site-sharded-replica-15236 ++ mktemp + local LAST_OUT=/tmp/tmp.3r5ZhZoPJ9 ++ mktemp + local LAST_ERR=/tmp/tmp.526AyLuMEA + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl wait --for=delete namespace cross-site-sharded-replica-15236 + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.3r5ZhZoPJ9 + cat /tmp/tmp.526AyLuMEA + rm /tmp/tmp.3r5ZhZoPJ9 /tmp/tmp.526AyLuMEA + return 0 + desc 'create namespace cross-site-sharded-replica-15236' + set +o xtrace ----------------------------------------------------------------------------------- create namespace cross-site-sharded-replica-15236 ----------------------------------------------------------------------------------- + kubectl_bin create namespace cross-site-sharded-replica-15236 ++ mktemp + local LAST_OUT=/tmp/tmp.xP9eiBfXuj ++ mktemp + local LAST_ERR=/tmp/tmp.V0f4CLgbm5 + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl create namespace cross-site-sharded-replica-15236 + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.xP9eiBfXuj namespace/cross-site-sharded-replica-15236 created + cat /tmp/tmp.V0f4CLgbm5 + rm /tmp/tmp.xP9eiBfXuj /tmp/tmp.V0f4CLgbm5 + return 0 + set_kube_ctx cross-site-sharded-replica-15236 + local namespace=cross-site-sharded-replica-15236 ++ kubectl_bin config current-context +++ mktemp ++ local LAST_OUT=/tmp/tmp.pANMk7LY9q +++ mktemp ++ local LAST_ERR=/tmp/tmp.xVZ7eVc4vS ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl config current-context ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.pANMk7LY9q ++ cat /tmp/tmp.xVZ7eVc4vS ++ rm /tmp/tmp.pANMk7LY9q /tmp/tmp.xVZ7eVc4vS ++ return 0 + kubectl_bin config set-context gke_cloud-dev-112233_us-central1-a_jen-psmdb-2283-73ca3795-11-cluster3 --namespace=cross-site-sharded-replica-15236 ++ mktemp + local LAST_OUT=/tmp/tmp.GlyVMoCyuz ++ mktemp + local LAST_ERR=/tmp/tmp.3YxaLeOY37 + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl config set-context gke_cloud-dev-112233_us-central1-a_jen-psmdb-2283-73ca3795-11-cluster3 --namespace=cross-site-sharded-replica-15236 + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.GlyVMoCyuz Context "gke_cloud-dev-112233_us-central1-a_jen-psmdb-2283-73ca3795-11-cluster3" modified. + cat /tmp/tmp.3YxaLeOY37 + rm /tmp/tmp.GlyVMoCyuz /tmp/tmp.3YxaLeOY37 + return 0 + deploy_operator + desc 'start PSMDB operator: docker.io/perconalab/percona-server-mongodb-operator:PR-2283-73ca3795' + set +o xtrace ----------------------------------------------------------------------------------- start PSMDB operator: docker.io/perconalab/percona-server-mongodb-operator:PR-2283-73ca3795 ----------------------------------------------------------------------------------- + local cr_file + '[' -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2283/e2e-tests/cross-site-sharded/conf/crd.yaml ']' + cr_file=/mnt/jenkins/workspace/cloud-psmdb-operator_PR-2283/deploy/crd.yaml + kubectl_bin apply --server-side --force-conflicts -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2283/deploy/crd.yaml ++ mktemp + local LAST_OUT=/tmp/tmp.G6FecYO0xW ++ mktemp + local LAST_ERR=/tmp/tmp.yAM56Nwjmd + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl apply --server-side --force-conflicts -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2283/deploy/crd.yaml + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.G6FecYO0xW customresourcedefinition.apiextensions.k8s.io/perconaservermongodbbackups.psmdb.percona.com serverside-applied customresourcedefinition.apiextensions.k8s.io/perconaservermongodbrestores.psmdb.percona.com serverside-applied customresourcedefinition.apiextensions.k8s.io/perconaservermongodbs.psmdb.percona.com serverside-applied + cat /tmp/tmp.yAM56Nwjmd + rm /tmp/tmp.G6FecYO0xW /tmp/tmp.yAM56Nwjmd + return 0 + '[' -n '' ']' + apply_rbac rbac + local operator_namespace=psmdb-operator + local rbac=rbac + cat /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2283/deploy/rbac.yaml + sed -e 's^namespace: .*^namespace: psmdb-operator^' + kubectl_bin apply -f - ++ mktemp + local LAST_OUT=/tmp/tmp.NeYVHJhPt4 ++ mktemp + local LAST_ERR=/tmp/tmp.Ee2uJQ2i23 + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.NeYVHJhPt4 role.rbac.authorization.k8s.io/percona-server-mongodb-operator created serviceaccount/percona-server-mongodb-operator created rolebinding.rbac.authorization.k8s.io/service-account-percona-server-mongodb-operator created + cat /tmp/tmp.Ee2uJQ2i23 + rm /tmp/tmp.NeYVHJhPt4 /tmp/tmp.Ee2uJQ2i23 + return 0 + yq eval $'\n\t\t\t(.spec.template.spec.containers[].image = "docker.io/perconalab/percona-server-mongodb-operator:PR-2283-73ca3795") |\n\t\t\t((.. | select(.[] == "DISABLE_TELEMETRY")) |= .value="true") |\n\t\t\t((.. | select(.[] == "LOG_LEVEL")) |= .value="DEBUG")' /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2283/deploy/operator.yaml + kubectl_bin apply -f - ++ mktemp + local LAST_OUT=/tmp/tmp.jq4QYi2jw2 ++ mktemp + local LAST_ERR=/tmp/tmp.vmIQDYlavz + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.jq4QYi2jw2 deployment.apps/percona-server-mongodb-operator created + cat /tmp/tmp.vmIQDYlavz + rm /tmp/tmp.jq4QYi2jw2 /tmp/tmp.vmIQDYlavz + return 0 + sleep 20 ++ get_operator_pod ++ kubectl_bin get pods --selector=name=percona-server-mongodb-operator -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.0MF49XzvlR +++ mktemp ++ local LAST_ERR=/tmp/tmp.X5Lz8I8xEd ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pods --selector=name=percona-server-mongodb-operator -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.0MF49XzvlR ++ cat /tmp/tmp.X5Lz8I8xEd ++ rm /tmp/tmp.0MF49XzvlR /tmp/tmp.X5Lz8I8xEd ++ return 0 + wait_operator_pod percona-server-mongodb-operator-6d756c7f8f-vwnhh + local pod=percona-server-mongodb-operator-6d756c7f8f-vwnhh + set +o xtrace waiting for pod/percona-server-mongodb-operator-6d756c7f8f-vwnhh to be ready.OK + echo 'Print operator info from log' Print operator info from log + grep 'Manager starting up' ++ get_operator_pod ++ kubectl_bin get pods --selector=name=percona-server-mongodb-operator -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.xpbWCTmQqb +++ mktemp ++ local LAST_ERR=/tmp/tmp.pR1T4KAnWc ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pods --selector=name=percona-server-mongodb-operator -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.xpbWCTmQqb ++ cat /tmp/tmp.pR1T4KAnWc ++ rm /tmp/tmp.xpbWCTmQqb /tmp/tmp.pR1T4KAnWc ++ return 0 + kubectl_bin logs percona-server-mongodb-operator-6d756c7f8f-vwnhh ++ mktemp + local LAST_OUT=/tmp/tmp.3IX6gBnN3y ++ mktemp + local LAST_ERR=/tmp/tmp.N6Wvafgh63 + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl logs percona-server-mongodb-operator-6d756c7f8f-vwnhh + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.3IX6gBnN3y + cat /tmp/tmp.N6Wvafgh63 + rm /tmp/tmp.3IX6gBnN3y /tmp/tmp.N6Wvafgh63 + return 0 2026-04-09T16:25:40.298Z INFO setup Manager starting up {"gitCommit": "73ca37956356b4f3579a3ef1dce0c4105fda2da2", "gitBranch": "PR-2283-73ca3795", "buildTime": "", "goVersion": "go1.25.9", "os": "linux", "arch": "amd64"} + desc 'start client' + set +o xtrace ----------------------------------------------------------------------------------- start client ----------------------------------------------------------------------------------- + kubectl_bin apply -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2283/e2e-tests/conf/client.yml ++ mktemp + local LAST_OUT=/tmp/tmp.uAo9xycE9V ++ mktemp + local LAST_ERR=/tmp/tmp.KXGak3KsCC + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl apply -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2283/e2e-tests/conf/client.yml + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.uAo9xycE9V deployment.apps/psmdb-client created + cat /tmp/tmp.KXGak3KsCC + rm /tmp/tmp.uAo9xycE9V /tmp/tmp.KXGak3KsCC + return 0 + desc 'copy secrets from main to replica namespace and create all of them' + set +o xtrace ----------------------------------------------------------------------------------- copy secrets from main to replica namespace and create all of them ----------------------------------------------------------------------------------- + kubectl get secret cross-site-sharded-main-secrets -o yaml -n cross-site-sharded-2618 + yq eval $'\n\t\tdel(.metadata) |\n\t\t(.metadata.name = "cross-site-sharded-replica-secrets")' - + kubectl_bin apply -f - ++ mktemp + local LAST_OUT=/tmp/tmp.2V25T9yHDl ++ mktemp + local LAST_ERR=/tmp/tmp.Z8u3fz3Qw5 + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.2V25T9yHDl secret/cross-site-sharded-replica-secrets created + cat /tmp/tmp.Z8u3fz3Qw5 + rm /tmp/tmp.2V25T9yHDl /tmp/tmp.Z8u3fz3Qw5 + return 0 + kubectl_bin get secret cross-site-sharded-main-ssl-internal -o yaml -n cross-site-sharded-2618 + yq eval $'\n\t\tdel(.metadata) |\n\t\tdel(.status) |\n\t\t(.metadata.name = "cross-site-sharded-replica-ssl-internal")' - + kubectl_bin apply -f - ++ mktemp ++ mktemp + local LAST_OUT=/tmp/tmp.E8PPMI4fgR + local LAST_OUT=/tmp/tmp.pa5X8Apl8M ++ mktemp ++ mktemp + local LAST_ERR=/tmp/tmp.ekDGr4OqPH + local exit_status=0 + local timeout=4 ++ seq 0 2 + local LAST_ERR=/tmp/tmp.cTubVkDSev + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl apply -f - + for i in $(seq 0 2) + set +e + kubectl get secret cross-site-sharded-main-ssl-internal -o yaml -n cross-site-sharded-2618 + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.E8PPMI4fgR + cat /tmp/tmp.cTubVkDSev + rm /tmp/tmp.E8PPMI4fgR /tmp/tmp.cTubVkDSev + return 0 + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.pa5X8Apl8M secret/cross-site-sharded-replica-ssl-internal created + cat /tmp/tmp.ekDGr4OqPH + rm /tmp/tmp.pa5X8Apl8M /tmp/tmp.ekDGr4OqPH + return 0 + yq eval $'\n\t\tdel(.metadata) |\n\t\tdel(.status) |\n\t\t(.metadata.name = "cross-site-sharded-replica-ssl")' - + kubectl_bin apply -f - + kubectl_bin get secret cross-site-sharded-main-ssl -o yaml -n cross-site-sharded-2618 ++ mktemp ++ mktemp + local LAST_OUT=/tmp/tmp.tveTqcJzV8 ++ mktemp + local LAST_ERR=/tmp/tmp.HsGCiTWaUX + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl apply -f - + local LAST_OUT=/tmp/tmp.S35u0kKHnq ++ mktemp + local LAST_ERR=/tmp/tmp.ecJMHioDR8 + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl get secret cross-site-sharded-main-ssl -o yaml -n cross-site-sharded-2618 + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.S35u0kKHnq + cat /tmp/tmp.ecJMHioDR8 + rm /tmp/tmp.S35u0kKHnq /tmp/tmp.ecJMHioDR8 + return 0 + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.tveTqcJzV8 secret/cross-site-sharded-replica-ssl created + cat /tmp/tmp.HsGCiTWaUX + rm /tmp/tmp.tveTqcJzV8 /tmp/tmp.HsGCiTWaUX + return 0 + sleep 30 + desc 'create replica PSMDB cluster ' + set +o xtrace ----------------------------------------------------------------------------------- create replica PSMDB cluster ----------------------------------------------------------------------------------- + apply_cluster /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2283/e2e-tests/cross-site-sharded/conf/cross-site-sharded-replica.yml + '[' -z '' ']' + cat_config /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2283/e2e-tests/cross-site-sharded/conf/cross-site-sharded-replica.yml + kubectl_bin apply -f - + cat /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2283/e2e-tests/cross-site-sharded/conf/cross-site-sharded-replica.yml + yq eval '(.spec | select(.image == null)).image = "docker.io/perconalab/percona-server-mongodb-operator:main-mongod8.0"' ++ mktemp + yq eval '(.spec | select(has("pmm"))).pmm.image = "docker.io/percona/pmm-client:2.44.1-1"' + yq eval '(.spec | select(has("initImage"))).initImage = "docker.io/perconalab/percona-server-mongodb-operator:PR-2283-73ca3795"' + yq eval '(.spec | select(has("backup"))).backup.image = "docker.io/perconalab/percona-server-mongodb-operator:main-backup"' + /usr/sbin/sed -e s/NAME_SPACE/cross-site-sharded-2618/g + local LAST_OUT=/tmp/tmp.OfrUMHmqH3 + yq eval '.spec.upgradeOptions.apply="Never"' ++ mktemp + local LAST_ERR=/tmp/tmp.z5IvSiurQS + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.OfrUMHmqH3 perconaservermongodb.psmdb.percona.com/cross-site-sharded-replica created + cat /tmp/tmp.z5IvSiurQS + rm /tmp/tmp.OfrUMHmqH3 /tmp/tmp.z5IvSiurQS + return 0 + wait_for_running cross-site-sharded-replica-rs0 3 false + local name=cross-site-sharded-replica-rs0 + let last_pod=2 + local check_cluster_readyness=false + set_debug + [[ 1 == 1 ]] + set -o xtrace + local rs_name=rs0 + local cluster_name=cross-site-sharded-replica ++ seq 0 2 + for i in $(seq 0 $last_pod) + [[ 0 -eq 2 ]] + wait_pod cross-site-sharded-replica-rs0-0 + local pod=cross-site-sharded-replica-rs0-0 + set +o xtrace waiting for pod/cross-site-sharded-replica-rs0-0 to be ready.............OK + for i in $(seq 0 $last_pod) + [[ 1 -eq 2 ]] + wait_pod cross-site-sharded-replica-rs0-1 + local pod=cross-site-sharded-replica-rs0-1 + set +o xtrace waiting for pod/cross-site-sharded-replica-rs0-1 to be ready..............OK + for i in $(seq 0 $last_pod) + [[ 2 -eq 2 ]] ++ kubectl_bin get psmdb cross-site-sharded-replica -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].arbiter.enabled}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.95E3HKF1G7 +++ mktemp ++ local LAST_ERR=/tmp/tmp.SnBhrS65TV ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb cross-site-sharded-replica -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].arbiter.enabled}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.95E3HKF1G7 ++ cat /tmp/tmp.SnBhrS65TV ++ rm /tmp/tmp.95E3HKF1G7 /tmp/tmp.SnBhrS65TV ++ return 0 + [[ false == true ]] + wait_pod cross-site-sharded-replica-rs0-2 + local pod=cross-site-sharded-replica-rs0-2 + set +o xtrace waiting for pod/cross-site-sharded-replica-rs0-2 to be ready.............OK ++ kubectl_bin get psmdb cross-site-sharded-replica -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].nonvoting.enabled}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.yNobgGf0DW +++ mktemp ++ local LAST_ERR=/tmp/tmp.Uq0pF4X9fM ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb cross-site-sharded-replica -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].nonvoting.enabled}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.yNobgGf0DW ++ cat /tmp/tmp.Uq0pF4X9fM ++ rm /tmp/tmp.yNobgGf0DW /tmp/tmp.Uq0pF4X9fM ++ return 0 + [[ false == true ]] ++ kubectl_bin get psmdb cross-site-sharded-replica -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].hidden.enabled}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.RX7IyjC88I +++ mktemp ++ local LAST_ERR=/tmp/tmp.1XLjJoz90b ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb cross-site-sharded-replica -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].hidden.enabled}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.RX7IyjC88I ++ cat /tmp/tmp.1XLjJoz90b ++ rm /tmp/tmp.RX7IyjC88I /tmp/tmp.1XLjJoz90b ++ return 0 + [[ '' == true ]] + sleep 10 + [[ false == true ]] + wait_for_running cross-site-sharded-replica-rs1 3 false + local name=cross-site-sharded-replica-rs1 + let last_pod=2 + local check_cluster_readyness=false + set_debug + [[ 1 == 1 ]] + set -o xtrace + local rs_name=rs1 + local cluster_name=cross-site-sharded-replica ++ seq 0 2 + for i in $(seq 0 $last_pod) + [[ 0 -eq 2 ]] + wait_pod cross-site-sharded-replica-rs1-0 + local pod=cross-site-sharded-replica-rs1-0 + set +o xtrace waiting for pod/cross-site-sharded-replica-rs1-0 to be ready.OK + for i in $(seq 0 $last_pod) + [[ 1 -eq 2 ]] + wait_pod cross-site-sharded-replica-rs1-1 + local pod=cross-site-sharded-replica-rs1-1 + set +o xtrace waiting for pod/cross-site-sharded-replica-rs1-1 to be ready.OK + for i in $(seq 0 $last_pod) + [[ 2 -eq 2 ]] ++ kubectl_bin get psmdb cross-site-sharded-replica -o 'jsonpath={.spec.replsets[?(@.name=="rs1")].arbiter.enabled}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.N4ZxU7eUZB +++ mktemp ++ local LAST_ERR=/tmp/tmp.FiKEZ4Fwsx ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb cross-site-sharded-replica -o 'jsonpath={.spec.replsets[?(@.name=="rs1")].arbiter.enabled}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.N4ZxU7eUZB ++ cat /tmp/tmp.FiKEZ4Fwsx ++ rm /tmp/tmp.N4ZxU7eUZB /tmp/tmp.FiKEZ4Fwsx ++ return 0 + [[ false == true ]] + wait_pod cross-site-sharded-replica-rs1-2 + local pod=cross-site-sharded-replica-rs1-2 + set +o xtrace waiting for pod/cross-site-sharded-replica-rs1-2 to be ready.OK ++ kubectl_bin get psmdb cross-site-sharded-replica -o 'jsonpath={.spec.replsets[?(@.name=="rs1")].nonvoting.enabled}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.6UscnjMVJI +++ mktemp ++ local LAST_ERR=/tmp/tmp.j50FqGrMhF ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb cross-site-sharded-replica -o 'jsonpath={.spec.replsets[?(@.name=="rs1")].nonvoting.enabled}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.6UscnjMVJI ++ cat /tmp/tmp.j50FqGrMhF ++ rm /tmp/tmp.6UscnjMVJI /tmp/tmp.j50FqGrMhF ++ return 0 + [[ false == true ]] ++ kubectl_bin get psmdb cross-site-sharded-replica -o 'jsonpath={.spec.replsets[?(@.name=="rs1")].hidden.enabled}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.ZE75kZQKXh +++ mktemp ++ local LAST_ERR=/tmp/tmp.Jj97WwwLR0 ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb cross-site-sharded-replica -o 'jsonpath={.spec.replsets[?(@.name=="rs1")].hidden.enabled}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.ZE75kZQKXh ++ cat /tmp/tmp.Jj97WwwLR0 ++ rm /tmp/tmp.ZE75kZQKXh /tmp/tmp.Jj97WwwLR0 ++ return 0 + [[ '' == true ]] + sleep 10 + [[ false == true ]] + wait_for_running cross-site-sharded-replica-cfg 3 false + local name=cross-site-sharded-replica-cfg + let last_pod=2 + local check_cluster_readyness=false + set_debug + [[ 1 == 1 ]] + set -o xtrace + local rs_name=cfg + local cluster_name=cross-site-sharded-replica ++ seq 0 2 + for i in $(seq 0 $last_pod) + [[ 0 -eq 2 ]] + wait_pod cross-site-sharded-replica-cfg-0 + local pod=cross-site-sharded-replica-cfg-0 + set +o xtrace waiting for pod/cross-site-sharded-replica-cfg-0 to be ready.OK + for i in $(seq 0 $last_pod) + [[ 1 -eq 2 ]] + wait_pod cross-site-sharded-replica-cfg-1 + local pod=cross-site-sharded-replica-cfg-1 + set +o xtrace waiting for pod/cross-site-sharded-replica-cfg-1 to be ready.OK + for i in $(seq 0 $last_pod) + [[ 2 -eq 2 ]] ++ kubectl_bin get psmdb cross-site-sharded-replica -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].arbiter.enabled}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.4qdJf6fMXr +++ mktemp ++ local LAST_ERR=/tmp/tmp.GHJQ8eQuYZ ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb cross-site-sharded-replica -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].arbiter.enabled}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.4qdJf6fMXr ++ cat /tmp/tmp.GHJQ8eQuYZ ++ rm /tmp/tmp.4qdJf6fMXr /tmp/tmp.GHJQ8eQuYZ ++ return 0 + [[ '' == true ]] + wait_pod cross-site-sharded-replica-cfg-2 + local pod=cross-site-sharded-replica-cfg-2 + set +o xtrace waiting for pod/cross-site-sharded-replica-cfg-2 to be ready.OK ++ kubectl_bin get psmdb cross-site-sharded-replica -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].nonvoting.enabled}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.PoqnR08q0q +++ mktemp ++ local LAST_ERR=/tmp/tmp.mmlhWbbeSo ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb cross-site-sharded-replica -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].nonvoting.enabled}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.PoqnR08q0q ++ cat /tmp/tmp.mmlhWbbeSo ++ rm /tmp/tmp.PoqnR08q0q /tmp/tmp.mmlhWbbeSo ++ return 0 + [[ '' == true ]] ++ kubectl_bin get psmdb cross-site-sharded-replica -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].hidden.enabled}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.XPCK8OP00G +++ mktemp ++ local LAST_ERR=/tmp/tmp.3qQ4LarY2U ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb cross-site-sharded-replica -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].hidden.enabled}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.XPCK8OP00G ++ cat /tmp/tmp.3qQ4LarY2U ++ rm /tmp/tmp.XPCK8OP00G /tmp/tmp.3qQ4LarY2U ++ return 0 + [[ '' == true ]] + sleep 10 + [[ false == true ]] ++ get_service_ip cross-site-sharded-replica-cfg-0 cfg ++ local service=cross-site-sharded-replica-cfg-0 ++ local server_type=cfg +++ kubectl_bin get psmdb/cross-site-sharded-replica -o 'jsonpath={.spec.replsets[].expose.enabled}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.Ffvh7eczMI ++++ mktemp +++ local LAST_ERR=/tmp/tmp.yLiIZnocvY +++ local exit_status=0 +++ local timeout=4 ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl get psmdb/cross-site-sharded-replica -o 'jsonpath={.spec.replsets[].expose.enabled}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 -a -n 1 ']' +++ break +++ cat /tmp/tmp.Ffvh7eczMI +++ cat /tmp/tmp.yLiIZnocvY +++ rm /tmp/tmp.Ffvh7eczMI /tmp/tmp.yLiIZnocvY +++ return 0 ++ '[' true '!=' true ']' ++ kubectl_bin get service/cross-site-sharded-replica-cfg-0 -o 'jsonpath={.spec.type}' ++ grep -q NotFound +++ kubectl_bin get service/cross-site-sharded-replica-cfg-0 -o 'jsonpath={.spec.type}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.jBbtLVcbJk ++++ mktemp +++ local LAST_ERR=/tmp/tmp.YYjjaKlCHY +++ local exit_status=0 +++ local timeout=4 ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl get service/cross-site-sharded-replica-cfg-0 -o 'jsonpath={.spec.type}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 -a -n 1 ']' +++ break +++ cat /tmp/tmp.jBbtLVcbJk +++ cat /tmp/tmp.YYjjaKlCHY +++ rm /tmp/tmp.jBbtLVcbJk /tmp/tmp.YYjjaKlCHY +++ return 0 ++ service_type=ClusterIP ++ '[' ClusterIP = ClusterIP ']' ++ kubectl_bin get service/cross-site-sharded-replica-cfg-0 -o 'jsonpath={.spec.clusterIP}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.kHoF11aeMP +++ mktemp ++ local LAST_ERR=/tmp/tmp.WHOUCZN8gB ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get service/cross-site-sharded-replica-cfg-0 -o 'jsonpath={.spec.clusterIP}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.kHoF11aeMP ++ cat /tmp/tmp.WHOUCZN8gB ++ rm /tmp/tmp.kHoF11aeMP /tmp/tmp.WHOUCZN8gB ++ return 0 ++ return + replica_cfg_0_endpoint=34.118.234.136 ++ get_service_ip cross-site-sharded-replica-cfg-1 cfg ++ local service=cross-site-sharded-replica-cfg-1 ++ local server_type=cfg +++ kubectl_bin get psmdb/cross-site-sharded-replica -o 'jsonpath={.spec.replsets[].expose.enabled}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.J8hKsuy8Gr ++++ mktemp +++ local LAST_ERR=/tmp/tmp.JyuARUQtt2 +++ local exit_status=0 +++ local timeout=4 ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl get psmdb/cross-site-sharded-replica -o 'jsonpath={.spec.replsets[].expose.enabled}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 -a -n 1 ']' +++ break +++ cat /tmp/tmp.J8hKsuy8Gr +++ cat /tmp/tmp.JyuARUQtt2 +++ rm /tmp/tmp.J8hKsuy8Gr /tmp/tmp.JyuARUQtt2 +++ return 0 ++ '[' true '!=' true ']' ++ kubectl_bin get service/cross-site-sharded-replica-cfg-1 -o 'jsonpath={.spec.type}' ++ grep -q NotFound +++ kubectl_bin get service/cross-site-sharded-replica-cfg-1 -o 'jsonpath={.spec.type}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.43KD79iPnB ++++ mktemp +++ local LAST_ERR=/tmp/tmp.WXtz9d8BkT +++ local exit_status=0 +++ local timeout=4 ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl get service/cross-site-sharded-replica-cfg-1 -o 'jsonpath={.spec.type}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 -a -n 1 ']' +++ break +++ cat /tmp/tmp.43KD79iPnB +++ cat /tmp/tmp.WXtz9d8BkT +++ rm /tmp/tmp.43KD79iPnB /tmp/tmp.WXtz9d8BkT +++ return 0 ++ service_type=ClusterIP ++ '[' ClusterIP = ClusterIP ']' ++ kubectl_bin get service/cross-site-sharded-replica-cfg-1 -o 'jsonpath={.spec.clusterIP}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.J6eWEqXo6D +++ mktemp ++ local LAST_ERR=/tmp/tmp.mZ1CXLrgwq ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get service/cross-site-sharded-replica-cfg-1 -o 'jsonpath={.spec.clusterIP}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.J6eWEqXo6D ++ cat /tmp/tmp.mZ1CXLrgwq ++ rm /tmp/tmp.J6eWEqXo6D /tmp/tmp.mZ1CXLrgwq ++ return 0 ++ return + replica_cfg_1_endpoint=34.118.228.50 ++ get_service_ip cross-site-sharded-replica-cfg-2 cfg ++ local service=cross-site-sharded-replica-cfg-2 ++ local server_type=cfg +++ kubectl_bin get psmdb/cross-site-sharded-replica -o 'jsonpath={.spec.replsets[].expose.enabled}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.9yQ0RQO2aj ++++ mktemp +++ local LAST_ERR=/tmp/tmp.flrB4Stbf4 +++ local exit_status=0 +++ local timeout=4 ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl get psmdb/cross-site-sharded-replica -o 'jsonpath={.spec.replsets[].expose.enabled}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 -a -n 1 ']' +++ break +++ cat /tmp/tmp.9yQ0RQO2aj +++ cat /tmp/tmp.flrB4Stbf4 +++ rm /tmp/tmp.9yQ0RQO2aj /tmp/tmp.flrB4Stbf4 +++ return 0 ++ '[' true '!=' true ']' ++ kubectl_bin get service/cross-site-sharded-replica-cfg-2 -o 'jsonpath={.spec.type}' ++ grep -q NotFound +++ kubectl_bin get service/cross-site-sharded-replica-cfg-2 -o 'jsonpath={.spec.type}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.5fSI3IgMvS ++++ mktemp +++ local LAST_ERR=/tmp/tmp.awbrKqB1YG +++ local exit_status=0 +++ local timeout=4 ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl get service/cross-site-sharded-replica-cfg-2 -o 'jsonpath={.spec.type}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 -a -n 1 ']' +++ break +++ cat /tmp/tmp.5fSI3IgMvS +++ cat /tmp/tmp.awbrKqB1YG +++ rm /tmp/tmp.5fSI3IgMvS /tmp/tmp.awbrKqB1YG +++ return 0 ++ service_type=ClusterIP ++ '[' ClusterIP = ClusterIP ']' ++ kubectl_bin get service/cross-site-sharded-replica-cfg-2 -o 'jsonpath={.spec.clusterIP}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.0R2roDpxQy +++ mktemp ++ local LAST_ERR=/tmp/tmp.rjFw0ul6Ti ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get service/cross-site-sharded-replica-cfg-2 -o 'jsonpath={.spec.clusterIP}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.0R2roDpxQy ++ cat /tmp/tmp.rjFw0ul6Ti ++ rm /tmp/tmp.0R2roDpxQy /tmp/tmp.rjFw0ul6Ti ++ return 0 ++ return + replica_cfg_2_endpoint=34.118.226.86 ++ get_service_ip cross-site-sharded-replica-rs0-0 ++ local service=cross-site-sharded-replica-rs0-0 ++ local server_type=rs0 +++ kubectl_bin get psmdb/cross-site-sharded-replica -o 'jsonpath={.spec.replsets[].expose.enabled}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.eDMl7iaHbC ++++ mktemp +++ local LAST_ERR=/tmp/tmp.vMt52pnUHq +++ local exit_status=0 +++ local timeout=4 ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl get psmdb/cross-site-sharded-replica -o 'jsonpath={.spec.replsets[].expose.enabled}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 -a -n 1 ']' +++ break +++ cat /tmp/tmp.eDMl7iaHbC +++ cat /tmp/tmp.vMt52pnUHq +++ rm /tmp/tmp.eDMl7iaHbC /tmp/tmp.vMt52pnUHq +++ return 0 ++ '[' true '!=' true ']' ++ kubectl_bin get service/cross-site-sharded-replica-rs0-0 -o 'jsonpath={.spec.type}' ++ grep -q NotFound +++ kubectl_bin get service/cross-site-sharded-replica-rs0-0 -o 'jsonpath={.spec.type}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.RXGlbloEYL ++++ mktemp +++ local LAST_ERR=/tmp/tmp.kPweTpGEmg +++ local exit_status=0 +++ local timeout=4 ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl get service/cross-site-sharded-replica-rs0-0 -o 'jsonpath={.spec.type}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 -a -n 1 ']' +++ break +++ cat /tmp/tmp.RXGlbloEYL +++ cat /tmp/tmp.kPweTpGEmg +++ rm /tmp/tmp.RXGlbloEYL /tmp/tmp.kPweTpGEmg +++ return 0 ++ service_type=ClusterIP ++ '[' ClusterIP = ClusterIP ']' ++ kubectl_bin get service/cross-site-sharded-replica-rs0-0 -o 'jsonpath={.spec.clusterIP}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.1M9z2ooRCg +++ mktemp ++ local LAST_ERR=/tmp/tmp.G9SVdsSysC ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get service/cross-site-sharded-replica-rs0-0 -o 'jsonpath={.spec.clusterIP}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.1M9z2ooRCg ++ cat /tmp/tmp.G9SVdsSysC ++ rm /tmp/tmp.1M9z2ooRCg /tmp/tmp.G9SVdsSysC ++ return 0 ++ return + replica_rs0_0_endpoint=34.118.235.28 ++ get_service_ip cross-site-sharded-replica-rs0-1 ++ local service=cross-site-sharded-replica-rs0-1 ++ local server_type=rs0 +++ kubectl_bin get psmdb/cross-site-sharded-replica -o 'jsonpath={.spec.replsets[].expose.enabled}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.7cAyHdMFG2 ++++ mktemp +++ local LAST_ERR=/tmp/tmp.7XY8jx0B7u +++ local exit_status=0 +++ local timeout=4 ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl get psmdb/cross-site-sharded-replica -o 'jsonpath={.spec.replsets[].expose.enabled}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 -a -n 1 ']' +++ break +++ cat /tmp/tmp.7cAyHdMFG2 +++ cat /tmp/tmp.7XY8jx0B7u +++ rm /tmp/tmp.7cAyHdMFG2 /tmp/tmp.7XY8jx0B7u +++ return 0 ++ '[' true '!=' true ']' ++ kubectl_bin get service/cross-site-sharded-replica-rs0-1 -o 'jsonpath={.spec.type}' ++ grep -q NotFound +++ kubectl_bin get service/cross-site-sharded-replica-rs0-1 -o 'jsonpath={.spec.type}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.VS3hBXavyW ++++ mktemp +++ local LAST_ERR=/tmp/tmp.i6OgqOHP6D +++ local exit_status=0 +++ local timeout=4 ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl get service/cross-site-sharded-replica-rs0-1 -o 'jsonpath={.spec.type}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 -a -n 1 ']' +++ break +++ cat /tmp/tmp.VS3hBXavyW +++ cat /tmp/tmp.i6OgqOHP6D +++ rm /tmp/tmp.VS3hBXavyW /tmp/tmp.i6OgqOHP6D +++ return 0 ++ service_type=ClusterIP ++ '[' ClusterIP = ClusterIP ']' ++ kubectl_bin get service/cross-site-sharded-replica-rs0-1 -o 'jsonpath={.spec.clusterIP}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.c7yuCIlVCk +++ mktemp ++ local LAST_ERR=/tmp/tmp.eBD4FxmTHJ ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get service/cross-site-sharded-replica-rs0-1 -o 'jsonpath={.spec.clusterIP}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.c7yuCIlVCk ++ cat /tmp/tmp.eBD4FxmTHJ ++ rm /tmp/tmp.c7yuCIlVCk /tmp/tmp.eBD4FxmTHJ ++ return 0 ++ return + replica_rs0_1_endpoint=34.118.230.122 ++ get_service_ip cross-site-sharded-replica-rs0-2 ++ local service=cross-site-sharded-replica-rs0-2 ++ local server_type=rs0 +++ kubectl_bin get psmdb/cross-site-sharded-replica -o 'jsonpath={.spec.replsets[].expose.enabled}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.cOewcRxBJZ ++++ mktemp +++ local LAST_ERR=/tmp/tmp.x9js8lS35D +++ local exit_status=0 +++ local timeout=4 ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl get psmdb/cross-site-sharded-replica -o 'jsonpath={.spec.replsets[].expose.enabled}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 -a -n 1 ']' +++ break +++ cat /tmp/tmp.cOewcRxBJZ +++ cat /tmp/tmp.x9js8lS35D +++ rm /tmp/tmp.cOewcRxBJZ /tmp/tmp.x9js8lS35D +++ return 0 ++ '[' true '!=' true ']' ++ kubectl_bin get service/cross-site-sharded-replica-rs0-2 -o 'jsonpath={.spec.type}' ++ grep -q NotFound +++ kubectl_bin get service/cross-site-sharded-replica-rs0-2 -o 'jsonpath={.spec.type}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.bf3A4p2wpA ++++ mktemp +++ local LAST_ERR=/tmp/tmp.saFu4uarwK +++ local exit_status=0 +++ local timeout=4 ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl get service/cross-site-sharded-replica-rs0-2 -o 'jsonpath={.spec.type}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 -a -n 1 ']' +++ break +++ cat /tmp/tmp.bf3A4p2wpA +++ cat /tmp/tmp.saFu4uarwK +++ rm /tmp/tmp.bf3A4p2wpA /tmp/tmp.saFu4uarwK +++ return 0 ++ service_type=ClusterIP ++ '[' ClusterIP = ClusterIP ']' ++ kubectl_bin get service/cross-site-sharded-replica-rs0-2 -o 'jsonpath={.spec.clusterIP}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.4J6auKHvKG +++ mktemp ++ local LAST_ERR=/tmp/tmp.WMBKIAE6fd ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get service/cross-site-sharded-replica-rs0-2 -o 'jsonpath={.spec.clusterIP}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.4J6auKHvKG ++ cat /tmp/tmp.WMBKIAE6fd ++ rm /tmp/tmp.4J6auKHvKG /tmp/tmp.WMBKIAE6fd ++ return 0 ++ return + replica_rs0_2_endpoint=34.118.225.159 ++ get_service_ip cross-site-sharded-replica-rs1-0 rs1 ++ local service=cross-site-sharded-replica-rs1-0 ++ local server_type=rs1 +++ kubectl_bin get psmdb/cross-site-sharded-replica -o 'jsonpath={.spec.replsets[].expose.enabled}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.yUJ5mPxqLQ ++++ mktemp +++ local LAST_ERR=/tmp/tmp.Yj9ATNfXes +++ local exit_status=0 +++ local timeout=4 ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl get psmdb/cross-site-sharded-replica -o 'jsonpath={.spec.replsets[].expose.enabled}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 -a -n 1 ']' +++ break +++ cat /tmp/tmp.yUJ5mPxqLQ +++ cat /tmp/tmp.Yj9ATNfXes +++ rm /tmp/tmp.yUJ5mPxqLQ /tmp/tmp.Yj9ATNfXes +++ return 0 ++ '[' true '!=' true ']' ++ kubectl_bin get service/cross-site-sharded-replica-rs1-0 -o 'jsonpath={.spec.type}' ++ grep -q NotFound +++ kubectl_bin get service/cross-site-sharded-replica-rs1-0 -o 'jsonpath={.spec.type}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.b2lxqhp1ag ++++ mktemp +++ local LAST_ERR=/tmp/tmp.6cNXwOBpxr +++ local exit_status=0 +++ local timeout=4 ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl get service/cross-site-sharded-replica-rs1-0 -o 'jsonpath={.spec.type}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 -a -n 1 ']' +++ break +++ cat /tmp/tmp.b2lxqhp1ag +++ cat /tmp/tmp.6cNXwOBpxr +++ rm /tmp/tmp.b2lxqhp1ag /tmp/tmp.6cNXwOBpxr +++ return 0 ++ service_type=ClusterIP ++ '[' ClusterIP = ClusterIP ']' ++ kubectl_bin get service/cross-site-sharded-replica-rs1-0 -o 'jsonpath={.spec.clusterIP}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.6Ph1Af2XnZ +++ mktemp ++ local LAST_ERR=/tmp/tmp.FYsKFPfG54 ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get service/cross-site-sharded-replica-rs1-0 -o 'jsonpath={.spec.clusterIP}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.6Ph1Af2XnZ ++ cat /tmp/tmp.FYsKFPfG54 ++ rm /tmp/tmp.6Ph1Af2XnZ /tmp/tmp.FYsKFPfG54 ++ return 0 ++ return + replica_rs1_0_endpoint=34.118.231.49 ++ get_service_ip cross-site-sharded-replica-rs1-1 rs1 ++ local service=cross-site-sharded-replica-rs1-1 ++ local server_type=rs1 +++ kubectl_bin get psmdb/cross-site-sharded-replica -o 'jsonpath={.spec.replsets[].expose.enabled}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.i3UYJc342N ++++ mktemp +++ local LAST_ERR=/tmp/tmp.Cxu7QD73Jh +++ local exit_status=0 +++ local timeout=4 ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl get psmdb/cross-site-sharded-replica -o 'jsonpath={.spec.replsets[].expose.enabled}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 -a -n 1 ']' +++ break +++ cat /tmp/tmp.i3UYJc342N +++ cat /tmp/tmp.Cxu7QD73Jh +++ rm /tmp/tmp.i3UYJc342N /tmp/tmp.Cxu7QD73Jh +++ return 0 ++ '[' true '!=' true ']' ++ grep -q NotFound ++ kubectl_bin get service/cross-site-sharded-replica-rs1-1 -o 'jsonpath={.spec.type}' +++ kubectl_bin get service/cross-site-sharded-replica-rs1-1 -o 'jsonpath={.spec.type}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.xgLNj12Jq6 ++++ mktemp +++ local LAST_ERR=/tmp/tmp.4YkiRHggb2 +++ local exit_status=0 +++ local timeout=4 ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl get service/cross-site-sharded-replica-rs1-1 -o 'jsonpath={.spec.type}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 -a -n 1 ']' +++ break +++ cat /tmp/tmp.xgLNj12Jq6 +++ cat /tmp/tmp.4YkiRHggb2 +++ rm /tmp/tmp.xgLNj12Jq6 /tmp/tmp.4YkiRHggb2 +++ return 0 ++ service_type=ClusterIP ++ '[' ClusterIP = ClusterIP ']' ++ kubectl_bin get service/cross-site-sharded-replica-rs1-1 -o 'jsonpath={.spec.clusterIP}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.OuYpSfXmNy +++ mktemp ++ local LAST_ERR=/tmp/tmp.GyjQnaHepS ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get service/cross-site-sharded-replica-rs1-1 -o 'jsonpath={.spec.clusterIP}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.OuYpSfXmNy ++ cat /tmp/tmp.GyjQnaHepS ++ rm /tmp/tmp.OuYpSfXmNy /tmp/tmp.GyjQnaHepS ++ return 0 ++ return + replica_rs1_1_endpoint=34.118.230.137 ++ get_service_ip cross-site-sharded-replica-rs1-2 rs1 ++ local service=cross-site-sharded-replica-rs1-2 ++ local server_type=rs1 +++ kubectl_bin get psmdb/cross-site-sharded-replica -o 'jsonpath={.spec.replsets[].expose.enabled}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.0Puc9KU0wK ++++ mktemp +++ local LAST_ERR=/tmp/tmp.UM2oIBhEqb +++ local exit_status=0 +++ local timeout=4 ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl get psmdb/cross-site-sharded-replica -o 'jsonpath={.spec.replsets[].expose.enabled}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 -a -n 1 ']' +++ break +++ cat /tmp/tmp.0Puc9KU0wK +++ cat /tmp/tmp.UM2oIBhEqb +++ rm /tmp/tmp.0Puc9KU0wK /tmp/tmp.UM2oIBhEqb +++ return 0 ++ '[' true '!=' true ']' ++ kubectl_bin get service/cross-site-sharded-replica-rs1-2 -o 'jsonpath={.spec.type}' ++ grep -q NotFound +++ kubectl_bin get service/cross-site-sharded-replica-rs1-2 -o 'jsonpath={.spec.type}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.T9HRN3B3UZ ++++ mktemp +++ local LAST_ERR=/tmp/tmp.WkOb6bZxv3 +++ local exit_status=0 +++ local timeout=4 ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl get service/cross-site-sharded-replica-rs1-2 -o 'jsonpath={.spec.type}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 -a -n 1 ']' +++ break +++ cat /tmp/tmp.T9HRN3B3UZ +++ cat /tmp/tmp.WkOb6bZxv3 +++ rm /tmp/tmp.T9HRN3B3UZ /tmp/tmp.WkOb6bZxv3 +++ return 0 ++ service_type=ClusterIP ++ '[' ClusterIP = ClusterIP ']' ++ kubectl_bin get service/cross-site-sharded-replica-rs1-2 -o 'jsonpath={.spec.clusterIP}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.KhNTxLhMXv +++ mktemp ++ local LAST_ERR=/tmp/tmp.ufTDtSQ5Gz ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get service/cross-site-sharded-replica-rs1-2 -o 'jsonpath={.spec.clusterIP}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.KhNTxLhMXv ++ cat /tmp/tmp.ufTDtSQ5Gz ++ rm /tmp/tmp.KhNTxLhMXv /tmp/tmp.ufTDtSQ5Gz ++ return 0 ++ return + replica_rs1_2_endpoint=34.118.237.254 ++ kubectl_bin config current-context +++ mktemp ++ local LAST_OUT=/tmp/tmp.Iy1Aop4WIo +++ mktemp ++ local LAST_ERR=/tmp/tmp.nfGovvPma7 ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl config current-context ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.Iy1Aop4WIo ++ cat /tmp/tmp.nfGovvPma7 ++ rm /tmp/tmp.Iy1Aop4WIo /tmp/tmp.nfGovvPma7 ++ return 0 + kubectl_bin config set-context gke_cloud-dev-112233_us-central1-a_jen-psmdb-2283-73ca3795-11-cluster3 --namespace=cross-site-sharded-2618 ++ mktemp + local LAST_OUT=/tmp/tmp.sPrSOb0wep ++ mktemp + local LAST_ERR=/tmp/tmp.24JsDYDdsj + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl config set-context gke_cloud-dev-112233_us-central1-a_jen-psmdb-2283-73ca3795-11-cluster3 --namespace=cross-site-sharded-2618 + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.sPrSOb0wep Context "gke_cloud-dev-112233_us-central1-a_jen-psmdb-2283-73ca3795-11-cluster3" modified. + cat /tmp/tmp.24JsDYDdsj + rm /tmp/tmp.sPrSOb0wep /tmp/tmp.24JsDYDdsj + return 0 + kubectl_bin patch psmdb cross-site-sharded-main --type=merge --patch $'{\n\t\t\t"spec": {"replsets":[\n\t\t\t {"affinity":{"antiAffinityTopologyKey": "none"},"arbiter":{"affinity":{"antiAffinityTopologyKey": "none"},"enabled":false,"size":1},"expose":{"enabled":true,"type":"ClusterIp"},"externalNodes":[{"host":"34.118.235.28","priority":0,"votes":0},{"host":"34.118.230.122","port":27017,"priority":1,"votes":1},{"host":"34.118.225.159", "port":27017,"priority":1,"votes":1}],"name":"rs0","nonvoting":{"affinity":{"antiAffinityTopologyKey":"none"},"enabled":false,"podDisruptionBudget":{"maxUnavailable":1},"resources":{"limits":{"cpu":"300m","memory":"0.5G"},"requests":{"cpu":"300m","memory":"0.5G"}},"size":3,"volumeSpec":{"persistentVolumeClaim":{"resources":{"requests":{"storage":"1Gi"}}}}},"podDisruptionBudget":{"maxUnavailable":1},"resources":{"limits":{"cpu":"300m","memory":"0.5G"},"requests":{"cpu":"300m","memory":"0.5G"}},"size":3,"volumeSpec":{"persistentVolumeClaim":{"resources":{"requests":{"storage":"3Gi"}}}}},\n\t\t\t {"affinity":{"antiAffinityTopologyKey": "none"},"arbiter":{"affinity":{"antiAffinityTopologyKey": "none"},"enabled":false,"size":1},"expose":{"enabled":true,"type":"ClusterIp"},"externalNodes":[{"host":"34.118.231.49","priority":0,"votes":0},{"host":"34.118.230.137","port":27017,"priority":1,"votes":1},{"host":"34.118.237.254", "port":27017,"priority":1,"votes":1}],"name":"rs1","nonvoting":{"affinity":{"antiAffinityTopologyKey":"none"},"enabled":false,"podDisruptionBudget":{"maxUnavailable":1},"resources":{"limits":{"cpu":"300m","memory":"0.5G"},"requests":{"cpu":"300m","memory":"0.5G"}},"size":3,"volumeSpec":{"persistentVolumeClaim":{"resources":{"requests":{"storage":"1Gi"}}}}},"podDisruptionBudget":{"maxUnavailable":1},"resources":{"limits":{"cpu":"300m","memory":"0.5G"},"requests":{"cpu":"300m","memory":"0.5G"}},"size":3,"volumeSpec":{"persistentVolumeClaim":{"resources":{"requests":{"storage":"3Gi"}}}}}\n\t\t\t ],\n\t\t\t "sharding":{"configsvrReplSet":{ "externalNodes": [{"host":"34.118.234.136","priority":1,"votes":1 },{"host":"34.118.228.50", "priority":1,"votes":1},{"host":"34.118.226.86","priority":0,"votes":0}]}}\n\t\t\t }\n\t\t}' ++ mktemp + local LAST_OUT=/tmp/tmp.nF157fuw9f ++ mktemp + local LAST_ERR=/tmp/tmp.X3RS6vU9sK + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl patch psmdb cross-site-sharded-main --type=merge --patch $'{\n\t\t\t"spec": {"replsets":[\n\t\t\t {"affinity":{"antiAffinityTopologyKey": "none"},"arbiter":{"affinity":{"antiAffinityTopologyKey": "none"},"enabled":false,"size":1},"expose":{"enabled":true,"type":"ClusterIp"},"externalNodes":[{"host":"34.118.235.28","priority":0,"votes":0},{"host":"34.118.230.122","port":27017,"priority":1,"votes":1},{"host":"34.118.225.159", "port":27017,"priority":1,"votes":1}],"name":"rs0","nonvoting":{"affinity":{"antiAffinityTopologyKey":"none"},"enabled":false,"podDisruptionBudget":{"maxUnavailable":1},"resources":{"limits":{"cpu":"300m","memory":"0.5G"},"requests":{"cpu":"300m","memory":"0.5G"}},"size":3,"volumeSpec":{"persistentVolumeClaim":{"resources":{"requests":{"storage":"1Gi"}}}}},"podDisruptionBudget":{"maxUnavailable":1},"resources":{"limits":{"cpu":"300m","memory":"0.5G"},"requests":{"cpu":"300m","memory":"0.5G"}},"size":3,"volumeSpec":{"persistentVolumeClaim":{"resources":{"requests":{"storage":"3Gi"}}}}},\n\t\t\t {"affinity":{"antiAffinityTopologyKey": "none"},"arbiter":{"affinity":{"antiAffinityTopologyKey": "none"},"enabled":false,"size":1},"expose":{"enabled":true,"type":"ClusterIp"},"externalNodes":[{"host":"34.118.231.49","priority":0,"votes":0},{"host":"34.118.230.137","port":27017,"priority":1,"votes":1},{"host":"34.118.237.254", "port":27017,"priority":1,"votes":1}],"name":"rs1","nonvoting":{"affinity":{"antiAffinityTopologyKey":"none"},"enabled":false,"podDisruptionBudget":{"maxUnavailable":1},"resources":{"limits":{"cpu":"300m","memory":"0.5G"},"requests":{"cpu":"300m","memory":"0.5G"}},"size":3,"volumeSpec":{"persistentVolumeClaim":{"resources":{"requests":{"storage":"1Gi"}}}}},"podDisruptionBudget":{"maxUnavailable":1},"resources":{"limits":{"cpu":"300m","memory":"0.5G"},"requests":{"cpu":"300m","memory":"0.5G"}},"size":3,"volumeSpec":{"persistentVolumeClaim":{"resources":{"requests":{"storage":"3Gi"}}}}}\n\t\t\t ],\n\t\t\t "sharding":{"configsvrReplSet":{ "externalNodes": [{"host":"34.118.234.136","priority":1,"votes":1 },{"host":"34.118.228.50", "priority":1,"votes":1},{"host":"34.118.226.86","priority":0,"votes":0}]}}\n\t\t\t }\n\t\t}' + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.nF157fuw9f perconaservermongodb.psmdb.percona.com/cross-site-sharded-main patched + cat /tmp/tmp.X3RS6vU9sK + rm /tmp/tmp.nF157fuw9f /tmp/tmp.X3RS6vU9sK + return 0 + wait_for_members 34.118.234.136 cfg 6 + local endpoint=34.118.234.136 + local rsName=cfg + local target_count=6 + local nodes_count=0 + [[ 0 == 6 ]] ++ run_mongos 'rs.conf().members.length' clusterAdmin:clusterAdmin123456@34.118.234.136 mongodb :27017 ++ local 'command=rs.conf().members.length' ++ local uri=clusterAdmin:clusterAdmin123456@34.118.234.136 ++ local driver=mongodb ++ local suffix=:27017 ++ local mongo_flag= ++ grep -E -v 'I NETWORK|W NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|bye' ++ local port=27017 ++ local mongo_bin=mongo ++ /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/' +++ echo :27017 +++ awk -F: '{print $2}' ++ suffix_port=27017 ++ [[ -z 27017 ]] +++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.U2YjKYWBq9 ++++ mktemp +++ local LAST_ERR=/tmp/tmp.qEqi1cxgUw +++ local exit_status=0 +++ local timeout=4 ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 -a -n 1 ']' +++ break +++ cat /tmp/tmp.U2YjKYWBq9 +++ cat /tmp/tmp.qEqi1cxgUw +++ rm /tmp/tmp.U2YjKYWBq9 /tmp/tmp.qEqi1cxgUw +++ return 0 ++ local client_container=psmdb-client-bb8b97679-xl6jr ++ kubectl_bin exec psmdb-client-bb8b97679-xl6jr -- bash -c 'printf '\''rs.conf().members.length\n'\'' | mongo mongodb://clusterAdmin:clusterAdmin123456@34.118.234.136:27017/admin ' +++ mktemp ++ local LAST_OUT=/tmp/tmp.zR17np8xgR +++ mktemp ++ local LAST_ERR=/tmp/tmp.lWVrCUIRqU ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl exec psmdb-client-bb8b97679-xl6jr -- bash -c 'printf '\''rs.conf().members.length\n'\'' | mongo mongodb://clusterAdmin:clusterAdmin123456@34.118.234.136:27017/admin ' ++ exit_status=1 ++ set -e ++ '[' 1 '!=' 0 -a -n 1 ']' ++ cat /tmp/tmp.zR17np8xgR ++ cat /tmp/tmp.lWVrCUIRqU command terminated with exit code 1 ++ sleep 0 ++ for i in $(seq 0 2) ++ set +e ++ kubectl exec psmdb-client-bb8b97679-xl6jr -- bash -c 'printf '\''rs.conf().members.length\n'\'' | mongo mongodb://clusterAdmin:clusterAdmin123456@34.118.234.136:27017/admin ' ++ exit_status=1 ++ set -e ++ '[' 1 '!=' 0 -a -n 1 ']' ++ cat /tmp/tmp.zR17np8xgR ++ cat /tmp/tmp.lWVrCUIRqU command terminated with exit code 1 ++ sleep 4 ++ for i in $(seq 0 2) ++ set +e ++ kubectl exec psmdb-client-bb8b97679-xl6jr -- bash -c 'printf '\''rs.conf().members.length\n'\'' | mongo mongodb://clusterAdmin:clusterAdmin123456@34.118.234.136:27017/admin ' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.zR17np8xgR ++ cat /tmp/tmp.lWVrCUIRqU ++ rm /tmp/tmp.zR17np8xgR /tmp/tmp.lWVrCUIRqU ++ return 0 + nodes_count=$'Error: Authentication failed. :\nconnect@src/mongo/shell/mongo.js:374:17\n@(connect):2:6\nexception: connect failed\nexiting with code 1\nError: Authentication failed. :\nconnect@src/mongo/shell/mongo.js:374:17\n@(connect):2:6\nexception: connect failed\nexiting with code 1\n4' + echo -n 'waiting for all members to be configured in cfg' waiting for all members to be configured in cfg+ let retry+=1 + '[' 1 -ge 15 ']' + echo . . + sleep 10 + [[ Error: Authentication failed. : connect@src/mongo/shell/mongo.js:374:17 @(connect):2:6 exception: connect failed exiting with code 1 Error: Authentication failed. : connect@src/mongo/shell/mongo.js:374:17 @(connect):2:6 exception: connect failed exiting with code 1 4 == 6 ]] ++ run_mongos 'rs.conf().members.length' clusterAdmin:clusterAdmin123456@34.118.234.136 mongodb :27017 ++ grep -E -v 'I NETWORK|W NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|bye' ++ local 'command=rs.conf().members.length' ++ /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/' ++ local uri=clusterAdmin:clusterAdmin123456@34.118.234.136 ++ local driver=mongodb ++ local suffix=:27017 ++ local mongo_flag= ++ local port=27017 ++ local mongo_bin=mongo +++ echo :27017 +++ awk -F: '{print $2}' ++ suffix_port=27017 ++ [[ -z 27017 ]] +++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.9roMtBeYzl ++++ mktemp +++ local LAST_ERR=/tmp/tmp.OfWN3HOom7 +++ local exit_status=0 +++ local timeout=4 ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 -a -n 1 ']' +++ break +++ cat /tmp/tmp.9roMtBeYzl +++ cat /tmp/tmp.OfWN3HOom7 +++ rm /tmp/tmp.9roMtBeYzl /tmp/tmp.OfWN3HOom7 +++ return 0 ++ local client_container=psmdb-client-bb8b97679-xl6jr ++ kubectl_bin exec psmdb-client-bb8b97679-xl6jr -- bash -c 'printf '\''rs.conf().members.length\n'\'' | mongo mongodb://clusterAdmin:clusterAdmin123456@34.118.234.136:27017/admin ' +++ mktemp ++ local LAST_OUT=/tmp/tmp.0zz195aih1 +++ mktemp ++ local LAST_ERR=/tmp/tmp.7eRvB3qr2S ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl exec psmdb-client-bb8b97679-xl6jr -- bash -c 'printf '\''rs.conf().members.length\n'\'' | mongo mongodb://clusterAdmin:clusterAdmin123456@34.118.234.136:27017/admin ' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.0zz195aih1 ++ cat /tmp/tmp.7eRvB3qr2S ++ rm /tmp/tmp.0zz195aih1 /tmp/tmp.7eRvB3qr2S ++ return 0 + nodes_count=5 + echo -n 'waiting for all members to be configured in cfg' waiting for all members to be configured in cfg+ let retry+=1 + '[' 2 -ge 15 ']' + echo . . + sleep 10 + [[ 5 == 6 ]] ++ run_mongos 'rs.conf().members.length' clusterAdmin:clusterAdmin123456@34.118.234.136 mongodb :27017 ++ local 'command=rs.conf().members.length' ++ local uri=clusterAdmin:clusterAdmin123456@34.118.234.136 ++ local driver=mongodb ++ local suffix=:27017 ++ local mongo_flag= ++ local port=27017 ++ local mongo_bin=mongo ++ grep -E -v 'I NETWORK|W NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|bye' ++ /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/' +++ echo :27017 +++ awk -F: '{print $2}' ++ suffix_port=27017 ++ [[ -z 27017 ]] +++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.qDkHYKVDzE ++++ mktemp +++ local LAST_ERR=/tmp/tmp.aaKf15r2WA +++ local exit_status=0 +++ local timeout=4 ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 -a -n 1 ']' +++ break +++ cat /tmp/tmp.qDkHYKVDzE +++ cat /tmp/tmp.aaKf15r2WA +++ rm /tmp/tmp.qDkHYKVDzE /tmp/tmp.aaKf15r2WA +++ return 0 ++ local client_container=psmdb-client-bb8b97679-xl6jr ++ kubectl_bin exec psmdb-client-bb8b97679-xl6jr -- bash -c 'printf '\''rs.conf().members.length\n'\'' | mongo mongodb://clusterAdmin:clusterAdmin123456@34.118.234.136:27017/admin ' +++ mktemp ++ local LAST_OUT=/tmp/tmp.r1VAg1tZ7U +++ mktemp ++ local LAST_ERR=/tmp/tmp.qEmr7SbTyW ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl exec psmdb-client-bb8b97679-xl6jr -- bash -c 'printf '\''rs.conf().members.length\n'\'' | mongo mongodb://clusterAdmin:clusterAdmin123456@34.118.234.136:27017/admin ' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.r1VAg1tZ7U ++ cat /tmp/tmp.qEmr7SbTyW ++ rm /tmp/tmp.r1VAg1tZ7U /tmp/tmp.qEmr7SbTyW ++ return 0 + nodes_count=5 + echo -n 'waiting for all members to be configured in cfg' waiting for all members to be configured in cfg+ let retry+=1 + '[' 3 -ge 15 ']' + echo . . + sleep 10 + [[ 5 == 6 ]] ++ run_mongos 'rs.conf().members.length' clusterAdmin:clusterAdmin123456@34.118.234.136 mongodb :27017 ++ local 'command=rs.conf().members.length' ++ local uri=clusterAdmin:clusterAdmin123456@34.118.234.136 ++ local driver=mongodb ++ local suffix=:27017 ++ local mongo_flag= ++ local port=27017 ++ local mongo_bin=mongo ++ grep -E -v 'I NETWORK|W NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|bye' ++ /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/' +++ awk -F: '{print $2}' +++ echo :27017 ++ suffix_port=27017 ++ [[ -z 27017 ]] +++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.bQSU3LNywB ++++ mktemp +++ local LAST_ERR=/tmp/tmp.quCSlqGDRt +++ local exit_status=0 +++ local timeout=4 ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 -a -n 1 ']' +++ break +++ cat /tmp/tmp.bQSU3LNywB +++ cat /tmp/tmp.quCSlqGDRt +++ rm /tmp/tmp.bQSU3LNywB /tmp/tmp.quCSlqGDRt +++ return 0 ++ local client_container=psmdb-client-bb8b97679-xl6jr ++ kubectl_bin exec psmdb-client-bb8b97679-xl6jr -- bash -c 'printf '\''rs.conf().members.length\n'\'' | mongo mongodb://clusterAdmin:clusterAdmin123456@34.118.234.136:27017/admin ' +++ mktemp ++ local LAST_OUT=/tmp/tmp.WR7SiWy9XV +++ mktemp ++ local LAST_ERR=/tmp/tmp.WnztJ0LVxw ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl exec psmdb-client-bb8b97679-xl6jr -- bash -c 'printf '\''rs.conf().members.length\n'\'' | mongo mongodb://clusterAdmin:clusterAdmin123456@34.118.234.136:27017/admin ' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.WR7SiWy9XV ++ cat /tmp/tmp.WnztJ0LVxw ++ rm /tmp/tmp.WR7SiWy9XV /tmp/tmp.WnztJ0LVxw ++ return 0 + nodes_count=5 + echo -n 'waiting for all members to be configured in cfg' waiting for all members to be configured in cfg+ let retry+=1 + '[' 4 -ge 15 ']' + echo . . + sleep 10 + [[ 5 == 6 ]] ++ run_mongos 'rs.conf().members.length' clusterAdmin:clusterAdmin123456@34.118.234.136 mongodb :27017 ++ grep -E -v 'I NETWORK|W NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|bye' ++ local 'command=rs.conf().members.length' ++ local uri=clusterAdmin:clusterAdmin123456@34.118.234.136 ++ local driver=mongodb ++ local suffix=:27017 ++ local mongo_flag= ++ local port=27017 ++ local mongo_bin=mongo ++ /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/' +++ echo :27017 +++ awk -F: '{print $2}' ++ suffix_port=27017 ++ [[ -z 27017 ]] +++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.CvdCbqj3xN ++++ mktemp +++ local LAST_ERR=/tmp/tmp.tGftVoUixI +++ local exit_status=0 +++ local timeout=4 ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 -a -n 1 ']' +++ break +++ cat /tmp/tmp.CvdCbqj3xN +++ cat /tmp/tmp.tGftVoUixI +++ rm /tmp/tmp.CvdCbqj3xN /tmp/tmp.tGftVoUixI +++ return 0 ++ local client_container=psmdb-client-bb8b97679-xl6jr ++ kubectl_bin exec psmdb-client-bb8b97679-xl6jr -- bash -c 'printf '\''rs.conf().members.length\n'\'' | mongo mongodb://clusterAdmin:clusterAdmin123456@34.118.234.136:27017/admin ' +++ mktemp ++ local LAST_OUT=/tmp/tmp.NZaDKE7hw0 +++ mktemp ++ local LAST_ERR=/tmp/tmp.JrPlB5qpIk ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl exec psmdb-client-bb8b97679-xl6jr -- bash -c 'printf '\''rs.conf().members.length\n'\'' | mongo mongodb://clusterAdmin:clusterAdmin123456@34.118.234.136:27017/admin ' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.NZaDKE7hw0 ++ cat /tmp/tmp.JrPlB5qpIk ++ rm /tmp/tmp.NZaDKE7hw0 /tmp/tmp.JrPlB5qpIk ++ return 0 + nodes_count=5 + echo -n 'waiting for all members to be configured in cfg' waiting for all members to be configured in cfg+ let retry+=1 + '[' 5 -ge 15 ']' + echo . . + sleep 10 + [[ 5 == 6 ]] ++ run_mongos 'rs.conf().members.length' clusterAdmin:clusterAdmin123456@34.118.234.136 mongodb :27017 ++ local 'command=rs.conf().members.length' ++ grep -E -v 'I NETWORK|W NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|bye' ++ local uri=clusterAdmin:clusterAdmin123456@34.118.234.136 ++ local driver=mongodb ++ local suffix=:27017 ++ local mongo_flag= ++ local port=27017 ++ local mongo_bin=mongo ++ /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/' +++ echo :27017 +++ awk -F: '{print $2}' ++ suffix_port=27017 ++ [[ -z 27017 ]] +++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.m5deLS9fPt ++++ mktemp +++ local LAST_ERR=/tmp/tmp.lf69Zj0RjU +++ local exit_status=0 +++ local timeout=4 ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 -a -n 1 ']' +++ break +++ cat /tmp/tmp.m5deLS9fPt +++ cat /tmp/tmp.lf69Zj0RjU +++ rm /tmp/tmp.m5deLS9fPt /tmp/tmp.lf69Zj0RjU +++ return 0 ++ local client_container=psmdb-client-bb8b97679-xl6jr ++ kubectl_bin exec psmdb-client-bb8b97679-xl6jr -- bash -c 'printf '\''rs.conf().members.length\n'\'' | mongo mongodb://clusterAdmin:clusterAdmin123456@34.118.234.136:27017/admin ' +++ mktemp ++ local LAST_OUT=/tmp/tmp.WFYF3GcyL2 +++ mktemp ++ local LAST_ERR=/tmp/tmp.zyRNBUKYUs ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl exec psmdb-client-bb8b97679-xl6jr -- bash -c 'printf '\''rs.conf().members.length\n'\'' | mongo mongodb://clusterAdmin:clusterAdmin123456@34.118.234.136:27017/admin ' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.WFYF3GcyL2 ++ cat /tmp/tmp.zyRNBUKYUs ++ rm /tmp/tmp.WFYF3GcyL2 /tmp/tmp.zyRNBUKYUs ++ return 0 + nodes_count=5 + echo -n 'waiting for all members to be configured in cfg' waiting for all members to be configured in cfg+ let retry+=1 + '[' 6 -ge 15 ']' + echo . . + sleep 10 + [[ 5 == 6 ]] ++ run_mongos 'rs.conf().members.length' clusterAdmin:clusterAdmin123456@34.118.234.136 mongodb :27017 ++ local 'command=rs.conf().members.length' ++ local uri=clusterAdmin:clusterAdmin123456@34.118.234.136 ++ local driver=mongodb ++ local suffix=:27017 ++ local mongo_flag= ++ local port=27017 ++ local mongo_bin=mongo ++ /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/' ++ grep -E -v 'I NETWORK|W NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|bye' +++ echo :27017 +++ awk -F: '{print $2}' ++ suffix_port=27017 ++ [[ -z 27017 ]] +++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.EYK1MW0Fj5 ++++ mktemp +++ local LAST_ERR=/tmp/tmp.K6yuuX25Bx +++ local exit_status=0 +++ local timeout=4 ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 -a -n 1 ']' +++ break +++ cat /tmp/tmp.EYK1MW0Fj5 +++ cat /tmp/tmp.K6yuuX25Bx +++ rm /tmp/tmp.EYK1MW0Fj5 /tmp/tmp.K6yuuX25Bx +++ return 0 ++ local client_container=psmdb-client-bb8b97679-xl6jr ++ kubectl_bin exec psmdb-client-bb8b97679-xl6jr -- bash -c 'printf '\''rs.conf().members.length\n'\'' | mongo mongodb://clusterAdmin:clusterAdmin123456@34.118.234.136:27017/admin ' +++ mktemp ++ local LAST_OUT=/tmp/tmp.vWWgiWQJqe +++ mktemp ++ local LAST_ERR=/tmp/tmp.yb9dRzyNVZ ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl exec psmdb-client-bb8b97679-xl6jr -- bash -c 'printf '\''rs.conf().members.length\n'\'' | mongo mongodb://clusterAdmin:clusterAdmin123456@34.118.234.136:27017/admin ' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.vWWgiWQJqe ++ cat /tmp/tmp.yb9dRzyNVZ ++ rm /tmp/tmp.vWWgiWQJqe /tmp/tmp.yb9dRzyNVZ ++ return 0 + nodes_count=5 + echo -n 'waiting for all members to be configured in cfg' waiting for all members to be configured in cfg+ let retry+=1 + '[' 7 -ge 15 ']' + echo . . + sleep 10 + [[ 5 == 6 ]] ++ run_mongos 'rs.conf().members.length' clusterAdmin:clusterAdmin123456@34.118.234.136 mongodb :27017 ++ local 'command=rs.conf().members.length' ++ grep -E -v 'I NETWORK|W NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|bye' ++ local uri=clusterAdmin:clusterAdmin123456@34.118.234.136 ++ local driver=mongodb ++ local suffix=:27017 ++ local mongo_flag= ++ local port=27017 ++ local mongo_bin=mongo ++ /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/' +++ echo :27017 +++ awk -F: '{print $2}' ++ suffix_port=27017 ++ [[ -z 27017 ]] +++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.EHX509ED7b ++++ mktemp +++ local LAST_ERR=/tmp/tmp.gkHCL0dz56 +++ local exit_status=0 +++ local timeout=4 ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 -a -n 1 ']' +++ break +++ cat /tmp/tmp.EHX509ED7b +++ cat /tmp/tmp.gkHCL0dz56 +++ rm /tmp/tmp.EHX509ED7b /tmp/tmp.gkHCL0dz56 +++ return 0 ++ local client_container=psmdb-client-bb8b97679-xl6jr ++ kubectl_bin exec psmdb-client-bb8b97679-xl6jr -- bash -c 'printf '\''rs.conf().members.length\n'\'' | mongo mongodb://clusterAdmin:clusterAdmin123456@34.118.234.136:27017/admin ' +++ mktemp ++ local LAST_OUT=/tmp/tmp.Nd9wTJmTZi +++ mktemp ++ local LAST_ERR=/tmp/tmp.MEAVaW8t6t ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl exec psmdb-client-bb8b97679-xl6jr -- bash -c 'printf '\''rs.conf().members.length\n'\'' | mongo mongodb://clusterAdmin:clusterAdmin123456@34.118.234.136:27017/admin ' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.Nd9wTJmTZi ++ cat /tmp/tmp.MEAVaW8t6t ++ rm /tmp/tmp.Nd9wTJmTZi /tmp/tmp.MEAVaW8t6t ++ return 0 + nodes_count=5 + echo -n 'waiting for all members to be configured in cfg' waiting for all members to be configured in cfg+ let retry+=1 + '[' 8 -ge 15 ']' + echo . . + sleep 10 + [[ 5 == 6 ]] ++ run_mongos 'rs.conf().members.length' clusterAdmin:clusterAdmin123456@34.118.234.136 mongodb :27017 ++ local 'command=rs.conf().members.length' ++ local uri=clusterAdmin:clusterAdmin123456@34.118.234.136 ++ local driver=mongodb ++ local suffix=:27017 ++ grep -E -v 'I NETWORK|W NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|bye' ++ local mongo_flag= ++ local port=27017 ++ local mongo_bin=mongo +++ echo :27017 +++ awk -F: '{print $2}' ++ /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/' ++ suffix_port=27017 ++ [[ -z 27017 ]] +++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.5VJmSmynql ++++ mktemp +++ local LAST_ERR=/tmp/tmp.9sImAVEo38 +++ local exit_status=0 +++ local timeout=4 ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 -a -n 1 ']' +++ break +++ cat /tmp/tmp.5VJmSmynql +++ cat /tmp/tmp.9sImAVEo38 +++ rm /tmp/tmp.5VJmSmynql /tmp/tmp.9sImAVEo38 +++ return 0 ++ local client_container=psmdb-client-bb8b97679-xl6jr ++ kubectl_bin exec psmdb-client-bb8b97679-xl6jr -- bash -c 'printf '\''rs.conf().members.length\n'\'' | mongo mongodb://clusterAdmin:clusterAdmin123456@34.118.234.136:27017/admin ' +++ mktemp ++ local LAST_OUT=/tmp/tmp.VuJjPf03Dk +++ mktemp ++ local LAST_ERR=/tmp/tmp.XKaInuXcl9 ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl exec psmdb-client-bb8b97679-xl6jr -- bash -c 'printf '\''rs.conf().members.length\n'\'' | mongo mongodb://clusterAdmin:clusterAdmin123456@34.118.234.136:27017/admin ' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.VuJjPf03Dk ++ cat /tmp/tmp.XKaInuXcl9 ++ rm /tmp/tmp.VuJjPf03Dk /tmp/tmp.XKaInuXcl9 ++ return 0 + nodes_count=5 + echo -n 'waiting for all members to be configured in cfg' waiting for all members to be configured in cfg+ let retry+=1 + '[' 9 -ge 15 ']' + echo . . + sleep 10 + [[ 5 == 6 ]] ++ run_mongos 'rs.conf().members.length' clusterAdmin:clusterAdmin123456@34.118.234.136 mongodb :27017 ++ local 'command=rs.conf().members.length' ++ local uri=clusterAdmin:clusterAdmin123456@34.118.234.136 ++ local driver=mongodb ++ grep -E -v 'I NETWORK|W NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|bye' ++ local suffix=:27017 ++ local mongo_flag= ++ local port=27017 ++ local mongo_bin=mongo ++ /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/' +++ echo :27017 +++ awk -F: '{print $2}' ++ suffix_port=27017 ++ [[ -z 27017 ]] +++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.vFbma8VI6C ++++ mktemp +++ local LAST_ERR=/tmp/tmp.EEHacRwLQZ +++ local exit_status=0 +++ local timeout=4 ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 -a -n 1 ']' +++ break +++ cat /tmp/tmp.vFbma8VI6C +++ cat /tmp/tmp.EEHacRwLQZ +++ rm /tmp/tmp.vFbma8VI6C /tmp/tmp.EEHacRwLQZ +++ return 0 ++ local client_container=psmdb-client-bb8b97679-xl6jr ++ kubectl_bin exec psmdb-client-bb8b97679-xl6jr -- bash -c 'printf '\''rs.conf().members.length\n'\'' | mongo mongodb://clusterAdmin:clusterAdmin123456@34.118.234.136:27017/admin ' +++ mktemp ++ local LAST_OUT=/tmp/tmp.x599ZUO5RT +++ mktemp ++ local LAST_ERR=/tmp/tmp.bJcQpKFFK1 ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl exec psmdb-client-bb8b97679-xl6jr -- bash -c 'printf '\''rs.conf().members.length\n'\'' | mongo mongodb://clusterAdmin:clusterAdmin123456@34.118.234.136:27017/admin ' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.x599ZUO5RT ++ cat /tmp/tmp.bJcQpKFFK1 ++ rm /tmp/tmp.x599ZUO5RT /tmp/tmp.bJcQpKFFK1 ++ return 0 + nodes_count=6 + echo -n 'waiting for all members to be configured in cfg' waiting for all members to be configured in cfg+ let retry+=1 + '[' 10 -ge 15 ']' + echo . . + sleep 10 + [[ 6 == 6 ]] + wait_for_members 34.118.235.28 rs0 6 + local endpoint=34.118.235.28 + local rsName=rs0 + local target_count=6 + local nodes_count=0 + [[ 0 == 6 ]] ++ run_mongos 'rs.conf().members.length' clusterAdmin:clusterAdmin123456@34.118.235.28 mongodb :27017 ++ local 'command=rs.conf().members.length' ++ local uri=clusterAdmin:clusterAdmin123456@34.118.235.28 ++ local driver=mongodb ++ local suffix=:27017 ++ local mongo_flag= ++ local port=27017 ++ local mongo_bin=mongo ++ /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/' ++ grep -E -v 'I NETWORK|W NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|bye' +++ echo :27017 +++ awk -F: '{print $2}' ++ suffix_port=27017 ++ [[ -z 27017 ]] +++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.2hmXJlcAXy ++++ mktemp +++ local LAST_ERR=/tmp/tmp.4frO4iY4HH +++ local exit_status=0 +++ local timeout=4 ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 -a -n 1 ']' +++ break +++ cat /tmp/tmp.2hmXJlcAXy +++ cat /tmp/tmp.4frO4iY4HH +++ rm /tmp/tmp.2hmXJlcAXy /tmp/tmp.4frO4iY4HH +++ return 0 ++ local client_container=psmdb-client-bb8b97679-xl6jr ++ kubectl_bin exec psmdb-client-bb8b97679-xl6jr -- bash -c 'printf '\''rs.conf().members.length\n'\'' | mongo mongodb://clusterAdmin:clusterAdmin123456@34.118.235.28:27017/admin ' +++ mktemp ++ local LAST_OUT=/tmp/tmp.Nh6D4RMEKo +++ mktemp ++ local LAST_ERR=/tmp/tmp.rdW6SxoqXt ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl exec psmdb-client-bb8b97679-xl6jr -- bash -c 'printf '\''rs.conf().members.length\n'\'' | mongo mongodb://clusterAdmin:clusterAdmin123456@34.118.235.28:27017/admin ' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.Nh6D4RMEKo ++ cat /tmp/tmp.rdW6SxoqXt ++ rm /tmp/tmp.Nh6D4RMEKo /tmp/tmp.rdW6SxoqXt ++ return 0 + nodes_count=6 + echo -n 'waiting for all members to be configured in rs0' waiting for all members to be configured in rs0+ let retry+=1 + '[' 11 -ge 15 ']' + echo . . + sleep 10 + [[ 6 == 6 ]] + wait_for_members 34.118.231.49 rs1 6 + local endpoint=34.118.231.49 + local rsName=rs1 + local target_count=6 + local nodes_count=0 + [[ 0 == 6 ]] ++ run_mongos 'rs.conf().members.length' clusterAdmin:clusterAdmin123456@34.118.231.49 mongodb :27017 ++ /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/' ++ grep -E -v 'I NETWORK|W NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|bye' ++ local 'command=rs.conf().members.length' ++ local uri=clusterAdmin:clusterAdmin123456@34.118.231.49 ++ local driver=mongodb ++ local suffix=:27017 ++ local mongo_flag= ++ local port=27017 ++ local mongo_bin=mongo +++ awk -F: '{print $2}' +++ echo :27017 ++ suffix_port=27017 ++ [[ -z 27017 ]] +++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.GpAJYhLsob ++++ mktemp +++ local LAST_ERR=/tmp/tmp.cSPNL4M3Q6 +++ local exit_status=0 +++ local timeout=4 ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 -a -n 1 ']' +++ break +++ cat /tmp/tmp.GpAJYhLsob +++ cat /tmp/tmp.cSPNL4M3Q6 +++ rm /tmp/tmp.GpAJYhLsob /tmp/tmp.cSPNL4M3Q6 +++ return 0 ++ local client_container=psmdb-client-bb8b97679-xl6jr ++ kubectl_bin exec psmdb-client-bb8b97679-xl6jr -- bash -c 'printf '\''rs.conf().members.length\n'\'' | mongo mongodb://clusterAdmin:clusterAdmin123456@34.118.231.49:27017/admin ' +++ mktemp ++ local LAST_OUT=/tmp/tmp.ocdKjtRyGV +++ mktemp ++ local LAST_ERR=/tmp/tmp.mZ9CgecDPl ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl exec psmdb-client-bb8b97679-xl6jr -- bash -c 'printf '\''rs.conf().members.length\n'\'' | mongo mongodb://clusterAdmin:clusterAdmin123456@34.118.231.49:27017/admin ' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.ocdKjtRyGV ++ cat /tmp/tmp.mZ9CgecDPl ++ rm /tmp/tmp.ocdKjtRyGV /tmp/tmp.mZ9CgecDPl ++ return 0 + nodes_count=6 + echo -n 'waiting for all members to be configured in rs1' waiting for all members to be configured in rs1+ let retry+=1 + '[' 12 -ge 15 ']' + echo . . + sleep 10 + [[ 6 == 6 ]] ++ kubectl_bin config current-context +++ mktemp ++ local LAST_OUT=/tmp/tmp.2kkExdnj7m +++ mktemp ++ local LAST_ERR=/tmp/tmp.mvs7fKNOi0 ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl config current-context ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.2kkExdnj7m ++ cat /tmp/tmp.mvs7fKNOi0 ++ rm /tmp/tmp.2kkExdnj7m /tmp/tmp.mvs7fKNOi0 ++ return 0 + kubectl_bin config set-context gke_cloud-dev-112233_us-central1-a_jen-psmdb-2283-73ca3795-11-cluster3 --namespace=cross-site-sharded-replica-15236 ++ mktemp + local LAST_OUT=/tmp/tmp.jFJpQTk8DX ++ mktemp + local LAST_ERR=/tmp/tmp.CKjfUp4e3M + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl config set-context gke_cloud-dev-112233_us-central1-a_jen-psmdb-2283-73ca3795-11-cluster3 --namespace=cross-site-sharded-replica-15236 + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.jFJpQTk8DX Context "gke_cloud-dev-112233_us-central1-a_jen-psmdb-2283-73ca3795-11-cluster3" modified. + cat /tmp/tmp.CKjfUp4e3M + rm /tmp/tmp.jFJpQTk8DX /tmp/tmp.CKjfUp4e3M + return 0 + desc 'check if all 3 Pods started' + set +o xtrace ----------------------------------------------------------------------------------- check if all 3 Pods started ----------------------------------------------------------------------------------- + wait_for_running cross-site-sharded-replica-rs0 3 + local name=cross-site-sharded-replica-rs0 + let last_pod=2 + local check_cluster_readyness=true + set_debug + [[ 1 == 1 ]] + set -o xtrace + local rs_name=rs0 + local cluster_name=cross-site-sharded-replica ++ seq 0 2 + for i in $(seq 0 $last_pod) + [[ 0 -eq 2 ]] + wait_pod cross-site-sharded-replica-rs0-0 + local pod=cross-site-sharded-replica-rs0-0 + set +o xtrace waiting for pod/cross-site-sharded-replica-rs0-0 to be ready.OK + for i in $(seq 0 $last_pod) + [[ 1 -eq 2 ]] + wait_pod cross-site-sharded-replica-rs0-1 + local pod=cross-site-sharded-replica-rs0-1 + set +o xtrace waiting for pod/cross-site-sharded-replica-rs0-1 to be ready.OK + for i in $(seq 0 $last_pod) + [[ 2 -eq 2 ]] ++ kubectl_bin get psmdb cross-site-sharded-replica -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].arbiter.enabled}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.mQ4YVYJ2OI +++ mktemp ++ local LAST_ERR=/tmp/tmp.pRig7Lp5Xd ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb cross-site-sharded-replica -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].arbiter.enabled}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.mQ4YVYJ2OI ++ cat /tmp/tmp.pRig7Lp5Xd ++ rm /tmp/tmp.mQ4YVYJ2OI /tmp/tmp.pRig7Lp5Xd ++ return 0 + [[ false == true ]] + wait_pod cross-site-sharded-replica-rs0-2 + local pod=cross-site-sharded-replica-rs0-2 + set +o xtrace waiting for pod/cross-site-sharded-replica-rs0-2 to be ready.OK ++ kubectl_bin get psmdb cross-site-sharded-replica -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].nonvoting.enabled}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.bnbOLd82LX +++ mktemp ++ local LAST_ERR=/tmp/tmp.uz5VX09D22 ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb cross-site-sharded-replica -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].nonvoting.enabled}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.bnbOLd82LX ++ cat /tmp/tmp.uz5VX09D22 ++ rm /tmp/tmp.bnbOLd82LX /tmp/tmp.uz5VX09D22 ++ return 0 + [[ false == true ]] ++ kubectl_bin get psmdb cross-site-sharded-replica -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].hidden.enabled}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.hgi6ZrV8KP +++ mktemp ++ local LAST_ERR=/tmp/tmp.aMoYD7SVWp ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb cross-site-sharded-replica -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].hidden.enabled}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.hgi6ZrV8KP ++ cat /tmp/tmp.aMoYD7SVWp ++ rm /tmp/tmp.hgi6ZrV8KP /tmp/tmp.aMoYD7SVWp ++ return 0 + [[ '' == true ]] + sleep 10 + [[ true == true ]] + set +x Waiting for cluster readyness + wait_for_running cross-site-sharded-replica-cfg 3 false + local name=cross-site-sharded-replica-cfg + let last_pod=2 + local check_cluster_readyness=false + set_debug + [[ 1 == 1 ]] + set -o xtrace + local rs_name=cfg + local cluster_name=cross-site-sharded-replica ++ seq 0 2 + for i in $(seq 0 $last_pod) + [[ 0 -eq 2 ]] + wait_pod cross-site-sharded-replica-cfg-0 + local pod=cross-site-sharded-replica-cfg-0 + set +o xtrace waiting for pod/cross-site-sharded-replica-cfg-0 to be ready.OK + for i in $(seq 0 $last_pod) + [[ 1 -eq 2 ]] + wait_pod cross-site-sharded-replica-cfg-1 + local pod=cross-site-sharded-replica-cfg-1 + set +o xtrace waiting for pod/cross-site-sharded-replica-cfg-1 to be ready.OK + for i in $(seq 0 $last_pod) + [[ 2 -eq 2 ]] ++ kubectl_bin get psmdb cross-site-sharded-replica -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].arbiter.enabled}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.WcClV5S7D0 +++ mktemp ++ local LAST_ERR=/tmp/tmp.qrarychcjZ ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb cross-site-sharded-replica -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].arbiter.enabled}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.WcClV5S7D0 ++ cat /tmp/tmp.qrarychcjZ ++ rm /tmp/tmp.WcClV5S7D0 /tmp/tmp.qrarychcjZ ++ return 0 + [[ '' == true ]] + wait_pod cross-site-sharded-replica-cfg-2 + local pod=cross-site-sharded-replica-cfg-2 + set +o xtrace waiting for pod/cross-site-sharded-replica-cfg-2 to be ready.OK ++ kubectl_bin get psmdb cross-site-sharded-replica -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].nonvoting.enabled}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.dtrecwXBiF +++ mktemp ++ local LAST_ERR=/tmp/tmp.rgLAyh1R0A ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb cross-site-sharded-replica -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].nonvoting.enabled}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.dtrecwXBiF ++ cat /tmp/tmp.rgLAyh1R0A ++ rm /tmp/tmp.dtrecwXBiF /tmp/tmp.rgLAyh1R0A ++ return 0 + [[ '' == true ]] ++ kubectl_bin get psmdb cross-site-sharded-replica -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].hidden.enabled}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.mZ7PZbn0nu +++ mktemp ++ local LAST_ERR=/tmp/tmp.2IoFLWSMRw ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb cross-site-sharded-replica -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].hidden.enabled}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.mZ7PZbn0nu ++ cat /tmp/tmp.2IoFLWSMRw ++ rm /tmp/tmp.mZ7PZbn0nu /tmp/tmp.2IoFLWSMRw ++ return 0 + [[ '' == true ]] + sleep 10 + [[ false == true ]] + desc 'create user' + set +o xtrace ----------------------------------------------------------------------------------- create user ----------------------------------------------------------------------------------- + run_mongos 'db.createUser({user:"myApp",pwd:"myPass",roles:[{db:"myApp",role:"readWrite"}]})' userAdmin:userAdmin123456@cross-site-sharded-main-mongos.cross-site-sharded-2618 + local 'command=db.createUser({user:"myApp",pwd:"myPass",roles:[{db:"myApp",role:"readWrite"}]})' + local uri=userAdmin:userAdmin123456@cross-site-sharded-main-mongos.cross-site-sharded-2618 + local driver=mongodb + local suffix=.svc.cluster.local + local mongo_flag= + local port=27017 + local mongo_bin=mongo ++ echo .svc.cluster.local ++ awk -F: '{print $2}' + suffix_port= + [[ -z '' ]] + suffix=.svc.cluster.local:27017 ++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.MOHOnHuDla +++ mktemp ++ local LAST_ERR=/tmp/tmp.oI41WLRDXV ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.MOHOnHuDla ++ cat /tmp/tmp.oI41WLRDXV ++ rm /tmp/tmp.MOHOnHuDla /tmp/tmp.oI41WLRDXV ++ return 0 + local client_container=psmdb-client-bb8b97679-h8twb + kubectl_bin exec psmdb-client-bb8b97679-h8twb -- bash -c 'printf '\''db.createUser({user:"myApp",pwd:"myPass",roles:[{db:"myApp",role:"readWrite"}]})\n'\'' | mongo mongodb://userAdmin:userAdmin123456@cross-site-sharded-main-mongos.cross-site-sharded-2618.svc.cluster.local:27017/admin ' ++ mktemp + local LAST_OUT=/tmp/tmp.PiYzyeHXuC ++ mktemp + local LAST_ERR=/tmp/tmp.m9QgmmQjnc + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl exec psmdb-client-bb8b97679-h8twb -- bash -c 'printf '\''db.createUser({user:"myApp",pwd:"myPass",roles:[{db:"myApp",role:"readWrite"}]})\n'\'' | mongo mongodb://userAdmin:userAdmin123456@cross-site-sharded-main-mongos.cross-site-sharded-2618.svc.cluster.local:27017/admin ' + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.PiYzyeHXuC Percona Server for MongoDB shell version v4.4.29-28 connecting to: mongodb://cross-site-sharded-main-mongos.cross-site-sharded-2618.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb Implicit session: session { "id" : UUID("fa852589-6fff-4797-ad42-2863ea17382c") } Percona Server for MongoDB server version: v8.0.20-8 WARNING: shell and server versions do not match Successfully added user: { "user" : "myApp", "roles" : [ { "db" : "myApp", "role" : "readWrite" } ] } bye + cat /tmp/tmp.m9QgmmQjnc + rm /tmp/tmp.PiYzyeHXuC /tmp/tmp.m9QgmmQjnc + return 0 + sleep 2 + desc 'write data, read from all' + set +o xtrace ----------------------------------------------------------------------------------- write data, read from all ----------------------------------------------------------------------------------- + run_mongos 'use myApp\n db.test.insert({ x: 100500 })' myApp:myPass@cross-site-sharded-main-mongos.cross-site-sharded-2618 + local 'command=use myApp\n db.test.insert({ x: 100500 })' + local uri=myApp:myPass@cross-site-sharded-main-mongos.cross-site-sharded-2618 + local driver=mongodb + local suffix=.svc.cluster.local + local mongo_flag= + local port=27017 + local mongo_bin=mongo ++ echo .svc.cluster.local ++ awk -F: '{print $2}' + suffix_port= + [[ -z '' ]] + suffix=.svc.cluster.local:27017 ++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.IipWcgQZV9 +++ mktemp ++ local LAST_ERR=/tmp/tmp.1KST4P9RIK ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.IipWcgQZV9 ++ cat /tmp/tmp.1KST4P9RIK ++ rm /tmp/tmp.IipWcgQZV9 /tmp/tmp.1KST4P9RIK ++ return 0 + local client_container=psmdb-client-bb8b97679-h8twb + kubectl_bin exec psmdb-client-bb8b97679-h8twb -- bash -c 'printf '\''use myApp\n db.test.insert({ x: 100500 })\n'\'' | mongo mongodb://myApp:myPass@cross-site-sharded-main-mongos.cross-site-sharded-2618.svc.cluster.local:27017/admin ' ++ mktemp + local LAST_OUT=/tmp/tmp.asEbYOZn6Z ++ mktemp + local LAST_ERR=/tmp/tmp.kSXBLAPY7V + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl exec psmdb-client-bb8b97679-h8twb -- bash -c 'printf '\''use myApp\n db.test.insert({ x: 100500 })\n'\'' | mongo mongodb://myApp:myPass@cross-site-sharded-main-mongos.cross-site-sharded-2618.svc.cluster.local:27017/admin ' + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.asEbYOZn6Z Percona Server for MongoDB shell version v4.4.29-28 connecting to: mongodb://cross-site-sharded-main-mongos.cross-site-sharded-2618.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb Implicit session: session { "id" : UUID("bbf93a23-0d94-435a-bfd8-18a19fb404f9") } Percona Server for MongoDB server version: v8.0.20-8 WARNING: shell and server versions do not match switched to db myApp WriteResult({ "nInserted" : 1 }) bye + cat /tmp/tmp.kSXBLAPY7V + rm /tmp/tmp.asEbYOZn6Z /tmp/tmp.kSXBLAPY7V + return 0 + minikube_sleep + sleep_time=10 + [[ '' == 1 ]] + desc 'Compare data' + set +o xtrace ----------------------------------------------------------------------------------- Compare data ----------------------------------------------------------------------------------- + compare_mongos_cmd find myApp:myPass@cross-site-sharded-main-mongos.cross-site-sharded-2618 + local command=find + local uri=myApp:myPass@cross-site-sharded-main-mongos.cross-site-sharded-2618 + local postfix= + local suffix= + local database=myApp + local collection=test + local port=27017 + local tls=false + [[ false == true ]] + mongos_command=run_mongos + log 'running db.test.command() in myApp' + set +o xtrace [2026-04-09T16:33:04+0000] running db.test.command() in myApp + run_mongos 'use myApp\n db.test.find()' myApp:myPass@cross-site-sharded-main-mongos.cross-site-sharded-2618 mongodb '' '' 27017 + local 'command=use myApp\n db.test.find()' + local uri=myApp:myPass@cross-site-sharded-main-mongos.cross-site-sharded-2618 + local driver=mongodb + local suffix=.svc.cluster.local + local mongo_flag= + local port=27017 + local mongo_bin=mongo + grep -E -v 'I NETWORK|W NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|Started a new thread for the timer service' + /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/' ++ awk -F: '{print $2}' ++ echo .svc.cluster.local + suffix_port= + [[ -z '' ]] + suffix=.svc.cluster.local:27017 ++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.dXHz6pihx7 +++ mktemp ++ local LAST_ERR=/tmp/tmp.JyVyMZZWOI ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.dXHz6pihx7 ++ cat /tmp/tmp.JyVyMZZWOI ++ rm /tmp/tmp.dXHz6pihx7 /tmp/tmp.JyVyMZZWOI ++ return 0 + local client_container=psmdb-client-bb8b97679-h8twb + kubectl_bin exec psmdb-client-bb8b97679-h8twb -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@cross-site-sharded-main-mongos.cross-site-sharded-2618.svc.cluster.local:27017/admin ' ++ mktemp + local LAST_OUT=/tmp/tmp.PNHrYjYtBn ++ mktemp + local LAST_ERR=/tmp/tmp.IPrEod3Qrj + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl exec psmdb-client-bb8b97679-h8twb -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@cross-site-sharded-main-mongos.cross-site-sharded-2618.svc.cluster.local:27017/admin ' + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.PNHrYjYtBn + cat /tmp/tmp.IPrEod3Qrj + rm /tmp/tmp.PNHrYjYtBn /tmp/tmp.IPrEod3Qrj + return 0 + [[ 0 -eq 0 ]] + diff /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2283/e2e-tests/cross-site-sharded/compare/find.json /tmp/tmp.k96VpRrcFR/find + desc 'test failover' + set +o xtrace ----------------------------------------------------------------------------------- test failover ----------------------------------------------------------------------------------- ++ kubectl_bin config current-context +++ mktemp ++ local LAST_OUT=/tmp/tmp.JdsoD2Wt7j +++ mktemp ++ local LAST_ERR=/tmp/tmp.8xtE3KZCh6 ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl config current-context ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.JdsoD2Wt7j ++ cat /tmp/tmp.8xtE3KZCh6 ++ rm /tmp/tmp.JdsoD2Wt7j /tmp/tmp.8xtE3KZCh6 ++ return 0 + kubectl_bin config set-context gke_cloud-dev-112233_us-central1-a_jen-psmdb-2283-73ca3795-11-cluster3 --namespace=cross-site-sharded-2618 ++ mktemp + local LAST_OUT=/tmp/tmp.9Kq3C7RDqm ++ mktemp + local LAST_ERR=/tmp/tmp.xAQuXxp4Zj + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl config set-context gke_cloud-dev-112233_us-central1-a_jen-psmdb-2283-73ca3795-11-cluster3 --namespace=cross-site-sharded-2618 + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.9Kq3C7RDqm Context "gke_cloud-dev-112233_us-central1-a_jen-psmdb-2283-73ca3795-11-cluster3" modified. + cat /tmp/tmp.xAQuXxp4Zj + rm /tmp/tmp.9Kq3C7RDqm /tmp/tmp.xAQuXxp4Zj + return 0 + kubectl_bin delete psmdb cross-site-sharded-main ++ mktemp + local LAST_OUT=/tmp/tmp.N40NnU0RAm ++ mktemp + local LAST_ERR=/tmp/tmp.3uscSaCQjB + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl delete psmdb cross-site-sharded-main + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.N40NnU0RAm perconaservermongodb.psmdb.percona.com "cross-site-sharded-main" deleted from cross-site-sharded-2618 namespace + cat /tmp/tmp.3uscSaCQjB + rm /tmp/tmp.N40NnU0RAm /tmp/tmp.3uscSaCQjB + return 0 + desc 'run disaster recovery script for replset: cfg' + set +o xtrace ----------------------------------------------------------------------------------- run disaster recovery script for replset: cfg ----------------------------------------------------------------------------------- + run_script_mongos /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2283/e2e-tests/cross-site-sharded/disaster_recovery.js clusterAdmin:clusterAdmin123456@34.118.234.136 mongodb :27017 + local script=/mnt/jenkins/workspace/cloud-psmdb-operator_PR-2283/e2e-tests/cross-site-sharded/disaster_recovery.js + local uri=clusterAdmin:clusterAdmin123456@34.118.234.136 + local driver=mongodb + local suffix=:27017 + local mongo_flag= + local mongo_bin=mongo ++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.F9N8ut93z1 +++ mktemp ++ local LAST_ERR=/tmp/tmp.vgdCFbShTQ ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.F9N8ut93z1 ++ cat /tmp/tmp.vgdCFbShTQ ++ rm /tmp/tmp.F9N8ut93z1 /tmp/tmp.vgdCFbShTQ ++ return 0 + local client_container=psmdb-client-bb8b97679-xl6jr ++ basename /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2283/e2e-tests/cross-site-sharded/disaster_recovery.js + name=disaster_recovery.js + kubectl_bin cp /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2283/e2e-tests/cross-site-sharded/disaster_recovery.js cross-site-sharded-2618/psmdb-client-bb8b97679-xl6jr:/tmp ++ mktemp + local LAST_OUT=/tmp/tmp.OyMzzjQPnK ++ mktemp + local LAST_ERR=/tmp/tmp.YqT2DI8MWh + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl cp /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2283/e2e-tests/cross-site-sharded/disaster_recovery.js cross-site-sharded-2618/psmdb-client-bb8b97679-xl6jr:/tmp + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.OyMzzjQPnK + cat /tmp/tmp.YqT2DI8MWh + rm /tmp/tmp.OyMzzjQPnK /tmp/tmp.YqT2DI8MWh + return 0 + kubectl_bin exec psmdb-client-bb8b97679-xl6jr -- bash -c 'mongo mongodb://clusterAdmin:clusterAdmin123456@34.118.234.136:27017/admin /tmp/disaster_recovery.js' ++ mktemp + local LAST_OUT=/tmp/tmp.yxZXIJwl5i ++ mktemp + local LAST_ERR=/tmp/tmp.HGdLmc1YRK + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl exec psmdb-client-bb8b97679-xl6jr -- bash -c 'mongo mongodb://clusterAdmin:clusterAdmin123456@34.118.234.136:27017/admin /tmp/disaster_recovery.js' + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.yxZXIJwl5i Percona Server for MongoDB shell version v4.4.29-28 connecting to: mongodb://34.118.234.136:27017/admin?compressors=disabled&gssapiServiceName=mongodb Implicit session: session { "id" : UUID("acf0b500-22b2-4c16-b29b-9f2a1c304764") } Percona Server for MongoDB server version: v8.0.20-8 WARNING: shell and server versions do not match + cat /tmp/tmp.HGdLmc1YRK + rm /tmp/tmp.yxZXIJwl5i /tmp/tmp.HGdLmc1YRK + return 0 + desc 'run disaster recovery script for replset: rs0' + set +o xtrace ----------------------------------------------------------------------------------- run disaster recovery script for replset: rs0 ----------------------------------------------------------------------------------- + run_script_mongos /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2283/e2e-tests/cross-site-sharded/disaster_recovery.js clusterAdmin:clusterAdmin123456@34.118.235.28 mongodb :27017 + local script=/mnt/jenkins/workspace/cloud-psmdb-operator_PR-2283/e2e-tests/cross-site-sharded/disaster_recovery.js + local uri=clusterAdmin:clusterAdmin123456@34.118.235.28 + local driver=mongodb + local suffix=:27017 + local mongo_flag= + local mongo_bin=mongo ++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.1rEZCNk10r +++ mktemp ++ local LAST_ERR=/tmp/tmp.OvyH4goo0K ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.1rEZCNk10r ++ cat /tmp/tmp.OvyH4goo0K ++ rm /tmp/tmp.1rEZCNk10r /tmp/tmp.OvyH4goo0K ++ return 0 + local client_container=psmdb-client-bb8b97679-xl6jr ++ basename /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2283/e2e-tests/cross-site-sharded/disaster_recovery.js + name=disaster_recovery.js + kubectl_bin cp /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2283/e2e-tests/cross-site-sharded/disaster_recovery.js cross-site-sharded-2618/psmdb-client-bb8b97679-xl6jr:/tmp ++ mktemp + local LAST_OUT=/tmp/tmp.W1UXsCHOAO ++ mktemp + local LAST_ERR=/tmp/tmp.fwKWkH73OI + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl cp /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2283/e2e-tests/cross-site-sharded/disaster_recovery.js cross-site-sharded-2618/psmdb-client-bb8b97679-xl6jr:/tmp + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.W1UXsCHOAO + cat /tmp/tmp.fwKWkH73OI + rm /tmp/tmp.W1UXsCHOAO /tmp/tmp.fwKWkH73OI + return 0 + kubectl_bin exec psmdb-client-bb8b97679-xl6jr -- bash -c 'mongo mongodb://clusterAdmin:clusterAdmin123456@34.118.235.28:27017/admin /tmp/disaster_recovery.js' ++ mktemp + local LAST_OUT=/tmp/tmp.KxgGZtb4qE ++ mktemp + local LAST_ERR=/tmp/tmp.3OGcOsRgHF + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl exec psmdb-client-bb8b97679-xl6jr -- bash -c 'mongo mongodb://clusterAdmin:clusterAdmin123456@34.118.235.28:27017/admin /tmp/disaster_recovery.js' + exit_status=1 + set -e + '[' 1 '!=' 0 -a -n 1 ']' + cat /tmp/tmp.KxgGZtb4qE Percona Server for MongoDB shell version v4.4.29-28 connecting to: mongodb://34.118.235.28:27017/admin?compressors=disabled&gssapiServiceName=mongodb Error: The server is in quiesce mode and will shut down : connect@src/mongo/shell/mongo.js:374:17 @(connect):2:6 exception: connect failed exiting with code 1 + cat /tmp/tmp.3OGcOsRgHF command terminated with exit code 1 + sleep 0 + for i in $(seq 0 2) + set +e + kubectl exec psmdb-client-bb8b97679-xl6jr -- bash -c 'mongo mongodb://clusterAdmin:clusterAdmin123456@34.118.235.28:27017/admin /tmp/disaster_recovery.js' + exit_status=1 + set -e + '[' 1 '!=' 0 -a -n 1 ']' + cat /tmp/tmp.KxgGZtb4qE Percona Server for MongoDB shell version v4.4.29-28 connecting to: mongodb://34.118.235.28:27017/admin?compressors=disabled&gssapiServiceName=mongodb Error: The server is in quiesce mode and will shut down : connect@src/mongo/shell/mongo.js:374:17 @(connect):2:6 exception: connect failed exiting with code 1 + cat /tmp/tmp.3OGcOsRgHF command terminated with exit code 1 + sleep 4 + for i in $(seq 0 2) + set +e + kubectl exec psmdb-client-bb8b97679-xl6jr -- bash -c 'mongo mongodb://clusterAdmin:clusterAdmin123456@34.118.235.28:27017/admin /tmp/disaster_recovery.js' + exit_status=1 + set -e + '[' 1 '!=' 0 -a -n 1 ']' + cat /tmp/tmp.KxgGZtb4qE Percona Server for MongoDB shell version v4.4.29-28 connecting to: mongodb://34.118.235.28:27017/admin?compressors=disabled&gssapiServiceName=mongodb Error: couldn't connect to server 34.118.235.28:27017, connection attempt failed: SocketException: Error connecting to 34.118.235.28:27017 :: caused by :: Connection timed out : connect@src/mongo/shell/mongo.js:374:17 @(connect):2:6 exception: connect failed exiting with code 1 + cat /tmp/tmp.3OGcOsRgHF command terminated with exit code 1 + sleep 8 + cat /tmp/tmp.KxgGZtb4qE Percona Server for MongoDB shell version v4.4.29-28 connecting to: mongodb://34.118.235.28:27017/admin?compressors=disabled&gssapiServiceName=mongodb Error: couldn't connect to server 34.118.235.28:27017, connection attempt failed: SocketException: Error connecting to 34.118.235.28:27017 :: caused by :: Connection timed out : connect@src/mongo/shell/mongo.js:374:17 @(connect):2:6 exception: connect failed exiting with code 1 + cat /tmp/tmp.3OGcOsRgHF command terminated with exit code 1 + rm /tmp/tmp.KxgGZtb4qE /tmp/tmp.3OGcOsRgHF + return 1