Log: /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2287/e2e-tests/logs/cross-site-sharded.log Warning: version difference between client (1.35) and server (1.32) exceeds the supported minor version skew of +/-1 Warning: version difference between client (1.35) and server (1.32) exceeds the supported minor version skew of +/-1 Warning: version difference between client (1.35) and server (1.32) exceeds the supported minor version skew of +/-1 ++ get_mongod_ver_from_image docker.io/perconalab/percona-server-mongodb-operator:main-mongod8.0 ++ local image=docker.io/perconalab/percona-server-mongodb-operator:main-mongod8.0 +++ run_simple_cli_inside_image docker.io/perconalab/percona-server-mongodb-operator:main-mongod8.0 'mongod --version' +++ local image=docker.io/perconalab/percona-server-mongodb-operator:main-mongod8.0 +++ /usr/sbin/sed -r 's/^.*db version v(([0-9]+\.){2}[0-9]+-[0-9]+).*$/\1/g' +++ local 'cli=mongod --version' +++ local pod_name=6923 +++ kubectl_bin -n default run 6923 --image=docker.io/perconalab/percona-server-mongodb-operator:main-mongod8.0 --restart=Never --command -- sleep infinity ++++ mktemp +++ local LAST_OUT=/tmp/tmp.PsxH1PnQxQ ++++ mktemp +++ local LAST_ERR=/tmp/tmp.1fsMhHPq77 +++ local exit_status=0 +++ local timeout=4 ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl -n default run 6923 --image=docker.io/perconalab/percona-server-mongodb-operator:main-mongod8.0 --restart=Never --command -- sleep infinity +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 -a -n 1 ']' +++ break +++ cat /tmp/tmp.PsxH1PnQxQ +++ cat /tmp/tmp.1fsMhHPq77 +++ rm /tmp/tmp.PsxH1PnQxQ /tmp/tmp.1fsMhHPq77 +++ return 0 +++ kubectl_bin -n default wait --for=condition=Ready pod/6923 ++++ mktemp +++ local LAST_OUT=/tmp/tmp.VXF1YSwm5C ++++ mktemp +++ local LAST_ERR=/tmp/tmp.9NWL9rbxHy +++ local exit_status=0 +++ local timeout=4 ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl -n default wait --for=condition=Ready pod/6923 +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 -a -n 1 ']' +++ break +++ cat /tmp/tmp.VXF1YSwm5C +++ cat /tmp/tmp.9NWL9rbxHy +++ rm /tmp/tmp.VXF1YSwm5C /tmp/tmp.9NWL9rbxHy +++ return 0 ++++ kubectl_bin -n default exec 6923 -- bash -c 'mongod --version 2>&1' +++++ mktemp ++++ local LAST_OUT=/tmp/tmp.inBM0dp2gN +++++ mktemp ++++ local LAST_ERR=/tmp/tmp.dlr7KmJCZ0 ++++ local exit_status=0 ++++ local timeout=4 +++++ seq 0 2 ++++ for i in $(seq 0 2) ++++ set +e ++++ kubectl -n default exec 6923 -- bash -c 'mongod --version 2>&1' ++++ exit_status=0 ++++ set -e ++++ '[' 0 '!=' 0 -a -n 1 ']' ++++ break ++++ cat /tmp/tmp.inBM0dp2gN ++++ cat /tmp/tmp.dlr7KmJCZ0 ++++ rm /tmp/tmp.inBM0dp2gN /tmp/tmp.dlr7KmJCZ0 ++++ return 0 +++ local $'output=db version v8.0.20-8\nBuild Info: {\n "version": "8.0.20-8",\n "gitVersion": "5f8e208f2ce1ec988e9830397069fad1147bc257",\n "openSSLVersion": "OpenSSL 3.5.1 1 Jul 2025",\n "modules": [],\n "perconaFeatures": [\n "MemoryEngine",\n "HotBackup",\n "BackupCursorAggregationStage",\n "BackupCursorExtendAggregationStage",\n "AWSIAM",\n "Kerberos",\n "LDAP",\n "OIDC",\n "TDE",\n "FIPSMode",\n "FCBIS",\n "Auditing",\n "ProfilingRateLimit",\n "LogRedaction",\n "ngram"\n ],\n "allocator": "tcmalloc-google",\n "environment": {\n "distarch": "x86_64",\n "target_arch": "x86_64"\n }\n}' +++ kubectl_bin -n default delete pod/6923 --grace-period=0 --force ++++ mktemp +++ local LAST_OUT=/tmp/tmp.AuptJkrR7S ++++ mktemp +++ local LAST_ERR=/tmp/tmp.KeE51jS0FV +++ local exit_status=0 +++ local timeout=4 ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl -n default delete pod/6923 --grace-period=0 --force +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 -a -n 1 ']' +++ break +++ cat /tmp/tmp.AuptJkrR7S +++ cat /tmp/tmp.KeE51jS0FV Warning: Immediate deletion does not wait for confirmation that the running resource has been terminated. The resource may continue to run on the cluster indefinitely. +++ rm /tmp/tmp.AuptJkrR7S /tmp/tmp.KeE51jS0FV +++ return 0 +++ echo db version v8.0.20-8 Build Info: '{' '"version":' '"8.0.20-8",' '"gitVersion":' '"5f8e208f2ce1ec988e9830397069fad1147bc257",' '"openSSLVersion":' '"OpenSSL' 3.5.1 1 Jul '2025",' '"modules":' '[],' '"perconaFeatures":' '[' '"MemoryEngine",' '"HotBackup",' '"BackupCursorAggregationStage",' '"BackupCursorExtendAggregationStage",' '"AWSIAM",' '"Kerberos",' '"LDAP",' '"OIDC",' '"TDE",' '"FIPSMode",' '"FCBIS",' '"Auditing",' '"ProfilingRateLimit",' '"LogRedaction",' '"ngram"' '],' '"allocator":' '"tcmalloc-google",' '"environment":' '{' '"distarch":' '"x86_64",' '"target_arch":' '"x86_64"' '}' '}' ++ version_info=8.0.20-8 ++ [[ ! 8.0.20-8 =~ ^([0-9]+\.){2}[0-9]+-[0-9]+$ ]] ++ echo 8.0.20-8 + FULL_VER=8.0.20-8 + MONGO_VER=8.0 + unset OPERATOR_NS + main_cluster=cross-site-sharded-main + replica_cluster=cross-site-sharded-replica + desc 'create main cluster' + set +o xtrace ----------------------------------------------------------------------------------- create main cluster ----------------------------------------------------------------------------------- + create_infra cross-site-sharded-600 + local ns=cross-site-sharded-600 + [[ 1 == 1 ]] + delete_crd + desc 'get and delete old CRDs and RBAC' + set +o xtrace ----------------------------------------------------------------------------------- get and delete old CRDs and RBAC ----------------------------------------------------------------------------------- + kubectl_bin delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2287/deploy/crd.yaml --ignore-not-found --wait=false ++ mktemp + local LAST_OUT=/tmp/tmp.Z8PXWGquL7 ++ mktemp + local LAST_ERR=/tmp/tmp.vQpUtkxW2z + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2287/deploy/crd.yaml --ignore-not-found --wait=false + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.Z8PXWGquL7 customresourcedefinition.apiextensions.k8s.io "perconaservermongodbbackups.psmdb.percona.com" deleted customresourcedefinition.apiextensions.k8s.io "perconaservermongodbrestores.psmdb.percona.com" deleted customresourcedefinition.apiextensions.k8s.io "perconaservermongodbs.psmdb.percona.com" deleted + cat /tmp/tmp.vQpUtkxW2z + rm /tmp/tmp.Z8PXWGquL7 /tmp/tmp.vQpUtkxW2z + return 0 ++ grep -v '\-\-\-' ++ yq eval .metadata.name /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2287/deploy/crd.yaml grep: warning: stray \ before - grep: warning: stray \ before - + for crd_name in $(yq eval '.metadata.name' "${src_dir}/deploy/crd.yaml" | grep -v '\-\-\-') + kubectl get perconaservermongodbbackups.psmdb.percona.com --all-namespaces -o wide + grep -v NAMESPACE + xargs -L 1 sh -xc 'kubectl patch perconaservermongodbbackups.psmdb.percona.com -n $0 $1 --type=merge -p "{\"metadata\":{\"finalizers\":[]}}"' error: the server doesn't have a resource type "perconaservermongodbbackups" + kubectl patch perconaservermongodbbackups.psmdb.percona.com -n sh --type=merge -p '{"metadata":{"finalizers":[]}}' error: the server doesn't have a resource type "perconaservermongodbbackups" + : + kubectl_bin wait --for=delete crd perconaservermongodbbackups.psmdb.percona.com ++ mktemp + local LAST_OUT=/tmp/tmp.t8q8eat5GR ++ mktemp + local LAST_ERR=/tmp/tmp.n0aCMMMd75 + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl wait --for=delete crd perconaservermongodbbackups.psmdb.percona.com + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.t8q8eat5GR + cat /tmp/tmp.n0aCMMMd75 + rm /tmp/tmp.t8q8eat5GR /tmp/tmp.n0aCMMMd75 + return 0 + for crd_name in $(yq eval '.metadata.name' "${src_dir}/deploy/crd.yaml" | grep -v '\-\-\-') + kubectl get perconaservermongodbrestores.psmdb.percona.com --all-namespaces -o wide + grep -v NAMESPACE + xargs -L 1 sh -xc 'kubectl patch perconaservermongodbrestores.psmdb.percona.com -n $0 $1 --type=merge -p "{\"metadata\":{\"finalizers\":[]}}"' error: the server doesn't have a resource type "perconaservermongodbrestores" + kubectl patch perconaservermongodbrestores.psmdb.percona.com -n sh --type=merge -p '{"metadata":{"finalizers":[]}}' error: the server doesn't have a resource type "perconaservermongodbrestores" + : + kubectl_bin wait --for=delete crd perconaservermongodbrestores.psmdb.percona.com ++ mktemp + local LAST_OUT=/tmp/tmp.MqsVNqQXPX ++ mktemp + local LAST_ERR=/tmp/tmp.3gH3mhkykG + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl wait --for=delete crd perconaservermongodbrestores.psmdb.percona.com + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.MqsVNqQXPX + cat /tmp/tmp.3gH3mhkykG + rm /tmp/tmp.MqsVNqQXPX /tmp/tmp.3gH3mhkykG + return 0 + for crd_name in $(yq eval '.metadata.name' "${src_dir}/deploy/crd.yaml" | grep -v '\-\-\-') + kubectl get perconaservermongodbs.psmdb.percona.com --all-namespaces -o wide + grep -v NAMESPACE + xargs -L 1 sh -xc 'kubectl patch perconaservermongodbs.psmdb.percona.com -n $0 $1 --type=merge -p "{\"metadata\":{\"finalizers\":[]}}"' error: the server doesn't have a resource type "perconaservermongodbs" + kubectl patch perconaservermongodbs.psmdb.percona.com -n sh --type=merge -p '{"metadata":{"finalizers":[]}}' error: the server doesn't have a resource type "perconaservermongodbs" + : + kubectl_bin wait --for=delete crd perconaservermongodbs.psmdb.percona.com ++ mktemp + local LAST_OUT=/tmp/tmp.RsPChbHf2Q ++ mktemp + local LAST_ERR=/tmp/tmp.cxSMstLnR0 + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl wait --for=delete crd perconaservermongodbs.psmdb.percona.com + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.RsPChbHf2Q + cat /tmp/tmp.cxSMstLnR0 + rm /tmp/tmp.RsPChbHf2Q /tmp/tmp.cxSMstLnR0 + return 0 + local rbac_yaml=rbac.yaml + '[' -n '' ']' + kubectl_bin delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2287/deploy/rbac.yaml --ignore-not-found ++ mktemp + local LAST_OUT=/tmp/tmp.jDnWKbdOUV ++ mktemp + local LAST_ERR=/tmp/tmp.5CRtjRqVmJ + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2287/deploy/rbac.yaml --ignore-not-found + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.jDnWKbdOUV role.rbac.authorization.k8s.io "percona-server-mongodb-operator" deleted from cross-site-sharded-replica-21027 namespace serviceaccount "percona-server-mongodb-operator" deleted from cross-site-sharded-replica-21027 namespace rolebinding.rbac.authorization.k8s.io "service-account-percona-server-mongodb-operator" deleted from cross-site-sharded-replica-21027 namespace + cat /tmp/tmp.5CRtjRqVmJ + rm /tmp/tmp.jDnWKbdOUV /tmp/tmp.5CRtjRqVmJ + return 0 + check_crd_for_deletion PR-2287-fc474e65 + local git_tag=PR-2287-fc474e65 ++ curl -s https://raw.githubusercontent.com/percona/percona-server-mongodb-operator/PR-2287-fc474e65/deploy/crd.yaml ++ yq eval .metadata.name ++ /usr/sbin/sed s/---//g ++ /usr/sbin/sed ':a;N;$!ba;s/\n/ /g' + for crd_name in $(curl -s https://raw.githubusercontent.com/percona/percona-server-mongodb-operator/${git_tag}/deploy/crd.yaml | yq eval '.metadata.name' | $sed 's/---//g' | $sed ':a;N;$!ba;s/\n/ /g') ++ kubectl_bin get crd/null -o 'jsonpath={.status.conditions[-1].type}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.3n4CvHb8Qc +++ mktemp ++ local LAST_ERR=/tmp/tmp.PALzynSSCR ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get crd/null -o 'jsonpath={.status.conditions[-1].type}' ++ exit_status=1 ++ set -e ++ '[' 1 '!=' 0 -a -n 1 ']' ++ cat /tmp/tmp.3n4CvHb8Qc ++ cat /tmp/tmp.PALzynSSCR Error from server (NotFound): customresourcedefinitions.apiextensions.k8s.io "null" not found ++ sleep 0 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get crd/null -o 'jsonpath={.status.conditions[-1].type}' ++ exit_status=1 ++ set -e ++ '[' 1 '!=' 0 -a -n 1 ']' ++ cat /tmp/tmp.3n4CvHb8Qc ++ cat /tmp/tmp.PALzynSSCR Error from server (NotFound): customresourcedefinitions.apiextensions.k8s.io "null" not found ++ sleep 4 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get crd/null -o 'jsonpath={.status.conditions[-1].type}' ++ exit_status=1 ++ set -e ++ '[' 1 '!=' 0 -a -n 1 ']' ++ cat /tmp/tmp.3n4CvHb8Qc ++ cat /tmp/tmp.PALzynSSCR Error from server (NotFound): customresourcedefinitions.apiextensions.k8s.io "null" not found ++ sleep 8 ++ cat /tmp/tmp.3n4CvHb8Qc ++ cat /tmp/tmp.PALzynSSCR Error from server (NotFound): customresourcedefinitions.apiextensions.k8s.io "null" not found ++ rm /tmp/tmp.3n4CvHb8Qc /tmp/tmp.PALzynSSCR ++ return 1 + [[ '' == Terminating ]] + '[' -n '' ']' + create_namespace cross-site-sharded-600 + local namespace=cross-site-sharded-600 + local skip_clean_namespace= + [[ 1 == 1 ]] + [[ -z '' ]] + destroy_chaos_mesh ++ helm list --all-namespaces --filter chaos-mesh ++ tail -n1 ++ awk '-F ' '{print $2}' ++ sed s/NAMESPACE// + local chaos_mesh_ns= + desc 'destroy chaos-mesh' + set +o xtrace ----------------------------------------------------------------------------------- destroy chaos-mesh ----------------------------------------------------------------------------------- + '[' -n '' ']' ++ grep chaos-mesh ++ kubectl get MutatingWebhookConfiguration ++ awk '{print $1}' + timeout 30 kubectl delete MutatingWebhookConfiguration error: resource(s) were provided, but no name was specified + : ++ kubectl get ValidatingWebhookConfiguration ++ awk '{print $1}' ++ grep chaos-mesh + timeout 30 kubectl delete ValidatingWebhookConfiguration error: resource(s) were provided, but no name was specified + : ++ kubectl get ValidatingWebhookConfiguration ++ awk '{print $1}' ++ grep validate-auth + timeout 30 kubectl delete ValidatingWebhookConfiguration error: resource(s) were provided, but no name was specified + : ++ kubectl api-resources ++ grep chaos-mesh ++ awk '{print $1}' ++ kubectl get crd ++ grep chaos-mesh.org ++ awk '{print $1}' + timeout 30 kubectl delete crd error: resource(s) were provided, but no name was specified + : ++ grep chaos-mesh ++ kubectl get clusterrolebinding ++ awk '{print $1}' + timeout 30 kubectl delete clusterrolebinding error: resource(s) were provided, but no name was specified + : ++ kubectl get clusterrole ++ grep chaos-mesh ++ awk '{print $1}' + timeout 30 kubectl delete clusterrole error: resource(s) were provided, but no name was specified + : + desc 'cleaned up all old namespaces' + set +o xtrace ----------------------------------------------------------------------------------- cleaned up all old namespaces ----------------------------------------------------------------------------------- + kubectl_bin get ns ++ mktemp + '[' -n '' ']' + local LAST_OUT=/tmp/tmp.f8qGWpoE3w ++ mktemp + desc 'cleaned up old namespaces cross-site-sharded-600' + set +o xtrace ----------------------------------------------------------------------------------- cleaned up old namespaces cross-site-sharded-600 ----------------------------------------------------------------------------------- + kubectl_bin delete namespace cross-site-sharded-600 --ignore-not-found + local LAST_ERR=/tmp/tmp.BN5kaVKlDC + local exit_status=0 + local timeout=4 + awk '{print$1}' + grep -E -v '^kube-|^default|Terminating|psmdb-operator|openshift|^gke-|^gmp-|^NAME' ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl get ns + xargs kubectl delete ns ++ mktemp + local LAST_OUT=/tmp/tmp.c3tqVe71So ++ mktemp + local LAST_ERR=/tmp/tmp.xdVgCEleeX + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl delete namespace cross-site-sharded-600 --ignore-not-found + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.f8qGWpoE3w + cat /tmp/tmp.BN5kaVKlDC + rm /tmp/tmp.f8qGWpoE3w /tmp/tmp.BN5kaVKlDC + return 0 + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.c3tqVe71So + cat /tmp/tmp.xdVgCEleeX + rm /tmp/tmp.c3tqVe71So /tmp/tmp.xdVgCEleeX + return 0 + kubectl_bin wait --for=delete namespace cross-site-sharded-600 ++ mktemp + local LAST_OUT=/tmp/tmp.Ah9vnRl4ZF ++ mktemp + local LAST_ERR=/tmp/tmp.Fw6gFoHkeK + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl wait --for=delete namespace cross-site-sharded-600 namespace "cross-site-sharded-22886" deleted namespace "cross-site-sharded-replica-21027" deleted + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.Ah9vnRl4ZF + cat /tmp/tmp.Fw6gFoHkeK + rm /tmp/tmp.Ah9vnRl4ZF /tmp/tmp.Fw6gFoHkeK + return 0 + desc 'create namespace cross-site-sharded-600' + set +o xtrace ----------------------------------------------------------------------------------- create namespace cross-site-sharded-600 ----------------------------------------------------------------------------------- + kubectl_bin create namespace cross-site-sharded-600 ++ mktemp + local LAST_OUT=/tmp/tmp.vpLbBJYa5q ++ mktemp + local LAST_ERR=/tmp/tmp.LTpBLNvaEw + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl create namespace cross-site-sharded-600 + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.vpLbBJYa5q namespace/cross-site-sharded-600 created + cat /tmp/tmp.LTpBLNvaEw + rm /tmp/tmp.vpLbBJYa5q /tmp/tmp.LTpBLNvaEw + return 0 + set_kube_ctx cross-site-sharded-600 + local namespace=cross-site-sharded-600 ++ kubectl_bin config current-context +++ mktemp ++ local LAST_OUT=/tmp/tmp.L5NmtUgXUm +++ mktemp ++ local LAST_ERR=/tmp/tmp.mbjM9DNjcB ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl config current-context ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.L5NmtUgXUm ++ cat /tmp/tmp.mbjM9DNjcB ++ rm /tmp/tmp.L5NmtUgXUm /tmp/tmp.mbjM9DNjcB ++ return 0 + kubectl_bin config set-context gke_cloud-dev-112233_us-central1-a_jen-psmdb-2287-fc474e65-5-cluster3 --namespace=cross-site-sharded-600 ++ mktemp + local LAST_OUT=/tmp/tmp.dScBbKPTOg ++ mktemp + local LAST_ERR=/tmp/tmp.dxP5EEMT5C + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl config set-context gke_cloud-dev-112233_us-central1-a_jen-psmdb-2287-fc474e65-5-cluster3 --namespace=cross-site-sharded-600 + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.dScBbKPTOg Context "gke_cloud-dev-112233_us-central1-a_jen-psmdb-2287-fc474e65-5-cluster3" modified. + cat /tmp/tmp.dxP5EEMT5C + rm /tmp/tmp.dScBbKPTOg /tmp/tmp.dxP5EEMT5C + return 0 + deploy_operator + desc 'start PSMDB operator: docker.io/perconalab/percona-server-mongodb-operator:PR-2287-fc474e65' + set +o xtrace ----------------------------------------------------------------------------------- start PSMDB operator: docker.io/perconalab/percona-server-mongodb-operator:PR-2287-fc474e65 ----------------------------------------------------------------------------------- + local cr_file + '[' -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2287/e2e-tests/cross-site-sharded/conf/crd.yaml ']' + cr_file=/mnt/jenkins/workspace/cloud-psmdb-operator_PR-2287/deploy/crd.yaml + kubectl_bin apply --server-side --force-conflicts -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2287/deploy/crd.yaml ++ mktemp + local LAST_OUT=/tmp/tmp.wv8iKBb2ed ++ mktemp + local LAST_ERR=/tmp/tmp.okJPuRSlcd + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl apply --server-side --force-conflicts -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2287/deploy/crd.yaml + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.wv8iKBb2ed customresourcedefinition.apiextensions.k8s.io/perconaservermongodbbackups.psmdb.percona.com serverside-applied customresourcedefinition.apiextensions.k8s.io/perconaservermongodbrestores.psmdb.percona.com serverside-applied customresourcedefinition.apiextensions.k8s.io/perconaservermongodbs.psmdb.percona.com serverside-applied + cat /tmp/tmp.okJPuRSlcd + rm /tmp/tmp.wv8iKBb2ed /tmp/tmp.okJPuRSlcd + return 0 + '[' -n '' ']' + apply_rbac rbac + local operator_namespace=psmdb-operator + local rbac=rbac + cat /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2287/deploy/rbac.yaml + sed -e 's^namespace: .*^namespace: psmdb-operator^' + kubectl_bin apply -f - ++ mktemp + local LAST_OUT=/tmp/tmp.uaQwVhNxvh ++ mktemp + local LAST_ERR=/tmp/tmp.wUQl0AoeAM + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.uaQwVhNxvh role.rbac.authorization.k8s.io/percona-server-mongodb-operator created serviceaccount/percona-server-mongodb-operator created rolebinding.rbac.authorization.k8s.io/service-account-percona-server-mongodb-operator created + cat /tmp/tmp.wUQl0AoeAM + rm /tmp/tmp.uaQwVhNxvh /tmp/tmp.wUQl0AoeAM + return 0 + yq eval $'\n\t\t\t(.spec.template.spec.containers[].image = "docker.io/perconalab/percona-server-mongodb-operator:PR-2287-fc474e65") |\n\t\t\t((.. | select(.[] == "DISABLE_TELEMETRY")) |= .value="true") |\n\t\t\t((.. | select(.[] == "LOG_LEVEL")) |= .value="DEBUG")' /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2287/deploy/operator.yaml + kubectl_bin apply -f - ++ mktemp + local LAST_OUT=/tmp/tmp.XbPn3Fzu6C ++ mktemp + local LAST_ERR=/tmp/tmp.oiMDQxmmLH + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.XbPn3Fzu6C deployment.apps/percona-server-mongodb-operator created + cat /tmp/tmp.oiMDQxmmLH + rm /tmp/tmp.XbPn3Fzu6C /tmp/tmp.oiMDQxmmLH + return 0 + sleep 20 ++ get_operator_pod ++ kubectl_bin get pods --selector=name=percona-server-mongodb-operator -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.1eyTjXB7PJ +++ mktemp ++ local LAST_ERR=/tmp/tmp.mkoGSrcR9h ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pods --selector=name=percona-server-mongodb-operator -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.1eyTjXB7PJ ++ cat /tmp/tmp.mkoGSrcR9h ++ rm /tmp/tmp.1eyTjXB7PJ /tmp/tmp.mkoGSrcR9h ++ return 0 + wait_operator_pod percona-server-mongodb-operator-669854d959-cpcvc + local pod=percona-server-mongodb-operator-669854d959-cpcvc + set +o xtrace waiting for pod/percona-server-mongodb-operator-669854d959-cpcvc to be ready.OK + echo 'Print operator info from log' Print operator info from log + grep 'Manager starting up' ++ get_operator_pod ++ kubectl_bin get pods --selector=name=percona-server-mongodb-operator -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.YYkyaU2Im2 +++ mktemp ++ local LAST_ERR=/tmp/tmp.hz97wPu7Mn ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pods --selector=name=percona-server-mongodb-operator -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.YYkyaU2Im2 ++ cat /tmp/tmp.hz97wPu7Mn ++ rm /tmp/tmp.YYkyaU2Im2 /tmp/tmp.hz97wPu7Mn ++ return 0 + kubectl_bin logs percona-server-mongodb-operator-669854d959-cpcvc ++ mktemp + local LAST_OUT=/tmp/tmp.Ypz1kEDbq6 ++ mktemp + local LAST_ERR=/tmp/tmp.ydb7egr3nx + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl logs percona-server-mongodb-operator-669854d959-cpcvc + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.Ypz1kEDbq6 + cat /tmp/tmp.ydb7egr3nx + rm /tmp/tmp.Ypz1kEDbq6 /tmp/tmp.ydb7egr3nx + return 0 2026-04-09T12:32:08.595Z INFO setup Manager starting up {"gitCommit": "fc474e65109342c13c34c3ce7546452f0d6ad19d", "gitBranch": "PR-2287-fc474e65", "buildTime": "", "goVersion": "go1.25.9", "os": "linux", "arch": "amd64"} + desc 'create secrets and start client' + set +o xtrace ----------------------------------------------------------------------------------- create secrets and start client ----------------------------------------------------------------------------------- + kubectl_bin apply -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2287/e2e-tests/conf/client.yml -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2287/e2e-tests/cross-site-sharded/conf/secrets.yml ++ mktemp + local LAST_OUT=/tmp/tmp.v0lFR5lTeu ++ mktemp + local LAST_ERR=/tmp/tmp.9OkkHwF0JO + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl apply -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2287/e2e-tests/conf/client.yml -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2287/e2e-tests/cross-site-sharded/conf/secrets.yml + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.v0lFR5lTeu deployment.apps/psmdb-client created secret/cross-site-sharded-main-secrets created secret/cross-site-sharded-main-ssl created secret/cross-site-sharded-main-ssl-internal created + cat /tmp/tmp.9OkkHwF0JO + rm /tmp/tmp.v0lFR5lTeu /tmp/tmp.9OkkHwF0JO + return 0 + desc 'create main PSMDB cluster cross-site-sharded-main.' + set +o xtrace ----------------------------------------------------------------------------------- create main PSMDB cluster cross-site-sharded-main. ----------------------------------------------------------------------------------- + apply_cluster /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2287/e2e-tests/cross-site-sharded/conf/cross-site-sharded-main.yml + '[' -z '' ']' + cat_config /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2287/e2e-tests/cross-site-sharded/conf/cross-site-sharded-main.yml + kubectl_bin apply -f - + cat /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2287/e2e-tests/cross-site-sharded/conf/cross-site-sharded-main.yml ++ mktemp + yq eval '(.spec | select(.image == null)).image = "docker.io/perconalab/percona-server-mongodb-operator:main-mongod8.0"' + yq eval '(.spec | select(has("initImage"))).initImage = "docker.io/perconalab/percona-server-mongodb-operator:PR-2287-fc474e65"' + yq eval '(.spec | select(has("backup"))).backup.image = "docker.io/perconalab/percona-server-mongodb-operator:main-backup"' + /usr/sbin/sed -e s/NAME_SPACE/cross-site-sharded-600/g + yq eval '(.spec | select(has("pmm"))).pmm.image = "docker.io/percona/pmm-client:2.44.1-1"' + yq eval '.spec.upgradeOptions.apply="Never"' + local LAST_OUT=/tmp/tmp.Ctx9afHHEC ++ mktemp + local LAST_ERR=/tmp/tmp.jSsunIu1wz + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.Ctx9afHHEC perconaservermongodb.psmdb.percona.com/cross-site-sharded-main created + cat /tmp/tmp.jSsunIu1wz + rm /tmp/tmp.Ctx9afHHEC /tmp/tmp.jSsunIu1wz + return 0 + desc 'check if all 3 Pods started' + set +o xtrace ----------------------------------------------------------------------------------- check if all 3 Pods started ----------------------------------------------------------------------------------- + wait_for_running cross-site-sharded-main-rs0 3 + local name=cross-site-sharded-main-rs0 + let last_pod=2 + local check_cluster_readyness=true + set_debug + [[ 1 == 1 ]] + set -o xtrace + local rs_name=rs0 + local cluster_name=cross-site-sharded-main ++ seq 0 2 + for i in $(seq 0 $last_pod) + [[ 0 -eq 2 ]] + wait_pod cross-site-sharded-main-rs0-0 + local pod=cross-site-sharded-main-rs0-0 + set +o xtrace waiting for pod/cross-site-sharded-main-rs0-0 to be ready..............OK + for i in $(seq 0 $last_pod) + [[ 1 -eq 2 ]] + wait_pod cross-site-sharded-main-rs0-1 + local pod=cross-site-sharded-main-rs0-1 + set +o xtrace waiting for pod/cross-site-sharded-main-rs0-1 to be ready.............OK + for i in $(seq 0 $last_pod) + [[ 2 -eq 2 ]] ++ kubectl_bin get psmdb cross-site-sharded-main -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].arbiter.enabled}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.zr4yUqket9 +++ mktemp ++ local LAST_ERR=/tmp/tmp.bdDXdwUL9m ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb cross-site-sharded-main -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].arbiter.enabled}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.zr4yUqket9 ++ cat /tmp/tmp.bdDXdwUL9m ++ rm /tmp/tmp.zr4yUqket9 /tmp/tmp.bdDXdwUL9m ++ return 0 + [[ false == true ]] + wait_pod cross-site-sharded-main-rs0-2 + local pod=cross-site-sharded-main-rs0-2 + set +o xtrace waiting for pod/cross-site-sharded-main-rs0-2 to be ready.............OK ++ kubectl_bin get psmdb cross-site-sharded-main -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].nonvoting.enabled}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.M2qaXclXxm +++ mktemp ++ local LAST_ERR=/tmp/tmp.lnooQqRXvd ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb cross-site-sharded-main -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].nonvoting.enabled}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.M2qaXclXxm ++ cat /tmp/tmp.lnooQqRXvd ++ rm /tmp/tmp.M2qaXclXxm /tmp/tmp.lnooQqRXvd ++ return 0 + [[ false == true ]] ++ kubectl_bin get psmdb cross-site-sharded-main -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].hidden.enabled}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.VPUmcteSyv +++ mktemp ++ local LAST_ERR=/tmp/tmp.VxGsMpRKQ3 ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb cross-site-sharded-main -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].hidden.enabled}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.VPUmcteSyv ++ cat /tmp/tmp.VxGsMpRKQ3 ++ rm /tmp/tmp.VPUmcteSyv /tmp/tmp.VxGsMpRKQ3 ++ return 0 + [[ '' == true ]] + sleep 10 + [[ true == true ]] + set +x Waiting for cluster readyness.......................................................... + wait_for_running cross-site-sharded-main-cfg 3 false + local name=cross-site-sharded-main-cfg + let last_pod=2 + local check_cluster_readyness=false + set_debug + [[ 1 == 1 ]] + set -o xtrace + local rs_name=cfg + local cluster_name=cross-site-sharded-main ++ seq 0 2 + for i in $(seq 0 $last_pod) + [[ 0 -eq 2 ]] + wait_pod cross-site-sharded-main-cfg-0 + local pod=cross-site-sharded-main-cfg-0 + set +o xtrace waiting for pod/cross-site-sharded-main-cfg-0 to be ready.OK + for i in $(seq 0 $last_pod) + [[ 1 -eq 2 ]] + wait_pod cross-site-sharded-main-cfg-1 + local pod=cross-site-sharded-main-cfg-1 + set +o xtrace waiting for pod/cross-site-sharded-main-cfg-1 to be ready.OK + for i in $(seq 0 $last_pod) + [[ 2 -eq 2 ]] ++ kubectl_bin get psmdb cross-site-sharded-main -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].arbiter.enabled}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.oPIXtnWlW7 +++ mktemp ++ local LAST_ERR=/tmp/tmp.DkJeCJqzt0 ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb cross-site-sharded-main -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].arbiter.enabled}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.oPIXtnWlW7 ++ cat /tmp/tmp.DkJeCJqzt0 ++ rm /tmp/tmp.oPIXtnWlW7 /tmp/tmp.DkJeCJqzt0 ++ return 0 + [[ '' == true ]] + wait_pod cross-site-sharded-main-cfg-2 + local pod=cross-site-sharded-main-cfg-2 + set +o xtrace waiting for pod/cross-site-sharded-main-cfg-2 to be ready.OK ++ kubectl_bin get psmdb cross-site-sharded-main -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].nonvoting.enabled}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.3inCqAEnrG +++ mktemp ++ local LAST_ERR=/tmp/tmp.bDse6uwE2c ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb cross-site-sharded-main -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].nonvoting.enabled}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.3inCqAEnrG ++ cat /tmp/tmp.bDse6uwE2c ++ rm /tmp/tmp.3inCqAEnrG /tmp/tmp.bDse6uwE2c ++ return 0 + [[ '' == true ]] ++ kubectl_bin get psmdb cross-site-sharded-main -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].hidden.enabled}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.RzSjSXnU54 +++ mktemp ++ local LAST_ERR=/tmp/tmp.xtK2neeLN8 ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb cross-site-sharded-main -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].hidden.enabled}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.RzSjSXnU54 ++ cat /tmp/tmp.xtK2neeLN8 ++ rm /tmp/tmp.RzSjSXnU54 /tmp/tmp.xtK2neeLN8 ++ return 0 + [[ '' == true ]] + sleep 10 + [[ false == true ]] + desc 'create user' + set +o xtrace ----------------------------------------------------------------------------------- create user ----------------------------------------------------------------------------------- + run_mongos 'db.createUser({user:"user",pwd:"pass",roles:[{db:"app",role:"readWrite"}]})' userAdmin:userAdmin123456@cross-site-sharded-main-mongos.cross-site-sharded-600 + local 'command=db.createUser({user:"user",pwd:"pass",roles:[{db:"app",role:"readWrite"}]})' + local uri=userAdmin:userAdmin123456@cross-site-sharded-main-mongos.cross-site-sharded-600 + local driver=mongodb + local suffix=.svc.cluster.local + local mongo_flag= + local port=27017 + local mongo_bin=mongo ++ awk -F: '{print $2}' ++ echo .svc.cluster.local + suffix_port= + [[ -z '' ]] + suffix=.svc.cluster.local:27017 ++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.WTqRQnbGuu +++ mktemp ++ local LAST_ERR=/tmp/tmp.7BpIK0hD3a ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.WTqRQnbGuu ++ cat /tmp/tmp.7BpIK0hD3a ++ rm /tmp/tmp.WTqRQnbGuu /tmp/tmp.7BpIK0hD3a ++ return 0 + local client_container=psmdb-client-bb8b97679-gw8g4 + kubectl_bin exec psmdb-client-bb8b97679-gw8g4 -- bash -c 'printf '\''db.createUser({user:"user",pwd:"pass",roles:[{db:"app",role:"readWrite"}]})\n'\'' | mongo mongodb://userAdmin:userAdmin123456@cross-site-sharded-main-mongos.cross-site-sharded-600.svc.cluster.local:27017/admin ' ++ mktemp + local LAST_OUT=/tmp/tmp.EgwO0CeFUx ++ mktemp + local LAST_ERR=/tmp/tmp.j2zNmP6m1u + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl exec psmdb-client-bb8b97679-gw8g4 -- bash -c 'printf '\''db.createUser({user:"user",pwd:"pass",roles:[{db:"app",role:"readWrite"}]})\n'\'' | mongo mongodb://userAdmin:userAdmin123456@cross-site-sharded-main-mongos.cross-site-sharded-600.svc.cluster.local:27017/admin ' + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.EgwO0CeFUx Percona Server for MongoDB shell version v4.4.29-28 connecting to: mongodb://cross-site-sharded-main-mongos.cross-site-sharded-600.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb Implicit session: session { "id" : UUID("62b2ef93-559a-410c-8809-824cdcae2912") } Percona Server for MongoDB server version: v8.0.20-8 WARNING: shell and server versions do not match Successfully added user: { "user" : "user", "roles" : [ { "db" : "app", "role" : "readWrite" } ] } bye + cat /tmp/tmp.j2zNmP6m1u + rm /tmp/tmp.EgwO0CeFUx /tmp/tmp.j2zNmP6m1u + return 0 + sleep 2 + desc 'set chunk size to 2 MB' + set +o xtrace ----------------------------------------------------------------------------------- set chunk size to 2 MB ----------------------------------------------------------------------------------- + run_mongos 'use config\n db.settings.save( { _id:"chunksize", value: 2 } )' clusterAdmin:clusterAdmin123456@cross-site-sharded-main-mongos.cross-site-sharded-600 + local 'command=use config\n db.settings.save( { _id:"chunksize", value: 2 } )' + local uri=clusterAdmin:clusterAdmin123456@cross-site-sharded-main-mongos.cross-site-sharded-600 + local driver=mongodb + local suffix=.svc.cluster.local + local mongo_flag= + local port=27017 + local mongo_bin=mongo ++ echo .svc.cluster.local ++ awk -F: '{print $2}' + suffix_port= + [[ -z '' ]] + suffix=.svc.cluster.local:27017 ++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.smWMaFRQdG +++ mktemp ++ local LAST_ERR=/tmp/tmp.ESU3ZZ3A9N ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.smWMaFRQdG ++ cat /tmp/tmp.ESU3ZZ3A9N ++ rm /tmp/tmp.smWMaFRQdG /tmp/tmp.ESU3ZZ3A9N ++ return 0 + local client_container=psmdb-client-bb8b97679-gw8g4 + kubectl_bin exec psmdb-client-bb8b97679-gw8g4 -- bash -c 'printf '\''use config\n db.settings.save( { _id:"chunksize", value: 2 } )\n'\'' | mongo mongodb://clusterAdmin:clusterAdmin123456@cross-site-sharded-main-mongos.cross-site-sharded-600.svc.cluster.local:27017/admin ' ++ mktemp + local LAST_OUT=/tmp/tmp.BOryKH4snN ++ mktemp + local LAST_ERR=/tmp/tmp.sAEJAE4aK2 + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl exec psmdb-client-bb8b97679-gw8g4 -- bash -c 'printf '\''use config\n db.settings.save( { _id:"chunksize", value: 2 } )\n'\'' | mongo mongodb://clusterAdmin:clusterAdmin123456@cross-site-sharded-main-mongos.cross-site-sharded-600.svc.cluster.local:27017/admin ' + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.BOryKH4snN Percona Server for MongoDB shell version v4.4.29-28 connecting to: mongodb://cross-site-sharded-main-mongos.cross-site-sharded-600.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb Implicit session: session { "id" : UUID("172bfdc1-b589-4985-b690-43778dde8549") } Percona Server for MongoDB server version: v8.0.20-8 WARNING: shell and server versions do not match switched to db config WriteResult({ "nMatched" : 0, "nUpserted" : 1, "nModified" : 0, "_id" : "chunksize" }) bye + cat /tmp/tmp.sAEJAE4aK2 + rm /tmp/tmp.BOryKH4snN /tmp/tmp.sAEJAE4aK2 + return 0 + sleep 2 + desc 'write data' + set +o xtrace ----------------------------------------------------------------------------------- write data ----------------------------------------------------------------------------------- + run_script_mongos /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2287/e2e-tests/cross-site-sharded/data.js user:pass@cross-site-sharded-main-mongos.cross-site-sharded-600 + local script=/mnt/jenkins/workspace/cloud-psmdb-operator_PR-2287/e2e-tests/cross-site-sharded/data.js + local uri=user:pass@cross-site-sharded-main-mongos.cross-site-sharded-600 + local driver=mongodb + local suffix=.svc.cluster.local + local mongo_flag= + local mongo_bin=mongo ++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.jbQ7M3ODxN +++ mktemp ++ local LAST_ERR=/tmp/tmp.0hK0z2utIu ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.jbQ7M3ODxN ++ cat /tmp/tmp.0hK0z2utIu ++ rm /tmp/tmp.jbQ7M3ODxN /tmp/tmp.0hK0z2utIu ++ return 0 + local client_container=psmdb-client-bb8b97679-gw8g4 ++ basename /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2287/e2e-tests/cross-site-sharded/data.js + name=data.js + kubectl_bin cp /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2287/e2e-tests/cross-site-sharded/data.js cross-site-sharded-600/psmdb-client-bb8b97679-gw8g4:/tmp ++ mktemp + local LAST_OUT=/tmp/tmp.H6qpSOUmz9 ++ mktemp + local LAST_ERR=/tmp/tmp.XMWYNunmOf + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl cp /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2287/e2e-tests/cross-site-sharded/data.js cross-site-sharded-600/psmdb-client-bb8b97679-gw8g4:/tmp + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.H6qpSOUmz9 + cat /tmp/tmp.XMWYNunmOf + rm /tmp/tmp.H6qpSOUmz9 /tmp/tmp.XMWYNunmOf + return 0 + kubectl_bin exec psmdb-client-bb8b97679-gw8g4 -- bash -c 'mongo mongodb://user:pass@cross-site-sharded-main-mongos.cross-site-sharded-600.svc.cluster.local/admin /tmp/data.js' ++ mktemp + local LAST_OUT=/tmp/tmp.UXw92IKK60 ++ mktemp + local LAST_ERR=/tmp/tmp.vpWnZHOEpP + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl exec psmdb-client-bb8b97679-gw8g4 -- bash -c 'mongo mongodb://user:pass@cross-site-sharded-main-mongos.cross-site-sharded-600.svc.cluster.local/admin /tmp/data.js' + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.UXw92IKK60 Percona Server for MongoDB shell version v4.4.29-28 connecting to: mongodb://cross-site-sharded-main-mongos.cross-site-sharded-600.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb Implicit session: session { "id" : UUID("ee4d393e-8b41-4ea0-b7f1-57ff439cfa98") } Percona Server for MongoDB server version: v8.0.20-8 WARNING: shell and server versions do not match + cat /tmp/tmp.vpWnZHOEpP + rm /tmp/tmp.UXw92IKK60 /tmp/tmp.vpWnZHOEpP + return 0 + desc 'shard collection' + set +o xtrace ----------------------------------------------------------------------------------- shard collection ----------------------------------------------------------------------------------- + run_mongos 'sh.enableSharding("app")' clusterAdmin:clusterAdmin123456@cross-site-sharded-main-mongos.cross-site-sharded-600 + local 'command=sh.enableSharding("app")' + local uri=clusterAdmin:clusterAdmin123456@cross-site-sharded-main-mongos.cross-site-sharded-600 + local driver=mongodb + local suffix=.svc.cluster.local + local mongo_flag= + local port=27017 + local mongo_bin=mongo ++ echo .svc.cluster.local ++ awk -F: '{print $2}' + suffix_port= + [[ -z '' ]] + suffix=.svc.cluster.local:27017 ++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.vlaemaS0QU +++ mktemp ++ local LAST_ERR=/tmp/tmp.adiZR2FLvC ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.vlaemaS0QU ++ cat /tmp/tmp.adiZR2FLvC ++ rm /tmp/tmp.vlaemaS0QU /tmp/tmp.adiZR2FLvC ++ return 0 + local client_container=psmdb-client-bb8b97679-gw8g4 + kubectl_bin exec psmdb-client-bb8b97679-gw8g4 -- bash -c 'printf '\''sh.enableSharding("app")\n'\'' | mongo mongodb://clusterAdmin:clusterAdmin123456@cross-site-sharded-main-mongos.cross-site-sharded-600.svc.cluster.local:27017/admin ' ++ mktemp + local LAST_OUT=/tmp/tmp.EUFsnIjOY5 ++ mktemp + local LAST_ERR=/tmp/tmp.0N3SvzpUtX + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl exec psmdb-client-bb8b97679-gw8g4 -- bash -c 'printf '\''sh.enableSharding("app")\n'\'' | mongo mongodb://clusterAdmin:clusterAdmin123456@cross-site-sharded-main-mongos.cross-site-sharded-600.svc.cluster.local:27017/admin ' + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.EUFsnIjOY5 Percona Server for MongoDB shell version v4.4.29-28 connecting to: mongodb://cross-site-sharded-main-mongos.cross-site-sharded-600.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb Implicit session: session { "id" : UUID("104f1535-9dcf-470e-bc9a-0570ed45b4d8") } Percona Server for MongoDB server version: v8.0.20-8 WARNING: shell and server versions do not match { "ok" : 1, "$clusterTime" : { "clusterTime" : Timestamp(1775738244, 2), "signature" : { "hash" : BinData(0,"A4Es5sa3VPzzHwuKgIuzzfaCHbI="), "keyId" : NumberLong("7626736812358107150") } }, "operationTime" : Timestamp(1775738244, 2) } bye + cat /tmp/tmp.0N3SvzpUtX + rm /tmp/tmp.EUFsnIjOY5 /tmp/tmp.0N3SvzpUtX + return 0 + sleep 2 + run_mongos 'sh.shardCollection("app.city", { _id: 1 } )' clusterAdmin:clusterAdmin123456@cross-site-sharded-main-mongos.cross-site-sharded-600 + local 'command=sh.shardCollection("app.city", { _id: 1 } )' + local uri=clusterAdmin:clusterAdmin123456@cross-site-sharded-main-mongos.cross-site-sharded-600 + local driver=mongodb + local suffix=.svc.cluster.local + local mongo_flag= + local port=27017 + local mongo_bin=mongo ++ echo .svc.cluster.local ++ awk -F: '{print $2}' + suffix_port= + [[ -z '' ]] + suffix=.svc.cluster.local:27017 ++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.KKyBZ4iFCT +++ mktemp ++ local LAST_ERR=/tmp/tmp.JJpP032Tgi ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.KKyBZ4iFCT ++ cat /tmp/tmp.JJpP032Tgi ++ rm /tmp/tmp.KKyBZ4iFCT /tmp/tmp.JJpP032Tgi ++ return 0 + local client_container=psmdb-client-bb8b97679-gw8g4 + kubectl_bin exec psmdb-client-bb8b97679-gw8g4 -- bash -c 'printf '\''sh.shardCollection("app.city", { _id: 1 } )\n'\'' | mongo mongodb://clusterAdmin:clusterAdmin123456@cross-site-sharded-main-mongos.cross-site-sharded-600.svc.cluster.local:27017/admin ' ++ mktemp + local LAST_OUT=/tmp/tmp.3W1WtQDiTi ++ mktemp + local LAST_ERR=/tmp/tmp.PgGQMe4Pjy + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl exec psmdb-client-bb8b97679-gw8g4 -- bash -c 'printf '\''sh.shardCollection("app.city", { _id: 1 } )\n'\'' | mongo mongodb://clusterAdmin:clusterAdmin123456@cross-site-sharded-main-mongos.cross-site-sharded-600.svc.cluster.local:27017/admin ' + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.3W1WtQDiTi Percona Server for MongoDB shell version v4.4.29-28 connecting to: mongodb://cross-site-sharded-main-mongos.cross-site-sharded-600.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb Implicit session: session { "id" : UUID("0ffadac5-3992-4b11-9f24-5b00606771c9") } Percona Server for MongoDB server version: v8.0.20-8 WARNING: shell and server versions do not match { "collectionsharded" : "app.city", "ok" : 1, "$clusterTime" : { "clusterTime" : Timestamp(1775738250, 45), "signature" : { "hash" : BinData(0,"AFhyLV2gUjSqpuhqQq5Xts1isbg="), "keyId" : NumberLong("7626736812358107150") } }, "operationTime" : Timestamp(1775738250, 45) } bye + cat /tmp/tmp.PgGQMe4Pjy + rm /tmp/tmp.3W1WtQDiTi /tmp/tmp.PgGQMe4Pjy + return 0 + sleep 120 + desc 'Check chunks' + set +o xtrace ----------------------------------------------------------------------------------- Check chunks ----------------------------------------------------------------------------------- + chunks_param1=ns + chunks_param2='"app.city"' + [[ 8.0 != 4\.4 ]] + chunks_param1=uuid ++ run_mongos 'use app\n db.getCollectionInfos({ "name": "city" })[0].info.uuid' user:pass@cross-site-sharded-main-mongos.cross-site-sharded-600 ++ local 'command=use app\n db.getCollectionInfos({ "name": "city" })[0].info.uuid' ++ grep 'switched to db app' -A 1 ++ local uri=user:pass@cross-site-sharded-main-mongos.cross-site-sharded-600 ++ local driver=mongodb ++ local suffix=.svc.cluster.local ++ local mongo_flag= ++ local port=27017 ++ grep -v 'switched to db app' ++ local mongo_bin=mongo +++ echo .svc.cluster.local +++ awk -F: '{print $2}' ++ suffix_port= ++ [[ -z '' ]] ++ suffix=.svc.cluster.local:27017 +++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.vhVpyGG3Df ++++ mktemp +++ local LAST_ERR=/tmp/tmp.qaxJ91RdUn +++ local exit_status=0 +++ local timeout=4 ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 -a -n 1 ']' +++ break +++ cat /tmp/tmp.vhVpyGG3Df +++ cat /tmp/tmp.qaxJ91RdUn +++ rm /tmp/tmp.vhVpyGG3Df /tmp/tmp.qaxJ91RdUn +++ return 0 ++ local client_container=psmdb-client-bb8b97679-gw8g4 ++ kubectl_bin exec psmdb-client-bb8b97679-gw8g4 -- bash -c 'printf '\''use app\n db.getCollectionInfos({ "name": "city" })[0].info.uuid\n'\'' | mongo mongodb://user:pass@cross-site-sharded-main-mongos.cross-site-sharded-600.svc.cluster.local:27017/admin ' +++ mktemp ++ local LAST_OUT=/tmp/tmp.u71fk14s7i +++ mktemp ++ local LAST_ERR=/tmp/tmp.h5hyBCRYk5 ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl exec psmdb-client-bb8b97679-gw8g4 -- bash -c 'printf '\''use app\n db.getCollectionInfos({ "name": "city" })[0].info.uuid\n'\'' | mongo mongodb://user:pass@cross-site-sharded-main-mongos.cross-site-sharded-600.svc.cluster.local:27017/admin ' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.u71fk14s7i ++ cat /tmp/tmp.h5hyBCRYk5 ++ rm /tmp/tmp.u71fk14s7i /tmp/tmp.h5hyBCRYk5 ++ return 0 + chunks_param2='UUID("dfa6c42a-2689-4d1c-b87f-3f8e4f303ed7")' + shards=0 + for i in "rs0" "rs1" ++ run_mongos 'use config\n db.chunks.count({"uuid": UUID("dfa6c42a-2689-4d1c-b87f-3f8e4f303ed7"), "shard": "rs0"})' clusterAdmin:clusterAdmin123456@cross-site-sharded-main-mongos.cross-site-sharded-600 ++ local 'command=use config\n db.chunks.count({"uuid": UUID("dfa6c42a-2689-4d1c-b87f-3f8e4f303ed7"), "shard": "rs0"})' ++ local uri=clusterAdmin:clusterAdmin123456@cross-site-sharded-main-mongos.cross-site-sharded-600 ++ local driver=mongodb ++ local suffix=.svc.cluster.local ++ local mongo_flag= ++ grep 'switched to db config' -A 1 ++ local port=27017 ++ local mongo_bin=mongo ++ grep -v 'switched to db config' +++ echo .svc.cluster.local +++ awk -F: '{print $2}' ++ suffix_port= ++ [[ -z '' ]] ++ suffix=.svc.cluster.local:27017 +++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.my5w8BN5bc ++++ mktemp +++ local LAST_ERR=/tmp/tmp.ld1CodpFCK +++ local exit_status=0 +++ local timeout=4 ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 -a -n 1 ']' +++ break +++ cat /tmp/tmp.my5w8BN5bc +++ cat /tmp/tmp.ld1CodpFCK +++ rm /tmp/tmp.my5w8BN5bc /tmp/tmp.ld1CodpFCK +++ return 0 ++ local client_container=psmdb-client-bb8b97679-gw8g4 ++ kubectl_bin exec psmdb-client-bb8b97679-gw8g4 -- bash -c 'printf '\''use config\n db.chunks.count({"uuid": UUID("dfa6c42a-2689-4d1c-b87f-3f8e4f303ed7"), "shard": "rs0"})\n'\'' | mongo mongodb://clusterAdmin:clusterAdmin123456@cross-site-sharded-main-mongos.cross-site-sharded-600.svc.cluster.local:27017/admin ' +++ mktemp ++ local LAST_OUT=/tmp/tmp.u5efC5JtzR +++ mktemp ++ local LAST_ERR=/tmp/tmp.RPcKIfv380 ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl exec psmdb-client-bb8b97679-gw8g4 -- bash -c 'printf '\''use config\n db.chunks.count({"uuid": UUID("dfa6c42a-2689-4d1c-b87f-3f8e4f303ed7"), "shard": "rs0"})\n'\'' | mongo mongodb://clusterAdmin:clusterAdmin123456@cross-site-sharded-main-mongos.cross-site-sharded-600.svc.cluster.local:27017/admin ' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.u5efC5JtzR ++ cat /tmp/tmp.RPcKIfv380 ++ rm /tmp/tmp.u5efC5JtzR /tmp/tmp.RPcKIfv380 ++ return 0 + out=3 + desc 'rs0 has 3 chunks' + set +o xtrace ----------------------------------------------------------------------------------- rs0 has 3 chunks ----------------------------------------------------------------------------------- + [[ 3 -ne 0 ]] + (( shards = shards + 1 )) + for i in "rs0" "rs1" ++ run_mongos 'use config\n db.chunks.count({"uuid": UUID("dfa6c42a-2689-4d1c-b87f-3f8e4f303ed7"), "shard": "rs1"})' clusterAdmin:clusterAdmin123456@cross-site-sharded-main-mongos.cross-site-sharded-600 ++ local 'command=use config\n db.chunks.count({"uuid": UUID("dfa6c42a-2689-4d1c-b87f-3f8e4f303ed7"), "shard": "rs1"})' ++ local uri=clusterAdmin:clusterAdmin123456@cross-site-sharded-main-mongos.cross-site-sharded-600 ++ local driver=mongodb ++ local suffix=.svc.cluster.local ++ local mongo_flag= ++ local port=27017 ++ local mongo_bin=mongo ++ grep 'switched to db config' -A 1 ++ grep -v 'switched to db config' +++ echo .svc.cluster.local +++ awk -F: '{print $2}' ++ suffix_port= ++ [[ -z '' ]] ++ suffix=.svc.cluster.local:27017 +++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.hUHcRADNrt ++++ mktemp +++ local LAST_ERR=/tmp/tmp.4hDDFkxjV6 +++ local exit_status=0 +++ local timeout=4 ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 -a -n 1 ']' +++ break +++ cat /tmp/tmp.hUHcRADNrt +++ cat /tmp/tmp.4hDDFkxjV6 +++ rm /tmp/tmp.hUHcRADNrt /tmp/tmp.4hDDFkxjV6 +++ return 0 ++ local client_container=psmdb-client-bb8b97679-gw8g4 ++ kubectl_bin exec psmdb-client-bb8b97679-gw8g4 -- bash -c 'printf '\''use config\n db.chunks.count({"uuid": UUID("dfa6c42a-2689-4d1c-b87f-3f8e4f303ed7"), "shard": "rs1"})\n'\'' | mongo mongodb://clusterAdmin:clusterAdmin123456@cross-site-sharded-main-mongos.cross-site-sharded-600.svc.cluster.local:27017/admin ' +++ mktemp ++ local LAST_OUT=/tmp/tmp.EkAU1tGVnJ +++ mktemp ++ local LAST_ERR=/tmp/tmp.UhQDuI4MRX ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl exec psmdb-client-bb8b97679-gw8g4 -- bash -c 'printf '\''use config\n db.chunks.count({"uuid": UUID("dfa6c42a-2689-4d1c-b87f-3f8e4f303ed7"), "shard": "rs1"})\n'\'' | mongo mongodb://clusterAdmin:clusterAdmin123456@cross-site-sharded-main-mongos.cross-site-sharded-600.svc.cluster.local:27017/admin ' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.EkAU1tGVnJ ++ cat /tmp/tmp.UhQDuI4MRX ++ rm /tmp/tmp.EkAU1tGVnJ /tmp/tmp.UhQDuI4MRX ++ return 0 + out=1 + desc 'rs1 has 1 chunks' + set +o xtrace ----------------------------------------------------------------------------------- rs1 has 1 chunks ----------------------------------------------------------------------------------- + [[ 1 -ne 0 ]] + (( shards = shards + 1 )) + [[ 2 -lt 2 ]] + desc 'create replica cluster' + set +o xtrace ----------------------------------------------------------------------------------- create replica cluster ----------------------------------------------------------------------------------- + create_namespace cross-site-sharded-replica-2372 0 + local namespace=cross-site-sharded-replica-2372 + local skip_clean_namespace=0 + [[ 1 == 1 ]] + [[ -z 0 ]] + '[' -n '' ']' + desc 'cleaned up old namespaces cross-site-sharded-replica-2372' + set +o xtrace ----------------------------------------------------------------------------------- cleaned up old namespaces cross-site-sharded-replica-2372 ----------------------------------------------------------------------------------- + kubectl_bin delete namespace cross-site-sharded-replica-2372 --ignore-not-found ++ mktemp + local LAST_OUT=/tmp/tmp.Af6BN78G4G ++ mktemp + local LAST_ERR=/tmp/tmp.ctKroIM3BZ + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl delete namespace cross-site-sharded-replica-2372 --ignore-not-found + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.Af6BN78G4G + cat /tmp/tmp.ctKroIM3BZ + rm /tmp/tmp.Af6BN78G4G /tmp/tmp.ctKroIM3BZ + return 0 + kubectl_bin wait --for=delete namespace cross-site-sharded-replica-2372 ++ mktemp + local LAST_OUT=/tmp/tmp.I4hK6WNjhB ++ mktemp + local LAST_ERR=/tmp/tmp.gTesU7BtSJ + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl wait --for=delete namespace cross-site-sharded-replica-2372 + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.I4hK6WNjhB + cat /tmp/tmp.gTesU7BtSJ + rm /tmp/tmp.I4hK6WNjhB /tmp/tmp.gTesU7BtSJ + return 0 + desc 'create namespace cross-site-sharded-replica-2372' + set +o xtrace ----------------------------------------------------------------------------------- create namespace cross-site-sharded-replica-2372 ----------------------------------------------------------------------------------- + kubectl_bin create namespace cross-site-sharded-replica-2372 ++ mktemp + local LAST_OUT=/tmp/tmp.LomJK9gDNd ++ mktemp + local LAST_ERR=/tmp/tmp.AMbuUgPYIA + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl create namespace cross-site-sharded-replica-2372 + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.LomJK9gDNd namespace/cross-site-sharded-replica-2372 created + cat /tmp/tmp.AMbuUgPYIA + rm /tmp/tmp.LomJK9gDNd /tmp/tmp.AMbuUgPYIA + return 0 + set_kube_ctx cross-site-sharded-replica-2372 + local namespace=cross-site-sharded-replica-2372 ++ kubectl_bin config current-context +++ mktemp ++ local LAST_OUT=/tmp/tmp.mpnlXhjYBB +++ mktemp ++ local LAST_ERR=/tmp/tmp.Np7DetV3KP ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl config current-context ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.mpnlXhjYBB ++ cat /tmp/tmp.Np7DetV3KP ++ rm /tmp/tmp.mpnlXhjYBB /tmp/tmp.Np7DetV3KP ++ return 0 + kubectl_bin config set-context gke_cloud-dev-112233_us-central1-a_jen-psmdb-2287-fc474e65-5-cluster3 --namespace=cross-site-sharded-replica-2372 ++ mktemp + local LAST_OUT=/tmp/tmp.bSdXHDubpZ ++ mktemp + local LAST_ERR=/tmp/tmp.sn6E1d1xK3 + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl config set-context gke_cloud-dev-112233_us-central1-a_jen-psmdb-2287-fc474e65-5-cluster3 --namespace=cross-site-sharded-replica-2372 + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.bSdXHDubpZ Context "gke_cloud-dev-112233_us-central1-a_jen-psmdb-2287-fc474e65-5-cluster3" modified. + cat /tmp/tmp.sn6E1d1xK3 + rm /tmp/tmp.bSdXHDubpZ /tmp/tmp.sn6E1d1xK3 + return 0 + deploy_operator + desc 'start PSMDB operator: docker.io/perconalab/percona-server-mongodb-operator:PR-2287-fc474e65' + set +o xtrace ----------------------------------------------------------------------------------- start PSMDB operator: docker.io/perconalab/percona-server-mongodb-operator:PR-2287-fc474e65 ----------------------------------------------------------------------------------- + local cr_file + '[' -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2287/e2e-tests/cross-site-sharded/conf/crd.yaml ']' + cr_file=/mnt/jenkins/workspace/cloud-psmdb-operator_PR-2287/deploy/crd.yaml + kubectl_bin apply --server-side --force-conflicts -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2287/deploy/crd.yaml ++ mktemp + local LAST_OUT=/tmp/tmp.dIWRmKJmIS ++ mktemp + local LAST_ERR=/tmp/tmp.ZZAJP1VO0g + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl apply --server-side --force-conflicts -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2287/deploy/crd.yaml + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.dIWRmKJmIS customresourcedefinition.apiextensions.k8s.io/perconaservermongodbbackups.psmdb.percona.com serverside-applied customresourcedefinition.apiextensions.k8s.io/perconaservermongodbrestores.psmdb.percona.com serverside-applied customresourcedefinition.apiextensions.k8s.io/perconaservermongodbs.psmdb.percona.com serverside-applied + cat /tmp/tmp.ZZAJP1VO0g + rm /tmp/tmp.dIWRmKJmIS /tmp/tmp.ZZAJP1VO0g + return 0 + '[' -n '' ']' + apply_rbac rbac + local operator_namespace=psmdb-operator + local rbac=rbac + cat /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2287/deploy/rbac.yaml + sed -e 's^namespace: .*^namespace: psmdb-operator^' + kubectl_bin apply -f - ++ mktemp + local LAST_OUT=/tmp/tmp.l8pmDBbJxY ++ mktemp + local LAST_ERR=/tmp/tmp.rcxCWKO1Gn + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.l8pmDBbJxY role.rbac.authorization.k8s.io/percona-server-mongodb-operator created serviceaccount/percona-server-mongodb-operator created rolebinding.rbac.authorization.k8s.io/service-account-percona-server-mongodb-operator created + cat /tmp/tmp.rcxCWKO1Gn + rm /tmp/tmp.l8pmDBbJxY /tmp/tmp.rcxCWKO1Gn + return 0 + yq eval $'\n\t\t\t(.spec.template.spec.containers[].image = "docker.io/perconalab/percona-server-mongodb-operator:PR-2287-fc474e65") |\n\t\t\t((.. | select(.[] == "DISABLE_TELEMETRY")) |= .value="true") |\n\t\t\t((.. | select(.[] == "LOG_LEVEL")) |= .value="DEBUG")' /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2287/deploy/operator.yaml + kubectl_bin apply -f - ++ mktemp + local LAST_OUT=/tmp/tmp.uQaIcjc5ho ++ mktemp + local LAST_ERR=/tmp/tmp.463xMYpeN3 + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.uQaIcjc5ho deployment.apps/percona-server-mongodb-operator created + cat /tmp/tmp.463xMYpeN3 + rm /tmp/tmp.uQaIcjc5ho /tmp/tmp.463xMYpeN3 + return 0 + sleep 20 ++ get_operator_pod ++ kubectl_bin get pods --selector=name=percona-server-mongodb-operator -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.WVAGqGxTT2 +++ mktemp ++ local LAST_ERR=/tmp/tmp.9YbSIzdJTL ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pods --selector=name=percona-server-mongodb-operator -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.WVAGqGxTT2 ++ cat /tmp/tmp.9YbSIzdJTL ++ rm /tmp/tmp.WVAGqGxTT2 /tmp/tmp.9YbSIzdJTL ++ return 0 + wait_operator_pod percona-server-mongodb-operator-669854d959-hv64l + local pod=percona-server-mongodb-operator-669854d959-hv64l + set +o xtrace waiting for pod/percona-server-mongodb-operator-669854d959-hv64l to be ready.OK + echo 'Print operator info from log' Print operator info from log + grep 'Manager starting up' ++ get_operator_pod ++ kubectl_bin get pods --selector=name=percona-server-mongodb-operator -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.E9ZIcGgmGz +++ mktemp ++ local LAST_ERR=/tmp/tmp.3Mb3lbrLXI ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pods --selector=name=percona-server-mongodb-operator -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.E9ZIcGgmGz ++ cat /tmp/tmp.3Mb3lbrLXI ++ rm /tmp/tmp.E9ZIcGgmGz /tmp/tmp.3Mb3lbrLXI ++ return 0 + kubectl_bin logs percona-server-mongodb-operator-669854d959-hv64l ++ mktemp + local LAST_OUT=/tmp/tmp.jMcyqpU24u ++ mktemp + local LAST_ERR=/tmp/tmp.mD8XpaRlwu + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl logs percona-server-mongodb-operator-669854d959-hv64l + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.jMcyqpU24u + cat /tmp/tmp.mD8XpaRlwu + rm /tmp/tmp.jMcyqpU24u /tmp/tmp.mD8XpaRlwu + return 0 2026-04-09T12:39:55.794Z INFO setup Manager starting up {"gitCommit": "fc474e65109342c13c34c3ce7546452f0d6ad19d", "gitBranch": "PR-2287-fc474e65", "buildTime": "", "goVersion": "go1.25.9", "os": "linux", "arch": "amd64"} + desc 'start client' + set +o xtrace ----------------------------------------------------------------------------------- start client ----------------------------------------------------------------------------------- + kubectl_bin apply -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2287/e2e-tests/conf/client.yml ++ mktemp + local LAST_OUT=/tmp/tmp.65g0Iyzbz7 ++ mktemp + local LAST_ERR=/tmp/tmp.hTeILc0pH3 + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl apply -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2287/e2e-tests/conf/client.yml + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.65g0Iyzbz7 deployment.apps/psmdb-client created + cat /tmp/tmp.hTeILc0pH3 + rm /tmp/tmp.65g0Iyzbz7 /tmp/tmp.hTeILc0pH3 + return 0 + desc 'copy secrets from main to replica namespace and create all of them' + set +o xtrace ----------------------------------------------------------------------------------- copy secrets from main to replica namespace and create all of them ----------------------------------------------------------------------------------- + kubectl get secret cross-site-sharded-main-secrets -o yaml -n cross-site-sharded-600 + yq eval $'\n\t\tdel(.metadata) |\n\t\t(.metadata.name = "cross-site-sharded-replica-secrets")' - + kubectl_bin apply -f - ++ mktemp + local LAST_OUT=/tmp/tmp.HD6K9e9Yy5 ++ mktemp + local LAST_ERR=/tmp/tmp.GLrl4jtD6D + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.HD6K9e9Yy5 secret/cross-site-sharded-replica-secrets created + cat /tmp/tmp.GLrl4jtD6D + rm /tmp/tmp.HD6K9e9Yy5 /tmp/tmp.GLrl4jtD6D + return 0 + kubectl_bin get secret cross-site-sharded-main-ssl-internal -o yaml -n cross-site-sharded-600 + yq eval $'\n\t\tdel(.metadata) |\n\t\tdel(.status) |\n\t\t(.metadata.name = "cross-site-sharded-replica-ssl-internal")' - + kubectl_bin apply -f - ++ mktemp ++ mktemp + local LAST_OUT=/tmp/tmp.TqGUDzApl4 ++ mktemp + local LAST_OUT=/tmp/tmp.85mtJyRwFP ++ mktemp + local LAST_ERR=/tmp/tmp.6Tvifh35EX + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl get secret cross-site-sharded-main-ssl-internal -o yaml -n cross-site-sharded-600 + local LAST_ERR=/tmp/tmp.DNeSPxk4wD + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.TqGUDzApl4 + cat /tmp/tmp.6Tvifh35EX + rm /tmp/tmp.TqGUDzApl4 /tmp/tmp.6Tvifh35EX + return 0 + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.85mtJyRwFP secret/cross-site-sharded-replica-ssl-internal created + cat /tmp/tmp.DNeSPxk4wD + rm /tmp/tmp.85mtJyRwFP /tmp/tmp.DNeSPxk4wD + return 0 + kubectl_bin get secret cross-site-sharded-main-ssl -o yaml -n cross-site-sharded-600 + yq eval $'\n\t\tdel(.metadata) |\n\t\tdel(.status) |\n\t\t(.metadata.name = "cross-site-sharded-replica-ssl")' - + kubectl_bin apply -f - ++ mktemp ++ mktemp + local LAST_OUT=/tmp/tmp.ItkUldruMW + local LAST_OUT=/tmp/tmp.riq7xB7RQU ++ mktemp ++ mktemp + local LAST_ERR=/tmp/tmp.ucQkg7B6pM + local exit_status=0 + local timeout=4 + local LAST_ERR=/tmp/tmp.LTlmAzELuI + local exit_status=0 + local timeout=4 ++ seq 0 2 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl get secret cross-site-sharded-main-ssl -o yaml -n cross-site-sharded-600 + for i in $(seq 0 2) + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.ItkUldruMW + cat /tmp/tmp.ucQkg7B6pM + rm /tmp/tmp.ItkUldruMW /tmp/tmp.ucQkg7B6pM + return 0 + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.riq7xB7RQU secret/cross-site-sharded-replica-ssl created + cat /tmp/tmp.LTlmAzELuI + rm /tmp/tmp.riq7xB7RQU /tmp/tmp.LTlmAzELuI + return 0 + sleep 30 + desc 'create replica PSMDB cluster ' + set +o xtrace ----------------------------------------------------------------------------------- create replica PSMDB cluster ----------------------------------------------------------------------------------- + apply_cluster /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2287/e2e-tests/cross-site-sharded/conf/cross-site-sharded-replica.yml + '[' -z '' ']' + cat_config /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2287/e2e-tests/cross-site-sharded/conf/cross-site-sharded-replica.yml + kubectl_bin apply -f - + cat /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2287/e2e-tests/cross-site-sharded/conf/cross-site-sharded-replica.yml ++ mktemp + yq eval '(.spec | select(.image == null)).image = "docker.io/perconalab/percona-server-mongodb-operator:main-mongod8.0"' + yq eval '(.spec | select(has("pmm"))).pmm.image = "docker.io/percona/pmm-client:2.44.1-1"' + yq eval '(.spec | select(has("initImage"))).initImage = "docker.io/perconalab/percona-server-mongodb-operator:PR-2287-fc474e65"' + yq eval '(.spec | select(has("backup"))).backup.image = "docker.io/perconalab/percona-server-mongodb-operator:main-backup"' + /usr/sbin/sed -e s/NAME_SPACE/cross-site-sharded-600/g + local LAST_OUT=/tmp/tmp.Q9BBk9Y77E ++ mktemp + yq eval '.spec.upgradeOptions.apply="Never"' + local LAST_ERR=/tmp/tmp.2lZKtEQ4ZF + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.Q9BBk9Y77E perconaservermongodb.psmdb.percona.com/cross-site-sharded-replica created + cat /tmp/tmp.2lZKtEQ4ZF + rm /tmp/tmp.Q9BBk9Y77E /tmp/tmp.2lZKtEQ4ZF + return 0 + wait_for_running cross-site-sharded-replica-rs0 3 false + local name=cross-site-sharded-replica-rs0 + let last_pod=2 + local check_cluster_readyness=false + set_debug + [[ 1 == 1 ]] + set -o xtrace + local rs_name=rs0 + local cluster_name=cross-site-sharded-replica ++ seq 0 2 + for i in $(seq 0 $last_pod) + [[ 0 -eq 2 ]] + wait_pod cross-site-sharded-replica-rs0-0 + local pod=cross-site-sharded-replica-rs0-0 + set +o xtrace waiting for pod/cross-site-sharded-replica-rs0-0 to be ready.............OK + for i in $(seq 0 $last_pod) + [[ 1 -eq 2 ]] + wait_pod cross-site-sharded-replica-rs0-1 + local pod=cross-site-sharded-replica-rs0-1 + set +o xtrace waiting for pod/cross-site-sharded-replica-rs0-1 to be ready..............OK + for i in $(seq 0 $last_pod) + [[ 2 -eq 2 ]] ++ kubectl_bin get psmdb cross-site-sharded-replica -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].arbiter.enabled}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.PFuVDRqqWV +++ mktemp ++ local LAST_ERR=/tmp/tmp.EBxAI9uxmv ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb cross-site-sharded-replica -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].arbiter.enabled}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.PFuVDRqqWV ++ cat /tmp/tmp.EBxAI9uxmv ++ rm /tmp/tmp.PFuVDRqqWV /tmp/tmp.EBxAI9uxmv ++ return 0 + [[ false == true ]] + wait_pod cross-site-sharded-replica-rs0-2 + local pod=cross-site-sharded-replica-rs0-2 + set +o xtrace waiting for pod/cross-site-sharded-replica-rs0-2 to be ready.............OK ++ kubectl_bin get psmdb cross-site-sharded-replica -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].nonvoting.enabled}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.JoNK0eQydv +++ mktemp ++ local LAST_ERR=/tmp/tmp.w4SfTaHwA1 ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb cross-site-sharded-replica -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].nonvoting.enabled}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.JoNK0eQydv ++ cat /tmp/tmp.w4SfTaHwA1 ++ rm /tmp/tmp.JoNK0eQydv /tmp/tmp.w4SfTaHwA1 ++ return 0 + [[ false == true ]] ++ kubectl_bin get psmdb cross-site-sharded-replica -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].hidden.enabled}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.Xy4ZYvHOxW +++ mktemp ++ local LAST_ERR=/tmp/tmp.058QgP5SLZ ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb cross-site-sharded-replica -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].hidden.enabled}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.Xy4ZYvHOxW ++ cat /tmp/tmp.058QgP5SLZ ++ rm /tmp/tmp.Xy4ZYvHOxW /tmp/tmp.058QgP5SLZ ++ return 0 + [[ '' == true ]] + sleep 10 + [[ false == true ]] + wait_for_running cross-site-sharded-replica-rs1 3 false + local name=cross-site-sharded-replica-rs1 + let last_pod=2 + local check_cluster_readyness=false + set_debug + [[ 1 == 1 ]] + set -o xtrace + local rs_name=rs1 + local cluster_name=cross-site-sharded-replica ++ seq 0 2 + for i in $(seq 0 $last_pod) + [[ 0 -eq 2 ]] + wait_pod cross-site-sharded-replica-rs1-0 + local pod=cross-site-sharded-replica-rs1-0 + set +o xtrace waiting for pod/cross-site-sharded-replica-rs1-0 to be ready.OK + for i in $(seq 0 $last_pod) + [[ 1 -eq 2 ]] + wait_pod cross-site-sharded-replica-rs1-1 + local pod=cross-site-sharded-replica-rs1-1 + set +o xtrace waiting for pod/cross-site-sharded-replica-rs1-1 to be ready.OK + for i in $(seq 0 $last_pod) + [[ 2 -eq 2 ]] ++ kubectl_bin get psmdb cross-site-sharded-replica -o 'jsonpath={.spec.replsets[?(@.name=="rs1")].arbiter.enabled}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.iy0ftjpjp9 +++ mktemp ++ local LAST_ERR=/tmp/tmp.UIcHn9ABFu ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb cross-site-sharded-replica -o 'jsonpath={.spec.replsets[?(@.name=="rs1")].arbiter.enabled}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.iy0ftjpjp9 ++ cat /tmp/tmp.UIcHn9ABFu ++ rm /tmp/tmp.iy0ftjpjp9 /tmp/tmp.UIcHn9ABFu ++ return 0 + [[ false == true ]] + wait_pod cross-site-sharded-replica-rs1-2 + local pod=cross-site-sharded-replica-rs1-2 + set +o xtrace waiting for pod/cross-site-sharded-replica-rs1-2 to be ready.OK ++ kubectl_bin get psmdb cross-site-sharded-replica -o 'jsonpath={.spec.replsets[?(@.name=="rs1")].nonvoting.enabled}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.phzrpCsmbG +++ mktemp ++ local LAST_ERR=/tmp/tmp.OwWGrgdQbc ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb cross-site-sharded-replica -o 'jsonpath={.spec.replsets[?(@.name=="rs1")].nonvoting.enabled}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.phzrpCsmbG ++ cat /tmp/tmp.OwWGrgdQbc ++ rm /tmp/tmp.phzrpCsmbG /tmp/tmp.OwWGrgdQbc ++ return 0 + [[ false == true ]] ++ kubectl_bin get psmdb cross-site-sharded-replica -o 'jsonpath={.spec.replsets[?(@.name=="rs1")].hidden.enabled}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.n5bUj3ecr3 +++ mktemp ++ local LAST_ERR=/tmp/tmp.jMVUNJ6EvA ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb cross-site-sharded-replica -o 'jsonpath={.spec.replsets[?(@.name=="rs1")].hidden.enabled}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.n5bUj3ecr3 ++ cat /tmp/tmp.jMVUNJ6EvA ++ rm /tmp/tmp.n5bUj3ecr3 /tmp/tmp.jMVUNJ6EvA ++ return 0 + [[ '' == true ]] + sleep 10 + [[ false == true ]] + wait_for_running cross-site-sharded-replica-cfg 3 false + local name=cross-site-sharded-replica-cfg + let last_pod=2 + local check_cluster_readyness=false + set_debug + [[ 1 == 1 ]] + set -o xtrace + local rs_name=cfg + local cluster_name=cross-site-sharded-replica ++ seq 0 2 + for i in $(seq 0 $last_pod) + [[ 0 -eq 2 ]] + wait_pod cross-site-sharded-replica-cfg-0 + local pod=cross-site-sharded-replica-cfg-0 + set +o xtrace waiting for pod/cross-site-sharded-replica-cfg-0 to be ready.OK + for i in $(seq 0 $last_pod) + [[ 1 -eq 2 ]] + wait_pod cross-site-sharded-replica-cfg-1 + local pod=cross-site-sharded-replica-cfg-1 + set +o xtrace waiting for pod/cross-site-sharded-replica-cfg-1 to be ready.OK + for i in $(seq 0 $last_pod) + [[ 2 -eq 2 ]] ++ kubectl_bin get psmdb cross-site-sharded-replica -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].arbiter.enabled}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.Tar7vsKWBy +++ mktemp ++ local LAST_ERR=/tmp/tmp.YfKKUbaTL3 ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb cross-site-sharded-replica -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].arbiter.enabled}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.Tar7vsKWBy ++ cat /tmp/tmp.YfKKUbaTL3 ++ rm /tmp/tmp.Tar7vsKWBy /tmp/tmp.YfKKUbaTL3 ++ return 0 + [[ '' == true ]] + wait_pod cross-site-sharded-replica-cfg-2 + local pod=cross-site-sharded-replica-cfg-2 + set +o xtrace waiting for pod/cross-site-sharded-replica-cfg-2 to be ready.OK ++ kubectl_bin get psmdb cross-site-sharded-replica -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].nonvoting.enabled}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.2pim8CGRVu +++ mktemp ++ local LAST_ERR=/tmp/tmp.ljNxRPv8mj ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb cross-site-sharded-replica -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].nonvoting.enabled}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.2pim8CGRVu ++ cat /tmp/tmp.ljNxRPv8mj ++ rm /tmp/tmp.2pim8CGRVu /tmp/tmp.ljNxRPv8mj ++ return 0 + [[ '' == true ]] ++ kubectl_bin get psmdb cross-site-sharded-replica -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].hidden.enabled}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.Wo075O3kFC +++ mktemp ++ local LAST_ERR=/tmp/tmp.o1BPj3pptS ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb cross-site-sharded-replica -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].hidden.enabled}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.Wo075O3kFC ++ cat /tmp/tmp.o1BPj3pptS ++ rm /tmp/tmp.Wo075O3kFC /tmp/tmp.o1BPj3pptS ++ return 0 + [[ '' == true ]] + sleep 10 + [[ false == true ]] ++ get_service_ip cross-site-sharded-replica-cfg-0 cfg ++ local service=cross-site-sharded-replica-cfg-0 ++ local server_type=cfg +++ kubectl_bin get psmdb/cross-site-sharded-replica -o 'jsonpath={.spec.replsets[].expose.enabled}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.qfLWCqvlCn ++++ mktemp +++ local LAST_ERR=/tmp/tmp.0wZDyNYUov +++ local exit_status=0 +++ local timeout=4 ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl get psmdb/cross-site-sharded-replica -o 'jsonpath={.spec.replsets[].expose.enabled}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 -a -n 1 ']' +++ break +++ cat /tmp/tmp.qfLWCqvlCn +++ cat /tmp/tmp.0wZDyNYUov +++ rm /tmp/tmp.qfLWCqvlCn /tmp/tmp.0wZDyNYUov +++ return 0 ++ '[' true '!=' true ']' ++ kubectl_bin get service/cross-site-sharded-replica-cfg-0 -o 'jsonpath={.spec.type}' ++ grep -q NotFound +++ kubectl_bin get service/cross-site-sharded-replica-cfg-0 -o 'jsonpath={.spec.type}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.nGlUtrWGZO ++++ mktemp +++ local LAST_ERR=/tmp/tmp.LIAJzb38Ni +++ local exit_status=0 +++ local timeout=4 ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl get service/cross-site-sharded-replica-cfg-0 -o 'jsonpath={.spec.type}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 -a -n 1 ']' +++ break +++ cat /tmp/tmp.nGlUtrWGZO +++ cat /tmp/tmp.LIAJzb38Ni +++ rm /tmp/tmp.nGlUtrWGZO /tmp/tmp.LIAJzb38Ni +++ return 0 ++ service_type=ClusterIP ++ '[' ClusterIP = ClusterIP ']' ++ kubectl_bin get service/cross-site-sharded-replica-cfg-0 -o 'jsonpath={.spec.clusterIP}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.3OSNlhX8Bq +++ mktemp ++ local LAST_ERR=/tmp/tmp.4avR7ZMnsh ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get service/cross-site-sharded-replica-cfg-0 -o 'jsonpath={.spec.clusterIP}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.3OSNlhX8Bq ++ cat /tmp/tmp.4avR7ZMnsh ++ rm /tmp/tmp.3OSNlhX8Bq /tmp/tmp.4avR7ZMnsh ++ return 0 ++ return + replica_cfg_0_endpoint=34.118.226.232 ++ get_service_ip cross-site-sharded-replica-cfg-1 cfg ++ local service=cross-site-sharded-replica-cfg-1 ++ local server_type=cfg +++ kubectl_bin get psmdb/cross-site-sharded-replica -o 'jsonpath={.spec.replsets[].expose.enabled}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.a5ivv6a7Ha ++++ mktemp +++ local LAST_ERR=/tmp/tmp.4LY5tSklcZ +++ local exit_status=0 +++ local timeout=4 ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl get psmdb/cross-site-sharded-replica -o 'jsonpath={.spec.replsets[].expose.enabled}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 -a -n 1 ']' +++ break +++ cat /tmp/tmp.a5ivv6a7Ha +++ cat /tmp/tmp.4LY5tSklcZ +++ rm /tmp/tmp.a5ivv6a7Ha /tmp/tmp.4LY5tSklcZ +++ return 0 ++ '[' true '!=' true ']' ++ kubectl_bin get service/cross-site-sharded-replica-cfg-1 -o 'jsonpath={.spec.type}' ++ grep -q NotFound +++ kubectl_bin get service/cross-site-sharded-replica-cfg-1 -o 'jsonpath={.spec.type}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.ULj0yj5P75 ++++ mktemp +++ local LAST_ERR=/tmp/tmp.xTeXmdtp9u +++ local exit_status=0 +++ local timeout=4 ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl get service/cross-site-sharded-replica-cfg-1 -o 'jsonpath={.spec.type}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 -a -n 1 ']' +++ break +++ cat /tmp/tmp.ULj0yj5P75 +++ cat /tmp/tmp.xTeXmdtp9u +++ rm /tmp/tmp.ULj0yj5P75 /tmp/tmp.xTeXmdtp9u +++ return 0 ++ service_type=ClusterIP ++ '[' ClusterIP = ClusterIP ']' ++ kubectl_bin get service/cross-site-sharded-replica-cfg-1 -o 'jsonpath={.spec.clusterIP}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.1OQQFH0lqK +++ mktemp ++ local LAST_ERR=/tmp/tmp.BNCQvl0djg ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get service/cross-site-sharded-replica-cfg-1 -o 'jsonpath={.spec.clusterIP}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.1OQQFH0lqK ++ cat /tmp/tmp.BNCQvl0djg ++ rm /tmp/tmp.1OQQFH0lqK /tmp/tmp.BNCQvl0djg ++ return 0 ++ return + replica_cfg_1_endpoint=34.118.225.23 ++ get_service_ip cross-site-sharded-replica-cfg-2 cfg ++ local service=cross-site-sharded-replica-cfg-2 ++ local server_type=cfg +++ kubectl_bin get psmdb/cross-site-sharded-replica -o 'jsonpath={.spec.replsets[].expose.enabled}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.x6ghiE6RkT ++++ mktemp +++ local LAST_ERR=/tmp/tmp.nU7A6ADT3n +++ local exit_status=0 +++ local timeout=4 ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl get psmdb/cross-site-sharded-replica -o 'jsonpath={.spec.replsets[].expose.enabled}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 -a -n 1 ']' +++ break +++ cat /tmp/tmp.x6ghiE6RkT +++ cat /tmp/tmp.nU7A6ADT3n +++ rm /tmp/tmp.x6ghiE6RkT /tmp/tmp.nU7A6ADT3n +++ return 0 ++ '[' true '!=' true ']' ++ kubectl_bin get service/cross-site-sharded-replica-cfg-2 -o 'jsonpath={.spec.type}' ++ grep -q NotFound +++ kubectl_bin get service/cross-site-sharded-replica-cfg-2 -o 'jsonpath={.spec.type}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.ovmVtLIpp2 ++++ mktemp +++ local LAST_ERR=/tmp/tmp.97wRlVu5Xc +++ local exit_status=0 +++ local timeout=4 ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl get service/cross-site-sharded-replica-cfg-2 -o 'jsonpath={.spec.type}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 -a -n 1 ']' +++ break +++ cat /tmp/tmp.ovmVtLIpp2 +++ cat /tmp/tmp.97wRlVu5Xc +++ rm /tmp/tmp.ovmVtLIpp2 /tmp/tmp.97wRlVu5Xc +++ return 0 ++ service_type=ClusterIP ++ '[' ClusterIP = ClusterIP ']' ++ kubectl_bin get service/cross-site-sharded-replica-cfg-2 -o 'jsonpath={.spec.clusterIP}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.2JANQDlbH8 +++ mktemp ++ local LAST_ERR=/tmp/tmp.NFDhsiuYTG ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get service/cross-site-sharded-replica-cfg-2 -o 'jsonpath={.spec.clusterIP}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.2JANQDlbH8 ++ cat /tmp/tmp.NFDhsiuYTG ++ rm /tmp/tmp.2JANQDlbH8 /tmp/tmp.NFDhsiuYTG ++ return 0 ++ return + replica_cfg_2_endpoint=34.118.226.28 ++ get_service_ip cross-site-sharded-replica-rs0-0 ++ local service=cross-site-sharded-replica-rs0-0 ++ local server_type=rs0 +++ kubectl_bin get psmdb/cross-site-sharded-replica -o 'jsonpath={.spec.replsets[].expose.enabled}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.cibCBzLs4O ++++ mktemp +++ local LAST_ERR=/tmp/tmp.TK5MRMig89 +++ local exit_status=0 +++ local timeout=4 ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl get psmdb/cross-site-sharded-replica -o 'jsonpath={.spec.replsets[].expose.enabled}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 -a -n 1 ']' +++ break +++ cat /tmp/tmp.cibCBzLs4O +++ cat /tmp/tmp.TK5MRMig89 +++ rm /tmp/tmp.cibCBzLs4O /tmp/tmp.TK5MRMig89 +++ return 0 ++ '[' true '!=' true ']' ++ kubectl_bin get service/cross-site-sharded-replica-rs0-0 -o 'jsonpath={.spec.type}' ++ grep -q NotFound +++ kubectl_bin get service/cross-site-sharded-replica-rs0-0 -o 'jsonpath={.spec.type}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.4iREnRx86j ++++ mktemp +++ local LAST_ERR=/tmp/tmp.jmuhR8QaFo +++ local exit_status=0 +++ local timeout=4 ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl get service/cross-site-sharded-replica-rs0-0 -o 'jsonpath={.spec.type}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 -a -n 1 ']' +++ break +++ cat /tmp/tmp.4iREnRx86j +++ cat /tmp/tmp.jmuhR8QaFo +++ rm /tmp/tmp.4iREnRx86j /tmp/tmp.jmuhR8QaFo +++ return 0 ++ service_type=ClusterIP ++ '[' ClusterIP = ClusterIP ']' ++ kubectl_bin get service/cross-site-sharded-replica-rs0-0 -o 'jsonpath={.spec.clusterIP}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.BVVNFVcapL +++ mktemp ++ local LAST_ERR=/tmp/tmp.e0NLyeYk8x ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get service/cross-site-sharded-replica-rs0-0 -o 'jsonpath={.spec.clusterIP}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.BVVNFVcapL ++ cat /tmp/tmp.e0NLyeYk8x ++ rm /tmp/tmp.BVVNFVcapL /tmp/tmp.e0NLyeYk8x ++ return 0 ++ return + replica_rs0_0_endpoint=34.118.226.208 ++ get_service_ip cross-site-sharded-replica-rs0-1 ++ local service=cross-site-sharded-replica-rs0-1 ++ local server_type=rs0 +++ kubectl_bin get psmdb/cross-site-sharded-replica -o 'jsonpath={.spec.replsets[].expose.enabled}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.DBeWtKePSt ++++ mktemp +++ local LAST_ERR=/tmp/tmp.Ewt6QCKnaj +++ local exit_status=0 +++ local timeout=4 ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl get psmdb/cross-site-sharded-replica -o 'jsonpath={.spec.replsets[].expose.enabled}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 -a -n 1 ']' +++ break +++ cat /tmp/tmp.DBeWtKePSt +++ cat /tmp/tmp.Ewt6QCKnaj +++ rm /tmp/tmp.DBeWtKePSt /tmp/tmp.Ewt6QCKnaj +++ return 0 ++ '[' true '!=' true ']' ++ kubectl_bin get service/cross-site-sharded-replica-rs0-1 -o 'jsonpath={.spec.type}' ++ grep -q NotFound +++ kubectl_bin get service/cross-site-sharded-replica-rs0-1 -o 'jsonpath={.spec.type}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.TIfJpt3vXE ++++ mktemp +++ local LAST_ERR=/tmp/tmp.qabTX66YX9 +++ local exit_status=0 +++ local timeout=4 ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl get service/cross-site-sharded-replica-rs0-1 -o 'jsonpath={.spec.type}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 -a -n 1 ']' +++ break +++ cat /tmp/tmp.TIfJpt3vXE +++ cat /tmp/tmp.qabTX66YX9 +++ rm /tmp/tmp.TIfJpt3vXE /tmp/tmp.qabTX66YX9 +++ return 0 ++ service_type=ClusterIP ++ '[' ClusterIP = ClusterIP ']' ++ kubectl_bin get service/cross-site-sharded-replica-rs0-1 -o 'jsonpath={.spec.clusterIP}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.Ud7xJJWvSg +++ mktemp ++ local LAST_ERR=/tmp/tmp.L6T5JrdxmD ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get service/cross-site-sharded-replica-rs0-1 -o 'jsonpath={.spec.clusterIP}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.Ud7xJJWvSg ++ cat /tmp/tmp.L6T5JrdxmD ++ rm /tmp/tmp.Ud7xJJWvSg /tmp/tmp.L6T5JrdxmD ++ return 0 ++ return + replica_rs0_1_endpoint=34.118.228.45 ++ get_service_ip cross-site-sharded-replica-rs0-2 ++ local service=cross-site-sharded-replica-rs0-2 ++ local server_type=rs0 +++ kubectl_bin get psmdb/cross-site-sharded-replica -o 'jsonpath={.spec.replsets[].expose.enabled}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.Jjt8fgwL7d ++++ mktemp +++ local LAST_ERR=/tmp/tmp.FhSZdrwoYE +++ local exit_status=0 +++ local timeout=4 ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl get psmdb/cross-site-sharded-replica -o 'jsonpath={.spec.replsets[].expose.enabled}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 -a -n 1 ']' +++ break +++ cat /tmp/tmp.Jjt8fgwL7d +++ cat /tmp/tmp.FhSZdrwoYE +++ rm /tmp/tmp.Jjt8fgwL7d /tmp/tmp.FhSZdrwoYE +++ return 0 ++ '[' true '!=' true ']' ++ kubectl_bin get service/cross-site-sharded-replica-rs0-2 -o 'jsonpath={.spec.type}' ++ grep -q NotFound +++ kubectl_bin get service/cross-site-sharded-replica-rs0-2 -o 'jsonpath={.spec.type}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.DDbbwujAZo ++++ mktemp +++ local LAST_ERR=/tmp/tmp.BzGYlErGeZ +++ local exit_status=0 +++ local timeout=4 ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl get service/cross-site-sharded-replica-rs0-2 -o 'jsonpath={.spec.type}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 -a -n 1 ']' +++ break +++ cat /tmp/tmp.DDbbwujAZo +++ cat /tmp/tmp.BzGYlErGeZ +++ rm /tmp/tmp.DDbbwujAZo /tmp/tmp.BzGYlErGeZ +++ return 0 ++ service_type=ClusterIP ++ '[' ClusterIP = ClusterIP ']' ++ kubectl_bin get service/cross-site-sharded-replica-rs0-2 -o 'jsonpath={.spec.clusterIP}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.630nXaxupU +++ mktemp ++ local LAST_ERR=/tmp/tmp.QTbGzDndjv ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get service/cross-site-sharded-replica-rs0-2 -o 'jsonpath={.spec.clusterIP}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.630nXaxupU ++ cat /tmp/tmp.QTbGzDndjv ++ rm /tmp/tmp.630nXaxupU /tmp/tmp.QTbGzDndjv ++ return 0 ++ return + replica_rs0_2_endpoint=34.118.225.18 ++ get_service_ip cross-site-sharded-replica-rs1-0 rs1 ++ local service=cross-site-sharded-replica-rs1-0 ++ local server_type=rs1 +++ kubectl_bin get psmdb/cross-site-sharded-replica -o 'jsonpath={.spec.replsets[].expose.enabled}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.7RRwznai9e ++++ mktemp +++ local LAST_ERR=/tmp/tmp.Pjh4m7Su8U +++ local exit_status=0 +++ local timeout=4 ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl get psmdb/cross-site-sharded-replica -o 'jsonpath={.spec.replsets[].expose.enabled}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 -a -n 1 ']' +++ break +++ cat /tmp/tmp.7RRwznai9e +++ cat /tmp/tmp.Pjh4m7Su8U +++ rm /tmp/tmp.7RRwznai9e /tmp/tmp.Pjh4m7Su8U +++ return 0 ++ '[' true '!=' true ']' ++ kubectl_bin get service/cross-site-sharded-replica-rs1-0 -o 'jsonpath={.spec.type}' ++ grep -q NotFound +++ kubectl_bin get service/cross-site-sharded-replica-rs1-0 -o 'jsonpath={.spec.type}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.1zO9HBOSNt ++++ mktemp +++ local LAST_ERR=/tmp/tmp.SknbidUaqa +++ local exit_status=0 +++ local timeout=4 ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl get service/cross-site-sharded-replica-rs1-0 -o 'jsonpath={.spec.type}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 -a -n 1 ']' +++ break +++ cat /tmp/tmp.1zO9HBOSNt +++ cat /tmp/tmp.SknbidUaqa +++ rm /tmp/tmp.1zO9HBOSNt /tmp/tmp.SknbidUaqa +++ return 0 ++ service_type=ClusterIP ++ '[' ClusterIP = ClusterIP ']' ++ kubectl_bin get service/cross-site-sharded-replica-rs1-0 -o 'jsonpath={.spec.clusterIP}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.V79onO7XU2 +++ mktemp ++ local LAST_ERR=/tmp/tmp.8LhT9xynQs ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get service/cross-site-sharded-replica-rs1-0 -o 'jsonpath={.spec.clusterIP}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.V79onO7XU2 ++ cat /tmp/tmp.8LhT9xynQs ++ rm /tmp/tmp.V79onO7XU2 /tmp/tmp.8LhT9xynQs ++ return 0 ++ return + replica_rs1_0_endpoint=34.118.234.70 ++ get_service_ip cross-site-sharded-replica-rs1-1 rs1 ++ local service=cross-site-sharded-replica-rs1-1 ++ local server_type=rs1 +++ kubectl_bin get psmdb/cross-site-sharded-replica -o 'jsonpath={.spec.replsets[].expose.enabled}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.TMhlvl8IBI ++++ mktemp +++ local LAST_ERR=/tmp/tmp.Qtz1RfYHVv +++ local exit_status=0 +++ local timeout=4 ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl get psmdb/cross-site-sharded-replica -o 'jsonpath={.spec.replsets[].expose.enabled}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 -a -n 1 ']' +++ break +++ cat /tmp/tmp.TMhlvl8IBI +++ cat /tmp/tmp.Qtz1RfYHVv +++ rm /tmp/tmp.TMhlvl8IBI /tmp/tmp.Qtz1RfYHVv +++ return 0 ++ '[' true '!=' true ']' ++ kubectl_bin get service/cross-site-sharded-replica-rs1-1 -o 'jsonpath={.spec.type}' ++ grep -q NotFound +++ kubectl_bin get service/cross-site-sharded-replica-rs1-1 -o 'jsonpath={.spec.type}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.5VApcSguWB ++++ mktemp +++ local LAST_ERR=/tmp/tmp.I86oz7z858 +++ local exit_status=0 +++ local timeout=4 ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl get service/cross-site-sharded-replica-rs1-1 -o 'jsonpath={.spec.type}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 -a -n 1 ']' +++ break +++ cat /tmp/tmp.5VApcSguWB +++ cat /tmp/tmp.I86oz7z858 +++ rm /tmp/tmp.5VApcSguWB /tmp/tmp.I86oz7z858 +++ return 0 ++ service_type=ClusterIP ++ '[' ClusterIP = ClusterIP ']' ++ kubectl_bin get service/cross-site-sharded-replica-rs1-1 -o 'jsonpath={.spec.clusterIP}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.8b3dYr1liI +++ mktemp ++ local LAST_ERR=/tmp/tmp.ms765nuPkR ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get service/cross-site-sharded-replica-rs1-1 -o 'jsonpath={.spec.clusterIP}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.8b3dYr1liI ++ cat /tmp/tmp.ms765nuPkR ++ rm /tmp/tmp.8b3dYr1liI /tmp/tmp.ms765nuPkR ++ return 0 ++ return + replica_rs1_1_endpoint=34.118.233.66 ++ get_service_ip cross-site-sharded-replica-rs1-2 rs1 ++ local service=cross-site-sharded-replica-rs1-2 ++ local server_type=rs1 +++ kubectl_bin get psmdb/cross-site-sharded-replica -o 'jsonpath={.spec.replsets[].expose.enabled}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.TVBjul4nHt ++++ mktemp +++ local LAST_ERR=/tmp/tmp.Stvuy0C2uw +++ local exit_status=0 +++ local timeout=4 ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl get psmdb/cross-site-sharded-replica -o 'jsonpath={.spec.replsets[].expose.enabled}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 -a -n 1 ']' +++ break +++ cat /tmp/tmp.TVBjul4nHt +++ cat /tmp/tmp.Stvuy0C2uw +++ rm /tmp/tmp.TVBjul4nHt /tmp/tmp.Stvuy0C2uw +++ return 0 ++ '[' true '!=' true ']' ++ kubectl_bin get service/cross-site-sharded-replica-rs1-2 -o 'jsonpath={.spec.type}' ++ grep -q NotFound +++ kubectl_bin get service/cross-site-sharded-replica-rs1-2 -o 'jsonpath={.spec.type}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.dm4yjjiawq ++++ mktemp +++ local LAST_ERR=/tmp/tmp.Ib2IDJ7gBp +++ local exit_status=0 +++ local timeout=4 ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl get service/cross-site-sharded-replica-rs1-2 -o 'jsonpath={.spec.type}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 -a -n 1 ']' +++ break +++ cat /tmp/tmp.dm4yjjiawq +++ cat /tmp/tmp.Ib2IDJ7gBp +++ rm /tmp/tmp.dm4yjjiawq /tmp/tmp.Ib2IDJ7gBp +++ return 0 ++ service_type=ClusterIP ++ '[' ClusterIP = ClusterIP ']' ++ kubectl_bin get service/cross-site-sharded-replica-rs1-2 -o 'jsonpath={.spec.clusterIP}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.uCDIO1Y3p2 +++ mktemp ++ local LAST_ERR=/tmp/tmp.AzTLEPUsnT ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get service/cross-site-sharded-replica-rs1-2 -o 'jsonpath={.spec.clusterIP}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.uCDIO1Y3p2 ++ cat /tmp/tmp.AzTLEPUsnT ++ rm /tmp/tmp.uCDIO1Y3p2 /tmp/tmp.AzTLEPUsnT ++ return 0 ++ return + replica_rs1_2_endpoint=34.118.233.141 ++ kubectl_bin config current-context +++ mktemp ++ local LAST_OUT=/tmp/tmp.bAZjFgroFn +++ mktemp ++ local LAST_ERR=/tmp/tmp.OmM9h0cUA0 ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl config current-context ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.bAZjFgroFn ++ cat /tmp/tmp.OmM9h0cUA0 ++ rm /tmp/tmp.bAZjFgroFn /tmp/tmp.OmM9h0cUA0 ++ return 0 + kubectl_bin config set-context gke_cloud-dev-112233_us-central1-a_jen-psmdb-2287-fc474e65-5-cluster3 --namespace=cross-site-sharded-600 ++ mktemp + local LAST_OUT=/tmp/tmp.Zt9j6fBfsh ++ mktemp + local LAST_ERR=/tmp/tmp.4z0kwm856l + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl config set-context gke_cloud-dev-112233_us-central1-a_jen-psmdb-2287-fc474e65-5-cluster3 --namespace=cross-site-sharded-600 + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.Zt9j6fBfsh Context "gke_cloud-dev-112233_us-central1-a_jen-psmdb-2287-fc474e65-5-cluster3" modified. + cat /tmp/tmp.4z0kwm856l + rm /tmp/tmp.Zt9j6fBfsh /tmp/tmp.4z0kwm856l + return 0 + kubectl_bin patch psmdb cross-site-sharded-main --type=merge --patch $'{\n\t\t\t"spec": {"replsets":[\n\t\t\t {"affinity":{"antiAffinityTopologyKey": "none"},"arbiter":{"affinity":{"antiAffinityTopologyKey": "none"},"enabled":false,"size":1},"expose":{"enabled":true,"type":"ClusterIp"},"externalNodes":[{"host":"34.118.226.208","priority":0,"votes":0},{"host":"34.118.228.45","port":27017,"priority":1,"votes":1},{"host":"34.118.225.18", "port":27017,"priority":1,"votes":1}],"name":"rs0","nonvoting":{"affinity":{"antiAffinityTopologyKey":"none"},"enabled":false,"podDisruptionBudget":{"maxUnavailable":1},"resources":{"limits":{"cpu":"300m","memory":"0.5G"},"requests":{"cpu":"300m","memory":"0.5G"}},"size":3,"volumeSpec":{"persistentVolumeClaim":{"resources":{"requests":{"storage":"1Gi"}}}}},"podDisruptionBudget":{"maxUnavailable":1},"resources":{"limits":{"cpu":"300m","memory":"0.5G"},"requests":{"cpu":"300m","memory":"0.5G"}},"size":3,"volumeSpec":{"persistentVolumeClaim":{"resources":{"requests":{"storage":"3Gi"}}}}},\n\t\t\t {"affinity":{"antiAffinityTopologyKey": "none"},"arbiter":{"affinity":{"antiAffinityTopologyKey": "none"},"enabled":false,"size":1},"expose":{"enabled":true,"type":"ClusterIp"},"externalNodes":[{"host":"34.118.234.70","priority":0,"votes":0},{"host":"34.118.233.66","port":27017,"priority":1,"votes":1},{"host":"34.118.233.141", "port":27017,"priority":1,"votes":1}],"name":"rs1","nonvoting":{"affinity":{"antiAffinityTopologyKey":"none"},"enabled":false,"podDisruptionBudget":{"maxUnavailable":1},"resources":{"limits":{"cpu":"300m","memory":"0.5G"},"requests":{"cpu":"300m","memory":"0.5G"}},"size":3,"volumeSpec":{"persistentVolumeClaim":{"resources":{"requests":{"storage":"1Gi"}}}}},"podDisruptionBudget":{"maxUnavailable":1},"resources":{"limits":{"cpu":"300m","memory":"0.5G"},"requests":{"cpu":"300m","memory":"0.5G"}},"size":3,"volumeSpec":{"persistentVolumeClaim":{"resources":{"requests":{"storage":"3Gi"}}}}}\n\t\t\t ],\n\t\t\t "sharding":{"configsvrReplSet":{ "externalNodes": [{"host":"34.118.226.232","priority":1,"votes":1 },{"host":"34.118.225.23", "priority":1,"votes":1},{"host":"34.118.226.28","priority":0,"votes":0}]}}\n\t\t\t }\n\t\t}' ++ mktemp + local LAST_OUT=/tmp/tmp.AdQoPedaeI ++ mktemp + local LAST_ERR=/tmp/tmp.Hv6ozDCQ5a + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl patch psmdb cross-site-sharded-main --type=merge --patch $'{\n\t\t\t"spec": {"replsets":[\n\t\t\t {"affinity":{"antiAffinityTopologyKey": "none"},"arbiter":{"affinity":{"antiAffinityTopologyKey": "none"},"enabled":false,"size":1},"expose":{"enabled":true,"type":"ClusterIp"},"externalNodes":[{"host":"34.118.226.208","priority":0,"votes":0},{"host":"34.118.228.45","port":27017,"priority":1,"votes":1},{"host":"34.118.225.18", "port":27017,"priority":1,"votes":1}],"name":"rs0","nonvoting":{"affinity":{"antiAffinityTopologyKey":"none"},"enabled":false,"podDisruptionBudget":{"maxUnavailable":1},"resources":{"limits":{"cpu":"300m","memory":"0.5G"},"requests":{"cpu":"300m","memory":"0.5G"}},"size":3,"volumeSpec":{"persistentVolumeClaim":{"resources":{"requests":{"storage":"1Gi"}}}}},"podDisruptionBudget":{"maxUnavailable":1},"resources":{"limits":{"cpu":"300m","memory":"0.5G"},"requests":{"cpu":"300m","memory":"0.5G"}},"size":3,"volumeSpec":{"persistentVolumeClaim":{"resources":{"requests":{"storage":"3Gi"}}}}},\n\t\t\t {"affinity":{"antiAffinityTopologyKey": "none"},"arbiter":{"affinity":{"antiAffinityTopologyKey": "none"},"enabled":false,"size":1},"expose":{"enabled":true,"type":"ClusterIp"},"externalNodes":[{"host":"34.118.234.70","priority":0,"votes":0},{"host":"34.118.233.66","port":27017,"priority":1,"votes":1},{"host":"34.118.233.141", "port":27017,"priority":1,"votes":1}],"name":"rs1","nonvoting":{"affinity":{"antiAffinityTopologyKey":"none"},"enabled":false,"podDisruptionBudget":{"maxUnavailable":1},"resources":{"limits":{"cpu":"300m","memory":"0.5G"},"requests":{"cpu":"300m","memory":"0.5G"}},"size":3,"volumeSpec":{"persistentVolumeClaim":{"resources":{"requests":{"storage":"1Gi"}}}}},"podDisruptionBudget":{"maxUnavailable":1},"resources":{"limits":{"cpu":"300m","memory":"0.5G"},"requests":{"cpu":"300m","memory":"0.5G"}},"size":3,"volumeSpec":{"persistentVolumeClaim":{"resources":{"requests":{"storage":"3Gi"}}}}}\n\t\t\t ],\n\t\t\t "sharding":{"configsvrReplSet":{ "externalNodes": [{"host":"34.118.226.232","priority":1,"votes":1 },{"host":"34.118.225.23", "priority":1,"votes":1},{"host":"34.118.226.28","priority":0,"votes":0}]}}\n\t\t\t }\n\t\t}' + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.AdQoPedaeI perconaservermongodb.psmdb.percona.com/cross-site-sharded-main patched + cat /tmp/tmp.Hv6ozDCQ5a + rm /tmp/tmp.AdQoPedaeI /tmp/tmp.Hv6ozDCQ5a + return 0 + wait_for_members 34.118.226.232 cfg 6 + local endpoint=34.118.226.232 + local rsName=cfg + local target_count=6 + local nodes_count=0 + [[ 0 == 6 ]] ++ run_mongos 'rs.conf().members.length' clusterAdmin:clusterAdmin123456@34.118.226.232 mongodb :27017 ++ local 'command=rs.conf().members.length' ++ local uri=clusterAdmin:clusterAdmin123456@34.118.226.232 ++ local driver=mongodb ++ local suffix=:27017 ++ local mongo_flag= ++ local port=27017 ++ local mongo_bin=mongo ++ grep -E -v 'I NETWORK|W NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|bye' ++ /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/' +++ echo :27017 +++ awk -F: '{print $2}' ++ suffix_port=27017 ++ [[ -z 27017 ]] +++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.N4t1Aj9wVi ++++ mktemp +++ local LAST_ERR=/tmp/tmp.PF6h0ZNVOx +++ local exit_status=0 +++ local timeout=4 ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 -a -n 1 ']' +++ break +++ cat /tmp/tmp.N4t1Aj9wVi +++ cat /tmp/tmp.PF6h0ZNVOx +++ rm /tmp/tmp.N4t1Aj9wVi /tmp/tmp.PF6h0ZNVOx +++ return 0 ++ local client_container=psmdb-client-bb8b97679-gw8g4 ++ kubectl_bin exec psmdb-client-bb8b97679-gw8g4 -- bash -c 'printf '\''rs.conf().members.length\n'\'' | mongo mongodb://clusterAdmin:clusterAdmin123456@34.118.226.232:27017/admin ' +++ mktemp ++ local LAST_OUT=/tmp/tmp.v0WaZPfWVN +++ mktemp ++ local LAST_ERR=/tmp/tmp.Qtfilw98gg ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl exec psmdb-client-bb8b97679-gw8g4 -- bash -c 'printf '\''rs.conf().members.length\n'\'' | mongo mongodb://clusterAdmin:clusterAdmin123456@34.118.226.232:27017/admin ' ++ exit_status=1 ++ set -e ++ '[' 1 '!=' 0 -a -n 1 ']' ++ cat /tmp/tmp.v0WaZPfWVN ++ cat /tmp/tmp.Qtfilw98gg command terminated with exit code 1 ++ sleep 0 ++ for i in $(seq 0 2) ++ set +e ++ kubectl exec psmdb-client-bb8b97679-gw8g4 -- bash -c 'printf '\''rs.conf().members.length\n'\'' | mongo mongodb://clusterAdmin:clusterAdmin123456@34.118.226.232:27017/admin ' ++ exit_status=1 ++ set -e ++ '[' 1 '!=' 0 -a -n 1 ']' ++ cat /tmp/tmp.v0WaZPfWVN ++ cat /tmp/tmp.Qtfilw98gg command terminated with exit code 1 ++ sleep 4 ++ for i in $(seq 0 2) ++ set +e ++ kubectl exec psmdb-client-bb8b97679-gw8g4 -- bash -c 'printf '\''rs.conf().members.length\n'\'' | mongo mongodb://clusterAdmin:clusterAdmin123456@34.118.226.232:27017/admin ' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.v0WaZPfWVN ++ cat /tmp/tmp.Qtfilw98gg ++ rm /tmp/tmp.v0WaZPfWVN /tmp/tmp.Qtfilw98gg ++ return 0 + nodes_count=$'Error: Authentication failed. :\nconnect@src/mongo/shell/mongo.js:374:17\n@(connect):2:6\nexception: connect failed\nexiting with code 1\nError: Authentication failed. :\nconnect@src/mongo/shell/mongo.js:374:17\n@(connect):2:6\nexception: connect failed\nexiting with code 1\n4' + echo -n 'waiting for all members to be configured in cfg' waiting for all members to be configured in cfg+ let retry+=1 + '[' 1 -ge 15 ']' + echo . . + sleep 10 + [[ Error: Authentication failed. : connect@src/mongo/shell/mongo.js:374:17 @(connect):2:6 exception: connect failed exiting with code 1 Error: Authentication failed. : connect@src/mongo/shell/mongo.js:374:17 @(connect):2:6 exception: connect failed exiting with code 1 4 == 6 ]] ++ run_mongos 'rs.conf().members.length' clusterAdmin:clusterAdmin123456@34.118.226.232 mongodb :27017 ++ local 'command=rs.conf().members.length' ++ local uri=clusterAdmin:clusterAdmin123456@34.118.226.232 ++ local driver=mongodb ++ local suffix=:27017 ++ local mongo_flag= ++ local port=27017 ++ grep -E -v 'I NETWORK|W NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|bye' ++ local mongo_bin=mongo ++ /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/' +++ echo :27017 +++ awk -F: '{print $2}' ++ suffix_port=27017 ++ [[ -z 27017 ]] +++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.FTagYLA0XS ++++ mktemp +++ local LAST_ERR=/tmp/tmp.c4MhnHVuf4 +++ local exit_status=0 +++ local timeout=4 ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 -a -n 1 ']' +++ break +++ cat /tmp/tmp.FTagYLA0XS +++ cat /tmp/tmp.c4MhnHVuf4 +++ rm /tmp/tmp.FTagYLA0XS /tmp/tmp.c4MhnHVuf4 +++ return 0 ++ local client_container=psmdb-client-bb8b97679-gw8g4 ++ kubectl_bin exec psmdb-client-bb8b97679-gw8g4 -- bash -c 'printf '\''rs.conf().members.length\n'\'' | mongo mongodb://clusterAdmin:clusterAdmin123456@34.118.226.232:27017/admin ' +++ mktemp ++ local LAST_OUT=/tmp/tmp.fcWyAIDMBU +++ mktemp ++ local LAST_ERR=/tmp/tmp.k2p00UpFxe ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl exec psmdb-client-bb8b97679-gw8g4 -- bash -c 'printf '\''rs.conf().members.length\n'\'' | mongo mongodb://clusterAdmin:clusterAdmin123456@34.118.226.232:27017/admin ' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.fcWyAIDMBU ++ cat /tmp/tmp.k2p00UpFxe ++ rm /tmp/tmp.fcWyAIDMBU /tmp/tmp.k2p00UpFxe ++ return 0 + nodes_count=5 + echo -n 'waiting for all members to be configured in cfg' waiting for all members to be configured in cfg+ let retry+=1 + '[' 2 -ge 15 ']' + echo . . + sleep 10 + [[ 5 == 6 ]] ++ run_mongos 'rs.conf().members.length' clusterAdmin:clusterAdmin123456@34.118.226.232 mongodb :27017 ++ local 'command=rs.conf().members.length' ++ local uri=clusterAdmin:clusterAdmin123456@34.118.226.232 ++ local driver=mongodb ++ local suffix=:27017 ++ local mongo_flag= ++ local port=27017 ++ local mongo_bin=mongo ++ grep -E -v 'I NETWORK|W NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|bye' ++ /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/' +++ echo :27017 +++ awk -F: '{print $2}' ++ suffix_port=27017 ++ [[ -z 27017 ]] +++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.BlUhZ7p0OJ ++++ mktemp +++ local LAST_ERR=/tmp/tmp.LeizcBVwha +++ local exit_status=0 +++ local timeout=4 ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 -a -n 1 ']' +++ break +++ cat /tmp/tmp.BlUhZ7p0OJ +++ cat /tmp/tmp.LeizcBVwha +++ rm /tmp/tmp.BlUhZ7p0OJ /tmp/tmp.LeizcBVwha +++ return 0 ++ local client_container=psmdb-client-bb8b97679-gw8g4 ++ kubectl_bin exec psmdb-client-bb8b97679-gw8g4 -- bash -c 'printf '\''rs.conf().members.length\n'\'' | mongo mongodb://clusterAdmin:clusterAdmin123456@34.118.226.232:27017/admin ' +++ mktemp ++ local LAST_OUT=/tmp/tmp.YPGYXoJe0j +++ mktemp ++ local LAST_ERR=/tmp/tmp.77Kfycbdod ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl exec psmdb-client-bb8b97679-gw8g4 -- bash -c 'printf '\''rs.conf().members.length\n'\'' | mongo mongodb://clusterAdmin:clusterAdmin123456@34.118.226.232:27017/admin ' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.YPGYXoJe0j ++ cat /tmp/tmp.77Kfycbdod ++ rm /tmp/tmp.YPGYXoJe0j /tmp/tmp.77Kfycbdod ++ return 0 + nodes_count=5 + echo -n 'waiting for all members to be configured in cfg' waiting for all members to be configured in cfg+ let retry+=1 + '[' 3 -ge 15 ']' + echo . . + sleep 10 + [[ 5 == 6 ]] ++ run_mongos 'rs.conf().members.length' clusterAdmin:clusterAdmin123456@34.118.226.232 mongodb :27017 ++ local 'command=rs.conf().members.length' ++ local uri=clusterAdmin:clusterAdmin123456@34.118.226.232 ++ local driver=mongodb ++ local suffix=:27017 ++ local mongo_flag= ++ local port=27017 ++ grep -E -v 'I NETWORK|W NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|bye' ++ local mongo_bin=mongo ++ /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/' +++ echo :27017 +++ awk -F: '{print $2}' ++ suffix_port=27017 ++ [[ -z 27017 ]] +++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.fHRjoTwRFe ++++ mktemp +++ local LAST_ERR=/tmp/tmp.GvH9WJuc0P +++ local exit_status=0 +++ local timeout=4 ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 -a -n 1 ']' +++ break +++ cat /tmp/tmp.fHRjoTwRFe +++ cat /tmp/tmp.GvH9WJuc0P +++ rm /tmp/tmp.fHRjoTwRFe /tmp/tmp.GvH9WJuc0P +++ return 0 ++ local client_container=psmdb-client-bb8b97679-gw8g4 ++ kubectl_bin exec psmdb-client-bb8b97679-gw8g4 -- bash -c 'printf '\''rs.conf().members.length\n'\'' | mongo mongodb://clusterAdmin:clusterAdmin123456@34.118.226.232:27017/admin ' +++ mktemp ++ local LAST_OUT=/tmp/tmp.ZtRO3gbbxd +++ mktemp ++ local LAST_ERR=/tmp/tmp.2GZjBGzN1U ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl exec psmdb-client-bb8b97679-gw8g4 -- bash -c 'printf '\''rs.conf().members.length\n'\'' | mongo mongodb://clusterAdmin:clusterAdmin123456@34.118.226.232:27017/admin ' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.ZtRO3gbbxd ++ cat /tmp/tmp.2GZjBGzN1U ++ rm /tmp/tmp.ZtRO3gbbxd /tmp/tmp.2GZjBGzN1U ++ return 0 + nodes_count=5 + echo -n 'waiting for all members to be configured in cfg' waiting for all members to be configured in cfg+ let retry+=1 + '[' 4 -ge 15 ']' + echo . . + sleep 10 + [[ 5 == 6 ]] ++ run_mongos 'rs.conf().members.length' clusterAdmin:clusterAdmin123456@34.118.226.232 mongodb :27017 ++ local 'command=rs.conf().members.length' ++ grep -E -v 'I NETWORK|W NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|bye' ++ local uri=clusterAdmin:clusterAdmin123456@34.118.226.232 ++ local driver=mongodb ++ local suffix=:27017 ++ local mongo_flag= ++ local port=27017 ++ local mongo_bin=mongo ++ /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/' +++ echo :27017 +++ awk -F: '{print $2}' ++ suffix_port=27017 ++ [[ -z 27017 ]] +++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.OZU70D8gQn ++++ mktemp +++ local LAST_ERR=/tmp/tmp.aRfnAguVHp +++ local exit_status=0 +++ local timeout=4 ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 -a -n 1 ']' +++ break +++ cat /tmp/tmp.OZU70D8gQn +++ cat /tmp/tmp.aRfnAguVHp +++ rm /tmp/tmp.OZU70D8gQn /tmp/tmp.aRfnAguVHp +++ return 0 ++ local client_container=psmdb-client-bb8b97679-gw8g4 ++ kubectl_bin exec psmdb-client-bb8b97679-gw8g4 -- bash -c 'printf '\''rs.conf().members.length\n'\'' | mongo mongodb://clusterAdmin:clusterAdmin123456@34.118.226.232:27017/admin ' +++ mktemp ++ local LAST_OUT=/tmp/tmp.H0fXT9pt1N +++ mktemp ++ local LAST_ERR=/tmp/tmp.ehtVQBhVfA ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl exec psmdb-client-bb8b97679-gw8g4 -- bash -c 'printf '\''rs.conf().members.length\n'\'' | mongo mongodb://clusterAdmin:clusterAdmin123456@34.118.226.232:27017/admin ' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.H0fXT9pt1N ++ cat /tmp/tmp.ehtVQBhVfA ++ rm /tmp/tmp.H0fXT9pt1N /tmp/tmp.ehtVQBhVfA ++ return 0 + nodes_count=5 + echo -n 'waiting for all members to be configured in cfg' waiting for all members to be configured in cfg+ let retry+=1 + '[' 5 -ge 15 ']' + echo . . + sleep 10 + [[ 5 == 6 ]] ++ run_mongos 'rs.conf().members.length' clusterAdmin:clusterAdmin123456@34.118.226.232 mongodb :27017 ++ local 'command=rs.conf().members.length' ++ local uri=clusterAdmin:clusterAdmin123456@34.118.226.232 ++ local driver=mongodb ++ local suffix=:27017 ++ local mongo_flag= ++ local port=27017 ++ local mongo_bin=mongo ++ grep -E -v 'I NETWORK|W NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|bye' ++ /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/' +++ echo :27017 +++ awk -F: '{print $2}' ++ suffix_port=27017 ++ [[ -z 27017 ]] +++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.sD18wc8YWW ++++ mktemp +++ local LAST_ERR=/tmp/tmp.f3rTAlxfGG +++ local exit_status=0 +++ local timeout=4 ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 -a -n 1 ']' +++ break +++ cat /tmp/tmp.sD18wc8YWW +++ cat /tmp/tmp.f3rTAlxfGG +++ rm /tmp/tmp.sD18wc8YWW /tmp/tmp.f3rTAlxfGG +++ return 0 ++ local client_container=psmdb-client-bb8b97679-gw8g4 ++ kubectl_bin exec psmdb-client-bb8b97679-gw8g4 -- bash -c 'printf '\''rs.conf().members.length\n'\'' | mongo mongodb://clusterAdmin:clusterAdmin123456@34.118.226.232:27017/admin ' +++ mktemp ++ local LAST_OUT=/tmp/tmp.JUVXoEDhDw +++ mktemp ++ local LAST_ERR=/tmp/tmp.MUPLZpXW7i ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl exec psmdb-client-bb8b97679-gw8g4 -- bash -c 'printf '\''rs.conf().members.length\n'\'' | mongo mongodb://clusterAdmin:clusterAdmin123456@34.118.226.232:27017/admin ' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.JUVXoEDhDw ++ cat /tmp/tmp.MUPLZpXW7i ++ rm /tmp/tmp.JUVXoEDhDw /tmp/tmp.MUPLZpXW7i ++ return 0 + nodes_count=5 + echo -n 'waiting for all members to be configured in cfg' waiting for all members to be configured in cfg+ let retry+=1 + '[' 6 -ge 15 ']' + echo . . + sleep 10 + [[ 5 == 6 ]] ++ run_mongos 'rs.conf().members.length' clusterAdmin:clusterAdmin123456@34.118.226.232 mongodb :27017 ++ local 'command=rs.conf().members.length' ++ grep -E -v 'I NETWORK|W NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|bye' ++ local uri=clusterAdmin:clusterAdmin123456@34.118.226.232 ++ local driver=mongodb ++ local suffix=:27017 ++ local mongo_flag= ++ local port=27017 ++ local mongo_bin=mongo ++ /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/' +++ echo :27017 +++ awk -F: '{print $2}' ++ suffix_port=27017 ++ [[ -z 27017 ]] +++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.4hFWqr45Ob ++++ mktemp +++ local LAST_ERR=/tmp/tmp.jvueAqGtCI +++ local exit_status=0 +++ local timeout=4 ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 -a -n 1 ']' +++ break +++ cat /tmp/tmp.4hFWqr45Ob +++ cat /tmp/tmp.jvueAqGtCI +++ rm /tmp/tmp.4hFWqr45Ob /tmp/tmp.jvueAqGtCI +++ return 0 ++ local client_container=psmdb-client-bb8b97679-gw8g4 ++ kubectl_bin exec psmdb-client-bb8b97679-gw8g4 -- bash -c 'printf '\''rs.conf().members.length\n'\'' | mongo mongodb://clusterAdmin:clusterAdmin123456@34.118.226.232:27017/admin ' +++ mktemp ++ local LAST_OUT=/tmp/tmp.K4UouwYQdc +++ mktemp ++ local LAST_ERR=/tmp/tmp.XNns0lnh3Y ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl exec psmdb-client-bb8b97679-gw8g4 -- bash -c 'printf '\''rs.conf().members.length\n'\'' | mongo mongodb://clusterAdmin:clusterAdmin123456@34.118.226.232:27017/admin ' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.K4UouwYQdc ++ cat /tmp/tmp.XNns0lnh3Y ++ rm /tmp/tmp.K4UouwYQdc /tmp/tmp.XNns0lnh3Y ++ return 0 + nodes_count=5 + echo -n 'waiting for all members to be configured in cfg' waiting for all members to be configured in cfg+ let retry+=1 + '[' 7 -ge 15 ']' + echo . . + sleep 10 + [[ 5 == 6 ]] ++ run_mongos 'rs.conf().members.length' clusterAdmin:clusterAdmin123456@34.118.226.232 mongodb :27017 ++ grep -E -v 'I NETWORK|W NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|bye' ++ local 'command=rs.conf().members.length' ++ local uri=clusterAdmin:clusterAdmin123456@34.118.226.232 ++ local driver=mongodb ++ local suffix=:27017 ++ local mongo_flag= ++ local port=27017 ++ local mongo_bin=mongo ++ /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/' +++ echo :27017 +++ awk -F: '{print $2}' ++ suffix_port=27017 ++ [[ -z 27017 ]] +++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.R295f4i1GM ++++ mktemp +++ local LAST_ERR=/tmp/tmp.TanaOGBiiS +++ local exit_status=0 +++ local timeout=4 ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 -a -n 1 ']' +++ break +++ cat /tmp/tmp.R295f4i1GM +++ cat /tmp/tmp.TanaOGBiiS +++ rm /tmp/tmp.R295f4i1GM /tmp/tmp.TanaOGBiiS +++ return 0 ++ local client_container=psmdb-client-bb8b97679-gw8g4 ++ kubectl_bin exec psmdb-client-bb8b97679-gw8g4 -- bash -c 'printf '\''rs.conf().members.length\n'\'' | mongo mongodb://clusterAdmin:clusterAdmin123456@34.118.226.232:27017/admin ' +++ mktemp ++ local LAST_OUT=/tmp/tmp.SDp9rDDwPp +++ mktemp ++ local LAST_ERR=/tmp/tmp.swZNShM5wd ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl exec psmdb-client-bb8b97679-gw8g4 -- bash -c 'printf '\''rs.conf().members.length\n'\'' | mongo mongodb://clusterAdmin:clusterAdmin123456@34.118.226.232:27017/admin ' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.SDp9rDDwPp ++ cat /tmp/tmp.swZNShM5wd ++ rm /tmp/tmp.SDp9rDDwPp /tmp/tmp.swZNShM5wd ++ return 0 + nodes_count=5 + echo -n 'waiting for all members to be configured in cfg' waiting for all members to be configured in cfg+ let retry+=1 + '[' 8 -ge 15 ']' + echo . . + sleep 10 + [[ 5 == 6 ]] ++ run_mongos 'rs.conf().members.length' clusterAdmin:clusterAdmin123456@34.118.226.232 mongodb :27017 ++ local 'command=rs.conf().members.length' ++ local uri=clusterAdmin:clusterAdmin123456@34.118.226.232 ++ local driver=mongodb ++ local suffix=:27017 ++ local mongo_flag= ++ local port=27017 ++ local mongo_bin=mongo ++ grep -E -v 'I NETWORK|W NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|bye' ++ /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/' +++ echo :27017 +++ awk -F: '{print $2}' ++ suffix_port=27017 ++ [[ -z 27017 ]] +++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.qb4EdzyrNr ++++ mktemp +++ local LAST_ERR=/tmp/tmp.kghrhDasn6 +++ local exit_status=0 +++ local timeout=4 ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 -a -n 1 ']' +++ break +++ cat /tmp/tmp.qb4EdzyrNr +++ cat /tmp/tmp.kghrhDasn6 +++ rm /tmp/tmp.qb4EdzyrNr /tmp/tmp.kghrhDasn6 +++ return 0 ++ local client_container=psmdb-client-bb8b97679-gw8g4 ++ kubectl_bin exec psmdb-client-bb8b97679-gw8g4 -- bash -c 'printf '\''rs.conf().members.length\n'\'' | mongo mongodb://clusterAdmin:clusterAdmin123456@34.118.226.232:27017/admin ' +++ mktemp ++ local LAST_OUT=/tmp/tmp.3zszHSmZCw +++ mktemp ++ local LAST_ERR=/tmp/tmp.hhenGZ3T7D ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl exec psmdb-client-bb8b97679-gw8g4 -- bash -c 'printf '\''rs.conf().members.length\n'\'' | mongo mongodb://clusterAdmin:clusterAdmin123456@34.118.226.232:27017/admin ' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.3zszHSmZCw ++ cat /tmp/tmp.hhenGZ3T7D ++ rm /tmp/tmp.3zszHSmZCw /tmp/tmp.hhenGZ3T7D ++ return 0 + nodes_count=6 + echo -n 'waiting for all members to be configured in cfg' waiting for all members to be configured in cfg+ let retry+=1 + '[' 9 -ge 15 ']' + echo . . + sleep 10 + [[ 6 == 6 ]] + wait_for_members 34.118.226.208 rs0 6 + local endpoint=34.118.226.208 + local rsName=rs0 + local target_count=6 + local nodes_count=0 + [[ 0 == 6 ]] ++ run_mongos 'rs.conf().members.length' clusterAdmin:clusterAdmin123456@34.118.226.208 mongodb :27017 ++ grep -E -v 'I NETWORK|W NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|bye' ++ local 'command=rs.conf().members.length' ++ local uri=clusterAdmin:clusterAdmin123456@34.118.226.208 ++ local driver=mongodb ++ local suffix=:27017 ++ local mongo_flag= ++ local port=27017 ++ local mongo_bin=mongo ++ /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/' +++ echo :27017 +++ awk -F: '{print $2}' ++ suffix_port=27017 ++ [[ -z 27017 ]] +++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.CsTfy8NcXO ++++ mktemp +++ local LAST_ERR=/tmp/tmp.isLMPrSwrJ +++ local exit_status=0 +++ local timeout=4 ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 -a -n 1 ']' +++ break +++ cat /tmp/tmp.CsTfy8NcXO +++ cat /tmp/tmp.isLMPrSwrJ +++ rm /tmp/tmp.CsTfy8NcXO /tmp/tmp.isLMPrSwrJ +++ return 0 ++ local client_container=psmdb-client-bb8b97679-gw8g4 ++ kubectl_bin exec psmdb-client-bb8b97679-gw8g4 -- bash -c 'printf '\''rs.conf().members.length\n'\'' | mongo mongodb://clusterAdmin:clusterAdmin123456@34.118.226.208:27017/admin ' +++ mktemp ++ local LAST_OUT=/tmp/tmp.c2hO5IM6X1 +++ mktemp ++ local LAST_ERR=/tmp/tmp.Qaoi3k9vaH ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl exec psmdb-client-bb8b97679-gw8g4 -- bash -c 'printf '\''rs.conf().members.length\n'\'' | mongo mongodb://clusterAdmin:clusterAdmin123456@34.118.226.208:27017/admin ' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.c2hO5IM6X1 ++ cat /tmp/tmp.Qaoi3k9vaH ++ rm /tmp/tmp.c2hO5IM6X1 /tmp/tmp.Qaoi3k9vaH ++ return 0 + nodes_count=6 + echo -n 'waiting for all members to be configured in rs0' waiting for all members to be configured in rs0+ let retry+=1 + '[' 10 -ge 15 ']' + echo . . + sleep 10 + [[ 6 == 6 ]] + wait_for_members 34.118.234.70 rs1 6 + local endpoint=34.118.234.70 + local rsName=rs1 + local target_count=6 + local nodes_count=0 + [[ 0 == 6 ]] ++ run_mongos 'rs.conf().members.length' clusterAdmin:clusterAdmin123456@34.118.234.70 mongodb :27017 ++ local 'command=rs.conf().members.length' ++ local uri=clusterAdmin:clusterAdmin123456@34.118.234.70 ++ local driver=mongodb ++ local suffix=:27017 ++ local mongo_flag= ++ local port=27017 ++ local mongo_bin=mongo ++ grep -E -v 'I NETWORK|W NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|bye' ++ /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/' +++ echo :27017 +++ awk -F: '{print $2}' ++ suffix_port=27017 ++ [[ -z 27017 ]] +++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.zsknA4lo9P ++++ mktemp +++ local LAST_ERR=/tmp/tmp.l4hvAe17Dd +++ local exit_status=0 +++ local timeout=4 ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 -a -n 1 ']' +++ break +++ cat /tmp/tmp.zsknA4lo9P +++ cat /tmp/tmp.l4hvAe17Dd +++ rm /tmp/tmp.zsknA4lo9P /tmp/tmp.l4hvAe17Dd +++ return 0 ++ local client_container=psmdb-client-bb8b97679-gw8g4 ++ kubectl_bin exec psmdb-client-bb8b97679-gw8g4 -- bash -c 'printf '\''rs.conf().members.length\n'\'' | mongo mongodb://clusterAdmin:clusterAdmin123456@34.118.234.70:27017/admin ' +++ mktemp ++ local LAST_OUT=/tmp/tmp.v5qQym4IzC +++ mktemp ++ local LAST_ERR=/tmp/tmp.6d2YRXsbgR ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl exec psmdb-client-bb8b97679-gw8g4 -- bash -c 'printf '\''rs.conf().members.length\n'\'' | mongo mongodb://clusterAdmin:clusterAdmin123456@34.118.234.70:27017/admin ' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.v5qQym4IzC ++ cat /tmp/tmp.6d2YRXsbgR ++ rm /tmp/tmp.v5qQym4IzC /tmp/tmp.6d2YRXsbgR ++ return 0 + nodes_count=6 + echo -n 'waiting for all members to be configured in rs1' waiting for all members to be configured in rs1+ let retry+=1 + '[' 11 -ge 15 ']' + echo . . + sleep 10 + [[ 6 == 6 ]] ++ kubectl_bin config current-context +++ mktemp ++ local LAST_OUT=/tmp/tmp.Vfhkkogtv7 +++ mktemp ++ local LAST_ERR=/tmp/tmp.xFnyHfTw1P ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl config current-context ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.Vfhkkogtv7 ++ cat /tmp/tmp.xFnyHfTw1P ++ rm /tmp/tmp.Vfhkkogtv7 /tmp/tmp.xFnyHfTw1P ++ return 0 + kubectl_bin config set-context gke_cloud-dev-112233_us-central1-a_jen-psmdb-2287-fc474e65-5-cluster3 --namespace=cross-site-sharded-replica-2372 ++ mktemp + local LAST_OUT=/tmp/tmp.tD0XLF2bNK ++ mktemp + local LAST_ERR=/tmp/tmp.W7NxvUPBgW + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl config set-context gke_cloud-dev-112233_us-central1-a_jen-psmdb-2287-fc474e65-5-cluster3 --namespace=cross-site-sharded-replica-2372 + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.tD0XLF2bNK Context "gke_cloud-dev-112233_us-central1-a_jen-psmdb-2287-fc474e65-5-cluster3" modified. + cat /tmp/tmp.W7NxvUPBgW + rm /tmp/tmp.tD0XLF2bNK /tmp/tmp.W7NxvUPBgW + return 0 + desc 'check if all 3 Pods started' + set +o xtrace ----------------------------------------------------------------------------------- check if all 3 Pods started ----------------------------------------------------------------------------------- + wait_for_running cross-site-sharded-replica-rs0 3 + local name=cross-site-sharded-replica-rs0 + let last_pod=2 + local check_cluster_readyness=true + set_debug + [[ 1 == 1 ]] + set -o xtrace + local rs_name=rs0 + local cluster_name=cross-site-sharded-replica ++ seq 0 2 + for i in $(seq 0 $last_pod) + [[ 0 -eq 2 ]] + wait_pod cross-site-sharded-replica-rs0-0 + local pod=cross-site-sharded-replica-rs0-0 + set +o xtrace waiting for pod/cross-site-sharded-replica-rs0-0 to be ready.OK + for i in $(seq 0 $last_pod) + [[ 1 -eq 2 ]] + wait_pod cross-site-sharded-replica-rs0-1 + local pod=cross-site-sharded-replica-rs0-1 + set +o xtrace waiting for pod/cross-site-sharded-replica-rs0-1 to be ready.OK + for i in $(seq 0 $last_pod) + [[ 2 -eq 2 ]] ++ kubectl_bin get psmdb cross-site-sharded-replica -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].arbiter.enabled}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.6G1FsDAo7s +++ mktemp ++ local LAST_ERR=/tmp/tmp.NLup9qq8RS ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb cross-site-sharded-replica -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].arbiter.enabled}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.6G1FsDAo7s ++ cat /tmp/tmp.NLup9qq8RS ++ rm /tmp/tmp.6G1FsDAo7s /tmp/tmp.NLup9qq8RS ++ return 0 + [[ false == true ]] + wait_pod cross-site-sharded-replica-rs0-2 + local pod=cross-site-sharded-replica-rs0-2 + set +o xtrace waiting for pod/cross-site-sharded-replica-rs0-2 to be ready.OK ++ kubectl_bin get psmdb cross-site-sharded-replica -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].nonvoting.enabled}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.rQ8btwWdq4 +++ mktemp ++ local LAST_ERR=/tmp/tmp.vFHaQ67lKv ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb cross-site-sharded-replica -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].nonvoting.enabled}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.rQ8btwWdq4 ++ cat /tmp/tmp.vFHaQ67lKv ++ rm /tmp/tmp.rQ8btwWdq4 /tmp/tmp.vFHaQ67lKv ++ return 0 + [[ false == true ]] ++ kubectl_bin get psmdb cross-site-sharded-replica -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].hidden.enabled}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.OK5JHZPpTX +++ mktemp ++ local LAST_ERR=/tmp/tmp.RiI0Yr6n7J ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb cross-site-sharded-replica -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].hidden.enabled}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.OK5JHZPpTX ++ cat /tmp/tmp.RiI0Yr6n7J ++ rm /tmp/tmp.OK5JHZPpTX /tmp/tmp.RiI0Yr6n7J ++ return 0 + [[ '' == true ]] + sleep 10 + [[ true == true ]] + set +x Waiting for cluster readyness + wait_for_running cross-site-sharded-replica-cfg 3 false + local name=cross-site-sharded-replica-cfg + let last_pod=2 + local check_cluster_readyness=false + set_debug + [[ 1 == 1 ]] + set -o xtrace + local rs_name=cfg + local cluster_name=cross-site-sharded-replica ++ seq 0 2 + for i in $(seq 0 $last_pod) + [[ 0 -eq 2 ]] + wait_pod cross-site-sharded-replica-cfg-0 + local pod=cross-site-sharded-replica-cfg-0 + set +o xtrace waiting for pod/cross-site-sharded-replica-cfg-0 to be ready.OK + for i in $(seq 0 $last_pod) + [[ 1 -eq 2 ]] + wait_pod cross-site-sharded-replica-cfg-1 + local pod=cross-site-sharded-replica-cfg-1 + set +o xtrace waiting for pod/cross-site-sharded-replica-cfg-1 to be ready.OK + for i in $(seq 0 $last_pod) + [[ 2 -eq 2 ]] ++ kubectl_bin get psmdb cross-site-sharded-replica -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].arbiter.enabled}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.Q6lnLEixA2 +++ mktemp ++ local LAST_ERR=/tmp/tmp.egI7W8f5Fv ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb cross-site-sharded-replica -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].arbiter.enabled}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.Q6lnLEixA2 ++ cat /tmp/tmp.egI7W8f5Fv ++ rm /tmp/tmp.Q6lnLEixA2 /tmp/tmp.egI7W8f5Fv ++ return 0 + [[ '' == true ]] + wait_pod cross-site-sharded-replica-cfg-2 + local pod=cross-site-sharded-replica-cfg-2 + set +o xtrace waiting for pod/cross-site-sharded-replica-cfg-2 to be ready.OK ++ kubectl_bin get psmdb cross-site-sharded-replica -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].nonvoting.enabled}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.K0ORshsqbq +++ mktemp ++ local LAST_ERR=/tmp/tmp.KwDRuoVNdz ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb cross-site-sharded-replica -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].nonvoting.enabled}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.K0ORshsqbq ++ cat /tmp/tmp.KwDRuoVNdz ++ rm /tmp/tmp.K0ORshsqbq /tmp/tmp.KwDRuoVNdz ++ return 0 + [[ '' == true ]] ++ kubectl_bin get psmdb cross-site-sharded-replica -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].hidden.enabled}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.LnWW6egmGZ +++ mktemp ++ local LAST_ERR=/tmp/tmp.kXB1CLhhwk ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb cross-site-sharded-replica -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].hidden.enabled}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.LnWW6egmGZ ++ cat /tmp/tmp.kXB1CLhhwk ++ rm /tmp/tmp.LnWW6egmGZ /tmp/tmp.kXB1CLhhwk ++ return 0 + [[ '' == true ]] + sleep 10 + [[ false == true ]] + desc 'create user' + set +o xtrace ----------------------------------------------------------------------------------- create user ----------------------------------------------------------------------------------- + run_mongos 'db.createUser({user:"myApp",pwd:"myPass",roles:[{db:"myApp",role:"readWrite"}]})' userAdmin:userAdmin123456@cross-site-sharded-main-mongos.cross-site-sharded-600 + local 'command=db.createUser({user:"myApp",pwd:"myPass",roles:[{db:"myApp",role:"readWrite"}]})' + local uri=userAdmin:userAdmin123456@cross-site-sharded-main-mongos.cross-site-sharded-600 + local driver=mongodb + local suffix=.svc.cluster.local + local mongo_flag= + local port=27017 + local mongo_bin=mongo ++ echo .svc.cluster.local ++ awk -F: '{print $2}' + suffix_port= + [[ -z '' ]] + suffix=.svc.cluster.local:27017 ++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.67bajZyZJB +++ mktemp ++ local LAST_ERR=/tmp/tmp.yxCBoBhjxm ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.67bajZyZJB ++ cat /tmp/tmp.yxCBoBhjxm ++ rm /tmp/tmp.67bajZyZJB /tmp/tmp.yxCBoBhjxm ++ return 0 + local client_container=psmdb-client-bb8b97679-hdlpq + kubectl_bin exec psmdb-client-bb8b97679-hdlpq -- bash -c 'printf '\''db.createUser({user:"myApp",pwd:"myPass",roles:[{db:"myApp",role:"readWrite"}]})\n'\'' | mongo mongodb://userAdmin:userAdmin123456@cross-site-sharded-main-mongos.cross-site-sharded-600.svc.cluster.local:27017/admin ' ++ mktemp + local LAST_OUT=/tmp/tmp.KiB7Ytkc7T ++ mktemp + local LAST_ERR=/tmp/tmp.NZB3B2sHkS + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl exec psmdb-client-bb8b97679-hdlpq -- bash -c 'printf '\''db.createUser({user:"myApp",pwd:"myPass",roles:[{db:"myApp",role:"readWrite"}]})\n'\'' | mongo mongodb://userAdmin:userAdmin123456@cross-site-sharded-main-mongos.cross-site-sharded-600.svc.cluster.local:27017/admin ' + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.KiB7Ytkc7T Percona Server for MongoDB shell version v4.4.29-28 connecting to: mongodb://cross-site-sharded-main-mongos.cross-site-sharded-600.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb Implicit session: session { "id" : UUID("db796f24-4230-4e6b-98df-6b865ab93253") } Percona Server for MongoDB server version: v8.0.20-8 WARNING: shell and server versions do not match Successfully added user: { "user" : "myApp", "roles" : [ { "db" : "myApp", "role" : "readWrite" } ] } bye + cat /tmp/tmp.NZB3B2sHkS + rm /tmp/tmp.KiB7Ytkc7T /tmp/tmp.NZB3B2sHkS + return 0 + sleep 2 + desc 'write data, read from all' + set +o xtrace ----------------------------------------------------------------------------------- write data, read from all ----------------------------------------------------------------------------------- + run_mongos 'use myApp\n db.test.insert({ x: 100500 })' myApp:myPass@cross-site-sharded-main-mongos.cross-site-sharded-600 + local 'command=use myApp\n db.test.insert({ x: 100500 })' + local uri=myApp:myPass@cross-site-sharded-main-mongos.cross-site-sharded-600 + local driver=mongodb + local suffix=.svc.cluster.local + local mongo_flag= + local port=27017 + local mongo_bin=mongo ++ echo .svc.cluster.local ++ awk -F: '{print $2}' + suffix_port= + [[ -z '' ]] + suffix=.svc.cluster.local:27017 ++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.731KXumoJn +++ mktemp ++ local LAST_ERR=/tmp/tmp.fKzOr4MnGk ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.731KXumoJn ++ cat /tmp/tmp.fKzOr4MnGk ++ rm /tmp/tmp.731KXumoJn /tmp/tmp.fKzOr4MnGk ++ return 0 + local client_container=psmdb-client-bb8b97679-hdlpq + kubectl_bin exec psmdb-client-bb8b97679-hdlpq -- bash -c 'printf '\''use myApp\n db.test.insert({ x: 100500 })\n'\'' | mongo mongodb://myApp:myPass@cross-site-sharded-main-mongos.cross-site-sharded-600.svc.cluster.local:27017/admin ' ++ mktemp + local LAST_OUT=/tmp/tmp.TwXu5Ogv2b ++ mktemp + local LAST_ERR=/tmp/tmp.2rZJjsl2XP + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl exec psmdb-client-bb8b97679-hdlpq -- bash -c 'printf '\''use myApp\n db.test.insert({ x: 100500 })\n'\'' | mongo mongodb://myApp:myPass@cross-site-sharded-main-mongos.cross-site-sharded-600.svc.cluster.local:27017/admin ' + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.TwXu5Ogv2b Percona Server for MongoDB shell version v4.4.29-28 connecting to: mongodb://cross-site-sharded-main-mongos.cross-site-sharded-600.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb Implicit session: session { "id" : UUID("52bcd30e-a941-4ee8-900f-c1b42ec6e92b") } Percona Server for MongoDB server version: v8.0.20-8 WARNING: shell and server versions do not match switched to db myApp WriteResult({ "nInserted" : 1 }) bye + cat /tmp/tmp.2rZJjsl2XP + rm /tmp/tmp.TwXu5Ogv2b /tmp/tmp.2rZJjsl2XP + return 0 + minikube_sleep + sleep_time=10 + [[ '' == 1 ]] + desc 'Compare data' + set +o xtrace ----------------------------------------------------------------------------------- Compare data ----------------------------------------------------------------------------------- + compare_mongos_cmd find myApp:myPass@cross-site-sharded-main-mongos.cross-site-sharded-600 + local command=find + local uri=myApp:myPass@cross-site-sharded-main-mongos.cross-site-sharded-600 + local postfix= + local suffix= + local database=myApp + local collection=test + local port=27017 + local tls=false + [[ false == true ]] + mongos_command=run_mongos + log 'running db.test.command() in myApp' + set +o xtrace [2026-04-09T12:46:57+0000] running db.test.command() in myApp + run_mongos 'use myApp\n db.test.find()' myApp:myPass@cross-site-sharded-main-mongos.cross-site-sharded-600 mongodb '' '' 27017 + grep -E -v 'I NETWORK|W NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|Started a new thread for the timer service' + local 'command=use myApp\n db.test.find()' + local uri=myApp:myPass@cross-site-sharded-main-mongos.cross-site-sharded-600 + local driver=mongodb + local suffix=.svc.cluster.local + local mongo_flag= + local port=27017 + local mongo_bin=mongo + /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/' ++ echo .svc.cluster.local ++ awk -F: '{print $2}' + suffix_port= + [[ -z '' ]] + suffix=.svc.cluster.local:27017 ++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.EPLlUob1Mt +++ mktemp ++ local LAST_ERR=/tmp/tmp.sYpx9Zveha ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.EPLlUob1Mt ++ cat /tmp/tmp.sYpx9Zveha ++ rm /tmp/tmp.EPLlUob1Mt /tmp/tmp.sYpx9Zveha ++ return 0 + local client_container=psmdb-client-bb8b97679-hdlpq + kubectl_bin exec psmdb-client-bb8b97679-hdlpq -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@cross-site-sharded-main-mongos.cross-site-sharded-600.svc.cluster.local:27017/admin ' ++ mktemp + local LAST_OUT=/tmp/tmp.PZnUpXVkMg ++ mktemp + local LAST_ERR=/tmp/tmp.akKFhRPcOk + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl exec psmdb-client-bb8b97679-hdlpq -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@cross-site-sharded-main-mongos.cross-site-sharded-600.svc.cluster.local:27017/admin ' + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.PZnUpXVkMg + cat /tmp/tmp.akKFhRPcOk + rm /tmp/tmp.PZnUpXVkMg /tmp/tmp.akKFhRPcOk + return 0 + [[ 0 -eq 0 ]] + diff /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2287/e2e-tests/cross-site-sharded/compare/find.json /tmp/tmp.U69oeCRYdg/find + desc 'test failover' + set +o xtrace ----------------------------------------------------------------------------------- test failover ----------------------------------------------------------------------------------- ++ kubectl_bin config current-context +++ mktemp ++ local LAST_OUT=/tmp/tmp.cVil376Tct +++ mktemp ++ local LAST_ERR=/tmp/tmp.wSGJO1gJBx ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl config current-context ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.cVil376Tct ++ cat /tmp/tmp.wSGJO1gJBx ++ rm /tmp/tmp.cVil376Tct /tmp/tmp.wSGJO1gJBx ++ return 0 + kubectl_bin config set-context gke_cloud-dev-112233_us-central1-a_jen-psmdb-2287-fc474e65-5-cluster3 --namespace=cross-site-sharded-600 ++ mktemp + local LAST_OUT=/tmp/tmp.CTNTPoVKPw ++ mktemp + local LAST_ERR=/tmp/tmp.2yoltyZm4R + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl config set-context gke_cloud-dev-112233_us-central1-a_jen-psmdb-2287-fc474e65-5-cluster3 --namespace=cross-site-sharded-600 + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.CTNTPoVKPw Context "gke_cloud-dev-112233_us-central1-a_jen-psmdb-2287-fc474e65-5-cluster3" modified. + cat /tmp/tmp.2yoltyZm4R + rm /tmp/tmp.CTNTPoVKPw /tmp/tmp.2yoltyZm4R + return 0 + kubectl_bin delete psmdb cross-site-sharded-main ++ mktemp + local LAST_OUT=/tmp/tmp.IkYmmazwIS ++ mktemp + local LAST_ERR=/tmp/tmp.yvA5zRmdz8 + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl delete psmdb cross-site-sharded-main + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.IkYmmazwIS perconaservermongodb.psmdb.percona.com "cross-site-sharded-main" deleted from cross-site-sharded-600 namespace + cat /tmp/tmp.yvA5zRmdz8 + rm /tmp/tmp.IkYmmazwIS /tmp/tmp.yvA5zRmdz8 + return 0 + desc 'run disaster recovery script for replset: cfg' + set +o xtrace ----------------------------------------------------------------------------------- run disaster recovery script for replset: cfg ----------------------------------------------------------------------------------- + run_script_mongos /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2287/e2e-tests/cross-site-sharded/disaster_recovery.js clusterAdmin:clusterAdmin123456@34.118.226.232 mongodb :27017 + local script=/mnt/jenkins/workspace/cloud-psmdb-operator_PR-2287/e2e-tests/cross-site-sharded/disaster_recovery.js + local uri=clusterAdmin:clusterAdmin123456@34.118.226.232 + local driver=mongodb + local suffix=:27017 + local mongo_flag= + local mongo_bin=mongo ++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.bqqvxq2l3s +++ mktemp ++ local LAST_ERR=/tmp/tmp.u6WmiNsJni ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.bqqvxq2l3s ++ cat /tmp/tmp.u6WmiNsJni ++ rm /tmp/tmp.bqqvxq2l3s /tmp/tmp.u6WmiNsJni ++ return 0 + local client_container=psmdb-client-bb8b97679-gw8g4 ++ basename /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2287/e2e-tests/cross-site-sharded/disaster_recovery.js + name=disaster_recovery.js + kubectl_bin cp /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2287/e2e-tests/cross-site-sharded/disaster_recovery.js cross-site-sharded-600/psmdb-client-bb8b97679-gw8g4:/tmp ++ mktemp + local LAST_OUT=/tmp/tmp.kSz0TIRnSs ++ mktemp + local LAST_ERR=/tmp/tmp.y9eYxIRvTk + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl cp /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2287/e2e-tests/cross-site-sharded/disaster_recovery.js cross-site-sharded-600/psmdb-client-bb8b97679-gw8g4:/tmp + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.kSz0TIRnSs + cat /tmp/tmp.y9eYxIRvTk + rm /tmp/tmp.kSz0TIRnSs /tmp/tmp.y9eYxIRvTk + return 0 + kubectl_bin exec psmdb-client-bb8b97679-gw8g4 -- bash -c 'mongo mongodb://clusterAdmin:clusterAdmin123456@34.118.226.232:27017/admin /tmp/disaster_recovery.js' ++ mktemp + local LAST_OUT=/tmp/tmp.3bbqLsQNSA ++ mktemp + local LAST_ERR=/tmp/tmp.yiDavDJN4h + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl exec psmdb-client-bb8b97679-gw8g4 -- bash -c 'mongo mongodb://clusterAdmin:clusterAdmin123456@34.118.226.232:27017/admin /tmp/disaster_recovery.js' + exit_status=253 + set -e + '[' 253 '!=' 0 -a -n 1 ']' + cat /tmp/tmp.3bbqLsQNSA Percona Server for MongoDB shell version v4.4.29-28 connecting to: mongodb://34.118.226.232:27017/admin?compressors=disabled&gssapiServiceName=mongodb Implicit session: session { "id" : UUID("a5a6a117-2291-4ee0-b8c6-7b96a9920429") } Percona Server for MongoDB server version: v8.0.20-8 WARNING: shell and server versions do not match uncaught exception: TypeError: member3 is undefined : @/tmp/disaster_recovery.js:12:1 failed to load: /tmp/disaster_recovery.js exiting with code -3 + cat /tmp/tmp.yiDavDJN4h command terminated with exit code 253 + sleep 0 + for i in $(seq 0 2) + set +e + kubectl exec psmdb-client-bb8b97679-gw8g4 -- bash -c 'mongo mongodb://clusterAdmin:clusterAdmin123456@34.118.226.232:27017/admin /tmp/disaster_recovery.js' + exit_status=253 + set -e + '[' 253 '!=' 0 -a -n 1 ']' + cat /tmp/tmp.3bbqLsQNSA Percona Server for MongoDB shell version v4.4.29-28 connecting to: mongodb://34.118.226.232:27017/admin?compressors=disabled&gssapiServiceName=mongodb Implicit session: session { "id" : UUID("bb0f979f-d1c6-46c6-a062-cbc2f2b30414") } Percona Server for MongoDB server version: v8.0.20-8 WARNING: shell and server versions do not match uncaught exception: TypeError: member3 is undefined : @/tmp/disaster_recovery.js:12:1 failed to load: /tmp/disaster_recovery.js exiting with code -3 + cat /tmp/tmp.yiDavDJN4h command terminated with exit code 253 + sleep 4 + for i in $(seq 0 2) + set +e + kubectl exec psmdb-client-bb8b97679-gw8g4 -- bash -c 'mongo mongodb://clusterAdmin:clusterAdmin123456@34.118.226.232:27017/admin /tmp/disaster_recovery.js' + exit_status=253 + set -e + '[' 253 '!=' 0 -a -n 1 ']' + cat /tmp/tmp.3bbqLsQNSA Percona Server for MongoDB shell version v4.4.29-28 connecting to: mongodb://34.118.226.232:27017/admin?compressors=disabled&gssapiServiceName=mongodb Implicit session: session { "id" : UUID("7730b325-fb0b-411b-9ec5-cf1b3832d2f5") } Percona Server for MongoDB server version: v8.0.20-8 WARNING: shell and server versions do not match uncaught exception: TypeError: member3 is undefined : @/tmp/disaster_recovery.js:12:1 failed to load: /tmp/disaster_recovery.js exiting with code -3 + cat /tmp/tmp.yiDavDJN4h command terminated with exit code 253 + sleep 8 + cat /tmp/tmp.3bbqLsQNSA Percona Server for MongoDB shell version v4.4.29-28 connecting to: mongodb://34.118.226.232:27017/admin?compressors=disabled&gssapiServiceName=mongodb Implicit session: session { "id" : UUID("7730b325-fb0b-411b-9ec5-cf1b3832d2f5") } Percona Server for MongoDB server version: v8.0.20-8 WARNING: shell and server versions do not match uncaught exception: TypeError: member3 is undefined : @/tmp/disaster_recovery.js:12:1 failed to load: /tmp/disaster_recovery.js exiting with code -3 + cat /tmp/tmp.yiDavDJN4h command terminated with exit code 253 + rm /tmp/tmp.3bbqLsQNSA /tmp/tmp.yiDavDJN4h + return 253