Log: /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/e2e-tests/logs/cross-site-sharded.log Warning: version difference between client (1.36) and server (1.33) exceeds the supported minor version skew of +/-1 Warning: version difference between client (1.36) and server (1.33) exceeds the supported minor version skew of +/-1 Warning: version difference between client (1.36) and server (1.33) exceeds the supported minor version skew of +/-1 ++ get_mongod_ver_from_image docker.io/perconalab/percona-server-mongodb-operator:main-mongod8.0 ++ local image=docker.io/perconalab/percona-server-mongodb-operator:main-mongod8.0 +++ run_simple_cli_inside_image docker.io/perconalab/percona-server-mongodb-operator:main-mongod8.0 'mongod --version' +++ /usr/sbin/sed -r 's/^.*db version v(([0-9]+\.){2}[0-9]+-[0-9]+).*$/\1/g' +++ local image=docker.io/perconalab/percona-server-mongodb-operator:main-mongod8.0 +++ local 'cli=mongod --version' +++ local pod_name=17788 +++ kubectl_bin -n default run 17788 --image=docker.io/perconalab/percona-server-mongodb-operator:main-mongod8.0 --restart=Never --command -- sleep infinity ++++ mktemp +++ local LAST_OUT=/tmp/tmp.qw0g7CkbXx ++++ mktemp +++ local LAST_ERR=/tmp/tmp.iVUiGe1y7R +++ local exit_status=0 +++ local timeout=4 ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl -n default run 17788 --image=docker.io/perconalab/percona-server-mongodb-operator:main-mongod8.0 --restart=Never --command -- sleep infinity +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 -a -n 1 ']' +++ break +++ cat /tmp/tmp.qw0g7CkbXx +++ cat /tmp/tmp.iVUiGe1y7R +++ rm /tmp/tmp.qw0g7CkbXx /tmp/tmp.iVUiGe1y7R +++ return 0 +++ kubectl_bin -n default wait --for=condition=Ready pod/17788 ++++ mktemp +++ local LAST_OUT=/tmp/tmp.CSP3tTiRDB ++++ mktemp +++ local LAST_ERR=/tmp/tmp.WMhPFDkjlh +++ local exit_status=0 +++ local timeout=4 ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl -n default wait --for=condition=Ready pod/17788 +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 -a -n 1 ']' +++ break +++ cat /tmp/tmp.CSP3tTiRDB +++ cat /tmp/tmp.WMhPFDkjlh +++ rm /tmp/tmp.CSP3tTiRDB /tmp/tmp.WMhPFDkjlh +++ return 0 ++++ kubectl_bin -n default exec 17788 -- bash -c 'mongod --version 2>&1' +++++ mktemp ++++ local LAST_OUT=/tmp/tmp.oboXXtufP7 +++++ mktemp ++++ local LAST_ERR=/tmp/tmp.nIJ5lCaudj ++++ local exit_status=0 ++++ local timeout=4 +++++ seq 0 2 ++++ for i in $(seq 0 2) ++++ set +e ++++ kubectl -n default exec 17788 -- bash -c 'mongod --version 2>&1' ++++ exit_status=0 ++++ set -e ++++ '[' 0 '!=' 0 -a -n 1 ']' ++++ break ++++ cat /tmp/tmp.oboXXtufP7 ++++ cat /tmp/tmp.nIJ5lCaudj ++++ rm /tmp/tmp.oboXXtufP7 /tmp/tmp.nIJ5lCaudj ++++ return 0 +++ local $'output=db version v8.0.23-10\nBuild Info: {\n "version": "8.0.23-10",\n "gitVersion": "10286e6c2a0ccd5d65c0b324c6fe29e5412db449",\n "openSSLVersion": "OpenSSL 3.5.5 27 Jan 2026",\n "modules": [],\n "perconaFeatures": [\n "MemoryEngine",\n "HotBackup",\n "BackupCursorAggregationStage",\n "BackupCursorExtendAggregationStage",\n "AWSIAM",\n "Kerberos",\n "LDAP",\n "OIDC",\n "TDE",\n "FIPSMode",\n "FCBIS",\n "Auditing",\n "ProfilingRateLimit",\n "LogRedaction",\n "ngram"\n ],\n "allocator": "tcmalloc-google",\n "environment": {\n "distarch": "x86_64",\n "target_arch": "x86_64"\n }\n}' +++ kubectl_bin -n default delete pod/17788 --grace-period=0 --force ++++ mktemp +++ local LAST_OUT=/tmp/tmp.WesEZfAg6o ++++ mktemp +++ local LAST_ERR=/tmp/tmp.KUMYhz6HfS +++ local exit_status=0 +++ local timeout=4 ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl -n default delete pod/17788 --grace-period=0 --force +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 -a -n 1 ']' +++ break +++ cat /tmp/tmp.WesEZfAg6o +++ cat /tmp/tmp.KUMYhz6HfS Warning: Immediate deletion does not wait for confirmation that the running resource has been terminated. The resource may continue to run on the cluster indefinitely. +++ rm /tmp/tmp.WesEZfAg6o /tmp/tmp.KUMYhz6HfS +++ return 0 +++ echo db version v8.0.23-10 Build Info: '{' '"version":' '"8.0.23-10",' '"gitVersion":' '"10286e6c2a0ccd5d65c0b324c6fe29e5412db449",' '"openSSLVersion":' '"OpenSSL' 3.5.5 27 Jan '2026",' '"modules":' '[],' '"perconaFeatures":' '[' '"MemoryEngine",' '"HotBackup",' '"BackupCursorAggregationStage",' '"BackupCursorExtendAggregationStage",' '"AWSIAM",' '"Kerberos",' '"LDAP",' '"OIDC",' '"TDE",' '"FIPSMode",' '"FCBIS",' '"Auditing",' '"ProfilingRateLimit",' '"LogRedaction",' '"ngram"' '],' '"allocator":' '"tcmalloc-google",' '"environment":' '{' '"distarch":' '"x86_64",' '"target_arch":' '"x86_64"' '}' '}' ++ version_info=8.0.23-10 ++ [[ ! 8.0.23-10 =~ ^([0-9]+\.){2}[0-9]+-[0-9]+$ ]] ++ echo 8.0.23-10 + FULL_VER=8.0.23-10 + MONGO_VER=8.0 + unset OPERATOR_NS + main_cluster=cross-site-sharded-main + replica_cluster=cross-site-sharded-replica + desc 'create main cluster' + set +o xtrace ----------------------------------------------------------------------------------- create main cluster ----------------------------------------------------------------------------------- + create_infra cross-site-sharded-5222 + local ns=cross-site-sharded-5222 + echo cross-site-sharded-5222 + [[ 0 == 1 ]] + '[' -n '' ']' + create_namespace cross-site-sharded-5222 + local namespace=cross-site-sharded-5222 + local skip_clean_namespace= + [[ 1 == 1 ]] + [[ -z '' ]] + destroy_chaos_mesh ++ helm list --all-namespaces --filter chaos-mesh ++ tail -n1 ++ awk '-F ' '{print $2}' ++ sed s/NAMESPACE// + local chaos_mesh_ns= + desc 'destroy chaos-mesh' + set +o xtrace ----------------------------------------------------------------------------------- destroy chaos-mesh ----------------------------------------------------------------------------------- + '[' -n '' ']' ++ kubectl get MutatingWebhookConfiguration ++ grep chaos-mesh ++ awk '{print $1}' + timeout 30 kubectl delete MutatingWebhookConfiguration error: resource(s) were provided, but no name was specified + : ++ kubectl get ValidatingWebhookConfiguration ++ grep chaos-mesh ++ awk '{print $1}' + timeout 30 kubectl delete ValidatingWebhookConfiguration error: resource(s) were provided, but no name was specified + : ++ kubectl get ValidatingWebhookConfiguration ++ grep validate-auth ++ awk '{print $1}' + timeout 30 kubectl delete ValidatingWebhookConfiguration error: resource(s) were provided, but no name was specified + : ++ kubectl api-resources ++ grep chaos-mesh ++ awk '{print $1}' ++ kubectl get crd ++ grep chaos-mesh.org ++ awk '{print $1}' + timeout 30 kubectl delete crd error: resource(s) were provided, but no name was specified + : ++ kubectl get clusterrolebinding ++ grep chaos-mesh ++ awk '{print $1}' + timeout 30 kubectl delete clusterrolebinding error: resource(s) were provided, but no name was specified + : ++ kubectl get clusterrole ++ grep chaos-mesh ++ awk '{print $1}' + timeout 30 kubectl delete clusterrole error: resource(s) were provided, but no name was specified + : + desc 'cleaned up all old namespaces' + set +o xtrace ----------------------------------------------------------------------------------- cleaned up all old namespaces ----------------------------------------------------------------------------------- + kubectl_bin get ns + grep -E -v '^kube-|^default|Terminating|psmdb-operator|openshift|^gke-|^gmp-|^NAME' + awk '{print$1}' + '[' -n '' ']' + desc 'cleaned up old namespaces cross-site-sharded-5222' + set +o xtrace ++ mktemp ----------------------------------------------------------------------------------- cleaned up old namespaces cross-site-sharded-5222 ----------------------------------------------------------------------------------- + kubectl_bin delete namespace cross-site-sharded-5222 --ignore-not-found + xargs kubectl delete ns ++ mktemp + local LAST_OUT=/tmp/tmp.1MRbfVshxz ++ mktemp + local LAST_OUT=/tmp/tmp.ScERBa8n53 ++ mktemp + local LAST_ERR=/tmp/tmp.FD5KAY9g0j + local exit_status=0 + local timeout=4 ++ seq 0 2 + local LAST_ERR=/tmp/tmp.GrwOdOI57x + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl get ns + for i in $(seq 0 2) + set +e + kubectl delete namespace cross-site-sharded-5222 --ignore-not-found + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.1MRbfVshxz + cat /tmp/tmp.FD5KAY9g0j + rm /tmp/tmp.1MRbfVshxz /tmp/tmp.FD5KAY9g0j + return 0 error: resource(s) were provided, but no name was specified + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.ScERBa8n53 + cat /tmp/tmp.GrwOdOI57x + rm /tmp/tmp.ScERBa8n53 /tmp/tmp.GrwOdOI57x + return 0 + kubectl_bin wait --for=delete namespace cross-site-sharded-5222 ++ mktemp + local LAST_OUT=/tmp/tmp.Gu9faisp9a ++ mktemp + local LAST_ERR=/tmp/tmp.cczgsxynRw + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl wait --for=delete namespace cross-site-sharded-5222 + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.Gu9faisp9a + cat /tmp/tmp.cczgsxynRw + rm /tmp/tmp.Gu9faisp9a /tmp/tmp.cczgsxynRw + return 0 + desc 'create namespace cross-site-sharded-5222' + set +o xtrace ----------------------------------------------------------------------------------- create namespace cross-site-sharded-5222 ----------------------------------------------------------------------------------- + kubectl_bin create namespace cross-site-sharded-5222 ++ mktemp + local LAST_OUT=/tmp/tmp.fkSm0dbaWK ++ mktemp + local LAST_ERR=/tmp/tmp.w10pGfl9OD + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl create namespace cross-site-sharded-5222 + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.fkSm0dbaWK namespace/cross-site-sharded-5222 created + cat /tmp/tmp.w10pGfl9OD + rm /tmp/tmp.fkSm0dbaWK /tmp/tmp.w10pGfl9OD + return 0 + set_kube_ctx cross-site-sharded-5222 + local namespace=cross-site-sharded-5222 ++ kubectl_bin config current-context +++ mktemp ++ local LAST_OUT=/tmp/tmp.yeX2bTlFfO +++ mktemp ++ local LAST_ERR=/tmp/tmp.kBj8AThp6e ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl config current-context ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.yeX2bTlFfO ++ cat /tmp/tmp.kBj8AThp6e ++ rm /tmp/tmp.yeX2bTlFfO /tmp/tmp.kBj8AThp6e ++ return 0 + kubectl_bin config set-context gke_cloud-dev-112233_us-central1-a_jen-psmdb-2058-fb8cc7187-17-cluster3 --namespace=cross-site-sharded-5222 ++ mktemp + local LAST_OUT=/tmp/tmp.yy5kwz7XJH ++ mktemp + local LAST_ERR=/tmp/tmp.Jh8ezC23FL + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl config set-context gke_cloud-dev-112233_us-central1-a_jen-psmdb-2058-fb8cc7187-17-cluster3 --namespace=cross-site-sharded-5222 + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.yy5kwz7XJH Context "gke_cloud-dev-112233_us-central1-a_jen-psmdb-2058-fb8cc7187-17-cluster3" modified. + cat /tmp/tmp.Jh8ezC23FL + rm /tmp/tmp.yy5kwz7XJH /tmp/tmp.Jh8ezC23FL + return 0 + deploy_operator + desc 'start PSMDB operator: docker.io/perconalab/percona-server-mongodb-operator:PR-2058-fb8cc7187' + set +o xtrace ----------------------------------------------------------------------------------- start PSMDB operator: docker.io/perconalab/percona-server-mongodb-operator:PR-2058-fb8cc7187 ----------------------------------------------------------------------------------- + local cr_file + '[' -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/e2e-tests/cross-site-sharded/conf/crd.yaml ']' + cr_file=/mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/deploy/crd.yaml + kubectl_bin apply --server-side --force-conflicts -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/deploy/crd.yaml ++ mktemp + local LAST_OUT=/tmp/tmp.eq5NfwVgpL ++ mktemp + local LAST_ERR=/tmp/tmp.RxAw10U01D + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl apply --server-side --force-conflicts -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/deploy/crd.yaml + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.eq5NfwVgpL customresourcedefinition.apiextensions.k8s.io/perconaservermongodbbackups.psmdb.percona.com serverside-applied customresourcedefinition.apiextensions.k8s.io/perconaservermongodbrestores.psmdb.percona.com serverside-applied customresourcedefinition.apiextensions.k8s.io/perconaservermongodbs.psmdb.percona.com serverside-applied + cat /tmp/tmp.RxAw10U01D + rm /tmp/tmp.eq5NfwVgpL /tmp/tmp.RxAw10U01D + return 0 + '[' -n '' ']' + apply_rbac rbac + local operator_namespace=psmdb-operator + local rbac=rbac + cat /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/deploy/rbac.yaml + sed -e 's^namespace: .*^namespace: psmdb-operator^' + kubectl_bin apply -f - ++ mktemp + local LAST_OUT=/tmp/tmp.J40yN5srdB ++ mktemp + local LAST_ERR=/tmp/tmp.c0wRFhb93i + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.J40yN5srdB role.rbac.authorization.k8s.io/percona-server-mongodb-operator created serviceaccount/percona-server-mongodb-operator created rolebinding.rbac.authorization.k8s.io/service-account-percona-server-mongodb-operator created + cat /tmp/tmp.c0wRFhb93i + rm /tmp/tmp.J40yN5srdB /tmp/tmp.c0wRFhb93i + return 0 + yq eval $'\n\t\t\t(.spec.template.spec.containers[].image = "docker.io/perconalab/percona-server-mongodb-operator:PR-2058-fb8cc7187") |\n\t\t\t((.. | select(.[] == "DISABLE_TELEMETRY")) |= .value="true") |\n\t\t\t((.. | select(.[] == "LOG_LEVEL")) |= .value="DEBUG")' /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/deploy/operator.yaml + kubectl_bin apply -f - ++ mktemp + local LAST_OUT=/tmp/tmp.ItObVq0LGf ++ mktemp + local LAST_ERR=/tmp/tmp.mRxa8iqL8Q + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.ItObVq0LGf deployment.apps/percona-server-mongodb-operator created + cat /tmp/tmp.mRxa8iqL8Q + rm /tmp/tmp.ItObVq0LGf /tmp/tmp.mRxa8iqL8Q + return 0 + sleep 20 ++ get_operator_pod ++ kubectl_bin get pods --selector=name=percona-server-mongodb-operator -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.JiRAGqcOzg +++ mktemp ++ local LAST_ERR=/tmp/tmp.2Dd8yp19NC ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pods --selector=name=percona-server-mongodb-operator -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.JiRAGqcOzg ++ cat /tmp/tmp.2Dd8yp19NC ++ rm /tmp/tmp.JiRAGqcOzg /tmp/tmp.2Dd8yp19NC ++ return 0 + wait_operator_pod percona-server-mongodb-operator-7b4b8bdd9d-jrx85 + local pod=percona-server-mongodb-operator-7b4b8bdd9d-jrx85 + set +o xtrace waiting for pod/percona-server-mongodb-operator-7b4b8bdd9d-jrx85 to be ready.OK + echo 'Print operator info from log' Print operator info from log + grep 'Manager starting up' ++ get_operator_pod ++ kubectl_bin get pods --selector=name=percona-server-mongodb-operator -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.xLehysEgDO +++ mktemp ++ local LAST_ERR=/tmp/tmp.gBk4jJ3Ky9 ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pods --selector=name=percona-server-mongodb-operator -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.xLehysEgDO ++ cat /tmp/tmp.gBk4jJ3Ky9 ++ rm /tmp/tmp.xLehysEgDO /tmp/tmp.gBk4jJ3Ky9 ++ return 0 + kubectl_bin logs percona-server-mongodb-operator-7b4b8bdd9d-jrx85 ++ mktemp + local LAST_OUT=/tmp/tmp.TauXYBmvLZ ++ mktemp + local LAST_ERR=/tmp/tmp.vB22Op0LMH + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl logs percona-server-mongodb-operator-7b4b8bdd9d-jrx85 + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.TauXYBmvLZ + cat /tmp/tmp.vB22Op0LMH + rm /tmp/tmp.TauXYBmvLZ /tmp/tmp.vB22Op0LMH + return 0 2026-06-09T07:35:14.912Z INFO setup Manager starting up {"gitCommit": "fb8cc718776dc8f9e011cd5f23fe48cf7f65143e", "gitBranch": "PR-2058-fb8cc7187", "buildTime": "", "goVersion": "go1.26.4", "os": "linux", "arch": "amd64"} + desc 'create secrets and start client' + set +o xtrace ----------------------------------------------------------------------------------- create secrets and start client ----------------------------------------------------------------------------------- + kubectl_bin apply -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/e2e-tests/conf/client.yml -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/e2e-tests/cross-site-sharded/conf/secrets.yml ++ mktemp + local LAST_OUT=/tmp/tmp.L4JGMToKuw ++ mktemp + local LAST_ERR=/tmp/tmp.Mzgdc7dJSs + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl apply -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/e2e-tests/conf/client.yml -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/e2e-tests/cross-site-sharded/conf/secrets.yml + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.L4JGMToKuw deployment.apps/psmdb-client created secret/cross-site-sharded-main-secrets created secret/cross-site-sharded-main-ssl created secret/cross-site-sharded-main-ssl-internal created + cat /tmp/tmp.Mzgdc7dJSs + rm /tmp/tmp.L4JGMToKuw /tmp/tmp.Mzgdc7dJSs + return 0 + desc 'create main PSMDB cluster cross-site-sharded-main.' + set +o xtrace ----------------------------------------------------------------------------------- create main PSMDB cluster cross-site-sharded-main. ----------------------------------------------------------------------------------- + apply_cluster /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/e2e-tests/cross-site-sharded/conf/cross-site-sharded-main.yml + '[' -z '' ']' + cat_config /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/e2e-tests/cross-site-sharded/conf/cross-site-sharded-main.yml + kubectl_bin apply -f - + cat /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/e2e-tests/cross-site-sharded/conf/cross-site-sharded-main.yml + yq eval '(.spec | select(.image == null)).image = "docker.io/perconalab/percona-server-mongodb-operator:main-mongod8.0"' ++ mktemp + yq eval '(.spec | select(has("pmm"))).pmm.image = "docker.io/percona/pmm-client:2.44.1-1"' + yq eval '(.spec | select(has("initImage"))).initImage = "docker.io/perconalab/percona-server-mongodb-operator:PR-2058-fb8cc7187"' + yq eval '(.spec | select(has("backup"))).backup.image = "docker.io/perconalab/percona-server-mongodb-operator:main-backup"' + /usr/sbin/sed -e s/NAME_SPACE/cross-site-sharded-5222/g + local LAST_OUT=/tmp/tmp.ZCdUhHe7dL + yq eval '.spec.upgradeOptions.apply="Never"' ++ mktemp + local LAST_ERR=/tmp/tmp.seiO1KmnPJ + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.ZCdUhHe7dL perconaservermongodb.psmdb.percona.com/cross-site-sharded-main created + cat /tmp/tmp.seiO1KmnPJ + rm /tmp/tmp.ZCdUhHe7dL /tmp/tmp.seiO1KmnPJ + return 0 + desc 'check if all 3 Pods started' + set +o xtrace ----------------------------------------------------------------------------------- check if all 3 Pods started ----------------------------------------------------------------------------------- + wait_for_running cross-site-sharded-main-rs0 3 + local name=cross-site-sharded-main-rs0 + let last_pod=2 + local check_cluster_readyness=true + set_debug + [[ 1 == 1 ]] + set -o xtrace + local rs_name=rs0 + local cluster_name=cross-site-sharded-main ++ seq 0 2 + for i in $(seq 0 $last_pod) + [[ 0 -eq 2 ]] + wait_pod cross-site-sharded-main-rs0-0 + local pod=cross-site-sharded-main-rs0-0 + set +o xtrace waiting for pod/cross-site-sharded-main-rs0-0 to be ready...................OK + for i in $(seq 0 $last_pod) + [[ 1 -eq 2 ]] + wait_pod cross-site-sharded-main-rs0-1 + local pod=cross-site-sharded-main-rs0-1 + set +o xtrace waiting for pod/cross-site-sharded-main-rs0-1 to be ready..............OK + for i in $(seq 0 $last_pod) + [[ 2 -eq 2 ]] ++ kubectl_bin get psmdb cross-site-sharded-main -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].arbiter.enabled}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.E0vRBMqnWP +++ mktemp ++ local LAST_ERR=/tmp/tmp.jR1lTntaDH ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb cross-site-sharded-main -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].arbiter.enabled}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.E0vRBMqnWP ++ cat /tmp/tmp.jR1lTntaDH ++ rm /tmp/tmp.E0vRBMqnWP /tmp/tmp.jR1lTntaDH ++ return 0 + [[ false == true ]] + wait_pod cross-site-sharded-main-rs0-2 + local pod=cross-site-sharded-main-rs0-2 + set +o xtrace waiting for pod/cross-site-sharded-main-rs0-2 to be ready...............OK ++ kubectl_bin get psmdb cross-site-sharded-main -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].nonvoting.enabled}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.tSzJGZwRGz +++ mktemp ++ local LAST_ERR=/tmp/tmp.vA7RENpIbw ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb cross-site-sharded-main -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].nonvoting.enabled}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.tSzJGZwRGz ++ cat /tmp/tmp.vA7RENpIbw ++ rm /tmp/tmp.tSzJGZwRGz /tmp/tmp.vA7RENpIbw ++ return 0 + [[ false == true ]] ++ kubectl_bin get psmdb cross-site-sharded-main -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].hidden.enabled}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.WevryP4GVa +++ mktemp ++ local LAST_ERR=/tmp/tmp.wv4HSAfZJU ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb cross-site-sharded-main -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].hidden.enabled}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.WevryP4GVa ++ cat /tmp/tmp.wv4HSAfZJU ++ rm /tmp/tmp.WevryP4GVa /tmp/tmp.wv4HSAfZJU ++ return 0 + [[ '' == true ]] + sleep 10 + [[ true == true ]] + set +x Waiting for cluster readyness.................................................... + wait_for_running cross-site-sharded-main-cfg 3 false + local name=cross-site-sharded-main-cfg + let last_pod=2 + local check_cluster_readyness=false + set_debug + [[ 1 == 1 ]] + set -o xtrace + local rs_name=cfg + local cluster_name=cross-site-sharded-main ++ seq 0 2 + for i in $(seq 0 $last_pod) + [[ 0 -eq 2 ]] + wait_pod cross-site-sharded-main-cfg-0 + local pod=cross-site-sharded-main-cfg-0 + set +o xtrace waiting for pod/cross-site-sharded-main-cfg-0 to be ready.OK + for i in $(seq 0 $last_pod) + [[ 1 -eq 2 ]] + wait_pod cross-site-sharded-main-cfg-1 + local pod=cross-site-sharded-main-cfg-1 + set +o xtrace waiting for pod/cross-site-sharded-main-cfg-1 to be ready.OK + for i in $(seq 0 $last_pod) + [[ 2 -eq 2 ]] ++ kubectl_bin get psmdb cross-site-sharded-main -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].arbiter.enabled}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.Ywn533JygW +++ mktemp ++ local LAST_ERR=/tmp/tmp.w6HAQIZD9Z ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb cross-site-sharded-main -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].arbiter.enabled}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.Ywn533JygW ++ cat /tmp/tmp.w6HAQIZD9Z ++ rm /tmp/tmp.Ywn533JygW /tmp/tmp.w6HAQIZD9Z ++ return 0 + [[ '' == true ]] + wait_pod cross-site-sharded-main-cfg-2 + local pod=cross-site-sharded-main-cfg-2 + set +o xtrace waiting for pod/cross-site-sharded-main-cfg-2 to be ready.OK ++ kubectl_bin get psmdb cross-site-sharded-main -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].nonvoting.enabled}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.voLwCka4e0 +++ mktemp ++ local LAST_ERR=/tmp/tmp.rXWRbE2QGv ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb cross-site-sharded-main -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].nonvoting.enabled}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.voLwCka4e0 ++ cat /tmp/tmp.rXWRbE2QGv ++ rm /tmp/tmp.voLwCka4e0 /tmp/tmp.rXWRbE2QGv ++ return 0 + [[ '' == true ]] ++ kubectl_bin get psmdb cross-site-sharded-main -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].hidden.enabled}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.xQ5h84Vihd +++ mktemp ++ local LAST_ERR=/tmp/tmp.cQAsFU6BRm ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb cross-site-sharded-main -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].hidden.enabled}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.xQ5h84Vihd ++ cat /tmp/tmp.cQAsFU6BRm ++ rm /tmp/tmp.xQ5h84Vihd /tmp/tmp.cQAsFU6BRm ++ return 0 + [[ '' == true ]] + sleep 10 + [[ false == true ]] + desc 'create user' + set +o xtrace ----------------------------------------------------------------------------------- create user ----------------------------------------------------------------------------------- + run_mongos 'db.createUser({user:"user",pwd:"pass",roles:[{db:"app",role:"readWrite"}]})' userAdmin:userAdmin123456@cross-site-sharded-main-mongos.cross-site-sharded-5222 + local 'command=db.createUser({user:"user",pwd:"pass",roles:[{db:"app",role:"readWrite"}]})' + local uri=userAdmin:userAdmin123456@cross-site-sharded-main-mongos.cross-site-sharded-5222 + local driver=mongodb + local suffix=.svc.cluster.local + local mongo_flag= + local port=27017 + local mongo_bin=mongo ++ echo .svc.cluster.local ++ awk -F: '{print $2}' + suffix_port= + [[ -z '' ]] + suffix=.svc.cluster.local:27017 ++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.DGf1gVCT1Q +++ mktemp ++ local LAST_ERR=/tmp/tmp.MBr6ryQbrk ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.DGf1gVCT1Q ++ cat /tmp/tmp.MBr6ryQbrk ++ rm /tmp/tmp.DGf1gVCT1Q /tmp/tmp.MBr6ryQbrk ++ return 0 + local client_container=psmdb-client-bb8b97679-hr2qb + kubectl_bin exec psmdb-client-bb8b97679-hr2qb -- bash -c 'printf '\''db.createUser({user:"user",pwd:"pass",roles:[{db:"app",role:"readWrite"}]})\n'\'' | mongo mongodb://userAdmin:userAdmin123456@cross-site-sharded-main-mongos.cross-site-sharded-5222.svc.cluster.local:27017/admin ' ++ mktemp + local LAST_OUT=/tmp/tmp.asU9flRXu2 ++ mktemp + local LAST_ERR=/tmp/tmp.B2TwypmNSi + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl exec psmdb-client-bb8b97679-hr2qb -- bash -c 'printf '\''db.createUser({user:"user",pwd:"pass",roles:[{db:"app",role:"readWrite"}]})\n'\'' | mongo mongodb://userAdmin:userAdmin123456@cross-site-sharded-main-mongos.cross-site-sharded-5222.svc.cluster.local:27017/admin ' + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.asU9flRXu2 Percona Server for MongoDB shell version v4.4.29-28 connecting to: mongodb://cross-site-sharded-main-mongos.cross-site-sharded-5222.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb Implicit session: session { "id" : UUID("036944be-3e08-4a57-96f9-cbabefedceb6") } Percona Server for MongoDB server version: v8.0.23-10 WARNING: shell and server versions do not match Successfully added user: { "user" : "user", "roles" : [ { "db" : "app", "role" : "readWrite" } ] } bye + cat /tmp/tmp.B2TwypmNSi + rm /tmp/tmp.asU9flRXu2 /tmp/tmp.B2TwypmNSi + return 0 + sleep 2 + desc 'set chunk size to 2 MB' + set +o xtrace ----------------------------------------------------------------------------------- set chunk size to 2 MB ----------------------------------------------------------------------------------- + run_mongos 'use config\n db.settings.save( { _id:"chunksize", value: 2 } )' clusterAdmin:clusterAdmin123456@cross-site-sharded-main-mongos.cross-site-sharded-5222 + local 'command=use config\n db.settings.save( { _id:"chunksize", value: 2 } )' + local uri=clusterAdmin:clusterAdmin123456@cross-site-sharded-main-mongos.cross-site-sharded-5222 + local driver=mongodb + local suffix=.svc.cluster.local + local mongo_flag= + local port=27017 + local mongo_bin=mongo ++ echo .svc.cluster.local ++ awk -F: '{print $2}' + suffix_port= + [[ -z '' ]] + suffix=.svc.cluster.local:27017 ++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.0NUpNAYasl +++ mktemp ++ local LAST_ERR=/tmp/tmp.0m0hx6FDBJ ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.0NUpNAYasl ++ cat /tmp/tmp.0m0hx6FDBJ ++ rm /tmp/tmp.0NUpNAYasl /tmp/tmp.0m0hx6FDBJ ++ return 0 + local client_container=psmdb-client-bb8b97679-hr2qb + kubectl_bin exec psmdb-client-bb8b97679-hr2qb -- bash -c 'printf '\''use config\n db.settings.save( { _id:"chunksize", value: 2 } )\n'\'' | mongo mongodb://clusterAdmin:clusterAdmin123456@cross-site-sharded-main-mongos.cross-site-sharded-5222.svc.cluster.local:27017/admin ' ++ mktemp + local LAST_OUT=/tmp/tmp.49UI2r1bpW ++ mktemp + local LAST_ERR=/tmp/tmp.prjsHqHjAc + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl exec psmdb-client-bb8b97679-hr2qb -- bash -c 'printf '\''use config\n db.settings.save( { _id:"chunksize", value: 2 } )\n'\'' | mongo mongodb://clusterAdmin:clusterAdmin123456@cross-site-sharded-main-mongos.cross-site-sharded-5222.svc.cluster.local:27017/admin ' + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.49UI2r1bpW Percona Server for MongoDB shell version v4.4.29-28 connecting to: mongodb://cross-site-sharded-main-mongos.cross-site-sharded-5222.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb Implicit session: session { "id" : UUID("5edee9dd-cae3-4057-b6db-e3b5a56fa61a") } Percona Server for MongoDB server version: v8.0.23-10 WARNING: shell and server versions do not match switched to db config WriteResult({ "nMatched" : 0, "nUpserted" : 1, "nModified" : 0, "_id" : "chunksize" }) bye + cat /tmp/tmp.prjsHqHjAc + rm /tmp/tmp.49UI2r1bpW /tmp/tmp.prjsHqHjAc + return 0 + sleep 2 + desc 'write data' + set +o xtrace ----------------------------------------------------------------------------------- write data ----------------------------------------------------------------------------------- + run_script_mongos /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/e2e-tests/cross-site-sharded/data.js user:pass@cross-site-sharded-main-mongos.cross-site-sharded-5222 + local script=/mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/e2e-tests/cross-site-sharded/data.js + local uri=user:pass@cross-site-sharded-main-mongos.cross-site-sharded-5222 + local driver=mongodb + local suffix=.svc.cluster.local + local mongo_flag= + local mongo_bin=mongo ++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.NheRxCBIJ4 +++ mktemp ++ local LAST_ERR=/tmp/tmp.nZZR5R7SYq ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.NheRxCBIJ4 ++ cat /tmp/tmp.nZZR5R7SYq ++ rm /tmp/tmp.NheRxCBIJ4 /tmp/tmp.nZZR5R7SYq ++ return 0 + local client_container=psmdb-client-bb8b97679-hr2qb ++ basename /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/e2e-tests/cross-site-sharded/data.js + name=data.js + kubectl_bin cp /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/e2e-tests/cross-site-sharded/data.js cross-site-sharded-5222/psmdb-client-bb8b97679-hr2qb:/tmp ++ mktemp + local LAST_OUT=/tmp/tmp.pgFFVxbuWa ++ mktemp + local LAST_ERR=/tmp/tmp.xDGl82QtcO + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl cp /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/e2e-tests/cross-site-sharded/data.js cross-site-sharded-5222/psmdb-client-bb8b97679-hr2qb:/tmp + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.pgFFVxbuWa + cat /tmp/tmp.xDGl82QtcO + rm /tmp/tmp.pgFFVxbuWa /tmp/tmp.xDGl82QtcO + return 0 + kubectl_bin exec psmdb-client-bb8b97679-hr2qb -- bash -c 'mongo mongodb://user:pass@cross-site-sharded-main-mongos.cross-site-sharded-5222.svc.cluster.local/admin /tmp/data.js' ++ mktemp + local LAST_OUT=/tmp/tmp.RHiE8vFEnV ++ mktemp + local LAST_ERR=/tmp/tmp.MF4NuGw8Ia + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl exec psmdb-client-bb8b97679-hr2qb -- bash -c 'mongo mongodb://user:pass@cross-site-sharded-main-mongos.cross-site-sharded-5222.svc.cluster.local/admin /tmp/data.js' + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.RHiE8vFEnV Percona Server for MongoDB shell version v4.4.29-28 connecting to: mongodb://cross-site-sharded-main-mongos.cross-site-sharded-5222.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb Implicit session: session { "id" : UUID("ca9b96a3-78cf-4481-b868-1f4847435332") } Percona Server for MongoDB server version: v8.0.23-10 WARNING: shell and server versions do not match + cat /tmp/tmp.MF4NuGw8Ia + rm /tmp/tmp.RHiE8vFEnV /tmp/tmp.MF4NuGw8Ia + return 0 + desc 'shard collection' + set +o xtrace ----------------------------------------------------------------------------------- shard collection ----------------------------------------------------------------------------------- + run_mongos 'sh.enableSharding("app")' clusterAdmin:clusterAdmin123456@cross-site-sharded-main-mongos.cross-site-sharded-5222 + local 'command=sh.enableSharding("app")' + local uri=clusterAdmin:clusterAdmin123456@cross-site-sharded-main-mongos.cross-site-sharded-5222 + local driver=mongodb + local suffix=.svc.cluster.local + local mongo_flag= + local port=27017 + local mongo_bin=mongo ++ echo .svc.cluster.local ++ awk -F: '{print $2}' + suffix_port= + [[ -z '' ]] + suffix=.svc.cluster.local:27017 ++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.K2NkV3iJHI +++ mktemp ++ local LAST_ERR=/tmp/tmp.uuXIN0PrNI ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.K2NkV3iJHI ++ cat /tmp/tmp.uuXIN0PrNI ++ rm /tmp/tmp.K2NkV3iJHI /tmp/tmp.uuXIN0PrNI ++ return 0 + local client_container=psmdb-client-bb8b97679-hr2qb + kubectl_bin exec psmdb-client-bb8b97679-hr2qb -- bash -c 'printf '\''sh.enableSharding("app")\n'\'' | mongo mongodb://clusterAdmin:clusterAdmin123456@cross-site-sharded-main-mongos.cross-site-sharded-5222.svc.cluster.local:27017/admin ' ++ mktemp + local LAST_OUT=/tmp/tmp.SIif7k8X2Y ++ mktemp + local LAST_ERR=/tmp/tmp.UfwnFC3ClP + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl exec psmdb-client-bb8b97679-hr2qb -- bash -c 'printf '\''sh.enableSharding("app")\n'\'' | mongo mongodb://clusterAdmin:clusterAdmin123456@cross-site-sharded-main-mongos.cross-site-sharded-5222.svc.cluster.local:27017/admin ' + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.SIif7k8X2Y Percona Server for MongoDB shell version v4.4.29-28 connecting to: mongodb://cross-site-sharded-main-mongos.cross-site-sharded-5222.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb Implicit session: session { "id" : UUID("8c286b51-5f54-4f92-b505-c0251535b41c") } Percona Server for MongoDB server version: v8.0.23-10 WARNING: shell and server versions do not match { "ok" : 1, "$clusterTime" : { "clusterTime" : Timestamp(1780990825, 3), "signature" : { "hash" : BinData(0,"47NHMFynZMdwCJpgbnhSQRWcwBk="), "keyId" : NumberLong("7649296523217338372") } }, "operationTime" : Timestamp(1780990825, 3) } bye + cat /tmp/tmp.UfwnFC3ClP + rm /tmp/tmp.SIif7k8X2Y /tmp/tmp.UfwnFC3ClP + return 0 + sleep 2 + run_mongos 'sh.shardCollection("app.city", { _id: 1 } )' clusterAdmin:clusterAdmin123456@cross-site-sharded-main-mongos.cross-site-sharded-5222 + local 'command=sh.shardCollection("app.city", { _id: 1 } )' + local uri=clusterAdmin:clusterAdmin123456@cross-site-sharded-main-mongos.cross-site-sharded-5222 + local driver=mongodb + local suffix=.svc.cluster.local + local mongo_flag= + local port=27017 + local mongo_bin=mongo ++ echo .svc.cluster.local ++ awk -F: '{print $2}' + suffix_port= + [[ -z '' ]] + suffix=.svc.cluster.local:27017 ++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.O7vcecJTuL +++ mktemp ++ local LAST_ERR=/tmp/tmp.bF2XLy6Rcu ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.O7vcecJTuL ++ cat /tmp/tmp.bF2XLy6Rcu ++ rm /tmp/tmp.O7vcecJTuL /tmp/tmp.bF2XLy6Rcu ++ return 0 + local client_container=psmdb-client-bb8b97679-hr2qb + kubectl_bin exec psmdb-client-bb8b97679-hr2qb -- bash -c 'printf '\''sh.shardCollection("app.city", { _id: 1 } )\n'\'' | mongo mongodb://clusterAdmin:clusterAdmin123456@cross-site-sharded-main-mongos.cross-site-sharded-5222.svc.cluster.local:27017/admin ' ++ mktemp + local LAST_OUT=/tmp/tmp.fMcACT6NrG ++ mktemp + local LAST_ERR=/tmp/tmp.ufcXHXTVre + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl exec psmdb-client-bb8b97679-hr2qb -- bash -c 'printf '\''sh.shardCollection("app.city", { _id: 1 } )\n'\'' | mongo mongodb://clusterAdmin:clusterAdmin123456@cross-site-sharded-main-mongos.cross-site-sharded-5222.svc.cluster.local:27017/admin ' + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.fMcACT6NrG Percona Server for MongoDB shell version v4.4.29-28 connecting to: mongodb://cross-site-sharded-main-mongos.cross-site-sharded-5222.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb Implicit session: session { "id" : UUID("2987d5b1-b6d4-4afd-acad-a621a4d92dea") } Percona Server for MongoDB server version: v8.0.23-10 WARNING: shell and server versions do not match { "collectionsharded" : "app.city", "ok" : 1, "$clusterTime" : { "clusterTime" : Timestamp(1780990832, 42), "signature" : { "hash" : BinData(0,"ivBB8YVHfNVXZm8gekHagMI3v+k="), "keyId" : NumberLong("7649296523217338372") } }, "operationTime" : Timestamp(1780990832, 41) } bye + cat /tmp/tmp.ufcXHXTVre + rm /tmp/tmp.fMcACT6NrG /tmp/tmp.ufcXHXTVre + return 0 + sleep 120 + desc 'Check chunks' + set +o xtrace ----------------------------------------------------------------------------------- Check chunks ----------------------------------------------------------------------------------- + chunks_param1=ns + chunks_param2='"app.city"' + [[ 8.0 != 4\.4 ]] + chunks_param1=uuid ++ run_mongos 'use app\n db.getCollectionInfos({ "name": "city" })[0].info.uuid' user:pass@cross-site-sharded-main-mongos.cross-site-sharded-5222 ++ local 'command=use app\n db.getCollectionInfos({ "name": "city" })[0].info.uuid' ++ local uri=user:pass@cross-site-sharded-main-mongos.cross-site-sharded-5222 ++ grep 'switched to db app' -A 1 ++ local driver=mongodb ++ local suffix=.svc.cluster.local ++ local mongo_flag= ++ local port=27017 ++ local mongo_bin=mongo ++ grep -v 'switched to db app' +++ echo .svc.cluster.local +++ awk -F: '{print $2}' ++ suffix_port= ++ [[ -z '' ]] ++ suffix=.svc.cluster.local:27017 +++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.xfGo2fYVm4 ++++ mktemp +++ local LAST_ERR=/tmp/tmp.p2IBh5tbOj +++ local exit_status=0 +++ local timeout=4 ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 -a -n 1 ']' +++ break +++ cat /tmp/tmp.xfGo2fYVm4 +++ cat /tmp/tmp.p2IBh5tbOj +++ rm /tmp/tmp.xfGo2fYVm4 /tmp/tmp.p2IBh5tbOj +++ return 0 ++ local client_container=psmdb-client-bb8b97679-hr2qb ++ kubectl_bin exec psmdb-client-bb8b97679-hr2qb -- bash -c 'printf '\''use app\n db.getCollectionInfos({ "name": "city" })[0].info.uuid\n'\'' | mongo mongodb://user:pass@cross-site-sharded-main-mongos.cross-site-sharded-5222.svc.cluster.local:27017/admin ' +++ mktemp ++ local LAST_OUT=/tmp/tmp.Js6daGxVhI +++ mktemp ++ local LAST_ERR=/tmp/tmp.N7dYixTj3g ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl exec psmdb-client-bb8b97679-hr2qb -- bash -c 'printf '\''use app\n db.getCollectionInfos({ "name": "city" })[0].info.uuid\n'\'' | mongo mongodb://user:pass@cross-site-sharded-main-mongos.cross-site-sharded-5222.svc.cluster.local:27017/admin ' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.Js6daGxVhI ++ cat /tmp/tmp.N7dYixTj3g ++ rm /tmp/tmp.Js6daGxVhI /tmp/tmp.N7dYixTj3g ++ return 0 + chunks_param2='UUID("37879da5-8bad-4140-9789-331d0b9d5e1c")' + shards=0 + for i in "rs0" "rs1" ++ run_mongos 'use config\n db.chunks.count({"uuid": UUID("37879da5-8bad-4140-9789-331d0b9d5e1c"), "shard": "rs0"})' clusterAdmin:clusterAdmin123456@cross-site-sharded-main-mongos.cross-site-sharded-5222 ++ local 'command=use config\n db.chunks.count({"uuid": UUID("37879da5-8bad-4140-9789-331d0b9d5e1c"), "shard": "rs0"})' ++ local uri=clusterAdmin:clusterAdmin123456@cross-site-sharded-main-mongos.cross-site-sharded-5222 ++ local driver=mongodb ++ grep 'switched to db config' -A 1 ++ local suffix=.svc.cluster.local ++ local mongo_flag= ++ local port=27017 ++ local mongo_bin=mongo ++ grep -v 'switched to db config' +++ echo .svc.cluster.local +++ awk -F: '{print $2}' ++ suffix_port= ++ [[ -z '' ]] ++ suffix=.svc.cluster.local:27017 +++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.9ez8qnuJk3 ++++ mktemp +++ local LAST_ERR=/tmp/tmp.9BlZQTbWRb +++ local exit_status=0 +++ local timeout=4 ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 -a -n 1 ']' +++ break +++ cat /tmp/tmp.9ez8qnuJk3 +++ cat /tmp/tmp.9BlZQTbWRb +++ rm /tmp/tmp.9ez8qnuJk3 /tmp/tmp.9BlZQTbWRb +++ return 0 ++ local client_container=psmdb-client-bb8b97679-hr2qb ++ kubectl_bin exec psmdb-client-bb8b97679-hr2qb -- bash -c 'printf '\''use config\n db.chunks.count({"uuid": UUID("37879da5-8bad-4140-9789-331d0b9d5e1c"), "shard": "rs0"})\n'\'' | mongo mongodb://clusterAdmin:clusterAdmin123456@cross-site-sharded-main-mongos.cross-site-sharded-5222.svc.cluster.local:27017/admin ' +++ mktemp ++ local LAST_OUT=/tmp/tmp.F7bFLX0yg7 +++ mktemp ++ local LAST_ERR=/tmp/tmp.dvkdrQsrM5 ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl exec psmdb-client-bb8b97679-hr2qb -- bash -c 'printf '\''use config\n db.chunks.count({"uuid": UUID("37879da5-8bad-4140-9789-331d0b9d5e1c"), "shard": "rs0"})\n'\'' | mongo mongodb://clusterAdmin:clusterAdmin123456@cross-site-sharded-main-mongos.cross-site-sharded-5222.svc.cluster.local:27017/admin ' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.F7bFLX0yg7 ++ cat /tmp/tmp.dvkdrQsrM5 ++ rm /tmp/tmp.F7bFLX0yg7 /tmp/tmp.dvkdrQsrM5 ++ return 0 + out=3 + desc 'rs0 has 3 chunks' + set +o xtrace ----------------------------------------------------------------------------------- rs0 has 3 chunks ----------------------------------------------------------------------------------- + [[ 3 -ne 0 ]] + (( shards = shards + 1 )) + for i in "rs0" "rs1" ++ grep 'switched to db config' -A 1 ++ run_mongos 'use config\n db.chunks.count({"uuid": UUID("37879da5-8bad-4140-9789-331d0b9d5e1c"), "shard": "rs1"})' clusterAdmin:clusterAdmin123456@cross-site-sharded-main-mongos.cross-site-sharded-5222 ++ local 'command=use config\n db.chunks.count({"uuid": UUID("37879da5-8bad-4140-9789-331d0b9d5e1c"), "shard": "rs1"})' ++ local uri=clusterAdmin:clusterAdmin123456@cross-site-sharded-main-mongos.cross-site-sharded-5222 ++ local driver=mongodb ++ local suffix=.svc.cluster.local ++ local mongo_flag= ++ local port=27017 ++ local mongo_bin=mongo ++ grep -v 'switched to db config' +++ echo .svc.cluster.local +++ awk -F: '{print $2}' ++ suffix_port= ++ [[ -z '' ]] ++ suffix=.svc.cluster.local:27017 +++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.mGvTmAqDG6 ++++ mktemp +++ local LAST_ERR=/tmp/tmp.2seFcTTOwp +++ local exit_status=0 +++ local timeout=4 ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 -a -n 1 ']' +++ break +++ cat /tmp/tmp.mGvTmAqDG6 +++ cat /tmp/tmp.2seFcTTOwp +++ rm /tmp/tmp.mGvTmAqDG6 /tmp/tmp.2seFcTTOwp +++ return 0 ++ local client_container=psmdb-client-bb8b97679-hr2qb ++ kubectl_bin exec psmdb-client-bb8b97679-hr2qb -- bash -c 'printf '\''use config\n db.chunks.count({"uuid": UUID("37879da5-8bad-4140-9789-331d0b9d5e1c"), "shard": "rs1"})\n'\'' | mongo mongodb://clusterAdmin:clusterAdmin123456@cross-site-sharded-main-mongos.cross-site-sharded-5222.svc.cluster.local:27017/admin ' +++ mktemp ++ local LAST_OUT=/tmp/tmp.IHI4G5AAWz +++ mktemp ++ local LAST_ERR=/tmp/tmp.fKFdHnuqim ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl exec psmdb-client-bb8b97679-hr2qb -- bash -c 'printf '\''use config\n db.chunks.count({"uuid": UUID("37879da5-8bad-4140-9789-331d0b9d5e1c"), "shard": "rs1"})\n'\'' | mongo mongodb://clusterAdmin:clusterAdmin123456@cross-site-sharded-main-mongos.cross-site-sharded-5222.svc.cluster.local:27017/admin ' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.IHI4G5AAWz ++ cat /tmp/tmp.fKFdHnuqim ++ rm /tmp/tmp.IHI4G5AAWz /tmp/tmp.fKFdHnuqim ++ return 0 + out=1 + desc 'rs1 has 1 chunks' + set +o xtrace ----------------------------------------------------------------------------------- rs1 has 1 chunks ----------------------------------------------------------------------------------- + [[ 1 -ne 0 ]] + (( shards = shards + 1 )) + [[ 2 -lt 2 ]] + desc 'create replica cluster' + set +o xtrace ----------------------------------------------------------------------------------- create replica cluster ----------------------------------------------------------------------------------- + create_namespace cross-site-sharded-replica-2749 0 + local namespace=cross-site-sharded-replica-2749 + local skip_clean_namespace=0 + [[ 1 == 1 ]] + [[ -z 0 ]] + '[' -n '' ']' + desc 'cleaned up old namespaces cross-site-sharded-replica-2749' + set +o xtrace ----------------------------------------------------------------------------------- cleaned up old namespaces cross-site-sharded-replica-2749 ----------------------------------------------------------------------------------- + kubectl_bin delete namespace cross-site-sharded-replica-2749 --ignore-not-found ++ mktemp + local LAST_OUT=/tmp/tmp.QGRbDTPdQv ++ mktemp + local LAST_ERR=/tmp/tmp.V1ToKViCol + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl delete namespace cross-site-sharded-replica-2749 --ignore-not-found + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.QGRbDTPdQv + cat /tmp/tmp.V1ToKViCol + rm /tmp/tmp.QGRbDTPdQv /tmp/tmp.V1ToKViCol + return 0 + kubectl_bin wait --for=delete namespace cross-site-sharded-replica-2749 ++ mktemp + local LAST_OUT=/tmp/tmp.IxJtYShHD0 ++ mktemp + local LAST_ERR=/tmp/tmp.SK2BDPr8im + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl wait --for=delete namespace cross-site-sharded-replica-2749 + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.IxJtYShHD0 + cat /tmp/tmp.SK2BDPr8im + rm /tmp/tmp.IxJtYShHD0 /tmp/tmp.SK2BDPr8im + return 0 + desc 'create namespace cross-site-sharded-replica-2749' + set +o xtrace ----------------------------------------------------------------------------------- create namespace cross-site-sharded-replica-2749 ----------------------------------------------------------------------------------- + kubectl_bin create namespace cross-site-sharded-replica-2749 ++ mktemp + local LAST_OUT=/tmp/tmp.PyZVdZiAIS ++ mktemp + local LAST_ERR=/tmp/tmp.2cufXRcMoO + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl create namespace cross-site-sharded-replica-2749 + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.PyZVdZiAIS namespace/cross-site-sharded-replica-2749 created + cat /tmp/tmp.2cufXRcMoO + rm /tmp/tmp.PyZVdZiAIS /tmp/tmp.2cufXRcMoO + return 0 + set_kube_ctx cross-site-sharded-replica-2749 + local namespace=cross-site-sharded-replica-2749 ++ kubectl_bin config current-context +++ mktemp ++ local LAST_OUT=/tmp/tmp.Z67Hz5jnOr +++ mktemp ++ local LAST_ERR=/tmp/tmp.bk9crmQDbo ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl config current-context ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.Z67Hz5jnOr ++ cat /tmp/tmp.bk9crmQDbo ++ rm /tmp/tmp.Z67Hz5jnOr /tmp/tmp.bk9crmQDbo ++ return 0 + kubectl_bin config set-context gke_cloud-dev-112233_us-central1-a_jen-psmdb-2058-fb8cc7187-17-cluster3 --namespace=cross-site-sharded-replica-2749 ++ mktemp + local LAST_OUT=/tmp/tmp.yKw2xBTHYg ++ mktemp + local LAST_ERR=/tmp/tmp.NBfai6UH7h + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl config set-context gke_cloud-dev-112233_us-central1-a_jen-psmdb-2058-fb8cc7187-17-cluster3 --namespace=cross-site-sharded-replica-2749 + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.yKw2xBTHYg Context "gke_cloud-dev-112233_us-central1-a_jen-psmdb-2058-fb8cc7187-17-cluster3" modified. + cat /tmp/tmp.NBfai6UH7h + rm /tmp/tmp.yKw2xBTHYg /tmp/tmp.NBfai6UH7h + return 0 + deploy_operator + desc 'start PSMDB operator: docker.io/perconalab/percona-server-mongodb-operator:PR-2058-fb8cc7187' + set +o xtrace ----------------------------------------------------------------------------------- start PSMDB operator: docker.io/perconalab/percona-server-mongodb-operator:PR-2058-fb8cc7187 ----------------------------------------------------------------------------------- + local cr_file + '[' -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/e2e-tests/cross-site-sharded/conf/crd.yaml ']' + cr_file=/mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/deploy/crd.yaml + kubectl_bin apply --server-side --force-conflicts -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/deploy/crd.yaml ++ mktemp + local LAST_OUT=/tmp/tmp.6qtgDbjxBq ++ mktemp + local LAST_ERR=/tmp/tmp.1Oz0XOZpSk + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl apply --server-side --force-conflicts -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/deploy/crd.yaml + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.6qtgDbjxBq customresourcedefinition.apiextensions.k8s.io/perconaservermongodbbackups.psmdb.percona.com serverside-applied customresourcedefinition.apiextensions.k8s.io/perconaservermongodbrestores.psmdb.percona.com serverside-applied customresourcedefinition.apiextensions.k8s.io/perconaservermongodbs.psmdb.percona.com serverside-applied + cat /tmp/tmp.1Oz0XOZpSk + rm /tmp/tmp.6qtgDbjxBq /tmp/tmp.1Oz0XOZpSk + return 0 + '[' -n '' ']' + apply_rbac rbac + local operator_namespace=psmdb-operator + local rbac=rbac + cat /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/deploy/rbac.yaml + sed -e 's^namespace: .*^namespace: psmdb-operator^' + kubectl_bin apply -f - ++ mktemp + local LAST_OUT=/tmp/tmp.HUJOaodWpc ++ mktemp + local LAST_ERR=/tmp/tmp.HIg7SbekBU + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.HUJOaodWpc role.rbac.authorization.k8s.io/percona-server-mongodb-operator created serviceaccount/percona-server-mongodb-operator created rolebinding.rbac.authorization.k8s.io/service-account-percona-server-mongodb-operator created + cat /tmp/tmp.HIg7SbekBU + rm /tmp/tmp.HUJOaodWpc /tmp/tmp.HIg7SbekBU + return 0 + yq eval $'\n\t\t\t(.spec.template.spec.containers[].image = "docker.io/perconalab/percona-server-mongodb-operator:PR-2058-fb8cc7187") |\n\t\t\t((.. | select(.[] == "DISABLE_TELEMETRY")) |= .value="true") |\n\t\t\t((.. | select(.[] == "LOG_LEVEL")) |= .value="DEBUG")' /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/deploy/operator.yaml + kubectl_bin apply -f - ++ mktemp + local LAST_OUT=/tmp/tmp.B9bcn4wzED ++ mktemp + local LAST_ERR=/tmp/tmp.AbxSn2k7mY + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.B9bcn4wzED deployment.apps/percona-server-mongodb-operator created + cat /tmp/tmp.AbxSn2k7mY + rm /tmp/tmp.B9bcn4wzED /tmp/tmp.AbxSn2k7mY + return 0 + sleep 20 ++ get_operator_pod ++ kubectl_bin get pods --selector=name=percona-server-mongodb-operator -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.Vgoriw4SGM +++ mktemp ++ local LAST_ERR=/tmp/tmp.YXpgtmz9Qw ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pods --selector=name=percona-server-mongodb-operator -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.Vgoriw4SGM ++ cat /tmp/tmp.YXpgtmz9Qw ++ rm /tmp/tmp.Vgoriw4SGM /tmp/tmp.YXpgtmz9Qw ++ return 0 + wait_operator_pod percona-server-mongodb-operator-7b4b8bdd9d-gxss2 + local pod=percona-server-mongodb-operator-7b4b8bdd9d-gxss2 + set +o xtrace waiting for pod/percona-server-mongodb-operator-7b4b8bdd9d-gxss2 to be ready.OK + echo 'Print operator info from log' Print operator info from log + grep 'Manager starting up' ++ get_operator_pod ++ kubectl_bin get pods --selector=name=percona-server-mongodb-operator -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.fgxUZb6c4x +++ mktemp ++ local LAST_ERR=/tmp/tmp.ZpBSIbJ62D ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pods --selector=name=percona-server-mongodb-operator -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.fgxUZb6c4x ++ cat /tmp/tmp.ZpBSIbJ62D ++ rm /tmp/tmp.fgxUZb6c4x /tmp/tmp.ZpBSIbJ62D ++ return 0 + kubectl_bin logs percona-server-mongodb-operator-7b4b8bdd9d-gxss2 ++ mktemp + local LAST_OUT=/tmp/tmp.x8djVtWMkD ++ mktemp + local LAST_ERR=/tmp/tmp.T1nz7ccufD + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl logs percona-server-mongodb-operator-7b4b8bdd9d-gxss2 + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.x8djVtWMkD + cat /tmp/tmp.T1nz7ccufD + rm /tmp/tmp.x8djVtWMkD /tmp/tmp.T1nz7ccufD + return 0 2026-06-09T07:42:55.124Z INFO setup Manager starting up {"gitCommit": "fb8cc718776dc8f9e011cd5f23fe48cf7f65143e", "gitBranch": "PR-2058-fb8cc7187", "buildTime": "", "goVersion": "go1.26.4", "os": "linux", "arch": "amd64"} + desc 'start client' + set +o xtrace ----------------------------------------------------------------------------------- start client ----------------------------------------------------------------------------------- + kubectl_bin apply -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/e2e-tests/conf/client.yml ++ mktemp + local LAST_OUT=/tmp/tmp.FeQdbkW6ft ++ mktemp + local LAST_ERR=/tmp/tmp.d8LUWTM03M + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl apply -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/e2e-tests/conf/client.yml + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.FeQdbkW6ft deployment.apps/psmdb-client created + cat /tmp/tmp.d8LUWTM03M + rm /tmp/tmp.FeQdbkW6ft /tmp/tmp.d8LUWTM03M + return 0 + desc 'copy secrets from main to replica namespace and create all of them' + set +o xtrace ----------------------------------------------------------------------------------- copy secrets from main to replica namespace and create all of them ----------------------------------------------------------------------------------- + kubectl get secret cross-site-sharded-main-secrets -o yaml -n cross-site-sharded-5222 + yq eval $'\n\t\tdel(.metadata) |\n\t\t(.metadata.name = "cross-site-sharded-replica-secrets")' - + kubectl_bin apply -f - ++ mktemp + local LAST_OUT=/tmp/tmp.OFToIwVbyK ++ mktemp + local LAST_ERR=/tmp/tmp.0ddmRZES6a + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.OFToIwVbyK secret/cross-site-sharded-replica-secrets created + cat /tmp/tmp.0ddmRZES6a + rm /tmp/tmp.OFToIwVbyK /tmp/tmp.0ddmRZES6a + return 0 + kubectl_bin get secret cross-site-sharded-main-ssl-internal -o yaml -n cross-site-sharded-5222 + yq eval $'\n\t\tdel(.metadata) |\n\t\tdel(.status) |\n\t\t(.metadata.name = "cross-site-sharded-replica-ssl-internal")' - + kubectl_bin apply -f - ++ mktemp ++ mktemp + local LAST_OUT=/tmp/tmp.Kh9gl3CeKS + local LAST_OUT=/tmp/tmp.QzZsDr1ZNg ++ mktemp ++ mktemp + local LAST_ERR=/tmp/tmp.MEJatKZp5O + local exit_status=0 + local timeout=4 + local LAST_ERR=/tmp/tmp.rHCthcr35d + local exit_status=0 + local timeout=4 ++ seq 0 2 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl get secret cross-site-sharded-main-ssl-internal -o yaml -n cross-site-sharded-5222 + for i in $(seq 0 2) + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.Kh9gl3CeKS + cat /tmp/tmp.MEJatKZp5O + rm /tmp/tmp.Kh9gl3CeKS /tmp/tmp.MEJatKZp5O + return 0 + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.QzZsDr1ZNg secret/cross-site-sharded-replica-ssl-internal created + cat /tmp/tmp.rHCthcr35d + rm /tmp/tmp.QzZsDr1ZNg /tmp/tmp.rHCthcr35d + return 0 + kubectl_bin get secret cross-site-sharded-main-ssl -o yaml -n cross-site-sharded-5222 + yq eval $'\n\t\tdel(.metadata) |\n\t\tdel(.status) |\n\t\t(.metadata.name = "cross-site-sharded-replica-ssl")' - ++ mktemp + kubectl_bin apply -f - ++ mktemp + local LAST_OUT=/tmp/tmp.dV7tPXKnPH ++ mktemp + local LAST_OUT=/tmp/tmp.bADPXDmD1Q + local LAST_ERR=/tmp/tmp.NbNWTUFOqc + local exit_status=0 + local timeout=4 ++ mktemp ++ seq 0 2 + for i in $(seq 0 2) + local LAST_ERR=/tmp/tmp.UandqRvmjs + set +e + kubectl get secret cross-site-sharded-main-ssl -o yaml -n cross-site-sharded-5222 + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.dV7tPXKnPH + cat /tmp/tmp.NbNWTUFOqc + rm /tmp/tmp.dV7tPXKnPH /tmp/tmp.NbNWTUFOqc + return 0 + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.bADPXDmD1Q secret/cross-site-sharded-replica-ssl created + cat /tmp/tmp.UandqRvmjs + rm /tmp/tmp.bADPXDmD1Q /tmp/tmp.UandqRvmjs + return 0 + sleep 30 + desc 'create replica PSMDB cluster ' + set +o xtrace ----------------------------------------------------------------------------------- create replica PSMDB cluster ----------------------------------------------------------------------------------- + apply_cluster /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/e2e-tests/cross-site-sharded/conf/cross-site-sharded-replica.yml + '[' -z '' ']' + cat_config /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/e2e-tests/cross-site-sharded/conf/cross-site-sharded-replica.yml + kubectl_bin apply -f - + cat /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/e2e-tests/cross-site-sharded/conf/cross-site-sharded-replica.yml + yq eval '(.spec | select(.image == null)).image = "docker.io/perconalab/percona-server-mongodb-operator:main-mongod8.0"' ++ mktemp + yq eval '(.spec | select(has("pmm"))).pmm.image = "docker.io/percona/pmm-client:2.44.1-1"' + yq eval '(.spec | select(has("initImage"))).initImage = "docker.io/perconalab/percona-server-mongodb-operator:PR-2058-fb8cc7187"' + local LAST_OUT=/tmp/tmp.9LD9K42B4J ++ mktemp + /usr/sbin/sed -e s/NAME_SPACE/cross-site-sharded-5222/g + yq eval '(.spec | select(has("backup"))).backup.image = "docker.io/perconalab/percona-server-mongodb-operator:main-backup"' + yq eval '.spec.upgradeOptions.apply="Never"' + local LAST_ERR=/tmp/tmp.ycEX6ykUty + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.9LD9K42B4J perconaservermongodb.psmdb.percona.com/cross-site-sharded-replica created + cat /tmp/tmp.ycEX6ykUty + rm /tmp/tmp.9LD9K42B4J /tmp/tmp.ycEX6ykUty + return 0 + wait_for_running cross-site-sharded-replica-rs0 3 false + local name=cross-site-sharded-replica-rs0 + let last_pod=2 + local check_cluster_readyness=false + set_debug + [[ 1 == 1 ]] + set -o xtrace + local rs_name=rs0 + local cluster_name=cross-site-sharded-replica ++ seq 0 2 + for i in $(seq 0 $last_pod) + [[ 0 -eq 2 ]] + wait_pod cross-site-sharded-replica-rs0-0 + local pod=cross-site-sharded-replica-rs0-0 + set +o xtrace waiting for pod/cross-site-sharded-replica-rs0-0 to be ready.............OK + for i in $(seq 0 $last_pod) + [[ 1 -eq 2 ]] + wait_pod cross-site-sharded-replica-rs0-1 + local pod=cross-site-sharded-replica-rs0-1 + set +o xtrace waiting for pod/cross-site-sharded-replica-rs0-1 to be ready.............OK + for i in $(seq 0 $last_pod) + [[ 2 -eq 2 ]] ++ kubectl_bin get psmdb cross-site-sharded-replica -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].arbiter.enabled}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.wz4P75DT2w +++ mktemp ++ local LAST_ERR=/tmp/tmp.aMlDnIu9ib ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb cross-site-sharded-replica -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].arbiter.enabled}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.wz4P75DT2w ++ cat /tmp/tmp.aMlDnIu9ib ++ rm /tmp/tmp.wz4P75DT2w /tmp/tmp.aMlDnIu9ib ++ return 0 + [[ false == true ]] + wait_pod cross-site-sharded-replica-rs0-2 + local pod=cross-site-sharded-replica-rs0-2 + set +o xtrace waiting for pod/cross-site-sharded-replica-rs0-2 to be ready.............OK ++ kubectl_bin get psmdb cross-site-sharded-replica -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].nonvoting.enabled}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.aej4KqTOCa +++ mktemp ++ local LAST_ERR=/tmp/tmp.JWi1GoEwcL ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb cross-site-sharded-replica -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].nonvoting.enabled}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.aej4KqTOCa ++ cat /tmp/tmp.JWi1GoEwcL ++ rm /tmp/tmp.aej4KqTOCa /tmp/tmp.JWi1GoEwcL ++ return 0 + [[ false == true ]] ++ kubectl_bin get psmdb cross-site-sharded-replica -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].hidden.enabled}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.8vuXVYstDT +++ mktemp ++ local LAST_ERR=/tmp/tmp.oGWdYQpS4P ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb cross-site-sharded-replica -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].hidden.enabled}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.8vuXVYstDT ++ cat /tmp/tmp.oGWdYQpS4P ++ rm /tmp/tmp.8vuXVYstDT /tmp/tmp.oGWdYQpS4P ++ return 0 + [[ '' == true ]] + sleep 10 + [[ false == true ]] + wait_for_running cross-site-sharded-replica-rs1 3 false + local name=cross-site-sharded-replica-rs1 + let last_pod=2 + local check_cluster_readyness=false + set_debug + [[ 1 == 1 ]] + set -o xtrace + local rs_name=rs1 + local cluster_name=cross-site-sharded-replica ++ seq 0 2 + for i in $(seq 0 $last_pod) + [[ 0 -eq 2 ]] + wait_pod cross-site-sharded-replica-rs1-0 + local pod=cross-site-sharded-replica-rs1-0 + set +o xtrace waiting for pod/cross-site-sharded-replica-rs1-0 to be ready.OK + for i in $(seq 0 $last_pod) + [[ 1 -eq 2 ]] + wait_pod cross-site-sharded-replica-rs1-1 + local pod=cross-site-sharded-replica-rs1-1 + set +o xtrace waiting for pod/cross-site-sharded-replica-rs1-1 to be ready.OK + for i in $(seq 0 $last_pod) + [[ 2 -eq 2 ]] ++ kubectl_bin get psmdb cross-site-sharded-replica -o 'jsonpath={.spec.replsets[?(@.name=="rs1")].arbiter.enabled}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.LL0gHlj9H3 +++ mktemp ++ local LAST_ERR=/tmp/tmp.EluGBzXkSo ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb cross-site-sharded-replica -o 'jsonpath={.spec.replsets[?(@.name=="rs1")].arbiter.enabled}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.LL0gHlj9H3 ++ cat /tmp/tmp.EluGBzXkSo ++ rm /tmp/tmp.LL0gHlj9H3 /tmp/tmp.EluGBzXkSo ++ return 0 + [[ false == true ]] + wait_pod cross-site-sharded-replica-rs1-2 + local pod=cross-site-sharded-replica-rs1-2 + set +o xtrace waiting for pod/cross-site-sharded-replica-rs1-2 to be ready.OK ++ kubectl_bin get psmdb cross-site-sharded-replica -o 'jsonpath={.spec.replsets[?(@.name=="rs1")].nonvoting.enabled}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.5thhXdPcKg +++ mktemp ++ local LAST_ERR=/tmp/tmp.0NzKAjbOV8 ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb cross-site-sharded-replica -o 'jsonpath={.spec.replsets[?(@.name=="rs1")].nonvoting.enabled}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.5thhXdPcKg ++ cat /tmp/tmp.0NzKAjbOV8 ++ rm /tmp/tmp.5thhXdPcKg /tmp/tmp.0NzKAjbOV8 ++ return 0 + [[ false == true ]] ++ kubectl_bin get psmdb cross-site-sharded-replica -o 'jsonpath={.spec.replsets[?(@.name=="rs1")].hidden.enabled}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.VMW448RP3Q +++ mktemp ++ local LAST_ERR=/tmp/tmp.X9mDAyC5TS ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb cross-site-sharded-replica -o 'jsonpath={.spec.replsets[?(@.name=="rs1")].hidden.enabled}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.VMW448RP3Q ++ cat /tmp/tmp.X9mDAyC5TS ++ rm /tmp/tmp.VMW448RP3Q /tmp/tmp.X9mDAyC5TS ++ return 0 + [[ '' == true ]] + sleep 10 + [[ false == true ]] + wait_for_running cross-site-sharded-replica-cfg 3 false + local name=cross-site-sharded-replica-cfg + let last_pod=2 + local check_cluster_readyness=false + set_debug + [[ 1 == 1 ]] + set -o xtrace + local rs_name=cfg + local cluster_name=cross-site-sharded-replica ++ seq 0 2 + for i in $(seq 0 $last_pod) + [[ 0 -eq 2 ]] + wait_pod cross-site-sharded-replica-cfg-0 + local pod=cross-site-sharded-replica-cfg-0 + set +o xtrace waiting for pod/cross-site-sharded-replica-cfg-0 to be ready.OK + for i in $(seq 0 $last_pod) + [[ 1 -eq 2 ]] + wait_pod cross-site-sharded-replica-cfg-1 + local pod=cross-site-sharded-replica-cfg-1 + set +o xtrace waiting for pod/cross-site-sharded-replica-cfg-1 to be ready.OK + for i in $(seq 0 $last_pod) + [[ 2 -eq 2 ]] ++ kubectl_bin get psmdb cross-site-sharded-replica -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].arbiter.enabled}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.45qgN6iwxq +++ mktemp ++ local LAST_ERR=/tmp/tmp.f6bhhm0CAa ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb cross-site-sharded-replica -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].arbiter.enabled}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.45qgN6iwxq ++ cat /tmp/tmp.f6bhhm0CAa ++ rm /tmp/tmp.45qgN6iwxq /tmp/tmp.f6bhhm0CAa ++ return 0 + [[ '' == true ]] + wait_pod cross-site-sharded-replica-cfg-2 + local pod=cross-site-sharded-replica-cfg-2 + set +o xtrace waiting for pod/cross-site-sharded-replica-cfg-2 to be ready.OK ++ kubectl_bin get psmdb cross-site-sharded-replica -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].nonvoting.enabled}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.YBvM3lKdUO +++ mktemp ++ local LAST_ERR=/tmp/tmp.n1LKxLlsTr ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb cross-site-sharded-replica -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].nonvoting.enabled}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.YBvM3lKdUO ++ cat /tmp/tmp.n1LKxLlsTr ++ rm /tmp/tmp.YBvM3lKdUO /tmp/tmp.n1LKxLlsTr ++ return 0 + [[ '' == true ]] ++ kubectl_bin get psmdb cross-site-sharded-replica -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].hidden.enabled}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.bP4ZxOlyjb +++ mktemp ++ local LAST_ERR=/tmp/tmp.ldzsR0gByS ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb cross-site-sharded-replica -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].hidden.enabled}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.bP4ZxOlyjb ++ cat /tmp/tmp.ldzsR0gByS ++ rm /tmp/tmp.bP4ZxOlyjb /tmp/tmp.ldzsR0gByS ++ return 0 + [[ '' == true ]] + sleep 10 + [[ false == true ]] ++ get_service_ip cross-site-sharded-replica-cfg-0 cfg ++ local service=cross-site-sharded-replica-cfg-0 ++ local server_type=cfg +++ kubectl_bin get psmdb/cross-site-sharded-replica -o 'jsonpath={.spec.replsets[].expose.enabled}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.hVKRrVlxtd ++++ mktemp +++ local LAST_ERR=/tmp/tmp.D6rx7C5n00 +++ local exit_status=0 +++ local timeout=4 ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl get psmdb/cross-site-sharded-replica -o 'jsonpath={.spec.replsets[].expose.enabled}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 -a -n 1 ']' +++ break +++ cat /tmp/tmp.hVKRrVlxtd +++ cat /tmp/tmp.D6rx7C5n00 +++ rm /tmp/tmp.hVKRrVlxtd /tmp/tmp.D6rx7C5n00 +++ return 0 ++ '[' true '!=' true ']' ++ kubectl_bin get service/cross-site-sharded-replica-cfg-0 -o 'jsonpath={.spec.type}' ++ grep -q NotFound +++ kubectl_bin get service/cross-site-sharded-replica-cfg-0 -o 'jsonpath={.spec.type}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.fNKP7RT7h4 ++++ mktemp +++ local LAST_ERR=/tmp/tmp.iX9JfI0Qiv +++ local exit_status=0 +++ local timeout=4 ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl get service/cross-site-sharded-replica-cfg-0 -o 'jsonpath={.spec.type}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 -a -n 1 ']' +++ break +++ cat /tmp/tmp.fNKP7RT7h4 +++ cat /tmp/tmp.iX9JfI0Qiv +++ rm /tmp/tmp.fNKP7RT7h4 /tmp/tmp.iX9JfI0Qiv +++ return 0 ++ service_type=ClusterIP ++ '[' ClusterIP = ClusterIP ']' ++ kubectl_bin get service/cross-site-sharded-replica-cfg-0 -o 'jsonpath={.spec.clusterIP}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.tpihF09r0a +++ mktemp ++ local LAST_ERR=/tmp/tmp.aHjC3lxL4g ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get service/cross-site-sharded-replica-cfg-0 -o 'jsonpath={.spec.clusterIP}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.tpihF09r0a ++ cat /tmp/tmp.aHjC3lxL4g ++ rm /tmp/tmp.tpihF09r0a /tmp/tmp.aHjC3lxL4g ++ return 0 ++ return + replica_cfg_0_endpoint=34.118.234.161 ++ get_service_ip cross-site-sharded-replica-cfg-1 cfg ++ local service=cross-site-sharded-replica-cfg-1 ++ local server_type=cfg +++ kubectl_bin get psmdb/cross-site-sharded-replica -o 'jsonpath={.spec.replsets[].expose.enabled}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.EYj5P47RMM ++++ mktemp +++ local LAST_ERR=/tmp/tmp.r8xicajEzz +++ local exit_status=0 +++ local timeout=4 ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl get psmdb/cross-site-sharded-replica -o 'jsonpath={.spec.replsets[].expose.enabled}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 -a -n 1 ']' +++ break +++ cat /tmp/tmp.EYj5P47RMM +++ cat /tmp/tmp.r8xicajEzz +++ rm /tmp/tmp.EYj5P47RMM /tmp/tmp.r8xicajEzz +++ return 0 ++ '[' true '!=' true ']' ++ kubectl_bin get service/cross-site-sharded-replica-cfg-1 -o 'jsonpath={.spec.type}' ++ grep -q NotFound +++ kubectl_bin get service/cross-site-sharded-replica-cfg-1 -o 'jsonpath={.spec.type}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.wYCJwO5z7J ++++ mktemp +++ local LAST_ERR=/tmp/tmp.j5KtmK3TEM +++ local exit_status=0 +++ local timeout=4 ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl get service/cross-site-sharded-replica-cfg-1 -o 'jsonpath={.spec.type}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 -a -n 1 ']' +++ break +++ cat /tmp/tmp.wYCJwO5z7J +++ cat /tmp/tmp.j5KtmK3TEM +++ rm /tmp/tmp.wYCJwO5z7J /tmp/tmp.j5KtmK3TEM +++ return 0 ++ service_type=ClusterIP ++ '[' ClusterIP = ClusterIP ']' ++ kubectl_bin get service/cross-site-sharded-replica-cfg-1 -o 'jsonpath={.spec.clusterIP}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.KJ4eNZiWON +++ mktemp ++ local LAST_ERR=/tmp/tmp.Wr88duvWnB ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get service/cross-site-sharded-replica-cfg-1 -o 'jsonpath={.spec.clusterIP}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.KJ4eNZiWON ++ cat /tmp/tmp.Wr88duvWnB ++ rm /tmp/tmp.KJ4eNZiWON /tmp/tmp.Wr88duvWnB ++ return 0 ++ return + replica_cfg_1_endpoint=34.118.230.133 ++ get_service_ip cross-site-sharded-replica-cfg-2 cfg ++ local service=cross-site-sharded-replica-cfg-2 ++ local server_type=cfg +++ kubectl_bin get psmdb/cross-site-sharded-replica -o 'jsonpath={.spec.replsets[].expose.enabled}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.CfVJzLUCNw ++++ mktemp +++ local LAST_ERR=/tmp/tmp.f2hqhKuRQn +++ local exit_status=0 +++ local timeout=4 ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl get psmdb/cross-site-sharded-replica -o 'jsonpath={.spec.replsets[].expose.enabled}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 -a -n 1 ']' +++ break +++ cat /tmp/tmp.CfVJzLUCNw +++ cat /tmp/tmp.f2hqhKuRQn +++ rm /tmp/tmp.CfVJzLUCNw /tmp/tmp.f2hqhKuRQn +++ return 0 ++ '[' true '!=' true ']' ++ kubectl_bin get service/cross-site-sharded-replica-cfg-2 -o 'jsonpath={.spec.type}' ++ grep -q NotFound +++ kubectl_bin get service/cross-site-sharded-replica-cfg-2 -o 'jsonpath={.spec.type}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.KdosANoWpT ++++ mktemp +++ local LAST_ERR=/tmp/tmp.PrBTvAXK0B +++ local exit_status=0 +++ local timeout=4 ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl get service/cross-site-sharded-replica-cfg-2 -o 'jsonpath={.spec.type}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 -a -n 1 ']' +++ break +++ cat /tmp/tmp.KdosANoWpT +++ cat /tmp/tmp.PrBTvAXK0B +++ rm /tmp/tmp.KdosANoWpT /tmp/tmp.PrBTvAXK0B +++ return 0 ++ service_type=ClusterIP ++ '[' ClusterIP = ClusterIP ']' ++ kubectl_bin get service/cross-site-sharded-replica-cfg-2 -o 'jsonpath={.spec.clusterIP}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.YD11a9MJlb +++ mktemp ++ local LAST_ERR=/tmp/tmp.xuremkaEgS ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get service/cross-site-sharded-replica-cfg-2 -o 'jsonpath={.spec.clusterIP}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.YD11a9MJlb ++ cat /tmp/tmp.xuremkaEgS ++ rm /tmp/tmp.YD11a9MJlb /tmp/tmp.xuremkaEgS ++ return 0 ++ return + replica_cfg_2_endpoint=34.118.235.97 ++ get_service_ip cross-site-sharded-replica-rs0-0 ++ local service=cross-site-sharded-replica-rs0-0 ++ local server_type=rs0 +++ kubectl_bin get psmdb/cross-site-sharded-replica -o 'jsonpath={.spec.replsets[].expose.enabled}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.e5KKx46XPa ++++ mktemp +++ local LAST_ERR=/tmp/tmp.RBduUCHkpP +++ local exit_status=0 +++ local timeout=4 ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl get psmdb/cross-site-sharded-replica -o 'jsonpath={.spec.replsets[].expose.enabled}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 -a -n 1 ']' +++ break +++ cat /tmp/tmp.e5KKx46XPa +++ cat /tmp/tmp.RBduUCHkpP +++ rm /tmp/tmp.e5KKx46XPa /tmp/tmp.RBduUCHkpP +++ return 0 ++ '[' true '!=' true ']' ++ kubectl_bin get service/cross-site-sharded-replica-rs0-0 -o 'jsonpath={.spec.type}' ++ grep -q NotFound +++ kubectl_bin get service/cross-site-sharded-replica-rs0-0 -o 'jsonpath={.spec.type}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.LxUhFl5WFR ++++ mktemp +++ local LAST_ERR=/tmp/tmp.cnUjrJqjhv +++ local exit_status=0 +++ local timeout=4 ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl get service/cross-site-sharded-replica-rs0-0 -o 'jsonpath={.spec.type}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 -a -n 1 ']' +++ break +++ cat /tmp/tmp.LxUhFl5WFR +++ cat /tmp/tmp.cnUjrJqjhv +++ rm /tmp/tmp.LxUhFl5WFR /tmp/tmp.cnUjrJqjhv +++ return 0 ++ service_type=ClusterIP ++ '[' ClusterIP = ClusterIP ']' ++ kubectl_bin get service/cross-site-sharded-replica-rs0-0 -o 'jsonpath={.spec.clusterIP}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.m42tjn5i69 +++ mktemp ++ local LAST_ERR=/tmp/tmp.rZdTApZmQK ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get service/cross-site-sharded-replica-rs0-0 -o 'jsonpath={.spec.clusterIP}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.m42tjn5i69 ++ cat /tmp/tmp.rZdTApZmQK ++ rm /tmp/tmp.m42tjn5i69 /tmp/tmp.rZdTApZmQK ++ return 0 ++ return + replica_rs0_0_endpoint=34.118.237.97 ++ get_service_ip cross-site-sharded-replica-rs0-1 ++ local service=cross-site-sharded-replica-rs0-1 ++ local server_type=rs0 +++ kubectl_bin get psmdb/cross-site-sharded-replica -o 'jsonpath={.spec.replsets[].expose.enabled}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.6E29M6xS1g ++++ mktemp +++ local LAST_ERR=/tmp/tmp.xDs4zxvuqR +++ local exit_status=0 +++ local timeout=4 ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl get psmdb/cross-site-sharded-replica -o 'jsonpath={.spec.replsets[].expose.enabled}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 -a -n 1 ']' +++ break +++ cat /tmp/tmp.6E29M6xS1g +++ cat /tmp/tmp.xDs4zxvuqR +++ rm /tmp/tmp.6E29M6xS1g /tmp/tmp.xDs4zxvuqR +++ return 0 ++ '[' true '!=' true ']' ++ kubectl_bin get service/cross-site-sharded-replica-rs0-1 -o 'jsonpath={.spec.type}' ++ grep -q NotFound +++ kubectl_bin get service/cross-site-sharded-replica-rs0-1 -o 'jsonpath={.spec.type}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.Nu6gD2urz7 ++++ mktemp +++ local LAST_ERR=/tmp/tmp.fpYQMBoKNu +++ local exit_status=0 +++ local timeout=4 ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl get service/cross-site-sharded-replica-rs0-1 -o 'jsonpath={.spec.type}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 -a -n 1 ']' +++ break +++ cat /tmp/tmp.Nu6gD2urz7 +++ cat /tmp/tmp.fpYQMBoKNu +++ rm /tmp/tmp.Nu6gD2urz7 /tmp/tmp.fpYQMBoKNu +++ return 0 ++ service_type=ClusterIP ++ '[' ClusterIP = ClusterIP ']' ++ kubectl_bin get service/cross-site-sharded-replica-rs0-1 -o 'jsonpath={.spec.clusterIP}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.nbyriVhZbP +++ mktemp ++ local LAST_ERR=/tmp/tmp.LrLq9q8Jca ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get service/cross-site-sharded-replica-rs0-1 -o 'jsonpath={.spec.clusterIP}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.nbyriVhZbP ++ cat /tmp/tmp.LrLq9q8Jca ++ rm /tmp/tmp.nbyriVhZbP /tmp/tmp.LrLq9q8Jca ++ return 0 ++ return + replica_rs0_1_endpoint=34.118.232.70 ++ get_service_ip cross-site-sharded-replica-rs0-2 ++ local service=cross-site-sharded-replica-rs0-2 ++ local server_type=rs0 +++ kubectl_bin get psmdb/cross-site-sharded-replica -o 'jsonpath={.spec.replsets[].expose.enabled}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.MhnK75ATYW ++++ mktemp +++ local LAST_ERR=/tmp/tmp.0tZhc9w6ZX +++ local exit_status=0 +++ local timeout=4 ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl get psmdb/cross-site-sharded-replica -o 'jsonpath={.spec.replsets[].expose.enabled}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 -a -n 1 ']' +++ break +++ cat /tmp/tmp.MhnK75ATYW +++ cat /tmp/tmp.0tZhc9w6ZX +++ rm /tmp/tmp.MhnK75ATYW /tmp/tmp.0tZhc9w6ZX +++ return 0 ++ '[' true '!=' true ']' ++ kubectl_bin get service/cross-site-sharded-replica-rs0-2 -o 'jsonpath={.spec.type}' ++ grep -q NotFound +++ kubectl_bin get service/cross-site-sharded-replica-rs0-2 -o 'jsonpath={.spec.type}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.2HvS8nEpco ++++ mktemp +++ local LAST_ERR=/tmp/tmp.YIPINWG8NX +++ local exit_status=0 +++ local timeout=4 ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl get service/cross-site-sharded-replica-rs0-2 -o 'jsonpath={.spec.type}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 -a -n 1 ']' +++ break +++ cat /tmp/tmp.2HvS8nEpco +++ cat /tmp/tmp.YIPINWG8NX +++ rm /tmp/tmp.2HvS8nEpco /tmp/tmp.YIPINWG8NX +++ return 0 ++ service_type=ClusterIP ++ '[' ClusterIP = ClusterIP ']' ++ kubectl_bin get service/cross-site-sharded-replica-rs0-2 -o 'jsonpath={.spec.clusterIP}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.BEtnCClArP +++ mktemp ++ local LAST_ERR=/tmp/tmp.lh27lXoFbM ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get service/cross-site-sharded-replica-rs0-2 -o 'jsonpath={.spec.clusterIP}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.BEtnCClArP ++ cat /tmp/tmp.lh27lXoFbM ++ rm /tmp/tmp.BEtnCClArP /tmp/tmp.lh27lXoFbM ++ return 0 ++ return + replica_rs0_2_endpoint=34.118.239.247 ++ get_service_ip cross-site-sharded-replica-rs1-0 rs1 ++ local service=cross-site-sharded-replica-rs1-0 ++ local server_type=rs1 +++ kubectl_bin get psmdb/cross-site-sharded-replica -o 'jsonpath={.spec.replsets[].expose.enabled}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.qgLsh0Ma4G ++++ mktemp +++ local LAST_ERR=/tmp/tmp.1s3H2hPbhc +++ local exit_status=0 +++ local timeout=4 ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl get psmdb/cross-site-sharded-replica -o 'jsonpath={.spec.replsets[].expose.enabled}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 -a -n 1 ']' +++ break +++ cat /tmp/tmp.qgLsh0Ma4G +++ cat /tmp/tmp.1s3H2hPbhc +++ rm /tmp/tmp.qgLsh0Ma4G /tmp/tmp.1s3H2hPbhc +++ return 0 ++ '[' true '!=' true ']' ++ kubectl_bin get service/cross-site-sharded-replica-rs1-0 -o 'jsonpath={.spec.type}' ++ grep -q NotFound +++ kubectl_bin get service/cross-site-sharded-replica-rs1-0 -o 'jsonpath={.spec.type}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.DmQjRXvtQN ++++ mktemp +++ local LAST_ERR=/tmp/tmp.OpHPvWZOIc +++ local exit_status=0 +++ local timeout=4 ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl get service/cross-site-sharded-replica-rs1-0 -o 'jsonpath={.spec.type}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 -a -n 1 ']' +++ break +++ cat /tmp/tmp.DmQjRXvtQN +++ cat /tmp/tmp.OpHPvWZOIc +++ rm /tmp/tmp.DmQjRXvtQN /tmp/tmp.OpHPvWZOIc +++ return 0 ++ service_type=ClusterIP ++ '[' ClusterIP = ClusterIP ']' ++ kubectl_bin get service/cross-site-sharded-replica-rs1-0 -o 'jsonpath={.spec.clusterIP}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.xki0YwbOSv +++ mktemp ++ local LAST_ERR=/tmp/tmp.ZOMrP0Bwmt ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get service/cross-site-sharded-replica-rs1-0 -o 'jsonpath={.spec.clusterIP}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.xki0YwbOSv ++ cat /tmp/tmp.ZOMrP0Bwmt ++ rm /tmp/tmp.xki0YwbOSv /tmp/tmp.ZOMrP0Bwmt ++ return 0 ++ return + replica_rs1_0_endpoint=34.118.234.85 ++ get_service_ip cross-site-sharded-replica-rs1-1 rs1 ++ local service=cross-site-sharded-replica-rs1-1 ++ local server_type=rs1 +++ kubectl_bin get psmdb/cross-site-sharded-replica -o 'jsonpath={.spec.replsets[].expose.enabled}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.TqV4hRiufz ++++ mktemp +++ local LAST_ERR=/tmp/tmp.ozNpZC3i0X +++ local exit_status=0 +++ local timeout=4 ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl get psmdb/cross-site-sharded-replica -o 'jsonpath={.spec.replsets[].expose.enabled}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 -a -n 1 ']' +++ break +++ cat /tmp/tmp.TqV4hRiufz +++ cat /tmp/tmp.ozNpZC3i0X +++ rm /tmp/tmp.TqV4hRiufz /tmp/tmp.ozNpZC3i0X +++ return 0 ++ '[' true '!=' true ']' ++ kubectl_bin get service/cross-site-sharded-replica-rs1-1 -o 'jsonpath={.spec.type}' ++ grep -q NotFound +++ kubectl_bin get service/cross-site-sharded-replica-rs1-1 -o 'jsonpath={.spec.type}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.HZ5Vb8Dowx ++++ mktemp +++ local LAST_ERR=/tmp/tmp.FJapxGWxB0 +++ local exit_status=0 +++ local timeout=4 ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl get service/cross-site-sharded-replica-rs1-1 -o 'jsonpath={.spec.type}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 -a -n 1 ']' +++ break +++ cat /tmp/tmp.HZ5Vb8Dowx +++ cat /tmp/tmp.FJapxGWxB0 +++ rm /tmp/tmp.HZ5Vb8Dowx /tmp/tmp.FJapxGWxB0 +++ return 0 ++ service_type=ClusterIP ++ '[' ClusterIP = ClusterIP ']' ++ kubectl_bin get service/cross-site-sharded-replica-rs1-1 -o 'jsonpath={.spec.clusterIP}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.zRgE4mRHpa +++ mktemp ++ local LAST_ERR=/tmp/tmp.zMP9Vor8w2 ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get service/cross-site-sharded-replica-rs1-1 -o 'jsonpath={.spec.clusterIP}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.zRgE4mRHpa ++ cat /tmp/tmp.zMP9Vor8w2 ++ rm /tmp/tmp.zRgE4mRHpa /tmp/tmp.zMP9Vor8w2 ++ return 0 ++ return + replica_rs1_1_endpoint=34.118.227.91 ++ get_service_ip cross-site-sharded-replica-rs1-2 rs1 ++ local service=cross-site-sharded-replica-rs1-2 ++ local server_type=rs1 +++ kubectl_bin get psmdb/cross-site-sharded-replica -o 'jsonpath={.spec.replsets[].expose.enabled}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.rfcWVgCrlk ++++ mktemp +++ local LAST_ERR=/tmp/tmp.oIBBiqfobR +++ local exit_status=0 +++ local timeout=4 ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl get psmdb/cross-site-sharded-replica -o 'jsonpath={.spec.replsets[].expose.enabled}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 -a -n 1 ']' +++ break +++ cat /tmp/tmp.rfcWVgCrlk +++ cat /tmp/tmp.oIBBiqfobR +++ rm /tmp/tmp.rfcWVgCrlk /tmp/tmp.oIBBiqfobR +++ return 0 ++ '[' true '!=' true ']' ++ kubectl_bin get service/cross-site-sharded-replica-rs1-2 -o 'jsonpath={.spec.type}' ++ grep -q NotFound +++ kubectl_bin get service/cross-site-sharded-replica-rs1-2 -o 'jsonpath={.spec.type}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.obKNEMIr91 ++++ mktemp +++ local LAST_ERR=/tmp/tmp.3n9zn7uDAF +++ local exit_status=0 +++ local timeout=4 ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl get service/cross-site-sharded-replica-rs1-2 -o 'jsonpath={.spec.type}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 -a -n 1 ']' +++ break +++ cat /tmp/tmp.obKNEMIr91 +++ cat /tmp/tmp.3n9zn7uDAF +++ rm /tmp/tmp.obKNEMIr91 /tmp/tmp.3n9zn7uDAF +++ return 0 ++ service_type=ClusterIP ++ '[' ClusterIP = ClusterIP ']' ++ kubectl_bin get service/cross-site-sharded-replica-rs1-2 -o 'jsonpath={.spec.clusterIP}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.DoNu23HTMF +++ mktemp ++ local LAST_ERR=/tmp/tmp.TWcgxl4Foe ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get service/cross-site-sharded-replica-rs1-2 -o 'jsonpath={.spec.clusterIP}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.DoNu23HTMF ++ cat /tmp/tmp.TWcgxl4Foe ++ rm /tmp/tmp.DoNu23HTMF /tmp/tmp.TWcgxl4Foe ++ return 0 ++ return + replica_rs1_2_endpoint=34.118.226.230 ++ kubectl_bin config current-context +++ mktemp ++ local LAST_OUT=/tmp/tmp.pyGvWqV8u9 +++ mktemp ++ local LAST_ERR=/tmp/tmp.635RqbB6Ds ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl config current-context ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.pyGvWqV8u9 ++ cat /tmp/tmp.635RqbB6Ds ++ rm /tmp/tmp.pyGvWqV8u9 /tmp/tmp.635RqbB6Ds ++ return 0 + kubectl_bin config set-context gke_cloud-dev-112233_us-central1-a_jen-psmdb-2058-fb8cc7187-17-cluster3 --namespace=cross-site-sharded-5222 ++ mktemp + local LAST_OUT=/tmp/tmp.8y4mLCLY9g ++ mktemp + local LAST_ERR=/tmp/tmp.BnhOPRruwt + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl config set-context gke_cloud-dev-112233_us-central1-a_jen-psmdb-2058-fb8cc7187-17-cluster3 --namespace=cross-site-sharded-5222 + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.8y4mLCLY9g Context "gke_cloud-dev-112233_us-central1-a_jen-psmdb-2058-fb8cc7187-17-cluster3" modified. + cat /tmp/tmp.BnhOPRruwt + rm /tmp/tmp.8y4mLCLY9g /tmp/tmp.BnhOPRruwt + return 0 + kubectl_bin patch psmdb cross-site-sharded-main --type=merge --patch $'{\n\t\t\t"spec": {"replsets":[\n\t\t\t {"affinity":{"antiAffinityTopologyKey": "none"},"arbiter":{"affinity":{"antiAffinityTopologyKey": "none"},"enabled":false,"size":1},"expose":{"enabled":true,"type":"ClusterIp"},"externalNodes":[{"host":"34.118.237.97","priority":0,"votes":0},{"host":"34.118.232.70","port":27017,"priority":1,"votes":1},{"host":"34.118.239.247", "port":27017,"priority":1,"votes":1}],"name":"rs0","nonvoting":{"affinity":{"antiAffinityTopologyKey":"none"},"enabled":false,"podDisruptionBudget":{"maxUnavailable":1},"resources":{"limits":{"cpu":"300m","memory":"0.5G"},"requests":{"cpu":"300m","memory":"0.5G"}},"size":3,"volumeSpec":{"persistentVolumeClaim":{"resources":{"requests":{"storage":"1Gi"}}}}},"podDisruptionBudget":{"maxUnavailable":1},"resources":{"limits":{"cpu":"300m","memory":"0.5G"},"requests":{"cpu":"300m","memory":"0.5G"}},"size":3,"volumeSpec":{"persistentVolumeClaim":{"resources":{"requests":{"storage":"3Gi"}}}}},\n\t\t\t {"affinity":{"antiAffinityTopologyKey": "none"},"arbiter":{"affinity":{"antiAffinityTopologyKey": "none"},"enabled":false,"size":1},"expose":{"enabled":true,"type":"ClusterIp"},"externalNodes":[{"host":"34.118.234.85","priority":0,"votes":0},{"host":"34.118.227.91","port":27017,"priority":1,"votes":1},{"host":"34.118.226.230", "port":27017,"priority":1,"votes":1}],"name":"rs1","nonvoting":{"affinity":{"antiAffinityTopologyKey":"none"},"enabled":false,"podDisruptionBudget":{"maxUnavailable":1},"resources":{"limits":{"cpu":"300m","memory":"0.5G"},"requests":{"cpu":"300m","memory":"0.5G"}},"size":3,"volumeSpec":{"persistentVolumeClaim":{"resources":{"requests":{"storage":"1Gi"}}}}},"podDisruptionBudget":{"maxUnavailable":1},"resources":{"limits":{"cpu":"300m","memory":"0.5G"},"requests":{"cpu":"300m","memory":"0.5G"}},"size":3,"volumeSpec":{"persistentVolumeClaim":{"resources":{"requests":{"storage":"3Gi"}}}}}\n\t\t\t ],\n\t\t\t "sharding":{"configsvrReplSet":{ "externalNodes": [{"host":"34.118.234.161","priority":1,"votes":1 },{"host":"34.118.230.133", "priority":1,"votes":1},{"host":"34.118.235.97","priority":0,"votes":0}]}}\n\t\t\t }\n\t\t}' ++ mktemp + local LAST_OUT=/tmp/tmp.oPlYZHEIYx ++ mktemp + local LAST_ERR=/tmp/tmp.lnNs6Z4xKW + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl patch psmdb cross-site-sharded-main --type=merge --patch $'{\n\t\t\t"spec": {"replsets":[\n\t\t\t {"affinity":{"antiAffinityTopologyKey": "none"},"arbiter":{"affinity":{"antiAffinityTopologyKey": "none"},"enabled":false,"size":1},"expose":{"enabled":true,"type":"ClusterIp"},"externalNodes":[{"host":"34.118.237.97","priority":0,"votes":0},{"host":"34.118.232.70","port":27017,"priority":1,"votes":1},{"host":"34.118.239.247", "port":27017,"priority":1,"votes":1}],"name":"rs0","nonvoting":{"affinity":{"antiAffinityTopologyKey":"none"},"enabled":false,"podDisruptionBudget":{"maxUnavailable":1},"resources":{"limits":{"cpu":"300m","memory":"0.5G"},"requests":{"cpu":"300m","memory":"0.5G"}},"size":3,"volumeSpec":{"persistentVolumeClaim":{"resources":{"requests":{"storage":"1Gi"}}}}},"podDisruptionBudget":{"maxUnavailable":1},"resources":{"limits":{"cpu":"300m","memory":"0.5G"},"requests":{"cpu":"300m","memory":"0.5G"}},"size":3,"volumeSpec":{"persistentVolumeClaim":{"resources":{"requests":{"storage":"3Gi"}}}}},\n\t\t\t {"affinity":{"antiAffinityTopologyKey": "none"},"arbiter":{"affinity":{"antiAffinityTopologyKey": "none"},"enabled":false,"size":1},"expose":{"enabled":true,"type":"ClusterIp"},"externalNodes":[{"host":"34.118.234.85","priority":0,"votes":0},{"host":"34.118.227.91","port":27017,"priority":1,"votes":1},{"host":"34.118.226.230", "port":27017,"priority":1,"votes":1}],"name":"rs1","nonvoting":{"affinity":{"antiAffinityTopologyKey":"none"},"enabled":false,"podDisruptionBudget":{"maxUnavailable":1},"resources":{"limits":{"cpu":"300m","memory":"0.5G"},"requests":{"cpu":"300m","memory":"0.5G"}},"size":3,"volumeSpec":{"persistentVolumeClaim":{"resources":{"requests":{"storage":"1Gi"}}}}},"podDisruptionBudget":{"maxUnavailable":1},"resources":{"limits":{"cpu":"300m","memory":"0.5G"},"requests":{"cpu":"300m","memory":"0.5G"}},"size":3,"volumeSpec":{"persistentVolumeClaim":{"resources":{"requests":{"storage":"3Gi"}}}}}\n\t\t\t ],\n\t\t\t "sharding":{"configsvrReplSet":{ "externalNodes": [{"host":"34.118.234.161","priority":1,"votes":1 },{"host":"34.118.230.133", "priority":1,"votes":1},{"host":"34.118.235.97","priority":0,"votes":0}]}}\n\t\t\t }\n\t\t}' + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.oPlYZHEIYx perconaservermongodb.psmdb.percona.com/cross-site-sharded-main patched + cat /tmp/tmp.lnNs6Z4xKW + rm /tmp/tmp.oPlYZHEIYx /tmp/tmp.lnNs6Z4xKW + return 0 + wait_for_members 34.118.234.161 cfg 6 + local endpoint=34.118.234.161 + local rsName=cfg + local target_count=6 + local nodes_count=0 + [[ 0 == 6 ]] ++ run_mongos 'rs.conf().members.length' clusterAdmin:clusterAdmin123456@34.118.234.161 mongodb :27017 ++ local 'command=rs.conf().members.length' ++ local uri=clusterAdmin:clusterAdmin123456@34.118.234.161 ++ grep -E -v 'I NETWORK|W NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|bye' ++ local driver=mongodb ++ local suffix=:27017 ++ local mongo_flag= ++ local port=27017 ++ local mongo_bin=mongo ++ /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/' +++ echo :27017 +++ awk -F: '{print $2}' ++ suffix_port=27017 ++ [[ -z 27017 ]] +++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.hsdC7hY3LA ++++ mktemp +++ local LAST_ERR=/tmp/tmp.q56zlyT4zs +++ local exit_status=0 +++ local timeout=4 ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 -a -n 1 ']' +++ break +++ cat /tmp/tmp.hsdC7hY3LA +++ cat /tmp/tmp.q56zlyT4zs +++ rm /tmp/tmp.hsdC7hY3LA /tmp/tmp.q56zlyT4zs +++ return 0 ++ local client_container=psmdb-client-bb8b97679-hr2qb ++ kubectl_bin exec psmdb-client-bb8b97679-hr2qb -- bash -c 'printf '\''rs.conf().members.length\n'\'' | mongo mongodb://clusterAdmin:clusterAdmin123456@34.118.234.161:27017/admin ' +++ mktemp ++ local LAST_OUT=/tmp/tmp.K4GlIq3AWe +++ mktemp ++ local LAST_ERR=/tmp/tmp.DicDPCwyPI ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl exec psmdb-client-bb8b97679-hr2qb -- bash -c 'printf '\''rs.conf().members.length\n'\'' | mongo mongodb://clusterAdmin:clusterAdmin123456@34.118.234.161:27017/admin ' ++ exit_status=1 ++ set -e ++ '[' 1 '!=' 0 -a -n 1 ']' ++ cat /tmp/tmp.K4GlIq3AWe ++ cat /tmp/tmp.DicDPCwyPI command terminated with exit code 1 ++ sleep 0 ++ for i in $(seq 0 2) ++ set +e ++ kubectl exec psmdb-client-bb8b97679-hr2qb -- bash -c 'printf '\''rs.conf().members.length\n'\'' | mongo mongodb://clusterAdmin:clusterAdmin123456@34.118.234.161:27017/admin ' ++ exit_status=1 ++ set -e ++ '[' 1 '!=' 0 -a -n 1 ']' ++ cat /tmp/tmp.K4GlIq3AWe ++ cat /tmp/tmp.DicDPCwyPI command terminated with exit code 1 ++ sleep 4 ++ for i in $(seq 0 2) ++ set +e ++ kubectl exec psmdb-client-bb8b97679-hr2qb -- bash -c 'printf '\''rs.conf().members.length\n'\'' | mongo mongodb://clusterAdmin:clusterAdmin123456@34.118.234.161:27017/admin ' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.K4GlIq3AWe ++ cat /tmp/tmp.DicDPCwyPI ++ rm /tmp/tmp.K4GlIq3AWe /tmp/tmp.DicDPCwyPI ++ return 0 + nodes_count=$'Error: Authentication failed. :\nconnect@src/mongo/shell/mongo.js:374:17\n@(connect):2:6\nexception: connect failed\nexiting with code 1\nError: Authentication failed. :\nconnect@src/mongo/shell/mongo.js:374:17\n@(connect):2:6\nexception: connect failed\nexiting with code 1\n4' + echo -n 'waiting for all members to be configured in cfg' waiting for all members to be configured in cfg+ let retry+=1 + '[' 1 -ge 15 ']' + echo . . + sleep 10 + [[ Error: Authentication failed. : connect@src/mongo/shell/mongo.js:374:17 @(connect):2:6 exception: connect failed exiting with code 1 Error: Authentication failed. : connect@src/mongo/shell/mongo.js:374:17 @(connect):2:6 exception: connect failed exiting with code 1 4 == 6 ]] ++ run_mongos 'rs.conf().members.length' clusterAdmin:clusterAdmin123456@34.118.234.161 mongodb :27017 ++ local 'command=rs.conf().members.length' ++ grep -E -v 'I NETWORK|W NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|bye' ++ local uri=clusterAdmin:clusterAdmin123456@34.118.234.161 ++ local driver=mongodb ++ local suffix=:27017 ++ local mongo_flag= ++ local port=27017 ++ local mongo_bin=mongo +++ echo :27017 ++ /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/' +++ awk -F: '{print $2}' ++ suffix_port=27017 ++ [[ -z 27017 ]] +++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.ewyxhtgyZ9 ++++ mktemp +++ local LAST_ERR=/tmp/tmp.X4n50Xzikx +++ local exit_status=0 +++ local timeout=4 ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 -a -n 1 ']' +++ break +++ cat /tmp/tmp.ewyxhtgyZ9 +++ cat /tmp/tmp.X4n50Xzikx +++ rm /tmp/tmp.ewyxhtgyZ9 /tmp/tmp.X4n50Xzikx +++ return 0 ++ local client_container=psmdb-client-bb8b97679-hr2qb ++ kubectl_bin exec psmdb-client-bb8b97679-hr2qb -- bash -c 'printf '\''rs.conf().members.length\n'\'' | mongo mongodb://clusterAdmin:clusterAdmin123456@34.118.234.161:27017/admin ' +++ mktemp ++ local LAST_OUT=/tmp/tmp.46Afav3izN +++ mktemp ++ local LAST_ERR=/tmp/tmp.7oGeTiSNqL ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl exec psmdb-client-bb8b97679-hr2qb -- bash -c 'printf '\''rs.conf().members.length\n'\'' | mongo mongodb://clusterAdmin:clusterAdmin123456@34.118.234.161:27017/admin ' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.46Afav3izN ++ cat /tmp/tmp.7oGeTiSNqL ++ rm /tmp/tmp.46Afav3izN /tmp/tmp.7oGeTiSNqL ++ return 0 + nodes_count=5 + echo -n 'waiting for all members to be configured in cfg' waiting for all members to be configured in cfg+ let retry+=1 + '[' 2 -ge 15 ']' + echo . . + sleep 10 + [[ 5 == 6 ]] ++ run_mongos 'rs.conf().members.length' clusterAdmin:clusterAdmin123456@34.118.234.161 mongodb :27017 ++ local 'command=rs.conf().members.length' ++ local uri=clusterAdmin:clusterAdmin123456@34.118.234.161 ++ local driver=mongodb ++ local suffix=:27017 ++ local mongo_flag= ++ local port=27017 ++ local mongo_bin=mongo ++ grep -E -v 'I NETWORK|W NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|bye' ++ /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/' +++ echo :27017 +++ awk -F: '{print $2}' ++ suffix_port=27017 ++ [[ -z 27017 ]] +++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.8s4nRTYLd9 ++++ mktemp +++ local LAST_ERR=/tmp/tmp.y47HZnYYHm +++ local exit_status=0 +++ local timeout=4 ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 -a -n 1 ']' +++ break +++ cat /tmp/tmp.8s4nRTYLd9 +++ cat /tmp/tmp.y47HZnYYHm +++ rm /tmp/tmp.8s4nRTYLd9 /tmp/tmp.y47HZnYYHm +++ return 0 ++ local client_container=psmdb-client-bb8b97679-hr2qb ++ kubectl_bin exec psmdb-client-bb8b97679-hr2qb -- bash -c 'printf '\''rs.conf().members.length\n'\'' | mongo mongodb://clusterAdmin:clusterAdmin123456@34.118.234.161:27017/admin ' +++ mktemp ++ local LAST_OUT=/tmp/tmp.uhiMJ4l18N +++ mktemp ++ local LAST_ERR=/tmp/tmp.8CEyCtbZp3 ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl exec psmdb-client-bb8b97679-hr2qb -- bash -c 'printf '\''rs.conf().members.length\n'\'' | mongo mongodb://clusterAdmin:clusterAdmin123456@34.118.234.161:27017/admin ' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.uhiMJ4l18N ++ cat /tmp/tmp.8CEyCtbZp3 ++ rm /tmp/tmp.uhiMJ4l18N /tmp/tmp.8CEyCtbZp3 ++ return 0 + nodes_count=5 + echo -n 'waiting for all members to be configured in cfg' waiting for all members to be configured in cfg+ let retry+=1 + '[' 3 -ge 15 ']' + echo . . + sleep 10 + [[ 5 == 6 ]] ++ run_mongos 'rs.conf().members.length' clusterAdmin:clusterAdmin123456@34.118.234.161 mongodb :27017 ++ local 'command=rs.conf().members.length' ++ local uri=clusterAdmin:clusterAdmin123456@34.118.234.161 ++ grep -E -v 'I NETWORK|W NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|bye' ++ local driver=mongodb ++ local suffix=:27017 ++ local mongo_flag= ++ local port=27017 ++ local mongo_bin=mongo ++ /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/' +++ echo :27017 +++ awk -F: '{print $2}' ++ suffix_port=27017 ++ [[ -z 27017 ]] +++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.8zpdVHmZSc ++++ mktemp +++ local LAST_ERR=/tmp/tmp.UoA2N9Vv3G +++ local exit_status=0 +++ local timeout=4 ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 -a -n 1 ']' +++ break +++ cat /tmp/tmp.8zpdVHmZSc +++ cat /tmp/tmp.UoA2N9Vv3G +++ rm /tmp/tmp.8zpdVHmZSc /tmp/tmp.UoA2N9Vv3G +++ return 0 ++ local client_container=psmdb-client-bb8b97679-hr2qb ++ kubectl_bin exec psmdb-client-bb8b97679-hr2qb -- bash -c 'printf '\''rs.conf().members.length\n'\'' | mongo mongodb://clusterAdmin:clusterAdmin123456@34.118.234.161:27017/admin ' +++ mktemp ++ local LAST_OUT=/tmp/tmp.5XwdoJzPe9 +++ mktemp ++ local LAST_ERR=/tmp/tmp.OpmfXsz6L3 ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl exec psmdb-client-bb8b97679-hr2qb -- bash -c 'printf '\''rs.conf().members.length\n'\'' | mongo mongodb://clusterAdmin:clusterAdmin123456@34.118.234.161:27017/admin ' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.5XwdoJzPe9 ++ cat /tmp/tmp.OpmfXsz6L3 ++ rm /tmp/tmp.5XwdoJzPe9 /tmp/tmp.OpmfXsz6L3 ++ return 0 + nodes_count=5 + echo -n 'waiting for all members to be configured in cfg' waiting for all members to be configured in cfg+ let retry+=1 + '[' 4 -ge 15 ']' + echo . . + sleep 10 + [[ 5 == 6 ]] ++ run_mongos 'rs.conf().members.length' clusterAdmin:clusterAdmin123456@34.118.234.161 mongodb :27017 ++ local 'command=rs.conf().members.length' ++ local uri=clusterAdmin:clusterAdmin123456@34.118.234.161 ++ local driver=mongodb ++ local suffix=:27017 ++ local mongo_flag= ++ local port=27017 ++ local mongo_bin=mongo ++ grep -E -v 'I NETWORK|W NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|bye' ++ /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/' +++ echo :27017 +++ awk -F: '{print $2}' ++ suffix_port=27017 ++ [[ -z 27017 ]] +++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.5OqSTq197z ++++ mktemp +++ local LAST_ERR=/tmp/tmp.HJVe8nz1Hg +++ local exit_status=0 +++ local timeout=4 ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 -a -n 1 ']' +++ break +++ cat /tmp/tmp.5OqSTq197z +++ cat /tmp/tmp.HJVe8nz1Hg +++ rm /tmp/tmp.5OqSTq197z /tmp/tmp.HJVe8nz1Hg +++ return 0 ++ local client_container=psmdb-client-bb8b97679-hr2qb ++ kubectl_bin exec psmdb-client-bb8b97679-hr2qb -- bash -c 'printf '\''rs.conf().members.length\n'\'' | mongo mongodb://clusterAdmin:clusterAdmin123456@34.118.234.161:27017/admin ' +++ mktemp ++ local LAST_OUT=/tmp/tmp.RRQ5aWY0Ou +++ mktemp ++ local LAST_ERR=/tmp/tmp.nVERXGJ4Ix ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl exec psmdb-client-bb8b97679-hr2qb -- bash -c 'printf '\''rs.conf().members.length\n'\'' | mongo mongodb://clusterAdmin:clusterAdmin123456@34.118.234.161:27017/admin ' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.RRQ5aWY0Ou ++ cat /tmp/tmp.nVERXGJ4Ix ++ rm /tmp/tmp.RRQ5aWY0Ou /tmp/tmp.nVERXGJ4Ix ++ return 0 + nodes_count=5 + echo -n 'waiting for all members to be configured in cfg' waiting for all members to be configured in cfg+ let retry+=1 + '[' 5 -ge 15 ']' + echo . . + sleep 10 + [[ 5 == 6 ]] ++ run_mongos 'rs.conf().members.length' clusterAdmin:clusterAdmin123456@34.118.234.161 mongodb :27017 ++ local 'command=rs.conf().members.length' ++ local uri=clusterAdmin:clusterAdmin123456@34.118.234.161 ++ local driver=mongodb ++ local suffix=:27017 ++ local mongo_flag= ++ local port=27017 ++ local mongo_bin=mongo ++ grep -E -v 'I NETWORK|W NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|bye' ++ /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/' +++ echo :27017 +++ awk -F: '{print $2}' ++ suffix_port=27017 ++ [[ -z 27017 ]] +++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.24G1HT2Dzz ++++ mktemp +++ local LAST_ERR=/tmp/tmp.3zXAFyOCFJ +++ local exit_status=0 +++ local timeout=4 ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 -a -n 1 ']' +++ break +++ cat /tmp/tmp.24G1HT2Dzz +++ cat /tmp/tmp.3zXAFyOCFJ +++ rm /tmp/tmp.24G1HT2Dzz /tmp/tmp.3zXAFyOCFJ +++ return 0 ++ local client_container=psmdb-client-bb8b97679-hr2qb ++ kubectl_bin exec psmdb-client-bb8b97679-hr2qb -- bash -c 'printf '\''rs.conf().members.length\n'\'' | mongo mongodb://clusterAdmin:clusterAdmin123456@34.118.234.161:27017/admin ' +++ mktemp ++ local LAST_OUT=/tmp/tmp.q7UTP3AHAL +++ mktemp ++ local LAST_ERR=/tmp/tmp.9gjeEYly6L ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl exec psmdb-client-bb8b97679-hr2qb -- bash -c 'printf '\''rs.conf().members.length\n'\'' | mongo mongodb://clusterAdmin:clusterAdmin123456@34.118.234.161:27017/admin ' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.q7UTP3AHAL ++ cat /tmp/tmp.9gjeEYly6L ++ rm /tmp/tmp.q7UTP3AHAL /tmp/tmp.9gjeEYly6L ++ return 0 + nodes_count=5 + echo -n 'waiting for all members to be configured in cfg' waiting for all members to be configured in cfg+ let retry+=1 + '[' 6 -ge 15 ']' + echo . . + sleep 10 + [[ 5 == 6 ]] ++ run_mongos 'rs.conf().members.length' clusterAdmin:clusterAdmin123456@34.118.234.161 mongodb :27017 ++ grep -E -v 'I NETWORK|W NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|bye' ++ local 'command=rs.conf().members.length' ++ local uri=clusterAdmin:clusterAdmin123456@34.118.234.161 ++ local driver=mongodb ++ local suffix=:27017 ++ local mongo_flag= ++ local port=27017 ++ local mongo_bin=mongo ++ /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/' +++ echo :27017 +++ awk -F: '{print $2}' ++ suffix_port=27017 ++ [[ -z 27017 ]] +++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.b2QvELMT2X ++++ mktemp +++ local LAST_ERR=/tmp/tmp.xNM14OuTHA +++ local exit_status=0 +++ local timeout=4 ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 -a -n 1 ']' +++ break +++ cat /tmp/tmp.b2QvELMT2X +++ cat /tmp/tmp.xNM14OuTHA +++ rm /tmp/tmp.b2QvELMT2X /tmp/tmp.xNM14OuTHA +++ return 0 ++ local client_container=psmdb-client-bb8b97679-hr2qb ++ kubectl_bin exec psmdb-client-bb8b97679-hr2qb -- bash -c 'printf '\''rs.conf().members.length\n'\'' | mongo mongodb://clusterAdmin:clusterAdmin123456@34.118.234.161:27017/admin ' +++ mktemp ++ local LAST_OUT=/tmp/tmp.4xr87lRypt +++ mktemp ++ local LAST_ERR=/tmp/tmp.31pE1IZ79s ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl exec psmdb-client-bb8b97679-hr2qb -- bash -c 'printf '\''rs.conf().members.length\n'\'' | mongo mongodb://clusterAdmin:clusterAdmin123456@34.118.234.161:27017/admin ' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.4xr87lRypt ++ cat /tmp/tmp.31pE1IZ79s ++ rm /tmp/tmp.4xr87lRypt /tmp/tmp.31pE1IZ79s ++ return 0 + nodes_count=5 + echo -n 'waiting for all members to be configured in cfg' waiting for all members to be configured in cfg+ let retry+=1 + '[' 7 -ge 15 ']' + echo . . + sleep 10 + [[ 5 == 6 ]] ++ run_mongos 'rs.conf().members.length' clusterAdmin:clusterAdmin123456@34.118.234.161 mongodb :27017 ++ local 'command=rs.conf().members.length' ++ local uri=clusterAdmin:clusterAdmin123456@34.118.234.161 ++ local driver=mongodb ++ local suffix=:27017 ++ local mongo_flag= ++ local port=27017 ++ local mongo_bin=mongo ++ grep -E -v 'I NETWORK|W NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|bye' ++ /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/' +++ echo :27017 +++ awk -F: '{print $2}' ++ suffix_port=27017 ++ [[ -z 27017 ]] +++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.abPvNNWgku ++++ mktemp +++ local LAST_ERR=/tmp/tmp.wnSF0iUeGA +++ local exit_status=0 +++ local timeout=4 ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 -a -n 1 ']' +++ break +++ cat /tmp/tmp.abPvNNWgku +++ cat /tmp/tmp.wnSF0iUeGA +++ rm /tmp/tmp.abPvNNWgku /tmp/tmp.wnSF0iUeGA +++ return 0 ++ local client_container=psmdb-client-bb8b97679-hr2qb ++ kubectl_bin exec psmdb-client-bb8b97679-hr2qb -- bash -c 'printf '\''rs.conf().members.length\n'\'' | mongo mongodb://clusterAdmin:clusterAdmin123456@34.118.234.161:27017/admin ' +++ mktemp ++ local LAST_OUT=/tmp/tmp.RKb9Wl1Nmc +++ mktemp ++ local LAST_ERR=/tmp/tmp.kvMeZMvpvS ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl exec psmdb-client-bb8b97679-hr2qb -- bash -c 'printf '\''rs.conf().members.length\n'\'' | mongo mongodb://clusterAdmin:clusterAdmin123456@34.118.234.161:27017/admin ' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.RKb9Wl1Nmc ++ cat /tmp/tmp.kvMeZMvpvS ++ rm /tmp/tmp.RKb9Wl1Nmc /tmp/tmp.kvMeZMvpvS ++ return 0 + nodes_count=5 + echo -n 'waiting for all members to be configured in cfg' waiting for all members to be configured in cfg+ let retry+=1 + '[' 8 -ge 15 ']' + echo . . + sleep 10 + [[ 5 == 6 ]] ++ run_mongos 'rs.conf().members.length' clusterAdmin:clusterAdmin123456@34.118.234.161 mongodb :27017 ++ /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/' ++ local 'command=rs.conf().members.length' ++ grep -E -v 'I NETWORK|W NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|bye' ++ local uri=clusterAdmin:clusterAdmin123456@34.118.234.161 ++ local driver=mongodb ++ local suffix=:27017 ++ local mongo_flag= ++ local port=27017 ++ local mongo_bin=mongo +++ awk -F: '{print $2}' +++ echo :27017 ++ suffix_port=27017 ++ [[ -z 27017 ]] +++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.FqlS5CAzG7 ++++ mktemp +++ local LAST_ERR=/tmp/tmp.1VuC8v1uUG +++ local exit_status=0 +++ local timeout=4 ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 -a -n 1 ']' +++ break +++ cat /tmp/tmp.FqlS5CAzG7 +++ cat /tmp/tmp.1VuC8v1uUG +++ rm /tmp/tmp.FqlS5CAzG7 /tmp/tmp.1VuC8v1uUG +++ return 0 ++ local client_container=psmdb-client-bb8b97679-hr2qb ++ kubectl_bin exec psmdb-client-bb8b97679-hr2qb -- bash -c 'printf '\''rs.conf().members.length\n'\'' | mongo mongodb://clusterAdmin:clusterAdmin123456@34.118.234.161:27017/admin ' +++ mktemp ++ local LAST_OUT=/tmp/tmp.kFvfNnRYHA +++ mktemp ++ local LAST_ERR=/tmp/tmp.PeZy2zyXxG ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl exec psmdb-client-bb8b97679-hr2qb -- bash -c 'printf '\''rs.conf().members.length\n'\'' | mongo mongodb://clusterAdmin:clusterAdmin123456@34.118.234.161:27017/admin ' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.kFvfNnRYHA ++ cat /tmp/tmp.PeZy2zyXxG ++ rm /tmp/tmp.kFvfNnRYHA /tmp/tmp.PeZy2zyXxG ++ return 0 + nodes_count=6 + echo -n 'waiting for all members to be configured in cfg' waiting for all members to be configured in cfg+ let retry+=1 + '[' 9 -ge 15 ']' + echo . . + sleep 10 + [[ 6 == 6 ]] + wait_for_members 34.118.237.97 rs0 6 + local endpoint=34.118.237.97 + local rsName=rs0 + local target_count=6 + local nodes_count=0 + [[ 0 == 6 ]] ++ run_mongos 'rs.conf().members.length' clusterAdmin:clusterAdmin123456@34.118.237.97 mongodb :27017 ++ local 'command=rs.conf().members.length' ++ local uri=clusterAdmin:clusterAdmin123456@34.118.237.97 ++ local driver=mongodb ++ local suffix=:27017 ++ local mongo_flag= ++ local port=27017 ++ local mongo_bin=mongo ++ grep -E -v 'I NETWORK|W NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|bye' ++ /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/' +++ echo :27017 +++ awk -F: '{print $2}' ++ suffix_port=27017 ++ [[ -z 27017 ]] +++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.w5HYdQkGB0 ++++ mktemp +++ local LAST_ERR=/tmp/tmp.DcpJFCxbqG +++ local exit_status=0 +++ local timeout=4 ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 -a -n 1 ']' +++ break +++ cat /tmp/tmp.w5HYdQkGB0 +++ cat /tmp/tmp.DcpJFCxbqG +++ rm /tmp/tmp.w5HYdQkGB0 /tmp/tmp.DcpJFCxbqG +++ return 0 ++ local client_container=psmdb-client-bb8b97679-hr2qb ++ kubectl_bin exec psmdb-client-bb8b97679-hr2qb -- bash -c 'printf '\''rs.conf().members.length\n'\'' | mongo mongodb://clusterAdmin:clusterAdmin123456@34.118.237.97:27017/admin ' +++ mktemp ++ local LAST_OUT=/tmp/tmp.z3CupPcApH +++ mktemp ++ local LAST_ERR=/tmp/tmp.RIKdYgsQUN ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl exec psmdb-client-bb8b97679-hr2qb -- bash -c 'printf '\''rs.conf().members.length\n'\'' | mongo mongodb://clusterAdmin:clusterAdmin123456@34.118.237.97:27017/admin ' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.z3CupPcApH ++ cat /tmp/tmp.RIKdYgsQUN ++ rm /tmp/tmp.z3CupPcApH /tmp/tmp.RIKdYgsQUN ++ return 0 + nodes_count=6 + echo -n 'waiting for all members to be configured in rs0' waiting for all members to be configured in rs0+ let retry+=1 + '[' 10 -ge 15 ']' + echo . . + sleep 10 + [[ 6 == 6 ]] + wait_for_members 34.118.234.85 rs1 6 + local endpoint=34.118.234.85 + local rsName=rs1 + local target_count=6 + local nodes_count=0 + [[ 0 == 6 ]] ++ run_mongos 'rs.conf().members.length' clusterAdmin:clusterAdmin123456@34.118.234.85 mongodb :27017 ++ local 'command=rs.conf().members.length' ++ local uri=clusterAdmin:clusterAdmin123456@34.118.234.85 ++ local driver=mongodb ++ local suffix=:27017 ++ local mongo_flag= ++ local port=27017 ++ local mongo_bin=mongo ++ grep -E -v 'I NETWORK|W NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|bye' ++ /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/' +++ echo :27017 +++ awk -F: '{print $2}' ++ suffix_port=27017 ++ [[ -z 27017 ]] +++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.q63HARiu2J ++++ mktemp +++ local LAST_ERR=/tmp/tmp.VfkgIrMv4p +++ local exit_status=0 +++ local timeout=4 ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 -a -n 1 ']' +++ break +++ cat /tmp/tmp.q63HARiu2J +++ cat /tmp/tmp.VfkgIrMv4p +++ rm /tmp/tmp.q63HARiu2J /tmp/tmp.VfkgIrMv4p +++ return 0 ++ local client_container=psmdb-client-bb8b97679-hr2qb ++ kubectl_bin exec psmdb-client-bb8b97679-hr2qb -- bash -c 'printf '\''rs.conf().members.length\n'\'' | mongo mongodb://clusterAdmin:clusterAdmin123456@34.118.234.85:27017/admin ' +++ mktemp ++ local LAST_OUT=/tmp/tmp.nzONfzyddi +++ mktemp ++ local LAST_ERR=/tmp/tmp.a2KWWLXapP ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl exec psmdb-client-bb8b97679-hr2qb -- bash -c 'printf '\''rs.conf().members.length\n'\'' | mongo mongodb://clusterAdmin:clusterAdmin123456@34.118.234.85:27017/admin ' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.nzONfzyddi ++ cat /tmp/tmp.a2KWWLXapP ++ rm /tmp/tmp.nzONfzyddi /tmp/tmp.a2KWWLXapP ++ return 0 + nodes_count=6 + echo -n 'waiting for all members to be configured in rs1' waiting for all members to be configured in rs1+ let retry+=1 + '[' 11 -ge 15 ']' + echo . . + sleep 10 + [[ 6 == 6 ]] ++ kubectl_bin config current-context +++ mktemp ++ local LAST_OUT=/tmp/tmp.7MkNTm6NOC +++ mktemp ++ local LAST_ERR=/tmp/tmp.q93JSjjLAZ ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl config current-context ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.7MkNTm6NOC ++ cat /tmp/tmp.q93JSjjLAZ ++ rm /tmp/tmp.7MkNTm6NOC /tmp/tmp.q93JSjjLAZ ++ return 0 + kubectl_bin config set-context gke_cloud-dev-112233_us-central1-a_jen-psmdb-2058-fb8cc7187-17-cluster3 --namespace=cross-site-sharded-replica-2749 ++ mktemp + local LAST_OUT=/tmp/tmp.zLL4yVTiGF ++ mktemp + local LAST_ERR=/tmp/tmp.vSliSHmi78 + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl config set-context gke_cloud-dev-112233_us-central1-a_jen-psmdb-2058-fb8cc7187-17-cluster3 --namespace=cross-site-sharded-replica-2749 + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.zLL4yVTiGF Context "gke_cloud-dev-112233_us-central1-a_jen-psmdb-2058-fb8cc7187-17-cluster3" modified. + cat /tmp/tmp.vSliSHmi78 + rm /tmp/tmp.zLL4yVTiGF /tmp/tmp.vSliSHmi78 + return 0 + desc 'check if all 3 Pods started' + set +o xtrace ----------------------------------------------------------------------------------- check if all 3 Pods started ----------------------------------------------------------------------------------- + wait_for_running cross-site-sharded-replica-rs0 3 + local name=cross-site-sharded-replica-rs0 + let last_pod=2 + local check_cluster_readyness=true + set_debug + [[ 1 == 1 ]] + set -o xtrace + local rs_name=rs0 + local cluster_name=cross-site-sharded-replica ++ seq 0 2 + for i in $(seq 0 $last_pod) + [[ 0 -eq 2 ]] + wait_pod cross-site-sharded-replica-rs0-0 + local pod=cross-site-sharded-replica-rs0-0 + set +o xtrace waiting for pod/cross-site-sharded-replica-rs0-0 to be ready.OK + for i in $(seq 0 $last_pod) + [[ 1 -eq 2 ]] + wait_pod cross-site-sharded-replica-rs0-1 + local pod=cross-site-sharded-replica-rs0-1 + set +o xtrace waiting for pod/cross-site-sharded-replica-rs0-1 to be ready.OK + for i in $(seq 0 $last_pod) + [[ 2 -eq 2 ]] ++ kubectl_bin get psmdb cross-site-sharded-replica -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].arbiter.enabled}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.dMGUFEtfDF +++ mktemp ++ local LAST_ERR=/tmp/tmp.BR6zK1itOa ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb cross-site-sharded-replica -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].arbiter.enabled}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.dMGUFEtfDF ++ cat /tmp/tmp.BR6zK1itOa ++ rm /tmp/tmp.dMGUFEtfDF /tmp/tmp.BR6zK1itOa ++ return 0 + [[ false == true ]] + wait_pod cross-site-sharded-replica-rs0-2 + local pod=cross-site-sharded-replica-rs0-2 + set +o xtrace waiting for pod/cross-site-sharded-replica-rs0-2 to be ready.OK ++ kubectl_bin get psmdb cross-site-sharded-replica -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].nonvoting.enabled}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.vYlpLIilo9 +++ mktemp ++ local LAST_ERR=/tmp/tmp.4dFMse6iFc ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb cross-site-sharded-replica -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].nonvoting.enabled}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.vYlpLIilo9 ++ cat /tmp/tmp.4dFMse6iFc ++ rm /tmp/tmp.vYlpLIilo9 /tmp/tmp.4dFMse6iFc ++ return 0 + [[ false == true ]] ++ kubectl_bin get psmdb cross-site-sharded-replica -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].hidden.enabled}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.o2P1BMm0wq +++ mktemp ++ local LAST_ERR=/tmp/tmp.DTzQHUPktU ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb cross-site-sharded-replica -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].hidden.enabled}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.o2P1BMm0wq ++ cat /tmp/tmp.DTzQHUPktU ++ rm /tmp/tmp.o2P1BMm0wq /tmp/tmp.DTzQHUPktU ++ return 0 + [[ '' == true ]] + sleep 10 + [[ true == true ]] + set +x Waiting for cluster readyness + wait_for_running cross-site-sharded-replica-cfg 3 false + local name=cross-site-sharded-replica-cfg + let last_pod=2 + local check_cluster_readyness=false + set_debug + [[ 1 == 1 ]] + set -o xtrace + local rs_name=cfg + local cluster_name=cross-site-sharded-replica ++ seq 0 2 + for i in $(seq 0 $last_pod) + [[ 0 -eq 2 ]] + wait_pod cross-site-sharded-replica-cfg-0 + local pod=cross-site-sharded-replica-cfg-0 + set +o xtrace waiting for pod/cross-site-sharded-replica-cfg-0 to be ready.OK + for i in $(seq 0 $last_pod) + [[ 1 -eq 2 ]] + wait_pod cross-site-sharded-replica-cfg-1 + local pod=cross-site-sharded-replica-cfg-1 + set +o xtrace waiting for pod/cross-site-sharded-replica-cfg-1 to be ready.OK + for i in $(seq 0 $last_pod) + [[ 2 -eq 2 ]] ++ kubectl_bin get psmdb cross-site-sharded-replica -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].arbiter.enabled}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.CRsLLz3KXR +++ mktemp ++ local LAST_ERR=/tmp/tmp.xQaUM9cyIo ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb cross-site-sharded-replica -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].arbiter.enabled}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.CRsLLz3KXR ++ cat /tmp/tmp.xQaUM9cyIo ++ rm /tmp/tmp.CRsLLz3KXR /tmp/tmp.xQaUM9cyIo ++ return 0 + [[ '' == true ]] + wait_pod cross-site-sharded-replica-cfg-2 + local pod=cross-site-sharded-replica-cfg-2 + set +o xtrace waiting for pod/cross-site-sharded-replica-cfg-2 to be ready.OK ++ kubectl_bin get psmdb cross-site-sharded-replica -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].nonvoting.enabled}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.ERLp1VNnMQ +++ mktemp ++ local LAST_ERR=/tmp/tmp.1VLcB0BFXK ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb cross-site-sharded-replica -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].nonvoting.enabled}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.ERLp1VNnMQ ++ cat /tmp/tmp.1VLcB0BFXK ++ rm /tmp/tmp.ERLp1VNnMQ /tmp/tmp.1VLcB0BFXK ++ return 0 + [[ '' == true ]] ++ kubectl_bin get psmdb cross-site-sharded-replica -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].hidden.enabled}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.hy9LdP1iVN +++ mktemp ++ local LAST_ERR=/tmp/tmp.GEVp5HvLo8 ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb cross-site-sharded-replica -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].hidden.enabled}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.hy9LdP1iVN ++ cat /tmp/tmp.GEVp5HvLo8 ++ rm /tmp/tmp.hy9LdP1iVN /tmp/tmp.GEVp5HvLo8 ++ return 0 + [[ '' == true ]] + sleep 10 + [[ false == true ]] + desc 'create user' + set +o xtrace ----------------------------------------------------------------------------------- create user ----------------------------------------------------------------------------------- + run_mongos 'db.createUser({user:"myApp",pwd:"myPass",roles:[{db:"myApp",role:"readWrite"}]})' userAdmin:userAdmin123456@cross-site-sharded-main-mongos.cross-site-sharded-5222 + local 'command=db.createUser({user:"myApp",pwd:"myPass",roles:[{db:"myApp",role:"readWrite"}]})' + local uri=userAdmin:userAdmin123456@cross-site-sharded-main-mongos.cross-site-sharded-5222 + local driver=mongodb + local suffix=.svc.cluster.local + local mongo_flag= + local port=27017 + local mongo_bin=mongo ++ echo .svc.cluster.local ++ awk -F: '{print $2}' + suffix_port= + [[ -z '' ]] + suffix=.svc.cluster.local:27017 ++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.isvqSXCoKq +++ mktemp ++ local LAST_ERR=/tmp/tmp.9MqDNM1gtC ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.isvqSXCoKq ++ cat /tmp/tmp.9MqDNM1gtC ++ rm /tmp/tmp.isvqSXCoKq /tmp/tmp.9MqDNM1gtC ++ return 0 + local client_container=psmdb-client-bb8b97679-zhhh5 + kubectl_bin exec psmdb-client-bb8b97679-zhhh5 -- bash -c 'printf '\''db.createUser({user:"myApp",pwd:"myPass",roles:[{db:"myApp",role:"readWrite"}]})\n'\'' | mongo mongodb://userAdmin:userAdmin123456@cross-site-sharded-main-mongos.cross-site-sharded-5222.svc.cluster.local:27017/admin ' ++ mktemp + local LAST_OUT=/tmp/tmp.SoNI4dFRrT ++ mktemp + local LAST_ERR=/tmp/tmp.eATO7iFzvD + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl exec psmdb-client-bb8b97679-zhhh5 -- bash -c 'printf '\''db.createUser({user:"myApp",pwd:"myPass",roles:[{db:"myApp",role:"readWrite"}]})\n'\'' | mongo mongodb://userAdmin:userAdmin123456@cross-site-sharded-main-mongos.cross-site-sharded-5222.svc.cluster.local:27017/admin ' + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.SoNI4dFRrT Percona Server for MongoDB shell version v4.4.29-28 connecting to: mongodb://cross-site-sharded-main-mongos.cross-site-sharded-5222.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb Implicit session: session { "id" : UUID("4ff43397-22b1-4c85-9c7d-367390d5ed75") } Percona Server for MongoDB server version: v8.0.23-10 WARNING: shell and server versions do not match Successfully added user: { "user" : "myApp", "roles" : [ { "db" : "myApp", "role" : "readWrite" } ] } bye + cat /tmp/tmp.eATO7iFzvD + rm /tmp/tmp.SoNI4dFRrT /tmp/tmp.eATO7iFzvD + return 0 + sleep 2 + desc 'write data, read from all' + set +o xtrace ----------------------------------------------------------------------------------- write data, read from all ----------------------------------------------------------------------------------- + run_mongos 'use myApp\n db.test.insert({ x: 100500 })' myApp:myPass@cross-site-sharded-main-mongos.cross-site-sharded-5222 + local 'command=use myApp\n db.test.insert({ x: 100500 })' + local uri=myApp:myPass@cross-site-sharded-main-mongos.cross-site-sharded-5222 + local driver=mongodb + local suffix=.svc.cluster.local + local mongo_flag= + local port=27017 + local mongo_bin=mongo ++ echo .svc.cluster.local ++ awk -F: '{print $2}' + suffix_port= + [[ -z '' ]] + suffix=.svc.cluster.local:27017 ++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.cxgz4E67zw +++ mktemp ++ local LAST_ERR=/tmp/tmp.xcggoXBxQt ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.cxgz4E67zw ++ cat /tmp/tmp.xcggoXBxQt ++ rm /tmp/tmp.cxgz4E67zw /tmp/tmp.xcggoXBxQt ++ return 0 + local client_container=psmdb-client-bb8b97679-zhhh5 + kubectl_bin exec psmdb-client-bb8b97679-zhhh5 -- bash -c 'printf '\''use myApp\n db.test.insert({ x: 100500 })\n'\'' | mongo mongodb://myApp:myPass@cross-site-sharded-main-mongos.cross-site-sharded-5222.svc.cluster.local:27017/admin ' ++ mktemp + local LAST_OUT=/tmp/tmp.gYdoEiGaBB ++ mktemp + local LAST_ERR=/tmp/tmp.wg6rfxXfMI + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl exec psmdb-client-bb8b97679-zhhh5 -- bash -c 'printf '\''use myApp\n db.test.insert({ x: 100500 })\n'\'' | mongo mongodb://myApp:myPass@cross-site-sharded-main-mongos.cross-site-sharded-5222.svc.cluster.local:27017/admin ' + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.gYdoEiGaBB Percona Server for MongoDB shell version v4.4.29-28 connecting to: mongodb://cross-site-sharded-main-mongos.cross-site-sharded-5222.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb Implicit session: session { "id" : UUID("f1f365f2-fbde-4250-b95d-3dc816e8a144") } Percona Server for MongoDB server version: v8.0.23-10 WARNING: shell and server versions do not match switched to db myApp WriteResult({ "nInserted" : 1 }) bye + cat /tmp/tmp.wg6rfxXfMI + rm /tmp/tmp.gYdoEiGaBB /tmp/tmp.wg6rfxXfMI + return 0 + minikube_sleep + sleep_time=10 + [[ '' == 1 ]] + desc 'Compare data' + set +o xtrace ----------------------------------------------------------------------------------- Compare data ----------------------------------------------------------------------------------- + compare_mongos_cmd find myApp:myPass@cross-site-sharded-main-mongos.cross-site-sharded-5222 + local command=find + local uri=myApp:myPass@cross-site-sharded-main-mongos.cross-site-sharded-5222 + local postfix= + local suffix= + local database=myApp + local collection=test + local port=27017 + local tls=false + [[ false == true ]] + mongos_command=run_mongos + log 'running db.test.command() in myApp' + set +o xtrace [2026-06-09T07:49:57+0000] running db.test.command() in myApp + run_mongos 'use myApp\n db.test.find()' myApp:myPass@cross-site-sharded-main-mongos.cross-site-sharded-5222 mongodb '' '' 27017 + grep -E -v 'I NETWORK|W NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|Started a new thread for the timer service' + local 'command=use myApp\n db.test.find()' + local uri=myApp:myPass@cross-site-sharded-main-mongos.cross-site-sharded-5222 + local driver=mongodb + local suffix=.svc.cluster.local + local mongo_flag= + /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/' + local port=27017 + local mongo_bin=mongo ++ echo .svc.cluster.local ++ awk -F: '{print $2}' + suffix_port= + [[ -z '' ]] + suffix=.svc.cluster.local:27017 ++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.xH9Z6GRGLQ +++ mktemp ++ local LAST_ERR=/tmp/tmp.I2rxfCIDSM ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.xH9Z6GRGLQ ++ cat /tmp/tmp.I2rxfCIDSM ++ rm /tmp/tmp.xH9Z6GRGLQ /tmp/tmp.I2rxfCIDSM ++ return 0 + local client_container=psmdb-client-bb8b97679-zhhh5 + kubectl_bin exec psmdb-client-bb8b97679-zhhh5 -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@cross-site-sharded-main-mongos.cross-site-sharded-5222.svc.cluster.local:27017/admin ' ++ mktemp + local LAST_OUT=/tmp/tmp.jh3IuBHBy8 ++ mktemp + local LAST_ERR=/tmp/tmp.Nb8INt1ncI + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl exec psmdb-client-bb8b97679-zhhh5 -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@cross-site-sharded-main-mongos.cross-site-sharded-5222.svc.cluster.local:27017/admin ' + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.jh3IuBHBy8 + cat /tmp/tmp.Nb8INt1ncI + rm /tmp/tmp.jh3IuBHBy8 /tmp/tmp.Nb8INt1ncI + return 0 + [[ 0 -eq 0 ]] + diff /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/e2e-tests/cross-site-sharded/compare/find.json /tmp/tmp.GcNXT2EgJ5/find + desc 'test failover' + set +o xtrace ----------------------------------------------------------------------------------- test failover ----------------------------------------------------------------------------------- ++ kubectl_bin config current-context +++ mktemp ++ local LAST_OUT=/tmp/tmp.QgGJ7xuHPN +++ mktemp ++ local LAST_ERR=/tmp/tmp.DrFlH0iFKD ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl config current-context ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.QgGJ7xuHPN ++ cat /tmp/tmp.DrFlH0iFKD ++ rm /tmp/tmp.QgGJ7xuHPN /tmp/tmp.DrFlH0iFKD ++ return 0 + kubectl_bin config set-context gke_cloud-dev-112233_us-central1-a_jen-psmdb-2058-fb8cc7187-17-cluster3 --namespace=cross-site-sharded-5222 ++ mktemp + local LAST_OUT=/tmp/tmp.7LuT2C7mmu ++ mktemp + local LAST_ERR=/tmp/tmp.RjHspbOd5G + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl config set-context gke_cloud-dev-112233_us-central1-a_jen-psmdb-2058-fb8cc7187-17-cluster3 --namespace=cross-site-sharded-5222 + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.7LuT2C7mmu Context "gke_cloud-dev-112233_us-central1-a_jen-psmdb-2058-fb8cc7187-17-cluster3" modified. + cat /tmp/tmp.RjHspbOd5G + rm /tmp/tmp.7LuT2C7mmu /tmp/tmp.RjHspbOd5G + return 0 + kubectl_bin delete psmdb cross-site-sharded-main ++ mktemp + local LAST_OUT=/tmp/tmp.q22MBAP6ty ++ mktemp + local LAST_ERR=/tmp/tmp.iKHZrT6bY3 + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl delete psmdb cross-site-sharded-main + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.q22MBAP6ty perconaservermongodb.psmdb.percona.com "cross-site-sharded-main" deleted from cross-site-sharded-5222 namespace + cat /tmp/tmp.iKHZrT6bY3 + rm /tmp/tmp.q22MBAP6ty /tmp/tmp.iKHZrT6bY3 + return 0 + desc 'run disaster recovery script for replset: cfg' + set +o xtrace ----------------------------------------------------------------------------------- run disaster recovery script for replset: cfg ----------------------------------------------------------------------------------- + run_script_mongos /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/e2e-tests/cross-site-sharded/disaster_recovery.js clusterAdmin:clusterAdmin123456@34.118.234.161 mongodb :27017 + local script=/mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/e2e-tests/cross-site-sharded/disaster_recovery.js + local uri=clusterAdmin:clusterAdmin123456@34.118.234.161 + local driver=mongodb + local suffix=:27017 + local mongo_flag= + local mongo_bin=mongo ++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.yTJTSbk9K9 +++ mktemp ++ local LAST_ERR=/tmp/tmp.XJhWdGuN9B ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.yTJTSbk9K9 ++ cat /tmp/tmp.XJhWdGuN9B ++ rm /tmp/tmp.yTJTSbk9K9 /tmp/tmp.XJhWdGuN9B ++ return 0 + local client_container=psmdb-client-bb8b97679-hr2qb ++ basename /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/e2e-tests/cross-site-sharded/disaster_recovery.js + name=disaster_recovery.js + kubectl_bin cp /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/e2e-tests/cross-site-sharded/disaster_recovery.js cross-site-sharded-5222/psmdb-client-bb8b97679-hr2qb:/tmp ++ mktemp + local LAST_OUT=/tmp/tmp.Ju5KvZpinC ++ mktemp + local LAST_ERR=/tmp/tmp.kV4szOB3ge + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl cp /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/e2e-tests/cross-site-sharded/disaster_recovery.js cross-site-sharded-5222/psmdb-client-bb8b97679-hr2qb:/tmp + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.Ju5KvZpinC + cat /tmp/tmp.kV4szOB3ge + rm /tmp/tmp.Ju5KvZpinC /tmp/tmp.kV4szOB3ge + return 0 + kubectl_bin exec psmdb-client-bb8b97679-hr2qb -- bash -c 'mongo mongodb://clusterAdmin:clusterAdmin123456@34.118.234.161:27017/admin /tmp/disaster_recovery.js' ++ mktemp + local LAST_OUT=/tmp/tmp.WZSZ879QtO ++ mktemp + local LAST_ERR=/tmp/tmp.zBUJy6LaCG + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl exec psmdb-client-bb8b97679-hr2qb -- bash -c 'mongo mongodb://clusterAdmin:clusterAdmin123456@34.118.234.161:27017/admin /tmp/disaster_recovery.js' + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.WZSZ879QtO Percona Server for MongoDB shell version v4.4.29-28 connecting to: mongodb://34.118.234.161:27017/admin?compressors=disabled&gssapiServiceName=mongodb Implicit session: session { "id" : UUID("f4c40e58-e3ff-47a9-9544-de1a5e929135") } Percona Server for MongoDB server version: v8.0.23-10 WARNING: shell and server versions do not match + cat /tmp/tmp.zBUJy6LaCG + rm /tmp/tmp.WZSZ879QtO /tmp/tmp.zBUJy6LaCG + return 0 + desc 'run disaster recovery script for replset: rs0' + set +o xtrace ----------------------------------------------------------------------------------- run disaster recovery script for replset: rs0 ----------------------------------------------------------------------------------- + run_script_mongos /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/e2e-tests/cross-site-sharded/disaster_recovery.js clusterAdmin:clusterAdmin123456@34.118.237.97 mongodb :27017 + local script=/mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/e2e-tests/cross-site-sharded/disaster_recovery.js + local uri=clusterAdmin:clusterAdmin123456@34.118.237.97 + local driver=mongodb + local suffix=:27017 + local mongo_flag= + local mongo_bin=mongo ++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.MgsPdhCbpt +++ mktemp ++ local LAST_ERR=/tmp/tmp.ZBHNB4XwBO ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.MgsPdhCbpt ++ cat /tmp/tmp.ZBHNB4XwBO ++ rm /tmp/tmp.MgsPdhCbpt /tmp/tmp.ZBHNB4XwBO ++ return 0 + local client_container=psmdb-client-bb8b97679-hr2qb ++ basename /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/e2e-tests/cross-site-sharded/disaster_recovery.js + name=disaster_recovery.js + kubectl_bin cp /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/e2e-tests/cross-site-sharded/disaster_recovery.js cross-site-sharded-5222/psmdb-client-bb8b97679-hr2qb:/tmp ++ mktemp + local LAST_OUT=/tmp/tmp.WstRD4vU0o ++ mktemp + local LAST_ERR=/tmp/tmp.R3LSPL3dC2 + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl cp /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/e2e-tests/cross-site-sharded/disaster_recovery.js cross-site-sharded-5222/psmdb-client-bb8b97679-hr2qb:/tmp + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.WstRD4vU0o + cat /tmp/tmp.R3LSPL3dC2 + rm /tmp/tmp.WstRD4vU0o /tmp/tmp.R3LSPL3dC2 + return 0 + kubectl_bin exec psmdb-client-bb8b97679-hr2qb -- bash -c 'mongo mongodb://clusterAdmin:clusterAdmin123456@34.118.237.97:27017/admin /tmp/disaster_recovery.js' ++ mktemp + local LAST_OUT=/tmp/tmp.MJxrg9ARrE ++ mktemp + local LAST_ERR=/tmp/tmp.5lJDfiNG6d + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl exec psmdb-client-bb8b97679-hr2qb -- bash -c 'mongo mongodb://clusterAdmin:clusterAdmin123456@34.118.237.97:27017/admin /tmp/disaster_recovery.js' + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.MJxrg9ARrE Percona Server for MongoDB shell version v4.4.29-28 connecting to: mongodb://34.118.237.97:27017/admin?compressors=disabled&gssapiServiceName=mongodb Implicit session: session { "id" : UUID("0b73a667-7d46-4090-bbc5-7b2d5ec19c35") } Percona Server for MongoDB server version: v8.0.23-10 WARNING: shell and server versions do not match + cat /tmp/tmp.5lJDfiNG6d + rm /tmp/tmp.MJxrg9ARrE /tmp/tmp.5lJDfiNG6d + return 0 + desc 'run disaster recovery script for replset: rs1' + set +o xtrace ----------------------------------------------------------------------------------- run disaster recovery script for replset: rs1 ----------------------------------------------------------------------------------- + run_script_mongos /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/e2e-tests/cross-site-sharded/disaster_recovery.js clusterAdmin:clusterAdmin123456@34.118.234.85 mongodb :27017 + local script=/mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/e2e-tests/cross-site-sharded/disaster_recovery.js + local uri=clusterAdmin:clusterAdmin123456@34.118.234.85 + local driver=mongodb + local suffix=:27017 + local mongo_flag= + local mongo_bin=mongo ++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.To3iHkana7 +++ mktemp ++ local LAST_ERR=/tmp/tmp.VSg1t8EfQJ ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.To3iHkana7 ++ cat /tmp/tmp.VSg1t8EfQJ ++ rm /tmp/tmp.To3iHkana7 /tmp/tmp.VSg1t8EfQJ ++ return 0 + local client_container=psmdb-client-bb8b97679-hr2qb ++ basename /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/e2e-tests/cross-site-sharded/disaster_recovery.js + name=disaster_recovery.js + kubectl_bin cp /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/e2e-tests/cross-site-sharded/disaster_recovery.js cross-site-sharded-5222/psmdb-client-bb8b97679-hr2qb:/tmp ++ mktemp + local LAST_OUT=/tmp/tmp.AUW9AvUM9F ++ mktemp + local LAST_ERR=/tmp/tmp.kjPphRUnYj + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl cp /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/e2e-tests/cross-site-sharded/disaster_recovery.js cross-site-sharded-5222/psmdb-client-bb8b97679-hr2qb:/tmp + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.AUW9AvUM9F + cat /tmp/tmp.kjPphRUnYj + rm /tmp/tmp.AUW9AvUM9F /tmp/tmp.kjPphRUnYj + return 0 + kubectl_bin exec psmdb-client-bb8b97679-hr2qb -- bash -c 'mongo mongodb://clusterAdmin:clusterAdmin123456@34.118.234.85:27017/admin /tmp/disaster_recovery.js' ++ mktemp + local LAST_OUT=/tmp/tmp.IoMdcIm848 ++ mktemp + local LAST_ERR=/tmp/tmp.IMFL1gehRH + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl exec psmdb-client-bb8b97679-hr2qb -- bash -c 'mongo mongodb://clusterAdmin:clusterAdmin123456@34.118.234.85:27017/admin /tmp/disaster_recovery.js' + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.IoMdcIm848 Percona Server for MongoDB shell version v4.4.29-28 connecting to: mongodb://34.118.234.85:27017/admin?compressors=disabled&gssapiServiceName=mongodb Implicit session: session { "id" : UUID("e7dc6ff3-5a82-42fc-8088-5b01d90f784b") } Percona Server for MongoDB server version: v8.0.23-10 WARNING: shell and server versions do not match + cat /tmp/tmp.IMFL1gehRH + rm /tmp/tmp.IoMdcIm848 /tmp/tmp.IMFL1gehRH + return 0 + desc 'make replica cluster managed' + set +o xtrace ----------------------------------------------------------------------------------- make replica cluster managed ----------------------------------------------------------------------------------- ++ kubectl_bin config current-context +++ mktemp ++ local LAST_OUT=/tmp/tmp.zlbDdIDUB3 +++ mktemp ++ local LAST_ERR=/tmp/tmp.JcswbrQHAN ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl config current-context ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.zlbDdIDUB3 ++ cat /tmp/tmp.JcswbrQHAN ++ rm /tmp/tmp.zlbDdIDUB3 /tmp/tmp.JcswbrQHAN ++ return 0 + kubectl_bin config set-context gke_cloud-dev-112233_us-central1-a_jen-psmdb-2058-fb8cc7187-17-cluster3 --namespace=cross-site-sharded-replica-2749 ++ mktemp + local LAST_OUT=/tmp/tmp.saj6Y6DzOI ++ mktemp + local LAST_ERR=/tmp/tmp.vLX3K7OKvC + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl config set-context gke_cloud-dev-112233_us-central1-a_jen-psmdb-2058-fb8cc7187-17-cluster3 --namespace=cross-site-sharded-replica-2749 + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.saj6Y6DzOI Context "gke_cloud-dev-112233_us-central1-a_jen-psmdb-2058-fb8cc7187-17-cluster3" modified. + cat /tmp/tmp.vLX3K7OKvC + rm /tmp/tmp.saj6Y6DzOI /tmp/tmp.vLX3K7OKvC + return 0 + kubectl_bin patch psmdb cross-site-sharded-replica --type=merge --patch '{"spec":{"unmanaged": false}}' ++ mktemp + local LAST_OUT=/tmp/tmp.O0hd26WgxC ++ mktemp + local LAST_ERR=/tmp/tmp.faNlVY4DIm + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl patch psmdb cross-site-sharded-replica --type=merge --patch '{"spec":{"unmanaged": false}}' + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.O0hd26WgxC perconaservermongodb.psmdb.percona.com/cross-site-sharded-replica patched + cat /tmp/tmp.faNlVY4DIm + rm /tmp/tmp.O0hd26WgxC /tmp/tmp.faNlVY4DIm + return 0 + wait_for_running cross-site-sharded-replica-rs0 3 + local name=cross-site-sharded-replica-rs0 + let last_pod=2 + local check_cluster_readyness=true + set_debug + [[ 1 == 1 ]] + set -o xtrace + local rs_name=rs0 + local cluster_name=cross-site-sharded-replica ++ seq 0 2 + for i in $(seq 0 $last_pod) + [[ 0 -eq 2 ]] + wait_pod cross-site-sharded-replica-rs0-0 + local pod=cross-site-sharded-replica-rs0-0 + set +o xtrace waiting for pod/cross-site-sharded-replica-rs0-0 to be ready.OK + for i in $(seq 0 $last_pod) + [[ 1 -eq 2 ]] + wait_pod cross-site-sharded-replica-rs0-1 + local pod=cross-site-sharded-replica-rs0-1 + set +o xtrace waiting for pod/cross-site-sharded-replica-rs0-1 to be ready.OK + for i in $(seq 0 $last_pod) + [[ 2 -eq 2 ]] ++ kubectl_bin get psmdb cross-site-sharded-replica -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].arbiter.enabled}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.dc8i9ItDTW +++ mktemp ++ local LAST_ERR=/tmp/tmp.KX8MIky7Mc ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb cross-site-sharded-replica -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].arbiter.enabled}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.dc8i9ItDTW ++ cat /tmp/tmp.KX8MIky7Mc ++ rm /tmp/tmp.dc8i9ItDTW /tmp/tmp.KX8MIky7Mc ++ return 0 + [[ false == true ]] + wait_pod cross-site-sharded-replica-rs0-2 + local pod=cross-site-sharded-replica-rs0-2 + set +o xtrace waiting for pod/cross-site-sharded-replica-rs0-2 to be ready.OK ++ kubectl_bin get psmdb cross-site-sharded-replica -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].nonvoting.enabled}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.Z0dpZsl3ba +++ mktemp ++ local LAST_ERR=/tmp/tmp.TMbkeuLmcw ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb cross-site-sharded-replica -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].nonvoting.enabled}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.Z0dpZsl3ba ++ cat /tmp/tmp.TMbkeuLmcw ++ rm /tmp/tmp.Z0dpZsl3ba /tmp/tmp.TMbkeuLmcw ++ return 0 + [[ false == true ]] ++ kubectl_bin get psmdb cross-site-sharded-replica -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].hidden.enabled}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.XMVvBJSP1P +++ mktemp ++ local LAST_ERR=/tmp/tmp.4OYzjmJAYM ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb cross-site-sharded-replica -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].hidden.enabled}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.XMVvBJSP1P ++ cat /tmp/tmp.4OYzjmJAYM ++ rm /tmp/tmp.XMVvBJSP1P /tmp/tmp.4OYzjmJAYM ++ return 0 + [[ '' == true ]] + sleep 10 + [[ true == true ]] + set +x Waiting for cluster readyness.. + wait_for_running cross-site-sharded-replica-cfg 3 + local name=cross-site-sharded-replica-cfg + let last_pod=2 + local check_cluster_readyness=true + set_debug + [[ 1 == 1 ]] + set -o xtrace + local rs_name=cfg + local cluster_name=cross-site-sharded-replica ++ seq 0 2 + for i in $(seq 0 $last_pod) + [[ 0 -eq 2 ]] + wait_pod cross-site-sharded-replica-cfg-0 + local pod=cross-site-sharded-replica-cfg-0 + set +o xtrace waiting for pod/cross-site-sharded-replica-cfg-0 to be ready.OK + for i in $(seq 0 $last_pod) + [[ 1 -eq 2 ]] + wait_pod cross-site-sharded-replica-cfg-1 + local pod=cross-site-sharded-replica-cfg-1 + set +o xtrace waiting for pod/cross-site-sharded-replica-cfg-1 to be ready.OK + for i in $(seq 0 $last_pod) + [[ 2 -eq 2 ]] ++ kubectl_bin get psmdb cross-site-sharded-replica -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].arbiter.enabled}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.L18ajPzUsr +++ mktemp ++ local LAST_ERR=/tmp/tmp.fhNrsZYy4I ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb cross-site-sharded-replica -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].arbiter.enabled}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.L18ajPzUsr ++ cat /tmp/tmp.fhNrsZYy4I ++ rm /tmp/tmp.L18ajPzUsr /tmp/tmp.fhNrsZYy4I ++ return 0 + [[ '' == true ]] + wait_pod cross-site-sharded-replica-cfg-2 + local pod=cross-site-sharded-replica-cfg-2 + set +o xtrace waiting for pod/cross-site-sharded-replica-cfg-2 to be ready.OK ++ kubectl_bin get psmdb cross-site-sharded-replica -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].nonvoting.enabled}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.QmlsOV1ROk +++ mktemp ++ local LAST_ERR=/tmp/tmp.93PpmrgPi4 ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb cross-site-sharded-replica -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].nonvoting.enabled}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.QmlsOV1ROk ++ cat /tmp/tmp.93PpmrgPi4 ++ rm /tmp/tmp.QmlsOV1ROk /tmp/tmp.93PpmrgPi4 ++ return 0 + [[ '' == true ]] ++ kubectl_bin get psmdb cross-site-sharded-replica -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].hidden.enabled}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.iokYnBo1Oz +++ mktemp ++ local LAST_ERR=/tmp/tmp.YZZnzLSxLS ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb cross-site-sharded-replica -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].hidden.enabled}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.iokYnBo1Oz ++ cat /tmp/tmp.YZZnzLSxLS ++ rm /tmp/tmp.iokYnBo1Oz /tmp/tmp.YZZnzLSxLS ++ return 0 + [[ '' == true ]] + sleep 10 + [[ true == true ]] + set +x Waiting for cluster readyness + desc 'check failover status' + set +o xtrace ----------------------------------------------------------------------------------- check failover status ----------------------------------------------------------------------------------- + compare_mongos_cmd find myApp:myPass@cross-site-sharded-replica-mongos.cross-site-sharded-replica-2749 + local command=find + local uri=myApp:myPass@cross-site-sharded-replica-mongos.cross-site-sharded-replica-2749 + local postfix= + local suffix= + local database=myApp + local collection=test + local port=27017 + local tls=false + [[ false == true ]] + mongos_command=run_mongos + log 'running db.test.command() in myApp' + set +o xtrace [2026-06-09T07:50:56+0000] running db.test.command() in myApp + run_mongos 'use myApp\n db.test.find()' myApp:myPass@cross-site-sharded-replica-mongos.cross-site-sharded-replica-2749 mongodb '' '' 27017 + local 'command=use myApp\n db.test.find()' + local uri=myApp:myPass@cross-site-sharded-replica-mongos.cross-site-sharded-replica-2749 + local driver=mongodb + local suffix=.svc.cluster.local + local mongo_flag= + local port=27017 + local mongo_bin=mongo + grep -E -v 'I NETWORK|W NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|Started a new thread for the timer service' + /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/' ++ awk -F: '{print $2}' ++ echo .svc.cluster.local + suffix_port= + [[ -z '' ]] + suffix=.svc.cluster.local:27017 ++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.xIcSmUVAko +++ mktemp ++ local LAST_ERR=/tmp/tmp.8QAyZAISLm ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.xIcSmUVAko ++ cat /tmp/tmp.8QAyZAISLm ++ rm /tmp/tmp.xIcSmUVAko /tmp/tmp.8QAyZAISLm ++ return 0 + local client_container=psmdb-client-bb8b97679-zhhh5 + kubectl_bin exec psmdb-client-bb8b97679-zhhh5 -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@cross-site-sharded-replica-mongos.cross-site-sharded-replica-2749.svc.cluster.local:27017/admin ' ++ mktemp + local LAST_OUT=/tmp/tmp.euoxIBUK2P ++ mktemp + local LAST_ERR=/tmp/tmp.OHGHpLJwYq + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl exec psmdb-client-bb8b97679-zhhh5 -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@cross-site-sharded-replica-mongos.cross-site-sharded-replica-2749.svc.cluster.local:27017/admin ' + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.euoxIBUK2P + cat /tmp/tmp.OHGHpLJwYq + rm /tmp/tmp.euoxIBUK2P /tmp/tmp.OHGHpLJwYq + return 0 + [[ 0 -eq 0 ]] + diff /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/e2e-tests/cross-site-sharded/compare/find.json /tmp/tmp.GcNXT2EgJ5/find + desc 'Failover check finished successfully' + set +o xtrace ----------------------------------------------------------------------------------- Failover check finished successfully ----------------------------------------------------------------------------------- + wait_cluster_consistency cross-site-sharded-replica + local cluster_name=cross-site-sharded-replica + local wait_time=32 + retry=0 + sleep 7 + echo -n 'waiting for cluster readyness' waiting for cluster readyness++ kubectl_bin get psmdb cross-site-sharded-replica -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.FmGxIwZBCY +++ mktemp ++ local LAST_ERR=/tmp/tmp.oFbIjblQ1B ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb cross-site-sharded-replica -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.FmGxIwZBCY ++ cat /tmp/tmp.oFbIjblQ1B ++ rm /tmp/tmp.FmGxIwZBCY /tmp/tmp.oFbIjblQ1B ++ return 0 + [[ ready == ready ]] + echo .OK .OK + destroy cross-site-sharded-5222 true + local namespace=cross-site-sharded-5222 + local ignore_logs=true + [[ 1 == 1 ]] + echo 'SKIP_DELETE=1, not destroying cross-site-sharded-5222' SKIP_DELETE=1, not destroying cross-site-sharded-5222 + return + destroy cross-site-sharded-replica-2749 true + local namespace=cross-site-sharded-replica-2749 + local ignore_logs=true + [[ 1 == 1 ]] + echo 'SKIP_DELETE=1, not destroying cross-site-sharded-replica-2749' SKIP_DELETE=1, not destroying cross-site-sharded-replica-2749 + return + desc 'test passed' + set +o xtrace ----------------------------------------------------------------------------------- test passed -----------------------------------------------------------------------------------