Log: /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2271/e2e-tests/logs/cross-site-sharded.log Warning: version difference between client (1.35) and server (1.32) exceeds the supported minor version skew of +/-1 Warning: version difference between client (1.35) and server (1.32) exceeds the supported minor version skew of +/-1 Warning: version difference between client (1.35) and server (1.32) exceeds the supported minor version skew of +/-1 ++ get_mongod_ver_from_image docker.io/perconalab/percona-server-mongodb-operator:main-mongod8.0 ++ local image=docker.io/perconalab/percona-server-mongodb-operator:main-mongod8.0 +++ run_simple_cli_inside_image docker.io/perconalab/percona-server-mongodb-operator:main-mongod8.0 'mongod --version' +++ local image=docker.io/perconalab/percona-server-mongodb-operator:main-mongod8.0 +++ local 'cli=mongod --version' +++ local pod_name=23198 +++ kubectl_bin -n default run 23198 --image=docker.io/perconalab/percona-server-mongodb-operator:main-mongod8.0 --restart=Never --command -- sleep infinity +++ /usr/sbin/sed -r 's/^.*db version v(([0-9]+\.){2}[0-9]+-[0-9]+).*$/\1/g' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.pbqr71Xeww ++++ mktemp +++ local LAST_ERR=/tmp/tmp.bqoa2j3U05 +++ local exit_status=0 +++ local timeout=4 ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl -n default run 23198 --image=docker.io/perconalab/percona-server-mongodb-operator:main-mongod8.0 --restart=Never --command -- sleep infinity +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 -a -n 1 ']' +++ break +++ cat /tmp/tmp.pbqr71Xeww +++ cat /tmp/tmp.bqoa2j3U05 +++ rm /tmp/tmp.pbqr71Xeww /tmp/tmp.bqoa2j3U05 +++ return 0 +++ kubectl_bin -n default wait --for=condition=Ready pod/23198 ++++ mktemp +++ local LAST_OUT=/tmp/tmp.gteKapT3I3 ++++ mktemp +++ local LAST_ERR=/tmp/tmp.URu2nYxRbY +++ local exit_status=0 +++ local timeout=4 ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl -n default wait --for=condition=Ready pod/23198 +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 -a -n 1 ']' +++ break +++ cat /tmp/tmp.gteKapT3I3 +++ cat /tmp/tmp.URu2nYxRbY +++ rm /tmp/tmp.gteKapT3I3 /tmp/tmp.URu2nYxRbY +++ return 0 ++++ kubectl_bin -n default exec 23198 -- bash -c 'mongod --version 2>&1' +++++ mktemp ++++ local LAST_OUT=/tmp/tmp.Tx1ohW4qfZ +++++ mktemp ++++ local LAST_ERR=/tmp/tmp.6QTWAL9eXv ++++ local exit_status=0 ++++ local timeout=4 +++++ seq 0 2 ++++ for i in $(seq 0 2) ++++ set +e ++++ kubectl -n default exec 23198 -- bash -c 'mongod --version 2>&1' ++++ exit_status=0 ++++ set -e ++++ '[' 0 '!=' 0 -a -n 1 ']' ++++ break ++++ cat /tmp/tmp.Tx1ohW4qfZ ++++ cat /tmp/tmp.6QTWAL9eXv ++++ rm /tmp/tmp.Tx1ohW4qfZ /tmp/tmp.6QTWAL9eXv ++++ return 0 +++ local 'output=db version v8.0.19-7 Build Info: { "version": "8.0.19-7", "gitVersion": "902b12dbea631904a8175499a56c8bb95b708483", "openSSLVersion": "OpenSSL 3.5.1 1 Jul 2025", "modules": [], "perconaFeatures": [ "MemoryEngine", "HotBackup", "BackupCursorAggregationStage", "BackupCursorExtendAggregationStage", "AWSIAM", "Kerberos", "LDAP", "OIDC", "TDE", "FIPSMode", "FCBIS", "Auditing", "ProfilingRateLimit", "LogRedaction", "ngram" ], "allocator": "tcmalloc-google", "environment": { "distarch": "x86_64", "target_arch": "x86_64" } }' +++ kubectl_bin -n default delete pod/23198 --grace-period=0 --force ++++ mktemp +++ local LAST_OUT=/tmp/tmp.9WmazckYMI ++++ mktemp +++ local LAST_ERR=/tmp/tmp.nVILyDITZQ +++ local exit_status=0 +++ local timeout=4 ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl -n default delete pod/23198 --grace-period=0 --force +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 -a -n 1 ']' +++ break +++ cat /tmp/tmp.9WmazckYMI +++ cat /tmp/tmp.nVILyDITZQ Warning: Immediate deletion does not wait for confirmation that the running resource has been terminated. The resource may continue to run on the cluster indefinitely. +++ rm /tmp/tmp.9WmazckYMI /tmp/tmp.nVILyDITZQ +++ return 0 +++ echo db version v8.0.19-7 Build Info: '{' '"version":' '"8.0.19-7",' '"gitVersion":' '"902b12dbea631904a8175499a56c8bb95b708483",' '"openSSLVersion":' '"OpenSSL' 3.5.1 1 Jul '2025",' '"modules":' '[],' '"perconaFeatures":' '[' '"MemoryEngine",' '"HotBackup",' '"BackupCursorAggregationStage",' '"BackupCursorExtendAggregationStage",' '"AWSIAM",' '"Kerberos",' '"LDAP",' '"OIDC",' '"TDE",' '"FIPSMode",' '"FCBIS",' '"Auditing",' '"ProfilingRateLimit",' '"LogRedaction",' '"ngram"' '],' '"allocator":' '"tcmalloc-google",' '"environment":' '{' '"distarch":' '"x86_64",' '"target_arch":' '"x86_64"' '}' '}' ++ version_info=8.0.19-7 ++ [[ ! 8.0.19-7 =~ ^([0-9]+\.){2}[0-9]+-[0-9]+$ ]] ++ echo 8.0.19-7 + FULL_VER=8.0.19-7 + MONGO_VER=8.0 + unset OPERATOR_NS + main_cluster=cross-site-sharded-main + replica_cluster=cross-site-sharded-replica + desc 'create main cluster' + set +o xtrace ----------------------------------------------------------------------------------- create main cluster ----------------------------------------------------------------------------------- + create_infra cross-site-sharded-17853 + local ns=cross-site-sharded-17853 + [[ 1 == 1 ]] + delete_crd + desc 'get and delete old CRDs and RBAC' + set +o xtrace ----------------------------------------------------------------------------------- get and delete old CRDs and RBAC ----------------------------------------------------------------------------------- + kubectl_bin delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2271/deploy/crd.yaml --ignore-not-found --wait=false ++ mktemp + local LAST_OUT=/tmp/tmp.FLqZ3T62vg ++ mktemp + local LAST_ERR=/tmp/tmp.G0nvYVSyvH + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2271/deploy/crd.yaml --ignore-not-found --wait=false + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.FLqZ3T62vg customresourcedefinition.apiextensions.k8s.io "perconaservermongodbbackups.psmdb.percona.com" deleted customresourcedefinition.apiextensions.k8s.io "perconaservermongodbrestores.psmdb.percona.com" deleted customresourcedefinition.apiextensions.k8s.io "perconaservermongodbs.psmdb.percona.com" deleted + cat /tmp/tmp.G0nvYVSyvH + rm /tmp/tmp.FLqZ3T62vg /tmp/tmp.G0nvYVSyvH + return 0 ++ grep -v '\-\-\-' ++ yq eval .metadata.name /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2271/deploy/crd.yaml grep: warning: stray \ before - grep: warning: stray \ before - + for crd_name in $(yq eval '.metadata.name' "${src_dir}/deploy/crd.yaml" | grep -v '\-\-\-') + grep -v NAMESPACE + kubectl get perconaservermongodbbackups.psmdb.percona.com --all-namespaces -o wide + xargs -L 1 sh -xc 'kubectl patch perconaservermongodbbackups.psmdb.percona.com -n $0 $1 --type=merge -p "{\"metadata\":{\"finalizers\":[]}}"' error: the server doesn't have a resource type "perconaservermongodbbackups" + kubectl patch perconaservermongodbbackups.psmdb.percona.com -n sh --type=merge -p '{"metadata":{"finalizers":[]}}' error: the server doesn't have a resource type "perconaservermongodbbackups" + : + kubectl_bin wait --for=delete crd perconaservermongodbbackups.psmdb.percona.com ++ mktemp + local LAST_OUT=/tmp/tmp.6zLofHQQx0 ++ mktemp + local LAST_ERR=/tmp/tmp.ktyleSwhxn + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl wait --for=delete crd perconaservermongodbbackups.psmdb.percona.com + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.6zLofHQQx0 + cat /tmp/tmp.ktyleSwhxn + rm /tmp/tmp.6zLofHQQx0 /tmp/tmp.ktyleSwhxn + return 0 + for crd_name in $(yq eval '.metadata.name' "${src_dir}/deploy/crd.yaml" | grep -v '\-\-\-') + grep -v NAMESPACE + kubectl get perconaservermongodbrestores.psmdb.percona.com --all-namespaces -o wide + xargs -L 1 sh -xc 'kubectl patch perconaservermongodbrestores.psmdb.percona.com -n $0 $1 --type=merge -p "{\"metadata\":{\"finalizers\":[]}}"' error: the server doesn't have a resource type "perconaservermongodbrestores" + kubectl patch perconaservermongodbrestores.psmdb.percona.com -n sh --type=merge -p '{"metadata":{"finalizers":[]}}' error: the server doesn't have a resource type "perconaservermongodbrestores" + : + kubectl_bin wait --for=delete crd perconaservermongodbrestores.psmdb.percona.com ++ mktemp + local LAST_OUT=/tmp/tmp.D08S7Z1Zpx ++ mktemp + local LAST_ERR=/tmp/tmp.LaLyyEskWQ + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl wait --for=delete crd perconaservermongodbrestores.psmdb.percona.com + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.D08S7Z1Zpx + cat /tmp/tmp.LaLyyEskWQ + rm /tmp/tmp.D08S7Z1Zpx /tmp/tmp.LaLyyEskWQ + return 0 + for crd_name in $(yq eval '.metadata.name' "${src_dir}/deploy/crd.yaml" | grep -v '\-\-\-') + kubectl get perconaservermongodbs.psmdb.percona.com --all-namespaces -o wide + grep -v NAMESPACE + xargs -L 1 sh -xc 'kubectl patch perconaservermongodbs.psmdb.percona.com -n $0 $1 --type=merge -p "{\"metadata\":{\"finalizers\":[]}}"' error: the server doesn't have a resource type "perconaservermongodbs" + kubectl patch perconaservermongodbs.psmdb.percona.com -n sh --type=merge -p '{"metadata":{"finalizers":[]}}' error: the server doesn't have a resource type "perconaservermongodbs" + : + kubectl_bin wait --for=delete crd perconaservermongodbs.psmdb.percona.com ++ mktemp + local LAST_OUT=/tmp/tmp.TKPJ63rxOY ++ mktemp + local LAST_ERR=/tmp/tmp.bKoGRQeAbU + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl wait --for=delete crd perconaservermongodbs.psmdb.percona.com + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.TKPJ63rxOY + cat /tmp/tmp.bKoGRQeAbU + rm /tmp/tmp.TKPJ63rxOY /tmp/tmp.bKoGRQeAbU + return 0 + local rbac_yaml=rbac.yaml + '[' -n '' ']' + kubectl_bin delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2271/deploy/rbac.yaml --ignore-not-found ++ mktemp + local LAST_OUT=/tmp/tmp.lKWIU0A2JX ++ mktemp + local LAST_ERR=/tmp/tmp.RTJlGlMo41 + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2271/deploy/rbac.yaml --ignore-not-found + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.lKWIU0A2JX role.rbac.authorization.k8s.io "percona-server-mongodb-operator" deleted from cross-site-sharded-2219 namespace serviceaccount "percona-server-mongodb-operator" deleted from cross-site-sharded-2219 namespace rolebinding.rbac.authorization.k8s.io "service-account-percona-server-mongodb-operator" deleted from cross-site-sharded-2219 namespace + cat /tmp/tmp.RTJlGlMo41 + rm /tmp/tmp.lKWIU0A2JX /tmp/tmp.RTJlGlMo41 + return 0 + check_crd_for_deletion PR-2271-c31e8c20 + local git_tag=PR-2271-c31e8c20 ++ curl -s https://raw.githubusercontent.com/percona/percona-server-mongodb-operator/PR-2271-c31e8c20/deploy/crd.yaml ++ yq eval .metadata.name ++ /usr/sbin/sed ':a;N;$!ba;s/\n/ /g' ++ /usr/sbin/sed s/---//g + for crd_name in $(curl -s https://raw.githubusercontent.com/percona/percona-server-mongodb-operator/${git_tag}/deploy/crd.yaml | yq eval '.metadata.name' | $sed 's/---//g' | $sed ':a;N;$!ba;s/\n/ /g') ++ kubectl_bin get crd/null -o 'jsonpath={.status.conditions[-1].type}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.Aa5uklSR6S +++ mktemp ++ local LAST_ERR=/tmp/tmp.Xl7iixVg2n ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get crd/null -o 'jsonpath={.status.conditions[-1].type}' ++ exit_status=1 ++ set -e ++ '[' 1 '!=' 0 -a -n 1 ']' ++ cat /tmp/tmp.Aa5uklSR6S ++ cat /tmp/tmp.Xl7iixVg2n Error from server (NotFound): customresourcedefinitions.apiextensions.k8s.io "null" not found ++ sleep 0 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get crd/null -o 'jsonpath={.status.conditions[-1].type}' ++ exit_status=1 ++ set -e ++ '[' 1 '!=' 0 -a -n 1 ']' ++ cat /tmp/tmp.Aa5uklSR6S ++ cat /tmp/tmp.Xl7iixVg2n Error from server (NotFound): customresourcedefinitions.apiextensions.k8s.io "null" not found ++ sleep 4 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get crd/null -o 'jsonpath={.status.conditions[-1].type}' ++ exit_status=1 ++ set -e ++ '[' 1 '!=' 0 -a -n 1 ']' ++ cat /tmp/tmp.Aa5uklSR6S ++ cat /tmp/tmp.Xl7iixVg2n Error from server (NotFound): customresourcedefinitions.apiextensions.k8s.io "null" not found ++ sleep 8 ++ cat /tmp/tmp.Aa5uklSR6S ++ cat /tmp/tmp.Xl7iixVg2n Error from server (NotFound): customresourcedefinitions.apiextensions.k8s.io "null" not found ++ rm /tmp/tmp.Aa5uklSR6S /tmp/tmp.Xl7iixVg2n ++ return 1 + [[ '' == \T\e\r\m\i\n\a\t\i\n\g ]] + '[' -n '' ']' + create_namespace cross-site-sharded-17853 + local namespace=cross-site-sharded-17853 + local skip_clean_namespace= + [[ 1 == 1 ]] + [[ -z '' ]] + destroy_chaos_mesh ++ helm list --all-namespaces --filter chaos-mesh ++ tail -n1 ++ awk '-F ' '{print $2}' ++ sed s/NAMESPACE// + local chaos_mesh_ns= + desc 'destroy chaos-mesh' + set +o xtrace ----------------------------------------------------------------------------------- destroy chaos-mesh ----------------------------------------------------------------------------------- + '[' -n '' ']' ++ kubectl get MutatingWebhookConfiguration ++ grep chaos-mesh ++ awk '{print $1}' + timeout 30 kubectl delete MutatingWebhookConfiguration error: resource(s) were provided, but no name was specified + : ++ grep chaos-mesh ++ kubectl get ValidatingWebhookConfiguration ++ awk '{print $1}' + timeout 30 kubectl delete ValidatingWebhookConfiguration error: resource(s) were provided, but no name was specified + : ++ kubectl get ValidatingWebhookConfiguration ++ grep validate-auth ++ awk '{print $1}' + timeout 30 kubectl delete ValidatingWebhookConfiguration error: resource(s) were provided, but no name was specified + : ++ kubectl api-resources ++ grep chaos-mesh ++ awk '{print $1}' ++ kubectl get crd ++ grep chaos-mesh.org ++ awk '{print $1}' + timeout 30 kubectl delete crd error: resource(s) were provided, but no name was specified + : ++ kubectl get clusterrolebinding ++ grep chaos-mesh ++ awk '{print $1}' + timeout 30 kubectl delete clusterrolebinding error: resource(s) were provided, but no name was specified + : ++ kubectl get clusterrole ++ grep chaos-mesh ++ awk '{print $1}' + timeout 30 kubectl delete clusterrole error: resource(s) were provided, but no name was specified + : + desc 'cleaned up all old namespaces' + set +o xtrace ----------------------------------------------------------------------------------- cleaned up all old namespaces ----------------------------------------------------------------------------------- + kubectl_bin get ns + grep -E -v '^kube-|^default|Terminating|psmdb-operator|openshift|^gke-|^gmp-|^NAME' + awk '{print$1}' ++ mktemp + '[' -n '' ']' + desc 'cleaned up old namespaces cross-site-sharded-17853' + set +o xtrace + xargs kubectl delete ns ----------------------------------------------------------------------------------- cleaned up old namespaces cross-site-sharded-17853 ----------------------------------------------------------------------------------- + kubectl_bin delete namespace cross-site-sharded-17853 --ignore-not-found ++ mktemp + local LAST_OUT=/tmp/tmp.g3xGbu6zvv ++ mktemp + local LAST_OUT=/tmp/tmp.7ZvitI8KHx ++ mktemp + local LAST_ERR=/tmp/tmp.ThHmGKsrvW + local exit_status=0 + local timeout=4 ++ seq 0 2 + local LAST_ERR=/tmp/tmp.FXfknQw9Mb + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl get ns + for i in $(seq 0 2) + set +e + kubectl delete namespace cross-site-sharded-17853 --ignore-not-found + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.g3xGbu6zvv + cat /tmp/tmp.ThHmGKsrvW + rm /tmp/tmp.g3xGbu6zvv /tmp/tmp.ThHmGKsrvW + return 0 + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.7ZvitI8KHx + cat /tmp/tmp.FXfknQw9Mb + rm /tmp/tmp.7ZvitI8KHx /tmp/tmp.FXfknQw9Mb + return 0 + kubectl_bin wait --for=delete namespace cross-site-sharded-17853 ++ mktemp + local LAST_OUT=/tmp/tmp.zq6CmlEcDp ++ mktemp + local LAST_ERR=/tmp/tmp.vA6puCsR0E + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl wait --for=delete namespace cross-site-sharded-17853 namespace "cross-site-sharded-2219" deleted + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.zq6CmlEcDp namespace "cross-site-sharded-replica-15448" deleted + cat /tmp/tmp.vA6puCsR0E + rm /tmp/tmp.zq6CmlEcDp /tmp/tmp.vA6puCsR0E + return 0 + desc 'create namespace cross-site-sharded-17853' + set +o xtrace ----------------------------------------------------------------------------------- create namespace cross-site-sharded-17853 ----------------------------------------------------------------------------------- + kubectl_bin create namespace cross-site-sharded-17853 ++ mktemp + local LAST_OUT=/tmp/tmp.cBfzBwAYwO ++ mktemp + local LAST_ERR=/tmp/tmp.zkxmlkhE9q + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl create namespace cross-site-sharded-17853 + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.cBfzBwAYwO namespace/cross-site-sharded-17853 created + cat /tmp/tmp.zkxmlkhE9q + rm /tmp/tmp.cBfzBwAYwO /tmp/tmp.zkxmlkhE9q + return 0 + set_kube_ctx cross-site-sharded-17853 + local namespace=cross-site-sharded-17853 ++ kubectl_bin config current-context +++ mktemp ++ local LAST_OUT=/tmp/tmp.tR9EQ3htDp +++ mktemp ++ local LAST_ERR=/tmp/tmp.QNcIrxEvOb ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl config current-context ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.tR9EQ3htDp ++ cat /tmp/tmp.QNcIrxEvOb ++ rm /tmp/tmp.tR9EQ3htDp /tmp/tmp.QNcIrxEvOb ++ return 0 + kubectl_bin config set-context gke_cloud-dev-112233_us-central1-a_jen-psmdb-2271-c31e8c20-4-cluster3 --namespace=cross-site-sharded-17853 ++ mktemp + local LAST_OUT=/tmp/tmp.yucC6ca3B0 ++ mktemp + local LAST_ERR=/tmp/tmp.M56vqDPRQ9 + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl config set-context gke_cloud-dev-112233_us-central1-a_jen-psmdb-2271-c31e8c20-4-cluster3 --namespace=cross-site-sharded-17853 + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.yucC6ca3B0 Context "gke_cloud-dev-112233_us-central1-a_jen-psmdb-2271-c31e8c20-4-cluster3" modified. + cat /tmp/tmp.M56vqDPRQ9 + rm /tmp/tmp.yucC6ca3B0 /tmp/tmp.M56vqDPRQ9 + return 0 + deploy_operator + desc 'start PSMDB operator: docker.io/perconalab/percona-server-mongodb-operator:PR-2271-c31e8c20' + set +o xtrace ----------------------------------------------------------------------------------- start PSMDB operator: docker.io/perconalab/percona-server-mongodb-operator:PR-2271-c31e8c20 ----------------------------------------------------------------------------------- + local cr_file + '[' -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2271/e2e-tests/cross-site-sharded/conf/crd.yaml ']' + cr_file=/mnt/jenkins/workspace/cloud-psmdb-operator_PR-2271/deploy/crd.yaml + kubectl_bin apply --server-side --force-conflicts -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2271/deploy/crd.yaml ++ mktemp + local LAST_OUT=/tmp/tmp.ghrL8cEPpF ++ mktemp + local LAST_ERR=/tmp/tmp.B9z0JZqEe7 + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl apply --server-side --force-conflicts -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2271/deploy/crd.yaml + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.ghrL8cEPpF customresourcedefinition.apiextensions.k8s.io/perconaservermongodbbackups.psmdb.percona.com serverside-applied customresourcedefinition.apiextensions.k8s.io/perconaservermongodbrestores.psmdb.percona.com serverside-applied customresourcedefinition.apiextensions.k8s.io/perconaservermongodbs.psmdb.percona.com serverside-applied + cat /tmp/tmp.B9z0JZqEe7 + rm /tmp/tmp.ghrL8cEPpF /tmp/tmp.B9z0JZqEe7 + return 0 + '[' -n '' ']' + apply_rbac rbac + local operator_namespace=psmdb-operator + local rbac=rbac + cat /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2271/deploy/rbac.yaml + sed -e 's^namespace: .*^namespace: psmdb-operator^' + kubectl_bin apply -f - ++ mktemp + local LAST_OUT=/tmp/tmp.gkoKMDXDK4 ++ mktemp + local LAST_ERR=/tmp/tmp.Taps5Vw9Tv + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.gkoKMDXDK4 role.rbac.authorization.k8s.io/percona-server-mongodb-operator created serviceaccount/percona-server-mongodb-operator created rolebinding.rbac.authorization.k8s.io/service-account-percona-server-mongodb-operator created + cat /tmp/tmp.Taps5Vw9Tv + rm /tmp/tmp.gkoKMDXDK4 /tmp/tmp.Taps5Vw9Tv + return 0 + yq eval ' (.spec.template.spec.containers[].image = "docker.io/perconalab/percona-server-mongodb-operator:PR-2271-c31e8c20") | ((.. | select(.[] == "DISABLE_TELEMETRY")) |= .value="true") | ((.. | select(.[] == "LOG_LEVEL")) |= .value="DEBUG")' /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2271/deploy/operator.yaml + kubectl_bin apply -f - ++ mktemp + local LAST_OUT=/tmp/tmp.AZaCYmTGK1 ++ mktemp + local LAST_ERR=/tmp/tmp.5Fc9tUc4hG + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.AZaCYmTGK1 deployment.apps/percona-server-mongodb-operator created + cat /tmp/tmp.5Fc9tUc4hG + rm /tmp/tmp.AZaCYmTGK1 /tmp/tmp.5Fc9tUc4hG + return 0 + sleep 20 ++ get_operator_pod ++ kubectl_bin get pods --selector=name=percona-server-mongodb-operator -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.SN0BMNONiJ +++ mktemp ++ local LAST_ERR=/tmp/tmp.h5DdMUuSLo ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pods --selector=name=percona-server-mongodb-operator -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.SN0BMNONiJ ++ cat /tmp/tmp.h5DdMUuSLo ++ rm /tmp/tmp.SN0BMNONiJ /tmp/tmp.h5DdMUuSLo ++ return 0 + wait_operator_pod percona-server-mongodb-operator-846894f879-s2dg6 + local pod=percona-server-mongodb-operator-846894f879-s2dg6 + set +o xtrace waiting for pod/percona-server-mongodb-operator-846894f879-s2dg6 to be ready.OK + echo 'Print operator info from log' Print operator info from log + grep 'Manager starting up' ++ get_operator_pod ++ kubectl_bin get pods --selector=name=percona-server-mongodb-operator -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.mStCReTmB6 +++ mktemp ++ local LAST_ERR=/tmp/tmp.RuP2zNHgIC ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pods --selector=name=percona-server-mongodb-operator -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.mStCReTmB6 ++ cat /tmp/tmp.RuP2zNHgIC ++ rm /tmp/tmp.mStCReTmB6 /tmp/tmp.RuP2zNHgIC ++ return 0 + kubectl_bin logs percona-server-mongodb-operator-846894f879-s2dg6 ++ mktemp + local LAST_OUT=/tmp/tmp.sEd4D91YvT ++ mktemp + local LAST_ERR=/tmp/tmp.hhZBDkjrUl + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl logs percona-server-mongodb-operator-846894f879-s2dg6 + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.sEd4D91YvT + cat /tmp/tmp.hhZBDkjrUl + rm /tmp/tmp.sEd4D91YvT /tmp/tmp.hhZBDkjrUl + return 0 2026-03-26T08:36:40.669Z INFO setup Manager starting up {"gitCommit": "c31e8c20f2a920146c4a2410dc38a83148f57a2b", "gitBranch": "PR-2271-c31e8c20", "buildTime": "", "goVersion": "go1.25.8", "os": "linux", "arch": "amd64"} + desc 'create secrets and start client' + set +o xtrace ----------------------------------------------------------------------------------- create secrets and start client ----------------------------------------------------------------------------------- + kubectl_bin apply -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2271/e2e-tests/conf/client.yml -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2271/e2e-tests/cross-site-sharded/conf/secrets.yml ++ mktemp + local LAST_OUT=/tmp/tmp.BbXCzkb3GX ++ mktemp + local LAST_ERR=/tmp/tmp.Z2SRIIoijI + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl apply -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2271/e2e-tests/conf/client.yml -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2271/e2e-tests/cross-site-sharded/conf/secrets.yml + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.BbXCzkb3GX deployment.apps/psmdb-client created secret/cross-site-sharded-main-secrets created secret/cross-site-sharded-main-ssl created secret/cross-site-sharded-main-ssl-internal created + cat /tmp/tmp.Z2SRIIoijI + rm /tmp/tmp.BbXCzkb3GX /tmp/tmp.Z2SRIIoijI + return 0 + desc 'create main PSMDB cluster cross-site-sharded-main.' + set +o xtrace ----------------------------------------------------------------------------------- create main PSMDB cluster cross-site-sharded-main. ----------------------------------------------------------------------------------- + apply_cluster /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2271/e2e-tests/cross-site-sharded/conf/cross-site-sharded-main.yml + '[' -z '' ']' + cat_config /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2271/e2e-tests/cross-site-sharded/conf/cross-site-sharded-main.yml + kubectl_bin apply -f - + cat /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2271/e2e-tests/cross-site-sharded/conf/cross-site-sharded-main.yml ++ mktemp + yq eval '(.spec | select(.image == null)).image = "docker.io/perconalab/percona-server-mongodb-operator:main-mongod8.0"' + yq eval '(.spec | select(has("pmm"))).pmm.image = "docker.io/percona/pmm-client:2.44.1-1"' + yq eval '(.spec | select(has("initImage"))).initImage = "docker.io/perconalab/percona-server-mongodb-operator:PR-2271-c31e8c20"' + yq eval '(.spec | select(has("backup"))).backup.image = "docker.io/perconalab/percona-server-mongodb-operator:main-backup"' + /usr/sbin/sed -e s/NAME_SPACE/cross-site-sharded-17853/g + local LAST_OUT=/tmp/tmp.wzW9pFmW0Z ++ mktemp + yq eval '.spec.upgradeOptions.apply="Never"' + local LAST_ERR=/tmp/tmp.AS8GFSLj0E + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.wzW9pFmW0Z perconaservermongodb.psmdb.percona.com/cross-site-sharded-main created + cat /tmp/tmp.AS8GFSLj0E + rm /tmp/tmp.wzW9pFmW0Z /tmp/tmp.AS8GFSLj0E + return 0 + desc 'check if all 3 Pods started' + set +o xtrace ----------------------------------------------------------------------------------- check if all 3 Pods started ----------------------------------------------------------------------------------- + wait_for_running cross-site-sharded-main-rs0 3 + local name=cross-site-sharded-main-rs0 + let last_pod=2 + local check_cluster_readyness=true + set_debug + [[ 1 == 1 ]] + set -o xtrace + local rs_name=rs0 + local cluster_name=cross-site-sharded-main ++ seq 0 2 + for i in $(seq 0 $last_pod) + [[ 0 -eq 2 ]] + wait_pod cross-site-sharded-main-rs0-0 + local pod=cross-site-sharded-main-rs0-0 + set +o xtrace waiting for pod/cross-site-sharded-main-rs0-0 to be ready.................OK + for i in $(seq 0 $last_pod) + [[ 1 -eq 2 ]] + wait_pod cross-site-sharded-main-rs0-1 + local pod=cross-site-sharded-main-rs0-1 + set +o xtrace waiting for pod/cross-site-sharded-main-rs0-1 to be ready..............OK + for i in $(seq 0 $last_pod) + [[ 2 -eq 2 ]] ++ kubectl_bin get psmdb cross-site-sharded-main -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].arbiter.enabled}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.MUHyzIOlOL +++ mktemp ++ local LAST_ERR=/tmp/tmp.LCakqbujnc ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb cross-site-sharded-main -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].arbiter.enabled}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.MUHyzIOlOL ++ cat /tmp/tmp.LCakqbujnc ++ rm /tmp/tmp.MUHyzIOlOL /tmp/tmp.LCakqbujnc ++ return 0 + [[ false == \t\r\u\e ]] + wait_pod cross-site-sharded-main-rs0-2 + local pod=cross-site-sharded-main-rs0-2 + set +o xtrace waiting for pod/cross-site-sharded-main-rs0-2 to be ready.............OK ++ kubectl_bin get psmdb cross-site-sharded-main -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].nonvoting.enabled}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.Os2w0wZlvR +++ mktemp ++ local LAST_ERR=/tmp/tmp.UUdgsm20QZ ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb cross-site-sharded-main -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].nonvoting.enabled}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.Os2w0wZlvR ++ cat /tmp/tmp.UUdgsm20QZ ++ rm /tmp/tmp.Os2w0wZlvR /tmp/tmp.UUdgsm20QZ ++ return 0 + [[ false == \t\r\u\e ]] ++ kubectl_bin get psmdb cross-site-sharded-main -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].hidden.enabled}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.sV2fJo20ly +++ mktemp ++ local LAST_ERR=/tmp/tmp.DFHXXq1T95 ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb cross-site-sharded-main -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].hidden.enabled}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.sV2fJo20ly ++ cat /tmp/tmp.DFHXXq1T95 ++ rm /tmp/tmp.sV2fJo20ly /tmp/tmp.DFHXXq1T95 ++ return 0 + [[ '' == \t\r\u\e ]] + sleep 10 + [[ true == \t\r\u\e ]] + set +x Waiting for cluster readyness.................................................. + wait_for_running cross-site-sharded-main-cfg 3 false + local name=cross-site-sharded-main-cfg + let last_pod=2 + local check_cluster_readyness=false + set_debug + [[ 1 == 1 ]] + set -o xtrace + local rs_name=cfg + local cluster_name=cross-site-sharded-main ++ seq 0 2 + for i in $(seq 0 $last_pod) + [[ 0 -eq 2 ]] + wait_pod cross-site-sharded-main-cfg-0 + local pod=cross-site-sharded-main-cfg-0 + set +o xtrace waiting for pod/cross-site-sharded-main-cfg-0 to be ready.OK + for i in $(seq 0 $last_pod) + [[ 1 -eq 2 ]] + wait_pod cross-site-sharded-main-cfg-1 + local pod=cross-site-sharded-main-cfg-1 + set +o xtrace waiting for pod/cross-site-sharded-main-cfg-1 to be ready.OK + for i in $(seq 0 $last_pod) + [[ 2 -eq 2 ]] ++ kubectl_bin get psmdb cross-site-sharded-main -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].arbiter.enabled}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.Kcn0vR4FVL +++ mktemp ++ local LAST_ERR=/tmp/tmp.Phkr3H56Ud ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb cross-site-sharded-main -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].arbiter.enabled}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.Kcn0vR4FVL ++ cat /tmp/tmp.Phkr3H56Ud ++ rm /tmp/tmp.Kcn0vR4FVL /tmp/tmp.Phkr3H56Ud ++ return 0 + [[ '' == \t\r\u\e ]] + wait_pod cross-site-sharded-main-cfg-2 + local pod=cross-site-sharded-main-cfg-2 + set +o xtrace waiting for pod/cross-site-sharded-main-cfg-2 to be ready.OK ++ kubectl_bin get psmdb cross-site-sharded-main -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].nonvoting.enabled}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.XRZm9ld8Kz +++ mktemp ++ local LAST_ERR=/tmp/tmp.3E4Q9Hhc3k ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb cross-site-sharded-main -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].nonvoting.enabled}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.XRZm9ld8Kz ++ cat /tmp/tmp.3E4Q9Hhc3k ++ rm /tmp/tmp.XRZm9ld8Kz /tmp/tmp.3E4Q9Hhc3k ++ return 0 + [[ '' == \t\r\u\e ]] ++ kubectl_bin get psmdb cross-site-sharded-main -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].hidden.enabled}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.USBQn2E5LK +++ mktemp ++ local LAST_ERR=/tmp/tmp.Dhp9XmKSmY ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb cross-site-sharded-main -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].hidden.enabled}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.USBQn2E5LK ++ cat /tmp/tmp.Dhp9XmKSmY ++ rm /tmp/tmp.USBQn2E5LK /tmp/tmp.Dhp9XmKSmY ++ return 0 + [[ '' == \t\r\u\e ]] + sleep 10 + [[ false == \t\r\u\e ]] + desc 'create user' + set +o xtrace ----------------------------------------------------------------------------------- create user ----------------------------------------------------------------------------------- + run_mongos 'db.createUser({user:"user",pwd:"pass",roles:[{db:"app",role:"readWrite"}]})' userAdmin:userAdmin123456@cross-site-sharded-main-mongos.cross-site-sharded-17853 + local 'command=db.createUser({user:"user",pwd:"pass",roles:[{db:"app",role:"readWrite"}]})' + local uri=userAdmin:userAdmin123456@cross-site-sharded-main-mongos.cross-site-sharded-17853 + local driver=mongodb + local suffix=.svc.cluster.local + local mongo_flag= + local port=27017 + local mongo_bin=mongo ++ echo .svc.cluster.local ++ awk -F: '{print $2}' + suffix_port= + [[ -z '' ]] + suffix=.svc.cluster.local:27017 ++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.czBLShiHI0 +++ mktemp ++ local LAST_ERR=/tmp/tmp.ClOyCFavgC ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.czBLShiHI0 ++ cat /tmp/tmp.ClOyCFavgC ++ rm /tmp/tmp.czBLShiHI0 /tmp/tmp.ClOyCFavgC ++ return 0 + local client_container=psmdb-client-bb8b97679-l4k9z + kubectl_bin exec psmdb-client-bb8b97679-l4k9z -- bash -c 'printf '\''db.createUser({user:"user",pwd:"pass",roles:[{db:"app",role:"readWrite"}]})\n'\'' | mongo mongodb://userAdmin:userAdmin123456@cross-site-sharded-main-mongos.cross-site-sharded-17853.svc.cluster.local:27017/admin ' ++ mktemp + local LAST_OUT=/tmp/tmp.WlM8GdRduh ++ mktemp + local LAST_ERR=/tmp/tmp.QStsZYyNyM + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl exec psmdb-client-bb8b97679-l4k9z -- bash -c 'printf '\''db.createUser({user:"user",pwd:"pass",roles:[{db:"app",role:"readWrite"}]})\n'\'' | mongo mongodb://userAdmin:userAdmin123456@cross-site-sharded-main-mongos.cross-site-sharded-17853.svc.cluster.local:27017/admin ' + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.WlM8GdRduh Percona Server for MongoDB shell version v4.4.29-28 connecting to: mongodb://cross-site-sharded-main-mongos.cross-site-sharded-17853.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb Implicit session: session { "id" : UUID("1247d25c-f9e5-4679-a641-7ccd53786603") } Percona Server for MongoDB server version: v8.0.19-7 WARNING: shell and server versions do not match Successfully added user: { "user" : "user", "roles" : [ { "db" : "app", "role" : "readWrite" } ] } bye + cat /tmp/tmp.QStsZYyNyM + rm /tmp/tmp.WlM8GdRduh /tmp/tmp.QStsZYyNyM + return 0 + sleep 2 + desc 'set chunk size to 2 MB' + set +o xtrace ----------------------------------------------------------------------------------- set chunk size to 2 MB ----------------------------------------------------------------------------------- + run_mongos 'use config\n db.settings.save( { _id:"chunksize", value: 2 } )' clusterAdmin:clusterAdmin123456@cross-site-sharded-main-mongos.cross-site-sharded-17853 + local 'command=use config\n db.settings.save( { _id:"chunksize", value: 2 } )' + local uri=clusterAdmin:clusterAdmin123456@cross-site-sharded-main-mongos.cross-site-sharded-17853 + local driver=mongodb + local suffix=.svc.cluster.local + local mongo_flag= + local port=27017 + local mongo_bin=mongo ++ echo .svc.cluster.local ++ awk -F: '{print $2}' + suffix_port= + [[ -z '' ]] + suffix=.svc.cluster.local:27017 ++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.k1UU1NDLLb +++ mktemp ++ local LAST_ERR=/tmp/tmp.Lal4KGwTEm ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.k1UU1NDLLb ++ cat /tmp/tmp.Lal4KGwTEm ++ rm /tmp/tmp.k1UU1NDLLb /tmp/tmp.Lal4KGwTEm ++ return 0 + local client_container=psmdb-client-bb8b97679-l4k9z + kubectl_bin exec psmdb-client-bb8b97679-l4k9z -- bash -c 'printf '\''use config\n db.settings.save( { _id:"chunksize", value: 2 } )\n'\'' | mongo mongodb://clusterAdmin:clusterAdmin123456@cross-site-sharded-main-mongos.cross-site-sharded-17853.svc.cluster.local:27017/admin ' ++ mktemp + local LAST_OUT=/tmp/tmp.9fRhp06f3E ++ mktemp + local LAST_ERR=/tmp/tmp.k9wCapBxyV + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl exec psmdb-client-bb8b97679-l4k9z -- bash -c 'printf '\''use config\n db.settings.save( { _id:"chunksize", value: 2 } )\n'\'' | mongo mongodb://clusterAdmin:clusterAdmin123456@cross-site-sharded-main-mongos.cross-site-sharded-17853.svc.cluster.local:27017/admin ' + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.9fRhp06f3E Percona Server for MongoDB shell version v4.4.29-28 connecting to: mongodb://cross-site-sharded-main-mongos.cross-site-sharded-17853.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb Implicit session: session { "id" : UUID("9d760bc1-7444-482f-a8c5-e1ebb7e7234e") } Percona Server for MongoDB server version: v8.0.19-7 WARNING: shell and server versions do not match switched to db config WriteResult({ "nMatched" : 0, "nUpserted" : 1, "nModified" : 0, "_id" : "chunksize" }) bye + cat /tmp/tmp.k9wCapBxyV + rm /tmp/tmp.9fRhp06f3E /tmp/tmp.k9wCapBxyV + return 0 + sleep 2 + desc 'write data' + set +o xtrace ----------------------------------------------------------------------------------- write data ----------------------------------------------------------------------------------- + run_script_mongos /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2271/e2e-tests/cross-site-sharded/data.js user:pass@cross-site-sharded-main-mongos.cross-site-sharded-17853 + local script=/mnt/jenkins/workspace/cloud-psmdb-operator_PR-2271/e2e-tests/cross-site-sharded/data.js + local uri=user:pass@cross-site-sharded-main-mongos.cross-site-sharded-17853 + local driver=mongodb + local suffix=.svc.cluster.local + local mongo_flag= + local mongo_bin=mongo ++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.NWL41CkW1Y +++ mktemp ++ local LAST_ERR=/tmp/tmp.v1RgOrOj74 ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.NWL41CkW1Y ++ cat /tmp/tmp.v1RgOrOj74 ++ rm /tmp/tmp.NWL41CkW1Y /tmp/tmp.v1RgOrOj74 ++ return 0 + local client_container=psmdb-client-bb8b97679-l4k9z ++ basename /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2271/e2e-tests/cross-site-sharded/data.js + name=data.js + kubectl_bin cp /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2271/e2e-tests/cross-site-sharded/data.js cross-site-sharded-17853/psmdb-client-bb8b97679-l4k9z:/tmp ++ mktemp + local LAST_OUT=/tmp/tmp.6lrokn2KGp ++ mktemp + local LAST_ERR=/tmp/tmp.IjXZXmxMRu + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl cp /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2271/e2e-tests/cross-site-sharded/data.js cross-site-sharded-17853/psmdb-client-bb8b97679-l4k9z:/tmp + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.6lrokn2KGp + cat /tmp/tmp.IjXZXmxMRu + rm /tmp/tmp.6lrokn2KGp /tmp/tmp.IjXZXmxMRu + return 0 + kubectl_bin exec psmdb-client-bb8b97679-l4k9z -- bash -c 'mongo mongodb://user:pass@cross-site-sharded-main-mongos.cross-site-sharded-17853.svc.cluster.local/admin /tmp/data.js' ++ mktemp + local LAST_OUT=/tmp/tmp.F13EFtA4LM ++ mktemp + local LAST_ERR=/tmp/tmp.iBrALkm4vu + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl exec psmdb-client-bb8b97679-l4k9z -- bash -c 'mongo mongodb://user:pass@cross-site-sharded-main-mongos.cross-site-sharded-17853.svc.cluster.local/admin /tmp/data.js' + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.F13EFtA4LM Percona Server for MongoDB shell version v4.4.29-28 connecting to: mongodb://cross-site-sharded-main-mongos.cross-site-sharded-17853.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb Implicit session: session { "id" : UUID("afdf442d-49d4-4098-8688-8fabaa5fca4e") } Percona Server for MongoDB server version: v8.0.19-7 WARNING: shell and server versions do not match + cat /tmp/tmp.iBrALkm4vu + rm /tmp/tmp.F13EFtA4LM /tmp/tmp.iBrALkm4vu + return 0 + desc 'shard collection' + set +o xtrace ----------------------------------------------------------------------------------- shard collection ----------------------------------------------------------------------------------- + run_mongos 'sh.enableSharding("app")' clusterAdmin:clusterAdmin123456@cross-site-sharded-main-mongos.cross-site-sharded-17853 + local 'command=sh.enableSharding("app")' + local uri=clusterAdmin:clusterAdmin123456@cross-site-sharded-main-mongos.cross-site-sharded-17853 + local driver=mongodb + local suffix=.svc.cluster.local + local mongo_flag= + local port=27017 + local mongo_bin=mongo ++ echo .svc.cluster.local ++ awk -F: '{print $2}' + suffix_port= + [[ -z '' ]] + suffix=.svc.cluster.local:27017 ++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.OH4UqaBtDn +++ mktemp ++ local LAST_ERR=/tmp/tmp.v1mkHj3DKS ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.OH4UqaBtDn ++ cat /tmp/tmp.v1mkHj3DKS ++ rm /tmp/tmp.OH4UqaBtDn /tmp/tmp.v1mkHj3DKS ++ return 0 + local client_container=psmdb-client-bb8b97679-l4k9z + kubectl_bin exec psmdb-client-bb8b97679-l4k9z -- bash -c 'printf '\''sh.enableSharding("app")\n'\'' | mongo mongodb://clusterAdmin:clusterAdmin123456@cross-site-sharded-main-mongos.cross-site-sharded-17853.svc.cluster.local:27017/admin ' ++ mktemp + local LAST_OUT=/tmp/tmp.UG5VxD0Ea7 ++ mktemp + local LAST_ERR=/tmp/tmp.q5Q0pg5y15 + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl exec psmdb-client-bb8b97679-l4k9z -- bash -c 'printf '\''sh.enableSharding("app")\n'\'' | mongo mongodb://clusterAdmin:clusterAdmin123456@cross-site-sharded-main-mongos.cross-site-sharded-17853.svc.cluster.local:27017/admin ' + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.UG5VxD0Ea7 Percona Server for MongoDB shell version v4.4.29-28 connecting to: mongodb://cross-site-sharded-main-mongos.cross-site-sharded-17853.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb Implicit session: session { "id" : UUID("b2fab135-cd53-4ac3-bf7a-dfcbc2a4fdf1") } Percona Server for MongoDB server version: v8.0.19-7 WARNING: shell and server versions do not match { "ok" : 1, "$clusterTime" : { "clusterTime" : Timestamp(1774514497, 1), "signature" : { "hash" : BinData(0,"/E+AOIxxNK4DX8VFXs7ZpaRey/8="), "keyId" : NumberLong("7621481241266618369") } }, "operationTime" : Timestamp(1774514497, 1) } bye + cat /tmp/tmp.q5Q0pg5y15 + rm /tmp/tmp.UG5VxD0Ea7 /tmp/tmp.q5Q0pg5y15 + return 0 + sleep 2 + run_mongos 'sh.shardCollection("app.city", { _id: 1 } )' clusterAdmin:clusterAdmin123456@cross-site-sharded-main-mongos.cross-site-sharded-17853 + local 'command=sh.shardCollection("app.city", { _id: 1 } )' + local uri=clusterAdmin:clusterAdmin123456@cross-site-sharded-main-mongos.cross-site-sharded-17853 + local driver=mongodb + local suffix=.svc.cluster.local + local mongo_flag= + local port=27017 + local mongo_bin=mongo ++ echo .svc.cluster.local ++ awk -F: '{print $2}' + suffix_port= + [[ -z '' ]] + suffix=.svc.cluster.local:27017 ++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.8AGsqOMLCO +++ mktemp ++ local LAST_ERR=/tmp/tmp.CY4Y9eadHX ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.8AGsqOMLCO ++ cat /tmp/tmp.CY4Y9eadHX ++ rm /tmp/tmp.8AGsqOMLCO /tmp/tmp.CY4Y9eadHX ++ return 0 + local client_container=psmdb-client-bb8b97679-l4k9z + kubectl_bin exec psmdb-client-bb8b97679-l4k9z -- bash -c 'printf '\''sh.shardCollection("app.city", { _id: 1 } )\n'\'' | mongo mongodb://clusterAdmin:clusterAdmin123456@cross-site-sharded-main-mongos.cross-site-sharded-17853.svc.cluster.local:27017/admin ' ++ mktemp + local LAST_OUT=/tmp/tmp.OZXwL18vJ9 ++ mktemp + local LAST_ERR=/tmp/tmp.ersqgXmHfH + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl exec psmdb-client-bb8b97679-l4k9z -- bash -c 'printf '\''sh.shardCollection("app.city", { _id: 1 } )\n'\'' | mongo mongodb://clusterAdmin:clusterAdmin123456@cross-site-sharded-main-mongos.cross-site-sharded-17853.svc.cluster.local:27017/admin ' + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.OZXwL18vJ9 Percona Server for MongoDB shell version v4.4.29-28 connecting to: mongodb://cross-site-sharded-main-mongos.cross-site-sharded-17853.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb Implicit session: session { "id" : UUID("6d540c5c-e569-4070-92e4-14bc8a366576") } Percona Server for MongoDB server version: v8.0.19-7 WARNING: shell and server versions do not match { "collectionsharded" : "app.city", "ok" : 1, "$clusterTime" : { "clusterTime" : Timestamp(1774514503, 29), "signature" : { "hash" : BinData(0,"Xgl+2k+EOtLPZhwqJcjR616pQ38="), "keyId" : NumberLong("7621481241266618369") } }, "operationTime" : Timestamp(1774514503, 28) } bye + cat /tmp/tmp.ersqgXmHfH + rm /tmp/tmp.OZXwL18vJ9 /tmp/tmp.ersqgXmHfH + return 0 + sleep 120 + desc 'Check chunks' + set +o xtrace ----------------------------------------------------------------------------------- Check chunks ----------------------------------------------------------------------------------- + chunks_param1=ns + chunks_param2='"app.city"' + [[ 8.0 != \4\.\4 ]] + chunks_param1=uuid ++ run_mongos 'use app\n db.getCollectionInfos({ "name": "city" })[0].info.uuid' user:pass@cross-site-sharded-main-mongos.cross-site-sharded-17853 ++ grep 'switched to db app' -A 1 ++ local 'command=use app\n db.getCollectionInfos({ "name": "city" })[0].info.uuid' ++ local uri=user:pass@cross-site-sharded-main-mongos.cross-site-sharded-17853 ++ local driver=mongodb ++ local suffix=.svc.cluster.local ++ local mongo_flag= ++ local port=27017 ++ local mongo_bin=mongo ++ grep -v 'switched to db app' +++ echo .svc.cluster.local +++ awk -F: '{print $2}' ++ suffix_port= ++ [[ -z '' ]] ++ suffix=.svc.cluster.local:27017 +++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.Ow64RwgSMA ++++ mktemp +++ local LAST_ERR=/tmp/tmp.MJe187f5VD +++ local exit_status=0 +++ local timeout=4 ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 -a -n 1 ']' +++ break +++ cat /tmp/tmp.Ow64RwgSMA +++ cat /tmp/tmp.MJe187f5VD +++ rm /tmp/tmp.Ow64RwgSMA /tmp/tmp.MJe187f5VD +++ return 0 ++ local client_container=psmdb-client-bb8b97679-l4k9z ++ kubectl_bin exec psmdb-client-bb8b97679-l4k9z -- bash -c 'printf '\''use app\n db.getCollectionInfos({ "name": "city" })[0].info.uuid\n'\'' | mongo mongodb://user:pass@cross-site-sharded-main-mongos.cross-site-sharded-17853.svc.cluster.local:27017/admin ' +++ mktemp ++ local LAST_OUT=/tmp/tmp.rcBekuilZ3 +++ mktemp ++ local LAST_ERR=/tmp/tmp.Fvd8qtbHKJ ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl exec psmdb-client-bb8b97679-l4k9z -- bash -c 'printf '\''use app\n db.getCollectionInfos({ "name": "city" })[0].info.uuid\n'\'' | mongo mongodb://user:pass@cross-site-sharded-main-mongos.cross-site-sharded-17853.svc.cluster.local:27017/admin ' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.rcBekuilZ3 ++ cat /tmp/tmp.Fvd8qtbHKJ ++ rm /tmp/tmp.rcBekuilZ3 /tmp/tmp.Fvd8qtbHKJ ++ return 0 + chunks_param2='UUID("ccecccec-2b31-4e62-b2ab-9b82d89704ff")' + shards=0 + for i in "rs0" "rs1" ++ grep 'switched to db config' -A 1 ++ run_mongos 'use config\n db.chunks.count({"uuid": UUID("ccecccec-2b31-4e62-b2ab-9b82d89704ff"), "shard": "rs0"})' clusterAdmin:clusterAdmin123456@cross-site-sharded-main-mongos.cross-site-sharded-17853 ++ local 'command=use config\n db.chunks.count({"uuid": UUID("ccecccec-2b31-4e62-b2ab-9b82d89704ff"), "shard": "rs0"})' ++ local uri=clusterAdmin:clusterAdmin123456@cross-site-sharded-main-mongos.cross-site-sharded-17853 ++ grep -v 'switched to db config' ++ local driver=mongodb ++ local suffix=.svc.cluster.local ++ local mongo_flag= ++ local port=27017 ++ local mongo_bin=mongo +++ echo .svc.cluster.local +++ awk -F: '{print $2}' ++ suffix_port= ++ [[ -z '' ]] ++ suffix=.svc.cluster.local:27017 +++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.MEbmb0npSR ++++ mktemp +++ local LAST_ERR=/tmp/tmp.Q1t9J0kP70 +++ local exit_status=0 +++ local timeout=4 ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 -a -n 1 ']' +++ break +++ cat /tmp/tmp.MEbmb0npSR +++ cat /tmp/tmp.Q1t9J0kP70 +++ rm /tmp/tmp.MEbmb0npSR /tmp/tmp.Q1t9J0kP70 +++ return 0 ++ local client_container=psmdb-client-bb8b97679-l4k9z ++ kubectl_bin exec psmdb-client-bb8b97679-l4k9z -- bash -c 'printf '\''use config\n db.chunks.count({"uuid": UUID("ccecccec-2b31-4e62-b2ab-9b82d89704ff"), "shard": "rs0"})\n'\'' | mongo mongodb://clusterAdmin:clusterAdmin123456@cross-site-sharded-main-mongos.cross-site-sharded-17853.svc.cluster.local:27017/admin ' +++ mktemp ++ local LAST_OUT=/tmp/tmp.RDsuXS4h9i +++ mktemp ++ local LAST_ERR=/tmp/tmp.Ntfir6KLPX ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl exec psmdb-client-bb8b97679-l4k9z -- bash -c 'printf '\''use config\n db.chunks.count({"uuid": UUID("ccecccec-2b31-4e62-b2ab-9b82d89704ff"), "shard": "rs0"})\n'\'' | mongo mongodb://clusterAdmin:clusterAdmin123456@cross-site-sharded-main-mongos.cross-site-sharded-17853.svc.cluster.local:27017/admin ' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.RDsuXS4h9i ++ cat /tmp/tmp.Ntfir6KLPX ++ rm /tmp/tmp.RDsuXS4h9i /tmp/tmp.Ntfir6KLPX ++ return 0 + out=1 + desc 'rs0 has 1 chunks' + set +o xtrace ----------------------------------------------------------------------------------- rs0 has 1 chunks ----------------------------------------------------------------------------------- + [[ 1 -ne 0 ]] + (( shards = shards + 1 )) + for i in "rs0" "rs1" ++ run_mongos 'use config\n db.chunks.count({"uuid": UUID("ccecccec-2b31-4e62-b2ab-9b82d89704ff"), "shard": "rs1"})' clusterAdmin:clusterAdmin123456@cross-site-sharded-main-mongos.cross-site-sharded-17853 ++ local 'command=use config\n db.chunks.count({"uuid": UUID("ccecccec-2b31-4e62-b2ab-9b82d89704ff"), "shard": "rs1"})' ++ local uri=clusterAdmin:clusterAdmin123456@cross-site-sharded-main-mongos.cross-site-sharded-17853 ++ local driver=mongodb ++ local suffix=.svc.cluster.local ++ local mongo_flag= ++ local port=27017 ++ local mongo_bin=mongo ++ grep 'switched to db config' -A 1 ++ grep -v 'switched to db config' +++ echo .svc.cluster.local +++ awk -F: '{print $2}' ++ suffix_port= ++ [[ -z '' ]] ++ suffix=.svc.cluster.local:27017 +++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.URtom3CZno ++++ mktemp +++ local LAST_ERR=/tmp/tmp.EQj4FHYJ9x +++ local exit_status=0 +++ local timeout=4 ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 -a -n 1 ']' +++ break +++ cat /tmp/tmp.URtom3CZno +++ cat /tmp/tmp.EQj4FHYJ9x +++ rm /tmp/tmp.URtom3CZno /tmp/tmp.EQj4FHYJ9x +++ return 0 ++ local client_container=psmdb-client-bb8b97679-l4k9z ++ kubectl_bin exec psmdb-client-bb8b97679-l4k9z -- bash -c 'printf '\''use config\n db.chunks.count({"uuid": UUID("ccecccec-2b31-4e62-b2ab-9b82d89704ff"), "shard": "rs1"})\n'\'' | mongo mongodb://clusterAdmin:clusterAdmin123456@cross-site-sharded-main-mongos.cross-site-sharded-17853.svc.cluster.local:27017/admin ' +++ mktemp ++ local LAST_OUT=/tmp/tmp.cKUOwAB1Pj +++ mktemp ++ local LAST_ERR=/tmp/tmp.KyGS1udBH0 ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl exec psmdb-client-bb8b97679-l4k9z -- bash -c 'printf '\''use config\n db.chunks.count({"uuid": UUID("ccecccec-2b31-4e62-b2ab-9b82d89704ff"), "shard": "rs1"})\n'\'' | mongo mongodb://clusterAdmin:clusterAdmin123456@cross-site-sharded-main-mongos.cross-site-sharded-17853.svc.cluster.local:27017/admin ' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.cKUOwAB1Pj ++ cat /tmp/tmp.KyGS1udBH0 ++ rm /tmp/tmp.cKUOwAB1Pj /tmp/tmp.KyGS1udBH0 ++ return 0 + out=3 + desc 'rs1 has 3 chunks' + set +o xtrace ----------------------------------------------------------------------------------- rs1 has 3 chunks ----------------------------------------------------------------------------------- + [[ 3 -ne 0 ]] + (( shards = shards + 1 )) + [[ 2 -lt 2 ]] + desc 'create replica cluster' + set +o xtrace ----------------------------------------------------------------------------------- create replica cluster ----------------------------------------------------------------------------------- + create_namespace cross-site-sharded-replica-32611 0 + local namespace=cross-site-sharded-replica-32611 + local skip_clean_namespace=0 + [[ 1 == 1 ]] + [[ -z 0 ]] + '[' -n '' ']' + desc 'cleaned up old namespaces cross-site-sharded-replica-32611' + set +o xtrace ----------------------------------------------------------------------------------- cleaned up old namespaces cross-site-sharded-replica-32611 ----------------------------------------------------------------------------------- + kubectl_bin delete namespace cross-site-sharded-replica-32611 --ignore-not-found ++ mktemp + local LAST_OUT=/tmp/tmp.AidJNrBPEv ++ mktemp + local LAST_ERR=/tmp/tmp.X2nphrHctH + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl delete namespace cross-site-sharded-replica-32611 --ignore-not-found + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.AidJNrBPEv + cat /tmp/tmp.X2nphrHctH + rm /tmp/tmp.AidJNrBPEv /tmp/tmp.X2nphrHctH + return 0 + kubectl_bin wait --for=delete namespace cross-site-sharded-replica-32611 ++ mktemp + local LAST_OUT=/tmp/tmp.93CzFiHIZ8 ++ mktemp + local LAST_ERR=/tmp/tmp.6hAFtnVU3v + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl wait --for=delete namespace cross-site-sharded-replica-32611 + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.93CzFiHIZ8 + cat /tmp/tmp.6hAFtnVU3v + rm /tmp/tmp.93CzFiHIZ8 /tmp/tmp.6hAFtnVU3v + return 0 + desc 'create namespace cross-site-sharded-replica-32611' + set +o xtrace ----------------------------------------------------------------------------------- create namespace cross-site-sharded-replica-32611 ----------------------------------------------------------------------------------- + kubectl_bin create namespace cross-site-sharded-replica-32611 ++ mktemp + local LAST_OUT=/tmp/tmp.QhNvVuQXLG ++ mktemp + local LAST_ERR=/tmp/tmp.lBGSG14AUx + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl create namespace cross-site-sharded-replica-32611 + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.QhNvVuQXLG namespace/cross-site-sharded-replica-32611 created + cat /tmp/tmp.lBGSG14AUx + rm /tmp/tmp.QhNvVuQXLG /tmp/tmp.lBGSG14AUx + return 0 + set_kube_ctx cross-site-sharded-replica-32611 + local namespace=cross-site-sharded-replica-32611 ++ kubectl_bin config current-context +++ mktemp ++ local LAST_OUT=/tmp/tmp.reOajIu0gB +++ mktemp ++ local LAST_ERR=/tmp/tmp.G0qzkRXX7E ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl config current-context ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.reOajIu0gB ++ cat /tmp/tmp.G0qzkRXX7E ++ rm /tmp/tmp.reOajIu0gB /tmp/tmp.G0qzkRXX7E ++ return 0 + kubectl_bin config set-context gke_cloud-dev-112233_us-central1-a_jen-psmdb-2271-c31e8c20-4-cluster3 --namespace=cross-site-sharded-replica-32611 ++ mktemp + local LAST_OUT=/tmp/tmp.MKyK2LcZOf ++ mktemp + local LAST_ERR=/tmp/tmp.ybtNUdM9Nt + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl config set-context gke_cloud-dev-112233_us-central1-a_jen-psmdb-2271-c31e8c20-4-cluster3 --namespace=cross-site-sharded-replica-32611 + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.MKyK2LcZOf Context "gke_cloud-dev-112233_us-central1-a_jen-psmdb-2271-c31e8c20-4-cluster3" modified. + cat /tmp/tmp.ybtNUdM9Nt + rm /tmp/tmp.MKyK2LcZOf /tmp/tmp.ybtNUdM9Nt + return 0 + deploy_operator + desc 'start PSMDB operator: docker.io/perconalab/percona-server-mongodb-operator:PR-2271-c31e8c20' + set +o xtrace ----------------------------------------------------------------------------------- start PSMDB operator: docker.io/perconalab/percona-server-mongodb-operator:PR-2271-c31e8c20 ----------------------------------------------------------------------------------- + local cr_file + '[' -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2271/e2e-tests/cross-site-sharded/conf/crd.yaml ']' + cr_file=/mnt/jenkins/workspace/cloud-psmdb-operator_PR-2271/deploy/crd.yaml + kubectl_bin apply --server-side --force-conflicts -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2271/deploy/crd.yaml ++ mktemp + local LAST_OUT=/tmp/tmp.fsBdw226ib ++ mktemp + local LAST_ERR=/tmp/tmp.0FnMKtTPZr + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl apply --server-side --force-conflicts -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2271/deploy/crd.yaml + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.fsBdw226ib customresourcedefinition.apiextensions.k8s.io/perconaservermongodbbackups.psmdb.percona.com serverside-applied customresourcedefinition.apiextensions.k8s.io/perconaservermongodbrestores.psmdb.percona.com serverside-applied customresourcedefinition.apiextensions.k8s.io/perconaservermongodbs.psmdb.percona.com serverside-applied + cat /tmp/tmp.0FnMKtTPZr + rm /tmp/tmp.fsBdw226ib /tmp/tmp.0FnMKtTPZr + return 0 + '[' -n '' ']' + apply_rbac rbac + local operator_namespace=psmdb-operator + local rbac=rbac + cat /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2271/deploy/rbac.yaml + sed -e 's^namespace: .*^namespace: psmdb-operator^' + kubectl_bin apply -f - ++ mktemp + local LAST_OUT=/tmp/tmp.av2zDsyTL7 ++ mktemp + local LAST_ERR=/tmp/tmp.jVV2LAxxFI + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.av2zDsyTL7 role.rbac.authorization.k8s.io/percona-server-mongodb-operator created serviceaccount/percona-server-mongodb-operator created rolebinding.rbac.authorization.k8s.io/service-account-percona-server-mongodb-operator created + cat /tmp/tmp.jVV2LAxxFI + rm /tmp/tmp.av2zDsyTL7 /tmp/tmp.jVV2LAxxFI + return 0 + yq eval ' (.spec.template.spec.containers[].image = "docker.io/perconalab/percona-server-mongodb-operator:PR-2271-c31e8c20") | ((.. | select(.[] == "DISABLE_TELEMETRY")) |= .value="true") | ((.. | select(.[] == "LOG_LEVEL")) |= .value="DEBUG")' /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2271/deploy/operator.yaml + kubectl_bin apply -f - ++ mktemp + local LAST_OUT=/tmp/tmp.z8DG5bH4gW ++ mktemp + local LAST_ERR=/tmp/tmp.YCvbpvUS8b + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.z8DG5bH4gW deployment.apps/percona-server-mongodb-operator created + cat /tmp/tmp.YCvbpvUS8b + rm /tmp/tmp.z8DG5bH4gW /tmp/tmp.YCvbpvUS8b + return 0 + sleep 20 ++ get_operator_pod ++ kubectl_bin get pods --selector=name=percona-server-mongodb-operator -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.83ZnxODPEK +++ mktemp ++ local LAST_ERR=/tmp/tmp.iAL3EPd3Qs ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pods --selector=name=percona-server-mongodb-operator -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.83ZnxODPEK ++ cat /tmp/tmp.iAL3EPd3Qs ++ rm /tmp/tmp.83ZnxODPEK /tmp/tmp.iAL3EPd3Qs ++ return 0 + wait_operator_pod percona-server-mongodb-operator-846894f879-2bzbx + local pod=percona-server-mongodb-operator-846894f879-2bzbx + set +o xtrace waiting for pod/percona-server-mongodb-operator-846894f879-2bzbx to be ready.OK + echo 'Print operator info from log' Print operator info from log + grep 'Manager starting up' ++ get_operator_pod ++ kubectl_bin get pods --selector=name=percona-server-mongodb-operator -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.IkkUcUuYhP +++ mktemp ++ local LAST_ERR=/tmp/tmp.RO5Y1YPOWf ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pods --selector=name=percona-server-mongodb-operator -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.IkkUcUuYhP ++ cat /tmp/tmp.RO5Y1YPOWf ++ rm /tmp/tmp.IkkUcUuYhP /tmp/tmp.RO5Y1YPOWf ++ return 0 + kubectl_bin logs percona-server-mongodb-operator-846894f879-2bzbx ++ mktemp + local LAST_OUT=/tmp/tmp.GiFerjaZ2X ++ mktemp + local LAST_ERR=/tmp/tmp.YayddnCQgs + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl logs percona-server-mongodb-operator-846894f879-2bzbx + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.GiFerjaZ2X + cat /tmp/tmp.YayddnCQgs + rm /tmp/tmp.GiFerjaZ2X /tmp/tmp.YayddnCQgs + return 0 2026-03-26T08:44:08.921Z INFO setup Manager starting up {"gitCommit": "c31e8c20f2a920146c4a2410dc38a83148f57a2b", "gitBranch": "PR-2271-c31e8c20", "buildTime": "", "goVersion": "go1.25.8", "os": "linux", "arch": "amd64"} + desc 'start client' + set +o xtrace ----------------------------------------------------------------------------------- start client ----------------------------------------------------------------------------------- + kubectl_bin apply -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2271/e2e-tests/conf/client.yml ++ mktemp + local LAST_OUT=/tmp/tmp.QFys9Z9YrI ++ mktemp + local LAST_ERR=/tmp/tmp.pnqBuRgdXy + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl apply -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2271/e2e-tests/conf/client.yml + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.QFys9Z9YrI deployment.apps/psmdb-client created + cat /tmp/tmp.pnqBuRgdXy + rm /tmp/tmp.QFys9Z9YrI /tmp/tmp.pnqBuRgdXy + return 0 + desc 'copy secrets from main to replica namespace and create all of them' + set +o xtrace ----------------------------------------------------------------------------------- copy secrets from main to replica namespace and create all of them ----------------------------------------------------------------------------------- + kubectl get secret cross-site-sharded-main-secrets -o yaml -n cross-site-sharded-17853 + yq eval ' del(.metadata) | (.metadata.name = "cross-site-sharded-replica-secrets")' - + kubectl_bin apply -f - ++ mktemp + local LAST_OUT=/tmp/tmp.yyVERDS9LJ ++ mktemp + local LAST_ERR=/tmp/tmp.BIpIaBFld7 + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.yyVERDS9LJ secret/cross-site-sharded-replica-secrets created + cat /tmp/tmp.BIpIaBFld7 + rm /tmp/tmp.yyVERDS9LJ /tmp/tmp.BIpIaBFld7 + return 0 + kubectl_bin get secret cross-site-sharded-main-ssl-internal -o yaml -n cross-site-sharded-17853 + yq eval ' del(.metadata) | del(.status) | (.metadata.name = "cross-site-sharded-replica-ssl-internal")' - + kubectl_bin apply -f - ++ mktemp ++ mktemp + local LAST_OUT=/tmp/tmp.uhPas2QbLU + local LAST_OUT=/tmp/tmp.esq1QyuhIN ++ mktemp ++ mktemp + local LAST_ERR=/tmp/tmp.cnxMNrH6WM + local exit_status=0 + local timeout=4 + local LAST_ERR=/tmp/tmp.uljHwC3bs5 + local exit_status=0 + local timeout=4 ++ seq 0 2 ++ seq 0 2 + for i in $(seq 0 2) + set +e + for i in $(seq 0 2) + kubectl get secret cross-site-sharded-main-ssl-internal -o yaml -n cross-site-sharded-17853 + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.uhPas2QbLU + cat /tmp/tmp.cnxMNrH6WM + rm /tmp/tmp.uhPas2QbLU /tmp/tmp.cnxMNrH6WM + return 0 + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.esq1QyuhIN secret/cross-site-sharded-replica-ssl-internal created + cat /tmp/tmp.uljHwC3bs5 + rm /tmp/tmp.esq1QyuhIN /tmp/tmp.uljHwC3bs5 + return 0 + kubectl_bin get secret cross-site-sharded-main-ssl -o yaml -n cross-site-sharded-17853 + yq eval ' del(.metadata) | del(.status) | (.metadata.name = "cross-site-sharded-replica-ssl")' - + kubectl_bin apply -f - ++ mktemp ++ mktemp + local LAST_OUT=/tmp/tmp.DcXjtpDpxs ++ mktemp + local LAST_OUT=/tmp/tmp.y8QBtFmWtP ++ mktemp + local LAST_ERR=/tmp/tmp.WyuFFUdCzs + local exit_status=0 + local timeout=4 ++ seq 0 2 + local LAST_ERR=/tmp/tmp.LCchVj9hoD + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl get secret cross-site-sharded-main-ssl -o yaml -n cross-site-sharded-17853 + for i in $(seq 0 2) + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.DcXjtpDpxs + cat /tmp/tmp.WyuFFUdCzs + rm /tmp/tmp.DcXjtpDpxs /tmp/tmp.WyuFFUdCzs + return 0 + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.y8QBtFmWtP secret/cross-site-sharded-replica-ssl created + cat /tmp/tmp.LCchVj9hoD + rm /tmp/tmp.y8QBtFmWtP /tmp/tmp.LCchVj9hoD + return 0 + sleep 30 + desc 'create replica PSMDB cluster ' + set +o xtrace ----------------------------------------------------------------------------------- create replica PSMDB cluster ----------------------------------------------------------------------------------- + apply_cluster /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2271/e2e-tests/cross-site-sharded/conf/cross-site-sharded-replica.yml + '[' -z '' ']' + kubectl_bin apply -f - + cat_config /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2271/e2e-tests/cross-site-sharded/conf/cross-site-sharded-replica.yml ++ mktemp + cat /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2271/e2e-tests/cross-site-sharded/conf/cross-site-sharded-replica.yml + yq eval '(.spec | select(.image == null)).image = "docker.io/perconalab/percona-server-mongodb-operator:main-mongod8.0"' + yq eval '(.spec | select(has("pmm"))).pmm.image = "docker.io/percona/pmm-client:2.44.1-1"' + yq eval '(.spec | select(has("initImage"))).initImage = "docker.io/perconalab/percona-server-mongodb-operator:PR-2271-c31e8c20"' + local LAST_OUT=/tmp/tmp.1NGTp9zKy1 + yq eval '(.spec | select(has("backup"))).backup.image = "docker.io/perconalab/percona-server-mongodb-operator:main-backup"' ++ mktemp + /usr/sbin/sed -e s/NAME_SPACE/cross-site-sharded-17853/g + local LAST_ERR=/tmp/tmp.q4LZeD7iJ4 + yq eval '.spec.upgradeOptions.apply="Never"' + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.1NGTp9zKy1 perconaservermongodb.psmdb.percona.com/cross-site-sharded-replica created + cat /tmp/tmp.q4LZeD7iJ4 + rm /tmp/tmp.1NGTp9zKy1 /tmp/tmp.q4LZeD7iJ4 + return 0 + wait_for_running cross-site-sharded-replica-rs0 3 false + local name=cross-site-sharded-replica-rs0 + let last_pod=2 + local check_cluster_readyness=false + set_debug + [[ 1 == 1 ]] + set -o xtrace + local rs_name=rs0 + local cluster_name=cross-site-sharded-replica ++ seq 0 2 + for i in $(seq 0 $last_pod) + [[ 0 -eq 2 ]] + wait_pod cross-site-sharded-replica-rs0-0 + local pod=cross-site-sharded-replica-rs0-0 + set +o xtrace waiting for pod/cross-site-sharded-replica-rs0-0 to be ready...........OK + for i in $(seq 0 $last_pod) + [[ 1 -eq 2 ]] + wait_pod cross-site-sharded-replica-rs0-1 + local pod=cross-site-sharded-replica-rs0-1 + set +o xtrace waiting for pod/cross-site-sharded-replica-rs0-1 to be ready.............OK + for i in $(seq 0 $last_pod) + [[ 2 -eq 2 ]] ++ kubectl_bin get psmdb cross-site-sharded-replica -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].arbiter.enabled}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.oJwq9rAOMw +++ mktemp ++ local LAST_ERR=/tmp/tmp.Gh8eUOGqXF ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb cross-site-sharded-replica -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].arbiter.enabled}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.oJwq9rAOMw ++ cat /tmp/tmp.Gh8eUOGqXF ++ rm /tmp/tmp.oJwq9rAOMw /tmp/tmp.Gh8eUOGqXF ++ return 0 + [[ false == \t\r\u\e ]] + wait_pod cross-site-sharded-replica-rs0-2 + local pod=cross-site-sharded-replica-rs0-2 + set +o xtrace waiting for pod/cross-site-sharded-replica-rs0-2 to be ready..............OK ++ kubectl_bin get psmdb cross-site-sharded-replica -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].nonvoting.enabled}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.YWf233Sgbq +++ mktemp ++ local LAST_ERR=/tmp/tmp.TL3ps6426z ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb cross-site-sharded-replica -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].nonvoting.enabled}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.YWf233Sgbq ++ cat /tmp/tmp.TL3ps6426z ++ rm /tmp/tmp.YWf233Sgbq /tmp/tmp.TL3ps6426z ++ return 0 + [[ false == \t\r\u\e ]] ++ kubectl_bin get psmdb cross-site-sharded-replica -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].hidden.enabled}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.o4oENEMWvq +++ mktemp ++ local LAST_ERR=/tmp/tmp.00XHzKS30Y ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb cross-site-sharded-replica -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].hidden.enabled}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.o4oENEMWvq ++ cat /tmp/tmp.00XHzKS30Y ++ rm /tmp/tmp.o4oENEMWvq /tmp/tmp.00XHzKS30Y ++ return 0 + [[ '' == \t\r\u\e ]] + sleep 10 + [[ false == \t\r\u\e ]] + wait_for_running cross-site-sharded-replica-rs1 3 false + local name=cross-site-sharded-replica-rs1 + let last_pod=2 + local check_cluster_readyness=false + set_debug + [[ 1 == 1 ]] + set -o xtrace + local rs_name=rs1 + local cluster_name=cross-site-sharded-replica ++ seq 0 2 + for i in $(seq 0 $last_pod) + [[ 0 -eq 2 ]] + wait_pod cross-site-sharded-replica-rs1-0 + local pod=cross-site-sharded-replica-rs1-0 + set +o xtrace waiting for pod/cross-site-sharded-replica-rs1-0 to be ready.OK + for i in $(seq 0 $last_pod) + [[ 1 -eq 2 ]] + wait_pod cross-site-sharded-replica-rs1-1 + local pod=cross-site-sharded-replica-rs1-1 + set +o xtrace waiting for pod/cross-site-sharded-replica-rs1-1 to be ready.OK + for i in $(seq 0 $last_pod) + [[ 2 -eq 2 ]] ++ kubectl_bin get psmdb cross-site-sharded-replica -o 'jsonpath={.spec.replsets[?(@.name=="rs1")].arbiter.enabled}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.da4HVxpA02 +++ mktemp ++ local LAST_ERR=/tmp/tmp.NdVRAF2YX0 ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb cross-site-sharded-replica -o 'jsonpath={.spec.replsets[?(@.name=="rs1")].arbiter.enabled}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.da4HVxpA02 ++ cat /tmp/tmp.NdVRAF2YX0 ++ rm /tmp/tmp.da4HVxpA02 /tmp/tmp.NdVRAF2YX0 ++ return 0 + [[ false == \t\r\u\e ]] + wait_pod cross-site-sharded-replica-rs1-2 + local pod=cross-site-sharded-replica-rs1-2 + set +o xtrace waiting for pod/cross-site-sharded-replica-rs1-2 to be ready.OK ++ kubectl_bin get psmdb cross-site-sharded-replica -o 'jsonpath={.spec.replsets[?(@.name=="rs1")].nonvoting.enabled}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.9WknEJgas8 +++ mktemp ++ local LAST_ERR=/tmp/tmp.yiZpSByTfp ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb cross-site-sharded-replica -o 'jsonpath={.spec.replsets[?(@.name=="rs1")].nonvoting.enabled}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.9WknEJgas8 ++ cat /tmp/tmp.yiZpSByTfp ++ rm /tmp/tmp.9WknEJgas8 /tmp/tmp.yiZpSByTfp ++ return 0 + [[ false == \t\r\u\e ]] ++ kubectl_bin get psmdb cross-site-sharded-replica -o 'jsonpath={.spec.replsets[?(@.name=="rs1")].hidden.enabled}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.mCa0rtWbkl +++ mktemp ++ local LAST_ERR=/tmp/tmp.sXFwnatqIA ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb cross-site-sharded-replica -o 'jsonpath={.spec.replsets[?(@.name=="rs1")].hidden.enabled}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.mCa0rtWbkl ++ cat /tmp/tmp.sXFwnatqIA ++ rm /tmp/tmp.mCa0rtWbkl /tmp/tmp.sXFwnatqIA ++ return 0 + [[ '' == \t\r\u\e ]] + sleep 10 + [[ false == \t\r\u\e ]] + wait_for_running cross-site-sharded-replica-cfg 3 false + local name=cross-site-sharded-replica-cfg + let last_pod=2 + local check_cluster_readyness=false + set_debug + [[ 1 == 1 ]] + set -o xtrace + local rs_name=cfg + local cluster_name=cross-site-sharded-replica ++ seq 0 2 + for i in $(seq 0 $last_pod) + [[ 0 -eq 2 ]] + wait_pod cross-site-sharded-replica-cfg-0 + local pod=cross-site-sharded-replica-cfg-0 + set +o xtrace waiting for pod/cross-site-sharded-replica-cfg-0 to be ready.OK + for i in $(seq 0 $last_pod) + [[ 1 -eq 2 ]] + wait_pod cross-site-sharded-replica-cfg-1 + local pod=cross-site-sharded-replica-cfg-1 + set +o xtrace waiting for pod/cross-site-sharded-replica-cfg-1 to be ready.OK + for i in $(seq 0 $last_pod) + [[ 2 -eq 2 ]] ++ kubectl_bin get psmdb cross-site-sharded-replica -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].arbiter.enabled}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.7VGJuWAU8i +++ mktemp ++ local LAST_ERR=/tmp/tmp.YSX1SsTG4R ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb cross-site-sharded-replica -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].arbiter.enabled}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.7VGJuWAU8i ++ cat /tmp/tmp.YSX1SsTG4R ++ rm /tmp/tmp.7VGJuWAU8i /tmp/tmp.YSX1SsTG4R ++ return 0 + [[ '' == \t\r\u\e ]] + wait_pod cross-site-sharded-replica-cfg-2 + local pod=cross-site-sharded-replica-cfg-2 + set +o xtrace waiting for pod/cross-site-sharded-replica-cfg-2 to be ready.OK ++ kubectl_bin get psmdb cross-site-sharded-replica -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].nonvoting.enabled}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.vauvzPl5cC +++ mktemp ++ local LAST_ERR=/tmp/tmp.MPG4apDaxT ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb cross-site-sharded-replica -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].nonvoting.enabled}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.vauvzPl5cC ++ cat /tmp/tmp.MPG4apDaxT ++ rm /tmp/tmp.vauvzPl5cC /tmp/tmp.MPG4apDaxT ++ return 0 + [[ '' == \t\r\u\e ]] ++ kubectl_bin get psmdb cross-site-sharded-replica -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].hidden.enabled}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.Q3LwyNQ6iU +++ mktemp ++ local LAST_ERR=/tmp/tmp.pf94Z1ffSR ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb cross-site-sharded-replica -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].hidden.enabled}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.Q3LwyNQ6iU ++ cat /tmp/tmp.pf94Z1ffSR ++ rm /tmp/tmp.Q3LwyNQ6iU /tmp/tmp.pf94Z1ffSR ++ return 0 + [[ '' == \t\r\u\e ]] + sleep 10 + [[ false == \t\r\u\e ]] ++ get_service_ip cross-site-sharded-replica-cfg-0 cfg ++ local service=cross-site-sharded-replica-cfg-0 ++ local server_type=cfg +++ kubectl_bin get psmdb/cross-site-sharded-replica -o 'jsonpath={.spec.replsets[].expose.enabled}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.UZp1ZWvYQw ++++ mktemp +++ local LAST_ERR=/tmp/tmp.hyIBFJhcEW +++ local exit_status=0 +++ local timeout=4 ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl get psmdb/cross-site-sharded-replica -o 'jsonpath={.spec.replsets[].expose.enabled}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 -a -n 1 ']' +++ break +++ cat /tmp/tmp.UZp1ZWvYQw +++ cat /tmp/tmp.hyIBFJhcEW +++ rm /tmp/tmp.UZp1ZWvYQw /tmp/tmp.hyIBFJhcEW +++ return 0 ++ '[' true '!=' true ']' ++ kubectl_bin get service/cross-site-sharded-replica-cfg-0 -o 'jsonpath={.spec.type}' ++ grep -q NotFound +++ kubectl_bin get service/cross-site-sharded-replica-cfg-0 -o 'jsonpath={.spec.type}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.E60m1B2zkS ++++ mktemp +++ local LAST_ERR=/tmp/tmp.emnilOM5iq +++ local exit_status=0 +++ local timeout=4 ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl get service/cross-site-sharded-replica-cfg-0 -o 'jsonpath={.spec.type}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 -a -n 1 ']' +++ break +++ cat /tmp/tmp.E60m1B2zkS +++ cat /tmp/tmp.emnilOM5iq +++ rm /tmp/tmp.E60m1B2zkS /tmp/tmp.emnilOM5iq +++ return 0 ++ service_type=ClusterIP ++ '[' ClusterIP = ClusterIP ']' ++ kubectl_bin get service/cross-site-sharded-replica-cfg-0 -o 'jsonpath={.spec.clusterIP}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.D6kQuJ6nyr +++ mktemp ++ local LAST_ERR=/tmp/tmp.nQHgr2Uw5M ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get service/cross-site-sharded-replica-cfg-0 -o 'jsonpath={.spec.clusterIP}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.D6kQuJ6nyr ++ cat /tmp/tmp.nQHgr2Uw5M ++ rm /tmp/tmp.D6kQuJ6nyr /tmp/tmp.nQHgr2Uw5M ++ return 0 ++ return + replica_cfg_0_endpoint=34.118.232.156 ++ get_service_ip cross-site-sharded-replica-cfg-1 cfg ++ local service=cross-site-sharded-replica-cfg-1 ++ local server_type=cfg +++ kubectl_bin get psmdb/cross-site-sharded-replica -o 'jsonpath={.spec.replsets[].expose.enabled}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.mvd6jIct7C ++++ mktemp +++ local LAST_ERR=/tmp/tmp.mlBNPrrUuf +++ local exit_status=0 +++ local timeout=4 ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl get psmdb/cross-site-sharded-replica -o 'jsonpath={.spec.replsets[].expose.enabled}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 -a -n 1 ']' +++ break +++ cat /tmp/tmp.mvd6jIct7C +++ cat /tmp/tmp.mlBNPrrUuf +++ rm /tmp/tmp.mvd6jIct7C /tmp/tmp.mlBNPrrUuf +++ return 0 ++ '[' true '!=' true ']' ++ kubectl_bin get service/cross-site-sharded-replica-cfg-1 -o 'jsonpath={.spec.type}' ++ grep -q NotFound +++ kubectl_bin get service/cross-site-sharded-replica-cfg-1 -o 'jsonpath={.spec.type}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.MVVUk49lKs ++++ mktemp +++ local LAST_ERR=/tmp/tmp.7gLHIO04O7 +++ local exit_status=0 +++ local timeout=4 ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl get service/cross-site-sharded-replica-cfg-1 -o 'jsonpath={.spec.type}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 -a -n 1 ']' +++ break +++ cat /tmp/tmp.MVVUk49lKs +++ cat /tmp/tmp.7gLHIO04O7 +++ rm /tmp/tmp.MVVUk49lKs /tmp/tmp.7gLHIO04O7 +++ return 0 ++ service_type=ClusterIP ++ '[' ClusterIP = ClusterIP ']' ++ kubectl_bin get service/cross-site-sharded-replica-cfg-1 -o 'jsonpath={.spec.clusterIP}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.ajABm9Lavb +++ mktemp ++ local LAST_ERR=/tmp/tmp.2sIaM844Jx ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get service/cross-site-sharded-replica-cfg-1 -o 'jsonpath={.spec.clusterIP}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.ajABm9Lavb ++ cat /tmp/tmp.2sIaM844Jx ++ rm /tmp/tmp.ajABm9Lavb /tmp/tmp.2sIaM844Jx ++ return 0 ++ return + replica_cfg_1_endpoint=34.118.225.129 ++ get_service_ip cross-site-sharded-replica-cfg-2 cfg ++ local service=cross-site-sharded-replica-cfg-2 ++ local server_type=cfg +++ kubectl_bin get psmdb/cross-site-sharded-replica -o 'jsonpath={.spec.replsets[].expose.enabled}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.5xoSgOsEqb ++++ mktemp +++ local LAST_ERR=/tmp/tmp.6Ur6a4zEh1 +++ local exit_status=0 +++ local timeout=4 ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl get psmdb/cross-site-sharded-replica -o 'jsonpath={.spec.replsets[].expose.enabled}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 -a -n 1 ']' +++ break +++ cat /tmp/tmp.5xoSgOsEqb +++ cat /tmp/tmp.6Ur6a4zEh1 +++ rm /tmp/tmp.5xoSgOsEqb /tmp/tmp.6Ur6a4zEh1 +++ return 0 ++ '[' true '!=' true ']' ++ kubectl_bin get service/cross-site-sharded-replica-cfg-2 -o 'jsonpath={.spec.type}' ++ grep -q NotFound +++ kubectl_bin get service/cross-site-sharded-replica-cfg-2 -o 'jsonpath={.spec.type}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.2C1bDk7OIl ++++ mktemp +++ local LAST_ERR=/tmp/tmp.z9ebCCwlzf +++ local exit_status=0 +++ local timeout=4 ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl get service/cross-site-sharded-replica-cfg-2 -o 'jsonpath={.spec.type}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 -a -n 1 ']' +++ break +++ cat /tmp/tmp.2C1bDk7OIl +++ cat /tmp/tmp.z9ebCCwlzf +++ rm /tmp/tmp.2C1bDk7OIl /tmp/tmp.z9ebCCwlzf +++ return 0 ++ service_type=ClusterIP ++ '[' ClusterIP = ClusterIP ']' ++ kubectl_bin get service/cross-site-sharded-replica-cfg-2 -o 'jsonpath={.spec.clusterIP}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.Pb8BoqEuRN +++ mktemp ++ local LAST_ERR=/tmp/tmp.pGuLb4TJQd ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get service/cross-site-sharded-replica-cfg-2 -o 'jsonpath={.spec.clusterIP}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.Pb8BoqEuRN ++ cat /tmp/tmp.pGuLb4TJQd ++ rm /tmp/tmp.Pb8BoqEuRN /tmp/tmp.pGuLb4TJQd ++ return 0 ++ return + replica_cfg_2_endpoint=34.118.228.22 ++ get_service_ip cross-site-sharded-replica-rs0-0 ++ local service=cross-site-sharded-replica-rs0-0 ++ local server_type=rs0 +++ kubectl_bin get psmdb/cross-site-sharded-replica -o 'jsonpath={.spec.replsets[].expose.enabled}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.jeIUd5GJID ++++ mktemp +++ local LAST_ERR=/tmp/tmp.CWCyrLwfmr +++ local exit_status=0 +++ local timeout=4 ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl get psmdb/cross-site-sharded-replica -o 'jsonpath={.spec.replsets[].expose.enabled}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 -a -n 1 ']' +++ break +++ cat /tmp/tmp.jeIUd5GJID +++ cat /tmp/tmp.CWCyrLwfmr +++ rm /tmp/tmp.jeIUd5GJID /tmp/tmp.CWCyrLwfmr +++ return 0 ++ '[' true '!=' true ']' ++ kubectl_bin get service/cross-site-sharded-replica-rs0-0 -o 'jsonpath={.spec.type}' ++ grep -q NotFound +++ kubectl_bin get service/cross-site-sharded-replica-rs0-0 -o 'jsonpath={.spec.type}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.dXdBigptm0 ++++ mktemp +++ local LAST_ERR=/tmp/tmp.TCoEzTlc0y +++ local exit_status=0 +++ local timeout=4 ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl get service/cross-site-sharded-replica-rs0-0 -o 'jsonpath={.spec.type}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 -a -n 1 ']' +++ break +++ cat /tmp/tmp.dXdBigptm0 +++ cat /tmp/tmp.TCoEzTlc0y +++ rm /tmp/tmp.dXdBigptm0 /tmp/tmp.TCoEzTlc0y +++ return 0 ++ service_type=ClusterIP ++ '[' ClusterIP = ClusterIP ']' ++ kubectl_bin get service/cross-site-sharded-replica-rs0-0 -o 'jsonpath={.spec.clusterIP}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.Ve4UeTS82W +++ mktemp ++ local LAST_ERR=/tmp/tmp.zoY8RnNiIN ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get service/cross-site-sharded-replica-rs0-0 -o 'jsonpath={.spec.clusterIP}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.Ve4UeTS82W ++ cat /tmp/tmp.zoY8RnNiIN ++ rm /tmp/tmp.Ve4UeTS82W /tmp/tmp.zoY8RnNiIN ++ return 0 ++ return + replica_rs0_0_endpoint=34.118.233.158 ++ get_service_ip cross-site-sharded-replica-rs0-1 ++ local service=cross-site-sharded-replica-rs0-1 ++ local server_type=rs0 +++ kubectl_bin get psmdb/cross-site-sharded-replica -o 'jsonpath={.spec.replsets[].expose.enabled}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.7cifIE8Csj ++++ mktemp +++ local LAST_ERR=/tmp/tmp.UzdN2liJqR +++ local exit_status=0 +++ local timeout=4 ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl get psmdb/cross-site-sharded-replica -o 'jsonpath={.spec.replsets[].expose.enabled}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 -a -n 1 ']' +++ break +++ cat /tmp/tmp.7cifIE8Csj +++ cat /tmp/tmp.UzdN2liJqR +++ rm /tmp/tmp.7cifIE8Csj /tmp/tmp.UzdN2liJqR +++ return 0 ++ '[' true '!=' true ']' ++ kubectl_bin get service/cross-site-sharded-replica-rs0-1 -o 'jsonpath={.spec.type}' ++ grep -q NotFound +++ kubectl_bin get service/cross-site-sharded-replica-rs0-1 -o 'jsonpath={.spec.type}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.kmnY0BpHdV ++++ mktemp +++ local LAST_ERR=/tmp/tmp.uF6tcCuxUV +++ local exit_status=0 +++ local timeout=4 ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl get service/cross-site-sharded-replica-rs0-1 -o 'jsonpath={.spec.type}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 -a -n 1 ']' +++ break +++ cat /tmp/tmp.kmnY0BpHdV +++ cat /tmp/tmp.uF6tcCuxUV +++ rm /tmp/tmp.kmnY0BpHdV /tmp/tmp.uF6tcCuxUV +++ return 0 ++ service_type=ClusterIP ++ '[' ClusterIP = ClusterIP ']' ++ kubectl_bin get service/cross-site-sharded-replica-rs0-1 -o 'jsonpath={.spec.clusterIP}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.5IkE6f0REu +++ mktemp ++ local LAST_ERR=/tmp/tmp.AdOBykssNQ ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get service/cross-site-sharded-replica-rs0-1 -o 'jsonpath={.spec.clusterIP}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.5IkE6f0REu ++ cat /tmp/tmp.AdOBykssNQ ++ rm /tmp/tmp.5IkE6f0REu /tmp/tmp.AdOBykssNQ ++ return 0 ++ return + replica_rs0_1_endpoint=34.118.231.9 ++ get_service_ip cross-site-sharded-replica-rs0-2 ++ local service=cross-site-sharded-replica-rs0-2 ++ local server_type=rs0 +++ kubectl_bin get psmdb/cross-site-sharded-replica -o 'jsonpath={.spec.replsets[].expose.enabled}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.PDcWBjocsm ++++ mktemp +++ local LAST_ERR=/tmp/tmp.VLM4GAwK7S +++ local exit_status=0 +++ local timeout=4 ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl get psmdb/cross-site-sharded-replica -o 'jsonpath={.spec.replsets[].expose.enabled}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 -a -n 1 ']' +++ break +++ cat /tmp/tmp.PDcWBjocsm +++ cat /tmp/tmp.VLM4GAwK7S +++ rm /tmp/tmp.PDcWBjocsm /tmp/tmp.VLM4GAwK7S +++ return 0 ++ '[' true '!=' true ']' ++ kubectl_bin get service/cross-site-sharded-replica-rs0-2 -o 'jsonpath={.spec.type}' ++ grep -q NotFound +++ kubectl_bin get service/cross-site-sharded-replica-rs0-2 -o 'jsonpath={.spec.type}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.Xwdrt5GqHa ++++ mktemp +++ local LAST_ERR=/tmp/tmp.nFZoU0vHvg +++ local exit_status=0 +++ local timeout=4 ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl get service/cross-site-sharded-replica-rs0-2 -o 'jsonpath={.spec.type}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 -a -n 1 ']' +++ break +++ cat /tmp/tmp.Xwdrt5GqHa +++ cat /tmp/tmp.nFZoU0vHvg +++ rm /tmp/tmp.Xwdrt5GqHa /tmp/tmp.nFZoU0vHvg +++ return 0 ++ service_type=ClusterIP ++ '[' ClusterIP = ClusterIP ']' ++ kubectl_bin get service/cross-site-sharded-replica-rs0-2 -o 'jsonpath={.spec.clusterIP}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.Dg7TA8TzAM +++ mktemp ++ local LAST_ERR=/tmp/tmp.DLTzSGl9lr ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get service/cross-site-sharded-replica-rs0-2 -o 'jsonpath={.spec.clusterIP}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.Dg7TA8TzAM ++ cat /tmp/tmp.DLTzSGl9lr ++ rm /tmp/tmp.Dg7TA8TzAM /tmp/tmp.DLTzSGl9lr ++ return 0 ++ return + replica_rs0_2_endpoint=34.118.225.248 ++ get_service_ip cross-site-sharded-replica-rs1-0 rs1 ++ local service=cross-site-sharded-replica-rs1-0 ++ local server_type=rs1 +++ kubectl_bin get psmdb/cross-site-sharded-replica -o 'jsonpath={.spec.replsets[].expose.enabled}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.PMeUQIhhGu ++++ mktemp +++ local LAST_ERR=/tmp/tmp.bVhkZxPTms +++ local exit_status=0 +++ local timeout=4 ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl get psmdb/cross-site-sharded-replica -o 'jsonpath={.spec.replsets[].expose.enabled}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 -a -n 1 ']' +++ break +++ cat /tmp/tmp.PMeUQIhhGu +++ cat /tmp/tmp.bVhkZxPTms +++ rm /tmp/tmp.PMeUQIhhGu /tmp/tmp.bVhkZxPTms +++ return 0 ++ '[' true '!=' true ']' ++ kubectl_bin get service/cross-site-sharded-replica-rs1-0 -o 'jsonpath={.spec.type}' ++ grep -q NotFound +++ kubectl_bin get service/cross-site-sharded-replica-rs1-0 -o 'jsonpath={.spec.type}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.m4ClOQ7pji ++++ mktemp +++ local LAST_ERR=/tmp/tmp.IeTvmVIKGw +++ local exit_status=0 +++ local timeout=4 ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl get service/cross-site-sharded-replica-rs1-0 -o 'jsonpath={.spec.type}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 -a -n 1 ']' +++ break +++ cat /tmp/tmp.m4ClOQ7pji +++ cat /tmp/tmp.IeTvmVIKGw +++ rm /tmp/tmp.m4ClOQ7pji /tmp/tmp.IeTvmVIKGw +++ return 0 ++ service_type=ClusterIP ++ '[' ClusterIP = ClusterIP ']' ++ kubectl_bin get service/cross-site-sharded-replica-rs1-0 -o 'jsonpath={.spec.clusterIP}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.G9U0kKRw50 +++ mktemp ++ local LAST_ERR=/tmp/tmp.OnOdMIE2DP ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get service/cross-site-sharded-replica-rs1-0 -o 'jsonpath={.spec.clusterIP}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.G9U0kKRw50 ++ cat /tmp/tmp.OnOdMIE2DP ++ rm /tmp/tmp.G9U0kKRw50 /tmp/tmp.OnOdMIE2DP ++ return 0 ++ return + replica_rs1_0_endpoint=34.118.235.226 ++ get_service_ip cross-site-sharded-replica-rs1-1 rs1 ++ local service=cross-site-sharded-replica-rs1-1 ++ local server_type=rs1 +++ kubectl_bin get psmdb/cross-site-sharded-replica -o 'jsonpath={.spec.replsets[].expose.enabled}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.5kVoH5ZheA ++++ mktemp +++ local LAST_ERR=/tmp/tmp.5tKJgyk3Sy +++ local exit_status=0 +++ local timeout=4 ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl get psmdb/cross-site-sharded-replica -o 'jsonpath={.spec.replsets[].expose.enabled}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 -a -n 1 ']' +++ break +++ cat /tmp/tmp.5kVoH5ZheA +++ cat /tmp/tmp.5tKJgyk3Sy +++ rm /tmp/tmp.5kVoH5ZheA /tmp/tmp.5tKJgyk3Sy +++ return 0 ++ '[' true '!=' true ']' ++ kubectl_bin get service/cross-site-sharded-replica-rs1-1 -o 'jsonpath={.spec.type}' ++ grep -q NotFound +++ kubectl_bin get service/cross-site-sharded-replica-rs1-1 -o 'jsonpath={.spec.type}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.IMcaZd7Sa3 ++++ mktemp +++ local LAST_ERR=/tmp/tmp.P4fqizujzs +++ local exit_status=0 +++ local timeout=4 ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl get service/cross-site-sharded-replica-rs1-1 -o 'jsonpath={.spec.type}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 -a -n 1 ']' +++ break +++ cat /tmp/tmp.IMcaZd7Sa3 +++ cat /tmp/tmp.P4fqizujzs +++ rm /tmp/tmp.IMcaZd7Sa3 /tmp/tmp.P4fqizujzs +++ return 0 ++ service_type=ClusterIP ++ '[' ClusterIP = ClusterIP ']' ++ kubectl_bin get service/cross-site-sharded-replica-rs1-1 -o 'jsonpath={.spec.clusterIP}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.MFtzEQJVUS +++ mktemp ++ local LAST_ERR=/tmp/tmp.IAsBhJ0j4T ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get service/cross-site-sharded-replica-rs1-1 -o 'jsonpath={.spec.clusterIP}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.MFtzEQJVUS ++ cat /tmp/tmp.IAsBhJ0j4T ++ rm /tmp/tmp.MFtzEQJVUS /tmp/tmp.IAsBhJ0j4T ++ return 0 ++ return + replica_rs1_1_endpoint=34.118.230.203 ++ get_service_ip cross-site-sharded-replica-rs1-2 rs1 ++ local service=cross-site-sharded-replica-rs1-2 ++ local server_type=rs1 +++ kubectl_bin get psmdb/cross-site-sharded-replica -o 'jsonpath={.spec.replsets[].expose.enabled}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.8z8OFhtDlo ++++ mktemp +++ local LAST_ERR=/tmp/tmp.fxQ8dmsF97 +++ local exit_status=0 +++ local timeout=4 ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl get psmdb/cross-site-sharded-replica -o 'jsonpath={.spec.replsets[].expose.enabled}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 -a -n 1 ']' +++ break +++ cat /tmp/tmp.8z8OFhtDlo +++ cat /tmp/tmp.fxQ8dmsF97 +++ rm /tmp/tmp.8z8OFhtDlo /tmp/tmp.fxQ8dmsF97 +++ return 0 ++ '[' true '!=' true ']' ++ kubectl_bin get service/cross-site-sharded-replica-rs1-2 -o 'jsonpath={.spec.type}' ++ grep -q NotFound +++ kubectl_bin get service/cross-site-sharded-replica-rs1-2 -o 'jsonpath={.spec.type}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.6kY5cryptQ ++++ mktemp +++ local LAST_ERR=/tmp/tmp.GdgED7T0ga +++ local exit_status=0 +++ local timeout=4 ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl get service/cross-site-sharded-replica-rs1-2 -o 'jsonpath={.spec.type}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 -a -n 1 ']' +++ break +++ cat /tmp/tmp.6kY5cryptQ +++ cat /tmp/tmp.GdgED7T0ga +++ rm /tmp/tmp.6kY5cryptQ /tmp/tmp.GdgED7T0ga +++ return 0 ++ service_type=ClusterIP ++ '[' ClusterIP = ClusterIP ']' ++ kubectl_bin get service/cross-site-sharded-replica-rs1-2 -o 'jsonpath={.spec.clusterIP}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.UsIB5UIT0p +++ mktemp ++ local LAST_ERR=/tmp/tmp.pv6RlteEWx ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get service/cross-site-sharded-replica-rs1-2 -o 'jsonpath={.spec.clusterIP}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.UsIB5UIT0p ++ cat /tmp/tmp.pv6RlteEWx ++ rm /tmp/tmp.UsIB5UIT0p /tmp/tmp.pv6RlteEWx ++ return 0 ++ return + replica_rs1_2_endpoint=34.118.232.78 ++ kubectl_bin config current-context +++ mktemp ++ local LAST_OUT=/tmp/tmp.HlRFk1Lpw4 +++ mktemp ++ local LAST_ERR=/tmp/tmp.aNQq2TZZmk ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl config current-context ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.HlRFk1Lpw4 ++ cat /tmp/tmp.aNQq2TZZmk ++ rm /tmp/tmp.HlRFk1Lpw4 /tmp/tmp.aNQq2TZZmk ++ return 0 + kubectl_bin config set-context gke_cloud-dev-112233_us-central1-a_jen-psmdb-2271-c31e8c20-4-cluster3 --namespace=cross-site-sharded-17853 ++ mktemp + local LAST_OUT=/tmp/tmp.Pxum3cVSaT ++ mktemp + local LAST_ERR=/tmp/tmp.My3ij3KIYa + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl config set-context gke_cloud-dev-112233_us-central1-a_jen-psmdb-2271-c31e8c20-4-cluster3 --namespace=cross-site-sharded-17853 + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.Pxum3cVSaT Context "gke_cloud-dev-112233_us-central1-a_jen-psmdb-2271-c31e8c20-4-cluster3" modified. + cat /tmp/tmp.My3ij3KIYa + rm /tmp/tmp.Pxum3cVSaT /tmp/tmp.My3ij3KIYa + return 0 + kubectl_bin patch psmdb cross-site-sharded-main --type=merge --patch '{ "spec": {"replsets":[ {"affinity":{"antiAffinityTopologyKey": "none"},"arbiter":{"affinity":{"antiAffinityTopologyKey": "none"},"enabled":false,"size":1},"expose":{"enabled":true,"type":"ClusterIp"},"externalNodes":[{"host":"34.118.233.158","priority":0,"votes":0},{"host":"34.118.231.9","port":27017,"priority":1,"votes":1},{"host":"34.118.225.248", "port":27017,"priority":1,"votes":1}],"name":"rs0","nonvoting":{"affinity":{"antiAffinityTopologyKey":"none"},"enabled":false,"podDisruptionBudget":{"maxUnavailable":1},"resources":{"limits":{"cpu":"300m","memory":"0.5G"},"requests":{"cpu":"300m","memory":"0.5G"}},"size":3,"volumeSpec":{"persistentVolumeClaim":{"resources":{"requests":{"storage":"1Gi"}}}}},"podDisruptionBudget":{"maxUnavailable":1},"resources":{"limits":{"cpu":"300m","memory":"0.5G"},"requests":{"cpu":"300m","memory":"0.5G"}},"size":3,"volumeSpec":{"persistentVolumeClaim":{"resources":{"requests":{"storage":"3Gi"}}}}}, {"affinity":{"antiAffinityTopologyKey": "none"},"arbiter":{"affinity":{"antiAffinityTopologyKey": "none"},"enabled":false,"size":1},"expose":{"enabled":true,"type":"ClusterIp"},"externalNodes":[{"host":"34.118.235.226","priority":0,"votes":0},{"host":"34.118.230.203","port":27017,"priority":1,"votes":1},{"host":"34.118.232.78", "port":27017,"priority":1,"votes":1}],"name":"rs1","nonvoting":{"affinity":{"antiAffinityTopologyKey":"none"},"enabled":false,"podDisruptionBudget":{"maxUnavailable":1},"resources":{"limits":{"cpu":"300m","memory":"0.5G"},"requests":{"cpu":"300m","memory":"0.5G"}},"size":3,"volumeSpec":{"persistentVolumeClaim":{"resources":{"requests":{"storage":"1Gi"}}}}},"podDisruptionBudget":{"maxUnavailable":1},"resources":{"limits":{"cpu":"300m","memory":"0.5G"},"requests":{"cpu":"300m","memory":"0.5G"}},"size":3,"volumeSpec":{"persistentVolumeClaim":{"resources":{"requests":{"storage":"3Gi"}}}}} ], "sharding":{"configsvrReplSet":{ "externalNodes": [{"host":"34.118.232.156","priority":1,"votes":1 },{"host":"34.118.225.129", "priority":1,"votes":1},{"host":"34.118.228.22","priority":0,"votes":0}]}} } }' ++ mktemp + local LAST_OUT=/tmp/tmp.gFMMom6ZMu ++ mktemp + local LAST_ERR=/tmp/tmp.NpsygJ8K2b + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl patch psmdb cross-site-sharded-main --type=merge --patch '{ "spec": {"replsets":[ {"affinity":{"antiAffinityTopologyKey": "none"},"arbiter":{"affinity":{"antiAffinityTopologyKey": "none"},"enabled":false,"size":1},"expose":{"enabled":true,"type":"ClusterIp"},"externalNodes":[{"host":"34.118.233.158","priority":0,"votes":0},{"host":"34.118.231.9","port":27017,"priority":1,"votes":1},{"host":"34.118.225.248", "port":27017,"priority":1,"votes":1}],"name":"rs0","nonvoting":{"affinity":{"antiAffinityTopologyKey":"none"},"enabled":false,"podDisruptionBudget":{"maxUnavailable":1},"resources":{"limits":{"cpu":"300m","memory":"0.5G"},"requests":{"cpu":"300m","memory":"0.5G"}},"size":3,"volumeSpec":{"persistentVolumeClaim":{"resources":{"requests":{"storage":"1Gi"}}}}},"podDisruptionBudget":{"maxUnavailable":1},"resources":{"limits":{"cpu":"300m","memory":"0.5G"},"requests":{"cpu":"300m","memory":"0.5G"}},"size":3,"volumeSpec":{"persistentVolumeClaim":{"resources":{"requests":{"storage":"3Gi"}}}}}, {"affinity":{"antiAffinityTopologyKey": "none"},"arbiter":{"affinity":{"antiAffinityTopologyKey": "none"},"enabled":false,"size":1},"expose":{"enabled":true,"type":"ClusterIp"},"externalNodes":[{"host":"34.118.235.226","priority":0,"votes":0},{"host":"34.118.230.203","port":27017,"priority":1,"votes":1},{"host":"34.118.232.78", "port":27017,"priority":1,"votes":1}],"name":"rs1","nonvoting":{"affinity":{"antiAffinityTopologyKey":"none"},"enabled":false,"podDisruptionBudget":{"maxUnavailable":1},"resources":{"limits":{"cpu":"300m","memory":"0.5G"},"requests":{"cpu":"300m","memory":"0.5G"}},"size":3,"volumeSpec":{"persistentVolumeClaim":{"resources":{"requests":{"storage":"1Gi"}}}}},"podDisruptionBudget":{"maxUnavailable":1},"resources":{"limits":{"cpu":"300m","memory":"0.5G"},"requests":{"cpu":"300m","memory":"0.5G"}},"size":3,"volumeSpec":{"persistentVolumeClaim":{"resources":{"requests":{"storage":"3Gi"}}}}} ], "sharding":{"configsvrReplSet":{ "externalNodes": [{"host":"34.118.232.156","priority":1,"votes":1 },{"host":"34.118.225.129", "priority":1,"votes":1},{"host":"34.118.228.22","priority":0,"votes":0}]}} } }' + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.gFMMom6ZMu perconaservermongodb.psmdb.percona.com/cross-site-sharded-main patched + cat /tmp/tmp.NpsygJ8K2b + rm /tmp/tmp.gFMMom6ZMu /tmp/tmp.NpsygJ8K2b + return 0 + wait_for_members 34.118.232.156 cfg 6 + local endpoint=34.118.232.156 + local rsName=cfg + local target_count=6 + local nodes_count=0 + [[ 0 == 6 ]] ++ run_mongos 'rs.conf().members.length' clusterAdmin:clusterAdmin123456@34.118.232.156 mongodb :27017 ++ local 'command=rs.conf().members.length' ++ local uri=clusterAdmin:clusterAdmin123456@34.118.232.156 ++ local driver=mongodb ++ local suffix=:27017 ++ local mongo_flag= ++ local port=27017 ++ local mongo_bin=mongo ++ grep -E -v 'I NETWORK|W NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|bye' ++ /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/' +++ echo :27017 +++ awk -F: '{print $2}' ++ suffix_port=27017 ++ [[ -z 27017 ]] +++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.4I4dNI1dvF ++++ mktemp +++ local LAST_ERR=/tmp/tmp.NhasnhjIRd +++ local exit_status=0 +++ local timeout=4 ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 -a -n 1 ']' +++ break +++ cat /tmp/tmp.4I4dNI1dvF +++ cat /tmp/tmp.NhasnhjIRd +++ rm /tmp/tmp.4I4dNI1dvF /tmp/tmp.NhasnhjIRd +++ return 0 ++ local client_container=psmdb-client-bb8b97679-l4k9z ++ kubectl_bin exec psmdb-client-bb8b97679-l4k9z -- bash -c 'printf '\''rs.conf().members.length\n'\'' | mongo mongodb://clusterAdmin:clusterAdmin123456@34.118.232.156:27017/admin ' +++ mktemp ++ local LAST_OUT=/tmp/tmp.nnCuxR7oqA +++ mktemp ++ local LAST_ERR=/tmp/tmp.iajFilXtDU ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl exec psmdb-client-bb8b97679-l4k9z -- bash -c 'printf '\''rs.conf().members.length\n'\'' | mongo mongodb://clusterAdmin:clusterAdmin123456@34.118.232.156:27017/admin ' ++ exit_status=1 ++ set -e ++ '[' 1 '!=' 0 -a -n 1 ']' ++ cat /tmp/tmp.nnCuxR7oqA ++ cat /tmp/tmp.iajFilXtDU command terminated with exit code 1 ++ sleep 0 ++ for i in $(seq 0 2) ++ set +e ++ kubectl exec psmdb-client-bb8b97679-l4k9z -- bash -c 'printf '\''rs.conf().members.length\n'\'' | mongo mongodb://clusterAdmin:clusterAdmin123456@34.118.232.156:27017/admin ' ++ exit_status=1 ++ set -e ++ '[' 1 '!=' 0 -a -n 1 ']' ++ cat /tmp/tmp.nnCuxR7oqA ++ cat /tmp/tmp.iajFilXtDU command terminated with exit code 1 ++ sleep 4 ++ for i in $(seq 0 2) ++ set +e ++ kubectl exec psmdb-client-bb8b97679-l4k9z -- bash -c 'printf '\''rs.conf().members.length\n'\'' | mongo mongodb://clusterAdmin:clusterAdmin123456@34.118.232.156:27017/admin ' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.nnCuxR7oqA ++ cat /tmp/tmp.iajFilXtDU ++ rm /tmp/tmp.nnCuxR7oqA /tmp/tmp.iajFilXtDU ++ return 0 + nodes_count='Error: Authentication failed. : connect@src/mongo/shell/mongo.js:374:17 @(connect):2:6 exception: connect failed exiting with code 1 Error: Authentication failed. : connect@src/mongo/shell/mongo.js:374:17 @(connect):2:6 exception: connect failed exiting with code 1 4' + echo -n 'waiting for all members to be configured in cfg' waiting for all members to be configured in cfg+ let retry+=1 + '[' 1 -ge 15 ']' + echo . . + sleep 10 + [[ Error: Authentication failed. : connect@src/mongo/shell/mongo.js:374:17 @(connect):2:6 exception: connect failed exiting with code 1 Error: Authentication failed. : connect@src/mongo/shell/mongo.js:374:17 @(connect):2:6 exception: connect failed exiting with code 1 4 == 6 ]] ++ run_mongos 'rs.conf().members.length' clusterAdmin:clusterAdmin123456@34.118.232.156 mongodb :27017 ++ local 'command=rs.conf().members.length' ++ local uri=clusterAdmin:clusterAdmin123456@34.118.232.156 ++ local driver=mongodb ++ local suffix=:27017 ++ local mongo_flag= ++ local port=27017 ++ local mongo_bin=mongo ++ grep -E -v 'I NETWORK|W NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|bye' ++ /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/' +++ echo :27017 +++ awk -F: '{print $2}' ++ suffix_port=27017 ++ [[ -z 27017 ]] +++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.YbuqBAMifC ++++ mktemp +++ local LAST_ERR=/tmp/tmp.vuhdLdC27r +++ local exit_status=0 +++ local timeout=4 ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 -a -n 1 ']' +++ break +++ cat /tmp/tmp.YbuqBAMifC +++ cat /tmp/tmp.vuhdLdC27r +++ rm /tmp/tmp.YbuqBAMifC /tmp/tmp.vuhdLdC27r +++ return 0 ++ local client_container=psmdb-client-bb8b97679-l4k9z ++ kubectl_bin exec psmdb-client-bb8b97679-l4k9z -- bash -c 'printf '\''rs.conf().members.length\n'\'' | mongo mongodb://clusterAdmin:clusterAdmin123456@34.118.232.156:27017/admin ' +++ mktemp ++ local LAST_OUT=/tmp/tmp.d8qERHgI49 +++ mktemp ++ local LAST_ERR=/tmp/tmp.gx3wu0zmQR ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl exec psmdb-client-bb8b97679-l4k9z -- bash -c 'printf '\''rs.conf().members.length\n'\'' | mongo mongodb://clusterAdmin:clusterAdmin123456@34.118.232.156:27017/admin ' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.d8qERHgI49 ++ cat /tmp/tmp.gx3wu0zmQR ++ rm /tmp/tmp.d8qERHgI49 /tmp/tmp.gx3wu0zmQR ++ return 0 + nodes_count=5 + echo -n 'waiting for all members to be configured in cfg' waiting for all members to be configured in cfg+ let retry+=1 + '[' 2 -ge 15 ']' + echo . . + sleep 10 + [[ 5 == 6 ]] ++ run_mongos 'rs.conf().members.length' clusterAdmin:clusterAdmin123456@34.118.232.156 mongodb :27017 ++ local 'command=rs.conf().members.length' ++ local uri=clusterAdmin:clusterAdmin123456@34.118.232.156 ++ local driver=mongodb ++ local suffix=:27017 ++ local mongo_flag= ++ local port=27017 ++ local mongo_bin=mongo ++ grep -E -v 'I NETWORK|W NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|bye' ++ /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/' +++ echo :27017 +++ awk -F: '{print $2}' ++ suffix_port=27017 ++ [[ -z 27017 ]] +++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.sjK0HizU7H ++++ mktemp +++ local LAST_ERR=/tmp/tmp.6aFooRLeEu +++ local exit_status=0 +++ local timeout=4 ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 -a -n 1 ']' +++ break +++ cat /tmp/tmp.sjK0HizU7H +++ cat /tmp/tmp.6aFooRLeEu +++ rm /tmp/tmp.sjK0HizU7H /tmp/tmp.6aFooRLeEu +++ return 0 ++ local client_container=psmdb-client-bb8b97679-l4k9z ++ kubectl_bin exec psmdb-client-bb8b97679-l4k9z -- bash -c 'printf '\''rs.conf().members.length\n'\'' | mongo mongodb://clusterAdmin:clusterAdmin123456@34.118.232.156:27017/admin ' +++ mktemp ++ local LAST_OUT=/tmp/tmp.lE7Yij9YxP +++ mktemp ++ local LAST_ERR=/tmp/tmp.uYB0juRDgO ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl exec psmdb-client-bb8b97679-l4k9z -- bash -c 'printf '\''rs.conf().members.length\n'\'' | mongo mongodb://clusterAdmin:clusterAdmin123456@34.118.232.156:27017/admin ' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.lE7Yij9YxP ++ cat /tmp/tmp.uYB0juRDgO ++ rm /tmp/tmp.lE7Yij9YxP /tmp/tmp.uYB0juRDgO ++ return 0 + nodes_count=5 + echo -n 'waiting for all members to be configured in cfg' waiting for all members to be configured in cfg+ let retry+=1 + '[' 3 -ge 15 ']' + echo . . + sleep 10 + [[ 5 == 6 ]] ++ run_mongos 'rs.conf().members.length' clusterAdmin:clusterAdmin123456@34.118.232.156 mongodb :27017 ++ local 'command=rs.conf().members.length' ++ local uri=clusterAdmin:clusterAdmin123456@34.118.232.156 ++ local driver=mongodb ++ local suffix=:27017 ++ grep -E -v 'I NETWORK|W NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|bye' ++ local mongo_flag= ++ local port=27017 ++ local mongo_bin=mongo ++ /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/' +++ echo :27017 +++ awk -F: '{print $2}' ++ suffix_port=27017 ++ [[ -z 27017 ]] +++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.ikXXgx1sVe ++++ mktemp +++ local LAST_ERR=/tmp/tmp.LIgkhTpVxl +++ local exit_status=0 +++ local timeout=4 ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 -a -n 1 ']' +++ break +++ cat /tmp/tmp.ikXXgx1sVe +++ cat /tmp/tmp.LIgkhTpVxl +++ rm /tmp/tmp.ikXXgx1sVe /tmp/tmp.LIgkhTpVxl +++ return 0 ++ local client_container=psmdb-client-bb8b97679-l4k9z ++ kubectl_bin exec psmdb-client-bb8b97679-l4k9z -- bash -c 'printf '\''rs.conf().members.length\n'\'' | mongo mongodb://clusterAdmin:clusterAdmin123456@34.118.232.156:27017/admin ' +++ mktemp ++ local LAST_OUT=/tmp/tmp.0sS08cx0Hc +++ mktemp ++ local LAST_ERR=/tmp/tmp.jaW1W9YQ7p ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl exec psmdb-client-bb8b97679-l4k9z -- bash -c 'printf '\''rs.conf().members.length\n'\'' | mongo mongodb://clusterAdmin:clusterAdmin123456@34.118.232.156:27017/admin ' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.0sS08cx0Hc ++ cat /tmp/tmp.jaW1W9YQ7p ++ rm /tmp/tmp.0sS08cx0Hc /tmp/tmp.jaW1W9YQ7p ++ return 0 + nodes_count=5 + echo -n 'waiting for all members to be configured in cfg' waiting for all members to be configured in cfg+ let retry+=1 + '[' 4 -ge 15 ']' + echo . . + sleep 10 + [[ 5 == 6 ]] ++ grep -E -v 'I NETWORK|W NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|bye' ++ run_mongos 'rs.conf().members.length' clusterAdmin:clusterAdmin123456@34.118.232.156 mongodb :27017 ++ /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/' ++ local 'command=rs.conf().members.length' ++ local uri=clusterAdmin:clusterAdmin123456@34.118.232.156 ++ local driver=mongodb ++ local suffix=:27017 ++ local mongo_flag= ++ local port=27017 ++ local mongo_bin=mongo +++ echo :27017 +++ awk -F: '{print $2}' ++ suffix_port=27017 ++ [[ -z 27017 ]] +++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.NTMNzMKxC8 ++++ mktemp +++ local LAST_ERR=/tmp/tmp.Y0gfs9Zh86 +++ local exit_status=0 +++ local timeout=4 ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 -a -n 1 ']' +++ break +++ cat /tmp/tmp.NTMNzMKxC8 +++ cat /tmp/tmp.Y0gfs9Zh86 +++ rm /tmp/tmp.NTMNzMKxC8 /tmp/tmp.Y0gfs9Zh86 +++ return 0 ++ local client_container=psmdb-client-bb8b97679-l4k9z ++ kubectl_bin exec psmdb-client-bb8b97679-l4k9z -- bash -c 'printf '\''rs.conf().members.length\n'\'' | mongo mongodb://clusterAdmin:clusterAdmin123456@34.118.232.156:27017/admin ' +++ mktemp ++ local LAST_OUT=/tmp/tmp.RS5Bi8l8WN +++ mktemp ++ local LAST_ERR=/tmp/tmp.QnbE29TQnC ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl exec psmdb-client-bb8b97679-l4k9z -- bash -c 'printf '\''rs.conf().members.length\n'\'' | mongo mongodb://clusterAdmin:clusterAdmin123456@34.118.232.156:27017/admin ' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.RS5Bi8l8WN ++ cat /tmp/tmp.QnbE29TQnC ++ rm /tmp/tmp.RS5Bi8l8WN /tmp/tmp.QnbE29TQnC ++ return 0 + nodes_count=5 + echo -n 'waiting for all members to be configured in cfg' waiting for all members to be configured in cfg+ let retry+=1 + '[' 5 -ge 15 ']' + echo . . + sleep 10 + [[ 5 == 6 ]] ++ run_mongos 'rs.conf().members.length' clusterAdmin:clusterAdmin123456@34.118.232.156 mongodb :27017 ++ local 'command=rs.conf().members.length' ++ local uri=clusterAdmin:clusterAdmin123456@34.118.232.156 ++ local driver=mongodb ++ local suffix=:27017 ++ local mongo_flag= ++ local port=27017 ++ local mongo_bin=mongo ++ grep -E -v 'I NETWORK|W NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|bye' ++ /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/' +++ echo :27017 +++ awk -F: '{print $2}' ++ suffix_port=27017 ++ [[ -z 27017 ]] +++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.T9JGtq44gf ++++ mktemp +++ local LAST_ERR=/tmp/tmp.gfKL3JnkMy +++ local exit_status=0 +++ local timeout=4 ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 -a -n 1 ']' +++ break +++ cat /tmp/tmp.T9JGtq44gf +++ cat /tmp/tmp.gfKL3JnkMy +++ rm /tmp/tmp.T9JGtq44gf /tmp/tmp.gfKL3JnkMy +++ return 0 ++ local client_container=psmdb-client-bb8b97679-l4k9z ++ kubectl_bin exec psmdb-client-bb8b97679-l4k9z -- bash -c 'printf '\''rs.conf().members.length\n'\'' | mongo mongodb://clusterAdmin:clusterAdmin123456@34.118.232.156:27017/admin ' +++ mktemp ++ local LAST_OUT=/tmp/tmp.HXjjLTlu9l +++ mktemp ++ local LAST_ERR=/tmp/tmp.XWZdK98gKq ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl exec psmdb-client-bb8b97679-l4k9z -- bash -c 'printf '\''rs.conf().members.length\n'\'' | mongo mongodb://clusterAdmin:clusterAdmin123456@34.118.232.156:27017/admin ' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.HXjjLTlu9l ++ cat /tmp/tmp.XWZdK98gKq ++ rm /tmp/tmp.HXjjLTlu9l /tmp/tmp.XWZdK98gKq ++ return 0 + nodes_count=5 + echo -n 'waiting for all members to be configured in cfg' waiting for all members to be configured in cfg+ let retry+=1 + '[' 6 -ge 15 ']' + echo . . + sleep 10 + [[ 5 == 6 ]] ++ run_mongos 'rs.conf().members.length' clusterAdmin:clusterAdmin123456@34.118.232.156 mongodb :27017 ++ local 'command=rs.conf().members.length' ++ local uri=clusterAdmin:clusterAdmin123456@34.118.232.156 ++ local driver=mongodb ++ local suffix=:27017 ++ local mongo_flag= ++ local port=27017 ++ local mongo_bin=mongo ++ grep -E -v 'I NETWORK|W NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|bye' ++ /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/' +++ echo :27017 +++ awk -F: '{print $2}' ++ suffix_port=27017 ++ [[ -z 27017 ]] +++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.0sb7CrwSWf ++++ mktemp +++ local LAST_ERR=/tmp/tmp.Lz7DuJ10vr +++ local exit_status=0 +++ local timeout=4 ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 -a -n 1 ']' +++ break +++ cat /tmp/tmp.0sb7CrwSWf +++ cat /tmp/tmp.Lz7DuJ10vr +++ rm /tmp/tmp.0sb7CrwSWf /tmp/tmp.Lz7DuJ10vr +++ return 0 ++ local client_container=psmdb-client-bb8b97679-l4k9z ++ kubectl_bin exec psmdb-client-bb8b97679-l4k9z -- bash -c 'printf '\''rs.conf().members.length\n'\'' | mongo mongodb://clusterAdmin:clusterAdmin123456@34.118.232.156:27017/admin ' +++ mktemp ++ local LAST_OUT=/tmp/tmp.ORZ1chOi2C +++ mktemp ++ local LAST_ERR=/tmp/tmp.GZa59515OU ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl exec psmdb-client-bb8b97679-l4k9z -- bash -c 'printf '\''rs.conf().members.length\n'\'' | mongo mongodb://clusterAdmin:clusterAdmin123456@34.118.232.156:27017/admin ' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.ORZ1chOi2C ++ cat /tmp/tmp.GZa59515OU ++ rm /tmp/tmp.ORZ1chOi2C /tmp/tmp.GZa59515OU ++ return 0 + nodes_count=5 + echo -n 'waiting for all members to be configured in cfg' waiting for all members to be configured in cfg+ let retry+=1 + '[' 7 -ge 15 ']' + echo . . + sleep 10 + [[ 5 == 6 ]] ++ run_mongos 'rs.conf().members.length' clusterAdmin:clusterAdmin123456@34.118.232.156 mongodb :27017 ++ local 'command=rs.conf().members.length' ++ local uri=clusterAdmin:clusterAdmin123456@34.118.232.156 ++ local driver=mongodb ++ local suffix=:27017 ++ local mongo_flag= ++ local port=27017 ++ local mongo_bin=mongo ++ grep -E -v 'I NETWORK|W NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|bye' ++ /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/' +++ echo :27017 +++ awk -F: '{print $2}' ++ suffix_port=27017 ++ [[ -z 27017 ]] +++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.Eberq55PDd ++++ mktemp +++ local LAST_ERR=/tmp/tmp.lHcEi0wTzE +++ local exit_status=0 +++ local timeout=4 ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 -a -n 1 ']' +++ break +++ cat /tmp/tmp.Eberq55PDd +++ cat /tmp/tmp.lHcEi0wTzE +++ rm /tmp/tmp.Eberq55PDd /tmp/tmp.lHcEi0wTzE +++ return 0 ++ local client_container=psmdb-client-bb8b97679-l4k9z ++ kubectl_bin exec psmdb-client-bb8b97679-l4k9z -- bash -c 'printf '\''rs.conf().members.length\n'\'' | mongo mongodb://clusterAdmin:clusterAdmin123456@34.118.232.156:27017/admin ' +++ mktemp ++ local LAST_OUT=/tmp/tmp.nvuiBdqlSh +++ mktemp ++ local LAST_ERR=/tmp/tmp.NXEPpnT98i ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl exec psmdb-client-bb8b97679-l4k9z -- bash -c 'printf '\''rs.conf().members.length\n'\'' | mongo mongodb://clusterAdmin:clusterAdmin123456@34.118.232.156:27017/admin ' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.nvuiBdqlSh ++ cat /tmp/tmp.NXEPpnT98i ++ rm /tmp/tmp.nvuiBdqlSh /tmp/tmp.NXEPpnT98i ++ return 0 + nodes_count=5 + echo -n 'waiting for all members to be configured in cfg' waiting for all members to be configured in cfg+ let retry+=1 + '[' 8 -ge 15 ']' + echo . . + sleep 10 + [[ 5 == 6 ]] ++ run_mongos 'rs.conf().members.length' clusterAdmin:clusterAdmin123456@34.118.232.156 mongodb :27017 ++ local 'command=rs.conf().members.length' ++ local uri=clusterAdmin:clusterAdmin123456@34.118.232.156 ++ local driver=mongodb ++ local suffix=:27017 ++ local mongo_flag= ++ local port=27017 ++ local mongo_bin=mongo ++ grep -E -v 'I NETWORK|W NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|bye' ++ /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/' +++ echo :27017 +++ awk -F: '{print $2}' ++ suffix_port=27017 ++ [[ -z 27017 ]] +++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.D8Wq44XEOW ++++ mktemp +++ local LAST_ERR=/tmp/tmp.1pSfZ77lbb +++ local exit_status=0 +++ local timeout=4 ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 -a -n 1 ']' +++ break +++ cat /tmp/tmp.D8Wq44XEOW +++ cat /tmp/tmp.1pSfZ77lbb +++ rm /tmp/tmp.D8Wq44XEOW /tmp/tmp.1pSfZ77lbb +++ return 0 ++ local client_container=psmdb-client-bb8b97679-l4k9z ++ kubectl_bin exec psmdb-client-bb8b97679-l4k9z -- bash -c 'printf '\''rs.conf().members.length\n'\'' | mongo mongodb://clusterAdmin:clusterAdmin123456@34.118.232.156:27017/admin ' +++ mktemp ++ local LAST_OUT=/tmp/tmp.HP7URe0qQI +++ mktemp ++ local LAST_ERR=/tmp/tmp.JAYq3Ex2JQ ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl exec psmdb-client-bb8b97679-l4k9z -- bash -c 'printf '\''rs.conf().members.length\n'\'' | mongo mongodb://clusterAdmin:clusterAdmin123456@34.118.232.156:27017/admin ' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.HP7URe0qQI ++ cat /tmp/tmp.JAYq3Ex2JQ ++ rm /tmp/tmp.HP7URe0qQI /tmp/tmp.JAYq3Ex2JQ ++ return 0 + nodes_count=5 + echo -n 'waiting for all members to be configured in cfg' waiting for all members to be configured in cfg+ let retry+=1 + '[' 9 -ge 15 ']' + echo . . + sleep 10 + [[ 5 == 6 ]] ++ run_mongos 'rs.conf().members.length' clusterAdmin:clusterAdmin123456@34.118.232.156 mongodb :27017 ++ local 'command=rs.conf().members.length' ++ local uri=clusterAdmin:clusterAdmin123456@34.118.232.156 ++ local driver=mongodb ++ local suffix=:27017 ++ grep -E -v 'I NETWORK|W NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|bye' ++ local mongo_flag= ++ local port=27017 ++ local mongo_bin=mongo ++ /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/' +++ echo :27017 +++ awk -F: '{print $2}' ++ suffix_port=27017 ++ [[ -z 27017 ]] +++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.vw7lwtx1GD ++++ mktemp +++ local LAST_ERR=/tmp/tmp.FfIiqASJce +++ local exit_status=0 +++ local timeout=4 ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 -a -n 1 ']' +++ break +++ cat /tmp/tmp.vw7lwtx1GD +++ cat /tmp/tmp.FfIiqASJce +++ rm /tmp/tmp.vw7lwtx1GD /tmp/tmp.FfIiqASJce +++ return 0 ++ local client_container=psmdb-client-bb8b97679-l4k9z ++ kubectl_bin exec psmdb-client-bb8b97679-l4k9z -- bash -c 'printf '\''rs.conf().members.length\n'\'' | mongo mongodb://clusterAdmin:clusterAdmin123456@34.118.232.156:27017/admin ' +++ mktemp ++ local LAST_OUT=/tmp/tmp.7rD0LCINUP +++ mktemp ++ local LAST_ERR=/tmp/tmp.ne7Vm8fYiu ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl exec psmdb-client-bb8b97679-l4k9z -- bash -c 'printf '\''rs.conf().members.length\n'\'' | mongo mongodb://clusterAdmin:clusterAdmin123456@34.118.232.156:27017/admin ' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.7rD0LCINUP ++ cat /tmp/tmp.ne7Vm8fYiu ++ rm /tmp/tmp.7rD0LCINUP /tmp/tmp.ne7Vm8fYiu ++ return 0 + nodes_count=5 + echo -n 'waiting for all members to be configured in cfg' waiting for all members to be configured in cfg+ let retry+=1 + '[' 10 -ge 15 ']' + echo . . + sleep 10 + [[ 5 == 6 ]] ++ run_mongos 'rs.conf().members.length' clusterAdmin:clusterAdmin123456@34.118.232.156 mongodb :27017 ++ local 'command=rs.conf().members.length' ++ grep -E -v 'I NETWORK|W NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|bye' ++ local uri=clusterAdmin:clusterAdmin123456@34.118.232.156 ++ local driver=mongodb ++ local suffix=:27017 ++ local mongo_flag= ++ local port=27017 ++ local mongo_bin=mongo ++ /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/' +++ echo :27017 +++ awk -F: '{print $2}' ++ suffix_port=27017 ++ [[ -z 27017 ]] +++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.tB1TFhOdxf ++++ mktemp +++ local LAST_ERR=/tmp/tmp.yJrtqJTYTp +++ local exit_status=0 +++ local timeout=4 ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 -a -n 1 ']' +++ break +++ cat /tmp/tmp.tB1TFhOdxf +++ cat /tmp/tmp.yJrtqJTYTp +++ rm /tmp/tmp.tB1TFhOdxf /tmp/tmp.yJrtqJTYTp +++ return 0 ++ local client_container=psmdb-client-bb8b97679-l4k9z ++ kubectl_bin exec psmdb-client-bb8b97679-l4k9z -- bash -c 'printf '\''rs.conf().members.length\n'\'' | mongo mongodb://clusterAdmin:clusterAdmin123456@34.118.232.156:27017/admin ' +++ mktemp ++ local LAST_OUT=/tmp/tmp.5IyH7GVtlO +++ mktemp ++ local LAST_ERR=/tmp/tmp.NlwmY7KYJh ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl exec psmdb-client-bb8b97679-l4k9z -- bash -c 'printf '\''rs.conf().members.length\n'\'' | mongo mongodb://clusterAdmin:clusterAdmin123456@34.118.232.156:27017/admin ' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.5IyH7GVtlO ++ cat /tmp/tmp.NlwmY7KYJh ++ rm /tmp/tmp.5IyH7GVtlO /tmp/tmp.NlwmY7KYJh ++ return 0 + nodes_count=6 + echo -n 'waiting for all members to be configured in cfg' waiting for all members to be configured in cfg+ let retry+=1 + '[' 11 -ge 15 ']' + echo . . + sleep 10 + [[ 6 == 6 ]] + wait_for_members 34.118.233.158 rs0 6 + local endpoint=34.118.233.158 + local rsName=rs0 + local target_count=6 + local nodes_count=0 + [[ 0 == 6 ]] ++ run_mongos 'rs.conf().members.length' clusterAdmin:clusterAdmin123456@34.118.233.158 mongodb :27017 ++ local 'command=rs.conf().members.length' ++ local uri=clusterAdmin:clusterAdmin123456@34.118.233.158 ++ local driver=mongodb ++ local suffix=:27017 ++ local mongo_flag= ++ local port=27017 ++ local mongo_bin=mongo +++ echo :27017 ++ grep -E -v 'I NETWORK|W NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|bye' +++ awk -F: '{print $2}' ++ /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/' ++ suffix_port=27017 ++ [[ -z 27017 ]] +++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.rvEevHe0yw ++++ mktemp +++ local LAST_ERR=/tmp/tmp.wkHTOQKL4p +++ local exit_status=0 +++ local timeout=4 ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 -a -n 1 ']' +++ break +++ cat /tmp/tmp.rvEevHe0yw +++ cat /tmp/tmp.wkHTOQKL4p +++ rm /tmp/tmp.rvEevHe0yw /tmp/tmp.wkHTOQKL4p +++ return 0 ++ local client_container=psmdb-client-bb8b97679-l4k9z ++ kubectl_bin exec psmdb-client-bb8b97679-l4k9z -- bash -c 'printf '\''rs.conf().members.length\n'\'' | mongo mongodb://clusterAdmin:clusterAdmin123456@34.118.233.158:27017/admin ' +++ mktemp ++ local LAST_OUT=/tmp/tmp.j3fInRYwZ2 +++ mktemp ++ local LAST_ERR=/tmp/tmp.NUpB43q4Rp ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl exec psmdb-client-bb8b97679-l4k9z -- bash -c 'printf '\''rs.conf().members.length\n'\'' | mongo mongodb://clusterAdmin:clusterAdmin123456@34.118.233.158:27017/admin ' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.j3fInRYwZ2 ++ cat /tmp/tmp.NUpB43q4Rp ++ rm /tmp/tmp.j3fInRYwZ2 /tmp/tmp.NUpB43q4Rp ++ return 0 + nodes_count=5 + echo -n 'waiting for all members to be configured in rs0' waiting for all members to be configured in rs0+ let retry+=1 + '[' 12 -ge 15 ']' + echo . . + sleep 10 + [[ 5 == 6 ]] ++ run_mongos 'rs.conf().members.length' clusterAdmin:clusterAdmin123456@34.118.233.158 mongodb :27017 ++ local 'command=rs.conf().members.length' ++ grep -E -v 'I NETWORK|W NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|bye' ++ local uri=clusterAdmin:clusterAdmin123456@34.118.233.158 ++ local driver=mongodb ++ local suffix=:27017 ++ local mongo_flag= ++ local port=27017 ++ local mongo_bin=mongo ++ /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/' +++ echo :27017 +++ awk -F: '{print $2}' ++ suffix_port=27017 ++ [[ -z 27017 ]] +++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.hRlCR8fmLf ++++ mktemp +++ local LAST_ERR=/tmp/tmp.BUn2oY22uG +++ local exit_status=0 +++ local timeout=4 ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 -a -n 1 ']' +++ break +++ cat /tmp/tmp.hRlCR8fmLf +++ cat /tmp/tmp.BUn2oY22uG +++ rm /tmp/tmp.hRlCR8fmLf /tmp/tmp.BUn2oY22uG +++ return 0 ++ local client_container=psmdb-client-bb8b97679-l4k9z ++ kubectl_bin exec psmdb-client-bb8b97679-l4k9z -- bash -c 'printf '\''rs.conf().members.length\n'\'' | mongo mongodb://clusterAdmin:clusterAdmin123456@34.118.233.158:27017/admin ' +++ mktemp ++ local LAST_OUT=/tmp/tmp.ddjo4ehlSG +++ mktemp ++ local LAST_ERR=/tmp/tmp.vZ5XxQj190 ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl exec psmdb-client-bb8b97679-l4k9z -- bash -c 'printf '\''rs.conf().members.length\n'\'' | mongo mongodb://clusterAdmin:clusterAdmin123456@34.118.233.158:27017/admin ' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.ddjo4ehlSG ++ cat /tmp/tmp.vZ5XxQj190 ++ rm /tmp/tmp.ddjo4ehlSG /tmp/tmp.vZ5XxQj190 ++ return 0 + nodes_count=5 + echo -n 'waiting for all members to be configured in rs0' waiting for all members to be configured in rs0+ let retry+=1 + '[' 13 -ge 15 ']' + echo . . + sleep 10 + [[ 5 == 6 ]] ++ run_mongos 'rs.conf().members.length' clusterAdmin:clusterAdmin123456@34.118.233.158 mongodb :27017 ++ grep -E -v 'I NETWORK|W NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|bye' ++ local 'command=rs.conf().members.length' ++ local uri=clusterAdmin:clusterAdmin123456@34.118.233.158 ++ local driver=mongodb ++ local suffix=:27017 ++ local mongo_flag= ++ local port=27017 ++ local mongo_bin=mongo ++ /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/' +++ echo :27017 +++ awk -F: '{print $2}' ++ suffix_port=27017 ++ [[ -z 27017 ]] +++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.y9ECQKt29m ++++ mktemp +++ local LAST_ERR=/tmp/tmp.bweS7ubZQ9 +++ local exit_status=0 +++ local timeout=4 ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 -a -n 1 ']' +++ break +++ cat /tmp/tmp.y9ECQKt29m +++ cat /tmp/tmp.bweS7ubZQ9 +++ rm /tmp/tmp.y9ECQKt29m /tmp/tmp.bweS7ubZQ9 +++ return 0 ++ local client_container=psmdb-client-bb8b97679-l4k9z ++ kubectl_bin exec psmdb-client-bb8b97679-l4k9z -- bash -c 'printf '\''rs.conf().members.length\n'\'' | mongo mongodb://clusterAdmin:clusterAdmin123456@34.118.233.158:27017/admin ' +++ mktemp ++ local LAST_OUT=/tmp/tmp.E0NJgHLAcc +++ mktemp ++ local LAST_ERR=/tmp/tmp.dBf9LVjM9k ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl exec psmdb-client-bb8b97679-l4k9z -- bash -c 'printf '\''rs.conf().members.length\n'\'' | mongo mongodb://clusterAdmin:clusterAdmin123456@34.118.233.158:27017/admin ' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.E0NJgHLAcc ++ cat /tmp/tmp.dBf9LVjM9k ++ rm /tmp/tmp.E0NJgHLAcc /tmp/tmp.dBf9LVjM9k ++ return 0 + nodes_count=5 + echo -n 'waiting for all members to be configured in rs0' waiting for all members to be configured in rs0+ let retry+=1 + '[' 14 -ge 15 ']' + echo . . + sleep 10 + [[ 5 == 6 ]] ++ run_mongos 'rs.conf().members.length' clusterAdmin:clusterAdmin123456@34.118.233.158 mongodb :27017 ++ local 'command=rs.conf().members.length' ++ local uri=clusterAdmin:clusterAdmin123456@34.118.233.158 ++ local driver=mongodb ++ local suffix=:27017 ++ local mongo_flag= ++ local port=27017 ++ local mongo_bin=mongo ++ grep -E -v 'I NETWORK|W NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|bye' ++ /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/' +++ echo :27017 +++ awk -F: '{print $2}' ++ suffix_port=27017 ++ [[ -z 27017 ]] +++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.BFvj9MelPE ++++ mktemp +++ local LAST_ERR=/tmp/tmp.XTJISRlKGa +++ local exit_status=0 +++ local timeout=4 ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 -a -n 1 ']' +++ break +++ cat /tmp/tmp.BFvj9MelPE +++ cat /tmp/tmp.XTJISRlKGa +++ rm /tmp/tmp.BFvj9MelPE /tmp/tmp.XTJISRlKGa +++ return 0 ++ local client_container=psmdb-client-bb8b97679-l4k9z ++ kubectl_bin exec psmdb-client-bb8b97679-l4k9z -- bash -c 'printf '\''rs.conf().members.length\n'\'' | mongo mongodb://clusterAdmin:clusterAdmin123456@34.118.233.158:27017/admin ' +++ mktemp ++ local LAST_OUT=/tmp/tmp.eEBKVtF3ta +++ mktemp ++ local LAST_ERR=/tmp/tmp.Doi0iakzSD ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl exec psmdb-client-bb8b97679-l4k9z -- bash -c 'printf '\''rs.conf().members.length\n'\'' | mongo mongodb://clusterAdmin:clusterAdmin123456@34.118.233.158:27017/admin ' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.eEBKVtF3ta ++ cat /tmp/tmp.Doi0iakzSD ++ rm /tmp/tmp.eEBKVtF3ta /tmp/tmp.Doi0iakzSD ++ return 0 + nodes_count=5 + echo -n 'waiting for all members to be configured in rs0' waiting for all members to be configured in rs0+ let retry+=1 + '[' 15 -ge 15 ']' + echo 'Max retry count 15 reached. something went wrong with mongo cluster. Config for endpoint 34.118.233.158 has 5 but expected 6.' Max retry count 15 reached. something went wrong with mongo cluster. Config for endpoint 34.118.233.158 has 5 but expected 6. + exit 1