============================= test session starts ============================== platform linux -- Python 3.14.5, pytest-9.1.1, pluggy-1.6.0 -- /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/.venv/bin/python cachedir: .pytest_cache metadata: {'Python': '3.14.5', 'Platform': 'Linux-7.0.8-100.fc43.x86_64-x86_64-with-glibc2.42', 'Packages': {'pytest': '9.1.1', 'pluggy': '1.6.0'}, 'Plugins': {'html': '4.2.0', 'metadata': '3.1.1', 'dependency': '0.6.1', 'json-report': '1.5.0'}, 'CI': 'true', 'BUILD_NUMBER': '22', 'BUILD_ID': '22', 'BUILD_URL': 'https://cloud.cd.percona.com/job/cloud-psmdb-operator/job/PR-2058/22/', 'NODE_NAME': 'hcloud-3hmxjv56rzynxord', 'JOB_NAME': 'cloud-psmdb-operator/PR-2058', 'BUILD_TAG': 'jenkins-cloud-psmdb-operator-PR-2058-22', 'EXECUTOR_NUMBER': '0', 'JENKINS_URL': 'https://cloud.cd.percona.com/', 'WORKSPACE': '/mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058', 'GIT_COMMIT': '78121c56762758785b75e9b7c017e15fc887ae5d', 'GIT_URL': 'https://github.com/percona/percona-server-mongodb-operator.git', 'GIT_BRANCH': 'PR-2058'} rootdir: /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058 configfile: pyproject.toml plugins: html-4.2.0, metadata-3.1.1, dependency-0.6.1, json-report-1.5.0 collecting ... collected 1 item e2e-tests/test_pytest_wrapper.py::test_bash_wrapper[cross-site-sharded] [00:01:13.664634] WARNING kubectl error: Warning: version difference between client (1.36) and server (1.33) exceeds the supported minor version skew of +/-1 [00:01:15.729286] INFO Environment variables: KUBE_VERSION=1.33 EKS=0 GKE=1 OPENSHIFT= MINIKUBE= API=psmdb.percona.com/v1 REGISTRY_NAME=docker.io GIT_COMMIT=78121c56762758785b75e9b7c017e15fc887ae5d GIT_BRANCH=PR-2058 OPERATOR_VERSION=1.23.0 IMAGE=docker.io/perconalab/percona-server-mongodb-ope rator:PR-2058-78121c567 IMAGE_MONGOD=docker.io/perconalab/percona-server-mong odb-operator:main-mongod8.0 IMAGE_MONGOD_CHAIN=docker.io/perconalab/percona-serve r-mongodb-operator:main-mongod6.0 docker.io/perconalab/percona-server-mongodb-operator: main-mongod7.0 docker.io/perconalab/percona-server-mongodb-operator: main-mongod8.0 IMAGE_SEARCH=perconalab/percona-server-mongodb-operat or:main-mongot IMAGE_BACKUP=docker.io/perconalab/percona-server-mong odb-operator:main-backup IMAGE_PMM_CLIENT=docker.io/percona/pmm-client:2.44.1- 1 IMAGE_PMM_SERVER=docker.io/perconalab/pmm-server:dev- latest IMAGE_PMM3_CLIENT=docker.io/perconalab/pmm-client:3-d ev-latest IMAGE_PMM3_SERVER=docker.io/perconalab/pmm-server:3-d ev-latest IMAGE_LOGCOLLECTOR=docker.io/perconalab/fluentbit:mai n-logcollector IMAGE_CLUSTERSYNC=docker.io/percona/percona-clustersy nc-mongodb:0.9.0 CERT_MANAGER_VER=1.19.3 CHAOS_MESH_VER=2.7.1 MINIO_VER=5.4.0 PMM_SERVER_VER=9.9.9 CLEAN_NAMESPACE=1 DELETE_CRD_ON_START=1 SKIP_DELETE=0 SKIP_BACKUPS_TO_AWS_GCP_AZURE= UPDATE_COMPARE_FILES=0 Warning: version difference between client (1.36) and server (1.33) exceeds the supported minor version skew of +/-1 Debug logging disabled. ++ get_mongod_ver_from_image docker.io/perconalab/percona-server-mongodb-operator:main-mongod8.0 ++ local image=docker.io/perconalab/percona-server-mongodb-operator:main-mongod8.0 +++ run_simple_cli_inside_image docker.io/perconalab/percona-server-mongodb-operator:main-mongod8.0 'mongod --version' +++ local image=docker.io/perconalab/percona-server-mongodb-operator:main-mongod8.0 +++ local 'cli=mongod --version' +++ local pod_name=27211 +++ run_args=() +++ local run_args +++ arch_scheduling_enabled +++ /usr/sbin/sed -r 's/^.*db version v(([0-9]+\.){2}[0-9]+-[0-9]+).*$/\1/g' +++ [[ -n '' ]] +++ kubectl_bin -n default run 27211 --image=docker.io/perconalab/percona-server-mongodb-operator:main-mongod8.0 --restart=Never --command -- sleep infinity ++++ mktemp +++ local LAST_OUT=/tmp/tmp.OtpTaTMr8J ++++ mktemp +++ local LAST_ERR=/tmp/tmp.GxQur0NykM +++ local exit_status=0 +++ local timeout=4 +++ args=('-n' 'default' 'run' '27211' '--image=docker.io/perconalab/percona-server-mongodb-operator:main-mongod8.0' '--restart=Never' '--command' '--' 'sleep' 'infinity') +++ local args +++ transformed_files=() +++ local transformed_files +++ arch_scheduling_enabled +++ [[ -n '' ]] +++ arch_scheduling_enabled +++ [[ -n '' ]] ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl -n default run 27211 --image=docker.io/perconalab/percona-server-mongodb-operator:main-mongod8.0 --restart=Never --command -- sleep infinity +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 -a -n 1 ']' +++ break +++ cat /tmp/tmp.OtpTaTMr8J +++ cat /tmp/tmp.GxQur0NykM +++ rm /tmp/tmp.OtpTaTMr8J /tmp/tmp.GxQur0NykM +++ return 0 +++ kubectl_bin -n default wait --for=condition=Ready pod/27211 ++++ mktemp +++ local LAST_OUT=/tmp/tmp.WWV4nOvfif ++++ mktemp +++ local LAST_ERR=/tmp/tmp.h00j7nbTX9 +++ local exit_status=0 +++ local timeout=4 +++ args=('-n' 'default' 'wait' '--for=condition=Ready' 'pod/27211') +++ local args +++ transformed_files=() +++ local transformed_files +++ arch_scheduling_enabled +++ [[ -n '' ]] +++ arch_scheduling_enabled +++ [[ -n '' ]] ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl -n default wait --for=condition=Ready pod/27211 +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 -a -n 1 ']' +++ break +++ cat /tmp/tmp.WWV4nOvfif +++ cat /tmp/tmp.h00j7nbTX9 +++ rm /tmp/tmp.WWV4nOvfif /tmp/tmp.h00j7nbTX9 +++ return 0 ++++ kubectl_bin -n default exec 27211 -- bash -c 'mongod --version 2>&1' +++++ mktemp ++++ local LAST_OUT=/tmp/tmp.nydy1Llyv9 +++++ mktemp ++++ local LAST_ERR=/tmp/tmp.8Ru0E3jtQa ++++ local exit_status=0 ++++ local timeout=4 ++++ args=('-n' 'default' 'exec' '27211' '--' 'bash' '-c' 'mongod --version 2>&1') ++++ local args ++++ transformed_files=() ++++ local transformed_files ++++ arch_scheduling_enabled ++++ [[ -n '' ]] ++++ arch_scheduling_enabled ++++ [[ -n '' ]] +++++ seq 0 2 ++++ for i in $(seq 0 2) ++++ set +e ++++ kubectl -n default exec 27211 -- bash -c 'mongod --version 2>&1' ++++ exit_status=0 ++++ set -e ++++ '[' 0 '!=' 0 -a -n 1 ']' ++++ break ++++ cat /tmp/tmp.nydy1Llyv9 ++++ cat /tmp/tmp.8Ru0E3jtQa ++++ rm /tmp/tmp.nydy1Llyv9 /tmp/tmp.8Ru0E3jtQa ++++ return 0 +++ local $'output=db version v8.0.26-11\nBuild Info: {\n "version": "8.0.26-11",\n "gitVersion": "70b8c97ecabb33a90d0906d1612639dcf5e5dd85",\n "openSSLVersion": "OpenSSL 3.5.5 27 Jan 2026",\n "modules": [],\n "perconaFeatures": [\n "MemoryEngine",\n "HotBackup",\n "BackupCursorAggregationStage",\n "BackupCursorExtendAggregationStage",\n "AWSIAM",\n "Kerberos",\n "LDAP",\n "OIDC",\n "TDE",\n "FIPSMode",\n "FCBIS",\n "Auditing",\n "ProfilingRateLimit",\n "LogRedaction",\n "ngram"\n ],\n "allocator": "tcmalloc-google",\n "environment": {\n "distarch": "x86_64",\n "target_arch": "x86_64"\n }\n}' +++ kubectl_bin -n default delete pod/27211 --grace-period=0 --force ++++ mktemp +++ local LAST_OUT=/tmp/tmp.KskTwAul18 ++++ mktemp +++ local LAST_ERR=/tmp/tmp.ChOFArFqOa +++ local exit_status=0 +++ local timeout=4 +++ args=('-n' 'default' 'delete' 'pod/27211' '--grace-period=0' '--force') +++ local args +++ transformed_files=() +++ local transformed_files +++ arch_scheduling_enabled +++ [[ -n '' ]] +++ arch_scheduling_enabled +++ [[ -n '' ]] ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl -n default delete pod/27211 --grace-period=0 --force +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 -a -n 1 ']' +++ break +++ cat /tmp/tmp.KskTwAul18 +++ cat /tmp/tmp.ChOFArFqOa Warning: Immediate deletion does not wait for confirmation that the running resource has been terminated. The resource may continue to run on the cluster indefinitely. +++ rm /tmp/tmp.KskTwAul18 /tmp/tmp.ChOFArFqOa +++ return 0 +++ echo db version v8.0.26-11 Build Info: '{' '"version":' '"8.0.26-11",' '"gitVersion":' '"70b8c97ecabb33a90d0906d1612639dcf5e5dd85",' '"openSSLVersion":' '"OpenSSL' 3.5.5 27 Jan '2026",' '"modules":' '[],' '"perconaFeatures":' '[' '"MemoryEngine",' '"HotBackup",' '"BackupCursorAggregationStage",' '"BackupCursorExtendAggregationStage",' '"AWSIAM",' '"Kerberos",' '"LDAP",' '"OIDC",' '"TDE",' '"FIPSMode",' '"FCBIS",' '"Auditing",' '"ProfilingRateLimit",' '"LogRedaction",' '"ngram"' '],' '"allocator":' '"tcmalloc-google",' '"environment":' '{' '"distarch":' '"x86_64",' '"target_arch":' '"x86_64"' '}' '}' ++ version_info=8.0.26-11 ++ [[ ! 8.0.26-11 =~ ^([0-9]+\.){2}[0-9]+-[0-9]+$ ]] ++ echo 8.0.26-11 + FULL_VER=8.0.26-11 + MONGO_VER=8.0 + unset OPERATOR_NS + main_cluster=cross-site-sharded-main + replica_cluster=cross-site-sharded-replica + desc 'create main cluster' + set +o xtrace ----------------------------------------------------------------------------------- create main cluster ----------------------------------------------------------------------------------- + create_infra cross-site-sharded-12730 + local ns=cross-site-sharded-12730 + echo cross-site-sharded-12730 + [[ 1 == 1 ]] + delete_crd + desc 'get and delete old CRDs and RBAC' + set +o xtrace ----------------------------------------------------------------------------------- get and delete old CRDs and RBAC ----------------------------------------------------------------------------------- + kubectl_bin delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/deploy/crd.yaml --ignore-not-found --wait=false ++ mktemp + local LAST_OUT=/tmp/tmp.sCKL99lNOr ++ mktemp + local LAST_ERR=/tmp/tmp.5ec5Nr2f9i + local exit_status=0 + local timeout=4 + args=('delete' '-f' '/mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/deploy/crd.yaml' '--ignore-not-found' '--wait=false') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/deploy/crd.yaml --ignore-not-found --wait=false + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.sCKL99lNOr + cat /tmp/tmp.5ec5Nr2f9i + rm /tmp/tmp.sCKL99lNOr /tmp/tmp.5ec5Nr2f9i + return 0 ++ yq eval .metadata.name /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/deploy/crd.yaml ++ grep -v '\-\-\-' grep: warning: stray \ before - grep: warning: stray \ before - + for crd_name in $(yq eval '.metadata.name' "${src_dir}/deploy/crd.yaml" | grep -v '\-\-\-') + kubectl get perconaservermongodbbackups.psmdb.percona.com --all-namespaces -o wide + grep -v NAMESPACE + xargs -L 1 sh -xc 'kubectl patch perconaservermongodbbackups.psmdb.percona.com -n $0 $1 --type=merge -p "{\"metadata\":{\"finalizers\":[]}}"' error: the server doesn't have a resource type "perconaservermongodbbackups" + kubectl patch perconaservermongodbbackups.psmdb.percona.com -n sh --type=merge -p '{"metadata":{"finalizers":[]}}' error: the server doesn't have a resource type "perconaservermongodbbackups" + : + kubectl_bin wait --for=delete crd perconaservermongodbbackups.psmdb.percona.com ++ mktemp + local LAST_OUT=/tmp/tmp.xayGilgt6j ++ mktemp + local LAST_ERR=/tmp/tmp.90BUWoMqOP + local exit_status=0 + local timeout=4 + args=('wait' '--for=delete' 'crd' 'perconaservermongodbbackups.psmdb.percona.com') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl wait --for=delete crd perconaservermongodbbackups.psmdb.percona.com + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.xayGilgt6j + cat /tmp/tmp.90BUWoMqOP + rm /tmp/tmp.xayGilgt6j /tmp/tmp.90BUWoMqOP + return 0 + for crd_name in $(yq eval '.metadata.name' "${src_dir}/deploy/crd.yaml" | grep -v '\-\-\-') + kubectl get perconaservermongodbclustersyncs.psmdb.percona.com --all-namespaces -o wide + xargs -L 1 sh -xc 'kubectl patch perconaservermongodbclustersyncs.psmdb.percona.com -n $0 $1 --type=merge -p "{\"metadata\":{\"finalizers\":[]}}"' + grep -v NAMESPACE error: the server doesn't have a resource type "perconaservermongodbclustersyncs" + kubectl patch perconaservermongodbclustersyncs.psmdb.percona.com -n sh --type=merge -p '{"metadata":{"finalizers":[]}}' error: the server doesn't have a resource type "perconaservermongodbclustersyncs" + : + kubectl_bin wait --for=delete crd perconaservermongodbclustersyncs.psmdb.percona.com ++ mktemp + local LAST_OUT=/tmp/tmp.JyTTMNWlVj ++ mktemp + local LAST_ERR=/tmp/tmp.dx9UwHDMtG + local exit_status=0 + local timeout=4 + args=('wait' '--for=delete' 'crd' 'perconaservermongodbclustersyncs.psmdb.percona.com') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl wait --for=delete crd perconaservermongodbclustersyncs.psmdb.percona.com + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.JyTTMNWlVj + cat /tmp/tmp.dx9UwHDMtG + rm /tmp/tmp.JyTTMNWlVj /tmp/tmp.dx9UwHDMtG + return 0 + for crd_name in $(yq eval '.metadata.name' "${src_dir}/deploy/crd.yaml" | grep -v '\-\-\-') + grep -v NAMESPACE + kubectl get perconaservermongodbrestores.psmdb.percona.com --all-namespaces -o wide + xargs -L 1 sh -xc 'kubectl patch perconaservermongodbrestores.psmdb.percona.com -n $0 $1 --type=merge -p "{\"metadata\":{\"finalizers\":[]}}"' error: the server doesn't have a resource type "perconaservermongodbrestores" + kubectl patch perconaservermongodbrestores.psmdb.percona.com -n sh --type=merge -p '{"metadata":{"finalizers":[]}}' error: the server doesn't have a resource type "perconaservermongodbrestores" + : + kubectl_bin wait --for=delete crd perconaservermongodbrestores.psmdb.percona.com ++ mktemp + local LAST_OUT=/tmp/tmp.Io0Ry0OS1l ++ mktemp + local LAST_ERR=/tmp/tmp.ALUVq5bGIE + local exit_status=0 + local timeout=4 + args=('wait' '--for=delete' 'crd' 'perconaservermongodbrestores.psmdb.percona.com') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl wait --for=delete crd perconaservermongodbrestores.psmdb.percona.com + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.Io0Ry0OS1l + cat /tmp/tmp.ALUVq5bGIE + rm /tmp/tmp.Io0Ry0OS1l /tmp/tmp.ALUVq5bGIE + return 0 + for crd_name in $(yq eval '.metadata.name' "${src_dir}/deploy/crd.yaml" | grep -v '\-\-\-') + kubectl get perconaservermongodbs.psmdb.percona.com --all-namespaces -o wide + grep -v NAMESPACE + xargs -L 1 sh -xc 'kubectl patch perconaservermongodbs.psmdb.percona.com -n $0 $1 --type=merge -p "{\"metadata\":{\"finalizers\":[]}}"' error: the server doesn't have a resource type "perconaservermongodbs" + kubectl patch perconaservermongodbs.psmdb.percona.com -n sh --type=merge -p '{"metadata":{"finalizers":[]}}' error: the server doesn't have a resource type "perconaservermongodbs" + : + kubectl_bin wait --for=delete crd perconaservermongodbs.psmdb.percona.com ++ mktemp + local LAST_OUT=/tmp/tmp.jJUHXOA9cK ++ mktemp + local LAST_ERR=/tmp/tmp.UiyAf8mSHz + local exit_status=0 + local timeout=4 + args=('wait' '--for=delete' 'crd' 'perconaservermongodbs.psmdb.percona.com') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl wait --for=delete crd perconaservermongodbs.psmdb.percona.com + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.jJUHXOA9cK + cat /tmp/tmp.UiyAf8mSHz + rm /tmp/tmp.jJUHXOA9cK /tmp/tmp.UiyAf8mSHz + return 0 + local rbac_yaml=rbac.yaml + '[' -n '' ']' + kubectl_bin delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/deploy/rbac.yaml --ignore-not-found ++ mktemp + local LAST_OUT=/tmp/tmp.1RycS3GASl ++ mktemp + local LAST_ERR=/tmp/tmp.7EuYTFsU5X + local exit_status=0 + local timeout=4 + args=('delete' '-f' '/mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/deploy/rbac.yaml' '--ignore-not-found') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/deploy/rbac.yaml --ignore-not-found + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.1RycS3GASl + cat /tmp/tmp.7EuYTFsU5X + rm /tmp/tmp.1RycS3GASl /tmp/tmp.7EuYTFsU5X + return 0 + check_crd_for_deletion PR-2058-78121c567 + local git_tag=PR-2058-78121c567 ++ curl -s https://raw.githubusercontent.com/percona/percona-server-mongodb-operator/PR-2058-78121c567/deploy/crd.yaml ++ yq eval .metadata.name ++ /usr/sbin/sed s/---//g ++ /usr/sbin/sed ':a;N;$!ba;s/\n/ /g' + for crd_name in $(curl -s https://raw.githubusercontent.com/percona/percona-server-mongodb-operator/${git_tag}/deploy/crd.yaml | yq eval '.metadata.name' | $sed 's/---//g' | $sed ':a;N;$!ba;s/\n/ /g') ++ kubectl_bin get crd/null -o 'jsonpath={.status.conditions[-1].type}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.WUoaUJHOUi +++ mktemp ++ local LAST_ERR=/tmp/tmp.pt8E1a8n1K ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'crd/null' '-o' 'jsonpath={.status.conditions[-1].type}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get crd/null -o 'jsonpath={.status.conditions[-1].type}' ++ exit_status=1 ++ set -e ++ '[' 1 '!=' 0 -a -n 1 ']' ++ cat /tmp/tmp.WUoaUJHOUi ++ cat /tmp/tmp.pt8E1a8n1K Error from server (NotFound): customresourcedefinitions.apiextensions.k8s.io "null" not found ++ sleep 0 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get crd/null -o 'jsonpath={.status.conditions[-1].type}' ++ exit_status=1 ++ set -e ++ '[' 1 '!=' 0 -a -n 1 ']' ++ cat /tmp/tmp.WUoaUJHOUi ++ cat /tmp/tmp.pt8E1a8n1K Error from server (NotFound): customresourcedefinitions.apiextensions.k8s.io "null" not found ++ sleep 4 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get crd/null -o 'jsonpath={.status.conditions[-1].type}' ++ exit_status=1 ++ set -e ++ '[' 1 '!=' 0 -a -n 1 ']' ++ cat /tmp/tmp.WUoaUJHOUi ++ cat /tmp/tmp.pt8E1a8n1K Error from server (NotFound): customresourcedefinitions.apiextensions.k8s.io "null" not found ++ sleep 8 ++ cat /tmp/tmp.WUoaUJHOUi ++ cat /tmp/tmp.pt8E1a8n1K Error from server (NotFound): customresourcedefinitions.apiextensions.k8s.io "null" not found ++ rm /tmp/tmp.WUoaUJHOUi /tmp/tmp.pt8E1a8n1K ++ return 1 + [[ '' == Terminating ]] + '[' -n '' ']' + create_namespace cross-site-sharded-12730 + local namespace=cross-site-sharded-12730 + local skip_clean_namespace= + [[ 1 == 1 ]] + [[ -z '' ]] + destroy_chaos_mesh ++ helm list --all-namespaces --filter chaos-mesh ++ tail -n1 ++ awk '-F ' '{print $2}' ++ sed s/NAMESPACE// + local chaos_mesh_ns= + desc 'destroy chaos-mesh' + set +o xtrace ----------------------------------------------------------------------------------- destroy chaos-mesh ----------------------------------------------------------------------------------- + '[' -n '' ']' ++ kubectl get MutatingWebhookConfiguration ++ awk '{print $1}' ++ grep chaos-mesh + timeout 30 kubectl delete MutatingWebhookConfiguration error: resource(s) were provided, but no name was specified + : ++ kubectl get ValidatingWebhookConfiguration ++ grep chaos-mesh ++ awk '{print $1}' + timeout 30 kubectl delete ValidatingWebhookConfiguration error: resource(s) were provided, but no name was specified + : ++ kubectl get ValidatingWebhookConfiguration ++ grep validate-auth ++ awk '{print $1}' + timeout 30 kubectl delete ValidatingWebhookConfiguration error: resource(s) were provided, but no name was specified + : ++ grep chaos-mesh ++ kubectl api-resources ++ awk '{print $1}' ++ kubectl get crd ++ awk '{print $1}' ++ grep chaos-mesh.org + timeout 30 kubectl delete crd error: resource(s) were provided, but no name was specified + : ++ kubectl get clusterrolebinding ++ grep chaos-mesh ++ awk '{print $1}' + timeout 30 kubectl delete clusterrolebinding error: resource(s) were provided, but no name was specified + : ++ kubectl get clusterrole ++ grep chaos-mesh ++ awk '{print $1}' + timeout 30 kubectl delete clusterrole error: resource(s) were provided, but no name was specified + : + desc 'cleaned up all old namespaces' + set +o xtrace ----------------------------------------------------------------------------------- cleaned up all old namespaces ----------------------------------------------------------------------------------- + kubectl_bin get ns + grep -E -v '^kube-|^default|Terminating|psmdb-operator|openshift|^gke-|^gmp-|^NAME' ++ mktemp + awk '{print$1}' + xargs kubectl delete ns + local LAST_OUT=/tmp/tmp.V5cyuDKBf7 ++ mktemp + [[ -n '' ]] + desc 'cleaned up old namespaces cross-site-sharded-12730' + set +o xtrace ----------------------------------------------------------------------------------- cleaned up old namespaces cross-site-sharded-12730 ----------------------------------------------------------------------------------- + local LAST_ERR=/tmp/tmp.KrtBrUlOFO + local exit_status=0 + local timeout=4 + kubectl_bin delete namespace cross-site-sharded-12730 --ignore-not-found + args=('get' 'ns') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 ++ mktemp + for i in $(seq 0 2) + set +e + kubectl get ns + local LAST_OUT=/tmp/tmp.ezV1d6SkV3 ++ mktemp + local LAST_ERR=/tmp/tmp.WzfulmPD68 + local exit_status=0 + local timeout=4 + args=('delete' 'namespace' 'cross-site-sharded-12730' '--ignore-not-found') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl delete namespace cross-site-sharded-12730 --ignore-not-found + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.V5cyuDKBf7 + cat /tmp/tmp.KrtBrUlOFO + rm /tmp/tmp.V5cyuDKBf7 /tmp/tmp.KrtBrUlOFO + return 0 error: resource(s) were provided, but no name was specified + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.ezV1d6SkV3 + cat /tmp/tmp.WzfulmPD68 + rm /tmp/tmp.ezV1d6SkV3 /tmp/tmp.WzfulmPD68 + return 0 + kubectl_bin wait --for=delete namespace cross-site-sharded-12730 ++ mktemp + local LAST_OUT=/tmp/tmp.01ONBcapog ++ mktemp + local LAST_ERR=/tmp/tmp.SetpVfEMVP + local exit_status=0 + local timeout=4 + args=('wait' '--for=delete' 'namespace' 'cross-site-sharded-12730') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl wait --for=delete namespace cross-site-sharded-12730 + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.01ONBcapog + cat /tmp/tmp.SetpVfEMVP + rm /tmp/tmp.01ONBcapog /tmp/tmp.SetpVfEMVP + return 0 + desc 'create namespace cross-site-sharded-12730' + set +o xtrace ----------------------------------------------------------------------------------- create namespace cross-site-sharded-12730 ----------------------------------------------------------------------------------- + kubectl_bin create namespace cross-site-sharded-12730 ++ mktemp + local LAST_OUT=/tmp/tmp.q1to1TCgz1 ++ mktemp + local LAST_ERR=/tmp/tmp.hpucB2n0GE + local exit_status=0 + local timeout=4 + args=('create' 'namespace' 'cross-site-sharded-12730') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl create namespace cross-site-sharded-12730 + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.q1to1TCgz1 namespace/cross-site-sharded-12730 created + cat /tmp/tmp.hpucB2n0GE + rm /tmp/tmp.q1to1TCgz1 /tmp/tmp.hpucB2n0GE + return 0 + set_kube_ctx cross-site-sharded-12730 + local namespace=cross-site-sharded-12730 ++ kubectl_bin config current-context +++ mktemp ++ local LAST_OUT=/tmp/tmp.2PUUxy2wo5 +++ mktemp ++ local LAST_ERR=/tmp/tmp.PmCQcLR5wb ++ local exit_status=0 ++ local timeout=4 ++ args=('config' 'current-context') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl config current-context ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.2PUUxy2wo5 ++ cat /tmp/tmp.PmCQcLR5wb ++ rm /tmp/tmp.2PUUxy2wo5 /tmp/tmp.PmCQcLR5wb ++ return 0 + kubectl_bin config set-context gke_cloud-dev-112233_us-central1_jen-psmdb-2058-78121c567-22-cluster5 --namespace=cross-site-sharded-12730 ++ mktemp + local LAST_OUT=/tmp/tmp.HSHLl31Rws ++ mktemp + local LAST_ERR=/tmp/tmp.39vaoUfvcg + local exit_status=0 + local timeout=4 + args=('config' 'set-context' 'gke_cloud-dev-112233_us-central1_jen-psmdb-2058-78121c567-22-cluster5' '--namespace=cross-site-sharded-12730') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl config set-context gke_cloud-dev-112233_us-central1_jen-psmdb-2058-78121c567-22-cluster5 --namespace=cross-site-sharded-12730 + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.HSHLl31Rws Context "gke_cloud-dev-112233_us-central1_jen-psmdb-2058-78121c567-22-cluster5" modified. + cat /tmp/tmp.39vaoUfvcg + rm /tmp/tmp.HSHLl31Rws /tmp/tmp.39vaoUfvcg + return 0 + deploy_operator + desc 'start PSMDB operator: docker.io/perconalab/percona-server-mongodb-operator:PR-2058-78121c567' + set +o xtrace ----------------------------------------------------------------------------------- start PSMDB operator: docker.io/perconalab/percona-server-mongodb-operator:PR-2058-78121c567 ----------------------------------------------------------------------------------- + local cr_file + '[' -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/e2e-tests/cross-site-sharded/conf/crd.yaml ']' + cr_file=/mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/deploy/crd.yaml + kubectl_bin apply --server-side --force-conflicts -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/deploy/crd.yaml ++ mktemp + local LAST_OUT=/tmp/tmp.z5LfiZyRBq ++ mktemp + local LAST_ERR=/tmp/tmp.1KSf2SL2od + local exit_status=0 + local timeout=4 + args=('apply' '--server-side' '--force-conflicts' '-f' '/mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/deploy/crd.yaml') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl apply --server-side --force-conflicts -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/deploy/crd.yaml + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.z5LfiZyRBq customresourcedefinition.apiextensions.k8s.io/perconaservermongodbbackups.psmdb.percona.com serverside-applied customresourcedefinition.apiextensions.k8s.io/perconaservermongodbclustersyncs.psmdb.percona.com serverside-applied customresourcedefinition.apiextensions.k8s.io/perconaservermongodbrestores.psmdb.percona.com serverside-applied customresourcedefinition.apiextensions.k8s.io/perconaservermongodbs.psmdb.percona.com serverside-applied + cat /tmp/tmp.1KSf2SL2od + rm /tmp/tmp.z5LfiZyRBq /tmp/tmp.1KSf2SL2od + return 0 + '[' -n '' ']' + apply_rbac rbac + local operator_namespace=psmdb-operator + local rbac=rbac + cat /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/deploy/rbac.yaml + sed -e 's^namespace: .*^namespace: psmdb-operator^' + kubectl_bin apply -f - ++ mktemp + local LAST_OUT=/tmp/tmp.u1xY3cSPtX ++ mktemp + local LAST_ERR=/tmp/tmp.OxkeY4ILzd + local exit_status=0 + local timeout=4 + args=('apply' '-f' '-') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.u1xY3cSPtX role.rbac.authorization.k8s.io/percona-server-mongodb-operator created serviceaccount/percona-server-mongodb-operator created rolebinding.rbac.authorization.k8s.io/service-account-percona-server-mongodb-operator created + cat /tmp/tmp.OxkeY4ILzd + rm /tmp/tmp.u1xY3cSPtX /tmp/tmp.OxkeY4ILzd + return 0 + yq eval $'\n\t\t\t(.spec.template.spec.containers[].image = "docker.io/perconalab/percona-server-mongodb-operator:PR-2058-78121c567") |\n\t\t\t((.. | select(.[] == "DISABLE_TELEMETRY")) |= .value="true") |\n\t\t\t((.. | select(.[] == "LOG_LEVEL")) |= .value="DEBUG")' /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/deploy/operator.yaml + kubectl_bin apply -f - + add_arch_to_podspecs - + arch_scheduling_enabled + [[ -n '' ]] + cat - ++ mktemp + local LAST_OUT=/tmp/tmp.JxM9gXPD81 ++ mktemp + local LAST_ERR=/tmp/tmp.I5kD9zWetD + local exit_status=0 + local timeout=4 + args=('apply' '-f' '-') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl apply -f - + return + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.JxM9gXPD81 deployment.apps/percona-server-mongodb-operator created + cat /tmp/tmp.I5kD9zWetD + rm /tmp/tmp.JxM9gXPD81 /tmp/tmp.I5kD9zWetD + return 0 + sleep 20 ++ get_operator_pod ++ kubectl_bin get pods --selector=name=percona-server-mongodb-operator -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.6Ngw2eC24i +++ mktemp ++ local LAST_ERR=/tmp/tmp.W55Dxx2z9r ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'pods' '--selector=name=percona-server-mongodb-operator' '-o' 'jsonpath={.items[].metadata.name}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pods --selector=name=percona-server-mongodb-operator -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.6Ngw2eC24i ++ cat /tmp/tmp.W55Dxx2z9r ++ rm /tmp/tmp.6Ngw2eC24i /tmp/tmp.W55Dxx2z9r ++ return 0 + wait_operator_pod percona-server-mongodb-operator-6695587bc6-fg4wm + local pod=percona-server-mongodb-operator-6695587bc6-fg4wm + set +o xtrace waiting for pod/percona-server-mongodb-operator-6695587bc6-fg4wm to be ready.OK + echo 'Print operator info from log' Print operator info from log + grep 'Manager starting up' ++ get_operator_pod ++ kubectl_bin get pods --selector=name=percona-server-mongodb-operator -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.CK6YbENUOk +++ mktemp ++ local LAST_ERR=/tmp/tmp.wC421We1i0 ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'pods' '--selector=name=percona-server-mongodb-operator' '-o' 'jsonpath={.items[].metadata.name}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pods --selector=name=percona-server-mongodb-operator -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.CK6YbENUOk ++ cat /tmp/tmp.wC421We1i0 ++ rm /tmp/tmp.CK6YbENUOk /tmp/tmp.wC421We1i0 ++ return 0 + kubectl_bin logs percona-server-mongodb-operator-6695587bc6-fg4wm ++ mktemp + local LAST_OUT=/tmp/tmp.aO3T6Jc3xM ++ mktemp + local LAST_ERR=/tmp/tmp.4NN6Ei6TNv + local exit_status=0 + local timeout=4 + args=('logs' 'percona-server-mongodb-operator-6695587bc6-fg4wm') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl logs percona-server-mongodb-operator-6695587bc6-fg4wm + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.aO3T6Jc3xM + cat /tmp/tmp.4NN6Ei6TNv + rm /tmp/tmp.aO3T6Jc3xM /tmp/tmp.4NN6Ei6TNv + return 0 2026-07-17T00:02:38.537Z INFO setup Manager starting up {"gitCommit": "78121c56762758785b75e9b7c017e15fc887ae5d", "gitBranch": "PR-2058-78121c567", "buildTime": "", "goVersion": "go1.26.5", "os": "linux", "arch": "amd64"} + desc 'create secrets and start client' + set +o xtrace ----------------------------------------------------------------------------------- create secrets and start client ----------------------------------------------------------------------------------- + kubectl_bin apply -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/e2e-tests/conf/client.yml -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/e2e-tests/cross-site-sharded/conf/secrets.yml ++ mktemp + local LAST_OUT=/tmp/tmp.vDyNAGHZWF ++ mktemp + local LAST_ERR=/tmp/tmp.e1X05WbrfX + local exit_status=0 + local timeout=4 + args=('apply' '-f' '/mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/e2e-tests/conf/client.yml' '-f' '/mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/e2e-tests/cross-site-sharded/conf/secrets.yml') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl apply -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/e2e-tests/conf/client.yml -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/e2e-tests/cross-site-sharded/conf/secrets.yml + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.vDyNAGHZWF deployment.apps/psmdb-client created secret/cross-site-sharded-main-secrets created secret/cross-site-sharded-main-ssl created secret/cross-site-sharded-main-ssl-internal created + cat /tmp/tmp.e1X05WbrfX + rm /tmp/tmp.vDyNAGHZWF /tmp/tmp.e1X05WbrfX + return 0 + desc 'create main PSMDB cluster cross-site-sharded-main.' + set +o xtrace ----------------------------------------------------------------------------------- create main PSMDB cluster cross-site-sharded-main. ----------------------------------------------------------------------------------- + apply_cluster /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/e2e-tests/cross-site-sharded/conf/cross-site-sharded-main.yml + '[' -z '' ']' + cat_config /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/e2e-tests/cross-site-sharded/conf/cross-site-sharded-main.yml + cat /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/e2e-tests/cross-site-sharded/conf/cross-site-sharded-main.yml + yq eval '(.spec | select(.image == null)).image = "docker.io/perconalab/percona-server-mongodb-operator:main-mongod8.0"' + yq eval '(.spec | select(has("pmm"))).pmm.image = "docker.io/percona/pmm-client:2.44.1-1"' + kubectl_bin apply -f - + yq eval '(.spec | select(has("backup"))).backup.image = "docker.io/perconalab/percona-server-mongodb-operator:main-backup"' ++ mktemp + yq eval '(.spec | select(has("initImage"))).initImage = "docker.io/perconalab/percona-server-mongodb-operator:PR-2058-78121c567"' + yq eval '(.spec | select(has("search"))).search.image = "perconalab/percona-server-mongodb-operator:main-mongot"' + local LAST_OUT=/tmp/tmp.s8O5pvqszp + yq eval '.spec.upgradeOptions.apply="Never"' + /usr/sbin/sed -e s/NAME_SPACE/cross-site-sharded-12730/g + add_arch_to_psmdb - + [[ -n docker.io/perconalab/fluentbit:main-logcollector ]] + arch_scheduling_enabled + [[ -n '' ]] + cat - + IMAGE_LOGCOLLECTOR=docker.io/perconalab/fluentbit:main-logcollector + yq eval '(.spec | select(has("logcollector"))).logcollector.image = strenv(IMAGE_LOGCOLLECTOR)' ++ mktemp + local LAST_ERR=/tmp/tmp.Ta62y2FZhE + local exit_status=0 + local timeout=4 + args=('apply' '-f' '-') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl apply -f - + return + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.s8O5pvqszp perconaservermongodb.psmdb.percona.com/cross-site-sharded-main created + cat /tmp/tmp.Ta62y2FZhE + rm /tmp/tmp.s8O5pvqszp /tmp/tmp.Ta62y2FZhE + return 0 + desc 'check if all 3 Pods started' + set +o xtrace ----------------------------------------------------------------------------------- check if all 3 Pods started ----------------------------------------------------------------------------------- + wait_for_running cross-site-sharded-main-rs0 3 + local name=cross-site-sharded-main-rs0 + let last_pod=2 + local check_cluster_readiness=true + set_debug + [[ 1 == 1 ]] + set -o xtrace + local rs_name=rs0 + local cluster_name=cross-site-sharded-main ++ seq 0 2 + for i in $(seq 0 $last_pod) + [[ 0 -eq 2 ]] + wait_pod cross-site-sharded-main-rs0-0 + local pod=cross-site-sharded-main-rs0-0 + set +o xtrace waiting for pod/cross-site-sharded-main-rs0-0 to be ready.......................OK + for i in $(seq 0 $last_pod) + [[ 1 -eq 2 ]] + wait_pod cross-site-sharded-main-rs0-1 + local pod=cross-site-sharded-main-rs0-1 + set +o xtrace waiting for pod/cross-site-sharded-main-rs0-1 to be ready.....................OK + for i in $(seq 0 $last_pod) + [[ 2 -eq 2 ]] ++ kubectl_bin get psmdb cross-site-sharded-main -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].arbiter.enabled}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.W5OlqeD24D +++ mktemp ++ local LAST_ERR=/tmp/tmp.LiUPme7s1k ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'psmdb' 'cross-site-sharded-main' '-o' 'jsonpath={.spec.replsets[?(@.name=="rs0")].arbiter.enabled}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb cross-site-sharded-main -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].arbiter.enabled}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.W5OlqeD24D ++ cat /tmp/tmp.LiUPme7s1k ++ rm /tmp/tmp.W5OlqeD24D /tmp/tmp.LiUPme7s1k ++ return 0 + [[ false == true ]] + wait_pod cross-site-sharded-main-rs0-2 + local pod=cross-site-sharded-main-rs0-2 + set +o xtrace waiting for pod/cross-site-sharded-main-rs0-2 to be ready.............OK ++ kubectl_bin get psmdb cross-site-sharded-main -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].nonvoting.enabled}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.4GvlVbAwOP +++ mktemp ++ local LAST_ERR=/tmp/tmp.Xjl47rDvXD ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'psmdb' 'cross-site-sharded-main' '-o' 'jsonpath={.spec.replsets[?(@.name=="rs0")].nonvoting.enabled}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb cross-site-sharded-main -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].nonvoting.enabled}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.4GvlVbAwOP ++ cat /tmp/tmp.Xjl47rDvXD ++ rm /tmp/tmp.4GvlVbAwOP /tmp/tmp.Xjl47rDvXD ++ return 0 + [[ false == true ]] ++ kubectl_bin get psmdb cross-site-sharded-main -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].hidden.enabled}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.XQ7NizGJf3 +++ mktemp ++ local LAST_ERR=/tmp/tmp.Ghe4kbaJ20 ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'psmdb' 'cross-site-sharded-main' '-o' 'jsonpath={.spec.replsets[?(@.name=="rs0")].hidden.enabled}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb cross-site-sharded-main -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].hidden.enabled}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.XQ7NizGJf3 ++ cat /tmp/tmp.Ghe4kbaJ20 ++ rm /tmp/tmp.XQ7NizGJf3 /tmp/tmp.Ghe4kbaJ20 ++ return 0 + [[ '' == true ]] + sleep 10 + [[ true == true ]] + set +x Waiting for cluster cross-site-sharded-main to be ready......................................... + wait_for_running cross-site-sharded-main-cfg 3 false + local name=cross-site-sharded-main-cfg + let last_pod=2 + local check_cluster_readiness=false + set_debug + [[ 1 == 1 ]] + set -o xtrace + local rs_name=cfg + local cluster_name=cross-site-sharded-main ++ seq 0 2 + for i in $(seq 0 $last_pod) + [[ 0 -eq 2 ]] + wait_pod cross-site-sharded-main-cfg-0 + local pod=cross-site-sharded-main-cfg-0 + set +o xtrace waiting for pod/cross-site-sharded-main-cfg-0 to be ready.OK + for i in $(seq 0 $last_pod) + [[ 1 -eq 2 ]] + wait_pod cross-site-sharded-main-cfg-1 + local pod=cross-site-sharded-main-cfg-1 + set +o xtrace waiting for pod/cross-site-sharded-main-cfg-1 to be ready.OK + for i in $(seq 0 $last_pod) + [[ 2 -eq 2 ]] ++ kubectl_bin get psmdb cross-site-sharded-main -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].arbiter.enabled}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.9Vrq0E8QZz +++ mktemp ++ local LAST_ERR=/tmp/tmp.93GABwxgMF ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'psmdb' 'cross-site-sharded-main' '-o' 'jsonpath={.spec.replsets[?(@.name=="cfg")].arbiter.enabled}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb cross-site-sharded-main -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].arbiter.enabled}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.9Vrq0E8QZz ++ cat /tmp/tmp.93GABwxgMF ++ rm /tmp/tmp.9Vrq0E8QZz /tmp/tmp.93GABwxgMF ++ return 0 + [[ '' == true ]] + wait_pod cross-site-sharded-main-cfg-2 + local pod=cross-site-sharded-main-cfg-2 + set +o xtrace waiting for pod/cross-site-sharded-main-cfg-2 to be ready.OK ++ kubectl_bin get psmdb cross-site-sharded-main -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].nonvoting.enabled}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.KhDhPaDMRn +++ mktemp ++ local LAST_ERR=/tmp/tmp.JjGEylsrJZ ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'psmdb' 'cross-site-sharded-main' '-o' 'jsonpath={.spec.replsets[?(@.name=="cfg")].nonvoting.enabled}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb cross-site-sharded-main -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].nonvoting.enabled}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.KhDhPaDMRn ++ cat /tmp/tmp.JjGEylsrJZ ++ rm /tmp/tmp.KhDhPaDMRn /tmp/tmp.JjGEylsrJZ ++ return 0 + [[ '' == true ]] ++ kubectl_bin get psmdb cross-site-sharded-main -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].hidden.enabled}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.udQ8gEyHPv +++ mktemp ++ local LAST_ERR=/tmp/tmp.Z4Ss1f9vMP ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'psmdb' 'cross-site-sharded-main' '-o' 'jsonpath={.spec.replsets[?(@.name=="cfg")].hidden.enabled}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb cross-site-sharded-main -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].hidden.enabled}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.udQ8gEyHPv ++ cat /tmp/tmp.Z4Ss1f9vMP ++ rm /tmp/tmp.udQ8gEyHPv /tmp/tmp.Z4Ss1f9vMP ++ return 0 + [[ '' == true ]] + sleep 10 + [[ false == true ]] + desc 'create user' + set +o xtrace ----------------------------------------------------------------------------------- create user ----------------------------------------------------------------------------------- + run_mongos 'db.createUser({user:"user",pwd:"pass",roles:[{db:"app",role:"readWrite"}]})' userAdmin:userAdmin123456@cross-site-sharded-main-mongos.cross-site-sharded-12730 + local 'command=db.createUser({user:"user",pwd:"pass",roles:[{db:"app",role:"readWrite"}]})' + local uri=userAdmin:userAdmin123456@cross-site-sharded-main-mongos.cross-site-sharded-12730 + local driver=mongodb + local suffix=.svc.cluster.local + local mongo_flag= + local port=27017 + local mongo_bin=mongo ++ echo .svc.cluster.local ++ awk -F: '{print $2}' + suffix_port= + [[ -z '' ]] + suffix=.svc.cluster.local:27017 ++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.TIkbMVJwog +++ mktemp ++ local LAST_ERR=/tmp/tmp.YQ7d7YCASD ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'pods' '--selector=name=psmdb-client' '-o' 'jsonpath={.items[].metadata.name}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.TIkbMVJwog ++ cat /tmp/tmp.YQ7d7YCASD ++ rm /tmp/tmp.TIkbMVJwog /tmp/tmp.YQ7d7YCASD ++ return 0 + local client_container=psmdb-client-757cd65695-xjsxw + kubectl_bin exec psmdb-client-757cd65695-xjsxw -- bash -c 'export HOME=/tmp; printf '\''db.createUser({user:"user",pwd:"pass",roles:[{db:"app",role:"readWrite"}]})\n'\'' | mongo mongodb://userAdmin:userAdmin123456@cross-site-sharded-main-mongos.cross-site-sharded-12730.svc.cluster.local:27017/admin ' ++ mktemp + local LAST_OUT=/tmp/tmp.pQADoqunv6 ++ mktemp + local LAST_ERR=/tmp/tmp.bfRH1JrhZ3 + local exit_status=0 + local timeout=4 + args=('exec' 'psmdb-client-757cd65695-xjsxw' '--' 'bash' '-c' 'export HOME=/tmp; printf '\''db.createUser({user:"user",pwd:"pass",roles:[{db:"app",role:"readWrite"}]})\n'\'' | mongo mongodb://userAdmin:userAdmin123456@cross-site-sharded-main-mongos.cross-site-sharded-12730.svc.cluster.local:27017/admin ') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl exec psmdb-client-757cd65695-xjsxw -- bash -c 'export HOME=/tmp; printf '\''db.createUser({user:"user",pwd:"pass",roles:[{db:"app",role:"readWrite"}]})\n'\'' | mongo mongodb://userAdmin:userAdmin123456@cross-site-sharded-main-mongos.cross-site-sharded-12730.svc.cluster.local:27017/admin ' + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.pQADoqunv6 Percona Server for MongoDB shell version v4.4.29-28 connecting to: mongodb://cross-site-sharded-main-mongos.cross-site-sharded-12730.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb Implicit session: session { "id" : UUID("fecca1c0-701a-42f7-8e76-1cf11723107d") } Percona Server for MongoDB server version: v8.0.26-11 WARNING: shell and server versions do not match Successfully added user: { "user" : "user", "roles" : [ { "db" : "app", "role" : "readWrite" } ] } bye + cat /tmp/tmp.bfRH1JrhZ3 + rm /tmp/tmp.pQADoqunv6 /tmp/tmp.bfRH1JrhZ3 + return 0 + sleep 2 + desc 'set chunk size to 2 MB' + set +o xtrace ----------------------------------------------------------------------------------- set chunk size to 2 MB ----------------------------------------------------------------------------------- + run_mongos 'use config\n db.settings.save( { _id:"chunksize", value: 2 } )' clusterAdmin:clusterAdmin123456@cross-site-sharded-main-mongos.cross-site-sharded-12730 + local 'command=use config\n db.settings.save( { _id:"chunksize", value: 2 } )' + local uri=clusterAdmin:clusterAdmin123456@cross-site-sharded-main-mongos.cross-site-sharded-12730 + local driver=mongodb + local suffix=.svc.cluster.local + local mongo_flag= + local port=27017 + local mongo_bin=mongo ++ echo .svc.cluster.local ++ awk -F: '{print $2}' + suffix_port= + [[ -z '' ]] + suffix=.svc.cluster.local:27017 ++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.dam06UK7kU +++ mktemp ++ local LAST_ERR=/tmp/tmp.omFevNKFhG ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'pods' '--selector=name=psmdb-client' '-o' 'jsonpath={.items[].metadata.name}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.dam06UK7kU ++ cat /tmp/tmp.omFevNKFhG ++ rm /tmp/tmp.dam06UK7kU /tmp/tmp.omFevNKFhG ++ return 0 + local client_container=psmdb-client-757cd65695-xjsxw + kubectl_bin exec psmdb-client-757cd65695-xjsxw -- bash -c 'export HOME=/tmp; printf '\''use config\n db.settings.save( { _id:"chunksize", value: 2 } )\n'\'' | mongo mongodb://clusterAdmin:clusterAdmin123456@cross-site-sharded-main-mongos.cross-site-sharded-12730.svc.cluster.local:27017/admin ' ++ mktemp + local LAST_OUT=/tmp/tmp.BZDx2Z2NZG ++ mktemp + local LAST_ERR=/tmp/tmp.i9hjmq4ZqS + local exit_status=0 + local timeout=4 + args=('exec' 'psmdb-client-757cd65695-xjsxw' '--' 'bash' '-c' 'export HOME=/tmp; printf '\''use config\n db.settings.save( { _id:"chunksize", value: 2 } )\n'\'' | mongo mongodb://clusterAdmin:clusterAdmin123456@cross-site-sharded-main-mongos.cross-site-sharded-12730.svc.cluster.local:27017/admin ') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl exec psmdb-client-757cd65695-xjsxw -- bash -c 'export HOME=/tmp; printf '\''use config\n db.settings.save( { _id:"chunksize", value: 2 } )\n'\'' | mongo mongodb://clusterAdmin:clusterAdmin123456@cross-site-sharded-main-mongos.cross-site-sharded-12730.svc.cluster.local:27017/admin ' + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.BZDx2Z2NZG Percona Server for MongoDB shell version v4.4.29-28 connecting to: mongodb://cross-site-sharded-main-mongos.cross-site-sharded-12730.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb Implicit session: session { "id" : UUID("b07a4207-9925-41d0-a168-331f4b9b1bce") } Percona Server for MongoDB server version: v8.0.26-11 WARNING: shell and server versions do not match switched to db config WriteResult({ "nMatched" : 0, "nUpserted" : 1, "nModified" : 0, "_id" : "chunksize" }) bye + cat /tmp/tmp.i9hjmq4ZqS + rm /tmp/tmp.BZDx2Z2NZG /tmp/tmp.i9hjmq4ZqS + return 0 + sleep 2 + desc 'write data' + set +o xtrace ----------------------------------------------------------------------------------- write data ----------------------------------------------------------------------------------- + run_script_mongos /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/e2e-tests/cross-site-sharded/data.js user:pass@cross-site-sharded-main-mongos.cross-site-sharded-12730 + local script=/mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/e2e-tests/cross-site-sharded/data.js + local uri=user:pass@cross-site-sharded-main-mongos.cross-site-sharded-12730 + local driver=mongodb + local suffix=.svc.cluster.local + local mongo_flag= + local mongo_bin=mongo ++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.xcHTBxwslD +++ mktemp ++ local LAST_ERR=/tmp/tmp.hb4X9SinPS ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'pods' '--selector=name=psmdb-client' '-o' 'jsonpath={.items[].metadata.name}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.xcHTBxwslD ++ cat /tmp/tmp.hb4X9SinPS ++ rm /tmp/tmp.xcHTBxwslD /tmp/tmp.hb4X9SinPS ++ return 0 + local client_container=psmdb-client-757cd65695-xjsxw ++ basename /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/e2e-tests/cross-site-sharded/data.js + name=data.js + kubectl_bin cp /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/e2e-tests/cross-site-sharded/data.js cross-site-sharded-12730/psmdb-client-757cd65695-xjsxw:/tmp ++ mktemp + local LAST_OUT=/tmp/tmp.IYPYxnSIUW ++ mktemp + local LAST_ERR=/tmp/tmp.qSoQ12h6Yh + local exit_status=0 + local timeout=4 + args=('cp' '/mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/e2e-tests/cross-site-sharded/data.js' 'cross-site-sharded-12730/psmdb-client-757cd65695-xjsxw:/tmp') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl cp /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/e2e-tests/cross-site-sharded/data.js cross-site-sharded-12730/psmdb-client-757cd65695-xjsxw:/tmp + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.IYPYxnSIUW + cat /tmp/tmp.qSoQ12h6Yh + rm /tmp/tmp.IYPYxnSIUW /tmp/tmp.qSoQ12h6Yh + return 0 + kubectl_bin exec psmdb-client-757cd65695-xjsxw -- bash -c 'export HOME=/tmp; mongo mongodb://user:pass@cross-site-sharded-main-mongos.cross-site-sharded-12730.svc.cluster.local/admin /tmp/data.js' ++ mktemp + local LAST_OUT=/tmp/tmp.ZWQDV0mCoe ++ mktemp + local LAST_ERR=/tmp/tmp.9ccStnAINd + local exit_status=0 + local timeout=4 + args=('exec' 'psmdb-client-757cd65695-xjsxw' '--' 'bash' '-c' 'export HOME=/tmp; mongo mongodb://user:pass@cross-site-sharded-main-mongos.cross-site-sharded-12730.svc.cluster.local/admin /tmp/data.js') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl exec psmdb-client-757cd65695-xjsxw -- bash -c 'export HOME=/tmp; mongo mongodb://user:pass@cross-site-sharded-main-mongos.cross-site-sharded-12730.svc.cluster.local/admin /tmp/data.js' + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.ZWQDV0mCoe Percona Server for MongoDB shell version v4.4.29-28 connecting to: mongodb://cross-site-sharded-main-mongos.cross-site-sharded-12730.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb Implicit session: session { "id" : UUID("e5cec524-a6a1-43a8-87c2-c20a1c1edaba") } Percona Server for MongoDB server version: v8.0.26-11 WARNING: shell and server versions do not match + cat /tmp/tmp.9ccStnAINd + rm /tmp/tmp.ZWQDV0mCoe /tmp/tmp.9ccStnAINd + return 0 + desc 'shard collection' + set +o xtrace ----------------------------------------------------------------------------------- shard collection ----------------------------------------------------------------------------------- + run_mongos 'sh.enableSharding("app")' clusterAdmin:clusterAdmin123456@cross-site-sharded-main-mongos.cross-site-sharded-12730 + local 'command=sh.enableSharding("app")' + local uri=clusterAdmin:clusterAdmin123456@cross-site-sharded-main-mongos.cross-site-sharded-12730 + local driver=mongodb + local suffix=.svc.cluster.local + local mongo_flag= + local port=27017 + local mongo_bin=mongo ++ echo .svc.cluster.local ++ awk -F: '{print $2}' + suffix_port= + [[ -z '' ]] + suffix=.svc.cluster.local:27017 ++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.2YAyzOkTG3 +++ mktemp ++ local LAST_ERR=/tmp/tmp.6QGyfJ7sPA ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'pods' '--selector=name=psmdb-client' '-o' 'jsonpath={.items[].metadata.name}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.2YAyzOkTG3 ++ cat /tmp/tmp.6QGyfJ7sPA ++ rm /tmp/tmp.2YAyzOkTG3 /tmp/tmp.6QGyfJ7sPA ++ return 0 + local client_container=psmdb-client-757cd65695-xjsxw + kubectl_bin exec psmdb-client-757cd65695-xjsxw -- bash -c 'export HOME=/tmp; printf '\''sh.enableSharding("app")\n'\'' | mongo mongodb://clusterAdmin:clusterAdmin123456@cross-site-sharded-main-mongos.cross-site-sharded-12730.svc.cluster.local:27017/admin ' ++ mktemp + local LAST_OUT=/tmp/tmp.gX6L5OPL7P ++ mktemp + local LAST_ERR=/tmp/tmp.JckxJMMNxr + local exit_status=0 + local timeout=4 + args=('exec' 'psmdb-client-757cd65695-xjsxw' '--' 'bash' '-c' 'export HOME=/tmp; printf '\''sh.enableSharding("app")\n'\'' | mongo mongodb://clusterAdmin:clusterAdmin123456@cross-site-sharded-main-mongos.cross-site-sharded-12730.svc.cluster.local:27017/admin ') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl exec psmdb-client-757cd65695-xjsxw -- bash -c 'export HOME=/tmp; printf '\''sh.enableSharding("app")\n'\'' | mongo mongodb://clusterAdmin:clusterAdmin123456@cross-site-sharded-main-mongos.cross-site-sharded-12730.svc.cluster.local:27017/admin ' + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.gX6L5OPL7P Percona Server for MongoDB shell version v4.4.29-28 connecting to: mongodb://cross-site-sharded-main-mongos.cross-site-sharded-12730.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb Implicit session: session { "id" : UUID("b415b875-5a9a-4a32-af2e-87aa5aaa999e") } Percona Server for MongoDB server version: v8.0.26-11 WARNING: shell and server versions do not match { "ok" : 1, "$clusterTime" : { "clusterTime" : Timestamp(1784246857, 1), "signature" : { "hash" : BinData(0,"vVwYVePO7c17gqaUJsylmujQvKM="), "keyId" : NumberLong("7663281108531806223") } }, "operationTime" : Timestamp(1784246857, 1) } bye + cat /tmp/tmp.JckxJMMNxr + rm /tmp/tmp.gX6L5OPL7P /tmp/tmp.JckxJMMNxr + return 0 + sleep 2 + run_mongos 'sh.shardCollection("app.city", { _id: 1 } )' clusterAdmin:clusterAdmin123456@cross-site-sharded-main-mongos.cross-site-sharded-12730 + local 'command=sh.shardCollection("app.city", { _id: 1 } )' + local uri=clusterAdmin:clusterAdmin123456@cross-site-sharded-main-mongos.cross-site-sharded-12730 + local driver=mongodb + local suffix=.svc.cluster.local + local mongo_flag= + local port=27017 + local mongo_bin=mongo ++ echo .svc.cluster.local ++ awk -F: '{print $2}' + suffix_port= + [[ -z '' ]] + suffix=.svc.cluster.local:27017 ++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.LsCgVqasaD +++ mktemp ++ local LAST_ERR=/tmp/tmp.FmeLTJJHls ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'pods' '--selector=name=psmdb-client' '-o' 'jsonpath={.items[].metadata.name}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.LsCgVqasaD ++ cat /tmp/tmp.FmeLTJJHls ++ rm /tmp/tmp.LsCgVqasaD /tmp/tmp.FmeLTJJHls ++ return 0 + local client_container=psmdb-client-757cd65695-xjsxw + kubectl_bin exec psmdb-client-757cd65695-xjsxw -- bash -c 'export HOME=/tmp; printf '\''sh.shardCollection("app.city", { _id: 1 } )\n'\'' | mongo mongodb://clusterAdmin:clusterAdmin123456@cross-site-sharded-main-mongos.cross-site-sharded-12730.svc.cluster.local:27017/admin ' ++ mktemp + local LAST_OUT=/tmp/tmp.bfw9ZLlrs8 ++ mktemp + local LAST_ERR=/tmp/tmp.RwjOEXavSB + local exit_status=0 + local timeout=4 + args=('exec' 'psmdb-client-757cd65695-xjsxw' '--' 'bash' '-c' 'export HOME=/tmp; printf '\''sh.shardCollection("app.city", { _id: 1 } )\n'\'' | mongo mongodb://clusterAdmin:clusterAdmin123456@cross-site-sharded-main-mongos.cross-site-sharded-12730.svc.cluster.local:27017/admin ') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl exec psmdb-client-757cd65695-xjsxw -- bash -c 'export HOME=/tmp; printf '\''sh.shardCollection("app.city", { _id: 1 } )\n'\'' | mongo mongodb://clusterAdmin:clusterAdmin123456@cross-site-sharded-main-mongos.cross-site-sharded-12730.svc.cluster.local:27017/admin ' + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.bfw9ZLlrs8 Percona Server for MongoDB shell version v4.4.29-28 connecting to: mongodb://cross-site-sharded-main-mongos.cross-site-sharded-12730.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb Implicit session: session { "id" : UUID("1a762c74-6dfb-4ff8-bb50-52a5f67c10d9") } Percona Server for MongoDB server version: v8.0.26-11 WARNING: shell and server versions do not match { "collectionsharded" : "app.city", "ok" : 1, "$clusterTime" : { "clusterTime" : Timestamp(1784246863, 18), "signature" : { "hash" : BinData(0,"xZCnSzCBUFbKbkuLuIlsOFPuncY="), "keyId" : NumberLong("7663281108531806223") } }, "operationTime" : Timestamp(1784246863, 17) } bye + cat /tmp/tmp.RwjOEXavSB + rm /tmp/tmp.bfw9ZLlrs8 /tmp/tmp.RwjOEXavSB + return 0 + sleep 120 + desc 'Check chunks' + set +o xtrace ----------------------------------------------------------------------------------- Check chunks ----------------------------------------------------------------------------------- + chunks_param1=ns + chunks_param2='"app.city"' + [[ 8.0 != 4\.4 ]] + chunks_param1=uuid ++ run_mongos 'use app\n db.getCollectionInfos({ "name": "city" })[0].info.uuid' user:pass@cross-site-sharded-main-mongos.cross-site-sharded-12730 ++ local 'command=use app\n db.getCollectionInfos({ "name": "city" })[0].info.uuid' ++ local uri=user:pass@cross-site-sharded-main-mongos.cross-site-sharded-12730 ++ local driver=mongodb ++ local suffix=.svc.cluster.local ++ local mongo_flag= ++ local port=27017 ++ local mongo_bin=mongo ++ grep 'switched to db app' -A 1 ++ grep -v 'switched to db app' +++ echo .svc.cluster.local +++ awk -F: '{print $2}' ++ suffix_port= ++ [[ -z '' ]] ++ suffix=.svc.cluster.local:27017 +++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.l90LU5cl1h ++++ mktemp +++ local LAST_ERR=/tmp/tmp.7QNgLApxLI +++ local exit_status=0 +++ local timeout=4 +++ args=('get' 'pods' '--selector=name=psmdb-client' '-o' 'jsonpath={.items[].metadata.name}') +++ local args +++ transformed_files=() +++ local transformed_files +++ arch_scheduling_enabled +++ [[ -n '' ]] +++ arch_scheduling_enabled +++ [[ -n '' ]] ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 -a -n 1 ']' +++ break +++ cat /tmp/tmp.l90LU5cl1h +++ cat /tmp/tmp.7QNgLApxLI +++ rm /tmp/tmp.l90LU5cl1h /tmp/tmp.7QNgLApxLI +++ return 0 ++ local client_container=psmdb-client-757cd65695-xjsxw ++ kubectl_bin exec psmdb-client-757cd65695-xjsxw -- bash -c 'export HOME=/tmp; printf '\''use app\n db.getCollectionInfos({ "name": "city" })[0].info.uuid\n'\'' | mongo mongodb://user:pass@cross-site-sharded-main-mongos.cross-site-sharded-12730.svc.cluster.local:27017/admin ' +++ mktemp ++ local LAST_OUT=/tmp/tmp.fThTBHhLO0 +++ mktemp ++ local LAST_ERR=/tmp/tmp.1GQzecITAd ++ local exit_status=0 ++ local timeout=4 ++ args=('exec' 'psmdb-client-757cd65695-xjsxw' '--' 'bash' '-c' 'export HOME=/tmp; printf '\''use app\n db.getCollectionInfos({ "name": "city" })[0].info.uuid\n'\'' | mongo mongodb://user:pass@cross-site-sharded-main-mongos.cross-site-sharded-12730.svc.cluster.local:27017/admin ') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl exec psmdb-client-757cd65695-xjsxw -- bash -c 'export HOME=/tmp; printf '\''use app\n db.getCollectionInfos({ "name": "city" })[0].info.uuid\n'\'' | mongo mongodb://user:pass@cross-site-sharded-main-mongos.cross-site-sharded-12730.svc.cluster.local:27017/admin ' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.fThTBHhLO0 ++ cat /tmp/tmp.1GQzecITAd ++ rm /tmp/tmp.fThTBHhLO0 /tmp/tmp.1GQzecITAd ++ return 0 + chunks_param2='UUID("6308c842-58e6-40ff-a9d9-07e496add940")' + shards=0 + for i in "rs0" "rs1" ++ run_mongos 'use config\n db.chunks.count({"uuid": UUID("6308c842-58e6-40ff-a9d9-07e496add940"), "shard": "rs0"})' clusterAdmin:clusterAdmin123456@cross-site-sharded-main-mongos.cross-site-sharded-12730 ++ local 'command=use config\n db.chunks.count({"uuid": UUID("6308c842-58e6-40ff-a9d9-07e496add940"), "shard": "rs0"})' ++ local uri=clusterAdmin:clusterAdmin123456@cross-site-sharded-main-mongos.cross-site-sharded-12730 ++ local driver=mongodb ++ local suffix=.svc.cluster.local ++ local mongo_flag= ++ local port=27017 ++ local mongo_bin=mongo ++ grep 'switched to db config' -A 1 ++ grep -v 'switched to db config' +++ echo .svc.cluster.local +++ awk -F: '{print $2}' ++ suffix_port= ++ [[ -z '' ]] ++ suffix=.svc.cluster.local:27017 +++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.bQFz0LRSDq ++++ mktemp +++ local LAST_ERR=/tmp/tmp.wpq4Kwnwov +++ local exit_status=0 +++ local timeout=4 +++ args=('get' 'pods' '--selector=name=psmdb-client' '-o' 'jsonpath={.items[].metadata.name}') +++ local args +++ transformed_files=() +++ local transformed_files +++ arch_scheduling_enabled +++ [[ -n '' ]] +++ arch_scheduling_enabled +++ [[ -n '' ]] ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 -a -n 1 ']' +++ break +++ cat /tmp/tmp.bQFz0LRSDq +++ cat /tmp/tmp.wpq4Kwnwov +++ rm /tmp/tmp.bQFz0LRSDq /tmp/tmp.wpq4Kwnwov +++ return 0 ++ local client_container=psmdb-client-757cd65695-xjsxw ++ kubectl_bin exec psmdb-client-757cd65695-xjsxw -- bash -c 'export HOME=/tmp; printf '\''use config\n db.chunks.count({"uuid": UUID("6308c842-58e6-40ff-a9d9-07e496add940"), "shard": "rs0"})\n'\'' | mongo mongodb://clusterAdmin:clusterAdmin123456@cross-site-sharded-main-mongos.cross-site-sharded-12730.svc.cluster.local:27017/admin ' +++ mktemp ++ local LAST_OUT=/tmp/tmp.k93V03FNW8 +++ mktemp ++ local LAST_ERR=/tmp/tmp.m7ye8gCtc6 ++ local exit_status=0 ++ local timeout=4 ++ args=('exec' 'psmdb-client-757cd65695-xjsxw' '--' 'bash' '-c' 'export HOME=/tmp; printf '\''use config\n db.chunks.count({"uuid": UUID("6308c842-58e6-40ff-a9d9-07e496add940"), "shard": "rs0"})\n'\'' | mongo mongodb://clusterAdmin:clusterAdmin123456@cross-site-sharded-main-mongos.cross-site-sharded-12730.svc.cluster.local:27017/admin ') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl exec psmdb-client-757cd65695-xjsxw -- bash -c 'export HOME=/tmp; printf '\''use config\n db.chunks.count({"uuid": UUID("6308c842-58e6-40ff-a9d9-07e496add940"), "shard": "rs0"})\n'\'' | mongo mongodb://clusterAdmin:clusterAdmin123456@cross-site-sharded-main-mongos.cross-site-sharded-12730.svc.cluster.local:27017/admin ' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.k93V03FNW8 ++ cat /tmp/tmp.m7ye8gCtc6 ++ rm /tmp/tmp.k93V03FNW8 /tmp/tmp.m7ye8gCtc6 ++ return 0 + out=3 + desc 'rs0 has 3 chunks' + set +o xtrace ----------------------------------------------------------------------------------- rs0 has 3 chunks ----------------------------------------------------------------------------------- + [[ 3 -ne 0 ]] + (( shards = shards + 1 )) + for i in "rs0" "rs1" ++ run_mongos 'use config\n db.chunks.count({"uuid": UUID("6308c842-58e6-40ff-a9d9-07e496add940"), "shard": "rs1"})' clusterAdmin:clusterAdmin123456@cross-site-sharded-main-mongos.cross-site-sharded-12730 ++ local 'command=use config\n db.chunks.count({"uuid": UUID("6308c842-58e6-40ff-a9d9-07e496add940"), "shard": "rs1"})' ++ local uri=clusterAdmin:clusterAdmin123456@cross-site-sharded-main-mongos.cross-site-sharded-12730 ++ local driver=mongodb ++ local suffix=.svc.cluster.local ++ local mongo_flag= ++ local port=27017 ++ local mongo_bin=mongo ++ grep 'switched to db config' -A 1 ++ grep -v 'switched to db config' +++ echo .svc.cluster.local +++ awk -F: '{print $2}' ++ suffix_port= ++ [[ -z '' ]] ++ suffix=.svc.cluster.local:27017 +++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.nXjtYOBJa0 ++++ mktemp +++ local LAST_ERR=/tmp/tmp.lWuoQYZ6Jf +++ local exit_status=0 +++ local timeout=4 +++ args=('get' 'pods' '--selector=name=psmdb-client' '-o' 'jsonpath={.items[].metadata.name}') +++ local args +++ transformed_files=() +++ local transformed_files +++ arch_scheduling_enabled +++ [[ -n '' ]] +++ arch_scheduling_enabled +++ [[ -n '' ]] ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 -a -n 1 ']' +++ break +++ cat /tmp/tmp.nXjtYOBJa0 +++ cat /tmp/tmp.lWuoQYZ6Jf +++ rm /tmp/tmp.nXjtYOBJa0 /tmp/tmp.lWuoQYZ6Jf +++ return 0 ++ local client_container=psmdb-client-757cd65695-xjsxw ++ kubectl_bin exec psmdb-client-757cd65695-xjsxw -- bash -c 'export HOME=/tmp; printf '\''use config\n db.chunks.count({"uuid": UUID("6308c842-58e6-40ff-a9d9-07e496add940"), "shard": "rs1"})\n'\'' | mongo mongodb://clusterAdmin:clusterAdmin123456@cross-site-sharded-main-mongos.cross-site-sharded-12730.svc.cluster.local:27017/admin ' +++ mktemp ++ local LAST_OUT=/tmp/tmp.xWA7MJghpi +++ mktemp ++ local LAST_ERR=/tmp/tmp.6EwjIuA3cz ++ local exit_status=0 ++ local timeout=4 ++ args=('exec' 'psmdb-client-757cd65695-xjsxw' '--' 'bash' '-c' 'export HOME=/tmp; printf '\''use config\n db.chunks.count({"uuid": UUID("6308c842-58e6-40ff-a9d9-07e496add940"), "shard": "rs1"})\n'\'' | mongo mongodb://clusterAdmin:clusterAdmin123456@cross-site-sharded-main-mongos.cross-site-sharded-12730.svc.cluster.local:27017/admin ') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl exec psmdb-client-757cd65695-xjsxw -- bash -c 'export HOME=/tmp; printf '\''use config\n db.chunks.count({"uuid": UUID("6308c842-58e6-40ff-a9d9-07e496add940"), "shard": "rs1"})\n'\'' | mongo mongodb://clusterAdmin:clusterAdmin123456@cross-site-sharded-main-mongos.cross-site-sharded-12730.svc.cluster.local:27017/admin ' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.xWA7MJghpi ++ cat /tmp/tmp.6EwjIuA3cz ++ rm /tmp/tmp.xWA7MJghpi /tmp/tmp.6EwjIuA3cz ++ return 0 + out=1 + desc 'rs1 has 1 chunks' + set +o xtrace ----------------------------------------------------------------------------------- rs1 has 1 chunks ----------------------------------------------------------------------------------- + [[ 1 -ne 0 ]] + (( shards = shards + 1 )) + [[ 2 -lt 2 ]] + desc 'create replica cluster' + set +o xtrace ----------------------------------------------------------------------------------- create replica cluster ----------------------------------------------------------------------------------- + create_namespace cross-site-sharded-replica-16001 0 + local namespace=cross-site-sharded-replica-16001 + local skip_clean_namespace=0 + [[ 1 == 1 ]] + [[ -z 0 ]] + [[ -n '' ]] + desc 'cleaned up old namespaces cross-site-sharded-replica-16001' + set +o xtrace ----------------------------------------------------------------------------------- cleaned up old namespaces cross-site-sharded-replica-16001 ----------------------------------------------------------------------------------- + kubectl_bin delete namespace cross-site-sharded-replica-16001 --ignore-not-found ++ mktemp + local LAST_OUT=/tmp/tmp.RqBIh5O9CV ++ mktemp + local LAST_ERR=/tmp/tmp.YQGWj6ewpN + local exit_status=0 + local timeout=4 + args=('delete' 'namespace' 'cross-site-sharded-replica-16001' '--ignore-not-found') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl delete namespace cross-site-sharded-replica-16001 --ignore-not-found + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.RqBIh5O9CV + cat /tmp/tmp.YQGWj6ewpN + rm /tmp/tmp.RqBIh5O9CV /tmp/tmp.YQGWj6ewpN + return 0 + kubectl_bin wait --for=delete namespace cross-site-sharded-replica-16001 ++ mktemp + local LAST_OUT=/tmp/tmp.R2H3GbT1mC ++ mktemp + local LAST_ERR=/tmp/tmp.rsse2jlRKA + local exit_status=0 + local timeout=4 + args=('wait' '--for=delete' 'namespace' 'cross-site-sharded-replica-16001') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl wait --for=delete namespace cross-site-sharded-replica-16001 + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.R2H3GbT1mC + cat /tmp/tmp.rsse2jlRKA + rm /tmp/tmp.R2H3GbT1mC /tmp/tmp.rsse2jlRKA + return 0 + desc 'create namespace cross-site-sharded-replica-16001' + set +o xtrace ----------------------------------------------------------------------------------- create namespace cross-site-sharded-replica-16001 ----------------------------------------------------------------------------------- + kubectl_bin create namespace cross-site-sharded-replica-16001 ++ mktemp + local LAST_OUT=/tmp/tmp.R1VYiqIkc5 ++ mktemp + local LAST_ERR=/tmp/tmp.CAPwf4CuET + local exit_status=0 + local timeout=4 + args=('create' 'namespace' 'cross-site-sharded-replica-16001') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl create namespace cross-site-sharded-replica-16001 + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.R1VYiqIkc5 namespace/cross-site-sharded-replica-16001 created + cat /tmp/tmp.CAPwf4CuET + rm /tmp/tmp.R1VYiqIkc5 /tmp/tmp.CAPwf4CuET + return 0 + set_kube_ctx cross-site-sharded-replica-16001 + local namespace=cross-site-sharded-replica-16001 ++ kubectl_bin config current-context +++ mktemp ++ local LAST_OUT=/tmp/tmp.wDn18AufN1 +++ mktemp ++ local LAST_ERR=/tmp/tmp.Id2bWaQgKv ++ local exit_status=0 ++ local timeout=4 ++ args=('config' 'current-context') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl config current-context ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.wDn18AufN1 ++ cat /tmp/tmp.Id2bWaQgKv ++ rm /tmp/tmp.wDn18AufN1 /tmp/tmp.Id2bWaQgKv ++ return 0 + kubectl_bin config set-context gke_cloud-dev-112233_us-central1_jen-psmdb-2058-78121c567-22-cluster5 --namespace=cross-site-sharded-replica-16001 ++ mktemp + local LAST_OUT=/tmp/tmp.vdhltBWYpm ++ mktemp + local LAST_ERR=/tmp/tmp.G2w8fjbDmj + local exit_status=0 + local timeout=4 + args=('config' 'set-context' 'gke_cloud-dev-112233_us-central1_jen-psmdb-2058-78121c567-22-cluster5' '--namespace=cross-site-sharded-replica-16001') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl config set-context gke_cloud-dev-112233_us-central1_jen-psmdb-2058-78121c567-22-cluster5 --namespace=cross-site-sharded-replica-16001 + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.vdhltBWYpm Context "gke_cloud-dev-112233_us-central1_jen-psmdb-2058-78121c567-22-cluster5" modified. + cat /tmp/tmp.G2w8fjbDmj + rm /tmp/tmp.vdhltBWYpm /tmp/tmp.G2w8fjbDmj + return 0 + deploy_operator + desc 'start PSMDB operator: docker.io/perconalab/percona-server-mongodb-operator:PR-2058-78121c567' + set +o xtrace ----------------------------------------------------------------------------------- start PSMDB operator: docker.io/perconalab/percona-server-mongodb-operator:PR-2058-78121c567 ----------------------------------------------------------------------------------- + local cr_file + '[' -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/e2e-tests/cross-site-sharded/conf/crd.yaml ']' + cr_file=/mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/deploy/crd.yaml + kubectl_bin apply --server-side --force-conflicts -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/deploy/crd.yaml ++ mktemp + local LAST_OUT=/tmp/tmp.NQXiyQotlx ++ mktemp + local LAST_ERR=/tmp/tmp.QO636E6bEl + local exit_status=0 + local timeout=4 + args=('apply' '--server-side' '--force-conflicts' '-f' '/mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/deploy/crd.yaml') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl apply --server-side --force-conflicts -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/deploy/crd.yaml + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.NQXiyQotlx customresourcedefinition.apiextensions.k8s.io/perconaservermongodbbackups.psmdb.percona.com serverside-applied customresourcedefinition.apiextensions.k8s.io/perconaservermongodbclustersyncs.psmdb.percona.com serverside-applied customresourcedefinition.apiextensions.k8s.io/perconaservermongodbrestores.psmdb.percona.com serverside-applied customresourcedefinition.apiextensions.k8s.io/perconaservermongodbs.psmdb.percona.com serverside-applied + cat /tmp/tmp.QO636E6bEl + rm /tmp/tmp.NQXiyQotlx /tmp/tmp.QO636E6bEl + return 0 + '[' -n '' ']' + apply_rbac rbac + local operator_namespace=psmdb-operator + local rbac=rbac + cat /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/deploy/rbac.yaml + sed -e 's^namespace: .*^namespace: psmdb-operator^' + kubectl_bin apply -f - ++ mktemp + local LAST_OUT=/tmp/tmp.Znl09a6cHs ++ mktemp + local LAST_ERR=/tmp/tmp.BZ3LtFHZSz + local exit_status=0 + local timeout=4 + args=('apply' '-f' '-') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.Znl09a6cHs role.rbac.authorization.k8s.io/percona-server-mongodb-operator created serviceaccount/percona-server-mongodb-operator created rolebinding.rbac.authorization.k8s.io/service-account-percona-server-mongodb-operator created + cat /tmp/tmp.BZ3LtFHZSz + rm /tmp/tmp.Znl09a6cHs /tmp/tmp.BZ3LtFHZSz + return 0 + yq eval $'\n\t\t\t(.spec.template.spec.containers[].image = "docker.io/perconalab/percona-server-mongodb-operator:PR-2058-78121c567") |\n\t\t\t((.. | select(.[] == "DISABLE_TELEMETRY")) |= .value="true") |\n\t\t\t((.. | select(.[] == "LOG_LEVEL")) |= .value="DEBUG")' /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/deploy/operator.yaml + add_arch_to_podspecs - + kubectl_bin apply -f - + arch_scheduling_enabled + [[ -n '' ]] + cat - ++ mktemp + local LAST_OUT=/tmp/tmp.OY1uIjyPro ++ mktemp + local LAST_ERR=/tmp/tmp.Og3og9N0Oj + local exit_status=0 + local timeout=4 + args=('apply' '-f' '-') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl apply -f - + return + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.OY1uIjyPro deployment.apps/percona-server-mongodb-operator created + cat /tmp/tmp.Og3og9N0Oj + rm /tmp/tmp.OY1uIjyPro /tmp/tmp.Og3og9N0Oj + return 0 + sleep 20 ++ get_operator_pod ++ kubectl_bin get pods --selector=name=percona-server-mongodb-operator -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.yxW21Wydpe +++ mktemp ++ local LAST_ERR=/tmp/tmp.GjyDDaJ0QG ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'pods' '--selector=name=percona-server-mongodb-operator' '-o' 'jsonpath={.items[].metadata.name}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pods --selector=name=percona-server-mongodb-operator -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.yxW21Wydpe ++ cat /tmp/tmp.GjyDDaJ0QG ++ rm /tmp/tmp.yxW21Wydpe /tmp/tmp.GjyDDaJ0QG ++ return 0 + wait_operator_pod percona-server-mongodb-operator-6695587bc6-k8bzr + local pod=percona-server-mongodb-operator-6695587bc6-k8bzr + set +o xtrace waiting for pod/percona-server-mongodb-operator-6695587bc6-k8bzr to be ready.OK + echo 'Print operator info from log' Print operator info from log + grep 'Manager starting up' ++ get_operator_pod ++ kubectl_bin get pods --selector=name=percona-server-mongodb-operator -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.muQBQLOiZ0 +++ mktemp ++ local LAST_ERR=/tmp/tmp.pFWa6TJQzC ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'pods' '--selector=name=percona-server-mongodb-operator' '-o' 'jsonpath={.items[].metadata.name}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pods --selector=name=percona-server-mongodb-operator -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.muQBQLOiZ0 ++ cat /tmp/tmp.pFWa6TJQzC ++ rm /tmp/tmp.muQBQLOiZ0 /tmp/tmp.pFWa6TJQzC ++ return 0 + kubectl_bin logs percona-server-mongodb-operator-6695587bc6-k8bzr ++ mktemp + local LAST_OUT=/tmp/tmp.kxQqSRgZJR ++ mktemp + local LAST_ERR=/tmp/tmp.S394huay7o + local exit_status=0 + local timeout=4 + args=('logs' 'percona-server-mongodb-operator-6695587bc6-k8bzr') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl logs percona-server-mongodb-operator-6695587bc6-k8bzr + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.kxQqSRgZJR + cat /tmp/tmp.S394huay7o + rm /tmp/tmp.kxQqSRgZJR /tmp/tmp.S394huay7o + return 0 2026-07-17T00:10:07.702Z INFO setup Manager starting up {"gitCommit": "78121c56762758785b75e9b7c017e15fc887ae5d", "gitBranch": "PR-2058-78121c567", "buildTime": "", "goVersion": "go1.26.5", "os": "linux", "arch": "amd64"} + desc 'start client' + set +o xtrace ----------------------------------------------------------------------------------- start client ----------------------------------------------------------------------------------- + kubectl_bin apply -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/e2e-tests/conf/client.yml ++ mktemp + local LAST_OUT=/tmp/tmp.62MHb1faLC ++ mktemp + local LAST_ERR=/tmp/tmp.P1ULh3pyaU + local exit_status=0 + local timeout=4 + args=('apply' '-f' '/mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/e2e-tests/conf/client.yml') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl apply -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/e2e-tests/conf/client.yml + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.62MHb1faLC deployment.apps/psmdb-client created + cat /tmp/tmp.P1ULh3pyaU + rm /tmp/tmp.62MHb1faLC /tmp/tmp.P1ULh3pyaU + return 0 + desc 'copy secrets from main to replica namespace and create all of them' + set +o xtrace ----------------------------------------------------------------------------------- copy secrets from main to replica namespace and create all of them ----------------------------------------------------------------------------------- + kubectl get secret cross-site-sharded-main-secrets -o yaml -n cross-site-sharded-12730 + yq eval $'\n\t\tdel(.metadata) |\n\t\t(.metadata.name = "cross-site-sharded-replica-secrets")' - + kubectl_bin apply -f - ++ mktemp + local LAST_OUT=/tmp/tmp.9dOKNksbPV ++ mktemp + local LAST_ERR=/tmp/tmp.PRY4pOZvyK + local exit_status=0 + local timeout=4 + args=('apply' '-f' '-') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.9dOKNksbPV secret/cross-site-sharded-replica-secrets created + cat /tmp/tmp.PRY4pOZvyK + rm /tmp/tmp.9dOKNksbPV /tmp/tmp.PRY4pOZvyK + return 0 + kubectl_bin get secret cross-site-sharded-main-ssl-internal -o yaml -n cross-site-sharded-12730 + kubectl_bin apply -f - + yq eval $'\n\t\tdel(.metadata) |\n\t\tdel(.status) |\n\t\t(.metadata.name = "cross-site-sharded-replica-ssl-internal")' - ++ mktemp ++ mktemp + local LAST_OUT=/tmp/tmp.eV5WSUqa4H ++ mktemp + local LAST_ERR=/tmp/tmp.ssoT38gDi9 + local exit_status=0 + local timeout=4 + args=('apply' '-f' '-') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + local LAST_OUT=/tmp/tmp.HVMJZgVvl6 + for i in $(seq 0 2) + set +e + kubectl apply -f - ++ mktemp + local LAST_ERR=/tmp/tmp.5TvxyTbyAa + local exit_status=0 + local timeout=4 + args=('get' 'secret' 'cross-site-sharded-main-ssl-internal' '-o' 'yaml' '-n' 'cross-site-sharded-12730') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl get secret cross-site-sharded-main-ssl-internal -o yaml -n cross-site-sharded-12730 + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.HVMJZgVvl6 + cat /tmp/tmp.5TvxyTbyAa + rm /tmp/tmp.HVMJZgVvl6 /tmp/tmp.5TvxyTbyAa + return 0 + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.eV5WSUqa4H secret/cross-site-sharded-replica-ssl-internal created + cat /tmp/tmp.ssoT38gDi9 + rm /tmp/tmp.eV5WSUqa4H /tmp/tmp.ssoT38gDi9 + return 0 + kubectl_bin get secret cross-site-sharded-main-ssl -o yaml -n cross-site-sharded-12730 + yq eval $'\n\t\tdel(.metadata) |\n\t\tdel(.status) |\n\t\t(.metadata.name = "cross-site-sharded-replica-ssl")' - + kubectl_bin apply -f - ++ mktemp ++ mktemp + local LAST_OUT=/tmp/tmp.CDqGCcwe14 + local LAST_OUT=/tmp/tmp.SpHP8WSQtt ++ mktemp ++ mktemp + local LAST_ERR=/tmp/tmp.OPaMqkqcta + local exit_status=0 + local timeout=4 + args=('apply' '-f' '-') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] + local LAST_ERR=/tmp/tmp.5vTbcQdPP0 + local exit_status=0 + local timeout=4 + args=('get' 'secret' 'cross-site-sharded-main-ssl' '-o' 'yaml' '-n' 'cross-site-sharded-12730') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl apply -f - + for i in $(seq 0 2) + set +e + kubectl get secret cross-site-sharded-main-ssl -o yaml -n cross-site-sharded-12730 + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.SpHP8WSQtt + cat /tmp/tmp.5vTbcQdPP0 + rm /tmp/tmp.SpHP8WSQtt /tmp/tmp.5vTbcQdPP0 + return 0 + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.CDqGCcwe14 secret/cross-site-sharded-replica-ssl created + cat /tmp/tmp.OPaMqkqcta + rm /tmp/tmp.CDqGCcwe14 /tmp/tmp.OPaMqkqcta + return 0 + sleep 30 + desc 'create replica PSMDB cluster ' + set +o xtrace ----------------------------------------------------------------------------------- create replica PSMDB cluster ----------------------------------------------------------------------------------- + apply_cluster /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/e2e-tests/cross-site-sharded/conf/cross-site-sharded-replica.yml + '[' -z '' ']' + cat_config /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/e2e-tests/cross-site-sharded/conf/cross-site-sharded-replica.yml + kubectl_bin apply -f - + yq eval '(.spec | select(.image == null)).image = "docker.io/perconalab/percona-server-mongodb-operator:main-mongod8.0"' ++ mktemp + cat /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/e2e-tests/cross-site-sharded/conf/cross-site-sharded-replica.yml + yq eval '(.spec | select(has("initImage"))).initImage = "docker.io/perconalab/percona-server-mongodb-operator:PR-2058-78121c567"' + yq eval '(.spec | select(has("pmm"))).pmm.image = "docker.io/percona/pmm-client:2.44.1-1"' + yq eval '(.spec | select(has("backup"))).backup.image = "docker.io/perconalab/percona-server-mongodb-operator:main-backup"' + local LAST_OUT=/tmp/tmp.RVHKZvsfzd ++ mktemp + yq eval '(.spec | select(has("search"))).search.image = "perconalab/percona-server-mongodb-operator:main-mongot"' + add_arch_to_psmdb - + [[ -n docker.io/perconalab/fluentbit:main-logcollector ]] + IMAGE_LOGCOLLECTOR=docker.io/perconalab/fluentbit:main-logcollector + yq eval '(.spec | select(has("logcollector"))).logcollector.image = strenv(IMAGE_LOGCOLLECTOR)' + arch_scheduling_enabled + [[ -n '' ]] + cat - + yq eval '.spec.upgradeOptions.apply="Never"' + /usr/sbin/sed -e s/NAME_SPACE/cross-site-sharded-12730/g + local LAST_ERR=/tmp/tmp.DZgEhP75la + local exit_status=0 + local timeout=4 + args=('apply' '-f' '-') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl apply -f - + return + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.RVHKZvsfzd perconaservermongodb.psmdb.percona.com/cross-site-sharded-replica created + cat /tmp/tmp.DZgEhP75la + rm /tmp/tmp.RVHKZvsfzd /tmp/tmp.DZgEhP75la + return 0 + wait_for_running cross-site-sharded-replica-rs0 3 false + local name=cross-site-sharded-replica-rs0 + let last_pod=2 + local check_cluster_readiness=false + set_debug + [[ 1 == 1 ]] + set -o xtrace + local rs_name=rs0 + local cluster_name=cross-site-sharded-replica ++ seq 0 2 + for i in $(seq 0 $last_pod) + [[ 0 -eq 2 ]] + wait_pod cross-site-sharded-replica-rs0-0 + local pod=cross-site-sharded-replica-rs0-0 + set +o xtrace waiting for pod/cross-site-sharded-replica-rs0-0 to be ready.............OK + for i in $(seq 0 $last_pod) + [[ 1 -eq 2 ]] + wait_pod cross-site-sharded-replica-rs0-1 + local pod=cross-site-sharded-replica-rs0-1 + set +o xtrace waiting for pod/cross-site-sharded-replica-rs0-1 to be ready............OK + for i in $(seq 0 $last_pod) + [[ 2 -eq 2 ]] ++ kubectl_bin get psmdb cross-site-sharded-replica -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].arbiter.enabled}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.jy4j2iF85d +++ mktemp ++ local LAST_ERR=/tmp/tmp.OpIGbIiKh5 ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'psmdb' 'cross-site-sharded-replica' '-o' 'jsonpath={.spec.replsets[?(@.name=="rs0")].arbiter.enabled}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb cross-site-sharded-replica -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].arbiter.enabled}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.jy4j2iF85d ++ cat /tmp/tmp.OpIGbIiKh5 ++ rm /tmp/tmp.jy4j2iF85d /tmp/tmp.OpIGbIiKh5 ++ return 0 + [[ false == true ]] + wait_pod cross-site-sharded-replica-rs0-2 + local pod=cross-site-sharded-replica-rs0-2 + set +o xtrace waiting for pod/cross-site-sharded-replica-rs0-2 to be ready.............OK ++ kubectl_bin get psmdb cross-site-sharded-replica -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].nonvoting.enabled}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.oKqWELsxHB +++ mktemp ++ local LAST_ERR=/tmp/tmp.7jqC3PXAmM ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'psmdb' 'cross-site-sharded-replica' '-o' 'jsonpath={.spec.replsets[?(@.name=="rs0")].nonvoting.enabled}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb cross-site-sharded-replica -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].nonvoting.enabled}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.oKqWELsxHB ++ cat /tmp/tmp.7jqC3PXAmM ++ rm /tmp/tmp.oKqWELsxHB /tmp/tmp.7jqC3PXAmM ++ return 0 + [[ false == true ]] ++ kubectl_bin get psmdb cross-site-sharded-replica -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].hidden.enabled}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.5iqEOC6AGS +++ mktemp ++ local LAST_ERR=/tmp/tmp.tvEoKitico ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'psmdb' 'cross-site-sharded-replica' '-o' 'jsonpath={.spec.replsets[?(@.name=="rs0")].hidden.enabled}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb cross-site-sharded-replica -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].hidden.enabled}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.5iqEOC6AGS ++ cat /tmp/tmp.tvEoKitico ++ rm /tmp/tmp.5iqEOC6AGS /tmp/tmp.tvEoKitico ++ return 0 + [[ '' == true ]] + sleep 10 + [[ false == true ]] + wait_for_running cross-site-sharded-replica-rs1 3 false + local name=cross-site-sharded-replica-rs1 + let last_pod=2 + local check_cluster_readiness=false + set_debug + [[ 1 == 1 ]] + set -o xtrace + local rs_name=rs1 + local cluster_name=cross-site-sharded-replica ++ seq 0 2 + for i in $(seq 0 $last_pod) + [[ 0 -eq 2 ]] + wait_pod cross-site-sharded-replica-rs1-0 + local pod=cross-site-sharded-replica-rs1-0 + set +o xtrace waiting for pod/cross-site-sharded-replica-rs1-0 to be ready.OK + for i in $(seq 0 $last_pod) + [[ 1 -eq 2 ]] + wait_pod cross-site-sharded-replica-rs1-1 + local pod=cross-site-sharded-replica-rs1-1 + set +o xtrace waiting for pod/cross-site-sharded-replica-rs1-1 to be ready.OK + for i in $(seq 0 $last_pod) + [[ 2 -eq 2 ]] ++ kubectl_bin get psmdb cross-site-sharded-replica -o 'jsonpath={.spec.replsets[?(@.name=="rs1")].arbiter.enabled}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.Pu2rBRGghM +++ mktemp ++ local LAST_ERR=/tmp/tmp.22VSOjhCj7 ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'psmdb' 'cross-site-sharded-replica' '-o' 'jsonpath={.spec.replsets[?(@.name=="rs1")].arbiter.enabled}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb cross-site-sharded-replica -o 'jsonpath={.spec.replsets[?(@.name=="rs1")].arbiter.enabled}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.Pu2rBRGghM ++ cat /tmp/tmp.22VSOjhCj7 ++ rm /tmp/tmp.Pu2rBRGghM /tmp/tmp.22VSOjhCj7 ++ return 0 + [[ false == true ]] + wait_pod cross-site-sharded-replica-rs1-2 + local pod=cross-site-sharded-replica-rs1-2 + set +o xtrace waiting for pod/cross-site-sharded-replica-rs1-2 to be ready.OK ++ kubectl_bin get psmdb cross-site-sharded-replica -o 'jsonpath={.spec.replsets[?(@.name=="rs1")].nonvoting.enabled}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.l8YBA6XFId +++ mktemp ++ local LAST_ERR=/tmp/tmp.8ws0mxK7VR ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'psmdb' 'cross-site-sharded-replica' '-o' 'jsonpath={.spec.replsets[?(@.name=="rs1")].nonvoting.enabled}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb cross-site-sharded-replica -o 'jsonpath={.spec.replsets[?(@.name=="rs1")].nonvoting.enabled}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.l8YBA6XFId ++ cat /tmp/tmp.8ws0mxK7VR ++ rm /tmp/tmp.l8YBA6XFId /tmp/tmp.8ws0mxK7VR ++ return 0 + [[ false == true ]] ++ kubectl_bin get psmdb cross-site-sharded-replica -o 'jsonpath={.spec.replsets[?(@.name=="rs1")].hidden.enabled}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.CMI6b6Su2U +++ mktemp ++ local LAST_ERR=/tmp/tmp.2h8qCuhrTv ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'psmdb' 'cross-site-sharded-replica' '-o' 'jsonpath={.spec.replsets[?(@.name=="rs1")].hidden.enabled}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb cross-site-sharded-replica -o 'jsonpath={.spec.replsets[?(@.name=="rs1")].hidden.enabled}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.CMI6b6Su2U ++ cat /tmp/tmp.2h8qCuhrTv ++ rm /tmp/tmp.CMI6b6Su2U /tmp/tmp.2h8qCuhrTv ++ return 0 + [[ '' == true ]] + sleep 10 + [[ false == true ]] + wait_for_running cross-site-sharded-replica-cfg 3 false + local name=cross-site-sharded-replica-cfg + let last_pod=2 + local check_cluster_readiness=false + set_debug + [[ 1 == 1 ]] + set -o xtrace + local rs_name=cfg + local cluster_name=cross-site-sharded-replica ++ seq 0 2 + for i in $(seq 0 $last_pod) + [[ 0 -eq 2 ]] + wait_pod cross-site-sharded-replica-cfg-0 + local pod=cross-site-sharded-replica-cfg-0 + set +o xtrace waiting for pod/cross-site-sharded-replica-cfg-0 to be ready.OK + for i in $(seq 0 $last_pod) + [[ 1 -eq 2 ]] + wait_pod cross-site-sharded-replica-cfg-1 + local pod=cross-site-sharded-replica-cfg-1 + set +o xtrace waiting for pod/cross-site-sharded-replica-cfg-1 to be ready.OK + for i in $(seq 0 $last_pod) + [[ 2 -eq 2 ]] ++ kubectl_bin get psmdb cross-site-sharded-replica -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].arbiter.enabled}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.zKDfXk4uoA +++ mktemp ++ local LAST_ERR=/tmp/tmp.t3CWgRow3u ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'psmdb' 'cross-site-sharded-replica' '-o' 'jsonpath={.spec.replsets[?(@.name=="cfg")].arbiter.enabled}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb cross-site-sharded-replica -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].arbiter.enabled}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.zKDfXk4uoA ++ cat /tmp/tmp.t3CWgRow3u ++ rm /tmp/tmp.zKDfXk4uoA /tmp/tmp.t3CWgRow3u ++ return 0 + [[ '' == true ]] + wait_pod cross-site-sharded-replica-cfg-2 + local pod=cross-site-sharded-replica-cfg-2 + set +o xtrace waiting for pod/cross-site-sharded-replica-cfg-2 to be ready.OK ++ kubectl_bin get psmdb cross-site-sharded-replica -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].nonvoting.enabled}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.JBDjNsLq5g +++ mktemp ++ local LAST_ERR=/tmp/tmp.gSIzsq1HCG ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'psmdb' 'cross-site-sharded-replica' '-o' 'jsonpath={.spec.replsets[?(@.name=="cfg")].nonvoting.enabled}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb cross-site-sharded-replica -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].nonvoting.enabled}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.JBDjNsLq5g ++ cat /tmp/tmp.gSIzsq1HCG ++ rm /tmp/tmp.JBDjNsLq5g /tmp/tmp.gSIzsq1HCG ++ return 0 + [[ '' == true ]] ++ kubectl_bin get psmdb cross-site-sharded-replica -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].hidden.enabled}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.gWQq7P3fVg +++ mktemp ++ local LAST_ERR=/tmp/tmp.xDH8y6Rd8U ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'psmdb' 'cross-site-sharded-replica' '-o' 'jsonpath={.spec.replsets[?(@.name=="cfg")].hidden.enabled}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb cross-site-sharded-replica -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].hidden.enabled}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.gWQq7P3fVg ++ cat /tmp/tmp.xDH8y6Rd8U ++ rm /tmp/tmp.gWQq7P3fVg /tmp/tmp.xDH8y6Rd8U ++ return 0 + [[ '' == true ]] + sleep 10 + [[ false == true ]] ++ get_service_ip cross-site-sharded-replica-cfg-0 cfg ++ local service=cross-site-sharded-replica-cfg-0 ++ local server_type=cfg +++ kubectl_bin get psmdb/cross-site-sharded-replica -o 'jsonpath={.spec.replsets[].expose.enabled}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.r54o7OhsBb ++++ mktemp +++ local LAST_ERR=/tmp/tmp.Z5Jufqml3s +++ local exit_status=0 +++ local timeout=4 +++ args=('get' 'psmdb/cross-site-sharded-replica' '-o' 'jsonpath={.spec.replsets[].expose.enabled}') +++ local args +++ transformed_files=() +++ local transformed_files +++ arch_scheduling_enabled +++ [[ -n '' ]] +++ arch_scheduling_enabled +++ [[ -n '' ]] ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl get psmdb/cross-site-sharded-replica -o 'jsonpath={.spec.replsets[].expose.enabled}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 -a -n 1 ']' +++ break +++ cat /tmp/tmp.r54o7OhsBb +++ cat /tmp/tmp.Z5Jufqml3s +++ rm /tmp/tmp.r54o7OhsBb /tmp/tmp.Z5Jufqml3s +++ return 0 ++ '[' true '!=' true ']' ++ grep -q NotFound ++ kubectl_bin get service/cross-site-sharded-replica-cfg-0 -o 'jsonpath={.spec.type}' +++ kubectl_bin get service/cross-site-sharded-replica-cfg-0 -o 'jsonpath={.spec.type}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.NLNQcIq7rO ++++ mktemp +++ local LAST_ERR=/tmp/tmp.mUHz6LeTCA +++ local exit_status=0 +++ local timeout=4 +++ args=('get' 'service/cross-site-sharded-replica-cfg-0' '-o' 'jsonpath={.spec.type}') +++ local args +++ transformed_files=() +++ local transformed_files +++ arch_scheduling_enabled +++ [[ -n '' ]] +++ arch_scheduling_enabled +++ [[ -n '' ]] ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl get service/cross-site-sharded-replica-cfg-0 -o 'jsonpath={.spec.type}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 -a -n 1 ']' +++ break +++ cat /tmp/tmp.NLNQcIq7rO +++ cat /tmp/tmp.mUHz6LeTCA +++ rm /tmp/tmp.NLNQcIq7rO /tmp/tmp.mUHz6LeTCA +++ return 0 ++ service_type=ClusterIP ++ '[' ClusterIP = ClusterIP ']' ++ kubectl_bin get service/cross-site-sharded-replica-cfg-0 -o 'jsonpath={.spec.clusterIP}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.eAaxRHzQ8u +++ mktemp ++ local LAST_ERR=/tmp/tmp.MTZwpsac8P ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'service/cross-site-sharded-replica-cfg-0' '-o' 'jsonpath={.spec.clusterIP}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get service/cross-site-sharded-replica-cfg-0 -o 'jsonpath={.spec.clusterIP}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.eAaxRHzQ8u ++ cat /tmp/tmp.MTZwpsac8P ++ rm /tmp/tmp.eAaxRHzQ8u /tmp/tmp.MTZwpsac8P ++ return 0 ++ return + replica_cfg_0_endpoint=34.118.238.71 ++ get_service_ip cross-site-sharded-replica-cfg-1 cfg ++ local service=cross-site-sharded-replica-cfg-1 ++ local server_type=cfg +++ kubectl_bin get psmdb/cross-site-sharded-replica -o 'jsonpath={.spec.replsets[].expose.enabled}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.ixxWDRTeP5 ++++ mktemp +++ local LAST_ERR=/tmp/tmp.yvu7YFGRoa +++ local exit_status=0 +++ local timeout=4 +++ args=('get' 'psmdb/cross-site-sharded-replica' '-o' 'jsonpath={.spec.replsets[].expose.enabled}') +++ local args +++ transformed_files=() +++ local transformed_files +++ arch_scheduling_enabled +++ [[ -n '' ]] +++ arch_scheduling_enabled +++ [[ -n '' ]] ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl get psmdb/cross-site-sharded-replica -o 'jsonpath={.spec.replsets[].expose.enabled}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 -a -n 1 ']' +++ break +++ cat /tmp/tmp.ixxWDRTeP5 +++ cat /tmp/tmp.yvu7YFGRoa +++ rm /tmp/tmp.ixxWDRTeP5 /tmp/tmp.yvu7YFGRoa +++ return 0 ++ '[' true '!=' true ']' ++ kubectl_bin get service/cross-site-sharded-replica-cfg-1 -o 'jsonpath={.spec.type}' ++ grep -q NotFound +++ kubectl_bin get service/cross-site-sharded-replica-cfg-1 -o 'jsonpath={.spec.type}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.3Jo3G8M97C ++++ mktemp +++ local LAST_ERR=/tmp/tmp.V0MxPQHVcO +++ local exit_status=0 +++ local timeout=4 +++ args=('get' 'service/cross-site-sharded-replica-cfg-1' '-o' 'jsonpath={.spec.type}') +++ local args +++ transformed_files=() +++ local transformed_files +++ arch_scheduling_enabled +++ [[ -n '' ]] +++ arch_scheduling_enabled +++ [[ -n '' ]] ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl get service/cross-site-sharded-replica-cfg-1 -o 'jsonpath={.spec.type}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 -a -n 1 ']' +++ break +++ cat /tmp/tmp.3Jo3G8M97C +++ cat /tmp/tmp.V0MxPQHVcO +++ rm /tmp/tmp.3Jo3G8M97C /tmp/tmp.V0MxPQHVcO +++ return 0 ++ service_type=ClusterIP ++ '[' ClusterIP = ClusterIP ']' ++ kubectl_bin get service/cross-site-sharded-replica-cfg-1 -o 'jsonpath={.spec.clusterIP}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.8Kd3hBrDtq +++ mktemp ++ local LAST_ERR=/tmp/tmp.bgJc9I5TO3 ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'service/cross-site-sharded-replica-cfg-1' '-o' 'jsonpath={.spec.clusterIP}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get service/cross-site-sharded-replica-cfg-1 -o 'jsonpath={.spec.clusterIP}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.8Kd3hBrDtq ++ cat /tmp/tmp.bgJc9I5TO3 ++ rm /tmp/tmp.8Kd3hBrDtq /tmp/tmp.bgJc9I5TO3 ++ return 0 ++ return + replica_cfg_1_endpoint=34.118.230.156 ++ get_service_ip cross-site-sharded-replica-cfg-2 cfg ++ local service=cross-site-sharded-replica-cfg-2 ++ local server_type=cfg +++ kubectl_bin get psmdb/cross-site-sharded-replica -o 'jsonpath={.spec.replsets[].expose.enabled}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.Jh4vTzgG7R ++++ mktemp +++ local LAST_ERR=/tmp/tmp.2wi4P52qIL +++ local exit_status=0 +++ local timeout=4 +++ args=('get' 'psmdb/cross-site-sharded-replica' '-o' 'jsonpath={.spec.replsets[].expose.enabled}') +++ local args +++ transformed_files=() +++ local transformed_files +++ arch_scheduling_enabled +++ [[ -n '' ]] +++ arch_scheduling_enabled +++ [[ -n '' ]] ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl get psmdb/cross-site-sharded-replica -o 'jsonpath={.spec.replsets[].expose.enabled}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 -a -n 1 ']' +++ break +++ cat /tmp/tmp.Jh4vTzgG7R +++ cat /tmp/tmp.2wi4P52qIL +++ rm /tmp/tmp.Jh4vTzgG7R /tmp/tmp.2wi4P52qIL +++ return 0 ++ '[' true '!=' true ']' ++ kubectl_bin get service/cross-site-sharded-replica-cfg-2 -o 'jsonpath={.spec.type}' ++ grep -q NotFound +++ kubectl_bin get service/cross-site-sharded-replica-cfg-2 -o 'jsonpath={.spec.type}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.7ktGUfw6K8 ++++ mktemp +++ local LAST_ERR=/tmp/tmp.RAeGsJWQHw +++ local exit_status=0 +++ local timeout=4 +++ args=('get' 'service/cross-site-sharded-replica-cfg-2' '-o' 'jsonpath={.spec.type}') +++ local args +++ transformed_files=() +++ local transformed_files +++ arch_scheduling_enabled +++ [[ -n '' ]] +++ arch_scheduling_enabled +++ [[ -n '' ]] ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl get service/cross-site-sharded-replica-cfg-2 -o 'jsonpath={.spec.type}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 -a -n 1 ']' +++ break +++ cat /tmp/tmp.7ktGUfw6K8 +++ cat /tmp/tmp.RAeGsJWQHw +++ rm /tmp/tmp.7ktGUfw6K8 /tmp/tmp.RAeGsJWQHw +++ return 0 ++ service_type=ClusterIP ++ '[' ClusterIP = ClusterIP ']' ++ kubectl_bin get service/cross-site-sharded-replica-cfg-2 -o 'jsonpath={.spec.clusterIP}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.i8eHQmpZZ7 +++ mktemp ++ local LAST_ERR=/tmp/tmp.6pSA9INcox ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'service/cross-site-sharded-replica-cfg-2' '-o' 'jsonpath={.spec.clusterIP}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get service/cross-site-sharded-replica-cfg-2 -o 'jsonpath={.spec.clusterIP}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.i8eHQmpZZ7 ++ cat /tmp/tmp.6pSA9INcox ++ rm /tmp/tmp.i8eHQmpZZ7 /tmp/tmp.6pSA9INcox ++ return 0 ++ return + replica_cfg_2_endpoint=34.118.238.168 ++ get_service_ip cross-site-sharded-replica-rs0-0 ++ local service=cross-site-sharded-replica-rs0-0 ++ local server_type=rs0 +++ kubectl_bin get psmdb/cross-site-sharded-replica -o 'jsonpath={.spec.replsets[].expose.enabled}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.EZihzfgPWo ++++ mktemp +++ local LAST_ERR=/tmp/tmp.89IcBrLUf8 +++ local exit_status=0 +++ local timeout=4 +++ args=('get' 'psmdb/cross-site-sharded-replica' '-o' 'jsonpath={.spec.replsets[].expose.enabled}') +++ local args +++ transformed_files=() +++ local transformed_files +++ arch_scheduling_enabled +++ [[ -n '' ]] +++ arch_scheduling_enabled +++ [[ -n '' ]] ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl get psmdb/cross-site-sharded-replica -o 'jsonpath={.spec.replsets[].expose.enabled}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 -a -n 1 ']' +++ break +++ cat /tmp/tmp.EZihzfgPWo +++ cat /tmp/tmp.89IcBrLUf8 +++ rm /tmp/tmp.EZihzfgPWo /tmp/tmp.89IcBrLUf8 +++ return 0 ++ '[' true '!=' true ']' ++ kubectl_bin get service/cross-site-sharded-replica-rs0-0 -o 'jsonpath={.spec.type}' ++ grep -q NotFound +++ kubectl_bin get service/cross-site-sharded-replica-rs0-0 -o 'jsonpath={.spec.type}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.ze8TaMpGcv ++++ mktemp +++ local LAST_ERR=/tmp/tmp.uhSRWMsnwR +++ local exit_status=0 +++ local timeout=4 +++ args=('get' 'service/cross-site-sharded-replica-rs0-0' '-o' 'jsonpath={.spec.type}') +++ local args +++ transformed_files=() +++ local transformed_files +++ arch_scheduling_enabled +++ [[ -n '' ]] +++ arch_scheduling_enabled +++ [[ -n '' ]] ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl get service/cross-site-sharded-replica-rs0-0 -o 'jsonpath={.spec.type}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 -a -n 1 ']' +++ break +++ cat /tmp/tmp.ze8TaMpGcv +++ cat /tmp/tmp.uhSRWMsnwR +++ rm /tmp/tmp.ze8TaMpGcv /tmp/tmp.uhSRWMsnwR +++ return 0 ++ service_type=ClusterIP ++ '[' ClusterIP = ClusterIP ']' ++ kubectl_bin get service/cross-site-sharded-replica-rs0-0 -o 'jsonpath={.spec.clusterIP}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.qhCxVKynR5 +++ mktemp ++ local LAST_ERR=/tmp/tmp.G0cwGiDH8H ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'service/cross-site-sharded-replica-rs0-0' '-o' 'jsonpath={.spec.clusterIP}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get service/cross-site-sharded-replica-rs0-0 -o 'jsonpath={.spec.clusterIP}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.qhCxVKynR5 ++ cat /tmp/tmp.G0cwGiDH8H ++ rm /tmp/tmp.qhCxVKynR5 /tmp/tmp.G0cwGiDH8H ++ return 0 ++ return + replica_rs0_0_endpoint=34.118.232.18 ++ get_service_ip cross-site-sharded-replica-rs0-1 ++ local service=cross-site-sharded-replica-rs0-1 ++ local server_type=rs0 +++ kubectl_bin get psmdb/cross-site-sharded-replica -o 'jsonpath={.spec.replsets[].expose.enabled}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.a1yWyANWJJ ++++ mktemp +++ local LAST_ERR=/tmp/tmp.yTFgUEbX86 +++ local exit_status=0 +++ local timeout=4 +++ args=('get' 'psmdb/cross-site-sharded-replica' '-o' 'jsonpath={.spec.replsets[].expose.enabled}') +++ local args +++ transformed_files=() +++ local transformed_files +++ arch_scheduling_enabled +++ [[ -n '' ]] +++ arch_scheduling_enabled +++ [[ -n '' ]] ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl get psmdb/cross-site-sharded-replica -o 'jsonpath={.spec.replsets[].expose.enabled}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 -a -n 1 ']' +++ break +++ cat /tmp/tmp.a1yWyANWJJ +++ cat /tmp/tmp.yTFgUEbX86 +++ rm /tmp/tmp.a1yWyANWJJ /tmp/tmp.yTFgUEbX86 +++ return 0 ++ '[' true '!=' true ']' ++ kubectl_bin get service/cross-site-sharded-replica-rs0-1 -o 'jsonpath={.spec.type}' ++ grep -q NotFound +++ kubectl_bin get service/cross-site-sharded-replica-rs0-1 -o 'jsonpath={.spec.type}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.1f5aFuzRYS ++++ mktemp +++ local LAST_ERR=/tmp/tmp.8eUPWJj6aQ +++ local exit_status=0 +++ local timeout=4 +++ args=('get' 'service/cross-site-sharded-replica-rs0-1' '-o' 'jsonpath={.spec.type}') +++ local args +++ transformed_files=() +++ local transformed_files +++ arch_scheduling_enabled +++ [[ -n '' ]] +++ arch_scheduling_enabled +++ [[ -n '' ]] ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl get service/cross-site-sharded-replica-rs0-1 -o 'jsonpath={.spec.type}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 -a -n 1 ']' +++ break +++ cat /tmp/tmp.1f5aFuzRYS +++ cat /tmp/tmp.8eUPWJj6aQ +++ rm /tmp/tmp.1f5aFuzRYS /tmp/tmp.8eUPWJj6aQ +++ return 0 ++ service_type=ClusterIP ++ '[' ClusterIP = ClusterIP ']' ++ kubectl_bin get service/cross-site-sharded-replica-rs0-1 -o 'jsonpath={.spec.clusterIP}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.6fPS6vNy4m +++ mktemp ++ local LAST_ERR=/tmp/tmp.oyQrqGSMSb ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'service/cross-site-sharded-replica-rs0-1' '-o' 'jsonpath={.spec.clusterIP}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get service/cross-site-sharded-replica-rs0-1 -o 'jsonpath={.spec.clusterIP}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.6fPS6vNy4m ++ cat /tmp/tmp.oyQrqGSMSb ++ rm /tmp/tmp.6fPS6vNy4m /tmp/tmp.oyQrqGSMSb ++ return 0 ++ return + replica_rs0_1_endpoint=34.118.233.30 ++ get_service_ip cross-site-sharded-replica-rs0-2 ++ local service=cross-site-sharded-replica-rs0-2 ++ local server_type=rs0 +++ kubectl_bin get psmdb/cross-site-sharded-replica -o 'jsonpath={.spec.replsets[].expose.enabled}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.2uQ2yXt7xx ++++ mktemp +++ local LAST_ERR=/tmp/tmp.0mb1wuwvpT +++ local exit_status=0 +++ local timeout=4 +++ args=('get' 'psmdb/cross-site-sharded-replica' '-o' 'jsonpath={.spec.replsets[].expose.enabled}') +++ local args +++ transformed_files=() +++ local transformed_files +++ arch_scheduling_enabled +++ [[ -n '' ]] +++ arch_scheduling_enabled +++ [[ -n '' ]] ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl get psmdb/cross-site-sharded-replica -o 'jsonpath={.spec.replsets[].expose.enabled}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 -a -n 1 ']' +++ break +++ cat /tmp/tmp.2uQ2yXt7xx +++ cat /tmp/tmp.0mb1wuwvpT +++ rm /tmp/tmp.2uQ2yXt7xx /tmp/tmp.0mb1wuwvpT +++ return 0 ++ '[' true '!=' true ']' ++ kubectl_bin get service/cross-site-sharded-replica-rs0-2 -o 'jsonpath={.spec.type}' ++ grep -q NotFound +++ kubectl_bin get service/cross-site-sharded-replica-rs0-2 -o 'jsonpath={.spec.type}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.ePbTVUSxCK ++++ mktemp +++ local LAST_ERR=/tmp/tmp.qRtVVTafx8 +++ local exit_status=0 +++ local timeout=4 +++ args=('get' 'service/cross-site-sharded-replica-rs0-2' '-o' 'jsonpath={.spec.type}') +++ local args +++ transformed_files=() +++ local transformed_files +++ arch_scheduling_enabled +++ [[ -n '' ]] +++ arch_scheduling_enabled +++ [[ -n '' ]] ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl get service/cross-site-sharded-replica-rs0-2 -o 'jsonpath={.spec.type}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 -a -n 1 ']' +++ break +++ cat /tmp/tmp.ePbTVUSxCK +++ cat /tmp/tmp.qRtVVTafx8 +++ rm /tmp/tmp.ePbTVUSxCK /tmp/tmp.qRtVVTafx8 +++ return 0 ++ service_type=ClusterIP ++ '[' ClusterIP = ClusterIP ']' ++ kubectl_bin get service/cross-site-sharded-replica-rs0-2 -o 'jsonpath={.spec.clusterIP}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.Kni0ZiuL67 +++ mktemp ++ local LAST_ERR=/tmp/tmp.SwR0mucQtc ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'service/cross-site-sharded-replica-rs0-2' '-o' 'jsonpath={.spec.clusterIP}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get service/cross-site-sharded-replica-rs0-2 -o 'jsonpath={.spec.clusterIP}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.Kni0ZiuL67 ++ cat /tmp/tmp.SwR0mucQtc ++ rm /tmp/tmp.Kni0ZiuL67 /tmp/tmp.SwR0mucQtc ++ return 0 ++ return + replica_rs0_2_endpoint=34.118.234.195 ++ get_service_ip cross-site-sharded-replica-rs1-0 rs1 ++ local service=cross-site-sharded-replica-rs1-0 ++ local server_type=rs1 +++ kubectl_bin get psmdb/cross-site-sharded-replica -o 'jsonpath={.spec.replsets[].expose.enabled}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.0PM0G9F35h ++++ mktemp +++ local LAST_ERR=/tmp/tmp.hySMjDP7C9 +++ local exit_status=0 +++ local timeout=4 +++ args=('get' 'psmdb/cross-site-sharded-replica' '-o' 'jsonpath={.spec.replsets[].expose.enabled}') +++ local args +++ transformed_files=() +++ local transformed_files +++ arch_scheduling_enabled +++ [[ -n '' ]] +++ arch_scheduling_enabled +++ [[ -n '' ]] ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl get psmdb/cross-site-sharded-replica -o 'jsonpath={.spec.replsets[].expose.enabled}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 -a -n 1 ']' +++ break +++ cat /tmp/tmp.0PM0G9F35h +++ cat /tmp/tmp.hySMjDP7C9 +++ rm /tmp/tmp.0PM0G9F35h /tmp/tmp.hySMjDP7C9 +++ return 0 ++ '[' true '!=' true ']' ++ kubectl_bin get service/cross-site-sharded-replica-rs1-0 -o 'jsonpath={.spec.type}' ++ grep -q NotFound +++ kubectl_bin get service/cross-site-sharded-replica-rs1-0 -o 'jsonpath={.spec.type}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.UswUj8i7Eh ++++ mktemp +++ local LAST_ERR=/tmp/tmp.WcY9waXUSb +++ local exit_status=0 +++ local timeout=4 +++ args=('get' 'service/cross-site-sharded-replica-rs1-0' '-o' 'jsonpath={.spec.type}') +++ local args +++ transformed_files=() +++ local transformed_files +++ arch_scheduling_enabled +++ [[ -n '' ]] +++ arch_scheduling_enabled +++ [[ -n '' ]] ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl get service/cross-site-sharded-replica-rs1-0 -o 'jsonpath={.spec.type}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 -a -n 1 ']' +++ break +++ cat /tmp/tmp.UswUj8i7Eh +++ cat /tmp/tmp.WcY9waXUSb +++ rm /tmp/tmp.UswUj8i7Eh /tmp/tmp.WcY9waXUSb +++ return 0 ++ service_type=ClusterIP ++ '[' ClusterIP = ClusterIP ']' ++ kubectl_bin get service/cross-site-sharded-replica-rs1-0 -o 'jsonpath={.spec.clusterIP}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.psgMJ9PgDL +++ mktemp ++ local LAST_ERR=/tmp/tmp.96i8PAotDk ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'service/cross-site-sharded-replica-rs1-0' '-o' 'jsonpath={.spec.clusterIP}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get service/cross-site-sharded-replica-rs1-0 -o 'jsonpath={.spec.clusterIP}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.psgMJ9PgDL ++ cat /tmp/tmp.96i8PAotDk ++ rm /tmp/tmp.psgMJ9PgDL /tmp/tmp.96i8PAotDk ++ return 0 ++ return + replica_rs1_0_endpoint=34.118.231.170 ++ get_service_ip cross-site-sharded-replica-rs1-1 rs1 ++ local service=cross-site-sharded-replica-rs1-1 ++ local server_type=rs1 +++ kubectl_bin get psmdb/cross-site-sharded-replica -o 'jsonpath={.spec.replsets[].expose.enabled}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.9dyxDpQECO ++++ mktemp +++ local LAST_ERR=/tmp/tmp.BPGdPu620f +++ local exit_status=0 +++ local timeout=4 +++ args=('get' 'psmdb/cross-site-sharded-replica' '-o' 'jsonpath={.spec.replsets[].expose.enabled}') +++ local args +++ transformed_files=() +++ local transformed_files +++ arch_scheduling_enabled +++ [[ -n '' ]] +++ arch_scheduling_enabled +++ [[ -n '' ]] ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl get psmdb/cross-site-sharded-replica -o 'jsonpath={.spec.replsets[].expose.enabled}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 -a -n 1 ']' +++ break +++ cat /tmp/tmp.9dyxDpQECO +++ cat /tmp/tmp.BPGdPu620f +++ rm /tmp/tmp.9dyxDpQECO /tmp/tmp.BPGdPu620f +++ return 0 ++ '[' true '!=' true ']' ++ kubectl_bin get service/cross-site-sharded-replica-rs1-1 -o 'jsonpath={.spec.type}' ++ grep -q NotFound +++ kubectl_bin get service/cross-site-sharded-replica-rs1-1 -o 'jsonpath={.spec.type}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.zaDIKqVaEg ++++ mktemp +++ local LAST_ERR=/tmp/tmp.sHsX5OmcgA +++ local exit_status=0 +++ local timeout=4 +++ args=('get' 'service/cross-site-sharded-replica-rs1-1' '-o' 'jsonpath={.spec.type}') +++ local args +++ transformed_files=() +++ local transformed_files +++ arch_scheduling_enabled +++ [[ -n '' ]] +++ arch_scheduling_enabled +++ [[ -n '' ]] ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl get service/cross-site-sharded-replica-rs1-1 -o 'jsonpath={.spec.type}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 -a -n 1 ']' +++ break +++ cat /tmp/tmp.zaDIKqVaEg +++ cat /tmp/tmp.sHsX5OmcgA +++ rm /tmp/tmp.zaDIKqVaEg /tmp/tmp.sHsX5OmcgA +++ return 0 ++ service_type=ClusterIP ++ '[' ClusterIP = ClusterIP ']' ++ kubectl_bin get service/cross-site-sharded-replica-rs1-1 -o 'jsonpath={.spec.clusterIP}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.8ZFE9ea5Nf +++ mktemp ++ local LAST_ERR=/tmp/tmp.ajlGVR2sWR ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'service/cross-site-sharded-replica-rs1-1' '-o' 'jsonpath={.spec.clusterIP}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get service/cross-site-sharded-replica-rs1-1 -o 'jsonpath={.spec.clusterIP}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.8ZFE9ea5Nf ++ cat /tmp/tmp.ajlGVR2sWR ++ rm /tmp/tmp.8ZFE9ea5Nf /tmp/tmp.ajlGVR2sWR ++ return 0 ++ return + replica_rs1_1_endpoint=34.118.235.45 ++ get_service_ip cross-site-sharded-replica-rs1-2 rs1 ++ local service=cross-site-sharded-replica-rs1-2 ++ local server_type=rs1 +++ kubectl_bin get psmdb/cross-site-sharded-replica -o 'jsonpath={.spec.replsets[].expose.enabled}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.ni7rJY2RXi ++++ mktemp +++ local LAST_ERR=/tmp/tmp.yRoVUTC7O1 +++ local exit_status=0 +++ local timeout=4 +++ args=('get' 'psmdb/cross-site-sharded-replica' '-o' 'jsonpath={.spec.replsets[].expose.enabled}') +++ local args +++ transformed_files=() +++ local transformed_files +++ arch_scheduling_enabled +++ [[ -n '' ]] +++ arch_scheduling_enabled +++ [[ -n '' ]] ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl get psmdb/cross-site-sharded-replica -o 'jsonpath={.spec.replsets[].expose.enabled}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 -a -n 1 ']' +++ break +++ cat /tmp/tmp.ni7rJY2RXi +++ cat /tmp/tmp.yRoVUTC7O1 +++ rm /tmp/tmp.ni7rJY2RXi /tmp/tmp.yRoVUTC7O1 +++ return 0 ++ '[' true '!=' true ']' ++ grep -q NotFound ++ kubectl_bin get service/cross-site-sharded-replica-rs1-2 -o 'jsonpath={.spec.type}' +++ kubectl_bin get service/cross-site-sharded-replica-rs1-2 -o 'jsonpath={.spec.type}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.9zjPi6Bxb0 ++++ mktemp +++ local LAST_ERR=/tmp/tmp.kMdwJGaUfF +++ local exit_status=0 +++ local timeout=4 +++ args=('get' 'service/cross-site-sharded-replica-rs1-2' '-o' 'jsonpath={.spec.type}') +++ local args +++ transformed_files=() +++ local transformed_files +++ arch_scheduling_enabled +++ [[ -n '' ]] +++ arch_scheduling_enabled +++ [[ -n '' ]] ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl get service/cross-site-sharded-replica-rs1-2 -o 'jsonpath={.spec.type}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 -a -n 1 ']' +++ break +++ cat /tmp/tmp.9zjPi6Bxb0 +++ cat /tmp/tmp.kMdwJGaUfF +++ rm /tmp/tmp.9zjPi6Bxb0 /tmp/tmp.kMdwJGaUfF +++ return 0 ++ service_type=ClusterIP ++ '[' ClusterIP = ClusterIP ']' ++ kubectl_bin get service/cross-site-sharded-replica-rs1-2 -o 'jsonpath={.spec.clusterIP}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.lCTlmFzv1S +++ mktemp ++ local LAST_ERR=/tmp/tmp.BVrSjNUcO0 ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'service/cross-site-sharded-replica-rs1-2' '-o' 'jsonpath={.spec.clusterIP}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get service/cross-site-sharded-replica-rs1-2 -o 'jsonpath={.spec.clusterIP}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.lCTlmFzv1S ++ cat /tmp/tmp.BVrSjNUcO0 ++ rm /tmp/tmp.lCTlmFzv1S /tmp/tmp.BVrSjNUcO0 ++ return 0 ++ return + replica_rs1_2_endpoint=34.118.238.190 ++ kubectl_bin config current-context +++ mktemp ++ local LAST_OUT=/tmp/tmp.JlMkNmnm2d +++ mktemp ++ local LAST_ERR=/tmp/tmp.hbyjLVicvn ++ local exit_status=0 ++ local timeout=4 ++ args=('config' 'current-context') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl config current-context ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.JlMkNmnm2d ++ cat /tmp/tmp.hbyjLVicvn ++ rm /tmp/tmp.JlMkNmnm2d /tmp/tmp.hbyjLVicvn ++ return 0 + kubectl_bin config set-context gke_cloud-dev-112233_us-central1_jen-psmdb-2058-78121c567-22-cluster5 --namespace=cross-site-sharded-12730 ++ mktemp + local LAST_OUT=/tmp/tmp.Kx94sge1zP ++ mktemp + local LAST_ERR=/tmp/tmp.NOdULHmOfl + local exit_status=0 + local timeout=4 + args=('config' 'set-context' 'gke_cloud-dev-112233_us-central1_jen-psmdb-2058-78121c567-22-cluster5' '--namespace=cross-site-sharded-12730') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl config set-context gke_cloud-dev-112233_us-central1_jen-psmdb-2058-78121c567-22-cluster5 --namespace=cross-site-sharded-12730 + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.Kx94sge1zP Context "gke_cloud-dev-112233_us-central1_jen-psmdb-2058-78121c567-22-cluster5" modified. + cat /tmp/tmp.NOdULHmOfl + rm /tmp/tmp.Kx94sge1zP /tmp/tmp.NOdULHmOfl + return 0 + kubectl_bin patch psmdb cross-site-sharded-main --type=merge --patch $'{\n\t\t\t"spec": {"replsets":[\n\t\t\t {"affinity":{"antiAffinityTopologyKey": "none"},"arbiter":{"affinity":{"antiAffinityTopologyKey": "none"},"enabled":false,"size":1},"expose":{"enabled":true,"type":"ClusterIp"},"externalNodes":[{"host":"34.118.232.18","priority":0,"votes":0},{"host":"34.118.233.30","port":27017,"priority":1,"votes":1},{"host":"34.118.234.195", "port":27017,"priority":1,"votes":1}],"name":"rs0","nonvoting":{"affinity":{"antiAffinityTopologyKey":"none"},"enabled":false,"podDisruptionBudget":{"maxUnavailable":1},"resources":{"limits":{"cpu":"300m","memory":"0.5G"},"requests":{"cpu":"300m","memory":"0.5G"}},"size":3,"volumeSpec":{"persistentVolumeClaim":{"resources":{"requests":{"storage":"1Gi"}}}}},"podDisruptionBudget":{"maxUnavailable":1},"resources":{"limits":{"cpu":"300m","memory":"0.5G"},"requests":{"cpu":"300m","memory":"0.5G"}},"size":3,"volumeSpec":{"persistentVolumeClaim":{"resources":{"requests":{"storage":"3Gi"}}}}},\n\t\t\t {"affinity":{"antiAffinityTopologyKey": "none"},"arbiter":{"affinity":{"antiAffinityTopologyKey": "none"},"enabled":false,"size":1},"expose":{"enabled":true,"type":"ClusterIp"},"externalNodes":[{"host":"34.118.231.170","priority":0,"votes":0},{"host":"34.118.235.45","port":27017,"priority":1,"votes":1},{"host":"34.118.238.190", "port":27017,"priority":1,"votes":1}],"name":"rs1","nonvoting":{"affinity":{"antiAffinityTopologyKey":"none"},"enabled":false,"podDisruptionBudget":{"maxUnavailable":1},"resources":{"limits":{"cpu":"300m","memory":"0.5G"},"requests":{"cpu":"300m","memory":"0.5G"}},"size":3,"volumeSpec":{"persistentVolumeClaim":{"resources":{"requests":{"storage":"1Gi"}}}}},"podDisruptionBudget":{"maxUnavailable":1},"resources":{"limits":{"cpu":"300m","memory":"0.5G"},"requests":{"cpu":"300m","memory":"0.5G"}},"size":3,"volumeSpec":{"persistentVolumeClaim":{"resources":{"requests":{"storage":"3Gi"}}}}}\n\t\t\t ],\n\t\t\t "sharding":{"configsvrReplSet":{ "externalNodes": [{"host":"34.118.238.71","priority":1,"votes":1 },{"host":"34.118.230.156", "priority":1,"votes":1},{"host":"34.118.238.168","priority":0,"votes":0}]}}\n\t\t\t }\n\t\t}' ++ mktemp + local LAST_OUT=/tmp/tmp.yvwcNUVDTl ++ mktemp + local LAST_ERR=/tmp/tmp.vTAkwiaF2r + local exit_status=0 + local timeout=4 + args=('patch' 'psmdb' 'cross-site-sharded-main' '--type=merge' '--patch' '{ "spec": {"replsets":[ {"affinity":{"antiAffinityTopologyKey": "none"},"arbiter":{"affinity":{"antiAffinityTopologyKey": "none"},"enabled":false,"size":1},"expose":{"enabled":true,"type":"ClusterIp"},"externalNodes":[{"host":"34.118.232.18","priority":0,"votes":0},{"host":"34.118.233.30","port":27017,"priority":1,"votes":1},{"host":"34.118.234.195", "port":27017,"priority":1,"votes":1}],"name":"rs0","nonvoting":{"affinity":{"antiAffinityTopologyKey":"none"},"enabled":false,"podDisruptionBudget":{"maxUnavailable":1},"resources":{"limits":{"cpu":"300m","memory":"0.5G"},"requests":{"cpu":"300m","memory":"0.5G"}},"size":3,"volumeSpec":{"persistentVolumeClaim":{"resources":{"requests":{"storage":"1Gi"}}}}},"podDisruptionBudget":{"maxUnavailable":1},"resources":{"limits":{"cpu":"300m","memory":"0.5G"},"requests":{"cpu":"300m","memory":"0.5G"}},"size":3,"volumeSpec":{"persistentVolumeClaim":{"resources":{"requests":{"storage":"3Gi"}}}}}, {"affinity":{"antiAffinityTopologyKey": "none"},"arbiter":{"affinity":{"antiAffinityTopologyKey": "none"},"enabled":false,"size":1},"expose":{"enabled":true,"type":"ClusterIp"},"externalNodes":[{"host":"34.118.231.170","priority":0,"votes":0},{"host":"34.118.235.45","port":27017,"priority":1,"votes":1},{"host":"34.118.238.190", "port":27017,"priority":1,"votes":1}],"name":"rs1","nonvoting":{"affinity":{"antiAffinityTopologyKey":"none"},"enabled":false,"podDisruptionBudget":{"maxUnavailable":1},"resources":{"limits":{"cpu":"300m","memory":"0.5G"},"requests":{"cpu":"300m","memory":"0.5G"}},"size":3,"volumeSpec":{"persistentVolumeClaim":{"resources":{"requests":{"storage":"1Gi"}}}}},"podDisruptionBudget":{"maxUnavailable":1},"resources":{"limits":{"cpu":"300m","memory":"0.5G"},"requests":{"cpu":"300m","memory":"0.5G"}},"size":3,"volumeSpec":{"persistentVolumeClaim":{"resources":{"requests":{"storage":"3Gi"}}}}} ], "sharding":{"configsvrReplSet":{ "externalNodes": [{"host":"34.118.238.71","priority":1,"votes":1 },{"host":"34.118.230.156", "priority":1,"votes":1},{"host":"34.118.238.168","priority":0,"votes":0}]}} } }') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl patch psmdb cross-site-sharded-main --type=merge --patch $'{\n\t\t\t"spec": {"replsets":[\n\t\t\t {"affinity":{"antiAffinityTopologyKey": "none"},"arbiter":{"affinity":{"antiAffinityTopologyKey": "none"},"enabled":false,"size":1},"expose":{"enabled":true,"type":"ClusterIp"},"externalNodes":[{"host":"34.118.232.18","priority":0,"votes":0},{"host":"34.118.233.30","port":27017,"priority":1,"votes":1},{"host":"34.118.234.195", "port":27017,"priority":1,"votes":1}],"name":"rs0","nonvoting":{"affinity":{"antiAffinityTopologyKey":"none"},"enabled":false,"podDisruptionBudget":{"maxUnavailable":1},"resources":{"limits":{"cpu":"300m","memory":"0.5G"},"requests":{"cpu":"300m","memory":"0.5G"}},"size":3,"volumeSpec":{"persistentVolumeClaim":{"resources":{"requests":{"storage":"1Gi"}}}}},"podDisruptionBudget":{"maxUnavailable":1},"resources":{"limits":{"cpu":"300m","memory":"0.5G"},"requests":{"cpu":"300m","memory":"0.5G"}},"size":3,"volumeSpec":{"persistentVolumeClaim":{"resources":{"requests":{"storage":"3Gi"}}}}},\n\t\t\t {"affinity":{"antiAffinityTopologyKey": "none"},"arbiter":{"affinity":{"antiAffinityTopologyKey": "none"},"enabled":false,"size":1},"expose":{"enabled":true,"type":"ClusterIp"},"externalNodes":[{"host":"34.118.231.170","priority":0,"votes":0},{"host":"34.118.235.45","port":27017,"priority":1,"votes":1},{"host":"34.118.238.190", "port":27017,"priority":1,"votes":1}],"name":"rs1","nonvoting":{"affinity":{"antiAffinityTopologyKey":"none"},"enabled":false,"podDisruptionBudget":{"maxUnavailable":1},"resources":{"limits":{"cpu":"300m","memory":"0.5G"},"requests":{"cpu":"300m","memory":"0.5G"}},"size":3,"volumeSpec":{"persistentVolumeClaim":{"resources":{"requests":{"storage":"1Gi"}}}}},"podDisruptionBudget":{"maxUnavailable":1},"resources":{"limits":{"cpu":"300m","memory":"0.5G"},"requests":{"cpu":"300m","memory":"0.5G"}},"size":3,"volumeSpec":{"persistentVolumeClaim":{"resources":{"requests":{"storage":"3Gi"}}}}}\n\t\t\t ],\n\t\t\t "sharding":{"configsvrReplSet":{ "externalNodes": [{"host":"34.118.238.71","priority":1,"votes":1 },{"host":"34.118.230.156", "priority":1,"votes":1},{"host":"34.118.238.168","priority":0,"votes":0}]}}\n\t\t\t }\n\t\t}' + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.yvwcNUVDTl perconaservermongodb.psmdb.percona.com/cross-site-sharded-main patched + cat /tmp/tmp.vTAkwiaF2r + rm /tmp/tmp.yvwcNUVDTl /tmp/tmp.vTAkwiaF2r + return 0 + arch_scheduling_enabled + [[ -n '' ]] + wait_for_members 34.118.238.71 cfg 6 + local endpoint=34.118.238.71 + local rsName=cfg + local target_count=6 + local nodes_count=0 + [[ 0 == 6 ]] ++ run_mongos 'rs.conf().members.length' clusterAdmin:clusterAdmin123456@34.118.238.71 mongodb :27017 ++ local 'command=rs.conf().members.length' ++ local uri=clusterAdmin:clusterAdmin123456@34.118.238.71 ++ local driver=mongodb ++ local suffix=:27017 ++ local mongo_flag= ++ local port=27017 ++ local mongo_bin=mongo ++ grep -E -v 'I NETWORK|W NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|bye' ++ /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/' +++ echo :27017 +++ awk -F: '{print $2}' ++ suffix_port=27017 ++ [[ -z 27017 ]] +++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.itRRwz7MzD ++++ mktemp +++ local LAST_ERR=/tmp/tmp.BsurwE4NX3 +++ local exit_status=0 +++ local timeout=4 +++ args=('get' 'pods' '--selector=name=psmdb-client' '-o' 'jsonpath={.items[].metadata.name}') +++ local args +++ transformed_files=() +++ local transformed_files +++ arch_scheduling_enabled +++ [[ -n '' ]] +++ arch_scheduling_enabled +++ [[ -n '' ]] ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 -a -n 1 ']' +++ break +++ cat /tmp/tmp.itRRwz7MzD +++ cat /tmp/tmp.BsurwE4NX3 +++ rm /tmp/tmp.itRRwz7MzD /tmp/tmp.BsurwE4NX3 +++ return 0 ++ local client_container=psmdb-client-757cd65695-xjsxw ++ kubectl_bin exec psmdb-client-757cd65695-xjsxw -- bash -c 'export HOME=/tmp; printf '\''rs.conf().members.length\n'\'' | mongo mongodb://clusterAdmin:clusterAdmin123456@34.118.238.71:27017/admin ' +++ mktemp ++ local LAST_OUT=/tmp/tmp.d1qiRcbgNs +++ mktemp ++ local LAST_ERR=/tmp/tmp.HSEQdAPK94 ++ local exit_status=0 ++ local timeout=4 ++ args=('exec' 'psmdb-client-757cd65695-xjsxw' '--' 'bash' '-c' 'export HOME=/tmp; printf '\''rs.conf().members.length\n'\'' | mongo mongodb://clusterAdmin:clusterAdmin123456@34.118.238.71:27017/admin ') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl exec psmdb-client-757cd65695-xjsxw -- bash -c 'export HOME=/tmp; printf '\''rs.conf().members.length\n'\'' | mongo mongodb://clusterAdmin:clusterAdmin123456@34.118.238.71:27017/admin ' ++ exit_status=1 ++ set -e ++ '[' 1 '!=' 0 -a -n 1 ']' ++ cat /tmp/tmp.d1qiRcbgNs ++ cat /tmp/tmp.HSEQdAPK94 command terminated with exit code 1 ++ sleep 0 ++ for i in $(seq 0 2) ++ set +e ++ kubectl exec psmdb-client-757cd65695-xjsxw -- bash -c 'export HOME=/tmp; printf '\''rs.conf().members.length\n'\'' | mongo mongodb://clusterAdmin:clusterAdmin123456@34.118.238.71:27017/admin ' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.d1qiRcbgNs ++ cat /tmp/tmp.HSEQdAPK94 ++ rm /tmp/tmp.d1qiRcbgNs /tmp/tmp.HSEQdAPK94 ++ return 0 + nodes_count=$'Error: Authentication failed. :\nconnect@src/mongo/shell/mongo.js:374:17\n@(connect):2:6\nexception: connect failed\nexiting with code 1\n5' + echo -n 'waiting for all members to be configured in cfg' waiting for all members to be configured in cfg+ let retry+=1 + '[' 1 -ge 15 ']' + echo . . + sleep 10 + [[ Error: Authentication failed. : connect@src/mongo/shell/mongo.js:374:17 @(connect):2:6 exception: connect failed exiting with code 1 5 == 6 ]] ++ run_mongos 'rs.conf().members.length' clusterAdmin:clusterAdmin123456@34.118.238.71 mongodb :27017 ++ local 'command=rs.conf().members.length' ++ local uri=clusterAdmin:clusterAdmin123456@34.118.238.71 ++ local driver=mongodb ++ local suffix=:27017 ++ local mongo_flag= ++ local port=27017 ++ local mongo_bin=mongo ++ grep -E -v 'I NETWORK|W NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|bye' ++ /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/' +++ echo :27017 +++ awk -F: '{print $2}' ++ suffix_port=27017 ++ [[ -z 27017 ]] +++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.XhLVXzNn9g ++++ mktemp +++ local LAST_ERR=/tmp/tmp.ZW2SUxpYhh +++ local exit_status=0 +++ local timeout=4 +++ args=('get' 'pods' '--selector=name=psmdb-client' '-o' 'jsonpath={.items[].metadata.name}') +++ local args +++ transformed_files=() +++ local transformed_files +++ arch_scheduling_enabled +++ [[ -n '' ]] +++ arch_scheduling_enabled +++ [[ -n '' ]] ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 -a -n 1 ']' +++ break +++ cat /tmp/tmp.XhLVXzNn9g +++ cat /tmp/tmp.ZW2SUxpYhh +++ rm /tmp/tmp.XhLVXzNn9g /tmp/tmp.ZW2SUxpYhh +++ return 0 ++ local client_container=psmdb-client-757cd65695-xjsxw ++ kubectl_bin exec psmdb-client-757cd65695-xjsxw -- bash -c 'export HOME=/tmp; printf '\''rs.conf().members.length\n'\'' | mongo mongodb://clusterAdmin:clusterAdmin123456@34.118.238.71:27017/admin ' +++ mktemp ++ local LAST_OUT=/tmp/tmp.VmTn6R5dhl +++ mktemp ++ local LAST_ERR=/tmp/tmp.Z4mXCmUNQ4 ++ local exit_status=0 ++ local timeout=4 ++ args=('exec' 'psmdb-client-757cd65695-xjsxw' '--' 'bash' '-c' 'export HOME=/tmp; printf '\''rs.conf().members.length\n'\'' | mongo mongodb://clusterAdmin:clusterAdmin123456@34.118.238.71:27017/admin ') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl exec psmdb-client-757cd65695-xjsxw -- bash -c 'export HOME=/tmp; printf '\''rs.conf().members.length\n'\'' | mongo mongodb://clusterAdmin:clusterAdmin123456@34.118.238.71:27017/admin ' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.VmTn6R5dhl ++ cat /tmp/tmp.Z4mXCmUNQ4 ++ rm /tmp/tmp.VmTn6R5dhl /tmp/tmp.Z4mXCmUNQ4 ++ return 0 + nodes_count=5 + echo -n 'waiting for all members to be configured in cfg' waiting for all members to be configured in cfg+ let retry+=1 + '[' 2 -ge 15 ']' + echo . . + sleep 10 + [[ 5 == 6 ]] ++ run_mongos 'rs.conf().members.length' clusterAdmin:clusterAdmin123456@34.118.238.71 mongodb :27017 ++ local 'command=rs.conf().members.length' ++ local uri=clusterAdmin:clusterAdmin123456@34.118.238.71 ++ local driver=mongodb ++ local suffix=:27017 ++ local mongo_flag= ++ local port=27017 ++ local mongo_bin=mongo ++ grep -E -v 'I NETWORK|W NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|bye' ++ /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/' +++ echo :27017 +++ awk -F: '{print $2}' ++ suffix_port=27017 ++ [[ -z 27017 ]] +++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.vhbjbpTNwi ++++ mktemp +++ local LAST_ERR=/tmp/tmp.8u1mOr5ihF +++ local exit_status=0 +++ local timeout=4 +++ args=('get' 'pods' '--selector=name=psmdb-client' '-o' 'jsonpath={.items[].metadata.name}') +++ local args +++ transformed_files=() +++ local transformed_files +++ arch_scheduling_enabled +++ [[ -n '' ]] +++ arch_scheduling_enabled +++ [[ -n '' ]] ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 -a -n 1 ']' +++ break +++ cat /tmp/tmp.vhbjbpTNwi +++ cat /tmp/tmp.8u1mOr5ihF +++ rm /tmp/tmp.vhbjbpTNwi /tmp/tmp.8u1mOr5ihF +++ return 0 ++ local client_container=psmdb-client-757cd65695-xjsxw ++ kubectl_bin exec psmdb-client-757cd65695-xjsxw -- bash -c 'export HOME=/tmp; printf '\''rs.conf().members.length\n'\'' | mongo mongodb://clusterAdmin:clusterAdmin123456@34.118.238.71:27017/admin ' +++ mktemp ++ local LAST_OUT=/tmp/tmp.7AnMkgG191 +++ mktemp ++ local LAST_ERR=/tmp/tmp.oj3Yge4e3r ++ local exit_status=0 ++ local timeout=4 ++ args=('exec' 'psmdb-client-757cd65695-xjsxw' '--' 'bash' '-c' 'export HOME=/tmp; printf '\''rs.conf().members.length\n'\'' | mongo mongodb://clusterAdmin:clusterAdmin123456@34.118.238.71:27017/admin ') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl exec psmdb-client-757cd65695-xjsxw -- bash -c 'export HOME=/tmp; printf '\''rs.conf().members.length\n'\'' | mongo mongodb://clusterAdmin:clusterAdmin123456@34.118.238.71:27017/admin ' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.7AnMkgG191 ++ cat /tmp/tmp.oj3Yge4e3r ++ rm /tmp/tmp.7AnMkgG191 /tmp/tmp.oj3Yge4e3r ++ return 0 + nodes_count=5 + echo -n 'waiting for all members to be configured in cfg' waiting for all members to be configured in cfg+ let retry+=1 + '[' 3 -ge 15 ']' + echo . . + sleep 10 + [[ 5 == 6 ]] ++ run_mongos 'rs.conf().members.length' clusterAdmin:clusterAdmin123456@34.118.238.71 mongodb :27017 ++ local 'command=rs.conf().members.length' ++ local uri=clusterAdmin:clusterAdmin123456@34.118.238.71 ++ local driver=mongodb ++ local suffix=:27017 ++ local mongo_flag= ++ local port=27017 ++ local mongo_bin=mongo ++ grep -E -v 'I NETWORK|W NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|bye' ++ /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/' +++ echo :27017 +++ awk -F: '{print $2}' ++ suffix_port=27017 ++ [[ -z 27017 ]] +++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.dWTNH1H6nn ++++ mktemp +++ local LAST_ERR=/tmp/tmp.a4BiYRZvO1 +++ local exit_status=0 +++ local timeout=4 +++ args=('get' 'pods' '--selector=name=psmdb-client' '-o' 'jsonpath={.items[].metadata.name}') +++ local args +++ transformed_files=() +++ local transformed_files +++ arch_scheduling_enabled +++ [[ -n '' ]] +++ arch_scheduling_enabled +++ [[ -n '' ]] ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 -a -n 1 ']' +++ break +++ cat /tmp/tmp.dWTNH1H6nn +++ cat /tmp/tmp.a4BiYRZvO1 +++ rm /tmp/tmp.dWTNH1H6nn /tmp/tmp.a4BiYRZvO1 +++ return 0 ++ local client_container=psmdb-client-757cd65695-xjsxw ++ kubectl_bin exec psmdb-client-757cd65695-xjsxw -- bash -c 'export HOME=/tmp; printf '\''rs.conf().members.length\n'\'' | mongo mongodb://clusterAdmin:clusterAdmin123456@34.118.238.71:27017/admin ' +++ mktemp ++ local LAST_OUT=/tmp/tmp.I9Fat4FNBo +++ mktemp ++ local LAST_ERR=/tmp/tmp.8sV2sWHP2D ++ local exit_status=0 ++ local timeout=4 ++ args=('exec' 'psmdb-client-757cd65695-xjsxw' '--' 'bash' '-c' 'export HOME=/tmp; printf '\''rs.conf().members.length\n'\'' | mongo mongodb://clusterAdmin:clusterAdmin123456@34.118.238.71:27017/admin ') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl exec psmdb-client-757cd65695-xjsxw -- bash -c 'export HOME=/tmp; printf '\''rs.conf().members.length\n'\'' | mongo mongodb://clusterAdmin:clusterAdmin123456@34.118.238.71:27017/admin ' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.I9Fat4FNBo ++ cat /tmp/tmp.8sV2sWHP2D ++ rm /tmp/tmp.I9Fat4FNBo /tmp/tmp.8sV2sWHP2D ++ return 0 + nodes_count=5 + echo -n 'waiting for all members to be configured in cfg' waiting for all members to be configured in cfg+ let retry+=1 + '[' 4 -ge 15 ']' + echo . . + sleep 10 + [[ 5 == 6 ]] ++ run_mongos 'rs.conf().members.length' clusterAdmin:clusterAdmin123456@34.118.238.71 mongodb :27017 ++ local 'command=rs.conf().members.length' ++ local uri=clusterAdmin:clusterAdmin123456@34.118.238.71 ++ local driver=mongodb ++ local suffix=:27017 ++ local mongo_flag= ++ local port=27017 ++ local mongo_bin=mongo ++ grep -E -v 'I NETWORK|W NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|bye' ++ /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/' +++ echo :27017 +++ awk -F: '{print $2}' ++ suffix_port=27017 ++ [[ -z 27017 ]] +++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.rVFN2CKRKX ++++ mktemp +++ local LAST_ERR=/tmp/tmp.lFSXJmlZBU +++ local exit_status=0 +++ local timeout=4 +++ args=('get' 'pods' '--selector=name=psmdb-client' '-o' 'jsonpath={.items[].metadata.name}') +++ local args +++ transformed_files=() +++ local transformed_files +++ arch_scheduling_enabled +++ [[ -n '' ]] +++ arch_scheduling_enabled +++ [[ -n '' ]] ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 -a -n 1 ']' +++ break +++ cat /tmp/tmp.rVFN2CKRKX +++ cat /tmp/tmp.lFSXJmlZBU +++ rm /tmp/tmp.rVFN2CKRKX /tmp/tmp.lFSXJmlZBU +++ return 0 ++ local client_container=psmdb-client-757cd65695-xjsxw ++ kubectl_bin exec psmdb-client-757cd65695-xjsxw -- bash -c 'export HOME=/tmp; printf '\''rs.conf().members.length\n'\'' | mongo mongodb://clusterAdmin:clusterAdmin123456@34.118.238.71:27017/admin ' +++ mktemp ++ local LAST_OUT=/tmp/tmp.RlLdqa32vw +++ mktemp ++ local LAST_ERR=/tmp/tmp.QO388Gh5w6 ++ local exit_status=0 ++ local timeout=4 ++ args=('exec' 'psmdb-client-757cd65695-xjsxw' '--' 'bash' '-c' 'export HOME=/tmp; printf '\''rs.conf().members.length\n'\'' | mongo mongodb://clusterAdmin:clusterAdmin123456@34.118.238.71:27017/admin ') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl exec psmdb-client-757cd65695-xjsxw -- bash -c 'export HOME=/tmp; printf '\''rs.conf().members.length\n'\'' | mongo mongodb://clusterAdmin:clusterAdmin123456@34.118.238.71:27017/admin ' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.RlLdqa32vw ++ cat /tmp/tmp.QO388Gh5w6 ++ rm /tmp/tmp.RlLdqa32vw /tmp/tmp.QO388Gh5w6 ++ return 0 + nodes_count=5 + echo -n 'waiting for all members to be configured in cfg' waiting for all members to be configured in cfg+ let retry+=1 + '[' 5 -ge 15 ']' + echo . . + sleep 10 + [[ 5 == 6 ]] ++ run_mongos 'rs.conf().members.length' clusterAdmin:clusterAdmin123456@34.118.238.71 mongodb :27017 ++ local 'command=rs.conf().members.length' ++ local uri=clusterAdmin:clusterAdmin123456@34.118.238.71 ++ local driver=mongodb ++ local suffix=:27017 ++ local mongo_flag= ++ local port=27017 ++ local mongo_bin=mongo ++ grep -E -v 'I NETWORK|W NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|bye' ++ /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/' +++ echo :27017 +++ awk -F: '{print $2}' ++ suffix_port=27017 ++ [[ -z 27017 ]] +++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.lFsD0vQxGb ++++ mktemp +++ local LAST_ERR=/tmp/tmp.RAGkwcSiWS +++ local exit_status=0 +++ local timeout=4 +++ args=('get' 'pods' '--selector=name=psmdb-client' '-o' 'jsonpath={.items[].metadata.name}') +++ local args +++ transformed_files=() +++ local transformed_files +++ arch_scheduling_enabled +++ [[ -n '' ]] +++ arch_scheduling_enabled +++ [[ -n '' ]] ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 -a -n 1 ']' +++ break +++ cat /tmp/tmp.lFsD0vQxGb +++ cat /tmp/tmp.RAGkwcSiWS +++ rm /tmp/tmp.lFsD0vQxGb /tmp/tmp.RAGkwcSiWS +++ return 0 ++ local client_container=psmdb-client-757cd65695-xjsxw ++ kubectl_bin exec psmdb-client-757cd65695-xjsxw -- bash -c 'export HOME=/tmp; printf '\''rs.conf().members.length\n'\'' | mongo mongodb://clusterAdmin:clusterAdmin123456@34.118.238.71:27017/admin ' +++ mktemp ++ local LAST_OUT=/tmp/tmp.iM4M1W7mBQ +++ mktemp ++ local LAST_ERR=/tmp/tmp.NCFHufTYh6 ++ local exit_status=0 ++ local timeout=4 ++ args=('exec' 'psmdb-client-757cd65695-xjsxw' '--' 'bash' '-c' 'export HOME=/tmp; printf '\''rs.conf().members.length\n'\'' | mongo mongodb://clusterAdmin:clusterAdmin123456@34.118.238.71:27017/admin ') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl exec psmdb-client-757cd65695-xjsxw -- bash -c 'export HOME=/tmp; printf '\''rs.conf().members.length\n'\'' | mongo mongodb://clusterAdmin:clusterAdmin123456@34.118.238.71:27017/admin ' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.iM4M1W7mBQ ++ cat /tmp/tmp.NCFHufTYh6 ++ rm /tmp/tmp.iM4M1W7mBQ /tmp/tmp.NCFHufTYh6 ++ return 0 + nodes_count=5 + echo -n 'waiting for all members to be configured in cfg' waiting for all members to be configured in cfg+ let retry+=1 + '[' 6 -ge 15 ']' + echo . . + sleep 10 + [[ 5 == 6 ]] ++ run_mongos 'rs.conf().members.length' clusterAdmin:clusterAdmin123456@34.118.238.71 mongodb :27017 ++ local 'command=rs.conf().members.length' ++ local uri=clusterAdmin:clusterAdmin123456@34.118.238.71 ++ local driver=mongodb ++ local suffix=:27017 ++ local mongo_flag= ++ local port=27017 ++ local mongo_bin=mongo ++ grep -E -v 'I NETWORK|W NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|bye' ++ /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/' +++ echo :27017 +++ awk -F: '{print $2}' ++ suffix_port=27017 ++ [[ -z 27017 ]] +++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.oPmD1NzhMd ++++ mktemp +++ local LAST_ERR=/tmp/tmp.vBYhcpNkVw +++ local exit_status=0 +++ local timeout=4 +++ args=('get' 'pods' '--selector=name=psmdb-client' '-o' 'jsonpath={.items[].metadata.name}') +++ local args +++ transformed_files=() +++ local transformed_files +++ arch_scheduling_enabled +++ [[ -n '' ]] +++ arch_scheduling_enabled +++ [[ -n '' ]] ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 -a -n 1 ']' +++ break +++ cat /tmp/tmp.oPmD1NzhMd +++ cat /tmp/tmp.vBYhcpNkVw +++ rm /tmp/tmp.oPmD1NzhMd /tmp/tmp.vBYhcpNkVw +++ return 0 ++ local client_container=psmdb-client-757cd65695-xjsxw ++ kubectl_bin exec psmdb-client-757cd65695-xjsxw -- bash -c 'export HOME=/tmp; printf '\''rs.conf().members.length\n'\'' | mongo mongodb://clusterAdmin:clusterAdmin123456@34.118.238.71:27017/admin ' +++ mktemp ++ local LAST_OUT=/tmp/tmp.v9CY27qxGq +++ mktemp ++ local LAST_ERR=/tmp/tmp.HgH2oqa9Ye ++ local exit_status=0 ++ local timeout=4 ++ args=('exec' 'psmdb-client-757cd65695-xjsxw' '--' 'bash' '-c' 'export HOME=/tmp; printf '\''rs.conf().members.length\n'\'' | mongo mongodb://clusterAdmin:clusterAdmin123456@34.118.238.71:27017/admin ') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl exec psmdb-client-757cd65695-xjsxw -- bash -c 'export HOME=/tmp; printf '\''rs.conf().members.length\n'\'' | mongo mongodb://clusterAdmin:clusterAdmin123456@34.118.238.71:27017/admin ' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.v9CY27qxGq ++ cat /tmp/tmp.HgH2oqa9Ye ++ rm /tmp/tmp.v9CY27qxGq /tmp/tmp.HgH2oqa9Ye ++ return 0 + nodes_count=5 + echo -n 'waiting for all members to be configured in cfg' waiting for all members to be configured in cfg+ let retry+=1 + '[' 7 -ge 15 ']' + echo . . + sleep 10 + [[ 5 == 6 ]] ++ run_mongos 'rs.conf().members.length' clusterAdmin:clusterAdmin123456@34.118.238.71 mongodb :27017 ++ local 'command=rs.conf().members.length' ++ local uri=clusterAdmin:clusterAdmin123456@34.118.238.71 ++ local driver=mongodb ++ local suffix=:27017 ++ local mongo_flag= ++ local port=27017 ++ local mongo_bin=mongo ++ grep -E -v 'I NETWORK|W NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|bye' ++ /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/' +++ echo :27017 +++ awk -F: '{print $2}' ++ suffix_port=27017 ++ [[ -z 27017 ]] +++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.FtcrCoKXr3 ++++ mktemp +++ local LAST_ERR=/tmp/tmp.mFI6IBf6Bz +++ local exit_status=0 +++ local timeout=4 +++ args=('get' 'pods' '--selector=name=psmdb-client' '-o' 'jsonpath={.items[].metadata.name}') +++ local args +++ transformed_files=() +++ local transformed_files +++ arch_scheduling_enabled +++ [[ -n '' ]] +++ arch_scheduling_enabled +++ [[ -n '' ]] ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 -a -n 1 ']' +++ break +++ cat /tmp/tmp.FtcrCoKXr3 +++ cat /tmp/tmp.mFI6IBf6Bz +++ rm /tmp/tmp.FtcrCoKXr3 /tmp/tmp.mFI6IBf6Bz +++ return 0 ++ local client_container=psmdb-client-757cd65695-xjsxw ++ kubectl_bin exec psmdb-client-757cd65695-xjsxw -- bash -c 'export HOME=/tmp; printf '\''rs.conf().members.length\n'\'' | mongo mongodb://clusterAdmin:clusterAdmin123456@34.118.238.71:27017/admin ' +++ mktemp ++ local LAST_OUT=/tmp/tmp.Zp6CsqZJwF +++ mktemp ++ local LAST_ERR=/tmp/tmp.rLxgLzMaS5 ++ local exit_status=0 ++ local timeout=4 ++ args=('exec' 'psmdb-client-757cd65695-xjsxw' '--' 'bash' '-c' 'export HOME=/tmp; printf '\''rs.conf().members.length\n'\'' | mongo mongodb://clusterAdmin:clusterAdmin123456@34.118.238.71:27017/admin ') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl exec psmdb-client-757cd65695-xjsxw -- bash -c 'export HOME=/tmp; printf '\''rs.conf().members.length\n'\'' | mongo mongodb://clusterAdmin:clusterAdmin123456@34.118.238.71:27017/admin ' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.Zp6CsqZJwF ++ cat /tmp/tmp.rLxgLzMaS5 ++ rm /tmp/tmp.Zp6CsqZJwF /tmp/tmp.rLxgLzMaS5 ++ return 0 + nodes_count=5 + echo -n 'waiting for all members to be configured in cfg' waiting for all members to be configured in cfg+ let retry+=1 + '[' 8 -ge 15 ']' + echo . . + sleep 10 + [[ 5 == 6 ]] ++ run_mongos 'rs.conf().members.length' clusterAdmin:clusterAdmin123456@34.118.238.71 mongodb :27017 ++ local 'command=rs.conf().members.length' ++ local uri=clusterAdmin:clusterAdmin123456@34.118.238.71 ++ local driver=mongodb ++ local suffix=:27017 ++ local mongo_flag= ++ local port=27017 ++ local mongo_bin=mongo ++ /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/' ++ grep -E -v 'I NETWORK|W NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|bye' +++ echo :27017 +++ awk -F: '{print $2}' ++ suffix_port=27017 ++ [[ -z 27017 ]] +++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.tcAnCQqEl9 ++++ mktemp +++ local LAST_ERR=/tmp/tmp.0cYI4aVIxV +++ local exit_status=0 +++ local timeout=4 +++ args=('get' 'pods' '--selector=name=psmdb-client' '-o' 'jsonpath={.items[].metadata.name}') +++ local args +++ transformed_files=() +++ local transformed_files +++ arch_scheduling_enabled +++ [[ -n '' ]] +++ arch_scheduling_enabled +++ [[ -n '' ]] ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 -a -n 1 ']' +++ break +++ cat /tmp/tmp.tcAnCQqEl9 +++ cat /tmp/tmp.0cYI4aVIxV +++ rm /tmp/tmp.tcAnCQqEl9 /tmp/tmp.0cYI4aVIxV +++ return 0 ++ local client_container=psmdb-client-757cd65695-xjsxw ++ kubectl_bin exec psmdb-client-757cd65695-xjsxw -- bash -c 'export HOME=/tmp; printf '\''rs.conf().members.length\n'\'' | mongo mongodb://clusterAdmin:clusterAdmin123456@34.118.238.71:27017/admin ' +++ mktemp ++ local LAST_OUT=/tmp/tmp.eEMLnWWu7l +++ mktemp ++ local LAST_ERR=/tmp/tmp.2DSi07wgKo ++ local exit_status=0 ++ local timeout=4 ++ args=('exec' 'psmdb-client-757cd65695-xjsxw' '--' 'bash' '-c' 'export HOME=/tmp; printf '\''rs.conf().members.length\n'\'' | mongo mongodb://clusterAdmin:clusterAdmin123456@34.118.238.71:27017/admin ') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl exec psmdb-client-757cd65695-xjsxw -- bash -c 'export HOME=/tmp; printf '\''rs.conf().members.length\n'\'' | mongo mongodb://clusterAdmin:clusterAdmin123456@34.118.238.71:27017/admin ' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.eEMLnWWu7l ++ cat /tmp/tmp.2DSi07wgKo ++ rm /tmp/tmp.eEMLnWWu7l /tmp/tmp.2DSi07wgKo ++ return 0 + nodes_count=6 + echo -n 'waiting for all members to be configured in cfg' waiting for all members to be configured in cfg+ let retry+=1 + '[' 9 -ge 15 ']' + echo . . + sleep 10 + [[ 6 == 6 ]] + wait_for_members 34.118.232.18 rs0 6 + local endpoint=34.118.232.18 + local rsName=rs0 + local target_count=6 + local nodes_count=0 + [[ 0 == 6 ]] ++ run_mongos 'rs.conf().members.length' clusterAdmin:clusterAdmin123456@34.118.232.18 mongodb :27017 ++ local 'command=rs.conf().members.length' ++ local uri=clusterAdmin:clusterAdmin123456@34.118.232.18 ++ local driver=mongodb ++ local suffix=:27017 ++ local mongo_flag= ++ local port=27017 ++ grep -E -v 'I NETWORK|W NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|bye' ++ local mongo_bin=mongo ++ /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/' +++ echo :27017 +++ awk -F: '{print $2}' ++ suffix_port=27017 ++ [[ -z 27017 ]] +++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.JQdB2XsZPW ++++ mktemp +++ local LAST_ERR=/tmp/tmp.wk2LWN6xsp +++ local exit_status=0 +++ local timeout=4 +++ args=('get' 'pods' '--selector=name=psmdb-client' '-o' 'jsonpath={.items[].metadata.name}') +++ local args +++ transformed_files=() +++ local transformed_files +++ arch_scheduling_enabled +++ [[ -n '' ]] +++ arch_scheduling_enabled +++ [[ -n '' ]] ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 -a -n 1 ']' +++ break +++ cat /tmp/tmp.JQdB2XsZPW +++ cat /tmp/tmp.wk2LWN6xsp +++ rm /tmp/tmp.JQdB2XsZPW /tmp/tmp.wk2LWN6xsp +++ return 0 ++ local client_container=psmdb-client-757cd65695-xjsxw ++ kubectl_bin exec psmdb-client-757cd65695-xjsxw -- bash -c 'export HOME=/tmp; printf '\''rs.conf().members.length\n'\'' | mongo mongodb://clusterAdmin:clusterAdmin123456@34.118.232.18:27017/admin ' +++ mktemp ++ local LAST_OUT=/tmp/tmp.S3jbfAPyTG +++ mktemp ++ local LAST_ERR=/tmp/tmp.lFWKqXiGJE ++ local exit_status=0 ++ local timeout=4 ++ args=('exec' 'psmdb-client-757cd65695-xjsxw' '--' 'bash' '-c' 'export HOME=/tmp; printf '\''rs.conf().members.length\n'\'' | mongo mongodb://clusterAdmin:clusterAdmin123456@34.118.232.18:27017/admin ') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl exec psmdb-client-757cd65695-xjsxw -- bash -c 'export HOME=/tmp; printf '\''rs.conf().members.length\n'\'' | mongo mongodb://clusterAdmin:clusterAdmin123456@34.118.232.18:27017/admin ' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.S3jbfAPyTG ++ cat /tmp/tmp.lFWKqXiGJE ++ rm /tmp/tmp.S3jbfAPyTG /tmp/tmp.lFWKqXiGJE ++ return 0 + nodes_count=6 + echo -n 'waiting for all members to be configured in rs0' waiting for all members to be configured in rs0+ let retry+=1 + '[' 10 -ge 15 ']' + echo . . + sleep 10 + [[ 6 == 6 ]] + wait_for_members 34.118.231.170 rs1 6 + local endpoint=34.118.231.170 + local rsName=rs1 + local target_count=6 + local nodes_count=0 + [[ 0 == 6 ]] ++ run_mongos 'rs.conf().members.length' clusterAdmin:clusterAdmin123456@34.118.231.170 mongodb :27017 ++ grep -E -v 'I NETWORK|W NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|bye' ++ local 'command=rs.conf().members.length' ++ local uri=clusterAdmin:clusterAdmin123456@34.118.231.170 ++ local driver=mongodb ++ local suffix=:27017 ++ local mongo_flag= ++ local port=27017 ++ local mongo_bin=mongo ++ /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/' +++ echo :27017 +++ awk -F: '{print $2}' ++ suffix_port=27017 ++ [[ -z 27017 ]] +++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.tYtTqEumzw ++++ mktemp +++ local LAST_ERR=/tmp/tmp.kg3fzeIryl +++ local exit_status=0 +++ local timeout=4 +++ args=('get' 'pods' '--selector=name=psmdb-client' '-o' 'jsonpath={.items[].metadata.name}') +++ local args +++ transformed_files=() +++ local transformed_files +++ arch_scheduling_enabled +++ [[ -n '' ]] +++ arch_scheduling_enabled +++ [[ -n '' ]] ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 -a -n 1 ']' +++ break +++ cat /tmp/tmp.tYtTqEumzw +++ cat /tmp/tmp.kg3fzeIryl +++ rm /tmp/tmp.tYtTqEumzw /tmp/tmp.kg3fzeIryl +++ return 0 ++ local client_container=psmdb-client-757cd65695-xjsxw ++ kubectl_bin exec psmdb-client-757cd65695-xjsxw -- bash -c 'export HOME=/tmp; printf '\''rs.conf().members.length\n'\'' | mongo mongodb://clusterAdmin:clusterAdmin123456@34.118.231.170:27017/admin ' +++ mktemp ++ local LAST_OUT=/tmp/tmp.JvWQl3SKnM +++ mktemp ++ local LAST_ERR=/tmp/tmp.rkaKfMxVT8 ++ local exit_status=0 ++ local timeout=4 ++ args=('exec' 'psmdb-client-757cd65695-xjsxw' '--' 'bash' '-c' 'export HOME=/tmp; printf '\''rs.conf().members.length\n'\'' | mongo mongodb://clusterAdmin:clusterAdmin123456@34.118.231.170:27017/admin ') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl exec psmdb-client-757cd65695-xjsxw -- bash -c 'export HOME=/tmp; printf '\''rs.conf().members.length\n'\'' | mongo mongodb://clusterAdmin:clusterAdmin123456@34.118.231.170:27017/admin ' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.JvWQl3SKnM ++ cat /tmp/tmp.rkaKfMxVT8 ++ rm /tmp/tmp.JvWQl3SKnM /tmp/tmp.rkaKfMxVT8 ++ return 0 + nodes_count=6 + echo -n 'waiting for all members to be configured in rs1' waiting for all members to be configured in rs1+ let retry+=1 + '[' 11 -ge 15 ']' + echo . . + sleep 10 + [[ 6 == 6 ]] ++ kubectl_bin config current-context +++ mktemp ++ local LAST_OUT=/tmp/tmp.xdI0UyvOcY +++ mktemp ++ local LAST_ERR=/tmp/tmp.PPxnELPRKT ++ local exit_status=0 ++ local timeout=4 ++ args=('config' 'current-context') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl config current-context ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.xdI0UyvOcY ++ cat /tmp/tmp.PPxnELPRKT ++ rm /tmp/tmp.xdI0UyvOcY /tmp/tmp.PPxnELPRKT ++ return 0 + kubectl_bin config set-context gke_cloud-dev-112233_us-central1_jen-psmdb-2058-78121c567-22-cluster5 --namespace=cross-site-sharded-replica-16001 ++ mktemp + local LAST_OUT=/tmp/tmp.DMb0uwioEq ++ mktemp + local LAST_ERR=/tmp/tmp.1lDcYaLw0I + local exit_status=0 + local timeout=4 + args=('config' 'set-context' 'gke_cloud-dev-112233_us-central1_jen-psmdb-2058-78121c567-22-cluster5' '--namespace=cross-site-sharded-replica-16001') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl config set-context gke_cloud-dev-112233_us-central1_jen-psmdb-2058-78121c567-22-cluster5 --namespace=cross-site-sharded-replica-16001 + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.DMb0uwioEq Context "gke_cloud-dev-112233_us-central1_jen-psmdb-2058-78121c567-22-cluster5" modified. + cat /tmp/tmp.1lDcYaLw0I + rm /tmp/tmp.DMb0uwioEq /tmp/tmp.1lDcYaLw0I + return 0 + desc 'check if all 3 Pods started' + set +o xtrace ----------------------------------------------------------------------------------- check if all 3 Pods started ----------------------------------------------------------------------------------- + wait_for_running cross-site-sharded-replica-rs0 3 + local name=cross-site-sharded-replica-rs0 + let last_pod=2 + local check_cluster_readiness=true + set_debug + [[ 1 == 1 ]] + set -o xtrace + local rs_name=rs0 + local cluster_name=cross-site-sharded-replica ++ seq 0 2 + for i in $(seq 0 $last_pod) + [[ 0 -eq 2 ]] + wait_pod cross-site-sharded-replica-rs0-0 + local pod=cross-site-sharded-replica-rs0-0 + set +o xtrace waiting for pod/cross-site-sharded-replica-rs0-0 to be ready.OK + for i in $(seq 0 $last_pod) + [[ 1 -eq 2 ]] + wait_pod cross-site-sharded-replica-rs0-1 + local pod=cross-site-sharded-replica-rs0-1 + set +o xtrace waiting for pod/cross-site-sharded-replica-rs0-1 to be ready.OK + for i in $(seq 0 $last_pod) + [[ 2 -eq 2 ]] ++ kubectl_bin get psmdb cross-site-sharded-replica -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].arbiter.enabled}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.wvJsoQ5o1w +++ mktemp ++ local LAST_ERR=/tmp/tmp.zskWFpmxm6 ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'psmdb' 'cross-site-sharded-replica' '-o' 'jsonpath={.spec.replsets[?(@.name=="rs0")].arbiter.enabled}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb cross-site-sharded-replica -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].arbiter.enabled}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.wvJsoQ5o1w ++ cat /tmp/tmp.zskWFpmxm6 ++ rm /tmp/tmp.wvJsoQ5o1w /tmp/tmp.zskWFpmxm6 ++ return 0 + [[ false == true ]] + wait_pod cross-site-sharded-replica-rs0-2 + local pod=cross-site-sharded-replica-rs0-2 + set +o xtrace waiting for pod/cross-site-sharded-replica-rs0-2 to be ready.OK ++ kubectl_bin get psmdb cross-site-sharded-replica -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].nonvoting.enabled}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.jJXuHYEpSq +++ mktemp ++ local LAST_ERR=/tmp/tmp.sVU6Th8g3F ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'psmdb' 'cross-site-sharded-replica' '-o' 'jsonpath={.spec.replsets[?(@.name=="rs0")].nonvoting.enabled}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb cross-site-sharded-replica -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].nonvoting.enabled}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.jJXuHYEpSq ++ cat /tmp/tmp.sVU6Th8g3F ++ rm /tmp/tmp.jJXuHYEpSq /tmp/tmp.sVU6Th8g3F ++ return 0 + [[ false == true ]] ++ kubectl_bin get psmdb cross-site-sharded-replica -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].hidden.enabled}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.VydeBrz6U3 +++ mktemp ++ local LAST_ERR=/tmp/tmp.QRQB1WPyLC ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'psmdb' 'cross-site-sharded-replica' '-o' 'jsonpath={.spec.replsets[?(@.name=="rs0")].hidden.enabled}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb cross-site-sharded-replica -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].hidden.enabled}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.VydeBrz6U3 ++ cat /tmp/tmp.QRQB1WPyLC ++ rm /tmp/tmp.VydeBrz6U3 /tmp/tmp.QRQB1WPyLC ++ return 0 + [[ '' == true ]] + sleep 10 + [[ true == true ]] + set +x Waiting for cluster cross-site-sharded-replica to be ready + wait_for_running cross-site-sharded-replica-cfg 3 false + local name=cross-site-sharded-replica-cfg + let last_pod=2 + local check_cluster_readiness=false + set_debug + [[ 1 == 1 ]] + set -o xtrace + local rs_name=cfg + local cluster_name=cross-site-sharded-replica ++ seq 0 2 + for i in $(seq 0 $last_pod) + [[ 0 -eq 2 ]] + wait_pod cross-site-sharded-replica-cfg-0 + local pod=cross-site-sharded-replica-cfg-0 + set +o xtrace waiting for pod/cross-site-sharded-replica-cfg-0 to be ready.OK + for i in $(seq 0 $last_pod) + [[ 1 -eq 2 ]] + wait_pod cross-site-sharded-replica-cfg-1 + local pod=cross-site-sharded-replica-cfg-1 + set +o xtrace waiting for pod/cross-site-sharded-replica-cfg-1 to be ready.OK + for i in $(seq 0 $last_pod) + [[ 2 -eq 2 ]] ++ kubectl_bin get psmdb cross-site-sharded-replica -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].arbiter.enabled}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.fVM2eL4ApU +++ mktemp ++ local LAST_ERR=/tmp/tmp.Pxr4UsOjSc ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'psmdb' 'cross-site-sharded-replica' '-o' 'jsonpath={.spec.replsets[?(@.name=="cfg")].arbiter.enabled}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb cross-site-sharded-replica -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].arbiter.enabled}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.fVM2eL4ApU ++ cat /tmp/tmp.Pxr4UsOjSc ++ rm /tmp/tmp.fVM2eL4ApU /tmp/tmp.Pxr4UsOjSc ++ return 0 + [[ '' == true ]] + wait_pod cross-site-sharded-replica-cfg-2 + local pod=cross-site-sharded-replica-cfg-2 + set +o xtrace waiting for pod/cross-site-sharded-replica-cfg-2 to be ready.OK ++ kubectl_bin get psmdb cross-site-sharded-replica -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].nonvoting.enabled}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.VefduDBpcT +++ mktemp ++ local LAST_ERR=/tmp/tmp.Nsy0w5F4od ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'psmdb' 'cross-site-sharded-replica' '-o' 'jsonpath={.spec.replsets[?(@.name=="cfg")].nonvoting.enabled}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb cross-site-sharded-replica -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].nonvoting.enabled}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.VefduDBpcT ++ cat /tmp/tmp.Nsy0w5F4od ++ rm /tmp/tmp.VefduDBpcT /tmp/tmp.Nsy0w5F4od ++ return 0 + [[ '' == true ]] ++ kubectl_bin get psmdb cross-site-sharded-replica -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].hidden.enabled}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.M4WiZsMvYl +++ mktemp ++ local LAST_ERR=/tmp/tmp.7g0IxesiMT ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'psmdb' 'cross-site-sharded-replica' '-o' 'jsonpath={.spec.replsets[?(@.name=="cfg")].hidden.enabled}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb cross-site-sharded-replica -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].hidden.enabled}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.M4WiZsMvYl ++ cat /tmp/tmp.7g0IxesiMT ++ rm /tmp/tmp.M4WiZsMvYl /tmp/tmp.7g0IxesiMT ++ return 0 + [[ '' == true ]] + sleep 10 + [[ false == true ]] + desc 'create user' + set +o xtrace ----------------------------------------------------------------------------------- create user ----------------------------------------------------------------------------------- + run_mongos 'db.createUser({user:"myApp",pwd:"myPass",roles:[{db:"myApp",role:"readWrite"}]})' userAdmin:userAdmin123456@cross-site-sharded-main-mongos.cross-site-sharded-12730 + local 'command=db.createUser({user:"myApp",pwd:"myPass",roles:[{db:"myApp",role:"readWrite"}]})' + local uri=userAdmin:userAdmin123456@cross-site-sharded-main-mongos.cross-site-sharded-12730 + local driver=mongodb + local suffix=.svc.cluster.local + local mongo_flag= + local port=27017 + local mongo_bin=mongo ++ echo .svc.cluster.local ++ awk -F: '{print $2}' + suffix_port= + [[ -z '' ]] + suffix=.svc.cluster.local:27017 ++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.jNH9FuoDad +++ mktemp ++ local LAST_ERR=/tmp/tmp.TWsfbQtKtP ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'pods' '--selector=name=psmdb-client' '-o' 'jsonpath={.items[].metadata.name}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.jNH9FuoDad ++ cat /tmp/tmp.TWsfbQtKtP ++ rm /tmp/tmp.jNH9FuoDad /tmp/tmp.TWsfbQtKtP ++ return 0 + local client_container=psmdb-client-757cd65695-5qzl6 + kubectl_bin exec psmdb-client-757cd65695-5qzl6 -- bash -c 'export HOME=/tmp; printf '\''db.createUser({user:"myApp",pwd:"myPass",roles:[{db:"myApp",role:"readWrite"}]})\n'\'' | mongo mongodb://userAdmin:userAdmin123456@cross-site-sharded-main-mongos.cross-site-sharded-12730.svc.cluster.local:27017/admin ' ++ mktemp + local LAST_OUT=/tmp/tmp.8nWWAkDvNI ++ mktemp + local LAST_ERR=/tmp/tmp.jFxCqfMeeb + local exit_status=0 + local timeout=4 + args=('exec' 'psmdb-client-757cd65695-5qzl6' '--' 'bash' '-c' 'export HOME=/tmp; printf '\''db.createUser({user:"myApp",pwd:"myPass",roles:[{db:"myApp",role:"readWrite"}]})\n'\'' | mongo mongodb://userAdmin:userAdmin123456@cross-site-sharded-main-mongos.cross-site-sharded-12730.svc.cluster.local:27017/admin ') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl exec psmdb-client-757cd65695-5qzl6 -- bash -c 'export HOME=/tmp; printf '\''db.createUser({user:"myApp",pwd:"myPass",roles:[{db:"myApp",role:"readWrite"}]})\n'\'' | mongo mongodb://userAdmin:userAdmin123456@cross-site-sharded-main-mongos.cross-site-sharded-12730.svc.cluster.local:27017/admin ' + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.8nWWAkDvNI Percona Server for MongoDB shell version v4.4.29-28 connecting to: mongodb://cross-site-sharded-main-mongos.cross-site-sharded-12730.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb Implicit session: session { "id" : UUID("506a0499-5136-4504-a0c8-8e933906503a") } Percona Server for MongoDB server version: v8.0.26-11 WARNING: shell and server versions do not match Successfully added user: { "user" : "myApp", "roles" : [ { "db" : "myApp", "role" : "readWrite" } ] } bye + cat /tmp/tmp.jFxCqfMeeb + rm /tmp/tmp.8nWWAkDvNI /tmp/tmp.jFxCqfMeeb + return 0 + sleep 2 + desc 'write data, read from all' + set +o xtrace ----------------------------------------------------------------------------------- write data, read from all ----------------------------------------------------------------------------------- + run_mongos 'use myApp\n db.test.insert({ x: 100500 })' myApp:myPass@cross-site-sharded-main-mongos.cross-site-sharded-12730 + local 'command=use myApp\n db.test.insert({ x: 100500 })' + local uri=myApp:myPass@cross-site-sharded-main-mongos.cross-site-sharded-12730 + local driver=mongodb + local suffix=.svc.cluster.local + local mongo_flag= + local port=27017 + local mongo_bin=mongo ++ echo .svc.cluster.local ++ awk -F: '{print $2}' + suffix_port= + [[ -z '' ]] + suffix=.svc.cluster.local:27017 ++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.Rx6jsSS2WK +++ mktemp ++ local LAST_ERR=/tmp/tmp.nEiKaHvKC8 ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'pods' '--selector=name=psmdb-client' '-o' 'jsonpath={.items[].metadata.name}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.Rx6jsSS2WK ++ cat /tmp/tmp.nEiKaHvKC8 ++ rm /tmp/tmp.Rx6jsSS2WK /tmp/tmp.nEiKaHvKC8 ++ return 0 + local client_container=psmdb-client-757cd65695-5qzl6 + kubectl_bin exec psmdb-client-757cd65695-5qzl6 -- bash -c 'export HOME=/tmp; printf '\''use myApp\n db.test.insert({ x: 100500 })\n'\'' | mongo mongodb://myApp:myPass@cross-site-sharded-main-mongos.cross-site-sharded-12730.svc.cluster.local:27017/admin ' ++ mktemp + local LAST_OUT=/tmp/tmp.JWuiFRrBYi ++ mktemp + local LAST_ERR=/tmp/tmp.ooZRflWkBE + local exit_status=0 + local timeout=4 + args=('exec' 'psmdb-client-757cd65695-5qzl6' '--' 'bash' '-c' 'export HOME=/tmp; printf '\''use myApp\n db.test.insert({ x: 100500 })\n'\'' | mongo mongodb://myApp:myPass@cross-site-sharded-main-mongos.cross-site-sharded-12730.svc.cluster.local:27017/admin ') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl exec psmdb-client-757cd65695-5qzl6 -- bash -c 'export HOME=/tmp; printf '\''use myApp\n db.test.insert({ x: 100500 })\n'\'' | mongo mongodb://myApp:myPass@cross-site-sharded-main-mongos.cross-site-sharded-12730.svc.cluster.local:27017/admin ' + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.JWuiFRrBYi Percona Server for MongoDB shell version v4.4.29-28 connecting to: mongodb://cross-site-sharded-main-mongos.cross-site-sharded-12730.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb Implicit session: session { "id" : UUID("3acf7e5d-65dd-4fd5-94f1-8048022d4313") } Percona Server for MongoDB server version: v8.0.26-11 WARNING: shell and server versions do not match switched to db myApp WriteResult({ "nInserted" : 1 }) bye + cat /tmp/tmp.ooZRflWkBE + rm /tmp/tmp.JWuiFRrBYi /tmp/tmp.ooZRflWkBE + return 0 + minikube_sleep + sleep_time=10 + [[ '' == 1 ]] + desc 'Compare data' + set +o xtrace ----------------------------------------------------------------------------------- Compare data ----------------------------------------------------------------------------------- + compare_mongos_cmd find myApp:myPass@cross-site-sharded-main-mongos.cross-site-sharded-12730 + local command=find + local uri=myApp:myPass@cross-site-sharded-main-mongos.cross-site-sharded-12730 + local postfix= + local suffix= + local database=myApp + local collection=test + local port=27017 + local tls=false + [[ false == true ]] + mongos_command=run_mongos + log 'running db.test.command() in myApp' + set +o xtrace [2026-07-17T00:16:50+0000] running db.test.command() in myApp + run_mongos 'use myApp\n db.test.find()' myApp:myPass@cross-site-sharded-main-mongos.cross-site-sharded-12730 mongodb '' '' 27017 + local 'command=use myApp\n db.test.find()' + local uri=myApp:myPass@cross-site-sharded-main-mongos.cross-site-sharded-12730 + local driver=mongodb + local suffix=.svc.cluster.local + local mongo_flag= + local port=27017 + local mongo_bin=mongo + grep -E -v 'I NETWORK|W NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|Started a new thread for the timer service' + /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/' ++ echo .svc.cluster.local ++ awk -F: '{print $2}' + suffix_port= + [[ -z '' ]] + suffix=.svc.cluster.local:27017 ++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.lKPHA8cC1A +++ mktemp ++ local LAST_ERR=/tmp/tmp.mtmHKCAV9e ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'pods' '--selector=name=psmdb-client' '-o' 'jsonpath={.items[].metadata.name}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.lKPHA8cC1A ++ cat /tmp/tmp.mtmHKCAV9e ++ rm /tmp/tmp.lKPHA8cC1A /tmp/tmp.mtmHKCAV9e ++ return 0 + local client_container=psmdb-client-757cd65695-5qzl6 + kubectl_bin exec psmdb-client-757cd65695-5qzl6 -- bash -c 'export HOME=/tmp; printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@cross-site-sharded-main-mongos.cross-site-sharded-12730.svc.cluster.local:27017/admin ' ++ mktemp + local LAST_OUT=/tmp/tmp.XKacHMDZMf ++ mktemp + local LAST_ERR=/tmp/tmp.dRtvPj0qFR + local exit_status=0 + local timeout=4 + args=('exec' 'psmdb-client-757cd65695-5qzl6' '--' 'bash' '-c' 'export HOME=/tmp; printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@cross-site-sharded-main-mongos.cross-site-sharded-12730.svc.cluster.local:27017/admin ') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl exec psmdb-client-757cd65695-5qzl6 -- bash -c 'export HOME=/tmp; printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@cross-site-sharded-main-mongos.cross-site-sharded-12730.svc.cluster.local:27017/admin ' + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.XKacHMDZMf + cat /tmp/tmp.dRtvPj0qFR + rm /tmp/tmp.XKacHMDZMf /tmp/tmp.dRtvPj0qFR + return 0 + [[ 0 -eq 0 ]] + diff /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/e2e-tests/cross-site-sharded/compare/find.json /tmp/tmp.yXIW4gRKyb/find + desc 'test failover' + set +o xtrace ----------------------------------------------------------------------------------- test failover ----------------------------------------------------------------------------------- ++ kubectl_bin config current-context +++ mktemp ++ local LAST_OUT=/tmp/tmp.bTyKUxoigq +++ mktemp ++ local LAST_ERR=/tmp/tmp.VEQWg1xQIV ++ local exit_status=0 ++ local timeout=4 ++ args=('config' 'current-context') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl config current-context ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.bTyKUxoigq ++ cat /tmp/tmp.VEQWg1xQIV ++ rm /tmp/tmp.bTyKUxoigq /tmp/tmp.VEQWg1xQIV ++ return 0 + kubectl_bin config set-context gke_cloud-dev-112233_us-central1_jen-psmdb-2058-78121c567-22-cluster5 --namespace=cross-site-sharded-12730 ++ mktemp + local LAST_OUT=/tmp/tmp.3kvfwnb7gP ++ mktemp + local LAST_ERR=/tmp/tmp.pMVnzN9Adc + local exit_status=0 + local timeout=4 + args=('config' 'set-context' 'gke_cloud-dev-112233_us-central1_jen-psmdb-2058-78121c567-22-cluster5' '--namespace=cross-site-sharded-12730') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl config set-context gke_cloud-dev-112233_us-central1_jen-psmdb-2058-78121c567-22-cluster5 --namespace=cross-site-sharded-12730 + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.3kvfwnb7gP Context "gke_cloud-dev-112233_us-central1_jen-psmdb-2058-78121c567-22-cluster5" modified. + cat /tmp/tmp.pMVnzN9Adc + rm /tmp/tmp.3kvfwnb7gP /tmp/tmp.pMVnzN9Adc + return 0 + kubectl_bin delete psmdb cross-site-sharded-main ++ mktemp + local LAST_OUT=/tmp/tmp.kHYkjKfnyR ++ mktemp + local LAST_ERR=/tmp/tmp.oWShfqima9 + local exit_status=0 + local timeout=4 + args=('delete' 'psmdb' 'cross-site-sharded-main') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl delete psmdb cross-site-sharded-main + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.kHYkjKfnyR perconaservermongodb.psmdb.percona.com "cross-site-sharded-main" deleted from cross-site-sharded-12730 namespace + cat /tmp/tmp.oWShfqima9 + rm /tmp/tmp.kHYkjKfnyR /tmp/tmp.oWShfqima9 + return 0 + desc 'run disaster recovery script for replset: cfg' + set +o xtrace ----------------------------------------------------------------------------------- run disaster recovery script for replset: cfg ----------------------------------------------------------------------------------- + run_script_mongos /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/e2e-tests/cross-site-sharded/disaster_recovery.js clusterAdmin:clusterAdmin123456@34.118.238.71 mongodb :27017 + local script=/mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/e2e-tests/cross-site-sharded/disaster_recovery.js + local uri=clusterAdmin:clusterAdmin123456@34.118.238.71 + local driver=mongodb + local suffix=:27017 + local mongo_flag= + local mongo_bin=mongo ++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.iWT96EsKwC +++ mktemp ++ local LAST_ERR=/tmp/tmp.TBdIAilJ80 ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'pods' '--selector=name=psmdb-client' '-o' 'jsonpath={.items[].metadata.name}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.iWT96EsKwC ++ cat /tmp/tmp.TBdIAilJ80 ++ rm /tmp/tmp.iWT96EsKwC /tmp/tmp.TBdIAilJ80 ++ return 0 + local client_container=psmdb-client-757cd65695-xjsxw ++ basename /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/e2e-tests/cross-site-sharded/disaster_recovery.js + name=disaster_recovery.js + kubectl_bin cp /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/e2e-tests/cross-site-sharded/disaster_recovery.js cross-site-sharded-12730/psmdb-client-757cd65695-xjsxw:/tmp ++ mktemp + local LAST_OUT=/tmp/tmp.TH9XBgBD24 ++ mktemp + local LAST_ERR=/tmp/tmp.yDxA0Lzttt + local exit_status=0 + local timeout=4 + args=('cp' '/mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/e2e-tests/cross-site-sharded/disaster_recovery.js' 'cross-site-sharded-12730/psmdb-client-757cd65695-xjsxw:/tmp') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl cp /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/e2e-tests/cross-site-sharded/disaster_recovery.js cross-site-sharded-12730/psmdb-client-757cd65695-xjsxw:/tmp + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.TH9XBgBD24 + cat /tmp/tmp.yDxA0Lzttt + rm /tmp/tmp.TH9XBgBD24 /tmp/tmp.yDxA0Lzttt + return 0 + kubectl_bin exec psmdb-client-757cd65695-xjsxw -- bash -c 'export HOME=/tmp; mongo mongodb://clusterAdmin:clusterAdmin123456@34.118.238.71:27017/admin /tmp/disaster_recovery.js' ++ mktemp + local LAST_OUT=/tmp/tmp.P02pjxZbe3 ++ mktemp + local LAST_ERR=/tmp/tmp.uMegOVNvu9 + local exit_status=0 + local timeout=4 + args=('exec' 'psmdb-client-757cd65695-xjsxw' '--' 'bash' '-c' 'export HOME=/tmp; mongo mongodb://clusterAdmin:clusterAdmin123456@34.118.238.71:27017/admin /tmp/disaster_recovery.js') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl exec psmdb-client-757cd65695-xjsxw -- bash -c 'export HOME=/tmp; mongo mongodb://clusterAdmin:clusterAdmin123456@34.118.238.71:27017/admin /tmp/disaster_recovery.js' + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.P02pjxZbe3 Percona Server for MongoDB shell version v4.4.29-28 connecting to: mongodb://34.118.238.71:27017/admin?compressors=disabled&gssapiServiceName=mongodb Implicit session: session { "id" : UUID("8ba96112-f21d-45e2-8ed3-0923ada05280") } Percona Server for MongoDB server version: v8.0.26-11 WARNING: shell and server versions do not match + cat /tmp/tmp.uMegOVNvu9 + rm /tmp/tmp.P02pjxZbe3 /tmp/tmp.uMegOVNvu9 + return 0 + desc 'run disaster recovery script for replset: rs0' + set +o xtrace ----------------------------------------------------------------------------------- run disaster recovery script for replset: rs0 ----------------------------------------------------------------------------------- + run_script_mongos /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/e2e-tests/cross-site-sharded/disaster_recovery.js clusterAdmin:clusterAdmin123456@34.118.232.18 mongodb :27017 + local script=/mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/e2e-tests/cross-site-sharded/disaster_recovery.js + local uri=clusterAdmin:clusterAdmin123456@34.118.232.18 + local driver=mongodb + local suffix=:27017 + local mongo_flag= + local mongo_bin=mongo ++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.ggx9xA5Szy +++ mktemp ++ local LAST_ERR=/tmp/tmp.eKXk1csVEb ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'pods' '--selector=name=psmdb-client' '-o' 'jsonpath={.items[].metadata.name}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.ggx9xA5Szy ++ cat /tmp/tmp.eKXk1csVEb ++ rm /tmp/tmp.ggx9xA5Szy /tmp/tmp.eKXk1csVEb ++ return 0 + local client_container=psmdb-client-757cd65695-xjsxw ++ basename /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/e2e-tests/cross-site-sharded/disaster_recovery.js + name=disaster_recovery.js + kubectl_bin cp /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/e2e-tests/cross-site-sharded/disaster_recovery.js cross-site-sharded-12730/psmdb-client-757cd65695-xjsxw:/tmp ++ mktemp + local LAST_OUT=/tmp/tmp.xvQGQyYHjr ++ mktemp + local LAST_ERR=/tmp/tmp.87PBWFU613 + local exit_status=0 + local timeout=4 + args=('cp' '/mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/e2e-tests/cross-site-sharded/disaster_recovery.js' 'cross-site-sharded-12730/psmdb-client-757cd65695-xjsxw:/tmp') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl cp /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/e2e-tests/cross-site-sharded/disaster_recovery.js cross-site-sharded-12730/psmdb-client-757cd65695-xjsxw:/tmp + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.xvQGQyYHjr + cat /tmp/tmp.87PBWFU613 + rm /tmp/tmp.xvQGQyYHjr /tmp/tmp.87PBWFU613 + return 0 + kubectl_bin exec psmdb-client-757cd65695-xjsxw -- bash -c 'export HOME=/tmp; mongo mongodb://clusterAdmin:clusterAdmin123456@34.118.232.18:27017/admin /tmp/disaster_recovery.js' ++ mktemp + local LAST_OUT=/tmp/tmp.dcULLElUlM ++ mktemp + local LAST_ERR=/tmp/tmp.vMEY5wo9IL + local exit_status=0 + local timeout=4 + args=('exec' 'psmdb-client-757cd65695-xjsxw' '--' 'bash' '-c' 'export HOME=/tmp; mongo mongodb://clusterAdmin:clusterAdmin123456@34.118.232.18:27017/admin /tmp/disaster_recovery.js') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl exec psmdb-client-757cd65695-xjsxw -- bash -c 'export HOME=/tmp; mongo mongodb://clusterAdmin:clusterAdmin123456@34.118.232.18:27017/admin /tmp/disaster_recovery.js' + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.dcULLElUlM Percona Server for MongoDB shell version v4.4.29-28 connecting to: mongodb://34.118.232.18:27017/admin?compressors=disabled&gssapiServiceName=mongodb Implicit session: session { "id" : UUID("5bcfafae-965f-48ea-9f31-cac6c296b92b") } Percona Server for MongoDB server version: v8.0.26-11 WARNING: shell and server versions do not match + cat /tmp/tmp.vMEY5wo9IL + rm /tmp/tmp.dcULLElUlM /tmp/tmp.vMEY5wo9IL + return 0 + desc 'run disaster recovery script for replset: rs1' + set +o xtrace ----------------------------------------------------------------------------------- run disaster recovery script for replset: rs1 ----------------------------------------------------------------------------------- + run_script_mongos /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/e2e-tests/cross-site-sharded/disaster_recovery.js clusterAdmin:clusterAdmin123456@34.118.231.170 mongodb :27017 + local script=/mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/e2e-tests/cross-site-sharded/disaster_recovery.js + local uri=clusterAdmin:clusterAdmin123456@34.118.231.170 + local driver=mongodb + local suffix=:27017 + local mongo_flag= + local mongo_bin=mongo ++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.dJAAOlaqm5 +++ mktemp ++ local LAST_ERR=/tmp/tmp.VrGbJsoueF ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'pods' '--selector=name=psmdb-client' '-o' 'jsonpath={.items[].metadata.name}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.dJAAOlaqm5 ++ cat /tmp/tmp.VrGbJsoueF ++ rm /tmp/tmp.dJAAOlaqm5 /tmp/tmp.VrGbJsoueF ++ return 0 + local client_container=psmdb-client-757cd65695-xjsxw ++ basename /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/e2e-tests/cross-site-sharded/disaster_recovery.js + name=disaster_recovery.js + kubectl_bin cp /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/e2e-tests/cross-site-sharded/disaster_recovery.js cross-site-sharded-12730/psmdb-client-757cd65695-xjsxw:/tmp ++ mktemp + local LAST_OUT=/tmp/tmp.sMUVrn1VXB ++ mktemp + local LAST_ERR=/tmp/tmp.FvEkx7n75M + local exit_status=0 + local timeout=4 + args=('cp' '/mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/e2e-tests/cross-site-sharded/disaster_recovery.js' 'cross-site-sharded-12730/psmdb-client-757cd65695-xjsxw:/tmp') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl cp /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/e2e-tests/cross-site-sharded/disaster_recovery.js cross-site-sharded-12730/psmdb-client-757cd65695-xjsxw:/tmp + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.sMUVrn1VXB + cat /tmp/tmp.FvEkx7n75M + rm /tmp/tmp.sMUVrn1VXB /tmp/tmp.FvEkx7n75M + return 0 + kubectl_bin exec psmdb-client-757cd65695-xjsxw -- bash -c 'export HOME=/tmp; mongo mongodb://clusterAdmin:clusterAdmin123456@34.118.231.170:27017/admin /tmp/disaster_recovery.js' ++ mktemp + local LAST_OUT=/tmp/tmp.G9zZl69IcQ ++ mktemp + local LAST_ERR=/tmp/tmp.BCFhX8rmBr + local exit_status=0 + local timeout=4 + args=('exec' 'psmdb-client-757cd65695-xjsxw' '--' 'bash' '-c' 'export HOME=/tmp; mongo mongodb://clusterAdmin:clusterAdmin123456@34.118.231.170:27017/admin /tmp/disaster_recovery.js') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl exec psmdb-client-757cd65695-xjsxw -- bash -c 'export HOME=/tmp; mongo mongodb://clusterAdmin:clusterAdmin123456@34.118.231.170:27017/admin /tmp/disaster_recovery.js' + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.G9zZl69IcQ Percona Server for MongoDB shell version v4.4.29-28 connecting to: mongodb://34.118.231.170:27017/admin?compressors=disabled&gssapiServiceName=mongodb Implicit session: session { "id" : UUID("face1807-8d62-46db-a2f8-2f9b28dd728e") } Percona Server for MongoDB server version: v8.0.26-11 WARNING: shell and server versions do not match + cat /tmp/tmp.BCFhX8rmBr + rm /tmp/tmp.G9zZl69IcQ /tmp/tmp.BCFhX8rmBr + return 0 + desc 'make replica cluster managed' + set +o xtrace ----------------------------------------------------------------------------------- make replica cluster managed ----------------------------------------------------------------------------------- ++ kubectl_bin config current-context +++ mktemp ++ local LAST_OUT=/tmp/tmp.9MkdvWRCIx +++ mktemp ++ local LAST_ERR=/tmp/tmp.oyAiCds79D ++ local exit_status=0 ++ local timeout=4 ++ args=('config' 'current-context') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl config current-context ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.9MkdvWRCIx ++ cat /tmp/tmp.oyAiCds79D ++ rm /tmp/tmp.9MkdvWRCIx /tmp/tmp.oyAiCds79D ++ return 0 + kubectl_bin config set-context gke_cloud-dev-112233_us-central1_jen-psmdb-2058-78121c567-22-cluster5 --namespace=cross-site-sharded-replica-16001 ++ mktemp + local LAST_OUT=/tmp/tmp.PNR9TuRDNU ++ mktemp + local LAST_ERR=/tmp/tmp.ns7i3jaZWU + local exit_status=0 + local timeout=4 + args=('config' 'set-context' 'gke_cloud-dev-112233_us-central1_jen-psmdb-2058-78121c567-22-cluster5' '--namespace=cross-site-sharded-replica-16001') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl config set-context gke_cloud-dev-112233_us-central1_jen-psmdb-2058-78121c567-22-cluster5 --namespace=cross-site-sharded-replica-16001 + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.PNR9TuRDNU Context "gke_cloud-dev-112233_us-central1_jen-psmdb-2058-78121c567-22-cluster5" modified. + cat /tmp/tmp.ns7i3jaZWU + rm /tmp/tmp.PNR9TuRDNU /tmp/tmp.ns7i3jaZWU + return 0 + kubectl_bin patch psmdb cross-site-sharded-replica --type=merge --patch '{"spec":{"unmanaged": false}}' ++ mktemp + local LAST_OUT=/tmp/tmp.eaTpeNipBJ ++ mktemp + local LAST_ERR=/tmp/tmp.UXWUdlqatp + local exit_status=0 + local timeout=4 + args=('patch' 'psmdb' 'cross-site-sharded-replica' '--type=merge' '--patch' '{"spec":{"unmanaged": false}}') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl patch psmdb cross-site-sharded-replica --type=merge --patch '{"spec":{"unmanaged": false}}' + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.eaTpeNipBJ perconaservermongodb.psmdb.percona.com/cross-site-sharded-replica patched + cat /tmp/tmp.UXWUdlqatp + rm /tmp/tmp.eaTpeNipBJ /tmp/tmp.UXWUdlqatp + return 0 + wait_for_running cross-site-sharded-replica-rs0 3 + local name=cross-site-sharded-replica-rs0 + let last_pod=2 + local check_cluster_readiness=true + set_debug + [[ 1 == 1 ]] + set -o xtrace + local rs_name=rs0 + local cluster_name=cross-site-sharded-replica ++ seq 0 2 + for i in $(seq 0 $last_pod) + [[ 0 -eq 2 ]] + wait_pod cross-site-sharded-replica-rs0-0 + local pod=cross-site-sharded-replica-rs0-0 + set +o xtrace waiting for pod/cross-site-sharded-replica-rs0-0 to be ready.OK + for i in $(seq 0 $last_pod) + [[ 1 -eq 2 ]] + wait_pod cross-site-sharded-replica-rs0-1 + local pod=cross-site-sharded-replica-rs0-1 + set +o xtrace waiting for pod/cross-site-sharded-replica-rs0-1 to be ready.OK + for i in $(seq 0 $last_pod) + [[ 2 -eq 2 ]] ++ kubectl_bin get psmdb cross-site-sharded-replica -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].arbiter.enabled}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.3oXx4DE9v2 +++ mktemp ++ local LAST_ERR=/tmp/tmp.F7TpAavHBj ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'psmdb' 'cross-site-sharded-replica' '-o' 'jsonpath={.spec.replsets[?(@.name=="rs0")].arbiter.enabled}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb cross-site-sharded-replica -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].arbiter.enabled}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.3oXx4DE9v2 ++ cat /tmp/tmp.F7TpAavHBj ++ rm /tmp/tmp.3oXx4DE9v2 /tmp/tmp.F7TpAavHBj ++ return 0 + [[ false == true ]] + wait_pod cross-site-sharded-replica-rs0-2 + local pod=cross-site-sharded-replica-rs0-2 + set +o xtrace waiting for pod/cross-site-sharded-replica-rs0-2 to be ready.OK ++ kubectl_bin get psmdb cross-site-sharded-replica -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].nonvoting.enabled}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.xAHl6ZPhsx +++ mktemp ++ local LAST_ERR=/tmp/tmp.Zup60ANqdz ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'psmdb' 'cross-site-sharded-replica' '-o' 'jsonpath={.spec.replsets[?(@.name=="rs0")].nonvoting.enabled}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb cross-site-sharded-replica -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].nonvoting.enabled}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.xAHl6ZPhsx ++ cat /tmp/tmp.Zup60ANqdz ++ rm /tmp/tmp.xAHl6ZPhsx /tmp/tmp.Zup60ANqdz ++ return 0 + [[ false == true ]] ++ kubectl_bin get psmdb cross-site-sharded-replica -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].hidden.enabled}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.zkdEqdaU8e +++ mktemp ++ local LAST_ERR=/tmp/tmp.xCWk84jmtq ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'psmdb' 'cross-site-sharded-replica' '-o' 'jsonpath={.spec.replsets[?(@.name=="rs0")].hidden.enabled}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb cross-site-sharded-replica -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].hidden.enabled}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.zkdEqdaU8e ++ cat /tmp/tmp.xCWk84jmtq ++ rm /tmp/tmp.zkdEqdaU8e /tmp/tmp.xCWk84jmtq ++ return 0 + [[ '' == true ]] + sleep 10 + [[ true == true ]] + set +x Waiting for cluster cross-site-sharded-replica to be ready + wait_for_running cross-site-sharded-replica-cfg 3 + local name=cross-site-sharded-replica-cfg + let last_pod=2 + local check_cluster_readiness=true + set_debug + [[ 1 == 1 ]] + set -o xtrace + local rs_name=cfg + local cluster_name=cross-site-sharded-replica ++ seq 0 2 + for i in $(seq 0 $last_pod) + [[ 0 -eq 2 ]] + wait_pod cross-site-sharded-replica-cfg-0 + local pod=cross-site-sharded-replica-cfg-0 + set +o xtrace waiting for pod/cross-site-sharded-replica-cfg-0 to be ready.OK + for i in $(seq 0 $last_pod) + [[ 1 -eq 2 ]] + wait_pod cross-site-sharded-replica-cfg-1 + local pod=cross-site-sharded-replica-cfg-1 + set +o xtrace waiting for pod/cross-site-sharded-replica-cfg-1 to be ready.OK + for i in $(seq 0 $last_pod) + [[ 2 -eq 2 ]] ++ kubectl_bin get psmdb cross-site-sharded-replica -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].arbiter.enabled}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.B0Agie0FvR +++ mktemp ++ local LAST_ERR=/tmp/tmp.EDpSDKiguE ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'psmdb' 'cross-site-sharded-replica' '-o' 'jsonpath={.spec.replsets[?(@.name=="cfg")].arbiter.enabled}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb cross-site-sharded-replica -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].arbiter.enabled}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.B0Agie0FvR ++ cat /tmp/tmp.EDpSDKiguE ++ rm /tmp/tmp.B0Agie0FvR /tmp/tmp.EDpSDKiguE ++ return 0 + [[ '' == true ]] + wait_pod cross-site-sharded-replica-cfg-2 + local pod=cross-site-sharded-replica-cfg-2 + set +o xtrace waiting for pod/cross-site-sharded-replica-cfg-2 to be ready.OK ++ kubectl_bin get psmdb cross-site-sharded-replica -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].nonvoting.enabled}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.Fl02s9N4YG +++ mktemp ++ local LAST_ERR=/tmp/tmp.2yg4ajksgB ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'psmdb' 'cross-site-sharded-replica' '-o' 'jsonpath={.spec.replsets[?(@.name=="cfg")].nonvoting.enabled}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb cross-site-sharded-replica -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].nonvoting.enabled}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.Fl02s9N4YG ++ cat /tmp/tmp.2yg4ajksgB ++ rm /tmp/tmp.Fl02s9N4YG /tmp/tmp.2yg4ajksgB ++ return 0 + [[ '' == true ]] ++ kubectl_bin get psmdb cross-site-sharded-replica -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].hidden.enabled}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.qKd5EO8hdB +++ mktemp ++ local LAST_ERR=/tmp/tmp.gRMpH11L7G ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'psmdb' 'cross-site-sharded-replica' '-o' 'jsonpath={.spec.replsets[?(@.name=="cfg")].hidden.enabled}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb cross-site-sharded-replica -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].hidden.enabled}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.qKd5EO8hdB ++ cat /tmp/tmp.gRMpH11L7G ++ rm /tmp/tmp.qKd5EO8hdB /tmp/tmp.gRMpH11L7G ++ return 0 + [[ '' == true ]] + sleep 10 + [[ true == true ]] + set +x Waiting for cluster cross-site-sharded-replica to be ready + desc 'check failover status' + set +o xtrace ----------------------------------------------------------------------------------- check failover status ----------------------------------------------------------------------------------- + compare_mongos_cmd find myApp:myPass@cross-site-sharded-replica-mongos.cross-site-sharded-replica-16001 + local command=find + local uri=myApp:myPass@cross-site-sharded-replica-mongos.cross-site-sharded-replica-16001 + local postfix= + local suffix= + local database=myApp + local collection=test + local port=27017 + local tls=false + [[ false == true ]] + mongos_command=run_mongos + log 'running db.test.command() in myApp' + set +o xtrace [2026-07-17T00:17:45+0000] running db.test.command() in myApp + run_mongos 'use myApp\n db.test.find()' myApp:myPass@cross-site-sharded-replica-mongos.cross-site-sharded-replica-16001 mongodb '' '' 27017 + /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/' + grep -E -v 'I NETWORK|W NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|Started a new thread for the timer service' + local 'command=use myApp\n db.test.find()' + local uri=myApp:myPass@cross-site-sharded-replica-mongos.cross-site-sharded-replica-16001 + local driver=mongodb + local suffix=.svc.cluster.local + local mongo_flag= + local port=27017 + local mongo_bin=mongo ++ echo .svc.cluster.local ++ awk -F: '{print $2}' + suffix_port= + [[ -z '' ]] + suffix=.svc.cluster.local:27017 ++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.d5mf4wdt49 +++ mktemp ++ local LAST_ERR=/tmp/tmp.4nQ0r16t0g ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'pods' '--selector=name=psmdb-client' '-o' 'jsonpath={.items[].metadata.name}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.d5mf4wdt49 ++ cat /tmp/tmp.4nQ0r16t0g ++ rm /tmp/tmp.d5mf4wdt49 /tmp/tmp.4nQ0r16t0g ++ return 0 + local client_container=psmdb-client-757cd65695-5qzl6 + kubectl_bin exec psmdb-client-757cd65695-5qzl6 -- bash -c 'export HOME=/tmp; printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@cross-site-sharded-replica-mongos.cross-site-sharded-replica-16001.svc.cluster.local:27017/admin ' ++ mktemp + local LAST_OUT=/tmp/tmp.RDthwfNnGP ++ mktemp + local LAST_ERR=/tmp/tmp.Dyj6qXqUQk + local exit_status=0 + local timeout=4 + args=('exec' 'psmdb-client-757cd65695-5qzl6' '--' 'bash' '-c' 'export HOME=/tmp; printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@cross-site-sharded-replica-mongos.cross-site-sharded-replica-16001.svc.cluster.local:27017/admin ') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl exec psmdb-client-757cd65695-5qzl6 -- bash -c 'export HOME=/tmp; printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@cross-site-sharded-replica-mongos.cross-site-sharded-replica-16001.svc.cluster.local:27017/admin ' + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.RDthwfNnGP + cat /tmp/tmp.Dyj6qXqUQk + rm /tmp/tmp.RDthwfNnGP /tmp/tmp.Dyj6qXqUQk + return 0 + [[ 0 -eq 0 ]] + diff /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/e2e-tests/cross-site-sharded/compare/find.json /tmp/tmp.yXIW4gRKyb/find + desc 'Failover check finished successfully' + set +o xtrace ----------------------------------------------------------------------------------- Failover check finished successfully ----------------------------------------------------------------------------------- + wait_cluster_consistency cross-site-sharded-replica + local cluster_name=cross-site-sharded-replica + local wait_time=32 + retry=0 + sleep 7 + echo -n 'waiting for cluster cross-site-sharded-replica to be ready' waiting for cluster cross-site-sharded-replica to be ready++ kubectl_bin get psmdb cross-site-sharded-replica -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.IrEDIR1lyK +++ mktemp ++ local LAST_ERR=/tmp/tmp.z2UKccfpf0 ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'psmdb' 'cross-site-sharded-replica' '-o' 'jsonpath={.status.state}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb cross-site-sharded-replica -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.IrEDIR1lyK ++ cat /tmp/tmp.z2UKccfpf0 ++ rm /tmp/tmp.IrEDIR1lyK /tmp/tmp.z2UKccfpf0 ++ return 0 + [[ ready == ready ]] + echo .OK .OK + destroy cross-site-sharded-12730 true + local namespace=cross-site-sharded-12730 + local ignore_logs=true + [[ 0 == 1 ]] + desc 'destroy cluster/operator and all other resources' + set +o xtrace ----------------------------------------------------------------------------------- destroy cluster/operator and all other resources ----------------------------------------------------------------------------------- + '[' true == false ']' + delete_backups + desc 'Delete psmdb-backup' + set +o xtrace ----------------------------------------------------------------------------------- Delete psmdb-backup ----------------------------------------------------------------------------------- ++ kubectl_bin get psmdb-backup --no-headers ++ wc -l +++ mktemp ++ local LAST_OUT=/tmp/tmp.sc8BlevdWD +++ mktemp ++ local LAST_ERR=/tmp/tmp.Kw6dZqgWf2 ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'psmdb-backup' '--no-headers') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb-backup --no-headers ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.sc8BlevdWD ++ cat /tmp/tmp.Kw6dZqgWf2 No resources found in cross-site-sharded-replica-16001 namespace. ++ rm /tmp/tmp.sc8BlevdWD /tmp/tmp.Kw6dZqgWf2 ++ return 0 + '[' 0 '!=' 0 ']' + delete_crd + desc 'get and delete old CRDs and RBAC' + set +o xtrace ----------------------------------------------------------------------------------- get and delete old CRDs and RBAC ----------------------------------------------------------------------------------- + kubectl_bin delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/deploy/crd.yaml --ignore-not-found --wait=false ++ mktemp + local LAST_OUT=/tmp/tmp.lkG38x2QQN ++ mktemp + local LAST_ERR=/tmp/tmp.BqiiuY4PlA + local exit_status=0 + local timeout=4 + args=('delete' '-f' '/mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/deploy/crd.yaml' '--ignore-not-found' '--wait=false') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/deploy/crd.yaml --ignore-not-found --wait=false + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.lkG38x2QQN customresourcedefinition.apiextensions.k8s.io "perconaservermongodbbackups.psmdb.percona.com" deleted customresourcedefinition.apiextensions.k8s.io "perconaservermongodbclustersyncs.psmdb.percona.com" deleted customresourcedefinition.apiextensions.k8s.io "perconaservermongodbrestores.psmdb.percona.com" deleted customresourcedefinition.apiextensions.k8s.io "perconaservermongodbs.psmdb.percona.com" deleted + cat /tmp/tmp.BqiiuY4PlA + rm /tmp/tmp.lkG38x2QQN /tmp/tmp.BqiiuY4PlA + return 0 ++ yq eval .metadata.name /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/deploy/crd.yaml ++ grep -v '\-\-\-' grep: warning: stray \ before - grep: warning: stray \ before - + for crd_name in $(yq eval '.metadata.name' "${src_dir}/deploy/crd.yaml" | grep -v '\-\-\-') + kubectl get perconaservermongodbbackups.psmdb.percona.com --all-namespaces -o wide + grep -v NAMESPACE + xargs -L 1 sh -xc 'kubectl patch perconaservermongodbbackups.psmdb.percona.com -n $0 $1 --type=merge -p "{\"metadata\":{\"finalizers\":[]}}"' error: the server doesn't have a resource type "perconaservermongodbbackups" + kubectl patch perconaservermongodbbackups.psmdb.percona.com -n sh --type=merge -p '{"metadata":{"finalizers":[]}}' error: the server doesn't have a resource type "perconaservermongodbbackups" + : + kubectl_bin wait --for=delete crd perconaservermongodbbackups.psmdb.percona.com ++ mktemp + local LAST_OUT=/tmp/tmp.W85EBDHkI8 ++ mktemp + local LAST_ERR=/tmp/tmp.zgibtd8hjq + local exit_status=0 + local timeout=4 + args=('wait' '--for=delete' 'crd' 'perconaservermongodbbackups.psmdb.percona.com') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl wait --for=delete crd perconaservermongodbbackups.psmdb.percona.com + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.W85EBDHkI8 + cat /tmp/tmp.zgibtd8hjq + rm /tmp/tmp.W85EBDHkI8 /tmp/tmp.zgibtd8hjq + return 0 + for crd_name in $(yq eval '.metadata.name' "${src_dir}/deploy/crd.yaml" | grep -v '\-\-\-') + kubectl get perconaservermongodbclustersyncs.psmdb.percona.com --all-namespaces -o wide + grep -v NAMESPACE + xargs -L 1 sh -xc 'kubectl patch perconaservermongodbclustersyncs.psmdb.percona.com -n $0 $1 --type=merge -p "{\"metadata\":{\"finalizers\":[]}}"' error: the server doesn't have a resource type "perconaservermongodbclustersyncs" + kubectl patch perconaservermongodbclustersyncs.psmdb.percona.com -n sh --type=merge -p '{"metadata":{"finalizers":[]}}' error: the server doesn't have a resource type "perconaservermongodbclustersyncs" + : + kubectl_bin wait --for=delete crd perconaservermongodbclustersyncs.psmdb.percona.com ++ mktemp + local LAST_OUT=/tmp/tmp.Ax7ExBSY4V ++ mktemp + local LAST_ERR=/tmp/tmp.a7dotzPcxa + local exit_status=0 + local timeout=4 + args=('wait' '--for=delete' 'crd' 'perconaservermongodbclustersyncs.psmdb.percona.com') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl wait --for=delete crd perconaservermongodbclustersyncs.psmdb.percona.com + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.Ax7ExBSY4V + cat /tmp/tmp.a7dotzPcxa + rm /tmp/tmp.Ax7ExBSY4V /tmp/tmp.a7dotzPcxa + return 0 + for crd_name in $(yq eval '.metadata.name' "${src_dir}/deploy/crd.yaml" | grep -v '\-\-\-') + kubectl get perconaservermongodbrestores.psmdb.percona.com --all-namespaces -o wide + grep -v NAMESPACE + xargs -L 1 sh -xc 'kubectl patch perconaservermongodbrestores.psmdb.percona.com -n $0 $1 --type=merge -p "{\"metadata\":{\"finalizers\":[]}}"' error: the server doesn't have a resource type "perconaservermongodbrestores" + kubectl patch perconaservermongodbrestores.psmdb.percona.com -n sh --type=merge -p '{"metadata":{"finalizers":[]}}' error: the server doesn't have a resource type "perconaservermongodbrestores" + : + kubectl_bin wait --for=delete crd perconaservermongodbrestores.psmdb.percona.com ++ mktemp + local LAST_OUT=/tmp/tmp.fyDXIgpZKb ++ mktemp + local LAST_ERR=/tmp/tmp.PzvniiQ09o + local exit_status=0 + local timeout=4 + args=('wait' '--for=delete' 'crd' 'perconaservermongodbrestores.psmdb.percona.com') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl wait --for=delete crd perconaservermongodbrestores.psmdb.percona.com + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.fyDXIgpZKb + cat /tmp/tmp.PzvniiQ09o + rm /tmp/tmp.fyDXIgpZKb /tmp/tmp.PzvniiQ09o + return 0 + for crd_name in $(yq eval '.metadata.name' "${src_dir}/deploy/crd.yaml" | grep -v '\-\-\-') + kubectl get perconaservermongodbs.psmdb.percona.com --all-namespaces -o wide + grep -v NAMESPACE + xargs -L 1 sh -xc 'kubectl patch perconaservermongodbs.psmdb.percona.com -n $0 $1 --type=merge -p "{\"metadata\":{\"finalizers\":[]}}"' error: the server doesn't have a resource type "perconaservermongodbs" + kubectl patch perconaservermongodbs.psmdb.percona.com -n sh --type=merge -p '{"metadata":{"finalizers":[]}}' error: the server doesn't have a resource type "perconaservermongodbs" + : + kubectl_bin wait --for=delete crd perconaservermongodbs.psmdb.percona.com ++ mktemp + local LAST_OUT=/tmp/tmp.ak1bCCHEfA ++ mktemp + local LAST_ERR=/tmp/tmp.uqtO5l2VWE + local exit_status=0 + local timeout=4 + args=('wait' '--for=delete' 'crd' 'perconaservermongodbs.psmdb.percona.com') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl wait --for=delete crd perconaservermongodbs.psmdb.percona.com + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.ak1bCCHEfA + cat /tmp/tmp.uqtO5l2VWE + rm /tmp/tmp.ak1bCCHEfA /tmp/tmp.uqtO5l2VWE + return 0 + local rbac_yaml=rbac.yaml + '[' -n '' ']' + kubectl_bin delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/deploy/rbac.yaml --ignore-not-found ++ mktemp + local LAST_OUT=/tmp/tmp.cE5zhbsu8u ++ mktemp + local LAST_ERR=/tmp/tmp.4aUph6LKlX + local exit_status=0 + local timeout=4 + args=('delete' '-f' '/mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/deploy/rbac.yaml' '--ignore-not-found') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/deploy/rbac.yaml --ignore-not-found + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.cE5zhbsu8u role.rbac.authorization.k8s.io "percona-server-mongodb-operator" deleted from cross-site-sharded-replica-16001 namespace serviceaccount "percona-server-mongodb-operator" deleted from cross-site-sharded-replica-16001 namespace rolebinding.rbac.authorization.k8s.io "service-account-percona-server-mongodb-operator" deleted from cross-site-sharded-replica-16001 namespace + cat /tmp/tmp.4aUph6LKlX + rm /tmp/tmp.cE5zhbsu8u /tmp/tmp.4aUph6LKlX + return 0 + destroy_cert_manager + [[ gke == rancher ]] + kubectl_bin delete -f https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml ++ mktemp + local LAST_OUT=/tmp/tmp.DyUbhUYKr7 ++ mktemp + local LAST_ERR=/tmp/tmp.IoSnXyakyY + local exit_status=0 + local timeout=4 + args=('delete' '-f' 'https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl delete -f https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml + exit_status=1 + set -e + '[' 1 '!=' 0 -a -n 1 ']' + cat /tmp/tmp.DyUbhUYKr7 + cat /tmp/tmp.IoSnXyakyY Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": namespaces "cert-manager" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "challenges.acme.cert-manager.io" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "orders.acme.cert-manager.io" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "certificaterequests.cert-manager.io" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "certificates.cert-manager.io" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "clusterissuers.cert-manager.io" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "issuers.cert-manager.io" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": serviceaccounts "cert-manager-cainjector" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": serviceaccounts "cert-manager" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": serviceaccounts "cert-manager-webhook" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-cainjector" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-issuers" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-clusterissuers" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-certificates" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-orders" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-challenges" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-ingress-shim" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-cluster-view" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-view" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-edit" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-approve:cert-manager-io" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-certificatesigningrequests" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-webhook:subjectaccessreviews" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-cainjector" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-issuers" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-clusterissuers" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-certificates" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-orders" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-challenges" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-ingress-shim" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-approve:cert-manager-io" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-certificatesigningrequests" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-webhook:subjectaccessreviews" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager-cainjector:leaderelection" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager:leaderelection" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager-tokenrequest" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager-webhook:dynamic-serving" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager-cainjector:leaderelection" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager:leaderelection" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager-tokenrequest" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager-webhook:dynamic-serving" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": services "cert-manager-cainjector" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": services "cert-manager" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": services "cert-manager-webhook" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": deployments.apps "cert-manager-cainjector" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": deployments.apps "cert-manager" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": deployments.apps "cert-manager-webhook" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": mutatingwebhookconfigurations.admissionregistration.k8s.io "cert-manager-webhook" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": validatingwebhookconfigurations.admissionregistration.k8s.io "cert-manager-webhook" not found + sleep 0 + for i in $(seq 0 2) + set +e + kubectl delete -f https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml + exit_status=1 + set -e + '[' 1 '!=' 0 -a -n 1 ']' + cat /tmp/tmp.DyUbhUYKr7 + cat /tmp/tmp.IoSnXyakyY Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": namespaces "cert-manager" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "challenges.acme.cert-manager.io" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "orders.acme.cert-manager.io" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "certificaterequests.cert-manager.io" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "certificates.cert-manager.io" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "clusterissuers.cert-manager.io" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "issuers.cert-manager.io" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": serviceaccounts "cert-manager-cainjector" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": serviceaccounts "cert-manager" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": serviceaccounts "cert-manager-webhook" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-cainjector" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-issuers" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-clusterissuers" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-certificates" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-orders" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-challenges" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-ingress-shim" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-cluster-view" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-view" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-edit" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-approve:cert-manager-io" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-certificatesigningrequests" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-webhook:subjectaccessreviews" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-cainjector" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-issuers" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-clusterissuers" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-certificates" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-orders" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-challenges" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-ingress-shim" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-approve:cert-manager-io" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-certificatesigningrequests" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-webhook:subjectaccessreviews" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager-cainjector:leaderelection" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager:leaderelection" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager-tokenrequest" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager-webhook:dynamic-serving" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager-cainjector:leaderelection" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager:leaderelection" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager-tokenrequest" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager-webhook:dynamic-serving" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": services "cert-manager-cainjector" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": services "cert-manager" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": services "cert-manager-webhook" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": deployments.apps "cert-manager-cainjector" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": deployments.apps "cert-manager" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": deployments.apps "cert-manager-webhook" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": mutatingwebhookconfigurations.admissionregistration.k8s.io "cert-manager-webhook" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": validatingwebhookconfigurations.admissionregistration.k8s.io "cert-manager-webhook" not found + sleep 4 + for i in $(seq 0 2) + set +e + kubectl delete -f https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml + exit_status=1 + set -e + '[' 1 '!=' 0 -a -n 1 ']' + cat /tmp/tmp.DyUbhUYKr7 + cat /tmp/tmp.IoSnXyakyY Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": namespaces "cert-manager" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "challenges.acme.cert-manager.io" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "orders.acme.cert-manager.io" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "certificaterequests.cert-manager.io" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "certificates.cert-manager.io" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "clusterissuers.cert-manager.io" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "issuers.cert-manager.io" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": serviceaccounts "cert-manager-cainjector" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": serviceaccounts "cert-manager" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": serviceaccounts "cert-manager-webhook" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-cainjector" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-issuers" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-clusterissuers" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-certificates" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-orders" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-challenges" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-ingress-shim" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-cluster-view" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-view" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-edit" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-approve:cert-manager-io" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-certificatesigningrequests" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-webhook:subjectaccessreviews" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-cainjector" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-issuers" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-clusterissuers" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-certificates" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-orders" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-challenges" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-ingress-shim" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-approve:cert-manager-io" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-certificatesigningrequests" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-webhook:subjectaccessreviews" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager-cainjector:leaderelection" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager:leaderelection" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager-tokenrequest" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager-webhook:dynamic-serving" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager-cainjector:leaderelection" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager:leaderelection" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager-tokenrequest" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager-webhook:dynamic-serving" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": services "cert-manager-cainjector" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": services "cert-manager" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": services "cert-manager-webhook" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": deployments.apps "cert-manager-cainjector" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": deployments.apps "cert-manager" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": deployments.apps "cert-manager-webhook" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": mutatingwebhookconfigurations.admissionregistration.k8s.io "cert-manager-webhook" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": validatingwebhookconfigurations.admissionregistration.k8s.io "cert-manager-webhook" not found + sleep 8 + cat /tmp/tmp.DyUbhUYKr7 + cat /tmp/tmp.IoSnXyakyY Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": namespaces "cert-manager" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "challenges.acme.cert-manager.io" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "orders.acme.cert-manager.io" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "certificaterequests.cert-manager.io" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "certificates.cert-manager.io" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "clusterissuers.cert-manager.io" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "issuers.cert-manager.io" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": serviceaccounts "cert-manager-cainjector" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": serviceaccounts "cert-manager" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": serviceaccounts "cert-manager-webhook" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-cainjector" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-issuers" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-clusterissuers" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-certificates" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-orders" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-challenges" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-ingress-shim" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-cluster-view" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-view" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-edit" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-approve:cert-manager-io" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-certificatesigningrequests" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-webhook:subjectaccessreviews" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-cainjector" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-issuers" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-clusterissuers" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-certificates" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-orders" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-challenges" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-ingress-shim" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-approve:cert-manager-io" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-certificatesigningrequests" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-webhook:subjectaccessreviews" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager-cainjector:leaderelection" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager:leaderelection" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager-tokenrequest" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager-webhook:dynamic-serving" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager-cainjector:leaderelection" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager:leaderelection" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager-tokenrequest" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager-webhook:dynamic-serving" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": services "cert-manager-cainjector" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": services "cert-manager" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": services "cert-manager-webhook" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": deployments.apps "cert-manager-cainjector" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": deployments.apps "cert-manager" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": deployments.apps "cert-manager-webhook" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": mutatingwebhookconfigurations.admissionregistration.k8s.io "cert-manager-webhook" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": validatingwebhookconfigurations.admissionregistration.k8s.io "cert-manager-webhook" not found + rm /tmp/tmp.DyUbhUYKr7 /tmp/tmp.IoSnXyakyY + return 1 + true + [[ -n '' ]] + '[' -n '' ']' + kubectl_bin delete --grace-period=0 --force=true namespace cross-site-sharded-12730 + rm -rf /tmp/tmp.yXIW4gRKyb ++ mktemp + destroy cross-site-sharded-replica-16001 true + local namespace=cross-site-sharded-replica-16001 + local ignore_logs=true + [[ 0 == 1 ]] + desc 'destroy cluster/operator and all other resources' + set +o xtrace ----------------------------------------------------------------------------------- destroy cluster/operator and all other resources ----------------------------------------------------------------------------------- + '[' true == false ']' + delete_backups + desc 'Delete psmdb-backup' + set +o xtrace ----------------------------------------------------------------------------------- Delete psmdb-backup ----------------------------------------------------------------------------------- + local LAST_OUT=/tmp/tmp.gpn2Lrt60N ++ mktemp ++ kubectl_bin get psmdb-backup --no-headers ++ wc -l +++ mktemp + local LAST_ERR=/tmp/tmp.4vjnfujtNM + local exit_status=0 + local timeout=4 + args=('delete' '--grace-period=0' '--force=true' 'namespace' 'cross-site-sharded-12730') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl delete --grace-period=0 --force=true namespace cross-site-sharded-12730 ++ local LAST_OUT=/tmp/tmp.GEe84S4Nqc +++ mktemp ++ local LAST_ERR=/tmp/tmp.equ61V08Gk ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'psmdb-backup' '--no-headers') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb-backup --no-headers ++ exit_status=1 ++ set -e ++ '[' 1 '!=' 0 -a -n 1 ']' ++ cat /tmp/tmp.GEe84S4Nqc ++ cat /tmp/tmp.equ61V08Gk error: the server doesn't have a resource type "psmdb-backup" ++ sleep 0 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb-backup --no-headers ++ exit_status=1 ++ set -e ++ '[' 1 '!=' 0 -a -n 1 ']' ++ cat /tmp/tmp.GEe84S4Nqc ++ cat /tmp/tmp.equ61V08Gk error: the server doesn't have a resource type "psmdb-backup" ++ sleep 4 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb-backup --no-headers ++ exit_status=1 ++ set -e ++ '[' 1 '!=' 0 -a -n 1 ']' ++ cat /tmp/tmp.GEe84S4Nqc ++ cat /tmp/tmp.equ61V08Gk error: the server doesn't have a resource type "psmdb-backup" ++ sleep 8 ++ cat /tmp/tmp.GEe84S4Nqc ++ cat /tmp/tmp.equ61V08Gk error: the server doesn't have a resource type "psmdb-backup" ++ rm /tmp/tmp.GEe84S4Nqc /tmp/tmp.equ61V08Gk ++ return 1 + '[' 0 '!=' 0 ']' + delete_crd + desc 'get and delete old CRDs and RBAC' + set +o xtrace ----------------------------------------------------------------------------------- get and delete old CRDs and RBAC ----------------------------------------------------------------------------------- + kubectl_bin delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/deploy/crd.yaml --ignore-not-found --wait=false ++ mktemp + local LAST_OUT=/tmp/tmp.CUG7t1qiDl ++ mktemp + local LAST_ERR=/tmp/tmp.8x3lWIoP7Z + local exit_status=0 + local timeout=4 + args=('delete' '-f' '/mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/deploy/crd.yaml' '--ignore-not-found' '--wait=false') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/deploy/crd.yaml --ignore-not-found --wait=false + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.CUG7t1qiDl + cat /tmp/tmp.8x3lWIoP7Z + rm /tmp/tmp.CUG7t1qiDl /tmp/tmp.8x3lWIoP7Z + return 0 ++ yq eval .metadata.name /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/deploy/crd.yaml ++ grep -v '\-\-\-' grep: warning: stray \ before - grep: warning: stray \ before - + for crd_name in $(yq eval '.metadata.name' "${src_dir}/deploy/crd.yaml" | grep -v '\-\-\-') + kubectl get perconaservermongodbbackups.psmdb.percona.com --all-namespaces -o wide + grep -v NAMESPACE + xargs -L 1 sh -xc 'kubectl patch perconaservermongodbbackups.psmdb.percona.com -n $0 $1 --type=merge -p "{\"metadata\":{\"finalizers\":[]}}"' error: the server doesn't have a resource type "perconaservermongodbbackups" + kubectl patch perconaservermongodbbackups.psmdb.percona.com -n sh --type=merge -p '{"metadata":{"finalizers":[]}}' error: the server doesn't have a resource type "perconaservermongodbbackups" + : + kubectl_bin wait --for=delete crd perconaservermongodbbackups.psmdb.percona.com ++ mktemp + local LAST_OUT=/tmp/tmp.kNkXABkD9T ++ mktemp + local LAST_ERR=/tmp/tmp.zcR7HsqgrD + local exit_status=0 + local timeout=4 + args=('wait' '--for=delete' 'crd' 'perconaservermongodbbackups.psmdb.percona.com') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl wait --for=delete crd perconaservermongodbbackups.psmdb.percona.com + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.kNkXABkD9T + cat /tmp/tmp.zcR7HsqgrD + rm /tmp/tmp.kNkXABkD9T /tmp/tmp.zcR7HsqgrD + return 0 + for crd_name in $(yq eval '.metadata.name' "${src_dir}/deploy/crd.yaml" | grep -v '\-\-\-') + kubectl get perconaservermongodbclustersyncs.psmdb.percona.com --all-namespaces -o wide + grep -v NAMESPACE + xargs -L 1 sh -xc 'kubectl patch perconaservermongodbclustersyncs.psmdb.percona.com -n $0 $1 --type=merge -p "{\"metadata\":{\"finalizers\":[]}}"' error: the server doesn't have a resource type "perconaservermongodbclustersyncs" + kubectl patch perconaservermongodbclustersyncs.psmdb.percona.com -n sh --type=merge -p '{"metadata":{"finalizers":[]}}' error: the server doesn't have a resource type "perconaservermongodbclustersyncs" + : + kubectl_bin wait --for=delete crd perconaservermongodbclustersyncs.psmdb.percona.com ++ mktemp + local LAST_OUT=/tmp/tmp.lyTNkBrwq5 ++ mktemp + local LAST_ERR=/tmp/tmp.f1lhhO9Ik6 + local exit_status=0 + local timeout=4 + args=('wait' '--for=delete' 'crd' 'perconaservermongodbclustersyncs.psmdb.percona.com') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl wait --for=delete crd perconaservermongodbclustersyncs.psmdb.percona.com + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.lyTNkBrwq5 + cat /tmp/tmp.f1lhhO9Ik6 + rm /tmp/tmp.lyTNkBrwq5 /tmp/tmp.f1lhhO9Ik6 + return 0 + for crd_name in $(yq eval '.metadata.name' "${src_dir}/deploy/crd.yaml" | grep -v '\-\-\-') + kubectl get perconaservermongodbrestores.psmdb.percona.com --all-namespaces -o wide + grep -v NAMESPACE + xargs -L 1 sh -xc 'kubectl patch perconaservermongodbrestores.psmdb.percona.com -n $0 $1 --type=merge -p "{\"metadata\":{\"finalizers\":[]}}"' error: the server doesn't have a resource type "perconaservermongodbrestores" + kubectl patch perconaservermongodbrestores.psmdb.percona.com -n sh --type=merge -p '{"metadata":{"finalizers":[]}}' error: the server doesn't have a resource type "perconaservermongodbrestores" + : + kubectl_bin wait --for=delete crd perconaservermongodbrestores.psmdb.percona.com ++ mktemp + local LAST_OUT=/tmp/tmp.VXnQhCLzOd ++ mktemp + local LAST_ERR=/tmp/tmp.rtaWoUyqIN + local exit_status=0 + local timeout=4 + args=('wait' '--for=delete' 'crd' 'perconaservermongodbrestores.psmdb.percona.com') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl wait --for=delete crd perconaservermongodbrestores.psmdb.percona.com + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.VXnQhCLzOd + cat /tmp/tmp.rtaWoUyqIN + rm /tmp/tmp.VXnQhCLzOd /tmp/tmp.rtaWoUyqIN + return 0 + for crd_name in $(yq eval '.metadata.name' "${src_dir}/deploy/crd.yaml" | grep -v '\-\-\-') + kubectl get perconaservermongodbs.psmdb.percona.com --all-namespaces -o wide + grep -v NAMESPACE + xargs -L 1 sh -xc 'kubectl patch perconaservermongodbs.psmdb.percona.com -n $0 $1 --type=merge -p "{\"metadata\":{\"finalizers\":[]}}"' error: the server doesn't have a resource type "perconaservermongodbs" + kubectl patch perconaservermongodbs.psmdb.percona.com -n sh --type=merge -p '{"metadata":{"finalizers":[]}}' error: the server doesn't have a resource type "perconaservermongodbs" + : + kubectl_bin wait --for=delete crd perconaservermongodbs.psmdb.percona.com ++ mktemp + local LAST_OUT=/tmp/tmp.FTHkdP7X4I ++ mktemp + local LAST_ERR=/tmp/tmp.Z1C5fefNg6 + local exit_status=0 + local timeout=4 + args=('wait' '--for=delete' 'crd' 'perconaservermongodbs.psmdb.percona.com') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl wait --for=delete crd perconaservermongodbs.psmdb.percona.com + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.FTHkdP7X4I + cat /tmp/tmp.Z1C5fefNg6 + rm /tmp/tmp.FTHkdP7X4I /tmp/tmp.Z1C5fefNg6 + return 0 + local rbac_yaml=rbac.yaml + '[' -n '' ']' + kubectl_bin delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/deploy/rbac.yaml --ignore-not-found ++ mktemp + local LAST_OUT=/tmp/tmp.1sumibUM2p ++ mktemp + local LAST_ERR=/tmp/tmp.sseqAJMsrc + local exit_status=0 + local timeout=4 + args=('delete' '-f' '/mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/deploy/rbac.yaml' '--ignore-not-found') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/deploy/rbac.yaml --ignore-not-found + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.1sumibUM2p + cat /tmp/tmp.sseqAJMsrc + rm /tmp/tmp.1sumibUM2p /tmp/tmp.sseqAJMsrc + return 0 + destroy_cert_manager + [[ gke == rancher ]] + kubectl_bin delete -f https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml ++ mktemp + local LAST_OUT=/tmp/tmp.MEcgdl8N1Q ++ mktemp + local LAST_ERR=/tmp/tmp.Pw0rvyWaLF + local exit_status=0 + local timeout=4 + args=('delete' '-f' 'https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl delete -f https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.gpn2Lrt60N namespace "cross-site-sharded-12730" force deleted + cat /tmp/tmp.4vjnfujtNM Warning: Immediate deletion does not wait for confirmation that the running resource has been terminated. The resource may continue to run on the cluster indefinitely. + rm /tmp/tmp.gpn2Lrt60N /tmp/tmp.4vjnfujtNM + return 0 + exit_status=1 + set -e + '[' 1 '!=' 0 -a -n 1 ']' + cat /tmp/tmp.MEcgdl8N1Q + cat /tmp/tmp.Pw0rvyWaLF Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": namespaces "cert-manager" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "challenges.acme.cert-manager.io" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "orders.acme.cert-manager.io" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "certificaterequests.cert-manager.io" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "certificates.cert-manager.io" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "clusterissuers.cert-manager.io" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "issuers.cert-manager.io" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": serviceaccounts "cert-manager-cainjector" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": serviceaccounts "cert-manager" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": serviceaccounts "cert-manager-webhook" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-cainjector" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-issuers" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-clusterissuers" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-certificates" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-orders" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-challenges" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-ingress-shim" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-cluster-view" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-view" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-edit" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-approve:cert-manager-io" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-certificatesigningrequests" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-webhook:subjectaccessreviews" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-cainjector" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-issuers" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-clusterissuers" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-certificates" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-orders" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-challenges" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-ingress-shim" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-approve:cert-manager-io" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-certificatesigningrequests" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-webhook:subjectaccessreviews" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager-cainjector:leaderelection" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager:leaderelection" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager-tokenrequest" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager-webhook:dynamic-serving" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager-cainjector:leaderelection" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager:leaderelection" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager-tokenrequest" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager-webhook:dynamic-serving" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": services "cert-manager-cainjector" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": services "cert-manager" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": services "cert-manager-webhook" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": deployments.apps "cert-manager-cainjector" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": deployments.apps "cert-manager" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": deployments.apps "cert-manager-webhook" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": mutatingwebhookconfigurations.admissionregistration.k8s.io "cert-manager-webhook" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": validatingwebhookconfigurations.admissionregistration.k8s.io "cert-manager-webhook" not found + sleep 0 + for i in $(seq 0 2) + set +e + kubectl delete -f https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml + exit_status=1 + set -e + '[' 1 '!=' 0 -a -n 1 ']' + cat /tmp/tmp.MEcgdl8N1Q + cat /tmp/tmp.Pw0rvyWaLF Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": namespaces "cert-manager" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "challenges.acme.cert-manager.io" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "orders.acme.cert-manager.io" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "certificaterequests.cert-manager.io" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "certificates.cert-manager.io" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "clusterissuers.cert-manager.io" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "issuers.cert-manager.io" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": serviceaccounts "cert-manager-cainjector" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": serviceaccounts "cert-manager" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": serviceaccounts "cert-manager-webhook" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-cainjector" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-issuers" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-clusterissuers" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-certificates" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-orders" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-challenges" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-ingress-shim" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-cluster-view" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-view" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-edit" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-approve:cert-manager-io" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-certificatesigningrequests" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-webhook:subjectaccessreviews" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-cainjector" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-issuers" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-clusterissuers" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-certificates" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-orders" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-challenges" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-ingress-shim" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-approve:cert-manager-io" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-certificatesigningrequests" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-webhook:subjectaccessreviews" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager-cainjector:leaderelection" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager:leaderelection" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager-tokenrequest" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager-webhook:dynamic-serving" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager-cainjector:leaderelection" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager:leaderelection" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager-tokenrequest" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager-webhook:dynamic-serving" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": services "cert-manager-cainjector" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": services "cert-manager" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": services "cert-manager-webhook" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": deployments.apps "cert-manager-cainjector" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": deployments.apps "cert-manager" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": deployments.apps "cert-manager-webhook" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": mutatingwebhookconfigurations.admissionregistration.k8s.io "cert-manager-webhook" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": validatingwebhookconfigurations.admissionregistration.k8s.io "cert-manager-webhook" not found + sleep 4 + for i in $(seq 0 2) + set +e + kubectl delete -f https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml + exit_status=1 + set -e + '[' 1 '!=' 0 -a -n 1 ']' + cat /tmp/tmp.MEcgdl8N1Q + cat /tmp/tmp.Pw0rvyWaLF Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": namespaces "cert-manager" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "challenges.acme.cert-manager.io" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "orders.acme.cert-manager.io" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "certificaterequests.cert-manager.io" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "certificates.cert-manager.io" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "clusterissuers.cert-manager.io" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "issuers.cert-manager.io" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": serviceaccounts "cert-manager-cainjector" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": serviceaccounts "cert-manager" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": serviceaccounts "cert-manager-webhook" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-cainjector" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-issuers" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-clusterissuers" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-certificates" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-orders" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-challenges" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-ingress-shim" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-cluster-view" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-view" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-edit" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-approve:cert-manager-io" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-certificatesigningrequests" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-webhook:subjectaccessreviews" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-cainjector" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-issuers" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-clusterissuers" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-certificates" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-orders" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-challenges" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-ingress-shim" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-approve:cert-manager-io" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-certificatesigningrequests" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-webhook:subjectaccessreviews" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager-cainjector:leaderelection" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager:leaderelection" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager-tokenrequest" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager-webhook:dynamic-serving" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager-cainjector:leaderelection" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager:leaderelection" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager-tokenrequest" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager-webhook:dynamic-serving" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": services "cert-manager-cainjector" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": services "cert-manager" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": services "cert-manager-webhook" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": deployments.apps "cert-manager-cainjector" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": deployments.apps "cert-manager" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": deployments.apps "cert-manager-webhook" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": mutatingwebhookconfigurations.admissionregistration.k8s.io "cert-manager-webhook" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": validatingwebhookconfigurations.admissionregistration.k8s.io "cert-manager-webhook" not found + sleep 8 + cat /tmp/tmp.MEcgdl8N1Q + cat /tmp/tmp.Pw0rvyWaLF Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": namespaces "cert-manager" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "challenges.acme.cert-manager.io" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "orders.acme.cert-manager.io" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "certificaterequests.cert-manager.io" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "certificates.cert-manager.io" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "clusterissuers.cert-manager.io" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "issuers.cert-manager.io" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": serviceaccounts "cert-manager-cainjector" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": serviceaccounts "cert-manager" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": serviceaccounts "cert-manager-webhook" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-cainjector" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-issuers" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-clusterissuers" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-certificates" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-orders" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-challenges" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-ingress-shim" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-cluster-view" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-view" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-edit" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-approve:cert-manager-io" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-certificatesigningrequests" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-webhook:subjectaccessreviews" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-cainjector" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-issuers" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-clusterissuers" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-certificates" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-orders" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-challenges" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-ingress-shim" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-approve:cert-manager-io" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-certificatesigningrequests" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-webhook:subjectaccessreviews" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager-cainjector:leaderelection" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager:leaderelection" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager-tokenrequest" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager-webhook:dynamic-serving" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager-cainjector:leaderelection" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager:leaderelection" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager-tokenrequest" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager-webhook:dynamic-serving" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": services "cert-manager-cainjector" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": services "cert-manager" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": services "cert-manager-webhook" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": deployments.apps "cert-manager-cainjector" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": deployments.apps "cert-manager" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": deployments.apps "cert-manager-webhook" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": mutatingwebhookconfigurations.admissionregistration.k8s.io "cert-manager-webhook" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": validatingwebhookconfigurations.admissionregistration.k8s.io "cert-manager-webhook" not found + rm /tmp/tmp.MEcgdl8N1Q /tmp/tmp.Pw0rvyWaLF + return 1 + true + [[ -n '' ]] + '[' -n '' ']' + rm -rf /tmp/tmp.yXIW4gRKyb + kubectl_bin delete --grace-period=0 --force=true namespace cross-site-sharded-replica-16001 ++ mktemp + desc 'test passed' + set +o xtrace ----------------------------------------------------------------------------------- test passed ----------------------------------------------------------------------------------- PASSED+ local LAST_OUT=/tmp/tmp.2cXTt48G2H ++ mktemp + local LAST_ERR=/tmp/tmp.rmWWn9xjdX + local exit_status=0 + local timeout=4 + args=('delete' '--grace-period=0' '--force=true' 'namespace' 'cross-site-sharded-replica-16001') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl delete --grace-period=0 --force=true namespace cross-site-sharded-replica-16001 - generated xml file: /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/e2e-tests/reports/cross-site-sharded.xml - - Generated html report: file:///mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/e2e-tests/reports/cross-site-sharded.html - ======================== 1 passed in 1121.34s (0:18:41) ======================== + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.2cXTt48G2H namespace "cross-site-sharded-replica-16001" force deleted + cat /tmp/tmp.rmWWn9xjdX Warning: Immediate deletion does not wait for confirmation that the running resource has been terminated. The resource may continue to run on the cluster indefinitely. + rm /tmp/tmp.2cXTt48G2H /tmp/tmp.rmWWn9xjdX + return 0