Log: /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1548/e2e-tests/logs/cross-site-sharded.log WARNING: version difference between client (1.30) and server (1.26) exceeds the supported minor version skew of +/-1 WARNING: version difference between client (1.30) and server (1.26) exceeds the supported minor version skew of +/-1 WARNING: version difference between client (1.30) and server (1.26) exceeds the supported minor version skew of +/-1 ++ get_mongod_ver_from_image perconalab/percona-server-mongodb-operator:main-mongod7.0 ++ local image=perconalab/percona-server-mongodb-operator:main-mongod7.0 +++ run_simple_cli_inside_image perconalab/percona-server-mongodb-operator:main-mongod7.0 'mongod --version' +++ local image=perconalab/percona-server-mongodb-operator:main-mongod7.0 +++ local 'cli=mongod --version' +++ local pod_name=25790 +++ kubectl_bin -n default run 25790 --image=perconalab/percona-server-mongodb-operator:main-mongod7.0 --restart=Never --command -- sleep infinity +++ /usr/bin/sed -r 's/^.*db version v(([0-9]+\.){2}[0-9]+-[0-9]+).*$/\1/g' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.KEEZfbMOvI ++++ mktemp +++ local LAST_ERR=/tmp/tmp.cRyDRg5e0E +++ local exit_status=0 +++ local timeout=4 ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ set +e +++ kubectl -n default run 25790 --image=perconalab/percona-server-mongodb-operator:main-mongod7.0 --restart=Never --command -- sleep infinity +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 -a -n 1 ']' +++ break +++ cat /tmp/tmp.KEEZfbMOvI +++ cat /tmp/tmp.cRyDRg5e0E +++ rm /tmp/tmp.KEEZfbMOvI /tmp/tmp.cRyDRg5e0E +++ return 0 +++ kubectl_bin -n default wait --for=condition=Ready pod/25790 ++++ mktemp +++ local LAST_OUT=/tmp/tmp.MC7yWrar8X ++++ mktemp +++ local LAST_ERR=/tmp/tmp.AvEmPf0OTY +++ local exit_status=0 +++ local timeout=4 ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ set +e +++ kubectl -n default wait --for=condition=Ready pod/25790 +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 -a -n 1 ']' +++ break +++ cat /tmp/tmp.MC7yWrar8X +++ cat /tmp/tmp.AvEmPf0OTY +++ rm /tmp/tmp.MC7yWrar8X /tmp/tmp.AvEmPf0OTY +++ return 0 ++++ kubectl_bin -n default exec 25790 -- mongod --version +++++ mktemp ++++ local LAST_OUT=/tmp/tmp.8PVTbTyxWs +++++ mktemp ++++ local LAST_ERR=/tmp/tmp.0dGVS5pgI2 ++++ local exit_status=0 ++++ local timeout=4 +++++ seq 0 2 ++++ for i in '$(seq 0 2)' ++++ set +e ++++ kubectl -n default exec 25790 -- mongod --version ++++ exit_status=0 ++++ set -e ++++ '[' 0 '!=' 0 -a -n 1 ']' ++++ break ++++ cat /tmp/tmp.8PVTbTyxWs ++++ cat /tmp/tmp.0dGVS5pgI2 ++++ rm /tmp/tmp.8PVTbTyxWs /tmp/tmp.0dGVS5pgI2 ++++ return 0 +++ local 'output=db version v7.0.8-5 Build Info: { "version": "7.0.8-5", "gitVersion": "197ff4d49589e8b5a444b7ebc6f4d33b6f00105e", "openSSLVersion": "OpenSSL 1.1.1k FIPS 25 Mar 2021", "modules": [], "proFeatures": [], "allocator": "tcmalloc", "environment": { "distarch": "x86_64", "target_arch": "x86_64" } }' +++ kubectl_bin -n default delete pod/25790 --grace-period=0 --force ++++ mktemp +++ local LAST_OUT=/tmp/tmp.U2XNxafNMi ++++ mktemp +++ local LAST_ERR=/tmp/tmp.uya2X0TUxi +++ local exit_status=0 +++ local timeout=4 ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ set +e +++ kubectl -n default delete pod/25790 --grace-period=0 --force +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 -a -n 1 ']' +++ break +++ cat /tmp/tmp.U2XNxafNMi +++ cat /tmp/tmp.uya2X0TUxi Warning: Immediate deletion does not wait for confirmation that the running resource has been terminated. The resource may continue to run on the cluster indefinitely. +++ rm /tmp/tmp.U2XNxafNMi /tmp/tmp.uya2X0TUxi +++ return 0 +++ echo db version v7.0.8-5 Build Info: '{' '"version":' '"7.0.8-5",' '"gitVersion":' '"197ff4d49589e8b5a444b7ebc6f4d33b6f00105e",' '"openSSLVersion":' '"OpenSSL' 1.1.1k FIPS 25 Mar '2021",' '"modules":' '[],' '"proFeatures":' '[],' '"allocator":' '"tcmalloc",' '"environment":' '{' '"distarch":' '"x86_64",' '"target_arch":' '"x86_64"' '}' '}' ++ version_info=7.0.8-5 ++ [[ ! 7.0.8-5 =~ ^([0-9]+\.){2}[0-9]+-[0-9]+$ ]] ++ echo 7.0.8-5 + FULL_VER=7.0.8-5 + MONGO_VER=7.0 + unset OPERATOR_NS + main_cluster=cross-site-sharded-main + replica_cluster=cross-site-sharded-replica + desc 'create main cluster' + set +o xtrace ----------------------------------------------------------------------------------- create main cluster ----------------------------------------------------------------------------------- + create_infra cross-site-sharded-21880 + local ns=cross-site-sharded-21880 + delete_crd + desc 'get and delete old CRDs and RBAC' + set +o xtrace ----------------------------------------------------------------------------------- get and delete old CRDs and RBAC ----------------------------------------------------------------------------------- + kubectl_bin delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1548/deploy/crd.yaml --ignore-not-found --wait=false ++ mktemp + local LAST_OUT=/tmp/tmp.j6L7oVxMm4 ++ mktemp + local LAST_ERR=/tmp/tmp.uT92xGW5r3 + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1548/deploy/crd.yaml --ignore-not-found --wait=false + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.j6L7oVxMm4 customresourcedefinition.apiextensions.k8s.io "perconaservermongodbbackups.psmdb.percona.com" deleted customresourcedefinition.apiextensions.k8s.io "perconaservermongodbrestores.psmdb.percona.com" deleted customresourcedefinition.apiextensions.k8s.io "perconaservermongodbs.psmdb.percona.com" deleted + cat /tmp/tmp.uT92xGW5r3 + rm /tmp/tmp.j6L7oVxMm4 /tmp/tmp.uT92xGW5r3 + return 0 ++ grep -v '\-\-\-' ++ yq eval .metadata.name /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1548/deploy/crd.yaml + for crd_name in '$(yq eval '\''.metadata.name'\'' "${src_dir}/deploy/crd.yaml" | grep -v '\''\-\-\-'\'')' + kubectl get perconaservermongodbbackups.psmdb.percona.com --all-namespaces -o wide + grep -v NAMESPACE + xargs -L 1 sh -xc 'kubectl patch perconaservermongodbbackups.psmdb.percona.com -n $0 $1 --type=merge -p "{\"metadata\":{\"finalizers\":[]}}"' error: the server doesn't have a resource type "perconaservermongodbbackups" + kubectl patch perconaservermongodbbackups.psmdb.percona.com -n sh --type=merge -p '{"metadata":{"finalizers":[]}}' error: the server doesn't have a resource type "perconaservermongodbbackups" + : + kubectl_bin wait --for=delete crd perconaservermongodbbackups.psmdb.percona.com ++ mktemp + local LAST_OUT=/tmp/tmp.1OyX87AAaD ++ mktemp + local LAST_ERR=/tmp/tmp.YEy3381OXp + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl wait --for=delete crd perconaservermongodbbackups.psmdb.percona.com + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.1OyX87AAaD + cat /tmp/tmp.YEy3381OXp + rm /tmp/tmp.1OyX87AAaD /tmp/tmp.YEy3381OXp + return 0 + for crd_name in '$(yq eval '\''.metadata.name'\'' "${src_dir}/deploy/crd.yaml" | grep -v '\''\-\-\-'\'')' + kubectl get perconaservermongodbrestores.psmdb.percona.com --all-namespaces -o wide + grep -v NAMESPACE + xargs -L 1 sh -xc 'kubectl patch perconaservermongodbrestores.psmdb.percona.com -n $0 $1 --type=merge -p "{\"metadata\":{\"finalizers\":[]}}"' E0513 01:50:22.248304 20312 memcache.go:287] couldn't get resource list for psmdb.percona.com/v1-10-0: the server could not find the requested resource E0513 01:50:22.248532 20312 memcache.go:287] couldn't get resource list for psmdb.percona.com/v1-11-0: the server could not find the requested resource E0513 01:50:22.248740 20312 memcache.go:287] couldn't get resource list for psmdb.percona.com/v1: the server could not find the requested resource E0513 01:50:22.248929 20312 memcache.go:287] couldn't get resource list for psmdb.percona.com/v1-12-0: the server could not find the requested resource error: the server doesn't have a resource type "perconaservermongodbrestores" + kubectl patch perconaservermongodbrestores.psmdb.percona.com -n sh --type=merge -p '{"metadata":{"finalizers":[]}}' error: the server doesn't have a resource type "perconaservermongodbrestores" + : + kubectl_bin wait --for=delete crd perconaservermongodbrestores.psmdb.percona.com ++ mktemp + local LAST_OUT=/tmp/tmp.pfXkEJop9F ++ mktemp + local LAST_ERR=/tmp/tmp.H2WP2Eg9vo + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl wait --for=delete crd perconaservermongodbrestores.psmdb.percona.com + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.pfXkEJop9F + cat /tmp/tmp.H2WP2Eg9vo + rm /tmp/tmp.pfXkEJop9F /tmp/tmp.H2WP2Eg9vo + return 0 + for crd_name in '$(yq eval '\''.metadata.name'\'' "${src_dir}/deploy/crd.yaml" | grep -v '\''\-\-\-'\'')' + grep -v NAMESPACE + kubectl get perconaservermongodbs.psmdb.percona.com --all-namespaces -o wide + xargs -L 1 sh -xc 'kubectl patch perconaservermongodbs.psmdb.percona.com -n $0 $1 --type=merge -p "{\"metadata\":{\"finalizers\":[]}}"' error: the server doesn't have a resource type "perconaservermongodbs" + kubectl patch perconaservermongodbs.psmdb.percona.com -n sh --type=merge -p '{"metadata":{"finalizers":[]}}' error: the server doesn't have a resource type "perconaservermongodbs" + : + kubectl_bin wait --for=delete crd perconaservermongodbs.psmdb.percona.com ++ mktemp + local LAST_OUT=/tmp/tmp.05S1IMy4xR ++ mktemp + local LAST_ERR=/tmp/tmp.yQzv0B4l7i + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl wait --for=delete crd perconaservermongodbs.psmdb.percona.com + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.05S1IMy4xR + cat /tmp/tmp.yQzv0B4l7i + rm /tmp/tmp.05S1IMy4xR /tmp/tmp.yQzv0B4l7i + return 0 + local rbac_yaml=rbac.yaml + '[' -n '' ']' + kubectl_bin delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1548/deploy/rbac.yaml --ignore-not-found ++ mktemp + local LAST_OUT=/tmp/tmp.D5UySjEiiU ++ mktemp + local LAST_ERR=/tmp/tmp.fmV2SrH1NB + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1548/deploy/rbac.yaml --ignore-not-found + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.D5UySjEiiU role.rbac.authorization.k8s.io "percona-server-mongodb-operator" deleted serviceaccount "percona-server-mongodb-operator" deleted rolebinding.rbac.authorization.k8s.io "service-account-percona-server-mongodb-operator" deleted + cat /tmp/tmp.fmV2SrH1NB + rm /tmp/tmp.D5UySjEiiU /tmp/tmp.fmV2SrH1NB + return 0 + check_crd_for_deletion PR-1548-d2731110 + local git_tag=PR-1548-d2731110 ++ curl -s https://raw.githubusercontent.com/percona/percona-server-mongodb-operator/PR-1548-d2731110/deploy/crd.yaml ++ /usr/bin/sed ':a;N;$!ba;s/\n/ /g' ++ /usr/bin/sed s/---//g ++ yq eval .metadata.name + for crd_name in '$(curl -s https://raw.githubusercontent.com/percona/percona-server-mongodb-operator/${git_tag}/deploy/crd.yaml | yq eval '\''.metadata.name'\'' | $sed '\''s/---//g'\'' | $sed '\'':a;N;$!ba;s/\n/ /g'\'')' ++ kubectl_bin get crd/null -o 'jsonpath={.status.conditions[-1].type}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.sSWyiMY8c9 +++ mktemp ++ local LAST_ERR=/tmp/tmp.cGa2ya3Dd4 ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get crd/null -o 'jsonpath={.status.conditions[-1].type}' ++ exit_status=1 ++ set -e ++ '[' 1 '!=' 0 -a -n 1 ']' ++ cat /tmp/tmp.sSWyiMY8c9 ++ cat /tmp/tmp.cGa2ya3Dd4 Error from server (NotFound): customresourcedefinitions.apiextensions.k8s.io "null" not found ++ sleep 0 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get crd/null -o 'jsonpath={.status.conditions[-1].type}' ++ exit_status=1 ++ set -e ++ '[' 1 '!=' 0 -a -n 1 ']' ++ cat /tmp/tmp.sSWyiMY8c9 ++ cat /tmp/tmp.cGa2ya3Dd4 Error from server (NotFound): customresourcedefinitions.apiextensions.k8s.io "null" not found ++ sleep 4 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get crd/null -o 'jsonpath={.status.conditions[-1].type}' ++ exit_status=1 ++ set -e ++ '[' 1 '!=' 0 -a -n 1 ']' ++ cat /tmp/tmp.sSWyiMY8c9 ++ cat /tmp/tmp.cGa2ya3Dd4 Error from server (NotFound): customresourcedefinitions.apiextensions.k8s.io "null" not found ++ sleep 8 ++ cat /tmp/tmp.sSWyiMY8c9 ++ cat /tmp/tmp.cGa2ya3Dd4 Error from server (NotFound): customresourcedefinitions.apiextensions.k8s.io "null" not found ++ rm /tmp/tmp.sSWyiMY8c9 /tmp/tmp.cGa2ya3Dd4 ++ return 1 + [[ '' == \T\e\r\m\i\n\a\t\i\n\g ]] + '[' -n '' ']' + create_namespace cross-site-sharded-21880 + local namespace=cross-site-sharded-21880 + local skip_clean_namespace= + [[ 1 == 1 ]] + [[ -z '' ]] + destroy_chaos_mesh ++ helm list --all-namespaces --filter chaos-mesh ++ tail -n1 ++ awk '-F ' '{print $2}' ++ sed s/NAMESPACE// + local chaos_mesh_ns= + desc 'destroy chaos-mesh' + set +o xtrace ----------------------------------------------------------------------------------- destroy chaos-mesh ----------------------------------------------------------------------------------- + '[' -n '' ']' ++ grep chaos-mesh ++ awk '{print $1}' ++ kubectl get MutatingWebhookConfiguration + timeout 30 kubectl delete MutatingWebhookConfiguration error: resource(s) were provided, but no name was specified + : ++ grep chaos-mesh ++ awk '{print $1}' ++ kubectl get ValidatingWebhookConfiguration + timeout 30 kubectl delete ValidatingWebhookConfiguration error: resource(s) were provided, but no name was specified + : ++ awk '{print $1}' ++ grep validate-auth ++ kubectl get ValidatingWebhookConfiguration + timeout 30 kubectl delete ValidatingWebhookConfiguration error: resource(s) were provided, but no name was specified + : ++ kubectl api-resources ++ grep chaos-mesh ++ awk '{print $1}' ++ grep chaos-mesh.org ++ kubectl get crd ++ awk '{print $1}' + timeout 30 kubectl delete crd error: resource(s) were provided, but no name was specified + : ++ kubectl get clusterrolebinding ++ grep chaos-mesh ++ awk '{print $1}' + timeout 30 kubectl delete clusterrolebinding error: resource(s) were provided, but no name was specified + : ++ grep chaos-mesh ++ awk '{print $1}' ++ kubectl get clusterrole + timeout 30 kubectl delete clusterrole error: resource(s) were provided, but no name was specified + : + desc 'cleaned up all old namespaces' + set +o xtrace ----------------------------------------------------------------------------------- cleaned up all old namespaces ----------------------------------------------------------------------------------- + kubectl_bin get ns + '[' -n '' ']' + desc 'cleaned up old namespaces cross-site-sharded-21880' + set +o xtrace ----------------------------------------------------------------------------------- cleaned up old namespaces cross-site-sharded-21880 ----------------------------------------------------------------------------------- + kubectl_bin delete namespace cross-site-sharded-21880 --ignore-not-found ++ mktemp + xargs kubectl delete ns + awk '{print$1}' + local LAST_OUT=/tmp/tmp.CFCgStmmG3 + egrep -v '^kube-|^default|Terminating|psmdb-operator|openshift|gke-mcs|^NAME' ++ mktemp + local LAST_ERR=/tmp/tmp.9UptRNNyJI + local exit_status=0 + local timeout=4 ++ mktemp ++ seq 0 2 + local LAST_OUT=/tmp/tmp.jENga5xl50 ++ mktemp + for i in '$(seq 0 2)' + set +e + kubectl delete namespace cross-site-sharded-21880 --ignore-not-found + local LAST_ERR=/tmp/tmp.LuPXeFuQv6 + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl get ns + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.CFCgStmmG3 + cat /tmp/tmp.9UptRNNyJI + rm /tmp/tmp.CFCgStmmG3 /tmp/tmp.9UptRNNyJI + return 0 + kubectl_bin wait --for=delete namespace cross-site-sharded-21880 ++ mktemp + local LAST_OUT=/tmp/tmp.ggey4jwbNR ++ mktemp + local LAST_ERR=/tmp/tmp.9zOD0LwD9m + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl wait --for=delete namespace cross-site-sharded-21880 + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.jENga5xl50 + cat /tmp/tmp.LuPXeFuQv6 + rm /tmp/tmp.jENga5xl50 /tmp/tmp.LuPXeFuQv6 + return 0 + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.ggey4jwbNR + cat /tmp/tmp.9zOD0LwD9m + rm /tmp/tmp.ggey4jwbNR /tmp/tmp.9zOD0LwD9m + return 0 + desc 'create namespace cross-site-sharded-21880' + set +o xtrace ----------------------------------------------------------------------------------- create namespace cross-site-sharded-21880 ----------------------------------------------------------------------------------- + kubectl_bin create namespace cross-site-sharded-21880 ++ mktemp + local LAST_OUT=/tmp/tmp.FP7IOLqJTj ++ mktemp + local LAST_ERR=/tmp/tmp.hpinyrYNm5 + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl create namespace cross-site-sharded-21880 namespace "cross-site-sharded-31358" deleted + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.FP7IOLqJTj namespace/cross-site-sharded-21880 created + cat /tmp/tmp.hpinyrYNm5 + rm /tmp/tmp.FP7IOLqJTj /tmp/tmp.hpinyrYNm5 + return 0 ++ kubectl_bin config current-context +++ mktemp ++ local LAST_OUT=/tmp/tmp.wJYan99TeS +++ mktemp ++ local LAST_ERR=/tmp/tmp.xcdVe8v0wM ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl config current-context namespace "cross-site-sharded-replica-1963" deleted ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.wJYan99TeS ++ cat /tmp/tmp.xcdVe8v0wM ++ rm /tmp/tmp.wJYan99TeS /tmp/tmp.xcdVe8v0wM ++ return 0 + kubectl_bin config set-context gke_cloud-dev-112233_us-central1-a_jen-psmdb-1548-d2731110-1-cluster4 --namespace=cross-site-sharded-21880 ++ mktemp + local LAST_OUT=/tmp/tmp.GLelTLUiEi ++ mktemp + local LAST_ERR=/tmp/tmp.bFgM0saCl1 + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl config set-context gke_cloud-dev-112233_us-central1-a_jen-psmdb-1548-d2731110-1-cluster4 --namespace=cross-site-sharded-21880 + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.GLelTLUiEi Context "gke_cloud-dev-112233_us-central1-a_jen-psmdb-1548-d2731110-1-cluster4" modified. + cat /tmp/tmp.bFgM0saCl1 + rm /tmp/tmp.GLelTLUiEi /tmp/tmp.bFgM0saCl1 + return 0 + deploy_operator + desc 'start PSMDB operator' + set +o xtrace ----------------------------------------------------------------------------------- start PSMDB operator ----------------------------------------------------------------------------------- + local cr_file + '[' -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1548/e2e-tests/cross-site-sharded/conf/crd.yaml ']' + cr_file=/mnt/jenkins/workspace/cloud-psmdb-operator_PR-1548/deploy/crd.yaml + kubectl_bin apply --server-side --force-conflicts -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1548/deploy/crd.yaml ++ mktemp + local LAST_OUT=/tmp/tmp.wmj7HLPeWo ++ mktemp + local LAST_ERR=/tmp/tmp.sTe59FVuRo + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply --server-side --force-conflicts -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1548/deploy/crd.yaml + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.wmj7HLPeWo customresourcedefinition.apiextensions.k8s.io/perconaservermongodbbackups.psmdb.percona.com serverside-applied customresourcedefinition.apiextensions.k8s.io/perconaservermongodbrestores.psmdb.percona.com serverside-applied customresourcedefinition.apiextensions.k8s.io/perconaservermongodbs.psmdb.percona.com serverside-applied + cat /tmp/tmp.sTe59FVuRo + rm /tmp/tmp.wmj7HLPeWo /tmp/tmp.sTe59FVuRo + return 0 + '[' -n '' ']' + apply_rbac rbac + local operator_namespace=psmdb-operator + local rbac=rbac + cat /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1548/deploy/rbac.yaml + sed -e 's^namespace: .*^namespace: psmdb-operator^' + kubectl_bin apply -f - ++ mktemp + local LAST_OUT=/tmp/tmp.z0q6voJqRl ++ mktemp + local LAST_ERR=/tmp/tmp.AqNVxGs2xV + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.z0q6voJqRl role.rbac.authorization.k8s.io/percona-server-mongodb-operator created serviceaccount/percona-server-mongodb-operator created rolebinding.rbac.authorization.k8s.io/service-account-percona-server-mongodb-operator created + cat /tmp/tmp.AqNVxGs2xV + rm /tmp/tmp.z0q6voJqRl /tmp/tmp.AqNVxGs2xV + return 0 + kubectl_bin apply -f - + yq eval ' (.spec.template.spec.containers[].image = "perconalab/percona-server-mongodb-operator:PR-1548-d2731110") | ((.. | select(.[] == "DISABLE_TELEMETRY")) |= .value="true") | ((.. | select(.[] == "LOG_LEVEL")) |= .value="DEBUG")' /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1548/deploy/operator.yaml ++ mktemp + local LAST_OUT=/tmp/tmp.aIJffPDwQy ++ mktemp + local LAST_ERR=/tmp/tmp.MCoHDxywIE + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.aIJffPDwQy deployment.apps/percona-server-mongodb-operator created + cat /tmp/tmp.MCoHDxywIE + rm /tmp/tmp.aIJffPDwQy /tmp/tmp.MCoHDxywIE + return 0 + sleep 2 ++ get_operator_pod ++ kubectl_bin get pods --selector=name=percona-server-mongodb-operator -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.dMWghn9QeI +++ mktemp ++ local LAST_ERR=/tmp/tmp.Y9t6qU22cY ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=percona-server-mongodb-operator -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.dMWghn9QeI ++ cat /tmp/tmp.Y9t6qU22cY ++ rm /tmp/tmp.dMWghn9QeI /tmp/tmp.Y9t6qU22cY ++ return 0 + wait_pod percona-server-mongodb-operator-56887dd869-ctft6 + local pod=percona-server-mongodb-operator-56887dd869-ctft6 + set +o xtrace waiting for pod/percona-server-mongodb-operator-56887dd869-ctft6 to be ready.OK + desc 'create secrets and start client' + set +o xtrace ----------------------------------------------------------------------------------- create secrets and start client ----------------------------------------------------------------------------------- + kubectl_bin apply -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1548/e2e-tests/conf/client.yml -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1548/e2e-tests/cross-site-sharded/conf/secrets.yml ++ mktemp + local LAST_OUT=/tmp/tmp.4UUuhDKDDA ++ mktemp + local LAST_ERR=/tmp/tmp.UmepOltKrh + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1548/e2e-tests/conf/client.yml -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1548/e2e-tests/cross-site-sharded/conf/secrets.yml + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.4UUuhDKDDA deployment.apps/psmdb-client created secret/cross-site-sharded-main-secrets created secret/cross-site-sharded-main-ssl created secret/cross-site-sharded-main-ssl-internal created + cat /tmp/tmp.UmepOltKrh + rm /tmp/tmp.4UUuhDKDDA /tmp/tmp.UmepOltKrh + return 0 + desc 'create main PSMDB cluster cross-site-sharded-main.' + set +o xtrace ----------------------------------------------------------------------------------- create main PSMDB cluster cross-site-sharded-main. ----------------------------------------------------------------------------------- + apply_cluster /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1548/e2e-tests/cross-site-sharded/conf/cross-site-sharded-main.yml + '[' -z '' ']' + cat_config /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1548/e2e-tests/cross-site-sharded/conf/cross-site-sharded-main.yml + cat /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1548/e2e-tests/cross-site-sharded/conf/cross-site-sharded-main.yml + yq eval '(.spec | select(has("pmm"))).pmm.image = "perconalab/pmm-client:dev-latest"' + yq eval '(.spec | select(has("initImage"))).initImage = "perconalab/percona-server-mongodb-operator:PR-1548-d2731110"' + kubectl_bin apply -f - + yq eval '.spec.upgradeOptions.apply="Never"' + yq eval '(.spec | select(has("backup"))).backup.image = "perconalab/percona-server-mongodb-operator:main-backup"' ++ mktemp + yq eval '(.spec | select(.image == null)).image = "perconalab/percona-server-mongodb-operator:main-mongod7.0"' + local LAST_OUT=/tmp/tmp.WZzyXcOMEv ++ mktemp + local LAST_ERR=/tmp/tmp.H3nTBG1cYI + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.WZzyXcOMEv perconaservermongodb.psmdb.percona.com/cross-site-sharded-main created + cat /tmp/tmp.H3nTBG1cYI + rm /tmp/tmp.WZzyXcOMEv /tmp/tmp.H3nTBG1cYI + return 0 + desc 'check if all 3 Pods started' + set +o xtrace ----------------------------------------------------------------------------------- check if all 3 Pods started ----------------------------------------------------------------------------------- + wait_for_running cross-site-sharded-main-rs0 3 + local name=cross-site-sharded-main-rs0 + let last_pod=2 + local check_cluster_readyness=true + set_debug + [[ 1 == 1 ]] + set -o xtrace + local rs_name=rs0 + local cluster_name=cross-site-sharded-main ++ seq 0 2 + for i in '$(seq 0 $last_pod)' + [[ 0 -eq 2 ]] + wait_pod cross-site-sharded-main-rs0-0 + local pod=cross-site-sharded-main-rs0-0 + set +o xtrace waiting for pod/cross-site-sharded-main-rs0-0 to be ready..............OK + for i in '$(seq 0 $last_pod)' + [[ 1 -eq 2 ]] + wait_pod cross-site-sharded-main-rs0-1 + local pod=cross-site-sharded-main-rs0-1 + set +o xtrace waiting for pod/cross-site-sharded-main-rs0-1 to be ready............OK + for i in '$(seq 0 $last_pod)' + [[ 2 -eq 2 ]] ++ kubectl_bin get psmdb cross-site-sharded-main -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].arbiter.enabled}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.6h1R0S3KCC +++ mktemp ++ local LAST_ERR=/tmp/tmp.4AMj0HiCWu ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get psmdb cross-site-sharded-main -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].arbiter.enabled}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.6h1R0S3KCC ++ cat /tmp/tmp.4AMj0HiCWu ++ rm /tmp/tmp.6h1R0S3KCC /tmp/tmp.4AMj0HiCWu ++ return 0 + [[ false == \t\r\u\e ]] + wait_pod cross-site-sharded-main-rs0-2 + local pod=cross-site-sharded-main-rs0-2 + set +o xtrace waiting for pod/cross-site-sharded-main-rs0-2 to be ready............OK ++ kubectl_bin get psmdb cross-site-sharded-main -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].non_voting.enabled}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.M1dLf0Lz2m +++ mktemp ++ local LAST_ERR=/tmp/tmp.dfDluSs7qy ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get psmdb cross-site-sharded-main -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].non_voting.enabled}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.M1dLf0Lz2m ++ cat /tmp/tmp.dfDluSs7qy ++ rm /tmp/tmp.M1dLf0Lz2m /tmp/tmp.dfDluSs7qy ++ return 0 + [[ '' == \t\r\u\e ]] + sleep 10 + [[ true == \t\r\u\e ]] + set +x Waiting for cluster readyness....................................... + wait_for_running cross-site-sharded-main-cfg 3 false + local name=cross-site-sharded-main-cfg + let last_pod=2 + local check_cluster_readyness=false + set_debug + [[ 1 == 1 ]] + set -o xtrace + local rs_name=cfg + local cluster_name=cross-site-sharded-main ++ seq 0 2 + for i in '$(seq 0 $last_pod)' + [[ 0 -eq 2 ]] + wait_pod cross-site-sharded-main-cfg-0 + local pod=cross-site-sharded-main-cfg-0 + set +o xtrace waiting for pod/cross-site-sharded-main-cfg-0 to be ready.OK + for i in '$(seq 0 $last_pod)' + [[ 1 -eq 2 ]] + wait_pod cross-site-sharded-main-cfg-1 + local pod=cross-site-sharded-main-cfg-1 + set +o xtrace waiting for pod/cross-site-sharded-main-cfg-1 to be ready.OK + for i in '$(seq 0 $last_pod)' + [[ 2 -eq 2 ]] ++ kubectl_bin get psmdb cross-site-sharded-main -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].arbiter.enabled}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.wXL6dnjn0R +++ mktemp ++ local LAST_ERR=/tmp/tmp.p8y7AeDdqh ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get psmdb cross-site-sharded-main -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].arbiter.enabled}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.wXL6dnjn0R ++ cat /tmp/tmp.p8y7AeDdqh ++ rm /tmp/tmp.wXL6dnjn0R /tmp/tmp.p8y7AeDdqh ++ return 0 + [[ '' == \t\r\u\e ]] + wait_pod cross-site-sharded-main-cfg-2 + local pod=cross-site-sharded-main-cfg-2 + set +o xtrace waiting for pod/cross-site-sharded-main-cfg-2 to be ready.OK ++ kubectl_bin get psmdb cross-site-sharded-main -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].non_voting.enabled}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.8roOzyRRiD +++ mktemp ++ local LAST_ERR=/tmp/tmp.1f3SMvuXXP ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get psmdb cross-site-sharded-main -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].non_voting.enabled}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.8roOzyRRiD ++ cat /tmp/tmp.1f3SMvuXXP ++ rm /tmp/tmp.8roOzyRRiD /tmp/tmp.1f3SMvuXXP ++ return 0 + [[ '' == \t\r\u\e ]] + sleep 10 + [[ false == \t\r\u\e ]] + desc 'create user' + set +o xtrace ----------------------------------------------------------------------------------- create user ----------------------------------------------------------------------------------- + run_mongos 'db.createUser({user:"user",pwd:"pass",roles:[{db:"app",role:"readWrite"}]})' userAdmin:userAdmin123456@cross-site-sharded-main-mongos.cross-site-sharded-21880 + local 'command=db.createUser({user:"user",pwd:"pass",roles:[{db:"app",role:"readWrite"}]})' + local uri=userAdmin:userAdmin123456@cross-site-sharded-main-mongos.cross-site-sharded-21880 + local driver=mongodb + local suffix=.svc.cluster.local ++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.b6bzo6dxXJ +++ mktemp ++ local LAST_ERR=/tmp/tmp.L16bTvgc8t ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.b6bzo6dxXJ ++ cat /tmp/tmp.L16bTvgc8t ++ rm /tmp/tmp.b6bzo6dxXJ /tmp/tmp.L16bTvgc8t ++ return 0 + local client_container=psmdb-client-7469665986-xhdrq + local mongo_flag= + kubectl_bin exec psmdb-client-7469665986-xhdrq -- bash -c 'printf '\''db.createUser({user:"user",pwd:"pass",roles:[{db:"app",role:"readWrite"}]})\n'\'' | mongo mongodb://userAdmin:userAdmin123456@cross-site-sharded-main-mongos.cross-site-sharded-21880.svc.cluster.local/admin ' ++ mktemp + local LAST_OUT=/tmp/tmp.CQvDFAo8c9 ++ mktemp + local LAST_ERR=/tmp/tmp.5DG71iMO5D + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl exec psmdb-client-7469665986-xhdrq -- bash -c 'printf '\''db.createUser({user:"user",pwd:"pass",roles:[{db:"app",role:"readWrite"}]})\n'\'' | mongo mongodb://userAdmin:userAdmin123456@cross-site-sharded-main-mongos.cross-site-sharded-21880.svc.cluster.local/admin ' + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.CQvDFAo8c9 Percona Server for MongoDB shell version v4.4.29-28 connecting to: mongodb://cross-site-sharded-main-mongos.cross-site-sharded-21880.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb Implicit session: session { "id" : UUID("72d2d022-3945-4025-a5f7-fa4a3601201e") } Percona Server for MongoDB server version: v7.0.8-5 WARNING: shell and server versions do not match Successfully added user: { "user" : "user", "roles" : [ { "db" : "app", "role" : "readWrite" } ] } bye + cat /tmp/tmp.5DG71iMO5D + rm /tmp/tmp.CQvDFAo8c9 /tmp/tmp.5DG71iMO5D + return 0 + sleep 2 + desc 'set chunk size to 2 MB' + set +o xtrace ----------------------------------------------------------------------------------- set chunk size to 2 MB ----------------------------------------------------------------------------------- + run_mongos 'use config\n db.settings.save( { _id:"chunksize", value: 2 } )' clusterAdmin:clusterAdmin123456@cross-site-sharded-main-mongos.cross-site-sharded-21880 + local 'command=use config\n db.settings.save( { _id:"chunksize", value: 2 } )' + local uri=clusterAdmin:clusterAdmin123456@cross-site-sharded-main-mongos.cross-site-sharded-21880 + local driver=mongodb + local suffix=.svc.cluster.local ++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.ercAykBlG9 +++ mktemp ++ local LAST_ERR=/tmp/tmp.0CCI9mUsJp ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.ercAykBlG9 ++ cat /tmp/tmp.0CCI9mUsJp ++ rm /tmp/tmp.ercAykBlG9 /tmp/tmp.0CCI9mUsJp ++ return 0 + local client_container=psmdb-client-7469665986-xhdrq + local mongo_flag= + kubectl_bin exec psmdb-client-7469665986-xhdrq -- bash -c 'printf '\''use config\n db.settings.save( { _id:"chunksize", value: 2 } )\n'\'' | mongo mongodb://clusterAdmin:clusterAdmin123456@cross-site-sharded-main-mongos.cross-site-sharded-21880.svc.cluster.local/admin ' ++ mktemp + local LAST_OUT=/tmp/tmp.1aEi1phfVC ++ mktemp + local LAST_ERR=/tmp/tmp.JDpOJuklAw + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl exec psmdb-client-7469665986-xhdrq -- bash -c 'printf '\''use config\n db.settings.save( { _id:"chunksize", value: 2 } )\n'\'' | mongo mongodb://clusterAdmin:clusterAdmin123456@cross-site-sharded-main-mongos.cross-site-sharded-21880.svc.cluster.local/admin ' + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.1aEi1phfVC Percona Server for MongoDB shell version v4.4.29-28 connecting to: mongodb://cross-site-sharded-main-mongos.cross-site-sharded-21880.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb Implicit session: session { "id" : UUID("68f794fc-2f69-4d43-9cf5-66670cc6f793") } Percona Server for MongoDB server version: v7.0.8-5 WARNING: shell and server versions do not match switched to db config WriteResult({ "nMatched" : 0, "nUpserted" : 1, "nModified" : 0, "_id" : "chunksize" }) bye + cat /tmp/tmp.JDpOJuklAw + rm /tmp/tmp.1aEi1phfVC /tmp/tmp.JDpOJuklAw + return 0 + sleep 2 + desc 'write data' + set +o xtrace ----------------------------------------------------------------------------------- write data ----------------------------------------------------------------------------------- + run_script_mongos /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1548/e2e-tests/cross-site-sharded/data.js user:pass@cross-site-sharded-main-mongos.cross-site-sharded-21880 + local script=/mnt/jenkins/workspace/cloud-psmdb-operator_PR-1548/e2e-tests/cross-site-sharded/data.js + local uri=user:pass@cross-site-sharded-main-mongos.cross-site-sharded-21880 + local driver=mongodb + local suffix=.svc.cluster.local ++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.vD04Jhflty +++ mktemp ++ local LAST_ERR=/tmp/tmp.TDKHbzjs1o ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.vD04Jhflty ++ cat /tmp/tmp.TDKHbzjs1o ++ rm /tmp/tmp.vD04Jhflty /tmp/tmp.TDKHbzjs1o ++ return 0 + local client_container=psmdb-client-7469665986-xhdrq + local mongo_flag= ++ basename /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1548/e2e-tests/cross-site-sharded/data.js + name=data.js + kubectl_bin cp /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1548/e2e-tests/cross-site-sharded/data.js cross-site-sharded-21880/psmdb-client-7469665986-xhdrq:/tmp ++ mktemp + local LAST_OUT=/tmp/tmp.78WV3VVyP3 ++ mktemp + local LAST_ERR=/tmp/tmp.vc4j3o9v6g + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl cp /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1548/e2e-tests/cross-site-sharded/data.js cross-site-sharded-21880/psmdb-client-7469665986-xhdrq:/tmp + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.78WV3VVyP3 + cat /tmp/tmp.vc4j3o9v6g + rm /tmp/tmp.78WV3VVyP3 /tmp/tmp.vc4j3o9v6g + return 0 + kubectl_bin exec psmdb-client-7469665986-xhdrq -- bash -c 'mongo mongodb://user:pass@cross-site-sharded-main-mongos.cross-site-sharded-21880.svc.cluster.local/admin /tmp/data.js' ++ mktemp + local LAST_OUT=/tmp/tmp.a1kqlEsDB8 ++ mktemp + local LAST_ERR=/tmp/tmp.R1xHgQhUnX + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl exec psmdb-client-7469665986-xhdrq -- bash -c 'mongo mongodb://user:pass@cross-site-sharded-main-mongos.cross-site-sharded-21880.svc.cluster.local/admin /tmp/data.js' + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.a1kqlEsDB8 Percona Server for MongoDB shell version v4.4.29-28 connecting to: mongodb://cross-site-sharded-main-mongos.cross-site-sharded-21880.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb Implicit session: session { "id" : UUID("680eb6a1-e6a9-4c53-a9a1-18739cf68c71") } Percona Server for MongoDB server version: v7.0.8-5 WARNING: shell and server versions do not match + cat /tmp/tmp.R1xHgQhUnX + rm /tmp/tmp.a1kqlEsDB8 /tmp/tmp.R1xHgQhUnX + return 0 + desc 'shard collection' + set +o xtrace ----------------------------------------------------------------------------------- shard collection ----------------------------------------------------------------------------------- + run_mongos 'sh.enableSharding("app")' clusterAdmin:clusterAdmin123456@cross-site-sharded-main-mongos.cross-site-sharded-21880 + local 'command=sh.enableSharding("app")' + local uri=clusterAdmin:clusterAdmin123456@cross-site-sharded-main-mongos.cross-site-sharded-21880 + local driver=mongodb + local suffix=.svc.cluster.local ++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.463MR3x9Op +++ mktemp ++ local LAST_ERR=/tmp/tmp.yeQOOZaxck ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.463MR3x9Op ++ cat /tmp/tmp.yeQOOZaxck ++ rm /tmp/tmp.463MR3x9Op /tmp/tmp.yeQOOZaxck ++ return 0 + local client_container=psmdb-client-7469665986-xhdrq + local mongo_flag= + kubectl_bin exec psmdb-client-7469665986-xhdrq -- bash -c 'printf '\''sh.enableSharding("app")\n'\'' | mongo mongodb://clusterAdmin:clusterAdmin123456@cross-site-sharded-main-mongos.cross-site-sharded-21880.svc.cluster.local/admin ' ++ mktemp + local LAST_OUT=/tmp/tmp.tyNr4Nnakk ++ mktemp + local LAST_ERR=/tmp/tmp.gYHAnfi9ZT + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl exec psmdb-client-7469665986-xhdrq -- bash -c 'printf '\''sh.enableSharding("app")\n'\'' | mongo mongodb://clusterAdmin:clusterAdmin123456@cross-site-sharded-main-mongos.cross-site-sharded-21880.svc.cluster.local/admin ' + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.tyNr4Nnakk Percona Server for MongoDB shell version v4.4.29-28 connecting to: mongodb://cross-site-sharded-main-mongos.cross-site-sharded-21880.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb Implicit session: session { "id" : UUID("3c5d8671-04e7-4544-b838-42e6a7ad7d53") } Percona Server for MongoDB server version: v7.0.8-5 WARNING: shell and server versions do not match { "ok" : 1, "$clusterTime" : { "clusterTime" : Timestamp(1715565340, 1), "signature" : { "hash" : BinData(0,"bOLp9TpkDwGOi426Dn9yBXcC/2Q="), "keyId" : NumberLong("7368296217702301711") } }, "operationTime" : Timestamp(1715565340, 1) } bye + cat /tmp/tmp.gYHAnfi9ZT + rm /tmp/tmp.tyNr4Nnakk /tmp/tmp.gYHAnfi9ZT + return 0 + sleep 2 + run_mongos 'sh.shardCollection("app.city", { _id: 1 } )' clusterAdmin:clusterAdmin123456@cross-site-sharded-main-mongos.cross-site-sharded-21880 + local 'command=sh.shardCollection("app.city", { _id: 1 } )' + local uri=clusterAdmin:clusterAdmin123456@cross-site-sharded-main-mongos.cross-site-sharded-21880 + local driver=mongodb + local suffix=.svc.cluster.local ++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.bBzHh8kx56 +++ mktemp ++ local LAST_ERR=/tmp/tmp.DMElD0ocfO ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.bBzHh8kx56 ++ cat /tmp/tmp.DMElD0ocfO ++ rm /tmp/tmp.bBzHh8kx56 /tmp/tmp.DMElD0ocfO ++ return 0 + local client_container=psmdb-client-7469665986-xhdrq + local mongo_flag= + kubectl_bin exec psmdb-client-7469665986-xhdrq -- bash -c 'printf '\''sh.shardCollection("app.city", { _id: 1 } )\n'\'' | mongo mongodb://clusterAdmin:clusterAdmin123456@cross-site-sharded-main-mongos.cross-site-sharded-21880.svc.cluster.local/admin ' ++ mktemp + local LAST_OUT=/tmp/tmp.ObZlZEoAM5 ++ mktemp + local LAST_ERR=/tmp/tmp.DKMMG0t53R + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl exec psmdb-client-7469665986-xhdrq -- bash -c 'printf '\''sh.shardCollection("app.city", { _id: 1 } )\n'\'' | mongo mongodb://clusterAdmin:clusterAdmin123456@cross-site-sharded-main-mongos.cross-site-sharded-21880.svc.cluster.local/admin ' + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.ObZlZEoAM5 Percona Server for MongoDB shell version v4.4.29-28 connecting to: mongodb://cross-site-sharded-main-mongos.cross-site-sharded-21880.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb Implicit session: session { "id" : UUID("ffaa6d07-c4f0-4256-9d36-4011e398fd34") } Percona Server for MongoDB server version: v7.0.8-5 WARNING: shell and server versions do not match { "collectionsharded" : "app.city", "ok" : 1, "$clusterTime" : { "clusterTime" : Timestamp(1715565346, 35), "signature" : { "hash" : BinData(0,"excdxCXCaRc4RxUPbtamYvbC+D4="), "keyId" : NumberLong("7368296217702301711") } }, "operationTime" : Timestamp(1715565346, 35) } bye + cat /tmp/tmp.DKMMG0t53R + rm /tmp/tmp.ObZlZEoAM5 /tmp/tmp.DKMMG0t53R + return 0 + sleep 120 + desc 'Check chunks' + set +o xtrace ----------------------------------------------------------------------------------- Check chunks ----------------------------------------------------------------------------------- + chunks_param1=ns + chunks_param2='"app.city"' + [[ 7.0 != \4\.\4 ]] + chunks_param1=uuid ++ run_mongos 'use app\n db.getCollectionInfos({ "name": "city" })[0].info.uuid' user:pass@cross-site-sharded-main-mongos.cross-site-sharded-21880 ++ local 'command=use app\n db.getCollectionInfos({ "name": "city" })[0].info.uuid' ++ local uri=user:pass@cross-site-sharded-main-mongos.cross-site-sharded-21880 ++ local driver=mongodb ++ local suffix=.svc.cluster.local ++ grep -v 'switched to db app' ++ grep 'switched to db app' -A 1 +++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.UDzExsy39o ++++ mktemp +++ local LAST_ERR=/tmp/tmp.qRoxLti1ZL +++ local exit_status=0 +++ local timeout=4 ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ set +e +++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 -a -n 1 ']' +++ break +++ cat /tmp/tmp.UDzExsy39o +++ cat /tmp/tmp.qRoxLti1ZL +++ rm /tmp/tmp.UDzExsy39o /tmp/tmp.qRoxLti1ZL +++ return 0 ++ local client_container=psmdb-client-7469665986-xhdrq ++ local mongo_flag= ++ kubectl_bin exec psmdb-client-7469665986-xhdrq -- bash -c 'printf '\''use app\n db.getCollectionInfos({ "name": "city" })[0].info.uuid\n'\'' | mongo mongodb://user:pass@cross-site-sharded-main-mongos.cross-site-sharded-21880.svc.cluster.local/admin ' +++ mktemp ++ local LAST_OUT=/tmp/tmp.yhCSUYWYDU +++ mktemp ++ local LAST_ERR=/tmp/tmp.zrKEpksGnL ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl exec psmdb-client-7469665986-xhdrq -- bash -c 'printf '\''use app\n db.getCollectionInfos({ "name": "city" })[0].info.uuid\n'\'' | mongo mongodb://user:pass@cross-site-sharded-main-mongos.cross-site-sharded-21880.svc.cluster.local/admin ' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.yhCSUYWYDU ++ cat /tmp/tmp.zrKEpksGnL ++ rm /tmp/tmp.yhCSUYWYDU /tmp/tmp.zrKEpksGnL ++ return 0 + chunks_param2='UUID("102ab4d1-f978-4bb6-82f6-3542af65eeb8")' + shards=0 + for i in '"rs0"' '"rs1"' ++ run_mongos 'use config\n db.chunks.count({"uuid": UUID("102ab4d1-f978-4bb6-82f6-3542af65eeb8"), "shard": "rs0"})' clusterAdmin:clusterAdmin123456@cross-site-sharded-main-mongos.cross-site-sharded-21880 ++ local 'command=use config\n db.chunks.count({"uuid": UUID("102ab4d1-f978-4bb6-82f6-3542af65eeb8"), "shard": "rs0"})' ++ local uri=clusterAdmin:clusterAdmin123456@cross-site-sharded-main-mongos.cross-site-sharded-21880 ++ local driver=mongodb ++ local suffix=.svc.cluster.local +++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.0iElMMPBFS ++++ mktemp +++ local LAST_ERR=/tmp/tmp.vRanFL5Nw2 +++ local exit_status=0 +++ local timeout=4 ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ set +e +++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++ grep -v 'switched to db config' ++ grep 'switched to db config' -A 1 +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 -a -n 1 ']' +++ break +++ cat /tmp/tmp.0iElMMPBFS +++ cat /tmp/tmp.vRanFL5Nw2 +++ rm /tmp/tmp.0iElMMPBFS /tmp/tmp.vRanFL5Nw2 +++ return 0 ++ local client_container=psmdb-client-7469665986-xhdrq ++ local mongo_flag= ++ kubectl_bin exec psmdb-client-7469665986-xhdrq -- bash -c 'printf '\''use config\n db.chunks.count({"uuid": UUID("102ab4d1-f978-4bb6-82f6-3542af65eeb8"), "shard": "rs0"})\n'\'' | mongo mongodb://clusterAdmin:clusterAdmin123456@cross-site-sharded-main-mongos.cross-site-sharded-21880.svc.cluster.local/admin ' +++ mktemp ++ local LAST_OUT=/tmp/tmp.dXV0FryVh2 +++ mktemp ++ local LAST_ERR=/tmp/tmp.6FR5ytACTq ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl exec psmdb-client-7469665986-xhdrq -- bash -c 'printf '\''use config\n db.chunks.count({"uuid": UUID("102ab4d1-f978-4bb6-82f6-3542af65eeb8"), "shard": "rs0"})\n'\'' | mongo mongodb://clusterAdmin:clusterAdmin123456@cross-site-sharded-main-mongos.cross-site-sharded-21880.svc.cluster.local/admin ' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.dXV0FryVh2 ++ cat /tmp/tmp.6FR5ytACTq ++ rm /tmp/tmp.dXV0FryVh2 /tmp/tmp.6FR5ytACTq ++ return 0 + out=3 + desc 'rs0 has 3 chunks' + set +o xtrace ----------------------------------------------------------------------------------- rs0 has 3 chunks ----------------------------------------------------------------------------------- + [[ 3 -ne 0 ]] + (( shards = shards + 1 )) + for i in '"rs0"' '"rs1"' ++ run_mongos 'use config\n db.chunks.count({"uuid": UUID("102ab4d1-f978-4bb6-82f6-3542af65eeb8"), "shard": "rs1"})' clusterAdmin:clusterAdmin123456@cross-site-sharded-main-mongos.cross-site-sharded-21880 ++ local 'command=use config\n db.chunks.count({"uuid": UUID("102ab4d1-f978-4bb6-82f6-3542af65eeb8"), "shard": "rs1"})' ++ local uri=clusterAdmin:clusterAdmin123456@cross-site-sharded-main-mongos.cross-site-sharded-21880 ++ local driver=mongodb ++ local suffix=.svc.cluster.local +++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.wQWZv6yFp6 ++++ mktemp +++ local LAST_ERR=/tmp/tmp.mTrh4wGOc5 +++ local exit_status=0 +++ local timeout=4 ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ set +e +++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++ grep 'switched to db config' -A 1 ++ grep -v 'switched to db config' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 -a -n 1 ']' +++ break +++ cat /tmp/tmp.wQWZv6yFp6 +++ cat /tmp/tmp.mTrh4wGOc5 +++ rm /tmp/tmp.wQWZv6yFp6 /tmp/tmp.mTrh4wGOc5 +++ return 0 ++ local client_container=psmdb-client-7469665986-xhdrq ++ local mongo_flag= ++ kubectl_bin exec psmdb-client-7469665986-xhdrq -- bash -c 'printf '\''use config\n db.chunks.count({"uuid": UUID("102ab4d1-f978-4bb6-82f6-3542af65eeb8"), "shard": "rs1"})\n'\'' | mongo mongodb://clusterAdmin:clusterAdmin123456@cross-site-sharded-main-mongos.cross-site-sharded-21880.svc.cluster.local/admin ' +++ mktemp ++ local LAST_OUT=/tmp/tmp.yRnrhNWZLa +++ mktemp ++ local LAST_ERR=/tmp/tmp.5n0m4vgL0C ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl exec psmdb-client-7469665986-xhdrq -- bash -c 'printf '\''use config\n db.chunks.count({"uuid": UUID("102ab4d1-f978-4bb6-82f6-3542af65eeb8"), "shard": "rs1"})\n'\'' | mongo mongodb://clusterAdmin:clusterAdmin123456@cross-site-sharded-main-mongos.cross-site-sharded-21880.svc.cluster.local/admin ' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.yRnrhNWZLa ++ cat /tmp/tmp.5n0m4vgL0C ++ rm /tmp/tmp.yRnrhNWZLa /tmp/tmp.5n0m4vgL0C ++ return 0 + out=1 + desc 'rs1 has 1 chunks' + set +o xtrace ----------------------------------------------------------------------------------- rs1 has 1 chunks ----------------------------------------------------------------------------------- + [[ 1 -ne 0 ]] + (( shards = shards + 1 )) + [[ 2 -lt 2 ]] + desc 'create replica cluster' + set +o xtrace ----------------------------------------------------------------------------------- create replica cluster ----------------------------------------------------------------------------------- + create_namespace cross-site-sharded-replica-17201 0 + local namespace=cross-site-sharded-replica-17201 + local skip_clean_namespace=0 + [[ 1 == 1 ]] + [[ -z 0 ]] + '[' -n '' ']' + desc 'cleaned up old namespaces cross-site-sharded-replica-17201' + set +o xtrace ----------------------------------------------------------------------------------- cleaned up old namespaces cross-site-sharded-replica-17201 ----------------------------------------------------------------------------------- + kubectl_bin delete namespace cross-site-sharded-replica-17201 --ignore-not-found ++ mktemp + local LAST_OUT=/tmp/tmp.unKcJL3ObG ++ mktemp + local LAST_ERR=/tmp/tmp.VwdVEN7Mp2 + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl delete namespace cross-site-sharded-replica-17201 --ignore-not-found + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.unKcJL3ObG + cat /tmp/tmp.VwdVEN7Mp2 + rm /tmp/tmp.unKcJL3ObG /tmp/tmp.VwdVEN7Mp2 + return 0 + kubectl_bin wait --for=delete namespace cross-site-sharded-replica-17201 ++ mktemp + local LAST_OUT=/tmp/tmp.K4JBRvpwGu ++ mktemp + local LAST_ERR=/tmp/tmp.cTynsVQVJ9 + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl wait --for=delete namespace cross-site-sharded-replica-17201 + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.K4JBRvpwGu + cat /tmp/tmp.cTynsVQVJ9 + rm /tmp/tmp.K4JBRvpwGu /tmp/tmp.cTynsVQVJ9 + return 0 + desc 'create namespace cross-site-sharded-replica-17201' + set +o xtrace ----------------------------------------------------------------------------------- create namespace cross-site-sharded-replica-17201 ----------------------------------------------------------------------------------- + kubectl_bin create namespace cross-site-sharded-replica-17201 ++ mktemp + local LAST_OUT=/tmp/tmp.tT8wYYZenU ++ mktemp + local LAST_ERR=/tmp/tmp.IJuN2aQtR5 + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl create namespace cross-site-sharded-replica-17201 + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.tT8wYYZenU namespace/cross-site-sharded-replica-17201 created + cat /tmp/tmp.IJuN2aQtR5 + rm /tmp/tmp.tT8wYYZenU /tmp/tmp.IJuN2aQtR5 + return 0 ++ kubectl_bin config current-context +++ mktemp ++ local LAST_OUT=/tmp/tmp.dUAMI7u5ou +++ mktemp ++ local LAST_ERR=/tmp/tmp.gZ0iazkyQR ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl config current-context ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.dUAMI7u5ou ++ cat /tmp/tmp.gZ0iazkyQR ++ rm /tmp/tmp.dUAMI7u5ou /tmp/tmp.gZ0iazkyQR ++ return 0 + kubectl_bin config set-context gke_cloud-dev-112233_us-central1-a_jen-psmdb-1548-d2731110-1-cluster4 --namespace=cross-site-sharded-replica-17201 ++ mktemp + local LAST_OUT=/tmp/tmp.jec78hOsig ++ mktemp + local LAST_ERR=/tmp/tmp.komnDeGdIW + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl config set-context gke_cloud-dev-112233_us-central1-a_jen-psmdb-1548-d2731110-1-cluster4 --namespace=cross-site-sharded-replica-17201 + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.jec78hOsig Context "gke_cloud-dev-112233_us-central1-a_jen-psmdb-1548-d2731110-1-cluster4" modified. + cat /tmp/tmp.komnDeGdIW + rm /tmp/tmp.jec78hOsig /tmp/tmp.komnDeGdIW + return 0 + deploy_operator + desc 'start PSMDB operator' + set +o xtrace ----------------------------------------------------------------------------------- start PSMDB operator ----------------------------------------------------------------------------------- + local cr_file + '[' -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1548/e2e-tests/cross-site-sharded/conf/crd.yaml ']' + cr_file=/mnt/jenkins/workspace/cloud-psmdb-operator_PR-1548/deploy/crd.yaml + kubectl_bin apply --server-side --force-conflicts -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1548/deploy/crd.yaml ++ mktemp + local LAST_OUT=/tmp/tmp.c9d7PlBGIL ++ mktemp + local LAST_ERR=/tmp/tmp.kUSs6xmCiP + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply --server-side --force-conflicts -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1548/deploy/crd.yaml + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.c9d7PlBGIL customresourcedefinition.apiextensions.k8s.io/perconaservermongodbbackups.psmdb.percona.com serverside-applied customresourcedefinition.apiextensions.k8s.io/perconaservermongodbrestores.psmdb.percona.com serverside-applied customresourcedefinition.apiextensions.k8s.io/perconaservermongodbs.psmdb.percona.com serverside-applied + cat /tmp/tmp.kUSs6xmCiP + rm /tmp/tmp.c9d7PlBGIL /tmp/tmp.kUSs6xmCiP + return 0 + '[' -n '' ']' + apply_rbac rbac + local operator_namespace=psmdb-operator + local rbac=rbac + cat /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1548/deploy/rbac.yaml + kubectl_bin apply -f - ++ mktemp + local LAST_OUT=/tmp/tmp.QAm4ld43ae + sed -e 's^namespace: .*^namespace: psmdb-operator^' ++ mktemp + local LAST_ERR=/tmp/tmp.jitnH4f4RJ + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.QAm4ld43ae role.rbac.authorization.k8s.io/percona-server-mongodb-operator created serviceaccount/percona-server-mongodb-operator created rolebinding.rbac.authorization.k8s.io/service-account-percona-server-mongodb-operator created + cat /tmp/tmp.jitnH4f4RJ + rm /tmp/tmp.QAm4ld43ae /tmp/tmp.jitnH4f4RJ + return 0 + yq eval ' (.spec.template.spec.containers[].image = "perconalab/percona-server-mongodb-operator:PR-1548-d2731110") | ((.. | select(.[] == "DISABLE_TELEMETRY")) |= .value="true") | ((.. | select(.[] == "LOG_LEVEL")) |= .value="DEBUG")' /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1548/deploy/operator.yaml + kubectl_bin apply -f - ++ mktemp + local LAST_OUT=/tmp/tmp.xoQckEaHEi ++ mktemp + local LAST_ERR=/tmp/tmp.NmP65n0XdW + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.xoQckEaHEi deployment.apps/percona-server-mongodb-operator created + cat /tmp/tmp.NmP65n0XdW + rm /tmp/tmp.xoQckEaHEi /tmp/tmp.NmP65n0XdW + return 0 + sleep 2 ++ get_operator_pod ++ kubectl_bin get pods --selector=name=percona-server-mongodb-operator -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.OkX0CuRyoO +++ mktemp ++ local LAST_ERR=/tmp/tmp.Qih2E8Lkgo ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=percona-server-mongodb-operator -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.OkX0CuRyoO ++ cat /tmp/tmp.Qih2E8Lkgo ++ rm /tmp/tmp.OkX0CuRyoO /tmp/tmp.Qih2E8Lkgo ++ return 0 + wait_pod percona-server-mongodb-operator-56887dd869-zz5xs + local pod=percona-server-mongodb-operator-56887dd869-zz5xs + set +o xtrace waiting for pod/percona-server-mongodb-operator-56887dd869-zz5xs to be ready.OK + desc 'start client' + set +o xtrace ----------------------------------------------------------------------------------- start client ----------------------------------------------------------------------------------- + kubectl_bin apply -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1548/e2e-tests/conf/client.yml ++ mktemp + local LAST_OUT=/tmp/tmp.qfAuyfyl7G ++ mktemp + local LAST_ERR=/tmp/tmp.D7jQM2ErvL + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1548/e2e-tests/conf/client.yml + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.qfAuyfyl7G deployment.apps/psmdb-client created + cat /tmp/tmp.D7jQM2ErvL + rm /tmp/tmp.qfAuyfyl7G /tmp/tmp.D7jQM2ErvL + return 0 + desc 'copy secrets from main to replica namespace and create all of them' + set +o xtrace ----------------------------------------------------------------------------------- copy secrets from main to replica namespace and create all of them ----------------------------------------------------------------------------------- + kubectl get secret cross-site-sharded-main-secrets -o yaml -n cross-site-sharded-21880 + kubectl_bin apply -f - + yq eval ' del(.metadata) | (.metadata.name = "cross-site-sharded-replica-secrets")' - ++ mktemp + local LAST_OUT=/tmp/tmp.AUveayTYiR ++ mktemp + local LAST_ERR=/tmp/tmp.gAVc7gxpUO + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.AUveayTYiR secret/cross-site-sharded-replica-secrets created + cat /tmp/tmp.gAVc7gxpUO + rm /tmp/tmp.AUveayTYiR /tmp/tmp.gAVc7gxpUO + return 0 + kubectl_bin get secret cross-site-sharded-main-ssl-internal -o yaml -n cross-site-sharded-21880 + kubectl_bin apply -f - ++ mktemp + local LAST_OUT=/tmp/tmp.pEtDPanOVS ++ mktemp + local LAST_ERR=/tmp/tmp.5w6DzyQa6H + local exit_status=0 + local timeout=4 ++ seq 0 2 ++ mktemp + for i in '$(seq 0 2)' + set +e + kubectl apply -f - + local LAST_OUT=/tmp/tmp.5XOz1nXl8X ++ mktemp + local LAST_ERR=/tmp/tmp.jOMr6Ee6Hl + local exit_status=0 + local timeout=4 + yq eval ' del(.metadata) | del(.status) | (.metadata.name = "cross-site-sharded-replica-ssl-internal")' - ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl get secret cross-site-sharded-main-ssl-internal -o yaml -n cross-site-sharded-21880 + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.5XOz1nXl8X + cat /tmp/tmp.jOMr6Ee6Hl + rm /tmp/tmp.5XOz1nXl8X /tmp/tmp.jOMr6Ee6Hl + return 0 + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.pEtDPanOVS secret/cross-site-sharded-replica-ssl-internal created + cat /tmp/tmp.5w6DzyQa6H + rm /tmp/tmp.pEtDPanOVS /tmp/tmp.5w6DzyQa6H + return 0 + yq eval ' del(.metadata) | del(.status) | (.metadata.name = "cross-site-sharded-replica-ssl")' - + kubectl_bin apply -f - + kubectl_bin get secret cross-site-sharded-main-ssl -o yaml -n cross-site-sharded-21880 ++ mktemp ++ mktemp + local LAST_OUT=/tmp/tmp.vvSrFnsQuB ++ mktemp + local LAST_OUT=/tmp/tmp.XUWkgeorCc ++ mktemp + local LAST_ERR=/tmp/tmp.mcm4UZVg3L + local exit_status=0 + local timeout=4 + local LAST_ERR=/tmp/tmp.Zbn3nlpccW + local exit_status=0 + local timeout=4 ++ seq 0 2 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl get secret cross-site-sharded-main-ssl -o yaml -n cross-site-sharded-21880 + for i in '$(seq 0 2)' + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.vvSrFnsQuB + cat /tmp/tmp.mcm4UZVg3L + rm /tmp/tmp.vvSrFnsQuB /tmp/tmp.mcm4UZVg3L + return 0 + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.XUWkgeorCc secret/cross-site-sharded-replica-ssl created + cat /tmp/tmp.Zbn3nlpccW + rm /tmp/tmp.XUWkgeorCc /tmp/tmp.Zbn3nlpccW + return 0 + sleep 30 + desc 'create replica PSMDB cluster ' + set +o xtrace ----------------------------------------------------------------------------------- create replica PSMDB cluster ----------------------------------------------------------------------------------- + apply_cluster /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1548/e2e-tests/cross-site-sharded/conf/cross-site-sharded-replica.yml + '[' -z '' ']' + kubectl_bin apply -f - ++ mktemp + local LAST_OUT=/tmp/tmp.FV8e4kBq0S ++ mktemp + local LAST_ERR=/tmp/tmp.A5fDwAfj8N + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f - + cat_config /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1548/e2e-tests/cross-site-sharded/conf/cross-site-sharded-replica.yml + yq eval '.spec.upgradeOptions.apply="Never"' + yq eval '(.spec | select(has("backup"))).backup.image = "perconalab/percona-server-mongodb-operator:main-backup"' + yq eval '(.spec | select(.image == null)).image = "perconalab/percona-server-mongodb-operator:main-mongod7.0"' + yq eval '(.spec | select(has("pmm"))).pmm.image = "perconalab/pmm-client:dev-latest"' + yq eval '(.spec | select(has("initImage"))).initImage = "perconalab/percona-server-mongodb-operator:PR-1548-d2731110"' + cat /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1548/e2e-tests/cross-site-sharded/conf/cross-site-sharded-replica.yml + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.FV8e4kBq0S perconaservermongodb.psmdb.percona.com/cross-site-sharded-replica created + cat /tmp/tmp.A5fDwAfj8N + rm /tmp/tmp.FV8e4kBq0S /tmp/tmp.A5fDwAfj8N + return 0 + wait_for_running cross-site-sharded-replica-rs0 3 false + local name=cross-site-sharded-replica-rs0 + let last_pod=2 + local check_cluster_readyness=false + set_debug + [[ 1 == 1 ]] + set -o xtrace + local rs_name=rs0 + local cluster_name=cross-site-sharded-replica ++ seq 0 2 + for i in '$(seq 0 $last_pod)' + [[ 0 -eq 2 ]] + wait_pod cross-site-sharded-replica-rs0-0 + local pod=cross-site-sharded-replica-rs0-0 + set +o xtrace waiting for pod/cross-site-sharded-replica-rs0-0 to be ready.............OK + for i in '$(seq 0 $last_pod)' + [[ 1 -eq 2 ]] + wait_pod cross-site-sharded-replica-rs0-1 + local pod=cross-site-sharded-replica-rs0-1 + set +o xtrace waiting for pod/cross-site-sharded-replica-rs0-1 to be ready............OK + for i in '$(seq 0 $last_pod)' + [[ 2 -eq 2 ]] ++ kubectl_bin get psmdb cross-site-sharded-replica -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].arbiter.enabled}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.alnmzOEpMt +++ mktemp ++ local LAST_ERR=/tmp/tmp.xoKCEfTQOI ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get psmdb cross-site-sharded-replica -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].arbiter.enabled}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.alnmzOEpMt ++ cat /tmp/tmp.xoKCEfTQOI ++ rm /tmp/tmp.alnmzOEpMt /tmp/tmp.xoKCEfTQOI ++ return 0 + [[ false == \t\r\u\e ]] + wait_pod cross-site-sharded-replica-rs0-2 + local pod=cross-site-sharded-replica-rs0-2 + set +o xtrace waiting for pod/cross-site-sharded-replica-rs0-2 to be ready...........OK ++ kubectl_bin get psmdb cross-site-sharded-replica -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].non_voting.enabled}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.62JW6O6ytp +++ mktemp ++ local LAST_ERR=/tmp/tmp.isMdNdvcYR ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get psmdb cross-site-sharded-replica -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].non_voting.enabled}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.62JW6O6ytp ++ cat /tmp/tmp.isMdNdvcYR ++ rm /tmp/tmp.62JW6O6ytp /tmp/tmp.isMdNdvcYR ++ return 0 + [[ '' == \t\r\u\e ]] + sleep 10 + [[ false == \t\r\u\e ]] + wait_for_running cross-site-sharded-replica-rs1 3 false + local name=cross-site-sharded-replica-rs1 + let last_pod=2 + local check_cluster_readyness=false + set_debug + [[ 1 == 1 ]] + set -o xtrace + local rs_name=rs1 + local cluster_name=cross-site-sharded-replica ++ seq 0 2 + for i in '$(seq 0 $last_pod)' + [[ 0 -eq 2 ]] + wait_pod cross-site-sharded-replica-rs1-0 + local pod=cross-site-sharded-replica-rs1-0 + set +o xtrace waiting for pod/cross-site-sharded-replica-rs1-0 to be ready.OK + for i in '$(seq 0 $last_pod)' + [[ 1 -eq 2 ]] + wait_pod cross-site-sharded-replica-rs1-1 + local pod=cross-site-sharded-replica-rs1-1 + set +o xtrace waiting for pod/cross-site-sharded-replica-rs1-1 to be ready.OK + for i in '$(seq 0 $last_pod)' + [[ 2 -eq 2 ]] ++ kubectl_bin get psmdb cross-site-sharded-replica -o 'jsonpath={.spec.replsets[?(@.name=="rs1")].arbiter.enabled}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.mwaNSFNnm6 +++ mktemp ++ local LAST_ERR=/tmp/tmp.ZaPG0IwKEP ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get psmdb cross-site-sharded-replica -o 'jsonpath={.spec.replsets[?(@.name=="rs1")].arbiter.enabled}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.mwaNSFNnm6 ++ cat /tmp/tmp.ZaPG0IwKEP ++ rm /tmp/tmp.mwaNSFNnm6 /tmp/tmp.ZaPG0IwKEP ++ return 0 + [[ false == \t\r\u\e ]] + wait_pod cross-site-sharded-replica-rs1-2 + local pod=cross-site-sharded-replica-rs1-2 + set +o xtrace waiting for pod/cross-site-sharded-replica-rs1-2 to be ready.OK ++ kubectl_bin get psmdb cross-site-sharded-replica -o 'jsonpath={.spec.replsets[?(@.name=="rs1")].non_voting.enabled}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.GthbApGMKf +++ mktemp ++ local LAST_ERR=/tmp/tmp.vbnx2DuGt7 ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get psmdb cross-site-sharded-replica -o 'jsonpath={.spec.replsets[?(@.name=="rs1")].non_voting.enabled}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.GthbApGMKf ++ cat /tmp/tmp.vbnx2DuGt7 ++ rm /tmp/tmp.GthbApGMKf /tmp/tmp.vbnx2DuGt7 ++ return 0 + [[ '' == \t\r\u\e ]] + sleep 10 + [[ false == \t\r\u\e ]] + wait_for_running cross-site-sharded-replica-cfg 3 false + local name=cross-site-sharded-replica-cfg + let last_pod=2 + local check_cluster_readyness=false + set_debug + [[ 1 == 1 ]] + set -o xtrace + local rs_name=cfg + local cluster_name=cross-site-sharded-replica ++ seq 0 2 + for i in '$(seq 0 $last_pod)' + [[ 0 -eq 2 ]] + wait_pod cross-site-sharded-replica-cfg-0 + local pod=cross-site-sharded-replica-cfg-0 + set +o xtrace waiting for pod/cross-site-sharded-replica-cfg-0 to be ready.OK + for i in '$(seq 0 $last_pod)' + [[ 1 -eq 2 ]] + wait_pod cross-site-sharded-replica-cfg-1 + local pod=cross-site-sharded-replica-cfg-1 + set +o xtrace waiting for pod/cross-site-sharded-replica-cfg-1 to be ready.OK + for i in '$(seq 0 $last_pod)' + [[ 2 -eq 2 ]] ++ kubectl_bin get psmdb cross-site-sharded-replica -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].arbiter.enabled}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.uxqHPetjzk +++ mktemp ++ local LAST_ERR=/tmp/tmp.gHSYV11FnN ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get psmdb cross-site-sharded-replica -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].arbiter.enabled}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.uxqHPetjzk ++ cat /tmp/tmp.gHSYV11FnN ++ rm /tmp/tmp.uxqHPetjzk /tmp/tmp.gHSYV11FnN ++ return 0 + [[ '' == \t\r\u\e ]] + wait_pod cross-site-sharded-replica-cfg-2 + local pod=cross-site-sharded-replica-cfg-2 + set +o xtrace waiting for pod/cross-site-sharded-replica-cfg-2 to be ready.OK ++ kubectl_bin get psmdb cross-site-sharded-replica -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].non_voting.enabled}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.VolBJiM1WZ +++ mktemp ++ local LAST_ERR=/tmp/tmp.ElyNZGF1GV ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get psmdb cross-site-sharded-replica -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].non_voting.enabled}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.VolBJiM1WZ ++ cat /tmp/tmp.ElyNZGF1GV ++ rm /tmp/tmp.VolBJiM1WZ /tmp/tmp.ElyNZGF1GV ++ return 0 + [[ '' == \t\r\u\e ]] + sleep 10 + [[ false == \t\r\u\e ]] ++ get_service_ip cross-site-sharded-replica-cfg-0 cfg ++ local service=cross-site-sharded-replica-cfg-0 ++ local server_type=cfg +++ kubectl_bin get psmdb/cross-site-sharded-replica -o 'jsonpath={.spec.replsets[].expose.enabled}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.e1avEgHnFm ++++ mktemp +++ local LAST_ERR=/tmp/tmp.6aj88BOYyw +++ local exit_status=0 +++ local timeout=4 ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ set +e +++ kubectl get psmdb/cross-site-sharded-replica -o 'jsonpath={.spec.replsets[].expose.enabled}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 -a -n 1 ']' +++ break +++ cat /tmp/tmp.e1avEgHnFm +++ cat /tmp/tmp.6aj88BOYyw +++ rm /tmp/tmp.e1avEgHnFm /tmp/tmp.6aj88BOYyw +++ return 0 ++ '[' true '!=' true ']' ++ kubectl_bin get service/cross-site-sharded-replica-cfg-0 -o 'jsonpath={.spec.type}' ++ grep -q NotFound +++ kubectl_bin get service/cross-site-sharded-replica-cfg-0 -o 'jsonpath={.spec.type}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.hGOhzC8xH6 ++++ mktemp +++ local LAST_ERR=/tmp/tmp.KKxzPlxDUM +++ local exit_status=0 +++ local timeout=4 ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ set +e +++ kubectl get service/cross-site-sharded-replica-cfg-0 -o 'jsonpath={.spec.type}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 -a -n 1 ']' +++ break +++ cat /tmp/tmp.hGOhzC8xH6 +++ cat /tmp/tmp.KKxzPlxDUM +++ rm /tmp/tmp.hGOhzC8xH6 /tmp/tmp.KKxzPlxDUM +++ return 0 ++ service_type=ClusterIP ++ '[' ClusterIP = ClusterIP ']' ++ kubectl_bin get service/cross-site-sharded-replica-cfg-0 -o 'jsonpath={.spec.clusterIP}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.RuCJxKcVLi +++ mktemp ++ local LAST_ERR=/tmp/tmp.8JDGLEQ4wU ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get service/cross-site-sharded-replica-cfg-0 -o 'jsonpath={.spec.clusterIP}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.RuCJxKcVLi ++ cat /tmp/tmp.8JDGLEQ4wU ++ rm /tmp/tmp.RuCJxKcVLi /tmp/tmp.8JDGLEQ4wU ++ return 0 ++ return + replica_cfg_0_endpoint=10.42.190.168 ++ get_service_ip cross-site-sharded-replica-cfg-1 cfg ++ local service=cross-site-sharded-replica-cfg-1 ++ local server_type=cfg +++ kubectl_bin get psmdb/cross-site-sharded-replica -o 'jsonpath={.spec.replsets[].expose.enabled}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.uaJxF91kTq ++++ mktemp +++ local LAST_ERR=/tmp/tmp.YTbAXkcJyf +++ local exit_status=0 +++ local timeout=4 ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ set +e +++ kubectl get psmdb/cross-site-sharded-replica -o 'jsonpath={.spec.replsets[].expose.enabled}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 -a -n 1 ']' +++ break +++ cat /tmp/tmp.uaJxF91kTq +++ cat /tmp/tmp.YTbAXkcJyf +++ rm /tmp/tmp.uaJxF91kTq /tmp/tmp.YTbAXkcJyf +++ return 0 ++ '[' true '!=' true ']' ++ grep -q NotFound ++ kubectl_bin get service/cross-site-sharded-replica-cfg-1 -o 'jsonpath={.spec.type}' +++ kubectl_bin get service/cross-site-sharded-replica-cfg-1 -o 'jsonpath={.spec.type}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.oZzhEkxiLE ++++ mktemp +++ local LAST_ERR=/tmp/tmp.7A8w2d56Nw +++ local exit_status=0 +++ local timeout=4 ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ set +e +++ kubectl get service/cross-site-sharded-replica-cfg-1 -o 'jsonpath={.spec.type}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 -a -n 1 ']' +++ break +++ cat /tmp/tmp.oZzhEkxiLE +++ cat /tmp/tmp.7A8w2d56Nw +++ rm /tmp/tmp.oZzhEkxiLE /tmp/tmp.7A8w2d56Nw +++ return 0 ++ service_type=ClusterIP ++ '[' ClusterIP = ClusterIP ']' ++ kubectl_bin get service/cross-site-sharded-replica-cfg-1 -o 'jsonpath={.spec.clusterIP}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.Fshg7dy3OL +++ mktemp ++ local LAST_ERR=/tmp/tmp.eNVhBifNG9 ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get service/cross-site-sharded-replica-cfg-1 -o 'jsonpath={.spec.clusterIP}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.Fshg7dy3OL ++ cat /tmp/tmp.eNVhBifNG9 ++ rm /tmp/tmp.Fshg7dy3OL /tmp/tmp.eNVhBifNG9 ++ return 0 ++ return + replica_cfg_1_endpoint=10.42.186.236 ++ get_service_ip cross-site-sharded-replica-cfg-2 cfg ++ local service=cross-site-sharded-replica-cfg-2 ++ local server_type=cfg +++ kubectl_bin get psmdb/cross-site-sharded-replica -o 'jsonpath={.spec.replsets[].expose.enabled}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.UCa1EWzfU4 ++++ mktemp +++ local LAST_ERR=/tmp/tmp.cz5BfBNqtK +++ local exit_status=0 +++ local timeout=4 ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ set +e +++ kubectl get psmdb/cross-site-sharded-replica -o 'jsonpath={.spec.replsets[].expose.enabled}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 -a -n 1 ']' +++ break +++ cat /tmp/tmp.UCa1EWzfU4 +++ cat /tmp/tmp.cz5BfBNqtK +++ rm /tmp/tmp.UCa1EWzfU4 /tmp/tmp.cz5BfBNqtK +++ return 0 ++ '[' true '!=' true ']' ++ kubectl_bin get service/cross-site-sharded-replica-cfg-2 -o 'jsonpath={.spec.type}' ++ grep -q NotFound +++ kubectl_bin get service/cross-site-sharded-replica-cfg-2 -o 'jsonpath={.spec.type}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.0zWQlv2sSy ++++ mktemp +++ local LAST_ERR=/tmp/tmp.SZqeXoinLq +++ local exit_status=0 +++ local timeout=4 ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ set +e +++ kubectl get service/cross-site-sharded-replica-cfg-2 -o 'jsonpath={.spec.type}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 -a -n 1 ']' +++ break +++ cat /tmp/tmp.0zWQlv2sSy +++ cat /tmp/tmp.SZqeXoinLq +++ rm /tmp/tmp.0zWQlv2sSy /tmp/tmp.SZqeXoinLq +++ return 0 ++ service_type=ClusterIP ++ '[' ClusterIP = ClusterIP ']' ++ kubectl_bin get service/cross-site-sharded-replica-cfg-2 -o 'jsonpath={.spec.clusterIP}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.xl6xm6NVCp +++ mktemp ++ local LAST_ERR=/tmp/tmp.ObEHALW4h5 ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get service/cross-site-sharded-replica-cfg-2 -o 'jsonpath={.spec.clusterIP}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.xl6xm6NVCp ++ cat /tmp/tmp.ObEHALW4h5 ++ rm /tmp/tmp.xl6xm6NVCp /tmp/tmp.ObEHALW4h5 ++ return 0 ++ return + replica_cfg_2_endpoint=10.42.177.236 ++ get_service_ip cross-site-sharded-replica-rs0-0 ++ local service=cross-site-sharded-replica-rs0-0 ++ local server_type=rs0 +++ kubectl_bin get psmdb/cross-site-sharded-replica -o 'jsonpath={.spec.replsets[].expose.enabled}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.oW46OsuSxl ++++ mktemp +++ local LAST_ERR=/tmp/tmp.mHCg4fvUlZ +++ local exit_status=0 +++ local timeout=4 ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ set +e +++ kubectl get psmdb/cross-site-sharded-replica -o 'jsonpath={.spec.replsets[].expose.enabled}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 -a -n 1 ']' +++ break +++ cat /tmp/tmp.oW46OsuSxl +++ cat /tmp/tmp.mHCg4fvUlZ +++ rm /tmp/tmp.oW46OsuSxl /tmp/tmp.mHCg4fvUlZ +++ return 0 ++ '[' true '!=' true ']' ++ kubectl_bin get service/cross-site-sharded-replica-rs0-0 -o 'jsonpath={.spec.type}' ++ grep -q NotFound +++ kubectl_bin get service/cross-site-sharded-replica-rs0-0 -o 'jsonpath={.spec.type}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.hRmtsgVem8 ++++ mktemp +++ local LAST_ERR=/tmp/tmp.eJyoorH8Xx +++ local exit_status=0 +++ local timeout=4 ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ set +e +++ kubectl get service/cross-site-sharded-replica-rs0-0 -o 'jsonpath={.spec.type}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 -a -n 1 ']' +++ break +++ cat /tmp/tmp.hRmtsgVem8 +++ cat /tmp/tmp.eJyoorH8Xx +++ rm /tmp/tmp.hRmtsgVem8 /tmp/tmp.eJyoorH8Xx +++ return 0 ++ service_type=ClusterIP ++ '[' ClusterIP = ClusterIP ']' ++ kubectl_bin get service/cross-site-sharded-replica-rs0-0 -o 'jsonpath={.spec.clusterIP}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.DEeqJO66pd +++ mktemp ++ local LAST_ERR=/tmp/tmp.HBtcwn0mcU ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get service/cross-site-sharded-replica-rs0-0 -o 'jsonpath={.spec.clusterIP}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.DEeqJO66pd ++ cat /tmp/tmp.HBtcwn0mcU ++ rm /tmp/tmp.DEeqJO66pd /tmp/tmp.HBtcwn0mcU ++ return 0 ++ return + replica_rs0_0_endpoint=10.42.184.66 ++ get_service_ip cross-site-sharded-replica-rs0-1 ++ local service=cross-site-sharded-replica-rs0-1 ++ local server_type=rs0 +++ kubectl_bin get psmdb/cross-site-sharded-replica -o 'jsonpath={.spec.replsets[].expose.enabled}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.9iufLdpHlm ++++ mktemp +++ local LAST_ERR=/tmp/tmp.LLWvqURZbI +++ local exit_status=0 +++ local timeout=4 ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ set +e +++ kubectl get psmdb/cross-site-sharded-replica -o 'jsonpath={.spec.replsets[].expose.enabled}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 -a -n 1 ']' +++ break +++ cat /tmp/tmp.9iufLdpHlm +++ cat /tmp/tmp.LLWvqURZbI +++ rm /tmp/tmp.9iufLdpHlm /tmp/tmp.LLWvqURZbI +++ return 0 ++ '[' true '!=' true ']' ++ kubectl_bin get service/cross-site-sharded-replica-rs0-1 -o 'jsonpath={.spec.type}' ++ grep -q NotFound +++ kubectl_bin get service/cross-site-sharded-replica-rs0-1 -o 'jsonpath={.spec.type}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.k7h5k4tOnv ++++ mktemp +++ local LAST_ERR=/tmp/tmp.nU6YMLLQhD +++ local exit_status=0 +++ local timeout=4 ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ set +e +++ kubectl get service/cross-site-sharded-replica-rs0-1 -o 'jsonpath={.spec.type}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 -a -n 1 ']' +++ break +++ cat /tmp/tmp.k7h5k4tOnv +++ cat /tmp/tmp.nU6YMLLQhD +++ rm /tmp/tmp.k7h5k4tOnv /tmp/tmp.nU6YMLLQhD +++ return 0 ++ service_type=ClusterIP ++ '[' ClusterIP = ClusterIP ']' ++ kubectl_bin get service/cross-site-sharded-replica-rs0-1 -o 'jsonpath={.spec.clusterIP}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.ed8hmBXejV +++ mktemp ++ local LAST_ERR=/tmp/tmp.cQe6ZGWa2F ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get service/cross-site-sharded-replica-rs0-1 -o 'jsonpath={.spec.clusterIP}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.ed8hmBXejV ++ cat /tmp/tmp.cQe6ZGWa2F ++ rm /tmp/tmp.ed8hmBXejV /tmp/tmp.cQe6ZGWa2F ++ return 0 ++ return + replica_rs0_1_endpoint=10.42.182.35 ++ get_service_ip cross-site-sharded-replica-rs0-2 ++ local service=cross-site-sharded-replica-rs0-2 ++ local server_type=rs0 +++ kubectl_bin get psmdb/cross-site-sharded-replica -o 'jsonpath={.spec.replsets[].expose.enabled}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.r4b3ago3x3 ++++ mktemp +++ local LAST_ERR=/tmp/tmp.9Kp3nq9A1p +++ local exit_status=0 +++ local timeout=4 ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ set +e +++ kubectl get psmdb/cross-site-sharded-replica -o 'jsonpath={.spec.replsets[].expose.enabled}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 -a -n 1 ']' +++ break +++ cat /tmp/tmp.r4b3ago3x3 +++ cat /tmp/tmp.9Kp3nq9A1p +++ rm /tmp/tmp.r4b3ago3x3 /tmp/tmp.9Kp3nq9A1p +++ return 0 ++ '[' true '!=' true ']' ++ kubectl_bin get service/cross-site-sharded-replica-rs0-2 -o 'jsonpath={.spec.type}' ++ grep -q NotFound +++ kubectl_bin get service/cross-site-sharded-replica-rs0-2 -o 'jsonpath={.spec.type}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.nqLn1G00aD ++++ mktemp +++ local LAST_ERR=/tmp/tmp.MJVb2kZ7yv +++ local exit_status=0 +++ local timeout=4 ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ set +e +++ kubectl get service/cross-site-sharded-replica-rs0-2 -o 'jsonpath={.spec.type}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 -a -n 1 ']' +++ break +++ cat /tmp/tmp.nqLn1G00aD +++ cat /tmp/tmp.MJVb2kZ7yv +++ rm /tmp/tmp.nqLn1G00aD /tmp/tmp.MJVb2kZ7yv +++ return 0 ++ service_type=ClusterIP ++ '[' ClusterIP = ClusterIP ']' ++ kubectl_bin get service/cross-site-sharded-replica-rs0-2 -o 'jsonpath={.spec.clusterIP}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.gMGw20RCKM +++ mktemp ++ local LAST_ERR=/tmp/tmp.YdQkFVTJ1P ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get service/cross-site-sharded-replica-rs0-2 -o 'jsonpath={.spec.clusterIP}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.gMGw20RCKM ++ cat /tmp/tmp.YdQkFVTJ1P ++ rm /tmp/tmp.gMGw20RCKM /tmp/tmp.YdQkFVTJ1P ++ return 0 ++ return + replica_rs0_2_endpoint=10.42.186.40 ++ get_service_ip cross-site-sharded-replica-rs1-0 rs1 ++ local service=cross-site-sharded-replica-rs1-0 ++ local server_type=rs1 +++ kubectl_bin get psmdb/cross-site-sharded-replica -o 'jsonpath={.spec.replsets[].expose.enabled}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.2GbKvH3E7Y ++++ mktemp +++ local LAST_ERR=/tmp/tmp.UwZLrfMk8q +++ local exit_status=0 +++ local timeout=4 ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ set +e +++ kubectl get psmdb/cross-site-sharded-replica -o 'jsonpath={.spec.replsets[].expose.enabled}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 -a -n 1 ']' +++ break +++ cat /tmp/tmp.2GbKvH3E7Y +++ cat /tmp/tmp.UwZLrfMk8q +++ rm /tmp/tmp.2GbKvH3E7Y /tmp/tmp.UwZLrfMk8q +++ return 0 ++ '[' true '!=' true ']' ++ kubectl_bin get service/cross-site-sharded-replica-rs1-0 -o 'jsonpath={.spec.type}' ++ grep -q NotFound +++ kubectl_bin get service/cross-site-sharded-replica-rs1-0 -o 'jsonpath={.spec.type}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.67AT5wklwI ++++ mktemp +++ local LAST_ERR=/tmp/tmp.lBR1NeDIGc +++ local exit_status=0 +++ local timeout=4 ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ set +e +++ kubectl get service/cross-site-sharded-replica-rs1-0 -o 'jsonpath={.spec.type}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 -a -n 1 ']' +++ break +++ cat /tmp/tmp.67AT5wklwI +++ cat /tmp/tmp.lBR1NeDIGc +++ rm /tmp/tmp.67AT5wklwI /tmp/tmp.lBR1NeDIGc +++ return 0 ++ service_type=ClusterIP ++ '[' ClusterIP = ClusterIP ']' ++ kubectl_bin get service/cross-site-sharded-replica-rs1-0 -o 'jsonpath={.spec.clusterIP}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.vtOLDXaqDs +++ mktemp ++ local LAST_ERR=/tmp/tmp.3sQaeh3tvz ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get service/cross-site-sharded-replica-rs1-0 -o 'jsonpath={.spec.clusterIP}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.vtOLDXaqDs ++ cat /tmp/tmp.3sQaeh3tvz ++ rm /tmp/tmp.vtOLDXaqDs /tmp/tmp.3sQaeh3tvz ++ return 0 ++ return + replica_rs1_0_endpoint=10.42.181.213 ++ get_service_ip cross-site-sharded-replica-rs1-1 rs1 ++ local service=cross-site-sharded-replica-rs1-1 ++ local server_type=rs1 +++ kubectl_bin get psmdb/cross-site-sharded-replica -o 'jsonpath={.spec.replsets[].expose.enabled}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.GzyRQekyMo ++++ mktemp +++ local LAST_ERR=/tmp/tmp.91qZtTQHqc +++ local exit_status=0 +++ local timeout=4 ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ set +e +++ kubectl get psmdb/cross-site-sharded-replica -o 'jsonpath={.spec.replsets[].expose.enabled}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 -a -n 1 ']' +++ break +++ cat /tmp/tmp.GzyRQekyMo +++ cat /tmp/tmp.91qZtTQHqc +++ rm /tmp/tmp.GzyRQekyMo /tmp/tmp.91qZtTQHqc +++ return 0 ++ '[' true '!=' true ']' ++ grep -q NotFound ++ kubectl_bin get service/cross-site-sharded-replica-rs1-1 -o 'jsonpath={.spec.type}' +++ kubectl_bin get service/cross-site-sharded-replica-rs1-1 -o 'jsonpath={.spec.type}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.4hs3WMiz6e ++++ mktemp +++ local LAST_ERR=/tmp/tmp.JTLAKha4Ap +++ local exit_status=0 +++ local timeout=4 ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ set +e +++ kubectl get service/cross-site-sharded-replica-rs1-1 -o 'jsonpath={.spec.type}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 -a -n 1 ']' +++ break +++ cat /tmp/tmp.4hs3WMiz6e +++ cat /tmp/tmp.JTLAKha4Ap +++ rm /tmp/tmp.4hs3WMiz6e /tmp/tmp.JTLAKha4Ap +++ return 0 ++ service_type=ClusterIP ++ '[' ClusterIP = ClusterIP ']' ++ kubectl_bin get service/cross-site-sharded-replica-rs1-1 -o 'jsonpath={.spec.clusterIP}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.y6fmyxeV89 +++ mktemp ++ local LAST_ERR=/tmp/tmp.JoQO6Y4bbY ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get service/cross-site-sharded-replica-rs1-1 -o 'jsonpath={.spec.clusterIP}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.y6fmyxeV89 ++ cat /tmp/tmp.JoQO6Y4bbY ++ rm /tmp/tmp.y6fmyxeV89 /tmp/tmp.JoQO6Y4bbY ++ return 0 ++ return + replica_rs1_1_endpoint=10.42.189.186 ++ get_service_ip cross-site-sharded-replica-rs1-2 rs1 ++ local service=cross-site-sharded-replica-rs1-2 ++ local server_type=rs1 +++ kubectl_bin get psmdb/cross-site-sharded-replica -o 'jsonpath={.spec.replsets[].expose.enabled}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.O0wQaA0FXI ++++ mktemp +++ local LAST_ERR=/tmp/tmp.aY0Hk3fNdB +++ local exit_status=0 +++ local timeout=4 ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ set +e +++ kubectl get psmdb/cross-site-sharded-replica -o 'jsonpath={.spec.replsets[].expose.enabled}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 -a -n 1 ']' +++ break +++ cat /tmp/tmp.O0wQaA0FXI +++ cat /tmp/tmp.aY0Hk3fNdB +++ rm /tmp/tmp.O0wQaA0FXI /tmp/tmp.aY0Hk3fNdB +++ return 0 ++ '[' true '!=' true ']' ++ kubectl_bin get service/cross-site-sharded-replica-rs1-2 -o 'jsonpath={.spec.type}' ++ grep -q NotFound +++ kubectl_bin get service/cross-site-sharded-replica-rs1-2 -o 'jsonpath={.spec.type}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.GJeiMqd6nG ++++ mktemp +++ local LAST_ERR=/tmp/tmp.PVcm9QJaIK +++ local exit_status=0 +++ local timeout=4 ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ set +e +++ kubectl get service/cross-site-sharded-replica-rs1-2 -o 'jsonpath={.spec.type}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 -a -n 1 ']' +++ break +++ cat /tmp/tmp.GJeiMqd6nG +++ cat /tmp/tmp.PVcm9QJaIK +++ rm /tmp/tmp.GJeiMqd6nG /tmp/tmp.PVcm9QJaIK +++ return 0 ++ service_type=ClusterIP ++ '[' ClusterIP = ClusterIP ']' ++ kubectl_bin get service/cross-site-sharded-replica-rs1-2 -o 'jsonpath={.spec.clusterIP}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.AX4v1FyyVu +++ mktemp ++ local LAST_ERR=/tmp/tmp.NLJsmTlW4J ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get service/cross-site-sharded-replica-rs1-2 -o 'jsonpath={.spec.clusterIP}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.AX4v1FyyVu ++ cat /tmp/tmp.NLJsmTlW4J ++ rm /tmp/tmp.AX4v1FyyVu /tmp/tmp.NLJsmTlW4J ++ return 0 ++ return + replica_rs1_2_endpoint=10.42.189.94 ++ kubectl_bin config current-context +++ mktemp ++ local LAST_OUT=/tmp/tmp.abWFNvcSta +++ mktemp ++ local LAST_ERR=/tmp/tmp.1Tj3pIKOWb ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl config current-context ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.abWFNvcSta ++ cat /tmp/tmp.1Tj3pIKOWb ++ rm /tmp/tmp.abWFNvcSta /tmp/tmp.1Tj3pIKOWb ++ return 0 + kubectl_bin config set-context gke_cloud-dev-112233_us-central1-a_jen-psmdb-1548-d2731110-1-cluster4 --namespace=cross-site-sharded-21880 ++ mktemp + local LAST_OUT=/tmp/tmp.zpDpQdA4Pu ++ mktemp + local LAST_ERR=/tmp/tmp.ghJzenvWHR + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl config set-context gke_cloud-dev-112233_us-central1-a_jen-psmdb-1548-d2731110-1-cluster4 --namespace=cross-site-sharded-21880 + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.zpDpQdA4Pu Context "gke_cloud-dev-112233_us-central1-a_jen-psmdb-1548-d2731110-1-cluster4" modified. + cat /tmp/tmp.ghJzenvWHR + rm /tmp/tmp.zpDpQdA4Pu /tmp/tmp.ghJzenvWHR + return 0 + kubectl_bin patch psmdb cross-site-sharded-main --type=merge --patch '{ "spec": {"replsets":[ {"affinity":{"antiAffinityTopologyKey": "none"},"arbiter":{"affinity":{"antiAffinityTopologyKey": "none"},"enabled":false,"size":1},"expose":{"enabled":true,"exposeType":"ClusterIp"},"externalNodes":[{"host":"10.42.184.66","priority":0,"votes":0},{"host":"10.42.182.35","port":27017,"priority":1,"votes":1},{"host":"10.42.186.40", "port":27017,"priority":1,"votes":1}],"name":"rs0","nonvoting":{"affinity":{"antiAffinityTopologyKey":"none"},"enabled":false,"podDisruptionBudget":{"maxUnavailable":1},"resources":{"limits":{"cpu":"300m","memory":"0.5G"},"requests":{"cpu":"300m","memory":"0.5G"}},"size":3,"volumeSpec":{"persistentVolumeClaim":{"resources":{"requests":{"storage":"1Gi"}}}}},"podDisruptionBudget":{"maxUnavailable":1},"resources":{"limits":{"cpu":"300m","memory":"0.5G"},"requests":{"cpu":"300m","memory":"0.5G"}},"size":3,"volumeSpec":{"persistentVolumeClaim":{"resources":{"requests":{"storage":"3Gi"}}}}}, {"affinity":{"antiAffinityTopologyKey": "none"},"arbiter":{"affinity":{"antiAffinityTopologyKey": "none"},"enabled":false,"size":1},"expose":{"enabled":true,"exposeType":"ClusterIp"},"externalNodes":[{"host":"10.42.181.213","priority":0,"votes":0},{"host":"10.42.189.186","port":27017,"priority":1,"votes":1},{"host":"10.42.189.94", "port":27017,"priority":1,"votes":1}],"name":"rs1","nonvoting":{"affinity":{"antiAffinityTopologyKey":"none"},"enabled":false,"podDisruptionBudget":{"maxUnavailable":1},"resources":{"limits":{"cpu":"300m","memory":"0.5G"},"requests":{"cpu":"300m","memory":"0.5G"}},"size":3,"volumeSpec":{"persistentVolumeClaim":{"resources":{"requests":{"storage":"1Gi"}}}}},"podDisruptionBudget":{"maxUnavailable":1},"resources":{"limits":{"cpu":"300m","memory":"0.5G"},"requests":{"cpu":"300m","memory":"0.5G"}},"size":3,"volumeSpec":{"persistentVolumeClaim":{"resources":{"requests":{"storage":"3Gi"}}}}} ], "sharding":{"configsvrReplSet":{ "externalNodes": [{"host":"10.42.190.168","priority":1,"votes":1 },{"host":"10.42.186.236", "priority":1,"votes":1},{"host":"10.42.177.236","priority":0,"votes":0}]}} } }' ++ mktemp + local LAST_OUT=/tmp/tmp.rMRFMTSv0w ++ mktemp + local LAST_ERR=/tmp/tmp.pJECvqfee4 + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl patch psmdb cross-site-sharded-main --type=merge --patch '{ "spec": {"replsets":[ {"affinity":{"antiAffinityTopologyKey": "none"},"arbiter":{"affinity":{"antiAffinityTopologyKey": "none"},"enabled":false,"size":1},"expose":{"enabled":true,"exposeType":"ClusterIp"},"externalNodes":[{"host":"10.42.184.66","priority":0,"votes":0},{"host":"10.42.182.35","port":27017,"priority":1,"votes":1},{"host":"10.42.186.40", "port":27017,"priority":1,"votes":1}],"name":"rs0","nonvoting":{"affinity":{"antiAffinityTopologyKey":"none"},"enabled":false,"podDisruptionBudget":{"maxUnavailable":1},"resources":{"limits":{"cpu":"300m","memory":"0.5G"},"requests":{"cpu":"300m","memory":"0.5G"}},"size":3,"volumeSpec":{"persistentVolumeClaim":{"resources":{"requests":{"storage":"1Gi"}}}}},"podDisruptionBudget":{"maxUnavailable":1},"resources":{"limits":{"cpu":"300m","memory":"0.5G"},"requests":{"cpu":"300m","memory":"0.5G"}},"size":3,"volumeSpec":{"persistentVolumeClaim":{"resources":{"requests":{"storage":"3Gi"}}}}}, {"affinity":{"antiAffinityTopologyKey": "none"},"arbiter":{"affinity":{"antiAffinityTopologyKey": "none"},"enabled":false,"size":1},"expose":{"enabled":true,"exposeType":"ClusterIp"},"externalNodes":[{"host":"10.42.181.213","priority":0,"votes":0},{"host":"10.42.189.186","port":27017,"priority":1,"votes":1},{"host":"10.42.189.94", "port":27017,"priority":1,"votes":1}],"name":"rs1","nonvoting":{"affinity":{"antiAffinityTopologyKey":"none"},"enabled":false,"podDisruptionBudget":{"maxUnavailable":1},"resources":{"limits":{"cpu":"300m","memory":"0.5G"},"requests":{"cpu":"300m","memory":"0.5G"}},"size":3,"volumeSpec":{"persistentVolumeClaim":{"resources":{"requests":{"storage":"1Gi"}}}}},"podDisruptionBudget":{"maxUnavailable":1},"resources":{"limits":{"cpu":"300m","memory":"0.5G"},"requests":{"cpu":"300m","memory":"0.5G"}},"size":3,"volumeSpec":{"persistentVolumeClaim":{"resources":{"requests":{"storage":"3Gi"}}}}} ], "sharding":{"configsvrReplSet":{ "externalNodes": [{"host":"10.42.190.168","priority":1,"votes":1 },{"host":"10.42.186.236", "priority":1,"votes":1},{"host":"10.42.177.236","priority":0,"votes":0}]}} } }' + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.rMRFMTSv0w perconaservermongodb.psmdb.percona.com/cross-site-sharded-main patched + cat /tmp/tmp.pJECvqfee4 + rm /tmp/tmp.rMRFMTSv0w /tmp/tmp.pJECvqfee4 + return 0 + wait_for_members 10.42.190.168 cfg + local endpoint=10.42.190.168 + local rsName=cfg + local nodes_amount=0 + [[ 0 == 6 ]] ++ run_mongos 'rs.conf().members.length' clusterAdmin:clusterAdmin123456@10.42.190.168 mongodb :27017 ++ local 'command=rs.conf().members.length' ++ local uri=clusterAdmin:clusterAdmin123456@10.42.190.168 ++ local driver=mongodb ++ local suffix=:27017 ++ /usr/bin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/' ++ egrep -v 'I NETWORK|W NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|bye' +++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.uLkJT2mg2D ++++ mktemp +++ local LAST_ERR=/tmp/tmp.Saco8QYJkq +++ local exit_status=0 +++ local timeout=4 ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ set +e +++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 -a -n 1 ']' +++ break +++ cat /tmp/tmp.uLkJT2mg2D +++ cat /tmp/tmp.Saco8QYJkq +++ rm /tmp/tmp.uLkJT2mg2D /tmp/tmp.Saco8QYJkq +++ return 0 ++ local client_container=psmdb-client-7469665986-xhdrq ++ local mongo_flag= ++ kubectl_bin exec psmdb-client-7469665986-xhdrq -- bash -c 'printf '\''rs.conf().members.length\n'\'' | mongo mongodb://clusterAdmin:clusterAdmin123456@10.42.190.168:27017/admin ' +++ mktemp ++ local LAST_OUT=/tmp/tmp.zt0DalEIyd +++ mktemp ++ local LAST_ERR=/tmp/tmp.1Tm0Ilkhj2 ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl exec psmdb-client-7469665986-xhdrq -- bash -c 'printf '\''rs.conf().members.length\n'\'' | mongo mongodb://clusterAdmin:clusterAdmin123456@10.42.190.168:27017/admin ' ++ exit_status=1 ++ set -e ++ '[' 1 '!=' 0 -a -n 1 ']' ++ cat /tmp/tmp.zt0DalEIyd ++ cat /tmp/tmp.1Tm0Ilkhj2 command terminated with exit code 1 ++ sleep 0 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl exec psmdb-client-7469665986-xhdrq -- bash -c 'printf '\''rs.conf().members.length\n'\'' | mongo mongodb://clusterAdmin:clusterAdmin123456@10.42.190.168:27017/admin ' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.zt0DalEIyd ++ cat /tmp/tmp.1Tm0Ilkhj2 ++ rm /tmp/tmp.zt0DalEIyd /tmp/tmp.1Tm0Ilkhj2 ++ return 0 + nodes_amount='Error: Authentication failed. : connect@src/mongo/shell/mongo.js:374:17 @(connect):2:6 exception: connect failed exiting with code 1 5' + echo 'waiting for all members to be configured in cfg' waiting for all members to be configured in cfg + let retry+=1 + '[' 1 -ge 15 ']' + echo -n . .+ sleep 10 + [[ Error: Authentication failed. : connect@src/mongo/shell/mongo.js:374:17 @(connect):2:6 exception: connect failed exiting with code 1 5 == 6 ]] ++ run_mongos 'rs.conf().members.length' clusterAdmin:clusterAdmin123456@10.42.190.168 mongodb :27017 ++ egrep -v 'I NETWORK|W NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|bye' ++ local 'command=rs.conf().members.length' ++ local uri=clusterAdmin:clusterAdmin123456@10.42.190.168 ++ /usr/bin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/' ++ local driver=mongodb ++ local suffix=:27017 +++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.rSOzXmSFJv ++++ mktemp +++ local LAST_ERR=/tmp/tmp.ABZ8hVCXsi +++ local exit_status=0 +++ local timeout=4 ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ set +e +++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 -a -n 1 ']' +++ break +++ cat /tmp/tmp.rSOzXmSFJv +++ cat /tmp/tmp.ABZ8hVCXsi +++ rm /tmp/tmp.rSOzXmSFJv /tmp/tmp.ABZ8hVCXsi +++ return 0 ++ local client_container=psmdb-client-7469665986-xhdrq ++ local mongo_flag= ++ kubectl_bin exec psmdb-client-7469665986-xhdrq -- bash -c 'printf '\''rs.conf().members.length\n'\'' | mongo mongodb://clusterAdmin:clusterAdmin123456@10.42.190.168:27017/admin ' +++ mktemp ++ local LAST_OUT=/tmp/tmp.U0r87oPiaH +++ mktemp ++ local LAST_ERR=/tmp/tmp.LXgcmDqPXr ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl exec psmdb-client-7469665986-xhdrq -- bash -c 'printf '\''rs.conf().members.length\n'\'' | mongo mongodb://clusterAdmin:clusterAdmin123456@10.42.190.168:27017/admin ' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.U0r87oPiaH ++ cat /tmp/tmp.LXgcmDqPXr ++ rm /tmp/tmp.U0r87oPiaH /tmp/tmp.LXgcmDqPXr ++ return 0 + nodes_amount=6 + echo 'waiting for all members to be configured in cfg' waiting for all members to be configured in cfg + let retry+=1 + '[' 2 -ge 15 ']' + echo -n . .+ sleep 10 + [[ 6 == 6 ]] + wait_for_members 10.42.184.66 rs0 + local endpoint=10.42.184.66 + local rsName=rs0 + local nodes_amount=0 + [[ 0 == 6 ]] ++ run_mongos 'rs.conf().members.length' clusterAdmin:clusterAdmin123456@10.42.184.66 mongodb :27017 ++ local 'command=rs.conf().members.length' ++ local uri=clusterAdmin:clusterAdmin123456@10.42.184.66 ++ egrep -v 'I NETWORK|W NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|bye' ++ local driver=mongodb ++ local suffix=:27017 ++ /usr/bin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/' +++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.QPRYCSNz70 ++++ mktemp +++ local LAST_ERR=/tmp/tmp.8T5JmlvWVg +++ local exit_status=0 +++ local timeout=4 ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ set +e +++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 -a -n 1 ']' +++ break +++ cat /tmp/tmp.QPRYCSNz70 +++ cat /tmp/tmp.8T5JmlvWVg +++ rm /tmp/tmp.QPRYCSNz70 /tmp/tmp.8T5JmlvWVg +++ return 0 ++ local client_container=psmdb-client-7469665986-xhdrq ++ local mongo_flag= ++ kubectl_bin exec psmdb-client-7469665986-xhdrq -- bash -c 'printf '\''rs.conf().members.length\n'\'' | mongo mongodb://clusterAdmin:clusterAdmin123456@10.42.184.66:27017/admin ' +++ mktemp ++ local LAST_OUT=/tmp/tmp.woPckJYQVR +++ mktemp ++ local LAST_ERR=/tmp/tmp.ATiHHfraPg ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl exec psmdb-client-7469665986-xhdrq -- bash -c 'printf '\''rs.conf().members.length\n'\'' | mongo mongodb://clusterAdmin:clusterAdmin123456@10.42.184.66:27017/admin ' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.woPckJYQVR ++ cat /tmp/tmp.ATiHHfraPg ++ rm /tmp/tmp.woPckJYQVR /tmp/tmp.ATiHHfraPg ++ return 0 + nodes_amount=6 + echo 'waiting for all members to be configured in rs0' waiting for all members to be configured in rs0 + let retry+=1 + '[' 3 -ge 15 ']' + echo -n . .+ sleep 10 + [[ 6 == 6 ]] + wait_for_members 10.42.181.213 rs1 + local endpoint=10.42.181.213 + local rsName=rs1 + local nodes_amount=0 + [[ 0 == 6 ]] ++ run_mongos 'rs.conf().members.length' clusterAdmin:clusterAdmin123456@10.42.181.213 mongodb :27017 ++ local 'command=rs.conf().members.length' ++ local uri=clusterAdmin:clusterAdmin123456@10.42.181.213 ++ egrep -v 'I NETWORK|W NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|bye' ++ local driver=mongodb ++ local suffix=:27017 ++ /usr/bin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/' +++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.jTq8D8at6X ++++ mktemp +++ local LAST_ERR=/tmp/tmp.kWuckYGjGP +++ local exit_status=0 +++ local timeout=4 ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ set +e +++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 -a -n 1 ']' +++ break +++ cat /tmp/tmp.jTq8D8at6X +++ cat /tmp/tmp.kWuckYGjGP +++ rm /tmp/tmp.jTq8D8at6X /tmp/tmp.kWuckYGjGP +++ return 0 ++ local client_container=psmdb-client-7469665986-xhdrq ++ local mongo_flag= ++ kubectl_bin exec psmdb-client-7469665986-xhdrq -- bash -c 'printf '\''rs.conf().members.length\n'\'' | mongo mongodb://clusterAdmin:clusterAdmin123456@10.42.181.213:27017/admin ' +++ mktemp ++ local LAST_OUT=/tmp/tmp.Oo03xXFM8t +++ mktemp ++ local LAST_ERR=/tmp/tmp.M1js10EbXL ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl exec psmdb-client-7469665986-xhdrq -- bash -c 'printf '\''rs.conf().members.length\n'\'' | mongo mongodb://clusterAdmin:clusterAdmin123456@10.42.181.213:27017/admin ' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.Oo03xXFM8t ++ cat /tmp/tmp.M1js10EbXL ++ rm /tmp/tmp.Oo03xXFM8t /tmp/tmp.M1js10EbXL ++ return 0 + nodes_amount=6 + echo 'waiting for all members to be configured in rs1' waiting for all members to be configured in rs1 + let retry+=1 + '[' 4 -ge 15 ']' + echo -n . .+ sleep 10 + [[ 6 == 6 ]] ++ kubectl_bin config current-context +++ mktemp ++ local LAST_OUT=/tmp/tmp.tkXSq1js9r +++ mktemp ++ local LAST_ERR=/tmp/tmp.pCiGLKbayc ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl config current-context ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.tkXSq1js9r ++ cat /tmp/tmp.pCiGLKbayc ++ rm /tmp/tmp.tkXSq1js9r /tmp/tmp.pCiGLKbayc ++ return 0 + kubectl_bin config set-context gke_cloud-dev-112233_us-central1-a_jen-psmdb-1548-d2731110-1-cluster4 --namespace=cross-site-sharded-replica-17201 ++ mktemp + local LAST_OUT=/tmp/tmp.HNImeA05j8 ++ mktemp + local LAST_ERR=/tmp/tmp.7V37F8N7uy + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl config set-context gke_cloud-dev-112233_us-central1-a_jen-psmdb-1548-d2731110-1-cluster4 --namespace=cross-site-sharded-replica-17201 + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.HNImeA05j8 Context "gke_cloud-dev-112233_us-central1-a_jen-psmdb-1548-d2731110-1-cluster4" modified. + cat /tmp/tmp.7V37F8N7uy + rm /tmp/tmp.HNImeA05j8 /tmp/tmp.7V37F8N7uy + return 0 + desc 'check if all 3 Pods started' + set +o xtrace ----------------------------------------------------------------------------------- check if all 3 Pods started ----------------------------------------------------------------------------------- + wait_for_running cross-site-sharded-replica-rs0 3 + local name=cross-site-sharded-replica-rs0 + let last_pod=2 + local check_cluster_readyness=true + set_debug + [[ 1 == 1 ]] + set -o xtrace + local rs_name=rs0 + local cluster_name=cross-site-sharded-replica ++ seq 0 2 + for i in '$(seq 0 $last_pod)' + [[ 0 -eq 2 ]] + wait_pod cross-site-sharded-replica-rs0-0 + local pod=cross-site-sharded-replica-rs0-0 + set +o xtrace waiting for pod/cross-site-sharded-replica-rs0-0 to be ready.OK + for i in '$(seq 0 $last_pod)' + [[ 1 -eq 2 ]] + wait_pod cross-site-sharded-replica-rs0-1 + local pod=cross-site-sharded-replica-rs0-1 + set +o xtrace waiting for pod/cross-site-sharded-replica-rs0-1 to be ready.OK + for i in '$(seq 0 $last_pod)' + [[ 2 -eq 2 ]] ++ kubectl_bin get psmdb cross-site-sharded-replica -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].arbiter.enabled}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.rCnps4NnNh +++ mktemp ++ local LAST_ERR=/tmp/tmp.FimKdiIy0S ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get psmdb cross-site-sharded-replica -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].arbiter.enabled}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.rCnps4NnNh ++ cat /tmp/tmp.FimKdiIy0S ++ rm /tmp/tmp.rCnps4NnNh /tmp/tmp.FimKdiIy0S ++ return 0 + [[ false == \t\r\u\e ]] + wait_pod cross-site-sharded-replica-rs0-2 + local pod=cross-site-sharded-replica-rs0-2 + set +o xtrace waiting for pod/cross-site-sharded-replica-rs0-2 to be ready.OK ++ kubectl_bin get psmdb cross-site-sharded-replica -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].non_voting.enabled}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.gJLVj312S0 +++ mktemp ++ local LAST_ERR=/tmp/tmp.D1VtJZgkdk ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get psmdb cross-site-sharded-replica -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].non_voting.enabled}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.gJLVj312S0 ++ cat /tmp/tmp.D1VtJZgkdk ++ rm /tmp/tmp.gJLVj312S0 /tmp/tmp.D1VtJZgkdk ++ return 0 + [[ '' == \t\r\u\e ]] + sleep 10 + [[ true == \t\r\u\e ]] + set +x Waiting for cluster readyness + wait_for_running cross-site-sharded-replica-cfg 3 false + local name=cross-site-sharded-replica-cfg + let last_pod=2 + local check_cluster_readyness=false + set_debug + [[ 1 == 1 ]] + set -o xtrace + local rs_name=cfg + local cluster_name=cross-site-sharded-replica ++ seq 0 2 + for i in '$(seq 0 $last_pod)' + [[ 0 -eq 2 ]] + wait_pod cross-site-sharded-replica-cfg-0 + local pod=cross-site-sharded-replica-cfg-0 + set +o xtrace waiting for pod/cross-site-sharded-replica-cfg-0 to be ready.OK + for i in '$(seq 0 $last_pod)' + [[ 1 -eq 2 ]] + wait_pod cross-site-sharded-replica-cfg-1 + local pod=cross-site-sharded-replica-cfg-1 + set +o xtrace waiting for pod/cross-site-sharded-replica-cfg-1 to be ready.OK + for i in '$(seq 0 $last_pod)' + [[ 2 -eq 2 ]] ++ kubectl_bin get psmdb cross-site-sharded-replica -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].arbiter.enabled}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.mUpMbLkzwB +++ mktemp ++ local LAST_ERR=/tmp/tmp.rdq335jwIS ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get psmdb cross-site-sharded-replica -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].arbiter.enabled}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.mUpMbLkzwB ++ cat /tmp/tmp.rdq335jwIS ++ rm /tmp/tmp.mUpMbLkzwB /tmp/tmp.rdq335jwIS ++ return 0 + [[ '' == \t\r\u\e ]] + wait_pod cross-site-sharded-replica-cfg-2 + local pod=cross-site-sharded-replica-cfg-2 + set +o xtrace waiting for pod/cross-site-sharded-replica-cfg-2 to be ready.OK ++ kubectl_bin get psmdb cross-site-sharded-replica -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].non_voting.enabled}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.pb9guUcAXD +++ mktemp ++ local LAST_ERR=/tmp/tmp.SPQrri3Hn3 ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get psmdb cross-site-sharded-replica -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].non_voting.enabled}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.pb9guUcAXD ++ cat /tmp/tmp.SPQrri3Hn3 ++ rm /tmp/tmp.pb9guUcAXD /tmp/tmp.SPQrri3Hn3 ++ return 0 + [[ '' == \t\r\u\e ]] + sleep 10 + [[ false == \t\r\u\e ]] + desc 'create user' + set +o xtrace ----------------------------------------------------------------------------------- create user ----------------------------------------------------------------------------------- + run_mongos 'db.createUser({user:"myApp",pwd:"myPass",roles:[{db:"myApp",role:"readWrite"}]})' userAdmin:userAdmin123456@cross-site-sharded-main-mongos.cross-site-sharded-21880 + local 'command=db.createUser({user:"myApp",pwd:"myPass",roles:[{db:"myApp",role:"readWrite"}]})' + local uri=userAdmin:userAdmin123456@cross-site-sharded-main-mongos.cross-site-sharded-21880 + local driver=mongodb + local suffix=.svc.cluster.local ++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.LZdNpoMwl0 +++ mktemp ++ local LAST_ERR=/tmp/tmp.FMjfP2nHaK ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.LZdNpoMwl0 ++ cat /tmp/tmp.FMjfP2nHaK ++ rm /tmp/tmp.LZdNpoMwl0 /tmp/tmp.FMjfP2nHaK ++ return 0 + local client_container=psmdb-client-7469665986-nvp8k + local mongo_flag= + kubectl_bin exec psmdb-client-7469665986-nvp8k -- bash -c 'printf '\''db.createUser({user:"myApp",pwd:"myPass",roles:[{db:"myApp",role:"readWrite"}]})\n'\'' | mongo mongodb://userAdmin:userAdmin123456@cross-site-sharded-main-mongos.cross-site-sharded-21880.svc.cluster.local/admin ' ++ mktemp + local LAST_OUT=/tmp/tmp.5dgPZqyxkf ++ mktemp + local LAST_ERR=/tmp/tmp.HPX1nLrq5d + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl exec psmdb-client-7469665986-nvp8k -- bash -c 'printf '\''db.createUser({user:"myApp",pwd:"myPass",roles:[{db:"myApp",role:"readWrite"}]})\n'\'' | mongo mongodb://userAdmin:userAdmin123456@cross-site-sharded-main-mongos.cross-site-sharded-21880.svc.cluster.local/admin ' + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.5dgPZqyxkf Percona Server for MongoDB shell version v4.4.29-28 connecting to: mongodb://cross-site-sharded-main-mongos.cross-site-sharded-21880.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb Implicit session: session { "id" : UUID("4c2b998e-b576-4cb7-9413-f86f8e338531") } Percona Server for MongoDB server version: v7.0.8-5 WARNING: shell and server versions do not match Successfully added user: { "user" : "myApp", "roles" : [ { "db" : "myApp", "role" : "readWrite" } ] } bye + cat /tmp/tmp.HPX1nLrq5d + rm /tmp/tmp.5dgPZqyxkf /tmp/tmp.HPX1nLrq5d + return 0 + sleep 2 + desc 'write data, read from all' + set +o xtrace ----------------------------------------------------------------------------------- write data, read from all ----------------------------------------------------------------------------------- + run_mongos 'use myApp\n db.test.insert({ x: 100500 })' myApp:myPass@cross-site-sharded-main-mongos.cross-site-sharded-21880 + local 'command=use myApp\n db.test.insert({ x: 100500 })' + local uri=myApp:myPass@cross-site-sharded-main-mongos.cross-site-sharded-21880 + local driver=mongodb + local suffix=.svc.cluster.local ++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.PlAUZMztoi +++ mktemp ++ local LAST_ERR=/tmp/tmp.TSJgyC3LUE ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.PlAUZMztoi ++ cat /tmp/tmp.TSJgyC3LUE ++ rm /tmp/tmp.PlAUZMztoi /tmp/tmp.TSJgyC3LUE ++ return 0 + local client_container=psmdb-client-7469665986-nvp8k + local mongo_flag= + kubectl_bin exec psmdb-client-7469665986-nvp8k -- bash -c 'printf '\''use myApp\n db.test.insert({ x: 100500 })\n'\'' | mongo mongodb://myApp:myPass@cross-site-sharded-main-mongos.cross-site-sharded-21880.svc.cluster.local/admin ' ++ mktemp + local LAST_OUT=/tmp/tmp.6WHYA6YeEv ++ mktemp + local LAST_ERR=/tmp/tmp.9tGbCdCsO0 + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl exec psmdb-client-7469665986-nvp8k -- bash -c 'printf '\''use myApp\n db.test.insert({ x: 100500 })\n'\'' | mongo mongodb://myApp:myPass@cross-site-sharded-main-mongos.cross-site-sharded-21880.svc.cluster.local/admin ' + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.6WHYA6YeEv Percona Server for MongoDB shell version v4.4.29-28 connecting to: mongodb://cross-site-sharded-main-mongos.cross-site-sharded-21880.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb Implicit session: session { "id" : UUID("598ae41f-61b5-451f-a76b-7ed6f95af74d") } Percona Server for MongoDB server version: v7.0.8-5 WARNING: shell and server versions do not match switched to db myApp WriteResult({ "nInserted" : 1 }) bye + cat /tmp/tmp.9tGbCdCsO0 + rm /tmp/tmp.6WHYA6YeEv /tmp/tmp.9tGbCdCsO0 + return 0 + minikube_sleep + sleep_time=10 + [[ '' == 1 ]] + desc 'Compare data' + set +o xtrace ----------------------------------------------------------------------------------- Compare data ----------------------------------------------------------------------------------- + compare_mongos_cmd find myApp:myPass@cross-site-sharded-main-mongos.cross-site-sharded-21880 + local command=find + local uri=myApp:myPass@cross-site-sharded-main-mongos.cross-site-sharded-21880 + local postfix= + local suffix= + local database=myApp + local collection=test + run_mongos 'use myApp\n db.test.find()' myApp:myPass@cross-site-sharded-main-mongos.cross-site-sharded-21880 mongodb '' + local 'command=use myApp\n db.test.find()' + egrep -v 'I NETWORK|W NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:' + local uri=myApp:myPass@cross-site-sharded-main-mongos.cross-site-sharded-21880 + local driver=mongodb + local suffix=.svc.cluster.local + /usr/bin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/' ++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.uGYkLAaBwp +++ mktemp ++ local LAST_ERR=/tmp/tmp.ADfBGhPGyW ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.uGYkLAaBwp ++ cat /tmp/tmp.ADfBGhPGyW ++ rm /tmp/tmp.uGYkLAaBwp /tmp/tmp.ADfBGhPGyW ++ return 0 + local client_container=psmdb-client-7469665986-nvp8k + local mongo_flag= + kubectl_bin exec psmdb-client-7469665986-nvp8k -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@cross-site-sharded-main-mongos.cross-site-sharded-21880.svc.cluster.local/admin ' ++ mktemp + local LAST_OUT=/tmp/tmp.93S453KgXh ++ mktemp + local LAST_ERR=/tmp/tmp.TmI7y9OULU + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl exec psmdb-client-7469665986-nvp8k -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@cross-site-sharded-main-mongos.cross-site-sharded-21880.svc.cluster.local/admin ' + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.93S453KgXh + cat /tmp/tmp.TmI7y9OULU + rm /tmp/tmp.93S453KgXh /tmp/tmp.TmI7y9OULU + return 0 + diff /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1548/e2e-tests/cross-site-sharded/compare/find.json /tmp/tmp.M9iiA1fJys/find + desc 'test failover' + set +o xtrace ----------------------------------------------------------------------------------- test failover ----------------------------------------------------------------------------------- ++ kubectl_bin config current-context +++ mktemp ++ local LAST_OUT=/tmp/tmp.rp9Kghx68g +++ mktemp ++ local LAST_ERR=/tmp/tmp.EBt5SnvNfD ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl config current-context ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.rp9Kghx68g ++ cat /tmp/tmp.EBt5SnvNfD ++ rm /tmp/tmp.rp9Kghx68g /tmp/tmp.EBt5SnvNfD ++ return 0 + kubectl_bin config set-context gke_cloud-dev-112233_us-central1-a_jen-psmdb-1548-d2731110-1-cluster4 --namespace=cross-site-sharded-21880 ++ mktemp + local LAST_OUT=/tmp/tmp.nvuX8I0Irr ++ mktemp + local LAST_ERR=/tmp/tmp.xWbz5VbW9d + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl config set-context gke_cloud-dev-112233_us-central1-a_jen-psmdb-1548-d2731110-1-cluster4 --namespace=cross-site-sharded-21880 + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.nvuX8I0Irr Context "gke_cloud-dev-112233_us-central1-a_jen-psmdb-1548-d2731110-1-cluster4" modified. + cat /tmp/tmp.xWbz5VbW9d + rm /tmp/tmp.nvuX8I0Irr /tmp/tmp.xWbz5VbW9d + return 0 + kubectl_bin delete psmdb cross-site-sharded-main ++ mktemp + local LAST_OUT=/tmp/tmp.yfrjH8DPGC ++ mktemp + local LAST_ERR=/tmp/tmp.zj0UDXUqGP + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl delete psmdb cross-site-sharded-main + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.yfrjH8DPGC perconaservermongodb.psmdb.percona.com "cross-site-sharded-main" deleted + cat /tmp/tmp.zj0UDXUqGP + rm /tmp/tmp.yfrjH8DPGC /tmp/tmp.zj0UDXUqGP + return 0 + desc 'run disaster recovery script for replset: cfg' + set +o xtrace ----------------------------------------------------------------------------------- run disaster recovery script for replset: cfg ----------------------------------------------------------------------------------- + run_script_mongos /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1548/e2e-tests/cross-site-sharded/disaster_recovery.js clusterAdmin:clusterAdmin123456@10.42.190.168 mongodb :27017 + local script=/mnt/jenkins/workspace/cloud-psmdb-operator_PR-1548/e2e-tests/cross-site-sharded/disaster_recovery.js + local uri=clusterAdmin:clusterAdmin123456@10.42.190.168 + local driver=mongodb + local suffix=:27017 ++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.NxqU7s2Emx +++ mktemp ++ local LAST_ERR=/tmp/tmp.5c4Xnwlabx ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.NxqU7s2Emx ++ cat /tmp/tmp.5c4Xnwlabx ++ rm /tmp/tmp.NxqU7s2Emx /tmp/tmp.5c4Xnwlabx ++ return 0 + local client_container=psmdb-client-7469665986-xhdrq + local mongo_flag= ++ basename /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1548/e2e-tests/cross-site-sharded/disaster_recovery.js + name=disaster_recovery.js + kubectl_bin cp /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1548/e2e-tests/cross-site-sharded/disaster_recovery.js cross-site-sharded-21880/psmdb-client-7469665986-xhdrq:/tmp ++ mktemp + local LAST_OUT=/tmp/tmp.QAT5QIUvOC ++ mktemp + local LAST_ERR=/tmp/tmp.JJPvlkqY1A + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl cp /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1548/e2e-tests/cross-site-sharded/disaster_recovery.js cross-site-sharded-21880/psmdb-client-7469665986-xhdrq:/tmp + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.QAT5QIUvOC + cat /tmp/tmp.JJPvlkqY1A + rm /tmp/tmp.QAT5QIUvOC /tmp/tmp.JJPvlkqY1A + return 0 + kubectl_bin exec psmdb-client-7469665986-xhdrq -- bash -c 'mongo mongodb://clusterAdmin:clusterAdmin123456@10.42.190.168:27017/admin /tmp/disaster_recovery.js' ++ mktemp + local LAST_OUT=/tmp/tmp.Brnxk1u3D4 ++ mktemp + local LAST_ERR=/tmp/tmp.QRzJMOoQxj + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl exec psmdb-client-7469665986-xhdrq -- bash -c 'mongo mongodb://clusterAdmin:clusterAdmin123456@10.42.190.168:27017/admin /tmp/disaster_recovery.js' + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.Brnxk1u3D4 Percona Server for MongoDB shell version v4.4.29-28 connecting to: mongodb://10.42.190.168:27017/admin?compressors=disabled&gssapiServiceName=mongodb Implicit session: session { "id" : UUID("302fe75e-fdde-411b-8568-4a6894239bd4") } Percona Server for MongoDB server version: v7.0.8-5 WARNING: shell and server versions do not match + cat /tmp/tmp.QRzJMOoQxj + rm /tmp/tmp.Brnxk1u3D4 /tmp/tmp.QRzJMOoQxj + return 0 + desc 'run disaster recovery script for replset: rs0' + set +o xtrace ----------------------------------------------------------------------------------- run disaster recovery script for replset: rs0 ----------------------------------------------------------------------------------- + run_script_mongos /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1548/e2e-tests/cross-site-sharded/disaster_recovery.js clusterAdmin:clusterAdmin123456@10.42.184.66 mongodb :27017 + local script=/mnt/jenkins/workspace/cloud-psmdb-operator_PR-1548/e2e-tests/cross-site-sharded/disaster_recovery.js + local uri=clusterAdmin:clusterAdmin123456@10.42.184.66 + local driver=mongodb + local suffix=:27017 ++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.h1RalpWdUX +++ mktemp ++ local LAST_ERR=/tmp/tmp.CSHD8cRtgf ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.h1RalpWdUX ++ cat /tmp/tmp.CSHD8cRtgf ++ rm /tmp/tmp.h1RalpWdUX /tmp/tmp.CSHD8cRtgf ++ return 0 + local client_container=psmdb-client-7469665986-xhdrq + local mongo_flag= ++ basename /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1548/e2e-tests/cross-site-sharded/disaster_recovery.js + name=disaster_recovery.js + kubectl_bin cp /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1548/e2e-tests/cross-site-sharded/disaster_recovery.js cross-site-sharded-21880/psmdb-client-7469665986-xhdrq:/tmp ++ mktemp + local LAST_OUT=/tmp/tmp.RQiuLqXQQi ++ mktemp + local LAST_ERR=/tmp/tmp.Ye1MmIhaA2 + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl cp /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1548/e2e-tests/cross-site-sharded/disaster_recovery.js cross-site-sharded-21880/psmdb-client-7469665986-xhdrq:/tmp + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.RQiuLqXQQi + cat /tmp/tmp.Ye1MmIhaA2 + rm /tmp/tmp.RQiuLqXQQi /tmp/tmp.Ye1MmIhaA2 + return 0 + kubectl_bin exec psmdb-client-7469665986-xhdrq -- bash -c 'mongo mongodb://clusterAdmin:clusterAdmin123456@10.42.184.66:27017/admin /tmp/disaster_recovery.js' ++ mktemp + local LAST_OUT=/tmp/tmp.DyEBABxRdh ++ mktemp + local LAST_ERR=/tmp/tmp.J7wXmMjvKL + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl exec psmdb-client-7469665986-xhdrq -- bash -c 'mongo mongodb://clusterAdmin:clusterAdmin123456@10.42.184.66:27017/admin /tmp/disaster_recovery.js' + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.DyEBABxRdh Percona Server for MongoDB shell version v4.4.29-28 connecting to: mongodb://10.42.184.66:27017/admin?compressors=disabled&gssapiServiceName=mongodb Implicit session: session { "id" : UUID("bed2d100-99e0-4bd0-96ed-8c0ce3181c7c") } Percona Server for MongoDB server version: v7.0.8-5 WARNING: shell and server versions do not match + cat /tmp/tmp.J7wXmMjvKL + rm /tmp/tmp.DyEBABxRdh /tmp/tmp.J7wXmMjvKL + return 0 + desc 'run disaster recovery script for replset: rs1' + set +o xtrace ----------------------------------------------------------------------------------- run disaster recovery script for replset: rs1 ----------------------------------------------------------------------------------- + run_script_mongos /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1548/e2e-tests/cross-site-sharded/disaster_recovery.js clusterAdmin:clusterAdmin123456@10.42.181.213 mongodb :27017 + local script=/mnt/jenkins/workspace/cloud-psmdb-operator_PR-1548/e2e-tests/cross-site-sharded/disaster_recovery.js + local uri=clusterAdmin:clusterAdmin123456@10.42.181.213 + local driver=mongodb + local suffix=:27017 ++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.rosE1evlfU +++ mktemp ++ local LAST_ERR=/tmp/tmp.8wzGfdVMwT ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.rosE1evlfU ++ cat /tmp/tmp.8wzGfdVMwT ++ rm /tmp/tmp.rosE1evlfU /tmp/tmp.8wzGfdVMwT ++ return 0 + local client_container=psmdb-client-7469665986-xhdrq + local mongo_flag= ++ basename /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1548/e2e-tests/cross-site-sharded/disaster_recovery.js + name=disaster_recovery.js + kubectl_bin cp /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1548/e2e-tests/cross-site-sharded/disaster_recovery.js cross-site-sharded-21880/psmdb-client-7469665986-xhdrq:/tmp ++ mktemp + local LAST_OUT=/tmp/tmp.Pbpvm81iVp ++ mktemp + local LAST_ERR=/tmp/tmp.82z7VYE96G + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl cp /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1548/e2e-tests/cross-site-sharded/disaster_recovery.js cross-site-sharded-21880/psmdb-client-7469665986-xhdrq:/tmp + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.Pbpvm81iVp + cat /tmp/tmp.82z7VYE96G + rm /tmp/tmp.Pbpvm81iVp /tmp/tmp.82z7VYE96G + return 0 + kubectl_bin exec psmdb-client-7469665986-xhdrq -- bash -c 'mongo mongodb://clusterAdmin:clusterAdmin123456@10.42.181.213:27017/admin /tmp/disaster_recovery.js' ++ mktemp + local LAST_OUT=/tmp/tmp.rnKR1gbuUo ++ mktemp + local LAST_ERR=/tmp/tmp.gDt6vfFllk + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl exec psmdb-client-7469665986-xhdrq -- bash -c 'mongo mongodb://clusterAdmin:clusterAdmin123456@10.42.181.213:27017/admin /tmp/disaster_recovery.js' + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.rnKR1gbuUo Percona Server for MongoDB shell version v4.4.29-28 connecting to: mongodb://10.42.181.213:27017/admin?compressors=disabled&gssapiServiceName=mongodb Implicit session: session { "id" : UUID("af31d98a-a13f-43a8-b004-89f9e610f3ff") } Percona Server for MongoDB server version: v7.0.8-5 WARNING: shell and server versions do not match + cat /tmp/tmp.gDt6vfFllk + rm /tmp/tmp.rnKR1gbuUo /tmp/tmp.gDt6vfFllk + return 0 + desc 'make replica cluster managed' + set +o xtrace ----------------------------------------------------------------------------------- make replica cluster managed ----------------------------------------------------------------------------------- ++ kubectl_bin config current-context +++ mktemp ++ local LAST_OUT=/tmp/tmp.XFKnBz98O2 +++ mktemp ++ local LAST_ERR=/tmp/tmp.czPl987k7x ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl config current-context ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.XFKnBz98O2 ++ cat /tmp/tmp.czPl987k7x ++ rm /tmp/tmp.XFKnBz98O2 /tmp/tmp.czPl987k7x ++ return 0 + kubectl_bin config set-context gke_cloud-dev-112233_us-central1-a_jen-psmdb-1548-d2731110-1-cluster4 --namespace=cross-site-sharded-replica-17201 ++ mktemp + local LAST_OUT=/tmp/tmp.EBNkKJXT7e ++ mktemp + local LAST_ERR=/tmp/tmp.GP9Wuqi73b + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl config set-context gke_cloud-dev-112233_us-central1-a_jen-psmdb-1548-d2731110-1-cluster4 --namespace=cross-site-sharded-replica-17201 + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.EBNkKJXT7e Context "gke_cloud-dev-112233_us-central1-a_jen-psmdb-1548-d2731110-1-cluster4" modified. + cat /tmp/tmp.GP9Wuqi73b + rm /tmp/tmp.EBNkKJXT7e /tmp/tmp.GP9Wuqi73b + return 0 + kubectl_bin patch psmdb cross-site-sharded-replica --type=merge --patch '{"spec":{"unmanaged": false}}' ++ mktemp + local LAST_OUT=/tmp/tmp.fodeN0Q7dO ++ mktemp + local LAST_ERR=/tmp/tmp.X2VpV5a98t + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl patch psmdb cross-site-sharded-replica --type=merge --patch '{"spec":{"unmanaged": false}}' + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.fodeN0Q7dO perconaservermongodb.psmdb.percona.com/cross-site-sharded-replica patched + cat /tmp/tmp.X2VpV5a98t + rm /tmp/tmp.fodeN0Q7dO /tmp/tmp.X2VpV5a98t + return 0 + wait_for_running cross-site-sharded-replica-rs0 3 + local name=cross-site-sharded-replica-rs0 + let last_pod=2 + local check_cluster_readyness=true + set_debug + [[ 1 == 1 ]] + set -o xtrace + local rs_name=rs0 + local cluster_name=cross-site-sharded-replica ++ seq 0 2 + for i in '$(seq 0 $last_pod)' + [[ 0 -eq 2 ]] + wait_pod cross-site-sharded-replica-rs0-0 + local pod=cross-site-sharded-replica-rs0-0 + set +o xtrace waiting for pod/cross-site-sharded-replica-rs0-0 to be ready.OK + for i in '$(seq 0 $last_pod)' + [[ 1 -eq 2 ]] + wait_pod cross-site-sharded-replica-rs0-1 + local pod=cross-site-sharded-replica-rs0-1 + set +o xtrace waiting for pod/cross-site-sharded-replica-rs0-1 to be ready.OK + for i in '$(seq 0 $last_pod)' + [[ 2 -eq 2 ]] ++ kubectl_bin get psmdb cross-site-sharded-replica -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].arbiter.enabled}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.PrTszbOcs7 +++ mktemp ++ local LAST_ERR=/tmp/tmp.4bq958N4Qy ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get psmdb cross-site-sharded-replica -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].arbiter.enabled}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.PrTszbOcs7 ++ cat /tmp/tmp.4bq958N4Qy ++ rm /tmp/tmp.PrTszbOcs7 /tmp/tmp.4bq958N4Qy ++ return 0 + [[ false == \t\r\u\e ]] + wait_pod cross-site-sharded-replica-rs0-2 + local pod=cross-site-sharded-replica-rs0-2 + set +o xtrace waiting for pod/cross-site-sharded-replica-rs0-2 to be ready.OK ++ kubectl_bin get psmdb cross-site-sharded-replica -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].non_voting.enabled}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.5ixuC3QBYM +++ mktemp ++ local LAST_ERR=/tmp/tmp.fEQKesInDK ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get psmdb cross-site-sharded-replica -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].non_voting.enabled}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.5ixuC3QBYM ++ cat /tmp/tmp.fEQKesInDK ++ rm /tmp/tmp.5ixuC3QBYM /tmp/tmp.fEQKesInDK ++ return 0 + [[ '' == \t\r\u\e ]] + sleep 10 + [[ true == \t\r\u\e ]] + set +x Waiting for cluster readyness + wait_for_running cross-site-sharded-replica-cfg 3 + local name=cross-site-sharded-replica-cfg + let last_pod=2 + local check_cluster_readyness=true + set_debug + [[ 1 == 1 ]] + set -o xtrace + local rs_name=cfg + local cluster_name=cross-site-sharded-replica ++ seq 0 2 + for i in '$(seq 0 $last_pod)' + [[ 0 -eq 2 ]] + wait_pod cross-site-sharded-replica-cfg-0 + local pod=cross-site-sharded-replica-cfg-0 + set +o xtrace waiting for pod/cross-site-sharded-replica-cfg-0 to be ready.OK + for i in '$(seq 0 $last_pod)' + [[ 1 -eq 2 ]] + wait_pod cross-site-sharded-replica-cfg-1 + local pod=cross-site-sharded-replica-cfg-1 + set +o xtrace waiting for pod/cross-site-sharded-replica-cfg-1 to be ready.OK + for i in '$(seq 0 $last_pod)' + [[ 2 -eq 2 ]] ++ kubectl_bin get psmdb cross-site-sharded-replica -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].arbiter.enabled}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.1NIGvMMt4e +++ mktemp ++ local LAST_ERR=/tmp/tmp.s6Yc5RMU52 ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get psmdb cross-site-sharded-replica -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].arbiter.enabled}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.1NIGvMMt4e ++ cat /tmp/tmp.s6Yc5RMU52 ++ rm /tmp/tmp.1NIGvMMt4e /tmp/tmp.s6Yc5RMU52 ++ return 0 + [[ '' == \t\r\u\e ]] + wait_pod cross-site-sharded-replica-cfg-2 + local pod=cross-site-sharded-replica-cfg-2 + set +o xtrace waiting for pod/cross-site-sharded-replica-cfg-2 to be ready.OK ++ kubectl_bin get psmdb cross-site-sharded-replica -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].non_voting.enabled}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.QYXrdJEUka +++ mktemp ++ local LAST_ERR=/tmp/tmp.VUx3yVNQxp ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get psmdb cross-site-sharded-replica -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].non_voting.enabled}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.QYXrdJEUka ++ cat /tmp/tmp.VUx3yVNQxp ++ rm /tmp/tmp.QYXrdJEUka /tmp/tmp.VUx3yVNQxp ++ return 0 + [[ '' == \t\r\u\e ]] + sleep 10 + [[ true == \t\r\u\e ]] + set +x Waiting for cluster readyness + desc 'check failover status' + set +o xtrace ----------------------------------------------------------------------------------- check failover status ----------------------------------------------------------------------------------- + compare_mongos_cmd find myApp:myPass@cross-site-sharded-replica-mongos.cross-site-sharded-replica-17201 + local command=find + local uri=myApp:myPass@cross-site-sharded-replica-mongos.cross-site-sharded-replica-17201 + local postfix= + local suffix= + local database=myApp + local collection=test + egrep -v 'I NETWORK|W NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:' + run_mongos 'use myApp\n db.test.find()' myApp:myPass@cross-site-sharded-replica-mongos.cross-site-sharded-replica-17201 mongodb '' + local 'command=use myApp\n db.test.find()' + local uri=myApp:myPass@cross-site-sharded-replica-mongos.cross-site-sharded-replica-17201 + local driver=mongodb + local suffix=.svc.cluster.local ++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.tXItA6TgBo +++ mktemp ++ local LAST_ERR=/tmp/tmp.ybzalHJYCb ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' + /usr/bin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.tXItA6TgBo ++ cat /tmp/tmp.ybzalHJYCb ++ rm /tmp/tmp.tXItA6TgBo /tmp/tmp.ybzalHJYCb ++ return 0 + local client_container=psmdb-client-7469665986-nvp8k + local mongo_flag= + kubectl_bin exec psmdb-client-7469665986-nvp8k -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@cross-site-sharded-replica-mongos.cross-site-sharded-replica-17201.svc.cluster.local/admin ' ++ mktemp + local LAST_OUT=/tmp/tmp.ks4RoqQVJm ++ mktemp + local LAST_ERR=/tmp/tmp.xPGzDCaf3D + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl exec psmdb-client-7469665986-nvp8k -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@cross-site-sharded-replica-mongos.cross-site-sharded-replica-17201.svc.cluster.local/admin ' + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.ks4RoqQVJm + cat /tmp/tmp.xPGzDCaf3D + rm /tmp/tmp.ks4RoqQVJm /tmp/tmp.xPGzDCaf3D + return 0 + diff /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1548/e2e-tests/cross-site-sharded/compare/find.json /tmp/tmp.M9iiA1fJys/find + desc 'Failover check finished successfully' + set +o xtrace ----------------------------------------------------------------------------------- Failover check finished successfully ----------------------------------------------------------------------------------- + wait_cluster_consistency cross-site-sharded-replica + local cluster_name=cross-site-sharded-replica + local wait_time=32 + retry=0 + sleep 7 + echo -n 'waiting for cluster readyness' waiting for cluster readyness++ kubectl_bin get psmdb cross-site-sharded-replica -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.fdgICoQta0 +++ mktemp ++ local LAST_ERR=/tmp/tmp.oumaFPFOdf ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get psmdb cross-site-sharded-replica -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.fdgICoQta0 ++ cat /tmp/tmp.oumaFPFOdf ++ rm /tmp/tmp.fdgICoQta0 /tmp/tmp.oumaFPFOdf ++ return 0 + [[ ready == \r\e\a\d\y ]] + destroy cross-site-sharded-21880 true + local namespace=cross-site-sharded-21880 + local ignore_logs=true + desc 'destroy cluster/operator and all other resources' + set +o xtrace ----------------------------------------------------------------------------------- destroy cluster/operator and all other resources ----------------------------------------------------------------------------------- + '[' true == false ']' + delete_crd + desc 'get and delete old CRDs and RBAC' + set +o xtrace ----------------------------------------------------------------------------------- get and delete old CRDs and RBAC ----------------------------------------------------------------------------------- + kubectl_bin delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1548/deploy/crd.yaml --ignore-not-found --wait=false ++ mktemp + local LAST_OUT=/tmp/tmp.XxzvOcNrca ++ mktemp + local LAST_ERR=/tmp/tmp.HHp78JY69b + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1548/deploy/crd.yaml --ignore-not-found --wait=false + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.XxzvOcNrca customresourcedefinition.apiextensions.k8s.io "perconaservermongodbbackups.psmdb.percona.com" deleted customresourcedefinition.apiextensions.k8s.io "perconaservermongodbrestores.psmdb.percona.com" deleted customresourcedefinition.apiextensions.k8s.io "perconaservermongodbs.psmdb.percona.com" deleted + cat /tmp/tmp.HHp78JY69b + rm /tmp/tmp.XxzvOcNrca /tmp/tmp.HHp78JY69b + return 0 ++ grep -v '\-\-\-' ++ yq eval .metadata.name /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1548/deploy/crd.yaml + for crd_name in '$(yq eval '\''.metadata.name'\'' "${src_dir}/deploy/crd.yaml" | grep -v '\''\-\-\-'\'')' + kubectl get perconaservermongodbbackups.psmdb.percona.com --all-namespaces -o wide + xargs -L 1 sh -xc 'kubectl patch perconaservermongodbbackups.psmdb.percona.com -n $0 $1 --type=merge -p "{\"metadata\":{\"finalizers\":[]}}"' + grep -v NAMESPACE error: the server doesn't have a resource type "perconaservermongodbbackups" + kubectl patch perconaservermongodbbackups.psmdb.percona.com -n sh --type=merge -p '{"metadata":{"finalizers":[]}}' error: the server doesn't have a resource type "perconaservermongodbbackups" + : + kubectl_bin wait --for=delete crd perconaservermongodbbackups.psmdb.percona.com ++ mktemp + local LAST_OUT=/tmp/tmp.91EIXbsVlh ++ mktemp + local LAST_ERR=/tmp/tmp.kKDdn9xqhi + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl wait --for=delete crd perconaservermongodbbackups.psmdb.percona.com + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.91EIXbsVlh + cat /tmp/tmp.kKDdn9xqhi + rm /tmp/tmp.91EIXbsVlh /tmp/tmp.kKDdn9xqhi + return 0 + for crd_name in '$(yq eval '\''.metadata.name'\'' "${src_dir}/deploy/crd.yaml" | grep -v '\''\-\-\-'\'')' + kubectl get perconaservermongodbrestores.psmdb.percona.com --all-namespaces -o wide + grep -v NAMESPACE + xargs -L 1 sh -xc 'kubectl patch perconaservermongodbrestores.psmdb.percona.com -n $0 $1 --type=merge -p "{\"metadata\":{\"finalizers\":[]}}"' error: the server doesn't have a resource type "perconaservermongodbrestores" + kubectl patch perconaservermongodbrestores.psmdb.percona.com -n sh --type=merge -p '{"metadata":{"finalizers":[]}}' error: the server doesn't have a resource type "perconaservermongodbrestores" + : + kubectl_bin wait --for=delete crd perconaservermongodbrestores.psmdb.percona.com ++ mktemp + local LAST_OUT=/tmp/tmp.bM9m8I2EF9 ++ mktemp + local LAST_ERR=/tmp/tmp.D9GkayQUB2 + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl wait --for=delete crd perconaservermongodbrestores.psmdb.percona.com + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.bM9m8I2EF9 + cat /tmp/tmp.D9GkayQUB2 + rm /tmp/tmp.bM9m8I2EF9 /tmp/tmp.D9GkayQUB2 + return 0 + for crd_name in '$(yq eval '\''.metadata.name'\'' "${src_dir}/deploy/crd.yaml" | grep -v '\''\-\-\-'\'')' + grep -v NAMESPACE + kubectl get perconaservermongodbs.psmdb.percona.com --all-namespaces -o wide + xargs -L 1 sh -xc 'kubectl patch perconaservermongodbs.psmdb.percona.com -n $0 $1 --type=merge -p "{\"metadata\":{\"finalizers\":[]}}"' error: the server doesn't have a resource type "perconaservermongodbs" + kubectl patch perconaservermongodbs.psmdb.percona.com -n sh --type=merge -p '{"metadata":{"finalizers":[]}}' error: the server doesn't have a resource type "perconaservermongodbs" + : + kubectl_bin wait --for=delete crd perconaservermongodbs.psmdb.percona.com ++ mktemp + local LAST_OUT=/tmp/tmp.p4T2acR8nE ++ mktemp + local LAST_ERR=/tmp/tmp.uJwKreceCK + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl wait --for=delete crd perconaservermongodbs.psmdb.percona.com + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.p4T2acR8nE + cat /tmp/tmp.uJwKreceCK + rm /tmp/tmp.p4T2acR8nE /tmp/tmp.uJwKreceCK + return 0 + local rbac_yaml=rbac.yaml + '[' -n '' ']' + kubectl_bin delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1548/deploy/rbac.yaml --ignore-not-found ++ mktemp + local LAST_OUT=/tmp/tmp.RWHAde2ELr ++ mktemp + local LAST_ERR=/tmp/tmp.AyRz7Vnl29 + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1548/deploy/rbac.yaml --ignore-not-found + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.RWHAde2ELr role.rbac.authorization.k8s.io "percona-server-mongodb-operator" deleted serviceaccount "percona-server-mongodb-operator" deleted rolebinding.rbac.authorization.k8s.io "service-account-percona-server-mongodb-operator" deleted + cat /tmp/tmp.AyRz7Vnl29 + rm /tmp/tmp.RWHAde2ELr /tmp/tmp.AyRz7Vnl29 + return 0 + kubectl_bin delete -f https://github.com/cert-manager/cert-manager/releases/download/v1.12.4/cert-manager.yaml + : + '[' -n '' ']' + '[' -n '' ']' + rm -rf /tmp/tmp.M9iiA1fJys + kubectl_bin delete --grace-period=0 --force=true namespace cross-site-sharded-21880 + destroy cross-site-sharded-replica-17201 true + local namespace=cross-site-sharded-replica-17201 + local ignore_logs=true + desc 'destroy cluster/operator and all other resources' + set +o xtrace ----------------------------------------------------------------------------------- destroy cluster/operator and all other resources ----------------------------------------------------------------------------------- + '[' true == false ']' + delete_crd + desc 'get and delete old CRDs and RBAC' + set +o xtrace ----------------------------------------------------------------------------------- get and delete old CRDs and RBAC ----------------------------------------------------------------------------------- + kubectl_bin delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1548/deploy/crd.yaml --ignore-not-found --wait=false ++ mktemp + local LAST_OUT=/tmp/tmp.KvJbEIC8Xa ++ mktemp + local LAST_OUT=/tmp/tmp.2YXWSKHEc3 ++ mktemp + local LAST_ERR=/tmp/tmp.PClg0Dd2NB + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl delete --grace-period=0 --force=true namespace cross-site-sharded-21880 ++ mktemp + local LAST_ERR=/tmp/tmp.305Tcuuwvt + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1548/deploy/crd.yaml --ignore-not-found --wait=false + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.KvJbEIC8Xa + cat /tmp/tmp.305Tcuuwvt + rm /tmp/tmp.KvJbEIC8Xa /tmp/tmp.305Tcuuwvt + return 0 ++ yq eval .metadata.name /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1548/deploy/crd.yaml ++ grep -v '\-\-\-' + for crd_name in '$(yq eval '\''.metadata.name'\'' "${src_dir}/deploy/crd.yaml" | grep -v '\''\-\-\-'\'')' + grep -v NAMESPACE + kubectl get perconaservermongodbbackups.psmdb.percona.com --all-namespaces -o wide + xargs -L 1 sh -xc 'kubectl patch perconaservermongodbbackups.psmdb.percona.com -n $0 $1 --type=merge -p "{\"metadata\":{\"finalizers\":[]}}"' error: the server doesn't have a resource type "perconaservermongodbbackups" + kubectl patch perconaservermongodbbackups.psmdb.percona.com -n sh --type=merge -p '{"metadata":{"finalizers":[]}}' error: the server doesn't have a resource type "perconaservermongodbbackups" + : + kubectl_bin wait --for=delete crd perconaservermongodbbackups.psmdb.percona.com ++ mktemp + local LAST_OUT=/tmp/tmp.w3V4PW0Tgv ++ mktemp + local LAST_ERR=/tmp/tmp.6tDEw591Ia + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl wait --for=delete crd perconaservermongodbbackups.psmdb.percona.com + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.w3V4PW0Tgv + cat /tmp/tmp.6tDEw591Ia + rm /tmp/tmp.w3V4PW0Tgv /tmp/tmp.6tDEw591Ia + return 0 + for crd_name in '$(yq eval '\''.metadata.name'\'' "${src_dir}/deploy/crd.yaml" | grep -v '\''\-\-\-'\'')' + kubectl get perconaservermongodbrestores.psmdb.percona.com --all-namespaces -o wide + grep -v NAMESPACE + xargs -L 1 sh -xc 'kubectl patch perconaservermongodbrestores.psmdb.percona.com -n $0 $1 --type=merge -p "{\"metadata\":{\"finalizers\":[]}}"' error: the server doesn't have a resource type "perconaservermongodbrestores" + kubectl patch perconaservermongodbrestores.psmdb.percona.com -n sh --type=merge -p '{"metadata":{"finalizers":[]}}' error: the server doesn't have a resource type "perconaservermongodbrestores" + : + kubectl_bin wait --for=delete crd perconaservermongodbrestores.psmdb.percona.com ++ mktemp + local LAST_OUT=/tmp/tmp.SG0KzlAGRR ++ mktemp + local LAST_ERR=/tmp/tmp.VyUHEPp7PK + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl wait --for=delete crd perconaservermongodbrestores.psmdb.percona.com + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.SG0KzlAGRR + cat /tmp/tmp.VyUHEPp7PK + rm /tmp/tmp.SG0KzlAGRR /tmp/tmp.VyUHEPp7PK + return 0 + for crd_name in '$(yq eval '\''.metadata.name'\'' "${src_dir}/deploy/crd.yaml" | grep -v '\''\-\-\-'\'')' + xargs -L 1 sh -xc 'kubectl patch perconaservermongodbs.psmdb.percona.com -n $0 $1 --type=merge -p "{\"metadata\":{\"finalizers\":[]}}"' + kubectl get perconaservermongodbs.psmdb.percona.com --all-namespaces -o wide + grep -v NAMESPACE error: the server doesn't have a resource type "perconaservermongodbs" + kubectl patch perconaservermongodbs.psmdb.percona.com -n sh --type=merge -p '{"metadata":{"finalizers":[]}}' error: the server doesn't have a resource type "perconaservermongodbs" + : + kubectl_bin wait --for=delete crd perconaservermongodbs.psmdb.percona.com ++ mktemp + local LAST_OUT=/tmp/tmp.UeLtsLyajq ++ mktemp + local LAST_ERR=/tmp/tmp.N4qqfpawlq + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl wait --for=delete crd perconaservermongodbs.psmdb.percona.com + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.UeLtsLyajq + cat /tmp/tmp.N4qqfpawlq + rm /tmp/tmp.UeLtsLyajq /tmp/tmp.N4qqfpawlq + return 0 + local rbac_yaml=rbac.yaml + '[' -n '' ']' + kubectl_bin delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1548/deploy/rbac.yaml --ignore-not-found ++ mktemp + local LAST_OUT=/tmp/tmp.NcPFRmYXzP ++ mktemp + local LAST_ERR=/tmp/tmp.mDKTIaejXu + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1548/deploy/rbac.yaml --ignore-not-found + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.NcPFRmYXzP + cat /tmp/tmp.mDKTIaejXu + rm /tmp/tmp.NcPFRmYXzP /tmp/tmp.mDKTIaejXu + return 0 + kubectl_bin delete -f https://github.com/cert-manager/cert-manager/releases/download/v1.12.4/cert-manager.yaml + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.2YXWSKHEc3 namespace "cross-site-sharded-21880" force deleted + cat /tmp/tmp.PClg0Dd2NB Warning: Immediate deletion does not wait for confirmation that the running resource has been terminated. The resource may continue to run on the cluster indefinitely. + rm /tmp/tmp.2YXWSKHEc3 /tmp/tmp.PClg0Dd2NB + return 0 + : + '[' -n '' ']' + '[' -n '' ']' + rm -rf /tmp/tmp.M9iiA1fJys + desc 'test passed' + set +o xtrace ----------------------------------------------------------------------------------- test passed ----------------------------------------------------------------------------------- + kubectl_bin delete --grace-period=0 --force=true namespace cross-site-sharded-replica-17201 ++ mktemp + local LAST_OUT=/tmp/tmp.YyCMKVwk1W ++ mktemp + local LAST_ERR=/tmp/tmp.CRY7azjKKJ + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl delete --grace-period=0 --force=true namespace cross-site-sharded-replica-17201