++ echo 'Log: /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2036/e2e-tests/logs/recover-no-primary.log'
Log: /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2036/e2e-tests/logs/recover-no-primary.log
++ '[' -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2036/e2e-tests/conf/cloud-secret.yml ']'
++ SKIP_BACKUPS_TO_AWS_GCP_AZURE=
++ oc get projects
++ kubectl get nodes
++ grep '^minikube'
+++ kubectl version -o json
+++ jq -r .serverVersion.gitVersion
+++ grep '\-eks\-'
grep: warning: stray \ before -
Warning: version difference between client (1.34) and server (1.30) exceeds the supported minor version skew of +/-1
++ '[' ']'
++ EKS=0
+++ kubectl version -o json
+++ jq -r .serverVersion.gitVersion
+++ grep gke
Warning: version difference between client (1.34) and server (1.30) exceeds the supported minor version skew of +/-1
++ '[' v1.30.14-gke.1267000 ']'
++ GKE=1
+++ kubectl version -o json
+++ jq -r '.serverVersion.major + "." + .serverVersion.minor'
+++ /usr/sbin/sed -r 's/[^0-9.]+//g'
Warning: version difference between client (1.34) and server (1.30) exceeds the supported minor version skew of +/-1
++ KUBE_VERSION=1.30
+ create_infra recover-no-primary-9198
+ local ns=recover-no-primary-9198
+ [[ 1 == 1 ]]
+ delete_crd
+ desc 'get and delete old CRDs and RBAC'
+ set +o xtrace


-----------------------------------------------------------------------------------
get and delete old CRDs and RBAC
-----------------------------------------------------------------------------------

+ kubectl_bin delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2036/deploy/crd.yaml --ignore-not-found --wait=false
++ mktemp
+ local LAST_OUT=/tmp/tmp.qAViaeCAWS
++ mktemp
+ local LAST_ERR=/tmp/tmp.Zr7eOtXpYk
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2036/deploy/crd.yaml --ignore-not-found --wait=false
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.qAViaeCAWS
customresourcedefinition.apiextensions.k8s.io "perconaservermongodbbackups.psmdb.percona.com" deleted
customresourcedefinition.apiextensions.k8s.io "perconaservermongodbrestores.psmdb.percona.com" deleted
customresourcedefinition.apiextensions.k8s.io "perconaservermongodbs.psmdb.percona.com" deleted
+ cat /tmp/tmp.Zr7eOtXpYk
+ rm /tmp/tmp.qAViaeCAWS /tmp/tmp.Zr7eOtXpYk
+ return 0
++ yq eval .metadata.name /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2036/deploy/crd.yaml
++ grep -v '\-\-\-'
grep: warning: stray \ before -
grep: warning: stray \ before -
+ for crd_name in $(yq eval '.metadata.name' "${src_dir}/deploy/crd.yaml" | grep -v '\-\-\-')
+ kubectl get perconaservermongodbbackups.psmdb.percona.com --all-namespaces -o wide
+ grep -v NAMESPACE
+ xargs -L 1 sh -xc 'kubectl patch perconaservermongodbbackups.psmdb.percona.com -n $0 $1 --type=merge -p "{\"metadata\":{\"finalizers\":[]}}"'
error: the server doesn't have a resource type "perconaservermongodbbackups"
+ kubectl patch perconaservermongodbbackups.psmdb.percona.com -n sh --type=merge -p '{"metadata":{"finalizers":[]}}'
error: the server doesn't have a resource type "perconaservermongodbbackups"
+ :
+ kubectl_bin wait --for=delete crd perconaservermongodbbackups.psmdb.percona.com
++ mktemp
+ local LAST_OUT=/tmp/tmp.JGwZRvSfyF
++ mktemp
+ local LAST_ERR=/tmp/tmp.52N0mPWQ17
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl wait --for=delete crd perconaservermongodbbackups.psmdb.percona.com
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.JGwZRvSfyF
+ cat /tmp/tmp.52N0mPWQ17
+ rm /tmp/tmp.JGwZRvSfyF /tmp/tmp.52N0mPWQ17
+ return 0
+ for crd_name in $(yq eval '.metadata.name' "${src_dir}/deploy/crd.yaml" | grep -v '\-\-\-')
+ kubectl get perconaservermongodbrestores.psmdb.percona.com --all-namespaces -o wide
+ grep -v NAMESPACE
+ xargs -L 1 sh -xc 'kubectl patch perconaservermongodbrestores.psmdb.percona.com -n $0 $1 --type=merge -p "{\"metadata\":{\"finalizers\":[]}}"'
error: the server doesn't have a resource type "perconaservermongodbrestores"
+ kubectl patch perconaservermongodbrestores.psmdb.percona.com -n sh --type=merge -p '{"metadata":{"finalizers":[]}}'
error: the server doesn't have a resource type "perconaservermongodbrestores"
+ :
+ kubectl_bin wait --for=delete crd perconaservermongodbrestores.psmdb.percona.com
++ mktemp
+ local LAST_OUT=/tmp/tmp.E4vpuQSlMT
++ mktemp
+ local LAST_ERR=/tmp/tmp.z5oL0dj6pA
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl wait --for=delete crd perconaservermongodbrestores.psmdb.percona.com
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.E4vpuQSlMT
+ cat /tmp/tmp.z5oL0dj6pA
+ rm /tmp/tmp.E4vpuQSlMT /tmp/tmp.z5oL0dj6pA
+ return 0
+ for crd_name in $(yq eval '.metadata.name' "${src_dir}/deploy/crd.yaml" | grep -v '\-\-\-')
+ kubectl get perconaservermongodbs.psmdb.percona.com --all-namespaces -o wide
+ grep -v NAMESPACE
+ xargs -L 1 sh -xc 'kubectl patch perconaservermongodbs.psmdb.percona.com -n $0 $1 --type=merge -p "{\"metadata\":{\"finalizers\":[]}}"'
No resources found
+ kubectl patch perconaservermongodbs.psmdb.percona.com -n sh --type=merge -p '{"metadata":{"finalizers":[]}}'
error: the server doesn't have a resource type "perconaservermongodbs"
+ :
+ kubectl_bin wait --for=delete crd perconaservermongodbs.psmdb.percona.com
++ mktemp
+ local LAST_OUT=/tmp/tmp.OFxVha9Yp5
++ mktemp
+ local LAST_ERR=/tmp/tmp.g8xdfg4orK
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl wait --for=delete crd perconaservermongodbs.psmdb.percona.com
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.OFxVha9Yp5
+ cat /tmp/tmp.g8xdfg4orK
+ rm /tmp/tmp.OFxVha9Yp5 /tmp/tmp.g8xdfg4orK
+ return 0
+ local rbac_yaml=rbac.yaml
+ '[' -n psmdb-operator ']'
+ rbac_yaml=cw-rbac.yaml
+ kubectl_bin delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2036/deploy/cw-rbac.yaml --ignore-not-found
++ mktemp
+ local LAST_OUT=/tmp/tmp.8tkecSelDX
++ mktemp
+ local LAST_ERR=/tmp/tmp.ugbRTDpIWV
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2036/deploy/cw-rbac.yaml --ignore-not-found
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.8tkecSelDX
clusterrole.rbac.authorization.k8s.io "percona-server-mongodb-operator" deleted
clusterrolebinding.rbac.authorization.k8s.io "service-account-percona-server-mongodb-operator" deleted
+ cat /tmp/tmp.ugbRTDpIWV
+ rm /tmp/tmp.8tkecSelDX /tmp/tmp.ugbRTDpIWV
+ return 0
+ check_crd_for_deletion PR-2036-6a5cff62
+ local git_tag=PR-2036-6a5cff62
++ curl -s https://raw.githubusercontent.com/percona/percona-server-mongodb-operator/PR-2036-6a5cff62/deploy/crd.yaml
++ yq eval .metadata.name
++ /usr/sbin/sed s/---//g
++ /usr/sbin/sed ':a;N;$!ba;s/\n/ /g'
+ for crd_name in $(curl -s https://raw.githubusercontent.com/percona/percona-server-mongodb-operator/${git_tag}/deploy/crd.yaml | yq eval '.metadata.name' | $sed 's/---//g' | $sed ':a;N;$!ba;s/\n/ /g')
++ kubectl_bin get crd/null -o 'jsonpath={.status.conditions[-1].type}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.0YiSJFVm7S
+++ mktemp
++ local LAST_ERR=/tmp/tmp.Mwlu7Z8S1i
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get crd/null -o 'jsonpath={.status.conditions[-1].type}'
++ exit_status=1
++ set -e
++ '[' 1 '!=' 0 -a -n 1 ']'
++ cat /tmp/tmp.0YiSJFVm7S
++ cat /tmp/tmp.Mwlu7Z8S1i
Error from server (NotFound): customresourcedefinitions.apiextensions.k8s.io "null" not found
++ sleep 0
++ for i in $(seq 0 2)
++ set +e
++ kubectl get crd/null -o 'jsonpath={.status.conditions[-1].type}'
++ exit_status=1
++ set -e
++ '[' 1 '!=' 0 -a -n 1 ']'
++ cat /tmp/tmp.0YiSJFVm7S
++ cat /tmp/tmp.Mwlu7Z8S1i
Error from server (NotFound): customresourcedefinitions.apiextensions.k8s.io "null" not found
++ sleep 4
++ for i in $(seq 0 2)
++ set +e
++ kubectl get crd/null -o 'jsonpath={.status.conditions[-1].type}'
++ exit_status=1
++ set -e
++ '[' 1 '!=' 0 -a -n 1 ']'
++ cat /tmp/tmp.0YiSJFVm7S
++ cat /tmp/tmp.Mwlu7Z8S1i
Error from server (NotFound): customresourcedefinitions.apiextensions.k8s.io "null" not found
++ sleep 8
++ cat /tmp/tmp.0YiSJFVm7S
++ cat /tmp/tmp.Mwlu7Z8S1i
Error from server (NotFound): customresourcedefinitions.apiextensions.k8s.io "null" not found
++ rm /tmp/tmp.0YiSJFVm7S /tmp/tmp.Mwlu7Z8S1i
++ return 1
+ [[ '' == \T\e\r\m\i\n\a\t\i\n\g ]]
+ '[' -n psmdb-operator ']'
+ create_namespace psmdb-operator
+ local namespace=psmdb-operator
+ local skip_clean_namespace=
+ [[ 1 == 1 ]]
+ [[ -z '' ]]
+ destroy_chaos_mesh
++ helm list --all-namespaces --filter chaos-mesh
++ tail -n1
++ awk '-F ' '{print $2}'
++ sed s/NAMESPACE//
+ local chaos_mesh_ns=
+ desc 'destroy chaos-mesh'
+ set +o xtrace


-----------------------------------------------------------------------------------
destroy chaos-mesh
-----------------------------------------------------------------------------------

+ '[' -n '' ']'
++ kubectl get MutatingWebhookConfiguration
++ grep chaos-mesh
++ awk '{print $1}'
+ timeout 30 kubectl delete MutatingWebhookConfiguration
error: resource(s) were provided, but no name was specified
+ :
++ kubectl get ValidatingWebhookConfiguration
++ grep chaos-mesh
++ awk '{print $1}'
+ timeout 30 kubectl delete ValidatingWebhookConfiguration
error: resource(s) were provided, but no name was specified
+ :
++ kubectl get ValidatingWebhookConfiguration
++ grep validate-auth
++ awk '{print $1}'
+ timeout 30 kubectl delete ValidatingWebhookConfiguration
error: resource(s) were provided, but no name was specified
+ :
++ kubectl api-resources
++ grep chaos-mesh
++ awk '{print $1}'
++ kubectl get crd
++ grep chaos-mesh.org
++ awk '{print $1}'
+ timeout 30 kubectl delete crd
error: resource(s) were provided, but no name was specified
+ :
++ kubectl get clusterrolebinding
++ grep chaos-mesh
++ awk '{print $1}'
+ timeout 30 kubectl delete clusterrolebinding
error: resource(s) were provided, but no name was specified
+ :
++ kubectl get clusterrole
++ grep chaos-mesh
++ awk '{print $1}'
+ timeout 30 kubectl delete clusterrole
error: resource(s) were provided, but no name was specified
+ :
+ desc 'cleaned up all old namespaces'
+ set +o xtrace


-----------------------------------------------------------------------------------
cleaned up all old namespaces
-----------------------------------------------------------------------------------

+ kubectl_bin get ns
+ egrep -v '^kube-|^default|Terminating|psmdb-operator|openshift|^gke-|^gmp-|^NAME'
+ awk '{print$1}'
+ '[' -n '' ']'
+ xargs kubectl delete ns
+ desc 'cleaned up old namespaces psmdb-operator'
+ set +o xtrace


-----------------------------------------------------------------------------------
++ mktemp
cleaned up old namespaces psmdb-operator
-----------------------------------------------------------------------------------

+ kubectl_bin delete namespace psmdb-operator --ignore-not-found
++ mktemp
egrep: warning: egrep is obsolescent; using grep -E
+ local LAST_OUT=/tmp/tmp.eVz68Ahrh4
++ mktemp
+ local LAST_OUT=/tmp/tmp.YMw05aQmPJ
++ mktemp
+ local LAST_ERR=/tmp/tmp.kvxZEzLHVF
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ local LAST_ERR=/tmp/tmp.mQUwH0XzZ6
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl get ns
+ for i in $(seq 0 2)
+ set +e
+ kubectl delete namespace psmdb-operator --ignore-not-found
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.eVz68Ahrh4
+ cat /tmp/tmp.kvxZEzLHVF
+ rm /tmp/tmp.eVz68Ahrh4 /tmp/tmp.kvxZEzLHVF
+ return 0
namespace "recover-no-primary-26208" deleted
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.YMw05aQmPJ
namespace "psmdb-operator" deleted
+ cat /tmp/tmp.mQUwH0XzZ6
+ rm /tmp/tmp.YMw05aQmPJ /tmp/tmp.mQUwH0XzZ6
+ return 0
+ kubectl_bin wait --for=delete namespace psmdb-operator
++ mktemp
+ local LAST_OUT=/tmp/tmp.JR72RQwnTl
++ mktemp
+ local LAST_ERR=/tmp/tmp.xgruIQCcFd
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl wait --for=delete namespace psmdb-operator
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.JR72RQwnTl
+ cat /tmp/tmp.xgruIQCcFd
+ rm /tmp/tmp.JR72RQwnTl /tmp/tmp.xgruIQCcFd
+ return 0
+ desc 'create namespace psmdb-operator'
+ set +o xtrace


-----------------------------------------------------------------------------------
create namespace psmdb-operator
-----------------------------------------------------------------------------------

+ kubectl_bin create namespace psmdb-operator
++ mktemp
+ local LAST_OUT=/tmp/tmp.XXCiZGIzbO
++ mktemp
+ local LAST_ERR=/tmp/tmp.Lfptj2NOTm
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl create namespace psmdb-operator
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.XXCiZGIzbO
namespace/psmdb-operator created
+ cat /tmp/tmp.Lfptj2NOTm
+ rm /tmp/tmp.XXCiZGIzbO /tmp/tmp.Lfptj2NOTm
+ return 0
+ set_kube_ctx psmdb-operator
+ local namespace=psmdb-operator
++ kubectl_bin config current-context
+++ mktemp
++ local LAST_OUT=/tmp/tmp.tJJ6u4OTNK
+++ mktemp
++ local LAST_ERR=/tmp/tmp.vTOCFviMPW
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl config current-context
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.tJJ6u4OTNK
++ cat /tmp/tmp.vTOCFviMPW
++ rm /tmp/tmp.tJJ6u4OTNK /tmp/tmp.vTOCFviMPW
++ return 0
+ kubectl_bin config set-context gke_cloud-dev-112233_us-central1-a_jen-psmdb-2036-6a5cff62-2-cluster4 --namespace=psmdb-operator
++ mktemp
+ local LAST_OUT=/tmp/tmp.HAdG0UpCmn
++ mktemp
+ local LAST_ERR=/tmp/tmp.AsTAiiJcze
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl config set-context gke_cloud-dev-112233_us-central1-a_jen-psmdb-2036-6a5cff62-2-cluster4 --namespace=psmdb-operator
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.HAdG0UpCmn
Context "gke_cloud-dev-112233_us-central1-a_jen-psmdb-2036-6a5cff62-2-cluster4" modified.
+ cat /tmp/tmp.AsTAiiJcze
+ rm /tmp/tmp.HAdG0UpCmn /tmp/tmp.AsTAiiJcze
+ return 0
+ deploy_operator
+ desc 'start PSMDB operator'
+ set +o xtrace


-----------------------------------------------------------------------------------
start PSMDB operator
-----------------------------------------------------------------------------------

+ local cr_file
+ '[' -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2036/e2e-tests/recover-no-primary/conf/crd.yaml ']'
+ cr_file=/mnt/jenkins/workspace/cloud-psmdb-operator_PR-2036/deploy/crd.yaml
+ kubectl_bin apply --server-side --force-conflicts -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2036/deploy/crd.yaml
++ mktemp
+ local LAST_OUT=/tmp/tmp.5zuWVdT7xD
++ mktemp
+ local LAST_ERR=/tmp/tmp.GLh9bNkZW9
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl apply --server-side --force-conflicts -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2036/deploy/crd.yaml
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.5zuWVdT7xD
customresourcedefinition.apiextensions.k8s.io/perconaservermongodbbackups.psmdb.percona.com serverside-applied
customresourcedefinition.apiextensions.k8s.io/perconaservermongodbrestores.psmdb.percona.com serverside-applied
customresourcedefinition.apiextensions.k8s.io/perconaservermongodbs.psmdb.percona.com serverside-applied
+ cat /tmp/tmp.GLh9bNkZW9
+ rm /tmp/tmp.5zuWVdT7xD /tmp/tmp.GLh9bNkZW9
+ return 0
+ '[' -n psmdb-operator ']'
+ apply_rbac cw-rbac
+ local operator_namespace=psmdb-operator
+ local rbac=cw-rbac
+ cat /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2036/deploy/cw-rbac.yaml
+ sed -e 's^namespace: .*^namespace: psmdb-operator^'
+ kubectl_bin apply -n psmdb-operator -f -
++ mktemp
+ local LAST_OUT=/tmp/tmp.8Daz9cUiF9
++ mktemp
+ local LAST_ERR=/tmp/tmp.a4bRIyEQzd
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl apply -n psmdb-operator -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.8Daz9cUiF9
clusterrole.rbac.authorization.k8s.io/percona-server-mongodb-operator created
serviceaccount/percona-server-mongodb-operator created
clusterrolebinding.rbac.authorization.k8s.io/service-account-percona-server-mongodb-operator created
+ cat /tmp/tmp.a4bRIyEQzd
+ rm /tmp/tmp.8Daz9cUiF9 /tmp/tmp.a4bRIyEQzd
+ return 0
+ yq eval '
			(.spec.template.spec.containers[].image = "perconalab/percona-server-mongodb-operator:PR-2036-6a5cff62") |
			((.. | select(.[] == "DISABLE_TELEMETRY")) |= .value="true") |
			((.. | select(.[] == "LOG_LEVEL")) |= .value="DEBUG")' /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2036/deploy/cw-operator.yaml
+ kubectl_bin apply -f -
++ mktemp
+ local LAST_OUT=/tmp/tmp.9CobiXuY3s
++ mktemp
+ local LAST_ERR=/tmp/tmp.22QNymKnBC
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl apply -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.9CobiXuY3s
deployment.apps/percona-server-mongodb-operator created
+ cat /tmp/tmp.22QNymKnBC
+ rm /tmp/tmp.9CobiXuY3s /tmp/tmp.22QNymKnBC
+ return 0
+ sleep 2
++ get_operator_pod
++ kubectl_bin get pods --selector=name=percona-server-mongodb-operator -o 'jsonpath={.items[].metadata.name}' -n psmdb-operator
+++ mktemp
++ local LAST_OUT=/tmp/tmp.LbDMWT5yNr
+++ mktemp
++ local LAST_ERR=/tmp/tmp.T30Cqv8vVl
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=percona-server-mongodb-operator -o 'jsonpath={.items[].metadata.name}' -n psmdb-operator
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.LbDMWT5yNr
++ cat /tmp/tmp.T30Cqv8vVl
++ rm /tmp/tmp.LbDMWT5yNr /tmp/tmp.T30Cqv8vVl
++ return 0
+ wait_pod percona-server-mongodb-operator-dd8c7dbb4-56jcq
+ local pod=percona-server-mongodb-operator-dd8c7dbb4-56jcq
+ set +o xtrace
waiting for pod/percona-server-mongodb-operator-dd8c7dbb4-56jcq to be ready.OK
+ echo 'Print operator info from log'
Print operator info from log
+ grep 'Manager starting up'
++ get_operator_pod
++ kubectl_bin get pods --selector=name=percona-server-mongodb-operator -o 'jsonpath={.items[].metadata.name}' -n psmdb-operator
+++ mktemp
++ local LAST_OUT=/tmp/tmp.wu8MzZSbE5
+++ mktemp
++ local LAST_ERR=/tmp/tmp.YKykIOztg7
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=percona-server-mongodb-operator -o 'jsonpath={.items[].metadata.name}' -n psmdb-operator
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.wu8MzZSbE5
++ cat /tmp/tmp.YKykIOztg7
++ rm /tmp/tmp.wu8MzZSbE5 /tmp/tmp.YKykIOztg7
++ return 0
+ kubectl_bin logs percona-server-mongodb-operator-dd8c7dbb4-56jcq
++ mktemp
+ local LAST_OUT=/tmp/tmp.tzBXRLMwZY
++ mktemp
+ local LAST_ERR=/tmp/tmp.wWAhmhxQ2i
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl logs percona-server-mongodb-operator-dd8c7dbb4-56jcq
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.tzBXRLMwZY
+ cat /tmp/tmp.wWAhmhxQ2i
+ rm /tmp/tmp.tzBXRLMwZY /tmp/tmp.wWAhmhxQ2i
+ return 0
2025-09-24T18:00:39.396Z	INFO	setup	Manager starting up	{"gitCommit": "6a5cff6212b0846db032e80907cac572ed8d3194", "gitBranch": "PR-2036-6a5cff62", "buildTime": "", "goVersion": "go1.25.1", "os": "linux", "arch": "amd64"}
+ create_namespace recover-no-primary-9198
+ local namespace=recover-no-primary-9198
+ local skip_clean_namespace=
+ [[ 1 == 1 ]]
+ [[ -z '' ]]
+ destroy_chaos_mesh
++ helm list --all-namespaces --filter chaos-mesh
++ tail -n1
++ awk '-F ' '{print $2}'
++ sed s/NAMESPACE//
+ local chaos_mesh_ns=
+ desc 'destroy chaos-mesh'
+ set +o xtrace


-----------------------------------------------------------------------------------
destroy chaos-mesh
-----------------------------------------------------------------------------------

+ '[' -n '' ']'
++ kubectl get MutatingWebhookConfiguration
++ grep chaos-mesh
++ awk '{print $1}'
+ timeout 30 kubectl delete MutatingWebhookConfiguration
error: resource(s) were provided, but no name was specified
+ :
++ kubectl get ValidatingWebhookConfiguration
++ grep chaos-mesh
++ awk '{print $1}'
+ timeout 30 kubectl delete ValidatingWebhookConfiguration
error: resource(s) were provided, but no name was specified
+ :
++ kubectl get ValidatingWebhookConfiguration
++ grep validate-auth
++ awk '{print $1}'
+ timeout 30 kubectl delete ValidatingWebhookConfiguration
error: resource(s) were provided, but no name was specified
+ :
++ kubectl api-resources
++ grep chaos-mesh
++ awk '{print $1}'
++ kubectl get crd
++ grep chaos-mesh.org
++ awk '{print $1}'
+ timeout 30 kubectl delete crd
error: resource(s) were provided, but no name was specified
+ :
++ kubectl get clusterrolebinding
++ grep chaos-mesh
++ awk '{print $1}'
+ timeout 30 kubectl delete clusterrolebinding
error: resource(s) were provided, but no name was specified
+ :
++ kubectl get clusterrole
++ grep chaos-mesh
++ awk '{print $1}'
+ timeout 30 kubectl delete clusterrole
error: resource(s) were provided, but no name was specified
+ :
+ desc 'cleaned up all old namespaces'
+ set +o xtrace


-----------------------------------------------------------------------------------
cleaned up all old namespaces
-----------------------------------------------------------------------------------

+ kubectl_bin get ns
+ egrep -v '^kube-|^default|Terminating|psmdb-operator|openshift|^gke-|^gmp-|^NAME'
+ awk '{print$1}'
+ '[' -n '' ']'
+ desc 'cleaned up old namespaces recover-no-primary-9198'
+ set +o xtrace
+ xargs kubectl delete ns


-----------------------------------------------------------------------------------
cleaned up old namespaces recover-no-primary-9198
-----------------------------------------------------------------------------------

++ mktemp
+ kubectl_bin delete namespace recover-no-primary-9198 --ignore-not-found
++ mktemp
+ local LAST_OUT=/tmp/tmp.XuujnjP2Cr
egrep: warning: egrep is obsolescent; using grep -E
++ mktemp
+ local LAST_ERR=/tmp/tmp.TklL0xuXhF
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ local LAST_OUT=/tmp/tmp.DlH1sEaww3
++ mktemp
+ for i in $(seq 0 2)
+ set +e
+ kubectl get ns
+ local LAST_ERR=/tmp/tmp.n4uosbhBz9
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl delete namespace recover-no-primary-9198 --ignore-not-found
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.DlH1sEaww3
+ cat /tmp/tmp.n4uosbhBz9
+ rm /tmp/tmp.DlH1sEaww3 /tmp/tmp.n4uosbhBz9
+ return 0
+ kubectl_bin wait --for=delete namespace recover-no-primary-9198
++ mktemp
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.XuujnjP2Cr
+ local LAST_OUT=/tmp/tmp.B2Q73FMtrv
++ mktemp
+ cat /tmp/tmp.TklL0xuXhF
+ local LAST_ERR=/tmp/tmp.IPFVAntvEz
+ local exit_status=0
+ local timeout=4
+ rm /tmp/tmp.XuujnjP2Cr /tmp/tmp.TklL0xuXhF
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl wait --for=delete namespace recover-no-primary-9198
+ return 0
error: resource(s) were provided, but no name was specified
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.B2Q73FMtrv
+ cat /tmp/tmp.IPFVAntvEz
+ rm /tmp/tmp.B2Q73FMtrv /tmp/tmp.IPFVAntvEz
+ return 0
+ desc 'create namespace recover-no-primary-9198'
+ set +o xtrace


-----------------------------------------------------------------------------------
create namespace recover-no-primary-9198
-----------------------------------------------------------------------------------

+ kubectl_bin create namespace recover-no-primary-9198
++ mktemp
+ local LAST_OUT=/tmp/tmp.SpjmrrY5TX
++ mktemp
+ local LAST_ERR=/tmp/tmp.R6CZwFsCh9
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl create namespace recover-no-primary-9198
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.SpjmrrY5TX
namespace/recover-no-primary-9198 created
+ cat /tmp/tmp.R6CZwFsCh9
+ rm /tmp/tmp.SpjmrrY5TX /tmp/tmp.R6CZwFsCh9
+ return 0
+ set_kube_ctx recover-no-primary-9198
+ local namespace=recover-no-primary-9198
++ kubectl_bin config current-context
+++ mktemp
++ local LAST_OUT=/tmp/tmp.jwOzgE44A9
+++ mktemp
++ local LAST_ERR=/tmp/tmp.rJRix9k7Gy
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl config current-context
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.jwOzgE44A9
++ cat /tmp/tmp.rJRix9k7Gy
++ rm /tmp/tmp.jwOzgE44A9 /tmp/tmp.rJRix9k7Gy
++ return 0
+ kubectl_bin config set-context gke_cloud-dev-112233_us-central1-a_jen-psmdb-2036-6a5cff62-2-cluster4 --namespace=recover-no-primary-9198
++ mktemp
+ local LAST_OUT=/tmp/tmp.hbrnCYaCGJ
++ mktemp
+ local LAST_ERR=/tmp/tmp.0rlsbh2YCo
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl config set-context gke_cloud-dev-112233_us-central1-a_jen-psmdb-2036-6a5cff62-2-cluster4 --namespace=recover-no-primary-9198
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.hbrnCYaCGJ
Context "gke_cloud-dev-112233_us-central1-a_jen-psmdb-2036-6a5cff62-2-cluster4" modified.
+ cat /tmp/tmp.0rlsbh2YCo
+ rm /tmp/tmp.hbrnCYaCGJ /tmp/tmp.0rlsbh2YCo
+ return 0
+ cluster=some-name
+ kubectl_bin apply -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2036/e2e-tests/conf/secrets_with_tls.yml -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2036/e2e-tests/conf/client.yml
++ mktemp
+ local LAST_OUT=/tmp/tmp.SKAda9rNvj
++ mktemp
+ local LAST_ERR=/tmp/tmp.IYLYWoqnZN
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl apply -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2036/e2e-tests/conf/secrets_with_tls.yml -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2036/e2e-tests/conf/client.yml
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.SKAda9rNvj
secret/some-users created
secret/some-name-ssl created
secret/some-name-ssl-internal created
deployment.apps/psmdb-client created
+ cat /tmp/tmp.IYLYWoqnZN
+ rm /tmp/tmp.SKAda9rNvj /tmp/tmp.IYLYWoqnZN
+ return 0
+ desc 'testing unexposed single replset cluster'
+ set +o xtrace


-----------------------------------------------------------------------------------
testing unexposed single replset cluster
-----------------------------------------------------------------------------------

+ test_single_replset
+ apply_cluster /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2036/e2e-tests/recover-no-primary/conf/some-name.yml
+ '[' -z '' ']'
+ cat_config /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2036/e2e-tests/recover-no-primary/conf/some-name.yml
+ kubectl_bin apply -f -
+ cat /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2036/e2e-tests/recover-no-primary/conf/some-name.yml
++ mktemp
+ yq eval '(.spec | select(.image == null)).image = "perconalab/percona-server-mongodb-operator:main-mongod7.0"'
+ yq eval '(.spec | select(has("pmm"))).pmm.image = "percona/pmm-client:2.44.1-1"'
+ yq eval '(.spec | select(has("initImage"))).initImage = "perconalab/percona-server-mongodb-operator:PR-2036-6a5cff62"'
+ yq eval '(.spec | select(has("backup"))).backup.image = "perconalab/percona-server-mongodb-operator:main-backup"'
+ yq eval '.spec.upgradeOptions.apply="Never"'
+ local LAST_OUT=/tmp/tmp.exGGVFbcoD
++ mktemp
+ local LAST_ERR=/tmp/tmp.RwHUSHJjRZ
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl apply -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.exGGVFbcoD
perconaservermongodb.psmdb.percona.com/some-name created
+ cat /tmp/tmp.RwHUSHJjRZ
+ rm /tmp/tmp.exGGVFbcoD /tmp/tmp.RwHUSHJjRZ
+ return 0
+ wait_for_running some-name-rs0 3
+ local name=some-name-rs0
+ let last_pod=2
+ local check_cluster_readyness=true
+ set_debug
+ [[ 1 == 1 ]]
+ set -o xtrace
+ local rs_name=rs0
+ local cluster_name=some-name
++ seq 0 2
+ for i in $(seq 0 $last_pod)
+ [[ 0 -eq 2 ]]
+ wait_pod some-name-rs0-0
+ local pod=some-name-rs0-0
+ set +o xtrace
waiting for pod/some-name-rs0-0 to be ready..............OK
+ for i in $(seq 0 $last_pod)
+ [[ 1 -eq 2 ]]
+ wait_pod some-name-rs0-1
+ local pod=some-name-rs0-1
+ set +o xtrace
waiting for pod/some-name-rs0-1 to be ready............OK
+ for i in $(seq 0 $last_pod)
+ [[ 2 -eq 2 ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].arbiter.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.jJDXDJKumx
+++ mktemp
++ local LAST_ERR=/tmp/tmp.IhmqGzmsks
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].arbiter.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.jJDXDJKumx
++ cat /tmp/tmp.IhmqGzmsks
++ rm /tmp/tmp.jJDXDJKumx /tmp/tmp.IhmqGzmsks
++ return 0
+ [[ '' == \t\r\u\e ]]
+ wait_pod some-name-rs0-2
+ local pod=some-name-rs0-2
+ set +o xtrace
waiting for pod/some-name-rs0-2 to be ready..............OK
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].nonvoting.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.zXbmGLQL7I
+++ mktemp
++ local LAST_ERR=/tmp/tmp.i5p01gb3z1
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].nonvoting.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.zXbmGLQL7I
++ cat /tmp/tmp.i5p01gb3z1
++ rm /tmp/tmp.zXbmGLQL7I /tmp/tmp.i5p01gb3z1
++ return 0
+ [[ '' == \t\r\u\e ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].hidden.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.PnfZSL0m0N
+++ mktemp
++ local LAST_ERR=/tmp/tmp.ky6DrJKgrb
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].hidden.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.PnfZSL0m0N
++ cat /tmp/tmp.ky6DrJKgrb
++ rm /tmp/tmp.PnfZSL0m0N /tmp/tmp.ky6DrJKgrb
++ return 0
+ [[ '' == \t\r\u\e ]]
+ sleep 10
+ [[ true == \t\r\u\e ]]
+ set +x
Waiting for cluster readyness
+ wait_cluster_consistency some-name
+ local cluster_name=some-name
+ local wait_time=32
+ retry=0
+ sleep 7
+ echo -n 'waiting for cluster readyness'
waiting for cluster readyness++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.NWT1ARHlwa
+++ mktemp
++ local LAST_ERR=/tmp/tmp.newV2lqmv0
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.NWT1ARHlwa
++ cat /tmp/tmp.newV2lqmv0
++ rm /tmp/tmp.NWT1ARHlwa /tmp/tmp.newV2lqmv0
++ return 0
+ [[ ready == \r\e\a\d\y ]]
+ echo

+ desc 'kill pods abruptly and wait for recovery'
+ set +o xtrace


-----------------------------------------------------------------------------------
kill pods abruptly and wait for recovery
-----------------------------------------------------------------------------------

++ seq 0 2
+ for i in $(seq 0 2)
+ kubectl_bin exec some-name-rs0-0 -c mongod -- kill 1
++ mktemp
+ local LAST_OUT=/tmp/tmp.higcdF98jl
++ mktemp
+ local LAST_ERR=/tmp/tmp.VEifQE24Ba
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec some-name-rs0-0 -c mongod -- kill 1
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.higcdF98jl
+ cat /tmp/tmp.VEifQE24Ba
+ rm /tmp/tmp.higcdF98jl /tmp/tmp.VEifQE24Ba
+ return 0
+ for i in $(seq 0 2)
+ kubectl_bin exec some-name-rs0-1 -c mongod -- kill 1
++ mktemp
+ local LAST_OUT=/tmp/tmp.UbzPibm4GT
++ mktemp
+ local LAST_ERR=/tmp/tmp.Xtxx1wJnyh
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec some-name-rs0-1 -c mongod -- kill 1
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.UbzPibm4GT
+ cat /tmp/tmp.Xtxx1wJnyh
+ rm /tmp/tmp.UbzPibm4GT /tmp/tmp.Xtxx1wJnyh
+ return 0
+ for i in $(seq 0 2)
+ kubectl_bin exec some-name-rs0-2 -c mongod -- kill 1
++ mktemp
+ local LAST_OUT=/tmp/tmp.G93AroN9oe
++ mktemp
+ local LAST_ERR=/tmp/tmp.KurorvYXJU
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec some-name-rs0-2 -c mongod -- kill 1
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.G93AroN9oe
+ cat /tmp/tmp.KurorvYXJU
+ rm /tmp/tmp.G93AroN9oe /tmp/tmp.KurorvYXJU
+ return 0
+ sleep 10
+ wait_for_running some-name-rs0 3
+ local name=some-name-rs0
+ let last_pod=2
+ local check_cluster_readyness=true
+ set_debug
+ [[ 1 == 1 ]]
+ set -o xtrace
+ local rs_name=rs0
+ local cluster_name=some-name
++ seq 0 2
+ for i in $(seq 0 $last_pod)
+ [[ 0 -eq 2 ]]
+ wait_pod some-name-rs0-0
+ local pod=some-name-rs0-0
+ set +o xtrace
waiting for pod/some-name-rs0-0 to be ready.OK
+ for i in $(seq 0 $last_pod)
+ [[ 1 -eq 2 ]]
+ wait_pod some-name-rs0-1
+ local pod=some-name-rs0-1
+ set +o xtrace
waiting for pod/some-name-rs0-1 to be ready.OK
+ for i in $(seq 0 $last_pod)
+ [[ 2 -eq 2 ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].arbiter.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.RHmhykYWYW
+++ mktemp
++ local LAST_ERR=/tmp/tmp.kH1pnHYmFR
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].arbiter.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.RHmhykYWYW
++ cat /tmp/tmp.kH1pnHYmFR
++ rm /tmp/tmp.RHmhykYWYW /tmp/tmp.kH1pnHYmFR
++ return 0
+ [[ '' == \t\r\u\e ]]
+ wait_pod some-name-rs0-2
+ local pod=some-name-rs0-2
+ set +o xtrace
waiting for pod/some-name-rs0-2 to be ready.OK
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].nonvoting.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.dz5AqFMSXp
+++ mktemp
++ local LAST_ERR=/tmp/tmp.DNJzVv6MOH
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].nonvoting.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.dz5AqFMSXp
++ cat /tmp/tmp.DNJzVv6MOH
++ rm /tmp/tmp.dz5AqFMSXp /tmp/tmp.DNJzVv6MOH
++ return 0
+ [[ '' == \t\r\u\e ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].hidden.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.q9B0NtNvv8
+++ mktemp
++ local LAST_ERR=/tmp/tmp.ACMEnafEKP
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].hidden.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.q9B0NtNvv8
++ cat /tmp/tmp.ACMEnafEKP
++ rm /tmp/tmp.q9B0NtNvv8 /tmp/tmp.ACMEnafEKP
++ return 0
+ [[ '' == \t\r\u\e ]]
+ sleep 10
+ [[ true == \t\r\u\e ]]
+ set +x
Waiting for cluster readyness...
+ wait_cluster_consistency some-name
+ local cluster_name=some-name
+ local wait_time=32
+ retry=0
+ sleep 7
+ echo -n 'waiting for cluster readyness'
waiting for cluster readyness++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.UCBzhsS696
+++ mktemp
++ local LAST_ERR=/tmp/tmp.GYUArs82Fk
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.UCBzhsS696
++ cat /tmp/tmp.GYUArs82Fk
++ rm /tmp/tmp.UCBzhsS696 /tmp/tmp.GYUArs82Fk
++ return 0
+ [[ ready == \r\e\a\d\y ]]
+ echo

+ desc 'delete statefulset and wait for recovery'
+ set +o xtrace


-----------------------------------------------------------------------------------
delete statefulset and wait for recovery
-----------------------------------------------------------------------------------

+ kubectl delete sts some-name-rs0
statefulset.apps "some-name-rs0" deleted from recover-no-primary-9198 namespace
+ sleep 5
+ wait_for_running some-name-rs0 3
+ local name=some-name-rs0
+ let last_pod=2
+ local check_cluster_readyness=true
+ set_debug
+ [[ 1 == 1 ]]
+ set -o xtrace
+ local rs_name=rs0
+ local cluster_name=some-name
++ seq 0 2
+ for i in $(seq 0 $last_pod)
+ [[ 0 -eq 2 ]]
+ wait_pod some-name-rs0-0
+ local pod=some-name-rs0-0
+ set +o xtrace
waiting for pod/some-name-rs0-0 to be ready.OK
+ for i in $(seq 0 $last_pod)
+ [[ 1 -eq 2 ]]
+ wait_pod some-name-rs0-1
+ local pod=some-name-rs0-1
+ set +o xtrace
waiting for pod/some-name-rs0-1 to be ready.OK
+ for i in $(seq 0 $last_pod)
+ [[ 2 -eq 2 ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].arbiter.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.NKLK3smV7Z
+++ mktemp
++ local LAST_ERR=/tmp/tmp.PjIcCkXWOh
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].arbiter.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.NKLK3smV7Z
++ cat /tmp/tmp.PjIcCkXWOh
++ rm /tmp/tmp.NKLK3smV7Z /tmp/tmp.PjIcCkXWOh
++ return 0
+ [[ '' == \t\r\u\e ]]
+ wait_pod some-name-rs0-2
+ local pod=some-name-rs0-2
+ set +o xtrace
waiting for pod/some-name-rs0-2 to be ready.OK
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].nonvoting.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.2t74Lnhb8f
+++ mktemp
++ local LAST_ERR=/tmp/tmp.LqWsJ3Duow
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].nonvoting.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.2t74Lnhb8f
++ cat /tmp/tmp.LqWsJ3Duow
++ rm /tmp/tmp.2t74Lnhb8f /tmp/tmp.LqWsJ3Duow
++ return 0
+ [[ '' == \t\r\u\e ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].hidden.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.z4SDfq3XDa
+++ mktemp
++ local LAST_ERR=/tmp/tmp.Kbh1QkHLGl
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].hidden.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.z4SDfq3XDa
++ cat /tmp/tmp.Kbh1QkHLGl
++ rm /tmp/tmp.z4SDfq3XDa /tmp/tmp.Kbh1QkHLGl
++ return 0
+ [[ '' == \t\r\u\e ]]
+ sleep 10
+ [[ true == \t\r\u\e ]]
+ set +x
Waiting for cluster readyness
+ wait_cluster_consistency some-name
+ local cluster_name=some-name
+ local wait_time=32
+ retry=0
+ sleep 7
+ echo -n 'waiting for cluster readyness'
waiting for cluster readyness++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.11dHry4uPg
+++ mktemp
++ local LAST_ERR=/tmp/tmp.agmFQmkabI
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.11dHry4uPg
++ cat /tmp/tmp.agmFQmkabI
++ rm /tmp/tmp.11dHry4uPg /tmp/tmp.agmFQmkabI
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 1 -ge 32 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.qnwoTkhCZH
+++ mktemp
++ local LAST_ERR=/tmp/tmp.dMrBnDBFRx
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.qnwoTkhCZH
++ cat /tmp/tmp.dMrBnDBFRx
++ rm /tmp/tmp.qnwoTkhCZH /tmp/tmp.dMrBnDBFRx
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 2 -ge 32 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.MDol8dWFN0
+++ mktemp
++ local LAST_ERR=/tmp/tmp.20SjsUV69Q
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.MDol8dWFN0
++ cat /tmp/tmp.20SjsUV69Q
++ rm /tmp/tmp.MDol8dWFN0 /tmp/tmp.20SjsUV69Q
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 3 -ge 32 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.WX5Y8sdtp6
+++ mktemp
++ local LAST_ERR=/tmp/tmp.Ju2oSM2UvB
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.WX5Y8sdtp6
++ cat /tmp/tmp.Ju2oSM2UvB
++ rm /tmp/tmp.WX5Y8sdtp6 /tmp/tmp.Ju2oSM2UvB
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 4 -ge 32 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.yC2YgE1ram
+++ mktemp
++ local LAST_ERR=/tmp/tmp.FliyxynFZi
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.yC2YgE1ram
++ cat /tmp/tmp.FliyxynFZi
++ rm /tmp/tmp.yC2YgE1ram /tmp/tmp.FliyxynFZi
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 5 -ge 32 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.zt8barkIOI
+++ mktemp
++ local LAST_ERR=/tmp/tmp.3ft5OSEEug
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.zt8barkIOI
++ cat /tmp/tmp.3ft5OSEEug
++ rm /tmp/tmp.zt8barkIOI /tmp/tmp.3ft5OSEEug
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 6 -ge 32 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.VIboTOgCEN
+++ mktemp
++ local LAST_ERR=/tmp/tmp.zP6w0n3qwM
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.VIboTOgCEN
++ cat /tmp/tmp.zP6w0n3qwM
++ rm /tmp/tmp.VIboTOgCEN /tmp/tmp.zP6w0n3qwM
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 7 -ge 32 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.VwJMV72xqp
+++ mktemp
++ local LAST_ERR=/tmp/tmp.46cd6MZbXX
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.VwJMV72xqp
++ cat /tmp/tmp.46cd6MZbXX
++ rm /tmp/tmp.VwJMV72xqp /tmp/tmp.46cd6MZbXX
++ return 0
+ [[ ready == \r\e\a\d\y ]]
+ echo

+ desc 'delete cluster and recreate'
+ set +o xtrace


-----------------------------------------------------------------------------------
delete cluster and recreate
-----------------------------------------------------------------------------------

+ kubectl delete psmdb some-name
perconaservermongodb.psmdb.percona.com "some-name" deleted from recover-no-primary-9198 namespace
+ set +e
+ echo -n 'waiting for psmdb/some-name to be deleted'
waiting for psmdb/some-name to be deleted+ kubectl get psmdb some-name
Error from server (NotFound): perconaservermongodbs.psmdb.percona.com "some-name" not found
+ set -e
+ apply_cluster /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2036/e2e-tests/recover-no-primary/conf/some-name.yml
+ '[' -z '' ']'
+ cat_config /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2036/e2e-tests/recover-no-primary/conf/some-name.yml
+ kubectl_bin apply -f -
+ cat /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2036/e2e-tests/recover-no-primary/conf/some-name.yml
++ mktemp
+ yq eval '(.spec | select(.image == null)).image = "perconalab/percona-server-mongodb-operator:main-mongod7.0"'
+ yq eval '(.spec | select(has("pmm"))).pmm.image = "percona/pmm-client:2.44.1-1"'
+ yq eval '(.spec | select(has("initImage"))).initImage = "perconalab/percona-server-mongodb-operator:PR-2036-6a5cff62"'
+ yq eval '(.spec | select(has("backup"))).backup.image = "perconalab/percona-server-mongodb-operator:main-backup"'
+ yq eval '.spec.upgradeOptions.apply="Never"'
+ local LAST_OUT=/tmp/tmp.AenUYvGcrC
++ mktemp
+ local LAST_ERR=/tmp/tmp.lgAPuqGSHt
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl apply -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.AenUYvGcrC
perconaservermongodb.psmdb.percona.com/some-name created
+ cat /tmp/tmp.lgAPuqGSHt
+ rm /tmp/tmp.AenUYvGcrC /tmp/tmp.lgAPuqGSHt
+ return 0
+ wait_for_running some-name-rs0 3
+ local name=some-name-rs0
+ let last_pod=2
+ local check_cluster_readyness=true
+ set_debug
+ [[ 1 == 1 ]]
+ set -o xtrace
+ local rs_name=rs0
+ local cluster_name=some-name
++ seq 0 2
+ for i in $(seq 0 $last_pod)
+ [[ 0 -eq 2 ]]
+ wait_pod some-name-rs0-0
+ local pod=some-name-rs0-0
+ set +o xtrace
waiting for pod/some-name-rs0-0 to be ready.OK
+ for i in $(seq 0 $last_pod)
+ [[ 1 -eq 2 ]]
+ wait_pod some-name-rs0-1
+ local pod=some-name-rs0-1
+ set +o xtrace
waiting for pod/some-name-rs0-1 to be ready.OK
+ for i in $(seq 0 $last_pod)
+ [[ 2 -eq 2 ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].arbiter.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.Ag0io3usX2
+++ mktemp
++ local LAST_ERR=/tmp/tmp.CbKj4vVBd5
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].arbiter.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.Ag0io3usX2
++ cat /tmp/tmp.CbKj4vVBd5
++ rm /tmp/tmp.Ag0io3usX2 /tmp/tmp.CbKj4vVBd5
++ return 0
+ [[ '' == \t\r\u\e ]]
+ wait_pod some-name-rs0-2
+ local pod=some-name-rs0-2
+ set +o xtrace
waiting for pod/some-name-rs0-2 to be ready.OK
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].nonvoting.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.iTpoq2sqp2
+++ mktemp
++ local LAST_ERR=/tmp/tmp.82e6WbLRcp
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].nonvoting.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.iTpoq2sqp2
++ cat /tmp/tmp.82e6WbLRcp
++ rm /tmp/tmp.iTpoq2sqp2 /tmp/tmp.82e6WbLRcp
++ return 0
+ [[ '' == \t\r\u\e ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].hidden.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.fI78jz3tf0
+++ mktemp
++ local LAST_ERR=/tmp/tmp.HlrJ3tE4g3
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].hidden.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.fI78jz3tf0
++ cat /tmp/tmp.HlrJ3tE4g3
++ rm /tmp/tmp.fI78jz3tf0 /tmp/tmp.HlrJ3tE4g3
++ return 0
+ [[ '' == \t\r\u\e ]]
+ sleep 10
+ [[ true == \t\r\u\e ]]
+ set +x
Waiting for cluster readyness...........................................
+ wait_cluster_consistency some-name
+ local cluster_name=some-name
+ local wait_time=32
+ retry=0
+ sleep 7
+ echo -n 'waiting for cluster readyness'
waiting for cluster readyness++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.WVBciRQi2D
+++ mktemp
++ local LAST_ERR=/tmp/tmp.ncYvz44TRZ
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.WVBciRQi2D
++ cat /tmp/tmp.ncYvz44TRZ
++ rm /tmp/tmp.WVBciRQi2D /tmp/tmp.ncYvz44TRZ
++ return 0
+ [[ ready == \r\e\a\d\y ]]
+ echo

+ kubectl delete psmdb some-name
perconaservermongodb.psmdb.percona.com "some-name" deleted from recover-no-primary-9198 namespace
+ kubectl delete pvc --all
persistentvolumeclaim "mongod-data-some-name-rs0-0" deleted from recover-no-primary-9198 namespace
persistentvolumeclaim "mongod-data-some-name-rs0-1" deleted from recover-no-primary-9198 namespace
persistentvolumeclaim "mongod-data-some-name-rs0-2" deleted from recover-no-primary-9198 namespace
+ desc 'testing exposed single replset cluster'
+ set +o xtrace


-----------------------------------------------------------------------------------
testing exposed single replset cluster
-----------------------------------------------------------------------------------

+ test_exposed_single_replset
+ apply_cluster /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2036/e2e-tests/recover-no-primary/conf/some-name-exposed.yml
+ '[' -z '' ']'
+ cat_config /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2036/e2e-tests/recover-no-primary/conf/some-name-exposed.yml
+ kubectl_bin apply -f -
+ cat /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2036/e2e-tests/recover-no-primary/conf/some-name-exposed.yml
++ mktemp
+ yq eval '(.spec | select(.image == null)).image = "perconalab/percona-server-mongodb-operator:main-mongod7.0"'
+ yq eval '(.spec | select(has("pmm"))).pmm.image = "percona/pmm-client:2.44.1-1"'
+ yq eval '(.spec | select(has("initImage"))).initImage = "perconalab/percona-server-mongodb-operator:PR-2036-6a5cff62"'
+ yq eval '(.spec | select(has("backup"))).backup.image = "perconalab/percona-server-mongodb-operator:main-backup"'
+ yq eval '.spec.upgradeOptions.apply="Never"'
+ local LAST_OUT=/tmp/tmp.ZvOBTbXatD
++ mktemp
+ local LAST_ERR=/tmp/tmp.fnNhNGEoGO
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl apply -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.ZvOBTbXatD
perconaservermongodb.psmdb.percona.com/some-name created
+ cat /tmp/tmp.fnNhNGEoGO
+ rm /tmp/tmp.ZvOBTbXatD /tmp/tmp.fnNhNGEoGO
+ return 0
+ wait_for_running some-name-rs0 3
+ local name=some-name-rs0
+ let last_pod=2
+ local check_cluster_readyness=true
+ set_debug
+ [[ 1 == 1 ]]
+ set -o xtrace
+ local rs_name=rs0
+ local cluster_name=some-name
++ seq 0 2
+ for i in $(seq 0 $last_pod)
+ [[ 0 -eq 2 ]]
+ wait_pod some-name-rs0-0
+ local pod=some-name-rs0-0
+ set +o xtrace
waiting for pod/some-name-rs0-0 to be ready..............OK
+ for i in $(seq 0 $last_pod)
+ [[ 1 -eq 2 ]]
+ wait_pod some-name-rs0-1
+ local pod=some-name-rs0-1
+ set +o xtrace
waiting for pod/some-name-rs0-1 to be ready.............OK
+ for i in $(seq 0 $last_pod)
+ [[ 2 -eq 2 ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].arbiter.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.sSHZejIbzH
+++ mktemp
++ local LAST_ERR=/tmp/tmp.86x9v7qlna
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].arbiter.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.sSHZejIbzH
++ cat /tmp/tmp.86x9v7qlna
++ rm /tmp/tmp.sSHZejIbzH /tmp/tmp.86x9v7qlna
++ return 0
+ [[ '' == \t\r\u\e ]]
+ wait_pod some-name-rs0-2
+ local pod=some-name-rs0-2
+ set +o xtrace
waiting for pod/some-name-rs0-2 to be ready..............OK
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].nonvoting.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.ZwDgRyYwpC
+++ mktemp
++ local LAST_ERR=/tmp/tmp.Yf0wq2X9p1
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].nonvoting.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.ZwDgRyYwpC
++ cat /tmp/tmp.Yf0wq2X9p1
++ rm /tmp/tmp.ZwDgRyYwpC /tmp/tmp.Yf0wq2X9p1
++ return 0
+ [[ '' == \t\r\u\e ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].hidden.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.uY6yZMTjs2
+++ mktemp
++ local LAST_ERR=/tmp/tmp.vB5wb5mJtv
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].hidden.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.uY6yZMTjs2
++ cat /tmp/tmp.vB5wb5mJtv
++ rm /tmp/tmp.uY6yZMTjs2 /tmp/tmp.vB5wb5mJtv
++ return 0
+ [[ '' == \t\r\u\e ]]
+ sleep 10
+ [[ true == \t\r\u\e ]]
+ set +x
Waiting for cluster readyness
+ wait_cluster_consistency some-name
+ local cluster_name=some-name
+ local wait_time=32
+ retry=0
+ sleep 7
+ echo -n 'waiting for cluster readyness'
waiting for cluster readyness++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.FqDHSQe4tp
+++ mktemp
++ local LAST_ERR=/tmp/tmp.riWqif9I0l
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.FqDHSQe4tp
++ cat /tmp/tmp.riWqif9I0l
++ rm /tmp/tmp.FqDHSQe4tp /tmp/tmp.riWqif9I0l
++ return 0
+ [[ ready == \r\e\a\d\y ]]
+ echo

+ desc 'kill pods abruptly and wait for recovery'
+ set +o xtrace


-----------------------------------------------------------------------------------
kill pods abruptly and wait for recovery
-----------------------------------------------------------------------------------

++ seq 0 2
+ for i in $(seq 0 2)
+ kubectl_bin exec some-name-rs0-0 -c mongod -- kill 1
++ mktemp
+ local LAST_OUT=/tmp/tmp.VpCOMsKlzM
++ mktemp
+ local LAST_ERR=/tmp/tmp.POFvIY3kai
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec some-name-rs0-0 -c mongod -- kill 1
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.VpCOMsKlzM
+ cat /tmp/tmp.POFvIY3kai
+ rm /tmp/tmp.VpCOMsKlzM /tmp/tmp.POFvIY3kai
+ return 0
+ for i in $(seq 0 2)
+ kubectl_bin exec some-name-rs0-1 -c mongod -- kill 1
++ mktemp
+ local LAST_OUT=/tmp/tmp.2TsQ0eh9Je
++ mktemp
+ local LAST_ERR=/tmp/tmp.T2Nofc2WzE
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec some-name-rs0-1 -c mongod -- kill 1
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.2TsQ0eh9Je
+ cat /tmp/tmp.T2Nofc2WzE
+ rm /tmp/tmp.2TsQ0eh9Je /tmp/tmp.T2Nofc2WzE
+ return 0
+ for i in $(seq 0 2)
+ kubectl_bin exec some-name-rs0-2 -c mongod -- kill 1
++ mktemp
+ local LAST_OUT=/tmp/tmp.TkkHiCDBgc
++ mktemp
+ local LAST_ERR=/tmp/tmp.imzqbnoRGD
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec some-name-rs0-2 -c mongod -- kill 1
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.TkkHiCDBgc
+ cat /tmp/tmp.imzqbnoRGD
+ rm /tmp/tmp.TkkHiCDBgc /tmp/tmp.imzqbnoRGD
+ return 0
+ sleep 10
+ wait_for_running some-name-rs0 3
+ local name=some-name-rs0
+ let last_pod=2
+ local check_cluster_readyness=true
+ set_debug
+ [[ 1 == 1 ]]
+ set -o xtrace
+ local rs_name=rs0
+ local cluster_name=some-name
++ seq 0 2
+ for i in $(seq 0 $last_pod)
+ [[ 0 -eq 2 ]]
+ wait_pod some-name-rs0-0
+ local pod=some-name-rs0-0
+ set +o xtrace
waiting for pod/some-name-rs0-0 to be ready.OK
+ for i in $(seq 0 $last_pod)
+ [[ 1 -eq 2 ]]
+ wait_pod some-name-rs0-1
+ local pod=some-name-rs0-1
+ set +o xtrace
waiting for pod/some-name-rs0-1 to be ready.OK
+ for i in $(seq 0 $last_pod)
+ [[ 2 -eq 2 ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].arbiter.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.aLvcEW0FRI
+++ mktemp
++ local LAST_ERR=/tmp/tmp.fD8jJ5Y5g2
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].arbiter.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.aLvcEW0FRI
++ cat /tmp/tmp.fD8jJ5Y5g2
++ rm /tmp/tmp.aLvcEW0FRI /tmp/tmp.fD8jJ5Y5g2
++ return 0
+ [[ '' == \t\r\u\e ]]
+ wait_pod some-name-rs0-2
+ local pod=some-name-rs0-2
+ set +o xtrace
waiting for pod/some-name-rs0-2 to be ready.OK
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].nonvoting.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.I8tA3ick8J
+++ mktemp
++ local LAST_ERR=/tmp/tmp.YhOrI08NDI
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].nonvoting.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.I8tA3ick8J
++ cat /tmp/tmp.YhOrI08NDI
++ rm /tmp/tmp.I8tA3ick8J /tmp/tmp.YhOrI08NDI
++ return 0
+ [[ '' == \t\r\u\e ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].hidden.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.vOvuLi280B
+++ mktemp
++ local LAST_ERR=/tmp/tmp.Gb7bx2Axbh
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].hidden.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.vOvuLi280B
++ cat /tmp/tmp.Gb7bx2Axbh
++ rm /tmp/tmp.vOvuLi280B /tmp/tmp.Gb7bx2Axbh
++ return 0
+ [[ '' == \t\r\u\e ]]
+ sleep 10
+ [[ true == \t\r\u\e ]]
+ set +x
Waiting for cluster readyness...
+ wait_cluster_consistency some-name
+ local cluster_name=some-name
+ local wait_time=32
+ retry=0
+ sleep 7
+ echo -n 'waiting for cluster readyness'
waiting for cluster readyness++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.1N3K2sWyMb
+++ mktemp
++ local LAST_ERR=/tmp/tmp.3f6kvI9bzF
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.1N3K2sWyMb
++ cat /tmp/tmp.3f6kvI9bzF
++ rm /tmp/tmp.1N3K2sWyMb /tmp/tmp.3f6kvI9bzF
++ return 0
+ [[ ready == \r\e\a\d\y ]]
+ echo

+ desc 'delete statefulset and wait for recovery'
+ set +o xtrace


-----------------------------------------------------------------------------------
delete statefulset and wait for recovery
-----------------------------------------------------------------------------------

+ kubectl delete sts some-name-rs0
statefulset.apps "some-name-rs0" deleted from recover-no-primary-9198 namespace
+ sleep 5
+ wait_for_running some-name-rs0 3
+ local name=some-name-rs0
+ let last_pod=2
+ local check_cluster_readyness=true
+ set_debug
+ [[ 1 == 1 ]]
+ set -o xtrace
+ local rs_name=rs0
+ local cluster_name=some-name
++ seq 0 2
+ for i in $(seq 0 $last_pod)
+ [[ 0 -eq 2 ]]
+ wait_pod some-name-rs0-0
+ local pod=some-name-rs0-0
+ set +o xtrace
waiting for pod/some-name-rs0-0 to be ready.OK
+ for i in $(seq 0 $last_pod)
+ [[ 1 -eq 2 ]]
+ wait_pod some-name-rs0-1
+ local pod=some-name-rs0-1
+ set +o xtrace
waiting for pod/some-name-rs0-1 to be ready.OK
+ for i in $(seq 0 $last_pod)
+ [[ 2 -eq 2 ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].arbiter.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.HkxlSc7sFC
+++ mktemp
++ local LAST_ERR=/tmp/tmp.MtKUphDSB4
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].arbiter.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.HkxlSc7sFC
++ cat /tmp/tmp.MtKUphDSB4
++ rm /tmp/tmp.HkxlSc7sFC /tmp/tmp.MtKUphDSB4
++ return 0
+ [[ '' == \t\r\u\e ]]
+ wait_pod some-name-rs0-2
+ local pod=some-name-rs0-2
+ set +o xtrace
waiting for pod/some-name-rs0-2 to be ready.OK
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].nonvoting.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.ZDCPJCNC3w
+++ mktemp
++ local LAST_ERR=/tmp/tmp.SZTsfzehQs
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].nonvoting.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.ZDCPJCNC3w
++ cat /tmp/tmp.SZTsfzehQs
++ rm /tmp/tmp.ZDCPJCNC3w /tmp/tmp.SZTsfzehQs
++ return 0
+ [[ '' == \t\r\u\e ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].hidden.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.B51FTPQoiU
+++ mktemp
++ local LAST_ERR=/tmp/tmp.MLcgTXcfwR
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].hidden.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.B51FTPQoiU
++ cat /tmp/tmp.MLcgTXcfwR
++ rm /tmp/tmp.B51FTPQoiU /tmp/tmp.MLcgTXcfwR
++ return 0
+ [[ '' == \t\r\u\e ]]
+ sleep 10
+ [[ true == \t\r\u\e ]]
+ set +x
Waiting for cluster readyness
+ wait_cluster_consistency some-name
+ local cluster_name=some-name
+ local wait_time=32
+ retry=0
+ sleep 7
+ echo -n 'waiting for cluster readyness'
waiting for cluster readyness++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.VGsnSUPMbs
+++ mktemp
++ local LAST_ERR=/tmp/tmp.7ylzoywTyF
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.VGsnSUPMbs
++ cat /tmp/tmp.7ylzoywTyF
++ rm /tmp/tmp.VGsnSUPMbs /tmp/tmp.7ylzoywTyF
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 1 -ge 32 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.f6ly1uNPD3
+++ mktemp
++ local LAST_ERR=/tmp/tmp.aAdyy2X5TT
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.f6ly1uNPD3
++ cat /tmp/tmp.aAdyy2X5TT
++ rm /tmp/tmp.f6ly1uNPD3 /tmp/tmp.aAdyy2X5TT
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 2 -ge 32 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.fA3Aja7DTG
+++ mktemp
++ local LAST_ERR=/tmp/tmp.mQlZeFD9dt
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.fA3Aja7DTG
++ cat /tmp/tmp.mQlZeFD9dt
++ rm /tmp/tmp.fA3Aja7DTG /tmp/tmp.mQlZeFD9dt
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 3 -ge 32 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.GQDqYQzhcJ
+++ mktemp
++ local LAST_ERR=/tmp/tmp.aQMAh7VqWO
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.GQDqYQzhcJ
++ cat /tmp/tmp.aQMAh7VqWO
++ rm /tmp/tmp.GQDqYQzhcJ /tmp/tmp.aQMAh7VqWO
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 4 -ge 32 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.hyuU3RVfwu
+++ mktemp
++ local LAST_ERR=/tmp/tmp.3fF2U3hRZP
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.hyuU3RVfwu
++ cat /tmp/tmp.3fF2U3hRZP
++ rm /tmp/tmp.hyuU3RVfwu /tmp/tmp.3fF2U3hRZP
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 5 -ge 32 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.YzrFUkbCu7
+++ mktemp
++ local LAST_ERR=/tmp/tmp.0iKCPvXxw3
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.YzrFUkbCu7
++ cat /tmp/tmp.0iKCPvXxw3
++ rm /tmp/tmp.YzrFUkbCu7 /tmp/tmp.0iKCPvXxw3
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 6 -ge 32 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.Zn5YrWMFL0
+++ mktemp
++ local LAST_ERR=/tmp/tmp.HOvwwgs2ab
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.Zn5YrWMFL0
++ cat /tmp/tmp.HOvwwgs2ab
++ rm /tmp/tmp.Zn5YrWMFL0 /tmp/tmp.HOvwwgs2ab
++ return 0
+ [[ ready == \r\e\a\d\y ]]
+ echo

+ desc 'delete cluster and recreate'
+ set +o xtrace


-----------------------------------------------------------------------------------
delete cluster and recreate
-----------------------------------------------------------------------------------

+ kubectl delete psmdb some-name
perconaservermongodb.psmdb.percona.com "some-name" deleted from recover-no-primary-9198 namespace
+ set +e
+ echo -n 'waiting for psmdb/some-name to be deleted'
waiting for psmdb/some-name to be deleted+ kubectl get psmdb some-name
Error from server (NotFound): perconaservermongodbs.psmdb.percona.com "some-name" not found
+ set -e
+ apply_cluster /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2036/e2e-tests/recover-no-primary/conf/some-name-exposed.yml
+ '[' -z '' ']'
+ cat_config /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2036/e2e-tests/recover-no-primary/conf/some-name-exposed.yml
+ kubectl_bin apply -f -
+ cat /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2036/e2e-tests/recover-no-primary/conf/some-name-exposed.yml
++ mktemp
+ yq eval '(.spec | select(.image == null)).image = "perconalab/percona-server-mongodb-operator:main-mongod7.0"'
+ yq eval '(.spec | select(has("pmm"))).pmm.image = "percona/pmm-client:2.44.1-1"'
+ yq eval '(.spec | select(has("initImage"))).initImage = "perconalab/percona-server-mongodb-operator:PR-2036-6a5cff62"'
+ yq eval '(.spec | select(has("backup"))).backup.image = "perconalab/percona-server-mongodb-operator:main-backup"'
+ yq eval '.spec.upgradeOptions.apply="Never"'
+ local LAST_OUT=/tmp/tmp.iiGS0ToUU3
++ mktemp
+ local LAST_ERR=/tmp/tmp.2QjH6Z05NL
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl apply -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.iiGS0ToUU3
perconaservermongodb.psmdb.percona.com/some-name created
+ cat /tmp/tmp.2QjH6Z05NL
+ rm /tmp/tmp.iiGS0ToUU3 /tmp/tmp.2QjH6Z05NL
+ return 0
+ wait_for_running some-name-rs0 3
+ local name=some-name-rs0
+ let last_pod=2
+ local check_cluster_readyness=true
+ set_debug
+ [[ 1 == 1 ]]
+ set -o xtrace
+ local rs_name=rs0
+ local cluster_name=some-name
++ seq 0 2
+ for i in $(seq 0 $last_pod)
+ [[ 0 -eq 2 ]]
+ wait_pod some-name-rs0-0
+ local pod=some-name-rs0-0
+ set +o xtrace
waiting for pod/some-name-rs0-0 to be ready.OK
+ for i in $(seq 0 $last_pod)
+ [[ 1 -eq 2 ]]
+ wait_pod some-name-rs0-1
+ local pod=some-name-rs0-1
+ set +o xtrace
waiting for pod/some-name-rs0-1 to be ready.OK
+ for i in $(seq 0 $last_pod)
+ [[ 2 -eq 2 ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].arbiter.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.auLLmo3d9F
+++ mktemp
++ local LAST_ERR=/tmp/tmp.Qvkxp9UILV
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].arbiter.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.auLLmo3d9F
++ cat /tmp/tmp.Qvkxp9UILV
++ rm /tmp/tmp.auLLmo3d9F /tmp/tmp.Qvkxp9UILV
++ return 0
+ [[ '' == \t\r\u\e ]]
+ wait_pod some-name-rs0-2
+ local pod=some-name-rs0-2
+ set +o xtrace
waiting for pod/some-name-rs0-2 to be ready.OK
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].nonvoting.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.rjSK3Gg4kW
+++ mktemp
++ local LAST_ERR=/tmp/tmp.F7nx3ruO0I
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].nonvoting.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.rjSK3Gg4kW
++ cat /tmp/tmp.F7nx3ruO0I
++ rm /tmp/tmp.rjSK3Gg4kW /tmp/tmp.F7nx3ruO0I
++ return 0
+ [[ '' == \t\r\u\e ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].hidden.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.aiSFMONthM
+++ mktemp
++ local LAST_ERR=/tmp/tmp.HvysNghS9h
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].hidden.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.aiSFMONthM
++ cat /tmp/tmp.HvysNghS9h
++ rm /tmp/tmp.aiSFMONthM /tmp/tmp.HvysNghS9h
++ return 0
+ [[ '' == \t\r\u\e ]]
+ sleep 10
+ [[ true == \t\r\u\e ]]
+ set +x
Waiting for cluster readyness......................................
+ wait_cluster_consistency some-name
+ local cluster_name=some-name
+ local wait_time=32
+ retry=0
+ sleep 7
+ echo -n 'waiting for cluster readyness'
waiting for cluster readyness++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.OmkGjxXwrH
+++ mktemp
++ local LAST_ERR=/tmp/tmp.gFbKKcWmOy
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.OmkGjxXwrH
++ cat /tmp/tmp.gFbKKcWmOy
++ rm /tmp/tmp.OmkGjxXwrH /tmp/tmp.gFbKKcWmOy
++ return 0
+ [[ ready == \r\e\a\d\y ]]
+ echo

+ kubectl delete psmdb some-name
perconaservermongodb.psmdb.percona.com "some-name" deleted from recover-no-primary-9198 namespace
+ kubectl delete pvc --all
persistentvolumeclaim "mongod-data-some-name-rs0-0" deleted from recover-no-primary-9198 namespace
persistentvolumeclaim "mongod-data-some-name-rs0-1" deleted from recover-no-primary-9198 namespace
persistentvolumeclaim "mongod-data-some-name-rs0-2" deleted from recover-no-primary-9198 namespace
+ desc 'testing exposed two replsets sharded cluster'
+ set +o xtrace


-----------------------------------------------------------------------------------
testing exposed two replsets sharded cluster
-----------------------------------------------------------------------------------

+ test_exposed_sharded_cluster
+ apply_cluster /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2036/e2e-tests/recover-no-primary/conf/some-name-sharded.yml
+ '[' -z '' ']'
+ cat_config /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2036/e2e-tests/recover-no-primary/conf/some-name-sharded.yml
+ kubectl_bin apply -f -
+ cat /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2036/e2e-tests/recover-no-primary/conf/some-name-sharded.yml
++ mktemp
+ yq eval '(.spec | select(.image == null)).image = "perconalab/percona-server-mongodb-operator:main-mongod7.0"'
+ yq eval '(.spec | select(has("pmm"))).pmm.image = "percona/pmm-client:2.44.1-1"'
+ yq eval '(.spec | select(has("backup"))).backup.image = "perconalab/percona-server-mongodb-operator:main-backup"'
+ yq eval '(.spec | select(has("initImage"))).initImage = "perconalab/percona-server-mongodb-operator:PR-2036-6a5cff62"'
+ yq eval '.spec.upgradeOptions.apply="Never"'
+ local LAST_OUT=/tmp/tmp.Lg8lTu1dTB
++ mktemp
+ local LAST_ERR=/tmp/tmp.QurDo1YelA
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl apply -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.Lg8lTu1dTB
perconaservermongodb.psmdb.percona.com/some-name created
+ cat /tmp/tmp.QurDo1YelA
+ rm /tmp/tmp.Lg8lTu1dTB /tmp/tmp.QurDo1YelA
+ return 0
+ wait_for_running some-name-rs0 3
+ local name=some-name-rs0
+ let last_pod=2
+ local check_cluster_readyness=true
+ set_debug
+ [[ 1 == 1 ]]
+ set -o xtrace
+ local rs_name=rs0
+ local cluster_name=some-name
++ seq 0 2
+ for i in $(seq 0 $last_pod)
+ [[ 0 -eq 2 ]]
+ wait_pod some-name-rs0-0
+ local pod=some-name-rs0-0
+ set +o xtrace
waiting for pod/some-name-rs0-0 to be ready.................OK
+ for i in $(seq 0 $last_pod)
+ [[ 1 -eq 2 ]]
+ wait_pod some-name-rs0-1
+ local pod=some-name-rs0-1
+ set +o xtrace
waiting for pod/some-name-rs0-1 to be ready.............OK
+ for i in $(seq 0 $last_pod)
+ [[ 2 -eq 2 ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].arbiter.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.cNOgQTFn9B
+++ mktemp
++ local LAST_ERR=/tmp/tmp.HrErWSW6fA
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].arbiter.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.cNOgQTFn9B
++ cat /tmp/tmp.HrErWSW6fA
++ rm /tmp/tmp.cNOgQTFn9B /tmp/tmp.HrErWSW6fA
++ return 0
+ [[ '' == \t\r\u\e ]]
+ wait_pod some-name-rs0-2
+ local pod=some-name-rs0-2
+ set +o xtrace
waiting for pod/some-name-rs0-2 to be ready..............OK
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].nonvoting.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.XqnkDrxaoC
+++ mktemp
++ local LAST_ERR=/tmp/tmp.sQfDirIFJw
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].nonvoting.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.XqnkDrxaoC
++ cat /tmp/tmp.sQfDirIFJw
++ rm /tmp/tmp.XqnkDrxaoC /tmp/tmp.sQfDirIFJw
++ return 0
+ [[ '' == \t\r\u\e ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].hidden.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.fosVHMpAzo
+++ mktemp
++ local LAST_ERR=/tmp/tmp.5hyQuEluRr
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].hidden.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.fosVHMpAzo
++ cat /tmp/tmp.5hyQuEluRr
++ rm /tmp/tmp.fosVHMpAzo /tmp/tmp.5hyQuEluRr
++ return 0
+ [[ '' == \t\r\u\e ]]
+ sleep 10
+ [[ true == \t\r\u\e ]]
+ set +x
Waiting for cluster readyness..............................
+ wait_cluster_consistency some-name
+ local cluster_name=some-name
+ local wait_time=32
+ retry=0
+ sleep 7
+ echo -n 'waiting for cluster readyness'
waiting for cluster readyness++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.jWUM86byfC
+++ mktemp
++ local LAST_ERR=/tmp/tmp.cWu7xRfNlS
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.jWUM86byfC
++ cat /tmp/tmp.cWu7xRfNlS
++ rm /tmp/tmp.jWUM86byfC /tmp/tmp.cWu7xRfNlS
++ return 0
+ [[ ready == \r\e\a\d\y ]]
+ echo

+ desc 'kill pods abruptly and wait for recovery'
+ set +o xtrace


-----------------------------------------------------------------------------------
kill pods abruptly and wait for recovery
-----------------------------------------------------------------------------------

++ seq 0 2
+ for i in $(seq 0 2)
+ kubectl_bin exec some-name-rs0-0 -c mongod -- kill 1
++ mktemp
+ local LAST_OUT=/tmp/tmp.JxrEE9H1hv
++ mktemp
+ local LAST_ERR=/tmp/tmp.QAjT48sx4N
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec some-name-rs0-0 -c mongod -- kill 1
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.JxrEE9H1hv
+ cat /tmp/tmp.QAjT48sx4N
+ rm /tmp/tmp.JxrEE9H1hv /tmp/tmp.QAjT48sx4N
+ return 0
+ for i in $(seq 0 2)
+ kubectl_bin exec some-name-rs0-1 -c mongod -- kill 1
++ mktemp
+ local LAST_OUT=/tmp/tmp.q4Ys39Uvfg
++ mktemp
+ local LAST_ERR=/tmp/tmp.4PYWzGn8zT
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec some-name-rs0-1 -c mongod -- kill 1
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.q4Ys39Uvfg
+ cat /tmp/tmp.4PYWzGn8zT
+ rm /tmp/tmp.q4Ys39Uvfg /tmp/tmp.4PYWzGn8zT
+ return 0
+ for i in $(seq 0 2)
+ kubectl_bin exec some-name-rs0-2 -c mongod -- kill 1
++ mktemp
+ local LAST_OUT=/tmp/tmp.FmJ7fbHNOQ
++ mktemp
+ local LAST_ERR=/tmp/tmp.yAq7Ai8Noh
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec some-name-rs0-2 -c mongod -- kill 1
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.FmJ7fbHNOQ
+ cat /tmp/tmp.yAq7Ai8Noh
+ rm /tmp/tmp.FmJ7fbHNOQ /tmp/tmp.yAq7Ai8Noh
+ return 0
+ sleep 10
+ wait_for_running some-name-rs0 3
+ local name=some-name-rs0
+ let last_pod=2
+ local check_cluster_readyness=true
+ set_debug
+ [[ 1 == 1 ]]
+ set -o xtrace
+ local rs_name=rs0
+ local cluster_name=some-name
++ seq 0 2
+ for i in $(seq 0 $last_pod)
+ [[ 0 -eq 2 ]]
+ wait_pod some-name-rs0-0
+ local pod=some-name-rs0-0
+ set +o xtrace
waiting for pod/some-name-rs0-0 to be ready.OK
+ for i in $(seq 0 $last_pod)
+ [[ 1 -eq 2 ]]
+ wait_pod some-name-rs0-1
+ local pod=some-name-rs0-1
+ set +o xtrace
waiting for pod/some-name-rs0-1 to be ready.OK
+ for i in $(seq 0 $last_pod)
+ [[ 2 -eq 2 ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].arbiter.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.5Yn5nqbvGs
+++ mktemp
++ local LAST_ERR=/tmp/tmp.0MTKG1gKrw
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].arbiter.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.5Yn5nqbvGs
++ cat /tmp/tmp.0MTKG1gKrw
++ rm /tmp/tmp.5Yn5nqbvGs /tmp/tmp.0MTKG1gKrw
++ return 0
+ [[ '' == \t\r\u\e ]]
+ wait_pod some-name-rs0-2
+ local pod=some-name-rs0-2
+ set +o xtrace
waiting for pod/some-name-rs0-2 to be ready.OK
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].nonvoting.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.GSaJO9cq0O
+++ mktemp
++ local LAST_ERR=/tmp/tmp.ploPgmDIqn
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].nonvoting.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.GSaJO9cq0O
++ cat /tmp/tmp.ploPgmDIqn
++ rm /tmp/tmp.GSaJO9cq0O /tmp/tmp.ploPgmDIqn
++ return 0
+ [[ '' == \t\r\u\e ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].hidden.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.LnE0rgr9VK
+++ mktemp
++ local LAST_ERR=/tmp/tmp.GVVjg8iVX9
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].hidden.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.LnE0rgr9VK
++ cat /tmp/tmp.GVVjg8iVX9
++ rm /tmp/tmp.LnE0rgr9VK /tmp/tmp.GVVjg8iVX9
++ return 0
+ [[ '' == \t\r\u\e ]]
+ sleep 10
+ [[ true == \t\r\u\e ]]
+ set +x
Waiting for cluster readyness...
+ wait_cluster_consistency some-name
+ local cluster_name=some-name
+ local wait_time=32
+ retry=0
+ sleep 7
+ echo -n 'waiting for cluster readyness'
waiting for cluster readyness++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.Q52vyFBQB1
+++ mktemp
++ local LAST_ERR=/tmp/tmp.uEFbHWJbCf
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.Q52vyFBQB1
++ cat /tmp/tmp.uEFbHWJbCf
++ rm /tmp/tmp.Q52vyFBQB1 /tmp/tmp.uEFbHWJbCf
++ return 0
+ [[ ready == \r\e\a\d\y ]]
+ echo

++ seq 0 2
+ for i in $(seq 0 2)
+ kubectl_bin exec some-name-rs1-0 -c mongod -- kill 1
++ mktemp
+ local LAST_OUT=/tmp/tmp.MDA28DJjhc
++ mktemp
+ local LAST_ERR=/tmp/tmp.HKeq6irO9E
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec some-name-rs1-0 -c mongod -- kill 1
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.MDA28DJjhc
+ cat /tmp/tmp.HKeq6irO9E
+ rm /tmp/tmp.MDA28DJjhc /tmp/tmp.HKeq6irO9E
+ return 0
+ for i in $(seq 0 2)
+ kubectl_bin exec some-name-rs1-1 -c mongod -- kill 1
++ mktemp
+ local LAST_OUT=/tmp/tmp.uF4MLQVqbi
++ mktemp
+ local LAST_ERR=/tmp/tmp.tt2mXFROuh
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec some-name-rs1-1 -c mongod -- kill 1
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.uF4MLQVqbi
+ cat /tmp/tmp.tt2mXFROuh
+ rm /tmp/tmp.uF4MLQVqbi /tmp/tmp.tt2mXFROuh
+ return 0
+ for i in $(seq 0 2)
+ kubectl_bin exec some-name-rs1-2 -c mongod -- kill 1
++ mktemp
+ local LAST_OUT=/tmp/tmp.h91lZS1Dix
++ mktemp
+ local LAST_ERR=/tmp/tmp.qHwvs6XgX0
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec some-name-rs1-2 -c mongod -- kill 1
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.h91lZS1Dix
+ cat /tmp/tmp.qHwvs6XgX0
+ rm /tmp/tmp.h91lZS1Dix /tmp/tmp.qHwvs6XgX0
+ return 0
+ sleep 10
+ wait_for_running some-name-rs1 3
+ local name=some-name-rs1
+ let last_pod=2
+ local check_cluster_readyness=true
+ set_debug
+ [[ 1 == 1 ]]
+ set -o xtrace
+ local rs_name=rs1
+ local cluster_name=some-name
++ seq 0 2
+ for i in $(seq 0 $last_pod)
+ [[ 0 -eq 2 ]]
+ wait_pod some-name-rs1-0
+ local pod=some-name-rs1-0
+ set +o xtrace
waiting for pod/some-name-rs1-0 to be ready.OK
+ for i in $(seq 0 $last_pod)
+ [[ 1 -eq 2 ]]
+ wait_pod some-name-rs1-1
+ local pod=some-name-rs1-1
+ set +o xtrace
waiting for pod/some-name-rs1-1 to be ready.OK
+ for i in $(seq 0 $last_pod)
+ [[ 2 -eq 2 ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs1")].arbiter.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.C0tSXDotB4
+++ mktemp
++ local LAST_ERR=/tmp/tmp.y2oe5MVMJu
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs1")].arbiter.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.C0tSXDotB4
++ cat /tmp/tmp.y2oe5MVMJu
++ rm /tmp/tmp.C0tSXDotB4 /tmp/tmp.y2oe5MVMJu
++ return 0
+ [[ '' == \t\r\u\e ]]
+ wait_pod some-name-rs1-2
+ local pod=some-name-rs1-2
+ set +o xtrace
waiting for pod/some-name-rs1-2 to be ready.OK
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs1")].nonvoting.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.68Nv524wdx
+++ mktemp
++ local LAST_ERR=/tmp/tmp.3QxDJsAfKx
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs1")].nonvoting.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.68Nv524wdx
++ cat /tmp/tmp.3QxDJsAfKx
++ rm /tmp/tmp.68Nv524wdx /tmp/tmp.3QxDJsAfKx
++ return 0
+ [[ '' == \t\r\u\e ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs1")].hidden.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.XGbItI02HD
+++ mktemp
++ local LAST_ERR=/tmp/tmp.zG82m4UOKx
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs1")].hidden.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.XGbItI02HD
++ cat /tmp/tmp.zG82m4UOKx
++ rm /tmp/tmp.XGbItI02HD /tmp/tmp.zG82m4UOKx
++ return 0
+ [[ '' == \t\r\u\e ]]
+ sleep 10
+ [[ true == \t\r\u\e ]]
+ set +x
Waiting for cluster readyness....
+ wait_cluster_consistency some-name
+ local cluster_name=some-name
+ local wait_time=32
+ retry=0
+ sleep 7
+ echo -n 'waiting for cluster readyness'
waiting for cluster readyness++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.H5RrJP1zBZ
+++ mktemp
++ local LAST_ERR=/tmp/tmp.LgtTDHHmaz
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.H5RrJP1zBZ
++ cat /tmp/tmp.LgtTDHHmaz
++ rm /tmp/tmp.H5RrJP1zBZ /tmp/tmp.LgtTDHHmaz
++ return 0
+ [[ ready == \r\e\a\d\y ]]
+ echo

++ seq 0 2
+ for i in $(seq 0 2)
+ kubectl_bin exec some-name-cfg-0 -c mongod -- kill 1
++ mktemp
+ local LAST_OUT=/tmp/tmp.u8qnfyiy5G
++ mktemp
+ local LAST_ERR=/tmp/tmp.VD5E4HNsUI
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec some-name-cfg-0 -c mongod -- kill 1
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.u8qnfyiy5G
+ cat /tmp/tmp.VD5E4HNsUI
+ rm /tmp/tmp.u8qnfyiy5G /tmp/tmp.VD5E4HNsUI
+ return 0
+ for i in $(seq 0 2)
+ kubectl_bin exec some-name-cfg-1 -c mongod -- kill 1
++ mktemp
+ local LAST_OUT=/tmp/tmp.yiC7VJRixS
++ mktemp
+ local LAST_ERR=/tmp/tmp.w1xtPBf7wk
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec some-name-cfg-1 -c mongod -- kill 1
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.yiC7VJRixS
+ cat /tmp/tmp.w1xtPBf7wk
+ rm /tmp/tmp.yiC7VJRixS /tmp/tmp.w1xtPBf7wk
+ return 0
+ for i in $(seq 0 2)
+ kubectl_bin exec some-name-cfg-2 -c mongod -- kill 1
++ mktemp
+ local LAST_OUT=/tmp/tmp.sDMdAg4TsZ
++ mktemp
+ local LAST_ERR=/tmp/tmp.026CzbVZB4
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec some-name-cfg-2 -c mongod -- kill 1
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.sDMdAg4TsZ
+ cat /tmp/tmp.026CzbVZB4
+ rm /tmp/tmp.sDMdAg4TsZ /tmp/tmp.026CzbVZB4
+ return 0
+ sleep 10
+ wait_for_running some-name-cfg 3
+ local name=some-name-cfg
+ let last_pod=2
+ local check_cluster_readyness=true
+ set_debug
+ [[ 1 == 1 ]]
+ set -o xtrace
+ local rs_name=cfg
+ local cluster_name=some-name
++ seq 0 2
+ for i in $(seq 0 $last_pod)
+ [[ 0 -eq 2 ]]
+ wait_pod some-name-cfg-0
+ local pod=some-name-cfg-0
+ set +o xtrace
waiting for pod/some-name-cfg-0 to be ready.OK
+ for i in $(seq 0 $last_pod)
+ [[ 1 -eq 2 ]]
+ wait_pod some-name-cfg-1
+ local pod=some-name-cfg-1
+ set +o xtrace
waiting for pod/some-name-cfg-1 to be ready.OK
+ for i in $(seq 0 $last_pod)
+ [[ 2 -eq 2 ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].arbiter.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.cmtwt1owt9
+++ mktemp
++ local LAST_ERR=/tmp/tmp.kr6Gx6Bg2p
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].arbiter.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.cmtwt1owt9
++ cat /tmp/tmp.kr6Gx6Bg2p
++ rm /tmp/tmp.cmtwt1owt9 /tmp/tmp.kr6Gx6Bg2p
++ return 0
+ [[ '' == \t\r\u\e ]]
+ wait_pod some-name-cfg-2
+ local pod=some-name-cfg-2
+ set +o xtrace
waiting for pod/some-name-cfg-2 to be ready.OK
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].nonvoting.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.yzaJGH3TlO
+++ mktemp
++ local LAST_ERR=/tmp/tmp.VJ0xRxBBKW
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].nonvoting.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.yzaJGH3TlO
++ cat /tmp/tmp.VJ0xRxBBKW
++ rm /tmp/tmp.yzaJGH3TlO /tmp/tmp.VJ0xRxBBKW
++ return 0
+ [[ '' == \t\r\u\e ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].hidden.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.BGD5Pgvt51
+++ mktemp
++ local LAST_ERR=/tmp/tmp.2ZfrEyNKiu
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].hidden.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.BGD5Pgvt51
++ cat /tmp/tmp.2ZfrEyNKiu
++ rm /tmp/tmp.BGD5Pgvt51 /tmp/tmp.2ZfrEyNKiu
++ return 0
+ [[ '' == \t\r\u\e ]]
+ sleep 10
+ [[ true == \t\r\u\e ]]
+ set +x
Waiting for cluster readyness...........
+ wait_cluster_consistency some-name
+ local cluster_name=some-name
+ local wait_time=32
+ retry=0
+ sleep 7
+ echo -n 'waiting for cluster readyness'
waiting for cluster readyness++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.chCotDT0Pf
+++ mktemp
++ local LAST_ERR=/tmp/tmp.7jSQzc13Od
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.chCotDT0Pf
++ cat /tmp/tmp.7jSQzc13Od
++ rm /tmp/tmp.chCotDT0Pf /tmp/tmp.7jSQzc13Od
++ return 0
+ [[ ready == \r\e\a\d\y ]]
+ echo

+ desc 'delete statefulsets and wait for recovery'
+ set +o xtrace


-----------------------------------------------------------------------------------
delete statefulsets and wait for recovery
-----------------------------------------------------------------------------------

+ kubectl delete sts some-name-rs0
statefulset.apps "some-name-rs0" deleted from recover-no-primary-9198 namespace
+ sleep 5
+ wait_for_running some-name-rs0 3
+ local name=some-name-rs0
+ let last_pod=2
+ local check_cluster_readyness=true
+ set_debug
+ [[ 1 == 1 ]]
+ set -o xtrace
+ local rs_name=rs0
+ local cluster_name=some-name
++ seq 0 2
+ for i in $(seq 0 $last_pod)
+ [[ 0 -eq 2 ]]
+ wait_pod some-name-rs0-0
+ local pod=some-name-rs0-0
+ set +o xtrace
waiting for pod/some-name-rs0-0 to be ready.OK
+ for i in $(seq 0 $last_pod)
+ [[ 1 -eq 2 ]]
+ wait_pod some-name-rs0-1
+ local pod=some-name-rs0-1
+ set +o xtrace
waiting for pod/some-name-rs0-1 to be ready.OK
+ for i in $(seq 0 $last_pod)
+ [[ 2 -eq 2 ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].arbiter.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.sxE0wkeZv0
+++ mktemp
++ local LAST_ERR=/tmp/tmp.CA9IEfBvf7
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].arbiter.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.sxE0wkeZv0
++ cat /tmp/tmp.CA9IEfBvf7
++ rm /tmp/tmp.sxE0wkeZv0 /tmp/tmp.CA9IEfBvf7
++ return 0
+ [[ '' == \t\r\u\e ]]
+ wait_pod some-name-rs0-2
+ local pod=some-name-rs0-2
+ set +o xtrace
waiting for pod/some-name-rs0-2 to be ready.OK
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].nonvoting.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.Dk6lCYdbyt
+++ mktemp
++ local LAST_ERR=/tmp/tmp.Re6GZjIqQ9
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].nonvoting.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.Dk6lCYdbyt
++ cat /tmp/tmp.Re6GZjIqQ9
++ rm /tmp/tmp.Dk6lCYdbyt /tmp/tmp.Re6GZjIqQ9
++ return 0
+ [[ '' == \t\r\u\e ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].hidden.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.yJhEhzbbSz
+++ mktemp
++ local LAST_ERR=/tmp/tmp.b7UrdaPRMG
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].hidden.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.yJhEhzbbSz
++ cat /tmp/tmp.b7UrdaPRMG
++ rm /tmp/tmp.yJhEhzbbSz /tmp/tmp.b7UrdaPRMG
++ return 0
+ [[ '' == \t\r\u\e ]]
+ sleep 10
+ [[ true == \t\r\u\e ]]
+ set +x
Waiting for cluster readyness
+ wait_cluster_consistency some-name
+ local cluster_name=some-name
+ local wait_time=32
+ retry=0
+ sleep 7
+ echo -n 'waiting for cluster readyness'
waiting for cluster readyness++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.nTtGMemVdZ
+++ mktemp
++ local LAST_ERR=/tmp/tmp.ALGbVqaly2
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.nTtGMemVdZ
++ cat /tmp/tmp.ALGbVqaly2
++ rm /tmp/tmp.nTtGMemVdZ /tmp/tmp.ALGbVqaly2
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 1 -ge 32 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.dB7hPDAq2K
+++ mktemp
++ local LAST_ERR=/tmp/tmp.vilE1X6Mcg
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.dB7hPDAq2K
++ cat /tmp/tmp.vilE1X6Mcg
++ rm /tmp/tmp.dB7hPDAq2K /tmp/tmp.vilE1X6Mcg
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 2 -ge 32 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.BFBb5m5UqB
+++ mktemp
++ local LAST_ERR=/tmp/tmp.pvbYoxoll2
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.BFBb5m5UqB
++ cat /tmp/tmp.pvbYoxoll2
++ rm /tmp/tmp.BFBb5m5UqB /tmp/tmp.pvbYoxoll2
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 3 -ge 32 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.5KwAwajFgH
+++ mktemp
++ local LAST_ERR=/tmp/tmp.JdE0gawSrK
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.5KwAwajFgH
++ cat /tmp/tmp.JdE0gawSrK
++ rm /tmp/tmp.5KwAwajFgH /tmp/tmp.JdE0gawSrK
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 4 -ge 32 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.BkEPSG3paY
+++ mktemp
++ local LAST_ERR=/tmp/tmp.GKW4eIWtkS
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.BkEPSG3paY
++ cat /tmp/tmp.GKW4eIWtkS
++ rm /tmp/tmp.BkEPSG3paY /tmp/tmp.GKW4eIWtkS
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 5 -ge 32 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.BwdRX3eVxO
+++ mktemp
++ local LAST_ERR=/tmp/tmp.oG9Yn4Q5lB
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.BwdRX3eVxO
++ cat /tmp/tmp.oG9Yn4Q5lB
++ rm /tmp/tmp.BwdRX3eVxO /tmp/tmp.oG9Yn4Q5lB
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 6 -ge 32 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.pZvh1Gv53t
+++ mktemp
++ local LAST_ERR=/tmp/tmp.7dBf0JhR4Z
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.pZvh1Gv53t
++ cat /tmp/tmp.7dBf0JhR4Z
++ rm /tmp/tmp.pZvh1Gv53t /tmp/tmp.7dBf0JhR4Z
++ return 0
+ [[ ready == \r\e\a\d\y ]]
+ echo

+ kubectl delete sts some-name-rs1
statefulset.apps "some-name-rs1" deleted from recover-no-primary-9198 namespace
+ sleep 5
+ wait_for_running some-name-rs1 3
+ local name=some-name-rs1
+ let last_pod=2
+ local check_cluster_readyness=true
+ set_debug
+ [[ 1 == 1 ]]
+ set -o xtrace
+ local rs_name=rs1
+ local cluster_name=some-name
++ seq 0 2
+ for i in $(seq 0 $last_pod)
+ [[ 0 -eq 2 ]]
+ wait_pod some-name-rs1-0
+ local pod=some-name-rs1-0
+ set +o xtrace
waiting for pod/some-name-rs1-0 to be ready.OK
+ for i in $(seq 0 $last_pod)
+ [[ 1 -eq 2 ]]
+ wait_pod some-name-rs1-1
+ local pod=some-name-rs1-1
+ set +o xtrace
waiting for pod/some-name-rs1-1 to be ready.OK
+ for i in $(seq 0 $last_pod)
+ [[ 2 -eq 2 ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs1")].arbiter.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.OIm2Vbfib0
+++ mktemp
++ local LAST_ERR=/tmp/tmp.HUo1AJ3ypb
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs1")].arbiter.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.OIm2Vbfib0
++ cat /tmp/tmp.HUo1AJ3ypb
++ rm /tmp/tmp.OIm2Vbfib0 /tmp/tmp.HUo1AJ3ypb
++ return 0
+ [[ '' == \t\r\u\e ]]
+ wait_pod some-name-rs1-2
+ local pod=some-name-rs1-2
+ set +o xtrace
waiting for pod/some-name-rs1-2 to be ready.OK
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs1")].nonvoting.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.wi6FKw6nJD
+++ mktemp
++ local LAST_ERR=/tmp/tmp.K822qbU9zk
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs1")].nonvoting.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.wi6FKw6nJD
++ cat /tmp/tmp.K822qbU9zk
++ rm /tmp/tmp.wi6FKw6nJD /tmp/tmp.K822qbU9zk
++ return 0
+ [[ '' == \t\r\u\e ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs1")].hidden.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.lUkVZ48pwA
+++ mktemp
++ local LAST_ERR=/tmp/tmp.R3HFOa64Xd
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs1")].hidden.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.lUkVZ48pwA
++ cat /tmp/tmp.R3HFOa64Xd
++ rm /tmp/tmp.lUkVZ48pwA /tmp/tmp.R3HFOa64Xd
++ return 0
+ [[ '' == \t\r\u\e ]]
+ sleep 10
+ [[ true == \t\r\u\e ]]
+ set +x
Waiting for cluster readyness..................................................
+ wait_cluster_consistency some-name
+ local cluster_name=some-name
+ local wait_time=32
+ retry=0
+ sleep 7
+ echo -n 'waiting for cluster readyness'
waiting for cluster readyness++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.K3RiW3QLbt
+++ mktemp
++ local LAST_ERR=/tmp/tmp.k6k9a54KQK
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.K3RiW3QLbt
++ cat /tmp/tmp.k6k9a54KQK
++ rm /tmp/tmp.K3RiW3QLbt /tmp/tmp.k6k9a54KQK
++ return 0
+ [[ ready == \r\e\a\d\y ]]
+ echo

+ kubectl delete sts some-name-cfg
statefulset.apps "some-name-cfg" deleted from recover-no-primary-9198 namespace
+ sleep 5
+ wait_for_running some-name-cfg 3
+ local name=some-name-cfg
+ let last_pod=2
+ local check_cluster_readyness=true
+ set_debug
+ [[ 1 == 1 ]]
+ set -o xtrace
+ local rs_name=cfg
+ local cluster_name=some-name
++ seq 0 2
+ for i in $(seq 0 $last_pod)
+ [[ 0 -eq 2 ]]
+ wait_pod some-name-cfg-0
+ local pod=some-name-cfg-0
+ set +o xtrace
waiting for pod/some-name-cfg-0 to be ready.OK
+ for i in $(seq 0 $last_pod)
+ [[ 1 -eq 2 ]]
+ wait_pod some-name-cfg-1
+ local pod=some-name-cfg-1
+ set +o xtrace
waiting for pod/some-name-cfg-1 to be ready.OK
+ for i in $(seq 0 $last_pod)
+ [[ 2 -eq 2 ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].arbiter.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.7W2k1Ai8nI
+++ mktemp
++ local LAST_ERR=/tmp/tmp.srqtPh2vZx
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].arbiter.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.7W2k1Ai8nI
++ cat /tmp/tmp.srqtPh2vZx
++ rm /tmp/tmp.7W2k1Ai8nI /tmp/tmp.srqtPh2vZx
++ return 0
+ [[ '' == \t\r\u\e ]]
+ wait_pod some-name-cfg-2
+ local pod=some-name-cfg-2
+ set +o xtrace
waiting for pod/some-name-cfg-2 to be ready.OK
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].nonvoting.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.hxh3RnEKZi
+++ mktemp
++ local LAST_ERR=/tmp/tmp.lJyZ0Y3u6t
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].nonvoting.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.hxh3RnEKZi
++ cat /tmp/tmp.lJyZ0Y3u6t
++ rm /tmp/tmp.hxh3RnEKZi /tmp/tmp.lJyZ0Y3u6t
++ return 0
+ [[ '' == \t\r\u\e ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].hidden.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.ty2ej85XFK
+++ mktemp
++ local LAST_ERR=/tmp/tmp.Wzg3axMj9I
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].hidden.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.ty2ej85XFK
++ cat /tmp/tmp.Wzg3axMj9I
++ rm /tmp/tmp.ty2ej85XFK /tmp/tmp.Wzg3axMj9I
++ return 0
+ [[ '' == \t\r\u\e ]]
+ sleep 10
+ [[ true == \t\r\u\e ]]
+ set +x
Waiting for cluster readyness
+ wait_cluster_consistency some-name
+ local cluster_name=some-name
+ local wait_time=32
+ retry=0
+ sleep 7
+ echo -n 'waiting for cluster readyness'
waiting for cluster readyness++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.dHYnV75K6T
+++ mktemp
++ local LAST_ERR=/tmp/tmp.AnkR6KWxJv
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.dHYnV75K6T
++ cat /tmp/tmp.AnkR6KWxJv
++ rm /tmp/tmp.dHYnV75K6T /tmp/tmp.AnkR6KWxJv
++ return 0
+ [[ ready == \r\e\a\d\y ]]
+ echo

+ desc 'expose and delete cluster then recreate'
+ set +o xtrace


-----------------------------------------------------------------------------------
expose and delete cluster then recreate
-----------------------------------------------------------------------------------

+ kubectl_bin patch psmdb some-name --type=json --patch '[
		{
			"op": "replace",
			"path": "/spec/replsets/0/expose",
			"value": {
				"enabled": true,
				"type" : "ClusterIP"
			}
		},
		{
			"op": "replace",
			"path": "/spec/replsets/1/expose",
			"value": {
				"enabled": true,
				"type" : "ClusterIP"
			}
		},
	]'
++ mktemp
+ local LAST_OUT=/tmp/tmp.G7v0LARXRA
++ mktemp
+ local LAST_ERR=/tmp/tmp.ruNouNipQs
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl patch psmdb some-name --type=json --patch '[
		{
			"op": "replace",
			"path": "/spec/replsets/0/expose",
			"value": {
				"enabled": true,
				"type" : "ClusterIP"
			}
		},
		{
			"op": "replace",
			"path": "/spec/replsets/1/expose",
			"value": {
				"enabled": true,
				"type" : "ClusterIP"
			}
		},
	]'
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.G7v0LARXRA
perconaservermongodb.psmdb.percona.com/some-name patched
+ cat /tmp/tmp.ruNouNipQs
+ rm /tmp/tmp.G7v0LARXRA /tmp/tmp.ruNouNipQs
+ return 0
+ kubectl delete psmdb some-name
perconaservermongodb.psmdb.percona.com "some-name" deleted from recover-no-primary-9198 namespace
+ set +e
+ echo -n 'waiting for psmdb/some-name to be deleted'
waiting for psmdb/some-name to be deleted+ kubectl get psmdb some-name
Error from server (NotFound): perconaservermongodbs.psmdb.percona.com "some-name" not found
+ set -e
+ apply_cluster /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2036/e2e-tests/recover-no-primary/conf/some-name-sharded.yml
+ '[' -z '' ']'
+ cat_config /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2036/e2e-tests/recover-no-primary/conf/some-name-sharded.yml
+ kubectl_bin apply -f -
+ cat /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2036/e2e-tests/recover-no-primary/conf/some-name-sharded.yml
++ mktemp
+ yq eval '(.spec | select(.image == null)).image = "perconalab/percona-server-mongodb-operator:main-mongod7.0"'
+ yq eval '(.spec | select(has("pmm"))).pmm.image = "percona/pmm-client:2.44.1-1"'
+ yq eval '(.spec | select(has("initImage"))).initImage = "perconalab/percona-server-mongodb-operator:PR-2036-6a5cff62"'
+ yq eval '(.spec | select(has("backup"))).backup.image = "perconalab/percona-server-mongodb-operator:main-backup"'
+ yq eval '.spec.upgradeOptions.apply="Never"'
+ local LAST_OUT=/tmp/tmp.PKAiJSpxGO
++ mktemp
+ local LAST_ERR=/tmp/tmp.O2FYF5ivo2
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl apply -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.PKAiJSpxGO
perconaservermongodb.psmdb.percona.com/some-name created
+ cat /tmp/tmp.O2FYF5ivo2
+ rm /tmp/tmp.PKAiJSpxGO /tmp/tmp.O2FYF5ivo2
+ return 0
+ wait_for_running some-name-rs0 3
+ local name=some-name-rs0
+ let last_pod=2
+ local check_cluster_readyness=true
+ set_debug
+ [[ 1 == 1 ]]
+ set -o xtrace
+ local rs_name=rs0
+ local cluster_name=some-name
++ seq 0 2
+ for i in $(seq 0 $last_pod)
+ [[ 0 -eq 2 ]]
+ wait_pod some-name-rs0-0
+ local pod=some-name-rs0-0
+ set +o xtrace
waiting for pod/some-name-rs0-0 to be ready.OK
+ for i in $(seq 0 $last_pod)
+ [[ 1 -eq 2 ]]
+ wait_pod some-name-rs0-1
+ local pod=some-name-rs0-1
+ set +o xtrace
waiting for pod/some-name-rs0-1 to be ready.OK
+ for i in $(seq 0 $last_pod)
+ [[ 2 -eq 2 ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].arbiter.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.0yj2i8naZn
+++ mktemp
++ local LAST_ERR=/tmp/tmp.4mXEkIH5AR
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].arbiter.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.0yj2i8naZn
++ cat /tmp/tmp.4mXEkIH5AR
++ rm /tmp/tmp.0yj2i8naZn /tmp/tmp.4mXEkIH5AR
++ return 0
+ [[ '' == \t\r\u\e ]]
+ wait_pod some-name-rs0-2
+ local pod=some-name-rs0-2
+ set +o xtrace
waiting for pod/some-name-rs0-2 to be ready.OK
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].nonvoting.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.xQa8ueymhc
+++ mktemp
++ local LAST_ERR=/tmp/tmp.008ZrfZXPx
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].nonvoting.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.xQa8ueymhc
++ cat /tmp/tmp.008ZrfZXPx
++ rm /tmp/tmp.xQa8ueymhc /tmp/tmp.008ZrfZXPx
++ return 0
+ [[ '' == \t\r\u\e ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].hidden.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.oeN7cbf0hp
+++ mktemp
++ local LAST_ERR=/tmp/tmp.r929tJ7sjT
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].hidden.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.oeN7cbf0hp
++ cat /tmp/tmp.r929tJ7sjT
++ rm /tmp/tmp.oeN7cbf0hp /tmp/tmp.r929tJ7sjT
++ return 0
+ [[ '' == \t\r\u\e ]]
+ sleep 10
+ [[ true == \t\r\u\e ]]
+ set +x
Waiting for cluster readyness........................................................
+ wait_for_running some-name-rs1 3
+ local name=some-name-rs1
+ let last_pod=2
+ local check_cluster_readyness=true
+ set_debug
+ [[ 1 == 1 ]]
+ set -o xtrace
+ local rs_name=rs1
+ local cluster_name=some-name
++ seq 0 2
+ for i in $(seq 0 $last_pod)
+ [[ 0 -eq 2 ]]
+ wait_pod some-name-rs1-0
+ local pod=some-name-rs1-0
+ set +o xtrace
waiting for pod/some-name-rs1-0 to be ready.OK
+ for i in $(seq 0 $last_pod)
+ [[ 1 -eq 2 ]]
+ wait_pod some-name-rs1-1
+ local pod=some-name-rs1-1
+ set +o xtrace
waiting for pod/some-name-rs1-1 to be ready.OK
+ for i in $(seq 0 $last_pod)
+ [[ 2 -eq 2 ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs1")].arbiter.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.AbwUzBbEof
+++ mktemp
++ local LAST_ERR=/tmp/tmp.mzBy35aKU3
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs1")].arbiter.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.AbwUzBbEof
++ cat /tmp/tmp.mzBy35aKU3
++ rm /tmp/tmp.AbwUzBbEof /tmp/tmp.mzBy35aKU3
++ return 0
+ [[ '' == \t\r\u\e ]]
+ wait_pod some-name-rs1-2
+ local pod=some-name-rs1-2
+ set +o xtrace
waiting for pod/some-name-rs1-2 to be ready.OK
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs1")].nonvoting.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.ohU2wYu23l
+++ mktemp
++ local LAST_ERR=/tmp/tmp.9iU6s5pLVq
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs1")].nonvoting.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.ohU2wYu23l
++ cat /tmp/tmp.9iU6s5pLVq
++ rm /tmp/tmp.ohU2wYu23l /tmp/tmp.9iU6s5pLVq
++ return 0
+ [[ '' == \t\r\u\e ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs1")].hidden.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.kLHblcuqiE
+++ mktemp
++ local LAST_ERR=/tmp/tmp.r3zKL8KESw
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs1")].hidden.enabled}'