++ echo 'Log: /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1569/e2e-tests/logs/recover-no-primary.log' Log: /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1569/e2e-tests/logs/recover-no-primary.log ++ '[' -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1569/e2e-tests/conf/cloud-secret.yml ']' ++ SKIP_BACKUPS_TO_AWS_GCP_AZURE= ++ oc get projects ++ kubectl get nodes ++ grep '^minikube' +++ kubectl version -o json +++ jq -r .serverVersion.gitVersion +++ grep '\-eks\-' WARNING: version difference between client (1.30) and server (1.26) exceeds the supported minor version skew of +/-1 ++ '[' ']' ++ EKS=0 +++ kubectl version -o json +++ grep gke +++ jq -r .serverVersion.gitVersion WARNING: version difference between client (1.30) and server (1.26) exceeds the supported minor version skew of +/-1 ++ '[' v1.26.15-gke.1390000 ']' ++ GKE=1 +++ jq -r '.serverVersion.major + "." + .serverVersion.minor' +++ /usr/bin/sed -r 's/[^0-9.]+//g' +++ kubectl version -o json WARNING: version difference between client (1.30) and server (1.26) exceeds the supported minor version skew of +/-1 ++ KUBE_VERSION=1.26 + create_infra recover-no-primary-5707 + local ns=recover-no-primary-5707 + delete_crd + desc 'get and delete old CRDs and RBAC' + set +o xtrace ----------------------------------------------------------------------------------- get and delete old CRDs and RBAC ----------------------------------------------------------------------------------- + kubectl_bin delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1569/deploy/crd.yaml --ignore-not-found --wait=false ++ mktemp + local LAST_OUT=/tmp/tmp.X40rrtWNDH ++ mktemp + local LAST_ERR=/tmp/tmp.IgkawAv4uA + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1569/deploy/crd.yaml --ignore-not-found --wait=false + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.X40rrtWNDH customresourcedefinition.apiextensions.k8s.io "perconaservermongodbbackups.psmdb.percona.com" deleted customresourcedefinition.apiextensions.k8s.io "perconaservermongodbrestores.psmdb.percona.com" deleted customresourcedefinition.apiextensions.k8s.io "perconaservermongodbs.psmdb.percona.com" deleted + cat /tmp/tmp.IgkawAv4uA + rm /tmp/tmp.X40rrtWNDH /tmp/tmp.IgkawAv4uA + return 0 ++ yq eval .metadata.name /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1569/deploy/crd.yaml ++ grep -v '\-\-\-' + for crd_name in '$(yq eval '\''.metadata.name'\'' "${src_dir}/deploy/crd.yaml" | grep -v '\''\-\-\-'\'')' + kubectl get perconaservermongodbbackups.psmdb.percona.com --all-namespaces -o wide + xargs -L 1 sh -xc 'kubectl patch perconaservermongodbbackups.psmdb.percona.com -n $0 $1 --type=merge -p "{\"metadata\":{\"finalizers\":[]}}"' + grep -v NAMESPACE error: the server doesn't have a resource type "perconaservermongodbbackups" + kubectl patch perconaservermongodbbackups.psmdb.percona.com -n sh --type=merge -p '{"metadata":{"finalizers":[]}}' error: the server doesn't have a resource type "perconaservermongodbbackups" + : + kubectl_bin wait --for=delete crd perconaservermongodbbackups.psmdb.percona.com ++ mktemp + local LAST_OUT=/tmp/tmp.JRDxJuWiT8 ++ mktemp + local LAST_ERR=/tmp/tmp.KEN2KW4APX + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl wait --for=delete crd perconaservermongodbbackups.psmdb.percona.com + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.JRDxJuWiT8 + cat /tmp/tmp.KEN2KW4APX + rm /tmp/tmp.JRDxJuWiT8 /tmp/tmp.KEN2KW4APX + return 0 + for crd_name in '$(yq eval '\''.metadata.name'\'' "${src_dir}/deploy/crd.yaml" | grep -v '\''\-\-\-'\'')' + xargs -L 1 sh -xc 'kubectl patch perconaservermongodbrestores.psmdb.percona.com -n $0 $1 --type=merge -p "{\"metadata\":{\"finalizers\":[]}}"' + grep -v NAMESPACE + kubectl get perconaservermongodbrestores.psmdb.percona.com --all-namespaces -o wide error: the server doesn't have a resource type "perconaservermongodbrestores" + kubectl patch perconaservermongodbrestores.psmdb.percona.com -n sh --type=merge -p '{"metadata":{"finalizers":[]}}' error: the server doesn't have a resource type "perconaservermongodbrestores" + : + kubectl_bin wait --for=delete crd perconaservermongodbrestores.psmdb.percona.com ++ mktemp + local LAST_OUT=/tmp/tmp.MHy5QkAimx ++ mktemp + local LAST_ERR=/tmp/tmp.HEWyCGSuje + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl wait --for=delete crd perconaservermongodbrestores.psmdb.percona.com + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.MHy5QkAimx + cat /tmp/tmp.HEWyCGSuje + rm /tmp/tmp.MHy5QkAimx /tmp/tmp.HEWyCGSuje + return 0 + for crd_name in '$(yq eval '\''.metadata.name'\'' "${src_dir}/deploy/crd.yaml" | grep -v '\''\-\-\-'\'')' + kubectl get perconaservermongodbs.psmdb.percona.com --all-namespaces -o wide + xargs -L 1 sh -xc 'kubectl patch perconaservermongodbs.psmdb.percona.com -n $0 $1 --type=merge -p "{\"metadata\":{\"finalizers\":[]}}"' + grep -v NAMESPACE error: the server doesn't have a resource type "perconaservermongodbs" + kubectl patch perconaservermongodbs.psmdb.percona.com -n sh --type=merge -p '{"metadata":{"finalizers":[]}}' error: the server doesn't have a resource type "perconaservermongodbs" + : + kubectl_bin wait --for=delete crd perconaservermongodbs.psmdb.percona.com ++ mktemp + local LAST_OUT=/tmp/tmp.vFRhrGjvxN ++ mktemp + local LAST_ERR=/tmp/tmp.HDNxO4XocV + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl wait --for=delete crd perconaservermongodbs.psmdb.percona.com + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.vFRhrGjvxN + cat /tmp/tmp.HDNxO4XocV + rm /tmp/tmp.vFRhrGjvxN /tmp/tmp.HDNxO4XocV + return 0 + local rbac_yaml=rbac.yaml + '[' -n psmdb-operator ']' + rbac_yaml=cw-rbac.yaml + kubectl_bin delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1569/deploy/cw-rbac.yaml --ignore-not-found ++ mktemp + local LAST_OUT=/tmp/tmp.a3EZo2ZIy0 ++ mktemp + local LAST_ERR=/tmp/tmp.pWKgeKubfG + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1569/deploy/cw-rbac.yaml --ignore-not-found + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.a3EZo2ZIy0 clusterrole.rbac.authorization.k8s.io "percona-server-mongodb-operator" deleted clusterrolebinding.rbac.authorization.k8s.io "service-account-percona-server-mongodb-operator" deleted + cat /tmp/tmp.pWKgeKubfG + rm /tmp/tmp.a3EZo2ZIy0 /tmp/tmp.pWKgeKubfG + return 0 + check_crd_for_deletion PR-1569-eb818ce7 + local git_tag=PR-1569-eb818ce7 ++ /usr/bin/sed ':a;N;$!ba;s/\n/ /g' ++ curl -s https://raw.githubusercontent.com/percona/percona-server-mongodb-operator/PR-1569-eb818ce7/deploy/crd.yaml ++ yq eval .metadata.name ++ /usr/bin/sed s/---//g + for crd_name in '$(curl -s https://raw.githubusercontent.com/percona/percona-server-mongodb-operator/${git_tag}/deploy/crd.yaml | yq eval '\''.metadata.name'\'' | $sed '\''s/---//g'\'' | $sed '\'':a;N;$!ba;s/\n/ /g'\'')' ++ kubectl_bin get crd/null -o 'jsonpath={.status.conditions[-1].type}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.f5EIsSiQJL +++ mktemp ++ local LAST_ERR=/tmp/tmp.VAk5AWKNIi ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get crd/null -o 'jsonpath={.status.conditions[-1].type}' ++ exit_status=1 ++ set -e ++ '[' 1 '!=' 0 -a -n 1 ']' ++ cat /tmp/tmp.f5EIsSiQJL ++ cat /tmp/tmp.VAk5AWKNIi Error from server (NotFound): customresourcedefinitions.apiextensions.k8s.io "null" not found ++ sleep 0 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get crd/null -o 'jsonpath={.status.conditions[-1].type}' ++ exit_status=1 ++ set -e ++ '[' 1 '!=' 0 -a -n 1 ']' ++ cat /tmp/tmp.f5EIsSiQJL ++ cat /tmp/tmp.VAk5AWKNIi Error from server (NotFound): customresourcedefinitions.apiextensions.k8s.io "null" not found ++ sleep 4 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get crd/null -o 'jsonpath={.status.conditions[-1].type}' ++ exit_status=1 ++ set -e ++ '[' 1 '!=' 0 -a -n 1 ']' ++ cat /tmp/tmp.f5EIsSiQJL ++ cat /tmp/tmp.VAk5AWKNIi Error from server (NotFound): customresourcedefinitions.apiextensions.k8s.io "null" not found ++ sleep 8 ++ cat /tmp/tmp.f5EIsSiQJL ++ cat /tmp/tmp.VAk5AWKNIi Error from server (NotFound): customresourcedefinitions.apiextensions.k8s.io "null" not found ++ rm /tmp/tmp.f5EIsSiQJL /tmp/tmp.VAk5AWKNIi ++ return 1 + [[ '' == \T\e\r\m\i\n\a\t\i\n\g ]] + '[' -n psmdb-operator ']' + create_namespace psmdb-operator + local namespace=psmdb-operator + local skip_clean_namespace= + [[ 1 == 1 ]] + [[ -z '' ]] + destroy_chaos_mesh ++ helm list --all-namespaces --filter chaos-mesh ++ tail -n1 ++ sed s/NAMESPACE// ++ awk '-F ' '{print $2}' + local chaos_mesh_ns= + desc 'destroy chaos-mesh' + set +o xtrace ----------------------------------------------------------------------------------- destroy chaos-mesh ----------------------------------------------------------------------------------- + '[' -n '' ']' ++ kubectl get MutatingWebhookConfiguration ++ awk '{print $1}' ++ grep chaos-mesh + timeout 30 kubectl delete MutatingWebhookConfiguration error: resource(s) were provided, but no name was specified + : ++ awk '{print $1}' ++ grep chaos-mesh ++ kubectl get ValidatingWebhookConfiguration + timeout 30 kubectl delete ValidatingWebhookConfiguration error: resource(s) were provided, but no name was specified + : ++ awk '{print $1}' ++ kubectl get ValidatingWebhookConfiguration ++ grep validate-auth + timeout 30 kubectl delete ValidatingWebhookConfiguration error: resource(s) were provided, but no name was specified + : ++ kubectl api-resources ++ grep chaos-mesh ++ awk '{print $1}' ++ kubectl get crd ++ grep chaos-mesh.org ++ awk '{print $1}' + timeout 30 kubectl delete crd error: resource(s) were provided, but no name was specified + : ++ grep chaos-mesh ++ awk '{print $1}' ++ kubectl get clusterrolebinding + timeout 30 kubectl delete clusterrolebinding error: resource(s) were provided, but no name was specified + : ++ kubectl get clusterrole ++ grep chaos-mesh ++ awk '{print $1}' + timeout 30 kubectl delete clusterrole error: resource(s) were provided, but no name was specified + : + desc 'cleaned up all old namespaces' + set +o xtrace ----------------------------------------------------------------------------------- cleaned up all old namespaces ----------------------------------------------------------------------------------- + '[' -n '' ']' + desc 'cleaned up old namespaces psmdb-operator' + set +o xtrace ----------------------------------------------------------------------------------- cleaned up old namespaces psmdb-operator ----------------------------------------------------------------------------------- + kubectl_bin delete namespace psmdb-operator --ignore-not-found + egrep -v '^kube-|^default|Terminating|psmdb-operator|openshift|gke-mcs|^NAME' ++ mktemp + local LAST_OUT=/tmp/tmp.L50GmCgN9s + xargs kubectl delete ns + kubectl_bin get ns + awk '{print$1}' ++ mktemp + local LAST_ERR=/tmp/tmp.3w0VenqoYu + local exit_status=0 + local timeout=4 ++ mktemp ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl delete namespace psmdb-operator --ignore-not-found + local LAST_OUT=/tmp/tmp.89Dwmg4Bsu ++ mktemp + local LAST_ERR=/tmp/tmp.m6w0kfZvXp + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl get ns + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.89Dwmg4Bsu + cat /tmp/tmp.m6w0kfZvXp + rm /tmp/tmp.89Dwmg4Bsu /tmp/tmp.m6w0kfZvXp + return 0 namespace "recover-no-primary-18169" deleted + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.L50GmCgN9s namespace "psmdb-operator" deleted + cat /tmp/tmp.3w0VenqoYu + rm /tmp/tmp.L50GmCgN9s /tmp/tmp.3w0VenqoYu + return 0 + kubectl_bin wait --for=delete namespace psmdb-operator ++ mktemp + local LAST_OUT=/tmp/tmp.dgGV0onZww ++ mktemp + local LAST_ERR=/tmp/tmp.jl2dn4ziem + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl wait --for=delete namespace psmdb-operator + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.dgGV0onZww + cat /tmp/tmp.jl2dn4ziem + rm /tmp/tmp.dgGV0onZww /tmp/tmp.jl2dn4ziem + return 0 + desc 'create namespace psmdb-operator' + set +o xtrace ----------------------------------------------------------------------------------- create namespace psmdb-operator ----------------------------------------------------------------------------------- + kubectl_bin create namespace psmdb-operator ++ mktemp + local LAST_OUT=/tmp/tmp.7gFdS1ZBQT ++ mktemp + local LAST_ERR=/tmp/tmp.h6sOwQlKih + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl create namespace psmdb-operator + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.7gFdS1ZBQT namespace/psmdb-operator created + cat /tmp/tmp.h6sOwQlKih + rm /tmp/tmp.7gFdS1ZBQT /tmp/tmp.h6sOwQlKih + return 0 ++ kubectl_bin config current-context +++ mktemp ++ local LAST_OUT=/tmp/tmp.iXrXgMmGUU +++ mktemp ++ local LAST_ERR=/tmp/tmp.kmDVOWh3FU ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl config current-context ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.iXrXgMmGUU ++ cat /tmp/tmp.kmDVOWh3FU ++ rm /tmp/tmp.iXrXgMmGUU /tmp/tmp.kmDVOWh3FU ++ return 0 + kubectl_bin config set-context gke_cloud-dev-112233_us-central1-a_jen-psmdb-1569-eb818ce7-1-cluster5 --namespace=psmdb-operator ++ mktemp + local LAST_OUT=/tmp/tmp.X80h7KeSYS ++ mktemp + local LAST_ERR=/tmp/tmp.FVVGG4ts0k + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl config set-context gke_cloud-dev-112233_us-central1-a_jen-psmdb-1569-eb818ce7-1-cluster5 --namespace=psmdb-operator + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.X80h7KeSYS Context "gke_cloud-dev-112233_us-central1-a_jen-psmdb-1569-eb818ce7-1-cluster5" modified. + cat /tmp/tmp.FVVGG4ts0k + rm /tmp/tmp.X80h7KeSYS /tmp/tmp.FVVGG4ts0k + return 0 + deploy_operator + desc 'start PSMDB operator' + set +o xtrace ----------------------------------------------------------------------------------- start PSMDB operator ----------------------------------------------------------------------------------- + local cr_file + '[' -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1569/e2e-tests/recover-no-primary/conf/crd.yaml ']' + cr_file=/mnt/jenkins/workspace/cloud-psmdb-operator_PR-1569/deploy/crd.yaml + kubectl_bin apply --server-side --force-conflicts -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1569/deploy/crd.yaml ++ mktemp + local LAST_OUT=/tmp/tmp.ozyf0CNnGf ++ mktemp + local LAST_ERR=/tmp/tmp.Ny8zhXSHF8 + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply --server-side --force-conflicts -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1569/deploy/crd.yaml + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.ozyf0CNnGf customresourcedefinition.apiextensions.k8s.io/perconaservermongodbbackups.psmdb.percona.com serverside-applied customresourcedefinition.apiextensions.k8s.io/perconaservermongodbrestores.psmdb.percona.com serverside-applied customresourcedefinition.apiextensions.k8s.io/perconaservermongodbs.psmdb.percona.com serverside-applied + cat /tmp/tmp.Ny8zhXSHF8 + rm /tmp/tmp.ozyf0CNnGf /tmp/tmp.Ny8zhXSHF8 + return 0 + '[' -n psmdb-operator ']' + apply_rbac cw-rbac + local operator_namespace=psmdb-operator + local rbac=cw-rbac + kubectl_bin apply -n psmdb-operator -f - + cat /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1569/deploy/cw-rbac.yaml + sed -e 's^namespace: .*^namespace: psmdb-operator^' ++ mktemp + local LAST_OUT=/tmp/tmp.dG9ZBa0WVr ++ mktemp + local LAST_ERR=/tmp/tmp.9eW1MOvvKJ + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -n psmdb-operator -f - + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.dG9ZBa0WVr clusterrole.rbac.authorization.k8s.io/percona-server-mongodb-operator created serviceaccount/percona-server-mongodb-operator created clusterrolebinding.rbac.authorization.k8s.io/service-account-percona-server-mongodb-operator created + cat /tmp/tmp.9eW1MOvvKJ + rm /tmp/tmp.dG9ZBa0WVr /tmp/tmp.9eW1MOvvKJ + return 0 + kubectl_bin apply -f - ++ mktemp + yq eval ' (.spec.template.spec.containers[].image = "perconalab/percona-server-mongodb-operator:PR-1569-eb818ce7") | ((.. | select(.[] == "DISABLE_TELEMETRY")) |= .value="true") | ((.. | select(.[] == "LOG_LEVEL")) |= .value="DEBUG")' /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1569/deploy/cw-operator.yaml + local LAST_OUT=/tmp/tmp.9yAKpIiOP6 ++ mktemp + local LAST_ERR=/tmp/tmp.unrZy3k098 + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.9yAKpIiOP6 deployment.apps/percona-server-mongodb-operator created + cat /tmp/tmp.unrZy3k098 + rm /tmp/tmp.9yAKpIiOP6 /tmp/tmp.unrZy3k098 + return 0 + sleep 2 ++ get_operator_pod ++ kubectl_bin get pods --selector=name=percona-server-mongodb-operator -o 'jsonpath={.items[].metadata.name}' -n psmdb-operator +++ mktemp ++ local LAST_OUT=/tmp/tmp.xWWdVu1OD2 +++ mktemp ++ local LAST_ERR=/tmp/tmp.xumG73HJyX ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=percona-server-mongodb-operator -o 'jsonpath={.items[].metadata.name}' -n psmdb-operator ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.xWWdVu1OD2 ++ cat /tmp/tmp.xumG73HJyX ++ rm /tmp/tmp.xWWdVu1OD2 /tmp/tmp.xumG73HJyX ++ return 0 + wait_pod percona-server-mongodb-operator-8d99b8f59-29zlf + local pod=percona-server-mongodb-operator-8d99b8f59-29zlf + set +o xtrace waiting for pod/percona-server-mongodb-operator-8d99b8f59-29zlf to be ready.OK + create_namespace recover-no-primary-5707 + local namespace=recover-no-primary-5707 + local skip_clean_namespace= + [[ 1 == 1 ]] + [[ -z '' ]] + destroy_chaos_mesh ++ helm list --all-namespaces --filter chaos-mesh ++ tail -n1 ++ awk '-F ' '{print $2}' ++ sed s/NAMESPACE// + local chaos_mesh_ns= + desc 'destroy chaos-mesh' + set +o xtrace ----------------------------------------------------------------------------------- destroy chaos-mesh ----------------------------------------------------------------------------------- + '[' -n '' ']' ++ awk '{print $1}' ++ grep chaos-mesh ++ kubectl get MutatingWebhookConfiguration + timeout 30 kubectl delete MutatingWebhookConfiguration error: resource(s) were provided, but no name was specified + : ++ grep chaos-mesh ++ awk '{print $1}' ++ kubectl get ValidatingWebhookConfiguration + timeout 30 kubectl delete ValidatingWebhookConfiguration error: resource(s) were provided, but no name was specified + : ++ grep validate-auth ++ awk '{print $1}' ++ kubectl get ValidatingWebhookConfiguration + timeout 30 kubectl delete ValidatingWebhookConfiguration error: resource(s) were provided, but no name was specified + : ++ awk '{print $1}' ++ grep chaos-mesh ++ kubectl api-resources ++ grep chaos-mesh.org ++ kubectl get crd ++ awk '{print $1}' + timeout 30 kubectl delete crd error: resource(s) were provided, but no name was specified + : ++ kubectl get clusterrolebinding ++ awk '{print $1}' ++ grep chaos-mesh + timeout 30 kubectl delete clusterrolebinding error: resource(s) were provided, but no name was specified + : ++ grep chaos-mesh ++ awk '{print $1}' ++ kubectl get clusterrole + timeout 30 kubectl delete clusterrole error: resource(s) were provided, but no name was specified + : + desc 'cleaned up all old namespaces' + set +o xtrace ----------------------------------------------------------------------------------- cleaned up all old namespaces ----------------------------------------------------------------------------------- + '[' -n '' ']' + desc 'cleaned up old namespaces recover-no-primary-5707' + set +o xtrace ----------------------------------------------------------------------------------- cleaned up old namespaces recover-no-primary-5707 ----------------------------------------------------------------------------------- + kubectl_bin delete namespace recover-no-primary-5707 --ignore-not-found + xargs kubectl delete ns ++ mktemp + awk '{print$1}' + local LAST_OUT=/tmp/tmp.jQapH2fmS3 + egrep -v '^kube-|^default|Terminating|psmdb-operator|openshift|gke-mcs|^NAME' + kubectl_bin get ns ++ mktemp ++ mktemp + local LAST_OUT=/tmp/tmp.nBePg7BeeK + local LAST_ERR=/tmp/tmp.vKB5OFAtM7 + local exit_status=0 + local timeout=4 ++ mktemp + local LAST_ERR=/tmp/tmp.ksGgzf7eKA + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl get ns ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl delete namespace recover-no-primary-5707 --ignore-not-found + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.jQapH2fmS3 + cat /tmp/tmp.vKB5OFAtM7 + rm /tmp/tmp.jQapH2fmS3 /tmp/tmp.vKB5OFAtM7 + return 0 + kubectl_bin wait --for=delete namespace recover-no-primary-5707 ++ mktemp + local LAST_OUT=/tmp/tmp.WkBxyF6XHO ++ mktemp + local LAST_ERR=/tmp/tmp.L0WCE9Mjc2 + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl wait --for=delete namespace recover-no-primary-5707 + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.nBePg7BeeK + cat /tmp/tmp.ksGgzf7eKA + rm /tmp/tmp.nBePg7BeeK /tmp/tmp.ksGgzf7eKA + return 0 error: resource(s) were provided, but no name was specified + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.WkBxyF6XHO + cat /tmp/tmp.L0WCE9Mjc2 + rm /tmp/tmp.WkBxyF6XHO /tmp/tmp.L0WCE9Mjc2 + return 0 + desc 'create namespace recover-no-primary-5707' + set +o xtrace ----------------------------------------------------------------------------------- create namespace recover-no-primary-5707 ----------------------------------------------------------------------------------- + kubectl_bin create namespace recover-no-primary-5707 ++ mktemp + local LAST_OUT=/tmp/tmp.BSuBA0gVbj ++ mktemp + local LAST_ERR=/tmp/tmp.0vjiNtYwX0 + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl create namespace recover-no-primary-5707 + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.BSuBA0gVbj namespace/recover-no-primary-5707 created + cat /tmp/tmp.0vjiNtYwX0 + rm /tmp/tmp.BSuBA0gVbj /tmp/tmp.0vjiNtYwX0 + return 0 ++ kubectl_bin config current-context +++ mktemp ++ local LAST_OUT=/tmp/tmp.hoOUcBbyFd +++ mktemp ++ local LAST_ERR=/tmp/tmp.p38C1xDwMq ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl config current-context ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.hoOUcBbyFd ++ cat /tmp/tmp.p38C1xDwMq ++ rm /tmp/tmp.hoOUcBbyFd /tmp/tmp.p38C1xDwMq ++ return 0 + kubectl_bin config set-context gke_cloud-dev-112233_us-central1-a_jen-psmdb-1569-eb818ce7-1-cluster5 --namespace=recover-no-primary-5707 ++ mktemp + local LAST_OUT=/tmp/tmp.1ch7OvdLq8 ++ mktemp + local LAST_ERR=/tmp/tmp.Wr4Esxd5HS + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl config set-context gke_cloud-dev-112233_us-central1-a_jen-psmdb-1569-eb818ce7-1-cluster5 --namespace=recover-no-primary-5707 + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.1ch7OvdLq8 Context "gke_cloud-dev-112233_us-central1-a_jen-psmdb-1569-eb818ce7-1-cluster5" modified. + cat /tmp/tmp.Wr4Esxd5HS + rm /tmp/tmp.1ch7OvdLq8 /tmp/tmp.Wr4Esxd5HS + return 0 + cluster=some-name + kubectl_bin apply -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1569/e2e-tests/conf/secrets_with_tls.yml -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1569/e2e-tests/conf/client.yml ++ mktemp + local LAST_OUT=/tmp/tmp.slaqGLbKeQ ++ mktemp + local LAST_ERR=/tmp/tmp.JNN4zmNpYX + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1569/e2e-tests/conf/secrets_with_tls.yml -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1569/e2e-tests/conf/client.yml + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.slaqGLbKeQ secret/some-users created secret/some-name-ssl created secret/some-name-ssl-internal created deployment.apps/psmdb-client created + cat /tmp/tmp.JNN4zmNpYX + rm /tmp/tmp.slaqGLbKeQ /tmp/tmp.JNN4zmNpYX + return 0 + desc 'testing unexposed single replset cluster' + set +o xtrace ----------------------------------------------------------------------------------- testing unexposed single replset cluster ----------------------------------------------------------------------------------- + test_single_replset + apply_cluster /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1569/e2e-tests/recover-no-primary/conf/some-name.yml + '[' -z '' ']' + kubectl_bin apply -f - + cat_config /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1569/e2e-tests/recover-no-primary/conf/some-name.yml + yq eval '(.spec | select(has("initImage"))).initImage = "perconalab/percona-server-mongodb-operator:PR-1569-eb818ce7"' + yq eval '(.spec | select(has("pmm"))).pmm.image = "perconalab/pmm-client:dev-latest"' + cat /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1569/e2e-tests/recover-no-primary/conf/some-name.yml + yq eval '.spec.upgradeOptions.apply="Never"' + yq eval '(.spec | select(has("backup"))).backup.image = "perconalab/percona-server-mongodb-operator:main-backup"' ++ mktemp + local LAST_OUT=/tmp/tmp.Y6BmakDTYV + yq eval '(.spec | select(.image == null)).image = "perconalab/percona-server-mongodb-operator:main-mongod7.0"' ++ mktemp + local LAST_ERR=/tmp/tmp.hkrOEItjHv + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.Y6BmakDTYV perconaservermongodb.psmdb.percona.com/some-name created + cat /tmp/tmp.hkrOEItjHv + rm /tmp/tmp.Y6BmakDTYV /tmp/tmp.hkrOEItjHv + return 0 + wait_for_running some-name-rs0 3 + local name=some-name-rs0 + let last_pod=2 + local check_cluster_readyness=true + set_debug + [[ 1 == 1 ]] + set -o xtrace + local rs_name=rs0 + local cluster_name=some-name ++ seq 0 2 + for i in '$(seq 0 $last_pod)' + [[ 0 -eq 2 ]] + wait_pod some-name-rs0-0 + local pod=some-name-rs0-0 + set +o xtrace waiting for pod/some-name-rs0-0 to be ready..............OK + for i in '$(seq 0 $last_pod)' + [[ 1 -eq 2 ]] + wait_pod some-name-rs0-1 + local pod=some-name-rs0-1 + set +o xtrace waiting for pod/some-name-rs0-1 to be ready.............OK + for i in '$(seq 0 $last_pod)' + [[ 2 -eq 2 ]] ++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].arbiter.enabled}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.YMzKJQNX47 +++ mktemp ++ local LAST_ERR=/tmp/tmp.Ctvb5U5A3g ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].arbiter.enabled}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.YMzKJQNX47 ++ cat /tmp/tmp.Ctvb5U5A3g ++ rm /tmp/tmp.YMzKJQNX47 /tmp/tmp.Ctvb5U5A3g ++ return 0 + [[ '' == \t\r\u\e ]] + wait_pod some-name-rs0-2 + local pod=some-name-rs0-2 + set +o xtrace waiting for pod/some-name-rs0-2 to be ready...........OK ++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].non_voting.enabled}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.e1qj78cXiY +++ mktemp ++ local LAST_ERR=/tmp/tmp.EMtcwYFIGx ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].non_voting.enabled}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.e1qj78cXiY ++ cat /tmp/tmp.EMtcwYFIGx ++ rm /tmp/tmp.e1qj78cXiY /tmp/tmp.EMtcwYFIGx ++ return 0 + [[ '' == \t\r\u\e ]] + sleep 10 + [[ true == \t\r\u\e ]] + set +x Waiting for cluster readyness + wait_cluster_consistency some-name + local cluster_name=some-name + local wait_time=32 + retry=0 + sleep 7 + echo -n 'waiting for cluster readyness' waiting for cluster readyness++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.z8b28VoKxO +++ mktemp ++ local LAST_ERR=/tmp/tmp.c8EOaPlkDd ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get psmdb some-name -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.z8b28VoKxO ++ cat /tmp/tmp.c8EOaPlkDd ++ rm /tmp/tmp.z8b28VoKxO /tmp/tmp.c8EOaPlkDd ++ return 0 + [[ ready == \r\e\a\d\y ]] + desc 'kill pods abruptly and wait for recovery' + set +o xtrace ----------------------------------------------------------------------------------- kill pods abruptly and wait for recovery ----------------------------------------------------------------------------------- ++ seq 0 2 + for i in '$(seq 0 2)' + kubectl_bin exec some-name-rs0-0 -c mongod -- kill 1 ++ mktemp + local LAST_OUT=/tmp/tmp.q8RkgR34bY ++ mktemp + local LAST_ERR=/tmp/tmp.bcTvOUg3h2 + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl exec some-name-rs0-0 -c mongod -- kill 1 + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.q8RkgR34bY + cat /tmp/tmp.bcTvOUg3h2 + rm /tmp/tmp.q8RkgR34bY /tmp/tmp.bcTvOUg3h2 + return 0 + for i in '$(seq 0 2)' + kubectl_bin exec some-name-rs0-1 -c mongod -- kill 1 ++ mktemp + local LAST_OUT=/tmp/tmp.M2OAR27eEv ++ mktemp + local LAST_ERR=/tmp/tmp.dstH8LCqRC + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl exec some-name-rs0-1 -c mongod -- kill 1 + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.M2OAR27eEv + cat /tmp/tmp.dstH8LCqRC + rm /tmp/tmp.M2OAR27eEv /tmp/tmp.dstH8LCqRC + return 0 + for i in '$(seq 0 2)' + kubectl_bin exec some-name-rs0-2 -c mongod -- kill 1 ++ mktemp + local LAST_OUT=/tmp/tmp.XcPsZFsFff ++ mktemp + local LAST_ERR=/tmp/tmp.YtMvKe1mvz + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl exec some-name-rs0-2 -c mongod -- kill 1 + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.XcPsZFsFff + cat /tmp/tmp.YtMvKe1mvz + rm /tmp/tmp.XcPsZFsFff /tmp/tmp.YtMvKe1mvz + return 0 + sleep 10 + wait_for_running some-name-rs0 3 + local name=some-name-rs0 + let last_pod=2 + local check_cluster_readyness=true + set_debug + [[ 1 == 1 ]] + set -o xtrace + local rs_name=rs0 + local cluster_name=some-name ++ seq 0 2 + for i in '$(seq 0 $last_pod)' + [[ 0 -eq 2 ]] + wait_pod some-name-rs0-0 + local pod=some-name-rs0-0 + set +o xtrace waiting for pod/some-name-rs0-0 to be ready.OK + for i in '$(seq 0 $last_pod)' + [[ 1 -eq 2 ]] + wait_pod some-name-rs0-1 + local pod=some-name-rs0-1 + set +o xtrace waiting for pod/some-name-rs0-1 to be ready.OK + for i in '$(seq 0 $last_pod)' + [[ 2 -eq 2 ]] ++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].arbiter.enabled}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.cL7FIJABzO +++ mktemp ++ local LAST_ERR=/tmp/tmp.WIx3bX0Exv ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].arbiter.enabled}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.cL7FIJABzO ++ cat /tmp/tmp.WIx3bX0Exv ++ rm /tmp/tmp.cL7FIJABzO /tmp/tmp.WIx3bX0Exv ++ return 0 + [[ '' == \t\r\u\e ]] + wait_pod some-name-rs0-2 + local pod=some-name-rs0-2 + set +o xtrace waiting for pod/some-name-rs0-2 to be ready.OK ++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].non_voting.enabled}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.SLtni1WFMf +++ mktemp ++ local LAST_ERR=/tmp/tmp.o09a5JjTge ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].non_voting.enabled}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.SLtni1WFMf ++ cat /tmp/tmp.o09a5JjTge ++ rm /tmp/tmp.SLtni1WFMf /tmp/tmp.o09a5JjTge ++ return 0 + [[ '' == \t\r\u\e ]] + sleep 10 + [[ true == \t\r\u\e ]] + set +x Waiting for cluster readyness... + wait_cluster_consistency some-name + local cluster_name=some-name + local wait_time=32 + retry=0 + sleep 7 + echo -n 'waiting for cluster readyness' waiting for cluster readyness++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.sWsboWHofj +++ mktemp ++ local LAST_ERR=/tmp/tmp.WCkQwdXU3K ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get psmdb some-name -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.sWsboWHofj ++ cat /tmp/tmp.WCkQwdXU3K ++ rm /tmp/tmp.sWsboWHofj /tmp/tmp.WCkQwdXU3K ++ return 0 + [[ ready == \r\e\a\d\y ]] + desc 'delete statefulset and wait for recovery' + set +o xtrace ----------------------------------------------------------------------------------- delete statefulset and wait for recovery ----------------------------------------------------------------------------------- + kubectl delete sts some-name-rs0 statefulset.apps "some-name-rs0" deleted + sleep 5 + wait_for_running some-name-rs0 3 + local name=some-name-rs0 + let last_pod=2 + local check_cluster_readyness=true + set_debug + [[ 1 == 1 ]] + set -o xtrace + local rs_name=rs0 + local cluster_name=some-name ++ seq 0 2 + for i in '$(seq 0 $last_pod)' + [[ 0 -eq 2 ]] + wait_pod some-name-rs0-0 + local pod=some-name-rs0-0 + set +o xtrace waiting for pod/some-name-rs0-0 to be ready.OK + for i in '$(seq 0 $last_pod)' + [[ 1 -eq 2 ]] + wait_pod some-name-rs0-1 + local pod=some-name-rs0-1 + set +o xtrace waiting for pod/some-name-rs0-1 to be ready.OK + for i in '$(seq 0 $last_pod)' + [[ 2 -eq 2 ]] ++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].arbiter.enabled}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.7CpqoVHlk7 +++ mktemp ++ local LAST_ERR=/tmp/tmp.00B5veCxva ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].arbiter.enabled}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.7CpqoVHlk7 ++ cat /tmp/tmp.00B5veCxva ++ rm /tmp/tmp.7CpqoVHlk7 /tmp/tmp.00B5veCxva ++ return 0 + [[ '' == \t\r\u\e ]] + wait_pod some-name-rs0-2 + local pod=some-name-rs0-2 + set +o xtrace waiting for pod/some-name-rs0-2 to be ready.OK ++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].non_voting.enabled}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.RK4QK65H6V +++ mktemp ++ local LAST_ERR=/tmp/tmp.hsXZmaro5d ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].non_voting.enabled}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.RK4QK65H6V ++ cat /tmp/tmp.hsXZmaro5d ++ rm /tmp/tmp.RK4QK65H6V /tmp/tmp.hsXZmaro5d ++ return 0 + [[ '' == \t\r\u\e ]] + sleep 10 + [[ true == \t\r\u\e ]] + set +x Waiting for cluster readyness..................................... + wait_cluster_consistency some-name + local cluster_name=some-name + local wait_time=32 + retry=0 + sleep 7 + echo -n 'waiting for cluster readyness' waiting for cluster readyness++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.5MgbjKzGkG +++ mktemp ++ local LAST_ERR=/tmp/tmp.ZF84V2A0eM ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get psmdb some-name -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.5MgbjKzGkG ++ cat /tmp/tmp.ZF84V2A0eM ++ rm /tmp/tmp.5MgbjKzGkG /tmp/tmp.ZF84V2A0eM ++ return 0 + [[ ready == \r\e\a\d\y ]] + desc 'delete cluster and recreate' + set +o xtrace ----------------------------------------------------------------------------------- delete cluster and recreate ----------------------------------------------------------------------------------- + kubectl delete psmdb some-name perconaservermongodb.psmdb.percona.com "some-name" deleted + set +e + echo -n 'waiting for psmdb/some-name to be deleted' waiting for psmdb/some-name to be deleted+ kubectl get psmdb some-name Error from server (NotFound): perconaservermongodbs.psmdb.percona.com "some-name" not found + set -e + apply_cluster /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1569/e2e-tests/recover-no-primary/conf/some-name.yml + '[' -z '' ']' + cat_config /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1569/e2e-tests/recover-no-primary/conf/some-name.yml + kubectl_bin apply -f - ++ mktemp + local LAST_OUT=/tmp/tmp.1zYcfupXiA + yq eval '(.spec | select(has("initImage"))).initImage = "perconalab/percona-server-mongodb-operator:PR-1569-eb818ce7"' ++ mktemp + yq eval '(.spec | select(has("backup"))).backup.image = "perconalab/percona-server-mongodb-operator:main-backup"' + local LAST_ERR=/tmp/tmp.JGEjQgz4GV + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f - + yq eval '.spec.upgradeOptions.apply="Never"' + cat /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1569/e2e-tests/recover-no-primary/conf/some-name.yml + yq eval '(.spec | select(.image == null)).image = "perconalab/percona-server-mongodb-operator:main-mongod7.0"' + yq eval '(.spec | select(has("pmm"))).pmm.image = "perconalab/pmm-client:dev-latest"' + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.1zYcfupXiA perconaservermongodb.psmdb.percona.com/some-name created + cat /tmp/tmp.JGEjQgz4GV + rm /tmp/tmp.1zYcfupXiA /tmp/tmp.JGEjQgz4GV + return 0 + wait_for_running some-name-rs0 3 + local name=some-name-rs0 + let last_pod=2 + local check_cluster_readyness=true + set_debug + [[ 1 == 1 ]] + set -o xtrace + local rs_name=rs0 + local cluster_name=some-name ++ seq 0 2 + for i in '$(seq 0 $last_pod)' + [[ 0 -eq 2 ]] + wait_pod some-name-rs0-0 + local pod=some-name-rs0-0 + set +o xtrace waiting for pod/some-name-rs0-0 to be ready.OK + for i in '$(seq 0 $last_pod)' + [[ 1 -eq 2 ]] + wait_pod some-name-rs0-1 + local pod=some-name-rs0-1 + set +o xtrace waiting for pod/some-name-rs0-1 to be ready.OK + for i in '$(seq 0 $last_pod)' + [[ 2 -eq 2 ]] ++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].arbiter.enabled}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.dmwTIxNZpS +++ mktemp ++ local LAST_ERR=/tmp/tmp.BFENgaaTAc ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].arbiter.enabled}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.dmwTIxNZpS ++ cat /tmp/tmp.BFENgaaTAc ++ rm /tmp/tmp.dmwTIxNZpS /tmp/tmp.BFENgaaTAc ++ return 0 + [[ '' == \t\r\u\e ]] + wait_pod some-name-rs0-2 + local pod=some-name-rs0-2 + set +o xtrace waiting for pod/some-name-rs0-2 to be ready.OK ++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].non_voting.enabled}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.2w6Mww4TGD +++ mktemp ++ local LAST_ERR=/tmp/tmp.rxYTapPORE ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].non_voting.enabled}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.2w6Mww4TGD ++ cat /tmp/tmp.rxYTapPORE ++ rm /tmp/tmp.2w6Mww4TGD /tmp/tmp.rxYTapPORE ++ return 0 + [[ '' == \t\r\u\e ]] + sleep 10 + [[ true == \t\r\u\e ]] + set +x Waiting for cluster readyness.................................... + wait_cluster_consistency some-name + local cluster_name=some-name + local wait_time=32 + retry=0 + sleep 7 + echo -n 'waiting for cluster readyness' waiting for cluster readyness++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.t7nrvJX2w7 +++ mktemp ++ local LAST_ERR=/tmp/tmp.kTi8oZtgzp ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get psmdb some-name -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.t7nrvJX2w7 ++ cat /tmp/tmp.kTi8oZtgzp ++ rm /tmp/tmp.t7nrvJX2w7 /tmp/tmp.kTi8oZtgzp ++ return 0 + [[ ready == \r\e\a\d\y ]] + kubectl delete psmdb some-name perconaservermongodb.psmdb.percona.com "some-name" deleted + kubectl delete pvc --all persistentvolumeclaim "mongod-data-some-name-rs0-0" deleted persistentvolumeclaim "mongod-data-some-name-rs0-1" deleted persistentvolumeclaim "mongod-data-some-name-rs0-2" deleted + desc 'testing exposed single replset cluster' + set +o xtrace ----------------------------------------------------------------------------------- testing exposed single replset cluster ----------------------------------------------------------------------------------- + test_exposed_single_replset + apply_cluster /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1569/e2e-tests/recover-no-primary/conf/some-name-exposed.yml + '[' -z '' ']' + cat_config /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1569/e2e-tests/recover-no-primary/conf/some-name-exposed.yml + kubectl_bin apply -f - + cat /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1569/e2e-tests/recover-no-primary/conf/some-name-exposed.yml + yq eval '(.spec | select(.image == null)).image = "perconalab/percona-server-mongodb-operator:main-mongod7.0"' ++ mktemp + yq eval '(.spec | select(has("pmm"))).pmm.image = "perconalab/pmm-client:dev-latest"' + yq eval '(.spec | select(has("initImage"))).initImage = "perconalab/percona-server-mongodb-operator:PR-1569-eb818ce7"' + yq eval '(.spec | select(has("backup"))).backup.image = "perconalab/percona-server-mongodb-operator:main-backup"' + yq eval '.spec.upgradeOptions.apply="Never"' + local LAST_OUT=/tmp/tmp.M4w4OehZ3s ++ mktemp + local LAST_ERR=/tmp/tmp.DECSCUnAYL + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.M4w4OehZ3s perconaservermongodb.psmdb.percona.com/some-name created + cat /tmp/tmp.DECSCUnAYL + rm /tmp/tmp.M4w4OehZ3s /tmp/tmp.DECSCUnAYL + return 0 + wait_for_running some-name-rs0 3 + local name=some-name-rs0 + let last_pod=2 + local check_cluster_readyness=true + set_debug + [[ 1 == 1 ]] + set -o xtrace + local rs_name=rs0 + local cluster_name=some-name ++ seq 0 2 + for i in '$(seq 0 $last_pod)' + [[ 0 -eq 2 ]] + wait_pod some-name-rs0-0 + local pod=some-name-rs0-0 + set +o xtrace waiting for pod/some-name-rs0-0 to be ready..............OK + for i in '$(seq 0 $last_pod)' + [[ 1 -eq 2 ]] + wait_pod some-name-rs0-1 + local pod=some-name-rs0-1 + set +o xtrace waiting for pod/some-name-rs0-1 to be ready............OK + for i in '$(seq 0 $last_pod)' + [[ 2 -eq 2 ]] ++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].arbiter.enabled}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.MpUwqZD12S +++ mktemp ++ local LAST_ERR=/tmp/tmp.whT7vLof1v ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].arbiter.enabled}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.MpUwqZD12S ++ cat /tmp/tmp.whT7vLof1v ++ rm /tmp/tmp.MpUwqZD12S /tmp/tmp.whT7vLof1v ++ return 0 + [[ '' == \t\r\u\e ]] + wait_pod some-name-rs0-2 + local pod=some-name-rs0-2 + set +o xtrace waiting for pod/some-name-rs0-2 to be ready...........OK ++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].non_voting.enabled}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.VTb3qDc8vE +++ mktemp ++ local LAST_ERR=/tmp/tmp.PD5Qvr7r8V ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].non_voting.enabled}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.VTb3qDc8vE ++ cat /tmp/tmp.PD5Qvr7r8V ++ rm /tmp/tmp.VTb3qDc8vE /tmp/tmp.PD5Qvr7r8V ++ return 0 + [[ '' == \t\r\u\e ]] + sleep 10 + [[ true == \t\r\u\e ]] + set +x Waiting for cluster readyness. + wait_cluster_consistency some-name + local cluster_name=some-name + local wait_time=32 + retry=0 + sleep 7 + echo -n 'waiting for cluster readyness' waiting for cluster readyness++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.7PmEUJq9lw +++ mktemp ++ local LAST_ERR=/tmp/tmp.XvVYDrlAC2 ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get psmdb some-name -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.7PmEUJq9lw ++ cat /tmp/tmp.XvVYDrlAC2 ++ rm /tmp/tmp.7PmEUJq9lw /tmp/tmp.XvVYDrlAC2 ++ return 0 + [[ ready == \r\e\a\d\y ]] + desc 'kill pods abruptly and wait for recovery' + set +o xtrace ----------------------------------------------------------------------------------- kill pods abruptly and wait for recovery ----------------------------------------------------------------------------------- ++ seq 0 2 + for i in '$(seq 0 2)' + kubectl_bin exec some-name-rs0-0 -c mongod -- kill 1 ++ mktemp + local LAST_OUT=/tmp/tmp.xcUSXpoExE ++ mktemp + local LAST_ERR=/tmp/tmp.hZZ7Z9pkXV + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl exec some-name-rs0-0 -c mongod -- kill 1 + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.xcUSXpoExE + cat /tmp/tmp.hZZ7Z9pkXV + rm /tmp/tmp.xcUSXpoExE /tmp/tmp.hZZ7Z9pkXV + return 0 + for i in '$(seq 0 2)' + kubectl_bin exec some-name-rs0-1 -c mongod -- kill 1 ++ mktemp + local LAST_OUT=/tmp/tmp.sCGDS0GD27 ++ mktemp + local LAST_ERR=/tmp/tmp.o7FUO7TDAk + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl exec some-name-rs0-1 -c mongod -- kill 1 + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.sCGDS0GD27 + cat /tmp/tmp.o7FUO7TDAk + rm /tmp/tmp.sCGDS0GD27 /tmp/tmp.o7FUO7TDAk + return 0 + for i in '$(seq 0 2)' + kubectl_bin exec some-name-rs0-2 -c mongod -- kill 1 ++ mktemp + local LAST_OUT=/tmp/tmp.x1Qf5gU6uv ++ mktemp + local LAST_ERR=/tmp/tmp.hcEInHZH3J + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl exec some-name-rs0-2 -c mongod -- kill 1 + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.x1Qf5gU6uv + cat /tmp/tmp.hcEInHZH3J + rm /tmp/tmp.x1Qf5gU6uv /tmp/tmp.hcEInHZH3J + return 0 + sleep 10 + wait_for_running some-name-rs0 3 + local name=some-name-rs0 + let last_pod=2 + local check_cluster_readyness=true + set_debug + [[ 1 == 1 ]] + set -o xtrace + local rs_name=rs0 + local cluster_name=some-name ++ seq 0 2 + for i in '$(seq 0 $last_pod)' + [[ 0 -eq 2 ]] + wait_pod some-name-rs0-0 + local pod=some-name-rs0-0 + set +o xtrace waiting for pod/some-name-rs0-0 to be ready......OK + for i in '$(seq 0 $last_pod)' + [[ 1 -eq 2 ]] + wait_pod some-name-rs0-1 + local pod=some-name-rs0-1 + set +o xtrace waiting for pod/some-name-rs0-1 to be ready..OK + for i in '$(seq 0 $last_pod)' + [[ 2 -eq 2 ]] ++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].arbiter.enabled}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.FomfuvKAgF +++ mktemp ++ local LAST_ERR=/tmp/tmp.y2cuSaFOv4 ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].arbiter.enabled}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.FomfuvKAgF ++ cat /tmp/tmp.y2cuSaFOv4 ++ rm /tmp/tmp.FomfuvKAgF /tmp/tmp.y2cuSaFOv4 ++ return 0 + [[ '' == \t\r\u\e ]] + wait_pod some-name-rs0-2 + local pod=some-name-rs0-2 + set +o xtrace waiting for pod/some-name-rs0-2 to be ready.OK ++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].non_voting.enabled}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.Msl0TbNS6Z +++ mktemp ++ local LAST_ERR=/tmp/tmp.JJwbaoJqjz ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].non_voting.enabled}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.Msl0TbNS6Z ++ cat /tmp/tmp.JJwbaoJqjz ++ rm /tmp/tmp.Msl0TbNS6Z /tmp/tmp.JJwbaoJqjz ++ return 0 + [[ '' == \t\r\u\e ]] + sleep 10 + [[ true == \t\r\u\e ]] + set +x Waiting for cluster readyness + wait_cluster_consistency some-name + local cluster_name=some-name + local wait_time=32 + retry=0 + sleep 7 + echo -n 'waiting for cluster readyness' waiting for cluster readyness++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.IWYc1atpcm +++ mktemp ++ local LAST_ERR=/tmp/tmp.LWJYI62OGz ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get psmdb some-name -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.IWYc1atpcm ++ cat /tmp/tmp.LWJYI62OGz ++ rm /tmp/tmp.IWYc1atpcm /tmp/tmp.LWJYI62OGz ++ return 0 + [[ ready == \r\e\a\d\y ]] + desc 'delete statefulset and wait for recovery' + set +o xtrace ----------------------------------------------------------------------------------- delete statefulset and wait for recovery ----------------------------------------------------------------------------------- + kubectl delete sts some-name-rs0 statefulset.apps "some-name-rs0" deleted + sleep 5 + wait_for_running some-name-rs0 3 + local name=some-name-rs0 + let last_pod=2 + local check_cluster_readyness=true + set_debug + [[ 1 == 1 ]] + set -o xtrace + local rs_name=rs0 + local cluster_name=some-name ++ seq 0 2 + for i in '$(seq 0 $last_pod)' + [[ 0 -eq 2 ]] + wait_pod some-name-rs0-0 + local pod=some-name-rs0-0 + set +o xtrace waiting for pod/some-name-rs0-0 to be ready.OK + for i in '$(seq 0 $last_pod)' + [[ 1 -eq 2 ]] + wait_pod some-name-rs0-1 + local pod=some-name-rs0-1 + set +o xtrace waiting for pod/some-name-rs0-1 to be ready.OK + for i in '$(seq 0 $last_pod)' + [[ 2 -eq 2 ]] ++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].arbiter.enabled}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.f8NT0Ey2rA +++ mktemp ++ local LAST_ERR=/tmp/tmp.Sp0gfhcM2E ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].arbiter.enabled}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.f8NT0Ey2rA ++ cat /tmp/tmp.Sp0gfhcM2E ++ rm /tmp/tmp.f8NT0Ey2rA /tmp/tmp.Sp0gfhcM2E ++ return 0 + [[ '' == \t\r\u\e ]] + wait_pod some-name-rs0-2 + local pod=some-name-rs0-2 + set +o xtrace waiting for pod/some-name-rs0-2 to be ready.OK ++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].non_voting.enabled}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.dUyWMoy7V4 +++ mktemp ++ local LAST_ERR=/tmp/tmp.mB5D39evbm ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].non_voting.enabled}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.dUyWMoy7V4 ++ cat /tmp/tmp.mB5D39evbm ++ rm /tmp/tmp.dUyWMoy7V4 /tmp/tmp.mB5D39evbm ++ return 0 + [[ '' == \t\r\u\e ]] + sleep 10 + [[ true == \t\r\u\e ]] + set +x Waiting for cluster readyness.................................... + wait_cluster_consistency some-name + local cluster_name=some-name + local wait_time=32 + retry=0 + sleep 7 + echo -n 'waiting for cluster readyness' waiting for cluster readyness++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.eLFrrEBTTZ +++ mktemp ++ local LAST_ERR=/tmp/tmp.eAv2KiKX8Z ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get psmdb some-name -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.eLFrrEBTTZ ++ cat /tmp/tmp.eAv2KiKX8Z ++ rm /tmp/tmp.eLFrrEBTTZ /tmp/tmp.eAv2KiKX8Z ++ return 0 + [[ ready == \r\e\a\d\y ]] + desc 'delete cluster and recreate' + set +o xtrace ----------------------------------------------------------------------------------- delete cluster and recreate ----------------------------------------------------------------------------------- + kubectl delete psmdb some-name perconaservermongodb.psmdb.percona.com "some-name" deleted + set +e + echo -n 'waiting for psmdb/some-name to be deleted' waiting for psmdb/some-name to be deleted+ kubectl get psmdb some-name Error from server (NotFound): perconaservermongodbs.psmdb.percona.com "some-name" not found + set -e + apply_cluster /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1569/e2e-tests/recover-no-primary/conf/some-name-exposed.yml + '[' -z '' ']' + cat_config /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1569/e2e-tests/recover-no-primary/conf/some-name-exposed.yml + kubectl_bin apply -f - + yq eval '(.spec | select(.image == null)).image = "perconalab/percona-server-mongodb-operator:main-mongod7.0"' ++ mktemp + yq eval '(.spec | select(has("initImage"))).initImage = "perconalab/percona-server-mongodb-operator:PR-1569-eb818ce7"' + yq eval '(.spec | select(has("pmm"))).pmm.image = "perconalab/pmm-client:dev-latest"' + cat /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1569/e2e-tests/recover-no-primary/conf/some-name-exposed.yml + local LAST_OUT=/tmp/tmp.nRTdLOZ1E7 + yq eval '(.spec | select(has("backup"))).backup.image = "perconalab/percona-server-mongodb-operator:main-backup"' + yq eval '.spec.upgradeOptions.apply="Never"' ++ mktemp + local LAST_ERR=/tmp/tmp.c8u4qwmPzw + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.nRTdLOZ1E7 perconaservermongodb.psmdb.percona.com/some-name created + cat /tmp/tmp.c8u4qwmPzw + rm /tmp/tmp.nRTdLOZ1E7 /tmp/tmp.c8u4qwmPzw + return 0 + wait_for_running some-name-rs0 3 + local name=some-name-rs0 + let last_pod=2 + local check_cluster_readyness=true + set_debug + [[ 1 == 1 ]] + set -o xtrace + local rs_name=rs0 + local cluster_name=some-name ++ seq 0 2 + for i in '$(seq 0 $last_pod)' + [[ 0 -eq 2 ]] + wait_pod some-name-rs0-0 + local pod=some-name-rs0-0 + set +o xtrace waiting for pod/some-name-rs0-0 to be ready.OK + for i in '$(seq 0 $last_pod)' + [[ 1 -eq 2 ]] + wait_pod some-name-rs0-1 + local pod=some-name-rs0-1 + set +o xtrace waiting for pod/some-name-rs0-1 to be ready.OK + for i in '$(seq 0 $last_pod)' + [[ 2 -eq 2 ]] ++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].arbiter.enabled}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.OgogWF76AF +++ mktemp ++ local LAST_ERR=/tmp/tmp.dQ7qLGYhbx ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].arbiter.enabled}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.OgogWF76AF ++ cat /tmp/tmp.dQ7qLGYhbx ++ rm /tmp/tmp.OgogWF76AF /tmp/tmp.dQ7qLGYhbx ++ return 0 + [[ '' == \t\r\u\e ]] + wait_pod some-name-rs0-2 + local pod=some-name-rs0-2 + set +o xtrace waiting for pod/some-name-rs0-2 to be ready.OK ++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].non_voting.enabled}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.qnh0VshvIm +++ mktemp ++ local LAST_ERR=/tmp/tmp.QNHerQsRd1 ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].non_voting.enabled}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.qnh0VshvIm ++ cat /tmp/tmp.QNHerQsRd1 ++ rm /tmp/tmp.qnh0VshvIm /tmp/tmp.QNHerQsRd1 ++ return 0 + [[ '' == \t\r\u\e ]] + sleep 10 + [[ true == \t\r\u\e ]] + set +x Waiting for cluster readyness.................................................. + wait_cluster_consistency some-name + local cluster_name=some-name + local wait_time=32 + retry=0 + sleep 7 + echo -n 'waiting for cluster readyness' waiting for cluster readyness++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.czHswXHiK3 +++ mktemp ++ local LAST_ERR=/tmp/tmp.skxYP5mk7M ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get psmdb some-name -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.czHswXHiK3 ++ cat /tmp/tmp.skxYP5mk7M ++ rm /tmp/tmp.czHswXHiK3 /tmp/tmp.skxYP5mk7M ++ return 0 + [[ ready == \r\e\a\d\y ]] + kubectl delete psmdb some-name perconaservermongodb.psmdb.percona.com "some-name" deleted + kubectl delete pvc --all persistentvolumeclaim "mongod-data-some-name-rs0-0" deleted persistentvolumeclaim "mongod-data-some-name-rs0-1" deleted persistentvolumeclaim "mongod-data-some-name-rs0-2" deleted + desc 'testing exposed two replsets sharded cluster' + set +o xtrace ----------------------------------------------------------------------------------- testing exposed two replsets sharded cluster ----------------------------------------------------------------------------------- + test_exposed_sharded_cluster + apply_cluster /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1569/e2e-tests/recover-no-primary/conf/some-name-sharded.yml + '[' -z '' ']' + kubectl_bin apply -f - + cat_config /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1569/e2e-tests/recover-no-primary/conf/some-name-sharded.yml + cat /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1569/e2e-tests/recover-no-primary/conf/some-name-sharded.yml + yq eval '(.spec | select(.image == null)).image = "perconalab/percona-server-mongodb-operator:main-mongod7.0"' + yq eval '(.spec | select(has("pmm"))).pmm.image = "perconalab/pmm-client:dev-latest"' ++ mktemp + yq eval '(.spec | select(has("backup"))).backup.image = "perconalab/percona-server-mongodb-operator:main-backup"' + yq eval '(.spec | select(has("initImage"))).initImage = "perconalab/percona-server-mongodb-operator:PR-1569-eb818ce7"' + yq eval '.spec.upgradeOptions.apply="Never"' + local LAST_OUT=/tmp/tmp.BmxrfgBMpa ++ mktemp + local LAST_ERR=/tmp/tmp.WGAkzyE1fA + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.BmxrfgBMpa perconaservermongodb.psmdb.percona.com/some-name created + cat /tmp/tmp.WGAkzyE1fA + rm /tmp/tmp.BmxrfgBMpa /tmp/tmp.WGAkzyE1fA + return 0 + wait_for_running some-name-rs0 3 + local name=some-name-rs0 + let last_pod=2 + local check_cluster_readyness=true + set_debug + [[ 1 == 1 ]] + set -o xtrace + local rs_name=rs0 + local cluster_name=some-name ++ seq 0 2 + for i in '$(seq 0 $last_pod)' + [[ 0 -eq 2 ]] + wait_pod some-name-rs0-0 + local pod=some-name-rs0-0 + set +o xtrace waiting for pod/some-name-rs0-0 to be ready.............OK + for i in '$(seq 0 $last_pod)' + [[ 1 -eq 2 ]] + wait_pod some-name-rs0-1 + local pod=some-name-rs0-1 + set +o xtrace waiting for pod/some-name-rs0-1 to be ready...........OK + for i in '$(seq 0 $last_pod)' + [[ 2 -eq 2 ]] ++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].arbiter.enabled}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.oz8y7H0Kk9 +++ mktemp ++ local LAST_ERR=/tmp/tmp.Bw2ghZ69DF ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].arbiter.enabled}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.oz8y7H0Kk9 ++ cat /tmp/tmp.Bw2ghZ69DF ++ rm /tmp/tmp.oz8y7H0Kk9 /tmp/tmp.Bw2ghZ69DF ++ return 0 + [[ '' == \t\r\u\e ]] + wait_pod some-name-rs0-2 + local pod=some-name-rs0-2 + set +o xtrace waiting for pod/some-name-rs0-2 to be ready...........OK ++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].non_voting.enabled}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.1r9IZLqWph +++ mktemp ++ local LAST_ERR=/tmp/tmp.DxRBCim2Ro ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].non_voting.enabled}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.1r9IZLqWph ++ cat /tmp/tmp.DxRBCim2Ro ++ rm /tmp/tmp.1r9IZLqWph /tmp/tmp.DxRBCim2Ro ++ return 0 + [[ '' == \t\r\u\e ]] + sleep 10 + [[ true == \t\r\u\e ]] + set +x Waiting for cluster readyness.............................. + wait_cluster_consistency some-name + local cluster_name=some-name + local wait_time=32 + retry=0 + sleep 7 + echo -n 'waiting for cluster readyness' waiting for cluster readyness++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.2UZgk0ryp1 +++ mktemp ++ local LAST_ERR=/tmp/tmp.mSINqnVUgp ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get psmdb some-name -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.2UZgk0ryp1 ++ cat /tmp/tmp.mSINqnVUgp ++ rm /tmp/tmp.2UZgk0ryp1 /tmp/tmp.mSINqnVUgp ++ return 0 + [[ ready == \r\e\a\d\y ]] + desc 'kill pods abruptly and wait for recovery' + set +o xtrace ----------------------------------------------------------------------------------- kill pods abruptly and wait for recovery ----------------------------------------------------------------------------------- ++ seq 0 2 + for i in '$(seq 0 2)' + kubectl_bin exec some-name-rs0-0 -c mongod -- kill 1 ++ mktemp + local LAST_OUT=/tmp/tmp.Y245rbC7l6 ++ mktemp + local LAST_ERR=/tmp/tmp.MWWvqa2Erw + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl exec some-name-rs0-0 -c mongod -- kill 1 + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.Y245rbC7l6 + cat /tmp/tmp.MWWvqa2Erw + rm /tmp/tmp.Y245rbC7l6 /tmp/tmp.MWWvqa2Erw + return 0 + for i in '$(seq 0 2)' + kubectl_bin exec some-name-rs0-1 -c mongod -- kill 1 ++ mktemp + local LAST_OUT=/tmp/tmp.7XfB13g2cU ++ mktemp + local LAST_ERR=/tmp/tmp.F6jrotzYRD + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl exec some-name-rs0-1 -c mongod -- kill 1 + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.7XfB13g2cU + cat /tmp/tmp.F6jrotzYRD + rm /tmp/tmp.7XfB13g2cU /tmp/tmp.F6jrotzYRD + return 0 + for i in '$(seq 0 2)' + kubectl_bin exec some-name-rs0-2 -c mongod -- kill 1 ++ mktemp + local LAST_OUT=/tmp/tmp.xlM4Cb2TTk ++ mktemp + local LAST_ERR=/tmp/tmp.sWgUj22zSE + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl exec some-name-rs0-2 -c mongod -- kill 1 + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.xlM4Cb2TTk + cat /tmp/tmp.sWgUj22zSE + rm /tmp/tmp.xlM4Cb2TTk /tmp/tmp.sWgUj22zSE + return 0 + sleep 10 + wait_for_running some-name-rs0 3 + local name=some-name-rs0 + let last_pod=2 + local check_cluster_readyness=true + set_debug + [[ 1 == 1 ]] + set -o xtrace + local rs_name=rs0 + local cluster_name=some-name ++ seq 0 2 + for i in '$(seq 0 $last_pod)' + [[ 0 -eq 2 ]] + wait_pod some-name-rs0-0 + local pod=some-name-rs0-0 + set +o xtrace waiting for pod/some-name-rs0-0 to be ready......OK + for i in '$(seq 0 $last_pod)' + [[ 1 -eq 2 ]] + wait_pod some-name-rs0-1 + local pod=some-name-rs0-1 + set +o xtrace waiting for pod/some-name-rs0-1 to be ready...OK + for i in '$(seq 0 $last_pod)' + [[ 2 -eq 2 ]] ++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].arbiter.enabled}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.wjys8ix3iR +++ mktemp ++ local LAST_ERR=/tmp/tmp.TgtZrvZI64 ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].arbiter.enabled}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.wjys8ix3iR ++ cat /tmp/tmp.TgtZrvZI64 ++ rm /tmp/tmp.wjys8ix3iR /tmp/tmp.TgtZrvZI64 ++ return 0 + [[ '' == \t\r\u\e ]] + wait_pod some-name-rs0-2 + local pod=some-name-rs0-2 + set +o xtrace waiting for pod/some-name-rs0-2 to be ready.OK ++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].non_voting.enabled}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.YIiZs2OU28 +++ mktemp ++ local LAST_ERR=/tmp/tmp.EtaF5tVeTv ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].non_voting.enabled}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.YIiZs2OU28 ++ cat /tmp/tmp.EtaF5tVeTv ++ rm /tmp/tmp.YIiZs2OU28 /tmp/tmp.EtaF5tVeTv ++ return 0 + [[ '' == \t\r\u\e ]] + sleep 10 + [[ true == \t\r\u\e ]] + set +x Waiting for cluster readyness + wait_cluster_consistency some-name + local cluster_name=some-name + local wait_time=32 + retry=0 + sleep 7 + echo -n 'waiting for cluster readyness' waiting for cluster readyness++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.zXptzA4qYz +++ mktemp ++ local LAST_ERR=/tmp/tmp.h0yw0pESap ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get psmdb some-name -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.zXptzA4qYz ++ cat /tmp/tmp.h0yw0pESap ++ rm /tmp/tmp.zXptzA4qYz /tmp/tmp.h0yw0pESap ++ return 0 + [[ ready == \r\e\a\d\y ]] ++ seq 0 2 + for i in '$(seq 0 2)' + kubectl_bin exec some-name-rs1-0 -c mongod -- kill 1 ++ mktemp + local LAST_OUT=/tmp/tmp.MfepcjvNE6 ++ mktemp + local LAST_ERR=/tmp/tmp.fJOLOjWKQw + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl exec some-name-rs1-0 -c mongod -- kill 1 + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.MfepcjvNE6 + cat /tmp/tmp.fJOLOjWKQw + rm /tmp/tmp.MfepcjvNE6 /tmp/tmp.fJOLOjWKQw + return 0 + for i in '$(seq 0 2)' + kubectl_bin exec some-name-rs1-1 -c mongod -- kill 1 ++ mktemp + local LAST_OUT=/tmp/tmp.zW78jE292r ++ mktemp + local LAST_ERR=/tmp/tmp.x5GiH6cy6n + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl exec some-name-rs1-1 -c mongod -- kill 1 + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.zW78jE292r + cat /tmp/tmp.x5GiH6cy6n + rm /tmp/tmp.zW78jE292r /tmp/tmp.x5GiH6cy6n + return 0 + for i in '$(seq 0 2)' + kubectl_bin exec some-name-rs1-2 -c mongod -- kill 1 ++ mktemp + local LAST_OUT=/tmp/tmp.CwCLPunwsn ++ mktemp + local LAST_ERR=/tmp/tmp.8Yqm4K2mfM + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl exec some-name-rs1-2 -c mongod -- kill 1 + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.CwCLPunwsn + cat /tmp/tmp.8Yqm4K2mfM + rm /tmp/tmp.CwCLPunwsn /tmp/tmp.8Yqm4K2mfM + return 0 + sleep 10 + wait_for_running some-name-rs1 3 + local name=some-name-rs1 + let last_pod=2 + local check_cluster_readyness=true + set_debug + [[ 1 == 1 ]] + set -o xtrace + local rs_name=rs1 + local cluster_name=some-name ++ seq 0 2 + for i in '$(seq 0 $last_pod)' + [[ 0 -eq 2 ]] + wait_pod some-name-rs1-0 + local pod=some-name-rs1-0 + set +o xtrace waiting for pod/some-name-rs1-0 to be ready......OK + for i in '$(seq 0 $last_pod)' + [[ 1 -eq 2 ]] + wait_pod some-name-rs1-1 + local pod=some-name-rs1-1 + set +o xtrace waiting for pod/some-name-rs1-1 to be ready..OK + for i in '$(seq 0 $last_pod)' + [[ 2 -eq 2 ]] ++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs1")].arbiter.enabled}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.eDJB6gkAxZ +++ mktemp ++ local LAST_ERR=/tmp/tmp.g6M4HosCLb ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs1")].arbiter.enabled}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.eDJB6gkAxZ ++ cat /tmp/tmp.g6M4HosCLb ++ rm /tmp/tmp.eDJB6gkAxZ /tmp/tmp.g6M4HosCLb ++ return 0 + [[ '' == \t\r\u\e ]] + wait_pod some-name-rs1-2 + local pod=some-name-rs1-2 + set +o xtrace waiting for pod/some-name-rs1-2 to be ready..OK ++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs1")].non_voting.enabled}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.7leWnXXtOb +++ mktemp ++ local LAST_ERR=/tmp/tmp.yF31eSeIHd ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs1")].non_voting.enabled}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.7leWnXXtOb ++ cat /tmp/tmp.yF31eSeIHd ++ rm /tmp/tmp.7leWnXXtOb /tmp/tmp.yF31eSeIHd ++ return 0 + [[ '' == \t\r\u\e ]] + sleep 10 + [[ true == \t\r\u\e ]] + set +x Waiting for cluster readyness + wait_cluster_consistency some-name + local cluster_name=some-name + local wait_time=32 + retry=0 + sleep 7 + echo -n 'waiting for cluster readyness' waiting for cluster readyness++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.uDvlWSyVLp +++ mktemp ++ local LAST_ERR=/tmp/tmp.sUxbPvsJ29 ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get psmdb some-name -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.uDvlWSyVLp ++ cat /tmp/tmp.sUxbPvsJ29 ++ rm /tmp/tmp.uDvlWSyVLp /tmp/tmp.sUxbPvsJ29 ++ return 0 + [[ ready == \r\e\a\d\y ]] ++ seq 0 2 + for i in '$(seq 0 2)' + kubectl_bin exec some-name-cfg-0 -c mongod -- kill 1 ++ mktemp + local LAST_OUT=/tmp/tmp.STajZhJFSz ++ mktemp + local LAST_ERR=/tmp/tmp.08EqZjbpo6 + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl exec some-name-cfg-0 -c mongod -- kill 1 + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.STajZhJFSz + cat /tmp/tmp.08EqZjbpo6 + rm /tmp/tmp.STajZhJFSz /tmp/tmp.08EqZjbpo6 + return 0 + for i in '$(seq 0 2)' + kubectl_bin exec some-name-cfg-1 -c mongod -- kill 1 ++ mktemp + local LAST_OUT=/tmp/tmp.keq0zVZwVH ++ mktemp + local LAST_ERR=/tmp/tmp.0Nhu6d6F1U + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl exec some-name-cfg-1 -c mongod -- kill 1 + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.keq0zVZwVH + cat /tmp/tmp.0Nhu6d6F1U + rm /tmp/tmp.keq0zVZwVH /tmp/tmp.0Nhu6d6F1U + return 0 + for i in '$(seq 0 2)' + kubectl_bin exec some-name-cfg-2 -c mongod -- kill 1 ++ mktemp + local LAST_OUT=/tmp/tmp.c7cRyKfvvf ++ mktemp + local LAST_ERR=/tmp/tmp.pA0kjGPvCr + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl exec some-name-cfg-2 -c mongod -- kill 1 + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.c7cRyKfvvf + cat /tmp/tmp.pA0kjGPvCr + rm /tmp/tmp.c7cRyKfvvf /tmp/tmp.pA0kjGPvCr + return 0 + sleep 10 + wait_for_running some-name-cfg 3 + local name=some-name-cfg + let last_pod=2 + local check_cluster_readyness=true + set_debug + [[ 1 == 1 ]] + set -o xtrace + local rs_name=cfg + local cluster_name=some-name ++ seq 0 2 + for i in '$(seq 0 $last_pod)' + [[ 0 -eq 2 ]] + wait_pod some-name-cfg-0 + local pod=some-name-cfg-0 + set +o xtrace waiting for pod/some-name-cfg-0 to be ready......OK + for i in '$(seq 0 $last_pod)' + [[ 1 -eq 2 ]] + wait_pod some-name-cfg-1 + local pod=some-name-cfg-1 + set +o xtrace waiting for pod/some-name-cfg-1 to be ready..OK + for i in '$(seq 0 $last_pod)' + [[ 2 -eq 2 ]] ++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].arbiter.enabled}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.Q4BaVPYI9F +++ mktemp ++ local LAST_ERR=/tmp/tmp.0hcrNI2qaO ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].arbiter.enabled}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.Q4BaVPYI9F ++ cat /tmp/tmp.0hcrNI2qaO ++ rm /tmp/tmp.Q4BaVPYI9F /tmp/tmp.0hcrNI2qaO ++ return 0 + [[ '' == \t\r\u\e ]] + wait_pod some-name-cfg-2 + local pod=some-name-cfg-2 + set +o xtrace waiting for pod/some-name-cfg-2 to be ready.OK ++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].non_voting.enabled}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.30kqMF0162 +++ mktemp ++ local LAST_ERR=/tmp/tmp.FRg1tXjIXa ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].non_voting.enabled}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.30kqMF0162 ++ cat /tmp/tmp.FRg1tXjIXa ++ rm /tmp/tmp.30kqMF0162 /tmp/tmp.FRg1tXjIXa ++ return 0 + [[ '' == \t\r\u\e ]] + sleep 10 + [[ true == \t\r\u\e ]] + set +x Waiting for cluster readyness + wait_cluster_consistency some-name + local cluster_name=some-name + local wait_time=32 + retry=0 + sleep 7 + echo -n 'waiting for cluster readyness' waiting for cluster readyness++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.fC9ZXrwYOb +++ mktemp ++ local LAST_ERR=/tmp/tmp.7LuF7Ki7d6 ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get psmdb some-name -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.fC9ZXrwYOb ++ cat /tmp/tmp.7LuF7Ki7d6 ++ rm /tmp/tmp.fC9ZXrwYOb /tmp/tmp.7LuF7Ki7d6 ++ return 0 + [[ ready == \r\e\a\d\y ]] + desc 'delete statefulsets and wait for recovery' + set +o xtrace ----------------------------------------------------------------------------------- delete statefulsets and wait for recovery ----------------------------------------------------------------------------------- + kubectl delete sts some-name-rs0 statefulset.apps "some-name-rs0" deleted + sleep 5 + wait_for_running some-name-rs0 3 + local name=some-name-rs0 + let last_pod=2 + local check_cluster_readyness=true + set_debug + [[ 1 == 1 ]] + set -o xtrace + local rs_name=rs0 + local cluster_name=some-name ++ seq 0 2 + for i in '$(seq 0 $last_pod)' + [[ 0 -eq 2 ]] + wait_pod some-name-rs0-0 + local pod=some-name-rs0-0 + set +o xtrace waiting for pod/some-name-rs0-0 to be ready.OK + for i in '$(seq 0 $last_pod)' + [[ 1 -eq 2 ]] + wait_pod some-name-rs0-1 + local pod=some-name-rs0-1 + set +o xtrace waiting for pod/some-name-rs0-1 to be ready.OK + for i in '$(seq 0 $last_pod)' + [[ 2 -eq 2 ]] ++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].arbiter.enabled}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.zh8tE0hjyo +++ mktemp ++ local LAST_ERR=/tmp/tmp.hqo7R24AGa ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].arbiter.enabled}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.zh8tE0hjyo ++ cat /tmp/tmp.hqo7R24AGa ++ rm /tmp/tmp.zh8tE0hjyo /tmp/tmp.hqo7R24AGa ++ return 0 + [[ '' == \t\r\u\e ]] + wait_pod some-name-rs0-2 + local pod=some-name-rs0-2 + set +o xtrace waiting for pod/some-name-rs0-2 to be ready.OK ++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].non_voting.enabled}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.xhECu2IEkj +++ mktemp ++ local LAST_ERR=/tmp/tmp.aE0187n7dS ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].non_voting.enabled}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.xhECu2IEkj ++ cat /tmp/tmp.aE0187n7dS ++ rm /tmp/tmp.xhECu2IEkj /tmp/tmp.aE0187n7dS ++ return 0 + [[ '' == \t\r\u\e ]] + sleep 10 + [[ true == \t\r\u\e ]] + set +x Waiting for cluster readyness............................. + wait_cluster_consistency some-name + local cluster_name=some-name + local wait_time=32 + retry=0 + sleep 7 + echo -n 'waiting for cluster readyness' waiting for cluster readyness++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.BpdfwalJjO +++ mktemp ++ local LAST_ERR=/tmp/tmp.Sp35l5jOIn ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get psmdb some-name -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.BpdfwalJjO ++ cat /tmp/tmp.Sp35l5jOIn ++ rm /tmp/tmp.BpdfwalJjO /tmp/tmp.Sp35l5jOIn ++ return 0 + [[ ready == \r\e\a\d\y ]] + kubectl delete sts some-name-rs1 statefulset.apps "some-name-rs1" deleted + sleep 5 + wait_for_running some-name-rs1 3 + local name=some-name-rs1 + let last_pod=2 + local check_cluster_readyness=true + set_debug + [[ 1 == 1 ]] + set -o xtrace + local rs_name=rs1 + local cluster_name=some-name ++ seq 0 2 + for i in '$(seq 0 $last_pod)' + [[ 0 -eq 2 ]] + wait_pod some-name-rs1-0 + local pod=some-name-rs1-0 + set +o xtrace waiting for pod/some-name-rs1-0 to be ready.OK + for i in '$(seq 0 $last_pod)' + [[ 1 -eq 2 ]] + wait_pod some-name-rs1-1 + local pod=some-name-rs1-1 + set +o xtrace waiting for pod/some-name-rs1-1 to be ready.OK + for i in '$(seq 0 $last_pod)' + [[ 2 -eq 2 ]] ++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs1")].arbiter.enabled}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.ys6acD73nX +++ mktemp ++ local LAST_ERR=/tmp/tmp.8obRw2SU5w ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs1")].arbiter.enabled}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.ys6acD73nX ++ cat /tmp/tmp.8obRw2SU5w ++ rm /tmp/tmp.ys6acD73nX /tmp/tmp.8obRw2SU5w ++ return 0 + [[ '' == \t\r\u\e ]] + wait_pod some-name-rs1-2 + local pod=some-name-rs1-2 + set +o xtrace waiting for pod/some-name-rs1-2 to be ready.OK ++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs1")].non_voting.enabled}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.2h4dcisjMj +++ mktemp ++ local LAST_ERR=/tmp/tmp.wVd86VaOrU ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs1")].non_voting.enabled}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.2h4dcisjMj ++ cat /tmp/tmp.wVd86VaOrU ++ rm /tmp/tmp.2h4dcisjMj /tmp/tmp.wVd86VaOrU ++ return 0 + [[ '' == \t\r\u\e ]] + sleep 10 + [[ true == \t\r\u\e ]] + set +x Waiting for cluster readyness.................................. + wait_cluster_consistency some-name + local cluster_name=some-name + local wait_time=32 + retry=0 + sleep 7 + echo -n 'waiting for cluster readyness' waiting for cluster readyness++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.BVnXeX8iK0 +++ mktemp ++ local LAST_ERR=/tmp/tmp.PM83sqdcwX ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get psmdb some-name -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.BVnXeX8iK0 ++ cat /tmp/tmp.PM83sqdcwX ++ rm /tmp/tmp.BVnXeX8iK0 /tmp/tmp.PM83sqdcwX ++ return 0 + [[ ready == \r\e\a\d\y ]] + kubectl delete sts some-name-cfg statefulset.apps "some-name-cfg" deleted + sleep 5 + wait_for_running some-name-cfg 3 + local name=some-name-cfg + let last_pod=2 + local check_cluster_readyness=true + set_debug + [[ 1 == 1 ]] + set -o xtrace + local rs_name=cfg + local cluster_name=some-name ++ seq 0 2 + for i in '$(seq 0 $last_pod)' + [[ 0 -eq 2 ]] + wait_pod some-name-cfg-0 + local pod=some-name-cfg-0 + set +o xtrace waiting for pod/some-name-cfg-0 to be ready.OK + for i in '$(seq 0 $last_pod)' + [[ 1 -eq 2 ]] + wait_pod some-name-cfg-1 + local pod=some-name-cfg-1 + set +o xtrace waiting for pod/some-name-cfg-1 to be ready.OK + for i in '$(seq 0 $last_pod)' + [[ 2 -eq 2 ]] ++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].arbiter.enabled}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.goBmIdjEVM +++ mktemp ++ local LAST_ERR=/tmp/tmp.R9gs5kL2Gu ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].arbiter.enabled}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.goBmIdjEVM ++ cat /tmp/tmp.R9gs5kL2Gu ++ rm /tmp/tmp.goBmIdjEVM /tmp/tmp.R9gs5kL2Gu ++ return 0 + [[ '' == \t\r\u\e ]] + wait_pod some-name-cfg-2 + local pod=some-name-cfg-2 + set +o xtrace waiting for pod/some-name-cfg-2 to be ready.OK ++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].non_voting.enabled}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.5cFLd5d5QJ +++ mktemp ++ local LAST_ERR=/tmp/tmp.f9ym3dWx41 ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].non_voting.enabled}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.5cFLd5d5QJ ++ cat /tmp/tmp.f9ym3dWx41 ++ rm /tmp/tmp.5cFLd5d5QJ /tmp/tmp.f9ym3dWx41 ++ return 0 + [[ '' == \t\r\u\e ]] + sleep 10 + [[ true == \t\r\u\e ]] + set +x Waiting for cluster readyness + wait_cluster_consistency some-name + local cluster_name=some-name + local wait_time=32 + retry=0 + sleep 7 + echo -n 'waiting for cluster readyness' waiting for cluster readyness++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.O1o39fgd4q +++ mktemp ++ local LAST_ERR=/tmp/tmp.nIuBPO5qpX ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get psmdb some-name -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.O1o39fgd4q ++ cat /tmp/tmp.nIuBPO5qpX ++ rm /tmp/tmp.O1o39fgd4q /tmp/tmp.nIuBPO5qpX ++ return 0 + [[ ready == \r\e\a\d\y ]] + desc 'expose and delete cluster then recreate' + set +o xtrace ----------------------------------------------------------------------------------- expose and delete cluster then recreate ----------------------------------------------------------------------------------- + kubectl_bin patch psmdb some-name --type=json --patch '[ { "op": "replace", "path": "/spec/replsets/0/expose", "value": { "enabled": true, "exposeType" : "ClusterIP" } }, { "op": "replace", "path": "/spec/replsets/1/expose", "value": { "enabled": true, "exposeType" : "ClusterIP" } }, ]' ++ mktemp + local LAST_OUT=/tmp/tmp.ldFc62eKU8 ++ mktemp + local LAST_ERR=/tmp/tmp.xOXPBnBBDT + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl patch psmdb some-name --type=json --patch '[ { "op": "replace", "path": "/spec/replsets/0/expose", "value": { "enabled": true, "exposeType" : "ClusterIP" } }, { "op": "replace", "path": "/spec/replsets/1/expose", "value": { "enabled": true, "exposeType" : "ClusterIP" } }, ]' + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.ldFc62eKU8 perconaservermongodb.psmdb.percona.com/some-name patched + cat /tmp/tmp.xOXPBnBBDT + rm /tmp/tmp.ldFc62eKU8 /tmp/tmp.xOXPBnBBDT + return 0 + kubectl delete psmdb some-name perconaservermongodb.psmdb.percona.com "some-name" deleted + set +e + echo -n 'waiting for psmdb/some-name to be deleted' waiting for psmdb/some-name to be deleted+ kubectl get psmdb some-name Error from server (NotFound): perconaservermongodbs.psmdb.percona.com "some-name" not found + set -e + apply_cluster /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1569/e2e-tests/recover-no-primary/conf/some-name-sharded.yml + '[' -z '' ']' + cat_config /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1569/e2e-tests/recover-no-primary/conf/some-name-sharded.yml + yq eval '(.spec | select(has("backup"))).backup.image = "perconalab/percona-server-mongodb-operator:main-backup"' + yq eval '.spec.upgradeOptions.apply="Never"' + yq eval '(.spec | select(has("pmm"))).pmm.image = "perconalab/pmm-client:dev-latest"' + yq eval '(.spec | select(.image == null)).image = "perconalab/percona-server-mongodb-operator:main-mongod7.0"' + kubectl_bin apply -f - + cat /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1569/e2e-tests/recover-no-primary/conf/some-name-sharded.yml + yq eval '(.spec | select(has("initImage"))).initImage = "perconalab/percona-server-mongodb-operator:PR-1569-eb818ce7"' ++ mktemp + local LAST_OUT=/tmp/tmp.mBMt7PvPF9 ++ mktemp + local LAST_ERR=/tmp/tmp.Z1jhSVZ4kh + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.mBMt7PvPF9 perconaservermongodb.psmdb.percona.com/some-name created + cat /tmp/tmp.Z1jhSVZ4kh + rm /tmp/tmp.mBMt7PvPF9 /tmp/tmp.Z1jhSVZ4kh + return 0 + wait_for_running some-name-rs0 3 + local name=some-name-rs0 + let last_pod=2 + local check_cluster_readyness=true + set_debug + [[ 1 == 1 ]] + set -o xtrace + local rs_name=rs0 + local cluster_name=some-name ++ seq 0 2 + for i in '$(seq 0 $last_pod)' + [[ 0 -eq 2 ]] + wait_pod some-name-rs0-0 + local pod=some-name-rs0-0 + set +o xtrace waiting for pod/some-name-rs0-0 to be ready.OK + for i in '$(seq 0 $last_pod)' + [[ 1 -eq 2 ]] + wait_pod some-name-rs0-1 + local pod=some-name-rs0-1 + set +o xtrace waiting for pod/some-name-rs0-1 to be ready.OK + for i in '$(seq 0 $last_pod)' + [[ 2 -eq 2 ]] ++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].arbiter.enabled}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.dD01XnS4sg +++ mktemp ++ local LAST_ERR=/tmp/tmp.RE8IsdNw5p ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].arbiter.enabled}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.dD01XnS4sg ++ cat /tmp/tmp.RE8IsdNw5p ++ rm /tmp/tmp.dD01XnS4sg /tmp/tmp.RE8IsdNw5p ++ return 0 + [[ '' == \t\r\u\e ]] + wait_pod some-name-rs0-2 + local pod=some-name-rs0-2 + set +o xtrace waiting for pod/some-name-rs0-2 to be ready.OK ++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].non_voting.enabled}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.yrpYyPRvSp +++ mktemp ++ local LAST_ERR=/tmp/tmp.36zZgieLz6 ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].non_voting.enabled}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.yrpYyPRvSp ++ cat /tmp/tmp.36zZgieLz6 ++ rm /tmp/tmp.yrpYyPRvSp /tmp/tmp.36zZgieLz6 ++ return 0 + [[ '' == \t\r\u\e ]] + sleep 10 + [[ true == \t\r\u\e ]] + set +x Waiting for cluster readyness................................................. + wait_for_running some-name-rs1 3 + local name=some-name-rs1 + let last_pod=2 + local check_cluster_readyness=true + set_debug + [[ 1 == 1 ]] + set -o xtrace + local rs_name=rs1 + local cluster_name=some-name ++ seq 0 2 + for i in '$(seq 0 $last_pod)' + [[ 0 -eq 2 ]] + wait_pod some-name-rs1-0 + local pod=some-name-rs1-0 + set +o xtrace waiting for pod/some-name-rs1-0 to be ready.OK + for i in '$(seq 0 $last_pod)' + [[ 1 -eq 2 ]] + wait_pod some-name-rs1-1 + local pod=some-name-rs1-1 + set +o xtrace waiting for pod/some-name-rs1-1 to be ready.OK + for i in '$(seq 0 $last_pod)' + [[ 2 -eq 2 ]] ++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs1")].arbiter.enabled}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.e9J5W3mmq6 +++ mktemp ++ local LAST_ERR=/tmp/tmp.C7HgkkkMYz ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs1")].arbiter.enabled}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.e9J5W3mmq6 ++ cat /tmp/tmp.C7HgkkkMYz ++ rm /tmp/tmp.e9J5W3mmq6 /tmp/tmp.C7HgkkkMYz ++ return 0 + [[ '' == \t\r\u\e ]] + wait_pod some-name-rs1-2 + local pod=some-name-rs1-2 + set +o xtrace waiting for pod/some-name-rs1-2 to be ready.OK ++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs1")].non_voting.enabled}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.oKaB48o5VL +++ mktemp ++ local LAST_ERR=/tmp/tmp.8i5hTcJ1ny ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs1")].non_voting.enabled}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.oKaB48o5VL ++ cat /tmp/tmp.8i5hTcJ1ny ++ rm /tmp/tmp.oKaB48o5VL /tmp/tmp.8i5hTcJ1ny ++ return 0 + [[ '' == \t\r\u\e ]] + sleep 10 + [[ true == \t\r\u\e ]] + set +x Waiting for cluster readyness + wait_for_running some-name-cfg 3 + local name=some-name-cfg + let last_pod=2 + local check_cluster_readyness=true + set_debug + [[ 1 == 1 ]] + set -o xtrace + local rs_name=cfg + local cluster_name=some-name ++ seq 0 2 + for i in '$(seq 0 $last_pod)' + [[ 0 -eq 2 ]] + wait_pod some-name-cfg-0 + local pod=some-name-cfg-0 + set +o xtrace waiting for pod/some-name-cfg-0 to be ready.OK + for i in '$(seq 0 $last_pod)' + [[ 1 -eq 2 ]] + wait_pod some-name-cfg-1 + local pod=some-name-cfg-1 + set +o xtrace waiting for pod/some-name-cfg-1 to be ready.OK + for i in '$(seq 0 $last_pod)' + [[ 2 -eq 2 ]] ++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].arbiter.enabled}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.0MPS5ZpUdt +++ mktemp ++ local LAST_ERR=/tmp/tmp.ONTjhm8YbY ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].arbiter.enabled}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.0MPS5ZpUdt ++ cat /tmp/tmp.ONTjhm8YbY ++ rm /tmp/tmp.0MPS5ZpUdt /tmp/tmp.ONTjhm8YbY ++ return 0 + [[ '' == \t\r\u\e ]] + wait_pod some-name-cfg-2 + local pod=some-name-cfg-2 + set +o xtrace waiting for pod/some-name-cfg-2 to be ready.OK ++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].non_voting.enabled}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.9NejCEPNBa +++ mktemp ++ local LAST_ERR=/tmp/tmp.NIHRbaCu86 ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].non_voting.enabled}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.9NejCEPNBa ++ cat /tmp/tmp.NIHRbaCu86 ++ rm /tmp/tmp.9NejCEPNBa /tmp/tmp.NIHRbaCu86 ++ return 0 + [[ '' == \t\r\u\e ]] + sleep 10 + [[ true == \t\r\u\e ]] + set +x Waiting for cluster readyness + wait_cluster_consistency some-name + local cluster_name=some-name + local wait_time=32 + retry=0 + sleep 7 + echo -n 'waiting for cluster readyness' waiting for cluster readyness++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.n8ksfzWQUB +++ mktemp ++ local LAST_ERR=/tmp/tmp.ib0IZHEkbw ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get psmdb some-name -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.n8ksfzWQUB ++ cat /tmp/tmp.ib0IZHEkbw ++ rm /tmp/tmp.n8ksfzWQUB /tmp/tmp.ib0IZHEkbw ++ return 0 + [[ ready == \r\e\a\d\y ]] + kubectl delete psmdb some-name perconaservermongodb.psmdb.percona.com "some-name" deleted + kubectl delete pvc --all persistentvolumeclaim "mongod-data-some-name-cfg-0" deleted persistentvolumeclaim "mongod-data-some-name-cfg-1" deleted persistentvolumeclaim "mongod-data-some-name-cfg-2" deleted persistentvolumeclaim "mongod-data-some-name-rs0-0" deleted persistentvolumeclaim "mongod-data-some-name-rs0-1" deleted persistentvolumeclaim "mongod-data-some-name-rs0-2" deleted persistentvolumeclaim "mongod-data-some-name-rs1-0" deleted persistentvolumeclaim "mongod-data-some-name-rs1-1" deleted persistentvolumeclaim "mongod-data-some-name-rs1-2" deleted + destroy recover-no-primary-5707 + local namespace=recover-no-primary-5707 + local ignore_logs=true + desc 'destroy cluster/operator and all other resources' + set +o xtrace ----------------------------------------------------------------------------------- destroy cluster/operator and all other resources ----------------------------------------------------------------------------------- + '[' true == false ']' + delete_crd + desc 'get and delete old CRDs and RBAC' + set +o xtrace ----------------------------------------------------------------------------------- get and delete old CRDs and RBAC ----------------------------------------------------------------------------------- + kubectl_bin delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1569/deploy/crd.yaml --ignore-not-found --wait=false ++ mktemp + local LAST_OUT=/tmp/tmp.rmOUQFwy6T ++ mktemp + local LAST_ERR=/tmp/tmp.mqLQ6Wo0vf + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1569/deploy/crd.yaml --ignore-not-found --wait=false + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.rmOUQFwy6T customresourcedefinition.apiextensions.k8s.io "perconaservermongodbbackups.psmdb.percona.com" deleted customresourcedefinition.apiextensions.k8s.io "perconaservermongodbrestores.psmdb.percona.com" deleted customresourcedefinition.apiextensions.k8s.io "perconaservermongodbs.psmdb.percona.com" deleted + cat /tmp/tmp.mqLQ6Wo0vf + rm /tmp/tmp.rmOUQFwy6T /tmp/tmp.mqLQ6Wo0vf + return 0 ++ yq eval .metadata.name /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1569/deploy/crd.yaml ++ grep -v '\-\-\-' + for crd_name in '$(yq eval '\''.metadata.name'\'' "${src_dir}/deploy/crd.yaml" | grep -v '\''\-\-\-'\'')' + kubectl get perconaservermongodbbackups.psmdb.percona.com --all-namespaces -o wide + grep -v NAMESPACE + xargs -L 1 sh -xc 'kubectl patch perconaservermongodbbackups.psmdb.percona.com -n $0 $1 --type=merge -p "{\"metadata\":{\"finalizers\":[]}}"' error: the server doesn't have a resource type "perconaservermongodbbackups" + kubectl patch perconaservermongodbbackups.psmdb.percona.com -n sh --type=merge -p '{"metadata":{"finalizers":[]}}' error: the server doesn't have a resource type "perconaservermongodbbackups" + : + kubectl_bin wait --for=delete crd perconaservermongodbbackups.psmdb.percona.com ++ mktemp + local LAST_OUT=/tmp/tmp.vH3keXRjJT ++ mktemp + local LAST_ERR=/tmp/tmp.dtniBUMdrr + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl wait --for=delete crd perconaservermongodbbackups.psmdb.percona.com + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.vH3keXRjJT + cat /tmp/tmp.dtniBUMdrr + rm /tmp/tmp.vH3keXRjJT /tmp/tmp.dtniBUMdrr + return 0 + for crd_name in '$(yq eval '\''.metadata.name'\'' "${src_dir}/deploy/crd.yaml" | grep -v '\''\-\-\-'\'')' + kubectl get perconaservermongodbrestores.psmdb.percona.com --all-namespaces -o wide + grep -v NAMESPACE + xargs -L 1 sh -xc 'kubectl patch perconaservermongodbrestores.psmdb.percona.com -n $0 $1 --type=merge -p "{\"metadata\":{\"finalizers\":[]}}"' error: the server doesn't have a resource type "perconaservermongodbrestores" + kubectl patch perconaservermongodbrestores.psmdb.percona.com -n sh --type=merge -p '{"metadata":{"finalizers":[]}}' error: the server doesn't have a resource type "perconaservermongodbrestores" + : + kubectl_bin wait --for=delete crd perconaservermongodbrestores.psmdb.percona.com ++ mktemp + local LAST_OUT=/tmp/tmp.Oph1r7Ndlp ++ mktemp + local LAST_ERR=/tmp/tmp.OcHUgsPKdD + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl wait --for=delete crd perconaservermongodbrestores.psmdb.percona.com + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.Oph1r7Ndlp + cat /tmp/tmp.OcHUgsPKdD + rm /tmp/tmp.Oph1r7Ndlp /tmp/tmp.OcHUgsPKdD + return 0 + for crd_name in '$(yq eval '\''.metadata.name'\'' "${src_dir}/deploy/crd.yaml" | grep -v '\''\-\-\-'\'')' + kubectl get perconaservermongodbs.psmdb.percona.com --all-namespaces -o wide + grep -v NAMESPACE + xargs -L 1 sh -xc 'kubectl patch perconaservermongodbs.psmdb.percona.com -n $0 $1 --type=merge -p "{\"metadata\":{\"finalizers\":[]}}"' error: the server doesn't have a resource type "perconaservermongodbs" + kubectl patch perconaservermongodbs.psmdb.percona.com -n sh --type=merge -p '{"metadata":{"finalizers":[]}}' error: the server doesn't have a resource type "perconaservermongodbs" + : + kubectl_bin wait --for=delete crd perconaservermongodbs.psmdb.percona.com ++ mktemp + local LAST_OUT=/tmp/tmp.fVEVb4ySif ++ mktemp + local LAST_ERR=/tmp/tmp.ZGZKB1RM2s + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl wait --for=delete crd perconaservermongodbs.psmdb.percona.com + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.fVEVb4ySif + cat /tmp/tmp.ZGZKB1RM2s + rm /tmp/tmp.fVEVb4ySif /tmp/tmp.ZGZKB1RM2s + return 0 + local rbac_yaml=rbac.yaml + '[' -n psmdb-operator ']' + rbac_yaml=cw-rbac.yaml + kubectl_bin delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1569/deploy/cw-rbac.yaml --ignore-not-found ++ mktemp + local LAST_OUT=/tmp/tmp.eHiDerVU3A ++ mktemp + local LAST_ERR=/tmp/tmp.gvxuvmod5u + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1569/deploy/cw-rbac.yaml --ignore-not-found + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.eHiDerVU3A clusterrole.rbac.authorization.k8s.io "percona-server-mongodb-operator" deleted clusterrolebinding.rbac.authorization.k8s.io "service-account-percona-server-mongodb-operator" deleted + cat /tmp/tmp.gvxuvmod5u + rm /tmp/tmp.eHiDerVU3A /tmp/tmp.gvxuvmod5u + return 0 + kubectl_bin delete -f https://github.com/cert-manager/cert-manager/releases/download/v1.14.5/cert-manager.yaml + : + '[' -n '' ']' + '[' -n psmdb-operator ']' + rm -rf /tmp/tmp.MB3F6OnWIO + kubectl_bin delete --grace-period=0 --force=true namespace psmdb-operator + kubectl_bin delete --grace-period=0 --force=true namespace recover-no-primary-5707 ++ mktemp ++ mktemp + local LAST_OUT=/tmp/tmp.RdyPlPyKDu ++ mktemp + local LAST_ERR=/tmp/tmp.zgc9zSxEkA + local exit_status=0 + local timeout=4 ++ seq 0 2 + local LAST_OUT=/tmp/tmp.tBPbq7oaxM + for i in '$(seq 0 2)' + set +e + kubectl delete --grace-period=0 --force=true namespace recover-no-primary-5707 ++ mktemp + local LAST_ERR=/tmp/tmp.y2V5fLOaOt + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl delete --grace-period=0 --force=true namespace psmdb-operator