++ echo 'Log: /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2156/e2e-tests/logs/recover-no-primary.log' Log: /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2156/e2e-tests/logs/recover-no-primary.log ++ '[' -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2156/e2e-tests/conf/cloud-secret.yml ']' ++ SKIP_BACKUPS_TO_AWS_GCP_AZURE= ++ oc get projects ++ kubectl get nodes ++ grep '^minikube' +++ kubectl version -o json +++ jq -r .serverVersion.gitVersion +++ grep '\-eks\-' grep: warning: stray \ before - Warning: version difference between client (1.35) and server (1.31) exceeds the supported minor version skew of +/-1 ++ '[' ']' ++ EKS=0 +++ kubectl version -o json +++ jq -r .serverVersion.gitVersion +++ grep gke Warning: version difference between client (1.35) and server (1.31) exceeds the supported minor version skew of +/-1 ++ '[' v1.31.14-gke.1081000 ']' ++ GKE=1 +++ kubectl version -o json +++ jq -r '.serverVersion.major + "." + .serverVersion.minor' +++ /usr/sbin/sed -r 's/[^0-9.]+//g' Warning: version difference between client (1.35) and server (1.31) exceeds the supported minor version skew of +/-1 ++ KUBE_VERSION=1.31 + create_infra recover-no-primary-29251 + local ns=recover-no-primary-29251 + [[ 1 == 1 ]] + delete_crd + desc 'get and delete old CRDs and RBAC' + set +o xtrace ----------------------------------------------------------------------------------- get and delete old CRDs and RBAC ----------------------------------------------------------------------------------- + kubectl_bin delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2156/deploy/crd.yaml --ignore-not-found --wait=false ++ mktemp + local LAST_OUT=/tmp/tmp.nYaUjG99nO ++ mktemp + local LAST_ERR=/tmp/tmp.xqMZThwByo + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2156/deploy/crd.yaml --ignore-not-found --wait=false + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.nYaUjG99nO customresourcedefinition.apiextensions.k8s.io "perconaservermongodbbackups.psmdb.percona.com" deleted customresourcedefinition.apiextensions.k8s.io "perconaservermongodbrestores.psmdb.percona.com" deleted customresourcedefinition.apiextensions.k8s.io "perconaservermongodbs.psmdb.percona.com" deleted + cat /tmp/tmp.xqMZThwByo + rm /tmp/tmp.nYaUjG99nO /tmp/tmp.xqMZThwByo + return 0 ++ yq eval .metadata.name /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2156/deploy/crd.yaml ++ grep -v '\-\-\-' grep: warning: stray \ before - grep: warning: stray \ before - + for crd_name in $(yq eval '.metadata.name' "${src_dir}/deploy/crd.yaml" | grep -v '\-\-\-') + kubectl get perconaservermongodbbackups.psmdb.percona.com --all-namespaces -o wide + grep -v NAMESPACE + xargs -L 1 sh -xc 'kubectl patch perconaservermongodbbackups.psmdb.percona.com -n $0 $1 --type=merge -p "{\"metadata\":{\"finalizers\":[]}}"' error: the server doesn't have a resource type "perconaservermongodbbackups" + kubectl patch perconaservermongodbbackups.psmdb.percona.com -n sh --type=merge -p '{"metadata":{"finalizers":[]}}' error: the server doesn't have a resource type "perconaservermongodbbackups" + : + kubectl_bin wait --for=delete crd perconaservermongodbbackups.psmdb.percona.com ++ mktemp + local LAST_OUT=/tmp/tmp.fj2hfsYLV5 ++ mktemp + local LAST_ERR=/tmp/tmp.KYvMTVLXpP + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl wait --for=delete crd perconaservermongodbbackups.psmdb.percona.com + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.fj2hfsYLV5 + cat /tmp/tmp.KYvMTVLXpP + rm /tmp/tmp.fj2hfsYLV5 /tmp/tmp.KYvMTVLXpP + return 0 + for crd_name in $(yq eval '.metadata.name' "${src_dir}/deploy/crd.yaml" | grep -v '\-\-\-') + kubectl get perconaservermongodbrestores.psmdb.percona.com --all-namespaces -o wide + grep -v NAMESPACE + xargs -L 1 sh -xc 'kubectl patch perconaservermongodbrestores.psmdb.percona.com -n $0 $1 --type=merge -p "{\"metadata\":{\"finalizers\":[]}}"' error: the server doesn't have a resource type "perconaservermongodbrestores" + kubectl patch perconaservermongodbrestores.psmdb.percona.com -n sh --type=merge -p '{"metadata":{"finalizers":[]}}' error: the server doesn't have a resource type "perconaservermongodbrestores" + : + kubectl_bin wait --for=delete crd perconaservermongodbrestores.psmdb.percona.com ++ mktemp + local LAST_OUT=/tmp/tmp.AYLSIS0dbN ++ mktemp + local LAST_ERR=/tmp/tmp.ohHblsu0hT + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl wait --for=delete crd perconaservermongodbrestores.psmdb.percona.com + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.AYLSIS0dbN + cat /tmp/tmp.ohHblsu0hT + rm /tmp/tmp.AYLSIS0dbN /tmp/tmp.ohHblsu0hT + return 0 + for crd_name in $(yq eval '.metadata.name' "${src_dir}/deploy/crd.yaml" | grep -v '\-\-\-') + kubectl get perconaservermongodbs.psmdb.percona.com --all-namespaces -o wide + xargs -L 1 sh -xc 'kubectl patch perconaservermongodbs.psmdb.percona.com -n $0 $1 --type=merge -p "{\"metadata\":{\"finalizers\":[]}}"' + grep -v NAMESPACE error: the server doesn't have a resource type "perconaservermongodbs" + kubectl patch perconaservermongodbs.psmdb.percona.com -n sh --type=merge -p '{"metadata":{"finalizers":[]}}' error: the server doesn't have a resource type "perconaservermongodbs" + : + kubectl_bin wait --for=delete crd perconaservermongodbs.psmdb.percona.com ++ mktemp + local LAST_OUT=/tmp/tmp.aylb7X9ohC ++ mktemp + local LAST_ERR=/tmp/tmp.8sKCaNXO7b + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl wait --for=delete crd perconaservermongodbs.psmdb.percona.com + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.aylb7X9ohC + cat /tmp/tmp.8sKCaNXO7b + rm /tmp/tmp.aylb7X9ohC /tmp/tmp.8sKCaNXO7b + return 0 + local rbac_yaml=rbac.yaml + '[' -n psmdb-operator ']' + rbac_yaml=cw-rbac.yaml + kubectl_bin delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2156/deploy/cw-rbac.yaml --ignore-not-found ++ mktemp + local LAST_OUT=/tmp/tmp.akrbWXF5pP ++ mktemp + local LAST_ERR=/tmp/tmp.nYfgcKEnwb + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2156/deploy/cw-rbac.yaml --ignore-not-found + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.akrbWXF5pP clusterrole.rbac.authorization.k8s.io "percona-server-mongodb-operator" deleted clusterrolebinding.rbac.authorization.k8s.io "service-account-percona-server-mongodb-operator" deleted + cat /tmp/tmp.nYfgcKEnwb + rm /tmp/tmp.akrbWXF5pP /tmp/tmp.nYfgcKEnwb + return 0 + check_crd_for_deletion PR-2156-df65e35b + local git_tag=PR-2156-df65e35b ++ curl -s https://raw.githubusercontent.com/percona/percona-server-mongodb-operator/PR-2156-df65e35b/deploy/crd.yaml ++ yq eval .metadata.name ++ /usr/sbin/sed s/---//g ++ /usr/sbin/sed ':a;N;$!ba;s/\n/ /g' + for crd_name in $(curl -s https://raw.githubusercontent.com/percona/percona-server-mongodb-operator/${git_tag}/deploy/crd.yaml | yq eval '.metadata.name' | $sed 's/---//g' | $sed ':a;N;$!ba;s/\n/ /g') ++ kubectl_bin get crd/null -o 'jsonpath={.status.conditions[-1].type}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.MBuu0rd9mH +++ mktemp ++ local LAST_ERR=/tmp/tmp.c8U91ZFlQj ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get crd/null -o 'jsonpath={.status.conditions[-1].type}' ++ exit_status=1 ++ set -e ++ '[' 1 '!=' 0 -a -n 1 ']' ++ cat /tmp/tmp.MBuu0rd9mH ++ cat /tmp/tmp.c8U91ZFlQj Error from server (NotFound): customresourcedefinitions.apiextensions.k8s.io "null" not found ++ sleep 0 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get crd/null -o 'jsonpath={.status.conditions[-1].type}' ++ exit_status=1 ++ set -e ++ '[' 1 '!=' 0 -a -n 1 ']' ++ cat /tmp/tmp.MBuu0rd9mH ++ cat /tmp/tmp.c8U91ZFlQj Error from server (NotFound): customresourcedefinitions.apiextensions.k8s.io "null" not found ++ sleep 4 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get crd/null -o 'jsonpath={.status.conditions[-1].type}' ++ exit_status=1 ++ set -e ++ '[' 1 '!=' 0 -a -n 1 ']' ++ cat /tmp/tmp.MBuu0rd9mH ++ cat /tmp/tmp.c8U91ZFlQj Error from server (NotFound): customresourcedefinitions.apiextensions.k8s.io "null" not found ++ sleep 8 ++ cat /tmp/tmp.MBuu0rd9mH ++ cat /tmp/tmp.c8U91ZFlQj Error from server (NotFound): customresourcedefinitions.apiextensions.k8s.io "null" not found ++ rm /tmp/tmp.MBuu0rd9mH /tmp/tmp.c8U91ZFlQj ++ return 1 + [[ '' == \T\e\r\m\i\n\a\t\i\n\g ]] + '[' -n psmdb-operator ']' + create_namespace psmdb-operator + local namespace=psmdb-operator + local skip_clean_namespace= + [[ 1 == 1 ]] + [[ -z '' ]] + destroy_chaos_mesh ++ helm list --all-namespaces --filter chaos-mesh ++ tail -n1 ++ awk '-F ' '{print $2}' ++ sed s/NAMESPACE// + local chaos_mesh_ns= + desc 'destroy chaos-mesh' + set +o xtrace ----------------------------------------------------------------------------------- destroy chaos-mesh ----------------------------------------------------------------------------------- + '[' -n '' ']' ++ kubectl get MutatingWebhookConfiguration ++ grep chaos-mesh ++ awk '{print $1}' + timeout 30 kubectl delete MutatingWebhookConfiguration error: resource(s) were provided, but no name was specified + : ++ kubectl get ValidatingWebhookConfiguration ++ grep chaos-mesh ++ awk '{print $1}' + timeout 30 kubectl delete ValidatingWebhookConfiguration error: resource(s) were provided, but no name was specified + : ++ kubectl get ValidatingWebhookConfiguration ++ grep validate-auth ++ awk '{print $1}' + timeout 30 kubectl delete ValidatingWebhookConfiguration error: resource(s) were provided, but no name was specified + : ++ kubectl api-resources ++ grep chaos-mesh ++ awk '{print $1}' ++ kubectl get crd ++ awk '{print $1}' ++ grep chaos-mesh.org + timeout 30 kubectl delete crd error: resource(s) were provided, but no name was specified + : ++ kubectl get clusterrolebinding ++ grep chaos-mesh ++ awk '{print $1}' + timeout 30 kubectl delete clusterrolebinding error: resource(s) were provided, but no name was specified + : ++ kubectl get clusterrole ++ awk '{print $1}' ++ grep chaos-mesh + timeout 30 kubectl delete clusterrole error: resource(s) were provided, but no name was specified + : + desc 'cleaned up all old namespaces' + set +o xtrace ----------------------------------------------------------------------------------- cleaned up all old namespaces ----------------------------------------------------------------------------------- + '[' -n '' ']' + kubectl_bin get ns + awk '{print$1}' + egrep -v '^kube-|^default|Terminating|psmdb-operator|openshift|^gke-|^gmp-|^NAME' ++ mktemp + xargs kubectl delete ns + desc 'cleaned up old namespaces psmdb-operator' + set +o xtrace ----------------------------------------------------------------------------------- cleaned up old namespaces psmdb-operator ----------------------------------------------------------------------------------- + kubectl_bin delete namespace psmdb-operator --ignore-not-found egrep: warning: egrep is obsolescent; using grep -E ++ mktemp + local LAST_OUT=/tmp/tmp.djWmmMUsZj ++ mktemp + local LAST_OUT=/tmp/tmp.XVsF7Jkmha + local LAST_ERR=/tmp/tmp.JC16XsTA2a ++ mktemp + local exit_status=0 + local timeout=4 ++ seq 0 2 + local LAST_ERR=/tmp/tmp.VUSDUYO1af + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl delete namespace psmdb-operator --ignore-not-found + for i in $(seq 0 2) + set +e + kubectl get ns + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.djWmmMUsZj + cat /tmp/tmp.JC16XsTA2a + rm /tmp/tmp.djWmmMUsZj /tmp/tmp.JC16XsTA2a + return 0 namespace "recover-no-primary-22557" deleted + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.XVsF7Jkmha namespace "psmdb-operator" deleted + cat /tmp/tmp.VUSDUYO1af + rm /tmp/tmp.XVsF7Jkmha /tmp/tmp.VUSDUYO1af + return 0 + kubectl_bin wait --for=delete namespace psmdb-operator ++ mktemp + local LAST_OUT=/tmp/tmp.6niyLRIVaI ++ mktemp + local LAST_ERR=/tmp/tmp.J33M09sPlw + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl wait --for=delete namespace psmdb-operator + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.6niyLRIVaI + cat /tmp/tmp.J33M09sPlw + rm /tmp/tmp.6niyLRIVaI /tmp/tmp.J33M09sPlw + return 0 + desc 'create namespace psmdb-operator' + set +o xtrace ----------------------------------------------------------------------------------- create namespace psmdb-operator ----------------------------------------------------------------------------------- + kubectl_bin create namespace psmdb-operator ++ mktemp + local LAST_OUT=/tmp/tmp.HnacaJvHgH ++ mktemp + local LAST_ERR=/tmp/tmp.nlGQXyjjV0 + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl create namespace psmdb-operator + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.HnacaJvHgH namespace/psmdb-operator created + cat /tmp/tmp.nlGQXyjjV0 + rm /tmp/tmp.HnacaJvHgH /tmp/tmp.nlGQXyjjV0 + return 0 + set_kube_ctx psmdb-operator + local namespace=psmdb-operator ++ kubectl_bin config current-context +++ mktemp ++ local LAST_OUT=/tmp/tmp.oAIpxIBNE4 +++ mktemp ++ local LAST_ERR=/tmp/tmp.otcIzand0I ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl config current-context ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.oAIpxIBNE4 ++ cat /tmp/tmp.otcIzand0I ++ rm /tmp/tmp.oAIpxIBNE4 /tmp/tmp.otcIzand0I ++ return 0 + kubectl_bin config set-context gke_cloud-dev-112233_us-central1-a_jen-psmdb-2156-df65e35b-5-cluster4 --namespace=psmdb-operator ++ mktemp + local LAST_OUT=/tmp/tmp.oIO3qpZojf ++ mktemp + local LAST_ERR=/tmp/tmp.Ji6T6Mii4e + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl config set-context gke_cloud-dev-112233_us-central1-a_jen-psmdb-2156-df65e35b-5-cluster4 --namespace=psmdb-operator + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.oIO3qpZojf Context "gke_cloud-dev-112233_us-central1-a_jen-psmdb-2156-df65e35b-5-cluster4" modified. + cat /tmp/tmp.Ji6T6Mii4e + rm /tmp/tmp.oIO3qpZojf /tmp/tmp.Ji6T6Mii4e + return 0 + deploy_operator + desc 'start PSMDB operator: perconalab/percona-server-mongodb-operator:PR-2156-df65e35b' + set +o xtrace ----------------------------------------------------------------------------------- start PSMDB operator: perconalab/percona-server-mongodb-operator:PR-2156-df65e35b ----------------------------------------------------------------------------------- + local cr_file + '[' -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2156/e2e-tests/recover-no-primary/conf/crd.yaml ']' + cr_file=/mnt/jenkins/workspace/cloud-psmdb-operator_PR-2156/deploy/crd.yaml + kubectl_bin apply --server-side --force-conflicts -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2156/deploy/crd.yaml ++ mktemp + local LAST_OUT=/tmp/tmp.mxfmzgisQ3 ++ mktemp + local LAST_ERR=/tmp/tmp.FCzepCMUYA + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl apply --server-side --force-conflicts -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2156/deploy/crd.yaml + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.mxfmzgisQ3 customresourcedefinition.apiextensions.k8s.io/perconaservermongodbbackups.psmdb.percona.com serverside-applied customresourcedefinition.apiextensions.k8s.io/perconaservermongodbrestores.psmdb.percona.com serverside-applied customresourcedefinition.apiextensions.k8s.io/perconaservermongodbs.psmdb.percona.com serverside-applied + cat /tmp/tmp.FCzepCMUYA + rm /tmp/tmp.mxfmzgisQ3 /tmp/tmp.FCzepCMUYA + return 0 + '[' -n psmdb-operator ']' + apply_rbac cw-rbac + local operator_namespace=psmdb-operator + local rbac=cw-rbac + cat /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2156/deploy/cw-rbac.yaml + sed -e 's^namespace: .*^namespace: psmdb-operator^' + kubectl_bin apply -n psmdb-operator -f - ++ mktemp + local LAST_OUT=/tmp/tmp.auYzDunYFT ++ mktemp + local LAST_ERR=/tmp/tmp.glY9Cmjsky + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl apply -n psmdb-operator -f - + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.auYzDunYFT clusterrole.rbac.authorization.k8s.io/percona-server-mongodb-operator created serviceaccount/percona-server-mongodb-operator created clusterrolebinding.rbac.authorization.k8s.io/service-account-percona-server-mongodb-operator created + cat /tmp/tmp.glY9Cmjsky + rm /tmp/tmp.auYzDunYFT /tmp/tmp.glY9Cmjsky + return 0 + yq eval ' (.spec.template.spec.containers[].image = "perconalab/percona-server-mongodb-operator:PR-2156-df65e35b") | ((.. | select(.[] == "DISABLE_TELEMETRY")) |= .value="true") | ((.. | select(.[] == "LOG_LEVEL")) |= .value="DEBUG")' /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2156/deploy/cw-operator.yaml + kubectl_bin apply -n psmdb-operator -f - ++ mktemp + local LAST_OUT=/tmp/tmp.eN70XSIjoo ++ mktemp + local LAST_ERR=/tmp/tmp.pYhhVSayJX + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl apply -n psmdb-operator -f - + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.eN70XSIjoo deployment.apps/percona-server-mongodb-operator created + cat /tmp/tmp.pYhhVSayJX + rm /tmp/tmp.eN70XSIjoo /tmp/tmp.pYhhVSayJX + return 0 + sleep 20 ++ get_operator_pod ++ kubectl_bin get pods --selector=name=percona-server-mongodb-operator -o 'jsonpath={.items[].metadata.name}' -n psmdb-operator +++ mktemp ++ local LAST_OUT=/tmp/tmp.TEqsQow4Jj +++ mktemp ++ local LAST_ERR=/tmp/tmp.8Fjggz4Rxo ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pods --selector=name=percona-server-mongodb-operator -o 'jsonpath={.items[].metadata.name}' -n psmdb-operator ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.TEqsQow4Jj ++ cat /tmp/tmp.8Fjggz4Rxo ++ rm /tmp/tmp.TEqsQow4Jj /tmp/tmp.8Fjggz4Rxo ++ return 0 + wait_operator_pod percona-server-mongodb-operator-7d4f6569ff-bt2xf + local pod=percona-server-mongodb-operator-7d4f6569ff-bt2xf + set +o xtrace waiting for pod/percona-server-mongodb-operator-7d4f6569ff-bt2xf to be ready.OK + echo 'Print operator info from log' Print operator info from log + grep 'Manager starting up' ++ get_operator_pod ++ kubectl_bin get pods --selector=name=percona-server-mongodb-operator -o 'jsonpath={.items[].metadata.name}' -n psmdb-operator +++ mktemp ++ local LAST_OUT=/tmp/tmp.r2kqrsQBPn +++ mktemp ++ local LAST_ERR=/tmp/tmp.qX4Mi5V5dv ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pods --selector=name=percona-server-mongodb-operator -o 'jsonpath={.items[].metadata.name}' -n psmdb-operator ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.r2kqrsQBPn ++ cat /tmp/tmp.qX4Mi5V5dv ++ rm /tmp/tmp.r2kqrsQBPn /tmp/tmp.qX4Mi5V5dv ++ return 0 + kubectl_bin logs -n psmdb-operator percona-server-mongodb-operator-7d4f6569ff-bt2xf ++ mktemp + local LAST_OUT=/tmp/tmp.AREBkp0sUF ++ mktemp + local LAST_ERR=/tmp/tmp.k5ttCcLC2F + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl logs -n psmdb-operator percona-server-mongodb-operator-7d4f6569ff-bt2xf + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.AREBkp0sUF + cat /tmp/tmp.k5ttCcLC2F + rm /tmp/tmp.AREBkp0sUF /tmp/tmp.k5ttCcLC2F + return 0 2025-12-22T15:29:59.758Z INFO setup Manager starting up {"gitCommit": "df65e35bfbc9e7f85c7fb9471637843450b96a5d", "gitBranch": "PR-2156-df65e35b", "buildTime": "", "goVersion": "go1.25.5", "os": "linux", "arch": "amd64"} + create_namespace recover-no-primary-29251 + local namespace=recover-no-primary-29251 + local skip_clean_namespace= + [[ 1 == 1 ]] + [[ -z '' ]] + destroy_chaos_mesh ++ helm list --all-namespaces --filter chaos-mesh ++ tail -n1 ++ awk '-F ' '{print $2}' ++ sed s/NAMESPACE// + local chaos_mesh_ns= + desc 'destroy chaos-mesh' + set +o xtrace ----------------------------------------------------------------------------------- destroy chaos-mesh ----------------------------------------------------------------------------------- + '[' -n '' ']' ++ kubectl get MutatingWebhookConfiguration ++ grep chaos-mesh ++ awk '{print $1}' + timeout 30 kubectl delete MutatingWebhookConfiguration error: resource(s) were provided, but no name was specified + : ++ kubectl get ValidatingWebhookConfiguration ++ grep chaos-mesh ++ awk '{print $1}' + timeout 30 kubectl delete ValidatingWebhookConfiguration error: resource(s) were provided, but no name was specified + : ++ kubectl get ValidatingWebhookConfiguration ++ grep validate-auth ++ awk '{print $1}' + timeout 30 kubectl delete ValidatingWebhookConfiguration error: resource(s) were provided, but no name was specified + : ++ kubectl api-resources ++ grep chaos-mesh ++ awk '{print $1}' ++ kubectl get crd ++ grep chaos-mesh.org ++ awk '{print $1}' + timeout 30 kubectl delete crd error: resource(s) were provided, but no name was specified + : ++ kubectl get clusterrolebinding ++ grep chaos-mesh ++ awk '{print $1}' + timeout 30 kubectl delete clusterrolebinding error: resource(s) were provided, but no name was specified + : ++ grep chaos-mesh ++ kubectl get clusterrole ++ awk '{print $1}' + timeout 30 kubectl delete clusterrole error: resource(s) were provided, but no name was specified + : + desc 'cleaned up all old namespaces' + set +o xtrace ----------------------------------------------------------------------------------- cleaned up all old namespaces ----------------------------------------------------------------------------------- + '[' -n '' ']' + desc 'cleaned up old namespaces recover-no-primary-29251' + set +o xtrace ----------------------------------------------------------------------------------- cleaned up old namespaces recover-no-primary-29251 ----------------------------------------------------------------------------------- + kubectl_bin delete namespace recover-no-primary-29251 --ignore-not-found + kubectl_bin get ns + egrep -v '^kube-|^default|Terminating|psmdb-operator|openshift|^gke-|^gmp-|^NAME' ++ mktemp egrep: warning: egrep is obsolescent; using grep -E + awk '{print$1}' ++ mktemp + xargs kubectl delete ns + local LAST_OUT=/tmp/tmp.LAzmBG3uQc + local LAST_OUT=/tmp/tmp.mFHas6H5R8 ++ mktemp ++ mktemp + local LAST_ERR=/tmp/tmp.40rCmON6Zu + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl delete namespace recover-no-primary-29251 --ignore-not-found + local LAST_ERR=/tmp/tmp.vGT2Gep7Of + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl get ns + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.mFHas6H5R8 + cat /tmp/tmp.40rCmON6Zu + rm /tmp/tmp.mFHas6H5R8 /tmp/tmp.40rCmON6Zu + return 0 + kubectl_bin wait --for=delete namespace recover-no-primary-29251 ++ mktemp + local LAST_OUT=/tmp/tmp.ux3laWAItk ++ mktemp + local LAST_ERR=/tmp/tmp.HKNynZGJcl + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl wait --for=delete namespace recover-no-primary-29251 + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.LAzmBG3uQc + cat /tmp/tmp.vGT2Gep7Of + rm /tmp/tmp.LAzmBG3uQc /tmp/tmp.vGT2Gep7Of + return 0 error: resource(s) were provided, but no name was specified + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.ux3laWAItk + cat /tmp/tmp.HKNynZGJcl + rm /tmp/tmp.ux3laWAItk /tmp/tmp.HKNynZGJcl + return 0 + desc 'create namespace recover-no-primary-29251' + set +o xtrace ----------------------------------------------------------------------------------- create namespace recover-no-primary-29251 ----------------------------------------------------------------------------------- + kubectl_bin create namespace recover-no-primary-29251 ++ mktemp + local LAST_OUT=/tmp/tmp.DvhKZWsQwR ++ mktemp + local LAST_ERR=/tmp/tmp.pBv2L9FE57 + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl create namespace recover-no-primary-29251 + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.DvhKZWsQwR namespace/recover-no-primary-29251 created + cat /tmp/tmp.pBv2L9FE57 + rm /tmp/tmp.DvhKZWsQwR /tmp/tmp.pBv2L9FE57 + return 0 + set_kube_ctx recover-no-primary-29251 + local namespace=recover-no-primary-29251 ++ kubectl_bin config current-context +++ mktemp ++ local LAST_OUT=/tmp/tmp.YHG2CrVjXr +++ mktemp ++ local LAST_ERR=/tmp/tmp.qoNMQsWkRt ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl config current-context ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.YHG2CrVjXr ++ cat /tmp/tmp.qoNMQsWkRt ++ rm /tmp/tmp.YHG2CrVjXr /tmp/tmp.qoNMQsWkRt ++ return 0 + kubectl_bin config set-context gke_cloud-dev-112233_us-central1-a_jen-psmdb-2156-df65e35b-5-cluster4 --namespace=recover-no-primary-29251 ++ mktemp + local LAST_OUT=/tmp/tmp.RJJ7SiCBEP ++ mktemp + local LAST_ERR=/tmp/tmp.isDI2huovK + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl config set-context gke_cloud-dev-112233_us-central1-a_jen-psmdb-2156-df65e35b-5-cluster4 --namespace=recover-no-primary-29251 + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.RJJ7SiCBEP Context "gke_cloud-dev-112233_us-central1-a_jen-psmdb-2156-df65e35b-5-cluster4" modified. + cat /tmp/tmp.isDI2huovK + rm /tmp/tmp.RJJ7SiCBEP /tmp/tmp.isDI2huovK + return 0 + cluster=some-name + kubectl_bin apply -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2156/e2e-tests/conf/secrets_with_tls.yml -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2156/e2e-tests/conf/client.yml ++ mktemp + local LAST_OUT=/tmp/tmp.GLyEPjyTIu ++ mktemp + local LAST_ERR=/tmp/tmp.GMhevDeUdo + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl apply -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2156/e2e-tests/conf/secrets_with_tls.yml -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2156/e2e-tests/conf/client.yml + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.GLyEPjyTIu secret/some-users created secret/some-name-ssl created secret/some-name-ssl-internal created deployment.apps/psmdb-client created + cat /tmp/tmp.GMhevDeUdo + rm /tmp/tmp.GLyEPjyTIu /tmp/tmp.GMhevDeUdo + return 0 + desc 'testing unexposed single replset cluster' + set +o xtrace ----------------------------------------------------------------------------------- testing unexposed single replset cluster ----------------------------------------------------------------------------------- + test_single_replset + apply_cluster /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2156/e2e-tests/recover-no-primary/conf/some-name.yml + '[' -z '' ']' + cat_config /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2156/e2e-tests/recover-no-primary/conf/some-name.yml + kubectl_bin apply -f - + cat /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2156/e2e-tests/recover-no-primary/conf/some-name.yml ++ mktemp + yq eval '(.spec | select(has("backup"))).backup.image = "perconalab/percona-server-mongodb-operator:main-backup"' + yq eval '(.spec | select(.image == null)).image = "perconalab/percona-server-mongodb-operator:main-mongod8.0"' + /usr/sbin/sed -e s/NAME_SPACE/recover-no-primary-29251/g + local LAST_OUT=/tmp/tmp.5P6RUnIADh + yq eval '(.spec | select(has("pmm"))).pmm.image = "percona/pmm-client:2.44.1-1"' + yq eval '(.spec | select(has("initImage"))).initImage = "perconalab/percona-server-mongodb-operator:PR-2156-df65e35b"' ++ mktemp + yq eval '.spec.upgradeOptions.apply="Never"' + local LAST_ERR=/tmp/tmp.fIRU1XaGCT + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.5P6RUnIADh perconaservermongodb.psmdb.percona.com/some-name created + cat /tmp/tmp.fIRU1XaGCT + rm /tmp/tmp.5P6RUnIADh /tmp/tmp.fIRU1XaGCT + return 0 + wait_for_running some-name-rs0 3 + local name=some-name-rs0 + let last_pod=2 + local check_cluster_readyness=true + set_debug + [[ 1 == 1 ]] + set -o xtrace + local rs_name=rs0 + local cluster_name=some-name ++ seq 0 2 + for i in $(seq 0 $last_pod) + [[ 0 -eq 2 ]] + wait_pod some-name-rs0-0 + local pod=some-name-rs0-0 + set +o xtrace waiting for pod/some-name-rs0-0 to be ready.............OK + for i in $(seq 0 $last_pod) + [[ 1 -eq 2 ]] + wait_pod some-name-rs0-1 + local pod=some-name-rs0-1 + set +o xtrace waiting for pod/some-name-rs0-1 to be ready..........OK + for i in $(seq 0 $last_pod) + [[ 2 -eq 2 ]] ++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].arbiter.enabled}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.moW9nr7gKV +++ mktemp ++ local LAST_ERR=/tmp/tmp.ONjLOzplp1 ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].arbiter.enabled}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.moW9nr7gKV ++ cat /tmp/tmp.ONjLOzplp1 ++ rm /tmp/tmp.moW9nr7gKV /tmp/tmp.ONjLOzplp1 ++ return 0 + [[ '' == \t\r\u\e ]] + wait_pod some-name-rs0-2 + local pod=some-name-rs0-2 + set +o xtrace waiting for pod/some-name-rs0-2 to be ready............OK ++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].nonvoting.enabled}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.kxB8RBxZHr +++ mktemp ++ local LAST_ERR=/tmp/tmp.6khp1Veg0R ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].nonvoting.enabled}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.kxB8RBxZHr ++ cat /tmp/tmp.6khp1Veg0R ++ rm /tmp/tmp.kxB8RBxZHr /tmp/tmp.6khp1Veg0R ++ return 0 + [[ '' == \t\r\u\e ]] ++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].hidden.enabled}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.77KO3wCYYS +++ mktemp ++ local LAST_ERR=/tmp/tmp.StWjh7MbL0 ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].hidden.enabled}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.77KO3wCYYS ++ cat /tmp/tmp.StWjh7MbL0 ++ rm /tmp/tmp.77KO3wCYYS /tmp/tmp.StWjh7MbL0 ++ return 0 + [[ '' == \t\r\u\e ]] + sleep 10 + [[ true == \t\r\u\e ]] + set +x Waiting for cluster readyness + wait_cluster_consistency some-name + local cluster_name=some-name + local wait_time=32 + retry=0 + sleep 7 + echo -n 'waiting for cluster readyness' waiting for cluster readyness++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.xxubDRUbjK +++ mktemp ++ local LAST_ERR=/tmp/tmp.e8KhVhOQlZ ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb some-name -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.xxubDRUbjK ++ cat /tmp/tmp.e8KhVhOQlZ ++ rm /tmp/tmp.xxubDRUbjK /tmp/tmp.e8KhVhOQlZ ++ return 0 + [[ ready == \r\e\a\d\y ]] + echo .OK .OK + desc 'kill pods abruptly and wait for recovery' + set +o xtrace ----------------------------------------------------------------------------------- kill pods abruptly and wait for recovery ----------------------------------------------------------------------------------- ++ seq 0 2 + for i in $(seq 0 2) + kubectl_bin exec some-name-rs0-0 -c mongod -- kill 1 ++ mktemp + local LAST_OUT=/tmp/tmp.D5bWRhdf1L ++ mktemp + local LAST_ERR=/tmp/tmp.vmR4ISVPAb + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl exec some-name-rs0-0 -c mongod -- kill 1 + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.D5bWRhdf1L + cat /tmp/tmp.vmR4ISVPAb + rm /tmp/tmp.D5bWRhdf1L /tmp/tmp.vmR4ISVPAb + return 0 + for i in $(seq 0 2) + kubectl_bin exec some-name-rs0-1 -c mongod -- kill 1 ++ mktemp + local LAST_OUT=/tmp/tmp.x4ZbyXWTiD ++ mktemp + local LAST_ERR=/tmp/tmp.66guMXUa3O + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl exec some-name-rs0-1 -c mongod -- kill 1 + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.x4ZbyXWTiD + cat /tmp/tmp.66guMXUa3O + rm /tmp/tmp.x4ZbyXWTiD /tmp/tmp.66guMXUa3O + return 0 + for i in $(seq 0 2) + kubectl_bin exec some-name-rs0-2 -c mongod -- kill 1 ++ mktemp + local LAST_OUT=/tmp/tmp.kDKkgpMUjT ++ mktemp + local LAST_ERR=/tmp/tmp.GnE79ckGrK + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl exec some-name-rs0-2 -c mongod -- kill 1 + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.kDKkgpMUjT + cat /tmp/tmp.GnE79ckGrK + rm /tmp/tmp.kDKkgpMUjT /tmp/tmp.GnE79ckGrK + return 0 + sleep 10 + wait_for_running some-name-rs0 3 + local name=some-name-rs0 + let last_pod=2 + local check_cluster_readyness=true + set_debug + [[ 1 == 1 ]] + set -o xtrace + local rs_name=rs0 + local cluster_name=some-name ++ seq 0 2 + for i in $(seq 0 $last_pod) + [[ 0 -eq 2 ]] + wait_pod some-name-rs0-0 + local pod=some-name-rs0-0 + set +o xtrace waiting for pod/some-name-rs0-0 to be ready.OK + for i in $(seq 0 $last_pod) + [[ 1 -eq 2 ]] + wait_pod some-name-rs0-1 + local pod=some-name-rs0-1 + set +o xtrace waiting for pod/some-name-rs0-1 to be ready.OK + for i in $(seq 0 $last_pod) + [[ 2 -eq 2 ]] ++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].arbiter.enabled}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.QH0xGawp8P +++ mktemp ++ local LAST_ERR=/tmp/tmp.DEW7C6hzZH ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].arbiter.enabled}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.QH0xGawp8P ++ cat /tmp/tmp.DEW7C6hzZH ++ rm /tmp/tmp.QH0xGawp8P /tmp/tmp.DEW7C6hzZH ++ return 0 + [[ '' == \t\r\u\e ]] + wait_pod some-name-rs0-2 + local pod=some-name-rs0-2 + set +o xtrace waiting for pod/some-name-rs0-2 to be ready.OK ++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].nonvoting.enabled}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.b7owtduzWO +++ mktemp ++ local LAST_ERR=/tmp/tmp.tLKsgFRxjj ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].nonvoting.enabled}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.b7owtduzWO ++ cat /tmp/tmp.tLKsgFRxjj ++ rm /tmp/tmp.b7owtduzWO /tmp/tmp.tLKsgFRxjj ++ return 0 + [[ '' == \t\r\u\e ]] ++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].hidden.enabled}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.bMJEEAMxL1 +++ mktemp ++ local LAST_ERR=/tmp/tmp.tToH4VGv2g ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].hidden.enabled}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.bMJEEAMxL1 ++ cat /tmp/tmp.tToH4VGv2g ++ rm /tmp/tmp.bMJEEAMxL1 /tmp/tmp.tToH4VGv2g ++ return 0 + [[ '' == \t\r\u\e ]] + sleep 10 + [[ true == \t\r\u\e ]] + set +x Waiting for cluster readyness. + wait_cluster_consistency some-name + local cluster_name=some-name + local wait_time=32 + retry=0 + sleep 7 + echo -n 'waiting for cluster readyness' waiting for cluster readyness++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.hk6EodUZiA +++ mktemp ++ local LAST_ERR=/tmp/tmp.tawSUpR8l5 ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb some-name -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.hk6EodUZiA ++ cat /tmp/tmp.tawSUpR8l5 ++ rm /tmp/tmp.hk6EodUZiA /tmp/tmp.tawSUpR8l5 ++ return 0 + [[ ready == \r\e\a\d\y ]] + echo .OK .OK + desc 'delete statefulset and wait for recovery' + set +o xtrace ----------------------------------------------------------------------------------- delete statefulset and wait for recovery ----------------------------------------------------------------------------------- + kubectl delete sts some-name-rs0 statefulset.apps "some-name-rs0" deleted from recover-no-primary-29251 namespace + sleep 5 + wait_for_running some-name-rs0 3 + local name=some-name-rs0 + let last_pod=2 + local check_cluster_readyness=true + set_debug + [[ 1 == 1 ]] + set -o xtrace + local rs_name=rs0 + local cluster_name=some-name ++ seq 0 2 + for i in $(seq 0 $last_pod) + [[ 0 -eq 2 ]] + wait_pod some-name-rs0-0 + local pod=some-name-rs0-0 + set +o xtrace waiting for pod/some-name-rs0-0 to be ready.OK + for i in $(seq 0 $last_pod) + [[ 1 -eq 2 ]] + wait_pod some-name-rs0-1 + local pod=some-name-rs0-1 + set +o xtrace waiting for pod/some-name-rs0-1 to be ready.OK + for i in $(seq 0 $last_pod) + [[ 2 -eq 2 ]] ++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].arbiter.enabled}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.DUWraMwLhz +++ mktemp ++ local LAST_ERR=/tmp/tmp.otnrzu3fgB ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].arbiter.enabled}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.DUWraMwLhz ++ cat /tmp/tmp.otnrzu3fgB ++ rm /tmp/tmp.DUWraMwLhz /tmp/tmp.otnrzu3fgB ++ return 0 + [[ '' == \t\r\u\e ]] + wait_pod some-name-rs0-2 + local pod=some-name-rs0-2 + set +o xtrace waiting for pod/some-name-rs0-2 to be ready.OK ++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].nonvoting.enabled}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.wnKSxo5fSY +++ mktemp ++ local LAST_ERR=/tmp/tmp.HTQTkgTHGd ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].nonvoting.enabled}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.wnKSxo5fSY ++ cat /tmp/tmp.HTQTkgTHGd ++ rm /tmp/tmp.wnKSxo5fSY /tmp/tmp.HTQTkgTHGd ++ return 0 + [[ '' == \t\r\u\e ]] ++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].hidden.enabled}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.RlwLFkgvyc +++ mktemp ++ local LAST_ERR=/tmp/tmp.lIR7WswpVk ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].hidden.enabled}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.RlwLFkgvyc ++ cat /tmp/tmp.lIR7WswpVk ++ rm /tmp/tmp.RlwLFkgvyc /tmp/tmp.lIR7WswpVk ++ return 0 + [[ '' == \t\r\u\e ]] + sleep 10 + [[ true == \t\r\u\e ]] + set +x Waiting for cluster readyness............................................................ + wait_cluster_consistency some-name + local cluster_name=some-name + local wait_time=32 + retry=0 + sleep 7 + echo -n 'waiting for cluster readyness' waiting for cluster readyness++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.r35riteRYr +++ mktemp ++ local LAST_ERR=/tmp/tmp.kr6qQY4zXD ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb some-name -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.r35riteRYr ++ cat /tmp/tmp.kr6qQY4zXD ++ rm /tmp/tmp.r35riteRYr /tmp/tmp.kr6qQY4zXD ++ return 0 + [[ ready == \r\e\a\d\y ]] + echo .OK .OK + desc 'delete cluster and recreate' + set +o xtrace ----------------------------------------------------------------------------------- delete cluster and recreate ----------------------------------------------------------------------------------- + kubectl delete psmdb some-name perconaservermongodb.psmdb.percona.com "some-name" deleted from recover-no-primary-29251 namespace + set +e + echo -n 'waiting for psmdb/some-name to be deleted' waiting for psmdb/some-name to be deleted+ kubectl get psmdb some-name Error from server (NotFound): perconaservermongodbs.psmdb.percona.com "some-name" not found + set -e + apply_cluster /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2156/e2e-tests/recover-no-primary/conf/some-name.yml + '[' -z '' ']' + cat_config /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2156/e2e-tests/recover-no-primary/conf/some-name.yml + kubectl_bin apply -f - + cat /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2156/e2e-tests/recover-no-primary/conf/some-name.yml + yq eval '(.spec | select(.image == null)).image = "perconalab/percona-server-mongodb-operator:main-mongod8.0"' + yq eval '(.spec | select(has("pmm"))).pmm.image = "percona/pmm-client:2.44.1-1"' ++ mktemp + yq eval '(.spec | select(has("initImage"))).initImage = "perconalab/percona-server-mongodb-operator:PR-2156-df65e35b"' + yq eval '(.spec | select(has("backup"))).backup.image = "perconalab/percona-server-mongodb-operator:main-backup"' + /usr/sbin/sed -e s/NAME_SPACE/recover-no-primary-29251/g + local LAST_OUT=/tmp/tmp.b1hOp2fhzs ++ mktemp + yq eval '.spec.upgradeOptions.apply="Never"' + local LAST_ERR=/tmp/tmp.f7AXyLUwQZ + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.b1hOp2fhzs perconaservermongodb.psmdb.percona.com/some-name created + cat /tmp/tmp.f7AXyLUwQZ + rm /tmp/tmp.b1hOp2fhzs /tmp/tmp.f7AXyLUwQZ + return 0 + wait_for_running some-name-rs0 3 + local name=some-name-rs0 + let last_pod=2 + local check_cluster_readyness=true + set_debug + [[ 1 == 1 ]] + set -o xtrace + local rs_name=rs0 + local cluster_name=some-name ++ seq 0 2 + for i in $(seq 0 $last_pod) + [[ 0 -eq 2 ]] + wait_pod some-name-rs0-0 + local pod=some-name-rs0-0 + set +o xtrace waiting for pod/some-name-rs0-0 to be ready.OK + for i in $(seq 0 $last_pod) + [[ 1 -eq 2 ]] + wait_pod some-name-rs0-1 + local pod=some-name-rs0-1 + set +o xtrace waiting for pod/some-name-rs0-1 to be ready.OK + for i in $(seq 0 $last_pod) + [[ 2 -eq 2 ]] ++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].arbiter.enabled}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.hDZltLGvXs +++ mktemp ++ local LAST_ERR=/tmp/tmp.6whaBFAk3p ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].arbiter.enabled}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.hDZltLGvXs ++ cat /tmp/tmp.6whaBFAk3p ++ rm /tmp/tmp.hDZltLGvXs /tmp/tmp.6whaBFAk3p ++ return 0 + [[ '' == \t\r\u\e ]] + wait_pod some-name-rs0-2 + local pod=some-name-rs0-2 + set +o xtrace waiting for pod/some-name-rs0-2 to be ready.OK ++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].nonvoting.enabled}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.Esud5wO1sK +++ mktemp ++ local LAST_ERR=/tmp/tmp.1qmhdnKHcH ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].nonvoting.enabled}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.Esud5wO1sK ++ cat /tmp/tmp.1qmhdnKHcH ++ rm /tmp/tmp.Esud5wO1sK /tmp/tmp.1qmhdnKHcH ++ return 0 + [[ '' == \t\r\u\e ]] ++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].hidden.enabled}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.AMtkG2Wlpk +++ mktemp ++ local LAST_ERR=/tmp/tmp.rUSrxSi968 ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].hidden.enabled}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.AMtkG2Wlpk ++ cat /tmp/tmp.rUSrxSi968 ++ rm /tmp/tmp.AMtkG2Wlpk /tmp/tmp.rUSrxSi968 ++ return 0 + [[ '' == \t\r\u\e ]] + sleep 10 + [[ true == \t\r\u\e ]] + set +x Waiting for cluster readyness.......................................... + wait_cluster_consistency some-name + local cluster_name=some-name + local wait_time=32 + retry=0 + sleep 7 + echo -n 'waiting for cluster readyness' waiting for cluster readyness++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.mXLNTwNiqy +++ mktemp ++ local LAST_ERR=/tmp/tmp.JJRdaHvWRM ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb some-name -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.mXLNTwNiqy ++ cat /tmp/tmp.JJRdaHvWRM ++ rm /tmp/tmp.mXLNTwNiqy /tmp/tmp.JJRdaHvWRM ++ return 0 + [[ ready == \r\e\a\d\y ]] + echo .OK .OK + kubectl delete psmdb some-name perconaservermongodb.psmdb.percona.com "some-name" deleted from recover-no-primary-29251 namespace + kubectl delete pvc --all persistentvolumeclaim "mongod-data-some-name-rs0-0" deleted from recover-no-primary-29251 namespace persistentvolumeclaim "mongod-data-some-name-rs0-1" deleted from recover-no-primary-29251 namespace persistentvolumeclaim "mongod-data-some-name-rs0-2" deleted from recover-no-primary-29251 namespace + desc 'testing exposed single replset cluster' + set +o xtrace ----------------------------------------------------------------------------------- testing exposed single replset cluster ----------------------------------------------------------------------------------- + test_exposed_single_replset + apply_cluster /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2156/e2e-tests/recover-no-primary/conf/some-name-exposed.yml + '[' -z '' ']' + cat_config /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2156/e2e-tests/recover-no-primary/conf/some-name-exposed.yml + kubectl_bin apply -f - + cat /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2156/e2e-tests/recover-no-primary/conf/some-name-exposed.yml + yq eval '(.spec | select(.image == null)).image = "perconalab/percona-server-mongodb-operator:main-mongod8.0"' ++ mktemp + yq eval '(.spec | select(has("pmm"))).pmm.image = "percona/pmm-client:2.44.1-1"' + yq eval '(.spec | select(has("initImage"))).initImage = "perconalab/percona-server-mongodb-operator:PR-2156-df65e35b"' + yq eval '(.spec | select(has("backup"))).backup.image = "perconalab/percona-server-mongodb-operator:main-backup"' + /usr/sbin/sed -e s/NAME_SPACE/recover-no-primary-29251/g + yq eval '.spec.upgradeOptions.apply="Never"' + local LAST_OUT=/tmp/tmp.zWqvMOmYfd ++ mktemp + local LAST_ERR=/tmp/tmp.tzQizmDpUq + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.zWqvMOmYfd perconaservermongodb.psmdb.percona.com/some-name created + cat /tmp/tmp.tzQizmDpUq + rm /tmp/tmp.zWqvMOmYfd /tmp/tmp.tzQizmDpUq + return 0 + wait_for_running some-name-rs0 3 + local name=some-name-rs0 + let last_pod=2 + local check_cluster_readyness=true + set_debug + [[ 1 == 1 ]] + set -o xtrace + local rs_name=rs0 + local cluster_name=some-name ++ seq 0 2 + for i in $(seq 0 $last_pod) + [[ 0 -eq 2 ]] + wait_pod some-name-rs0-0 + local pod=some-name-rs0-0 + set +o xtrace waiting for pod/some-name-rs0-0 to be ready...........OK + for i in $(seq 0 $last_pod) + [[ 1 -eq 2 ]] + wait_pod some-name-rs0-1 + local pod=some-name-rs0-1 + set +o xtrace waiting for pod/some-name-rs0-1 to be ready..........OK + for i in $(seq 0 $last_pod) + [[ 2 -eq 2 ]] ++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].arbiter.enabled}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.t21s5XBSX0 +++ mktemp ++ local LAST_ERR=/tmp/tmp.gll0GluQNw ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].arbiter.enabled}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.t21s5XBSX0 ++ cat /tmp/tmp.gll0GluQNw ++ rm /tmp/tmp.t21s5XBSX0 /tmp/tmp.gll0GluQNw ++ return 0 + [[ '' == \t\r\u\e ]] + wait_pod some-name-rs0-2 + local pod=some-name-rs0-2 + set +o xtrace waiting for pod/some-name-rs0-2 to be ready...........OK ++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].nonvoting.enabled}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.tjmo6ic5a0 +++ mktemp ++ local LAST_ERR=/tmp/tmp.8VTPyT8Bxs ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].nonvoting.enabled}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.tjmo6ic5a0 ++ cat /tmp/tmp.8VTPyT8Bxs ++ rm /tmp/tmp.tjmo6ic5a0 /tmp/tmp.8VTPyT8Bxs ++ return 0 + [[ '' == \t\r\u\e ]] ++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].hidden.enabled}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.BD7ehjlOSq +++ mktemp ++ local LAST_ERR=/tmp/tmp.457A0OLzZw ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].hidden.enabled}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.BD7ehjlOSq ++ cat /tmp/tmp.457A0OLzZw ++ rm /tmp/tmp.BD7ehjlOSq /tmp/tmp.457A0OLzZw ++ return 0 + [[ '' == \t\r\u\e ]] + sleep 10 + [[ true == \t\r\u\e ]] + set +x Waiting for cluster readyness + wait_cluster_consistency some-name + local cluster_name=some-name + local wait_time=32 + retry=0 + sleep 7 + echo -n 'waiting for cluster readyness' waiting for cluster readyness++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.I7ZNznuOLA +++ mktemp ++ local LAST_ERR=/tmp/tmp.wY1gQ5EEtA ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb some-name -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.I7ZNznuOLA ++ cat /tmp/tmp.wY1gQ5EEtA ++ rm /tmp/tmp.I7ZNznuOLA /tmp/tmp.wY1gQ5EEtA ++ return 0 + [[ ready == \r\e\a\d\y ]] + echo .OK .OK + desc 'kill pods abruptly and wait for recovery' + set +o xtrace ----------------------------------------------------------------------------------- kill pods abruptly and wait for recovery ----------------------------------------------------------------------------------- ++ seq 0 2 + for i in $(seq 0 2) + kubectl_bin exec some-name-rs0-0 -c mongod -- kill 1 ++ mktemp + local LAST_OUT=/tmp/tmp.RF6ZCZfFA0 ++ mktemp + local LAST_ERR=/tmp/tmp.rdD0zM7H9C + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl exec some-name-rs0-0 -c mongod -- kill 1 + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.RF6ZCZfFA0 + cat /tmp/tmp.rdD0zM7H9C + rm /tmp/tmp.RF6ZCZfFA0 /tmp/tmp.rdD0zM7H9C + return 0 + for i in $(seq 0 2) + kubectl_bin exec some-name-rs0-1 -c mongod -- kill 1 ++ mktemp + local LAST_OUT=/tmp/tmp.svFbwpxQ3e ++ mktemp + local LAST_ERR=/tmp/tmp.x1moE0vDAe + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl exec some-name-rs0-1 -c mongod -- kill 1 + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.svFbwpxQ3e + cat /tmp/tmp.x1moE0vDAe + rm /tmp/tmp.svFbwpxQ3e /tmp/tmp.x1moE0vDAe + return 0 + for i in $(seq 0 2) + kubectl_bin exec some-name-rs0-2 -c mongod -- kill 1 ++ mktemp + local LAST_OUT=/tmp/tmp.YEkug8MsDm ++ mktemp + local LAST_ERR=/tmp/tmp.siLxCxr34j + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl exec some-name-rs0-2 -c mongod -- kill 1 + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.YEkug8MsDm + cat /tmp/tmp.siLxCxr34j + rm /tmp/tmp.YEkug8MsDm /tmp/tmp.siLxCxr34j + return 0 + sleep 10 + wait_for_running some-name-rs0 3 + local name=some-name-rs0 + let last_pod=2 + local check_cluster_readyness=true + set_debug + [[ 1 == 1 ]] + set -o xtrace + local rs_name=rs0 + local cluster_name=some-name ++ seq 0 2 + for i in $(seq 0 $last_pod) + [[ 0 -eq 2 ]] + wait_pod some-name-rs0-0 + local pod=some-name-rs0-0 + set +o xtrace waiting for pod/some-name-rs0-0 to be ready.OK + for i in $(seq 0 $last_pod) + [[ 1 -eq 2 ]] + wait_pod some-name-rs0-1 + local pod=some-name-rs0-1 + set +o xtrace waiting for pod/some-name-rs0-1 to be ready.OK + for i in $(seq 0 $last_pod) + [[ 2 -eq 2 ]] ++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].arbiter.enabled}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.i5Rcqf3YNv +++ mktemp ++ local LAST_ERR=/tmp/tmp.0ScyYtvzCJ ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].arbiter.enabled}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.i5Rcqf3YNv ++ cat /tmp/tmp.0ScyYtvzCJ ++ rm /tmp/tmp.i5Rcqf3YNv /tmp/tmp.0ScyYtvzCJ ++ return 0 + [[ '' == \t\r\u\e ]] + wait_pod some-name-rs0-2 + local pod=some-name-rs0-2 + set +o xtrace waiting for pod/some-name-rs0-2 to be ready.OK ++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].nonvoting.enabled}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.q8f4mbicH8 +++ mktemp ++ local LAST_ERR=/tmp/tmp.Mrmx3KQVmy ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].nonvoting.enabled}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.q8f4mbicH8 ++ cat /tmp/tmp.Mrmx3KQVmy ++ rm /tmp/tmp.q8f4mbicH8 /tmp/tmp.Mrmx3KQVmy ++ return 0 + [[ '' == \t\r\u\e ]] ++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].hidden.enabled}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.ZLznIINcDd +++ mktemp ++ local LAST_ERR=/tmp/tmp.lY5mhqSQe3 ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].hidden.enabled}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.ZLznIINcDd ++ cat /tmp/tmp.lY5mhqSQe3 ++ rm /tmp/tmp.ZLznIINcDd /tmp/tmp.lY5mhqSQe3 ++ return 0 + [[ '' == \t\r\u\e ]] + sleep 10 + [[ true == \t\r\u\e ]] + set +x Waiting for cluster readyness. + wait_cluster_consistency some-name + local cluster_name=some-name + local wait_time=32 + retry=0 + sleep 7 + echo -n 'waiting for cluster readyness' waiting for cluster readyness++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.zx1vm0m8bB +++ mktemp ++ local LAST_ERR=/tmp/tmp.Z0ME0v103g ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb some-name -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.zx1vm0m8bB ++ cat /tmp/tmp.Z0ME0v103g ++ rm /tmp/tmp.zx1vm0m8bB /tmp/tmp.Z0ME0v103g ++ return 0 + [[ ready == \r\e\a\d\y ]] + echo .OK .OK + desc 'delete statefulset and wait for recovery' + set +o xtrace ----------------------------------------------------------------------------------- delete statefulset and wait for recovery ----------------------------------------------------------------------------------- + kubectl delete sts some-name-rs0 statefulset.apps "some-name-rs0" deleted from recover-no-primary-29251 namespace + sleep 5 + wait_for_running some-name-rs0 3 + local name=some-name-rs0 + let last_pod=2 + local check_cluster_readyness=true + set_debug + [[ 1 == 1 ]] + set -o xtrace + local rs_name=rs0 + local cluster_name=some-name ++ seq 0 2 + for i in $(seq 0 $last_pod) + [[ 0 -eq 2 ]] + wait_pod some-name-rs0-0 + local pod=some-name-rs0-0 + set +o xtrace waiting for pod/some-name-rs0-0 to be ready.OK + for i in $(seq 0 $last_pod) + [[ 1 -eq 2 ]] + wait_pod some-name-rs0-1 + local pod=some-name-rs0-1 + set +o xtrace waiting for pod/some-name-rs0-1 to be ready.OK + for i in $(seq 0 $last_pod) + [[ 2 -eq 2 ]] ++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].arbiter.enabled}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.LXFRXDYxzx +++ mktemp ++ local LAST_ERR=/tmp/tmp.90OcW9aSfu ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].arbiter.enabled}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.LXFRXDYxzx ++ cat /tmp/tmp.90OcW9aSfu ++ rm /tmp/tmp.LXFRXDYxzx /tmp/tmp.90OcW9aSfu ++ return 0 + [[ '' == \t\r\u\e ]] + wait_pod some-name-rs0-2 + local pod=some-name-rs0-2 + set +o xtrace waiting for pod/some-name-rs0-2 to be ready.OK ++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].nonvoting.enabled}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.RBHoYFCKya +++ mktemp ++ local LAST_ERR=/tmp/tmp.fe6ljrK08a ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].nonvoting.enabled}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.RBHoYFCKya ++ cat /tmp/tmp.fe6ljrK08a ++ rm /tmp/tmp.RBHoYFCKya /tmp/tmp.fe6ljrK08a ++ return 0 + [[ '' == \t\r\u\e ]] ++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].hidden.enabled}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.G0EW0Kn8wd +++ mktemp ++ local LAST_ERR=/tmp/tmp.YbPhbKbUA3 ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].hidden.enabled}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.G0EW0Kn8wd ++ cat /tmp/tmp.YbPhbKbUA3 ++ rm /tmp/tmp.G0EW0Kn8wd /tmp/tmp.YbPhbKbUA3 ++ return 0 + [[ '' == \t\r\u\e ]] + sleep 10 + [[ true == \t\r\u\e ]] + set +x Waiting for cluster readyness............................. + wait_cluster_consistency some-name + local cluster_name=some-name + local wait_time=32 + retry=0 + sleep 7 + echo -n 'waiting for cluster readyness' waiting for cluster readyness++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.jAu0mEjnCW +++ mktemp ++ local LAST_ERR=/tmp/tmp.jXq5fq5YUx ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb some-name -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.jAu0mEjnCW ++ cat /tmp/tmp.jXq5fq5YUx ++ rm /tmp/tmp.jAu0mEjnCW /tmp/tmp.jXq5fq5YUx ++ return 0 + [[ ready == \r\e\a\d\y ]] + echo .OK .OK + desc 'delete cluster and recreate' + set +o xtrace ----------------------------------------------------------------------------------- delete cluster and recreate ----------------------------------------------------------------------------------- + kubectl delete psmdb some-name perconaservermongodb.psmdb.percona.com "some-name" deleted from recover-no-primary-29251 namespace + set +e + echo -n 'waiting for psmdb/some-name to be deleted' waiting for psmdb/some-name to be deleted+ kubectl get psmdb some-name Error from server (NotFound): perconaservermongodbs.psmdb.percona.com "some-name" not found + set -e + apply_cluster /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2156/e2e-tests/recover-no-primary/conf/some-name-exposed.yml + '[' -z '' ']' + cat_config /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2156/e2e-tests/recover-no-primary/conf/some-name-exposed.yml + kubectl_bin apply -f - + cat /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2156/e2e-tests/recover-no-primary/conf/some-name-exposed.yml ++ mktemp + yq eval '(.spec | select(.image == null)).image = "perconalab/percona-server-mongodb-operator:main-mongod8.0"' + yq eval '(.spec | select(has("pmm"))).pmm.image = "percona/pmm-client:2.44.1-1"' + yq eval '(.spec | select(has("initImage"))).initImage = "perconalab/percona-server-mongodb-operator:PR-2156-df65e35b"' + yq eval '(.spec | select(has("backup"))).backup.image = "perconalab/percona-server-mongodb-operator:main-backup"' + local LAST_OUT=/tmp/tmp.QiLIiNN8rY ++ mktemp + /usr/sbin/sed -e s/NAME_SPACE/recover-no-primary-29251/g + local LAST_ERR=/tmp/tmp.XotYBHVd3O + local exit_status=0 + local timeout=4 + yq eval '.spec.upgradeOptions.apply="Never"' ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.QiLIiNN8rY perconaservermongodb.psmdb.percona.com/some-name created + cat /tmp/tmp.XotYBHVd3O + rm /tmp/tmp.QiLIiNN8rY /tmp/tmp.XotYBHVd3O + return 0 + wait_for_running some-name-rs0 3 + local name=some-name-rs0 + let last_pod=2 + local check_cluster_readyness=true + set_debug + [[ 1 == 1 ]] + set -o xtrace + local rs_name=rs0 + local cluster_name=some-name ++ seq 0 2 + for i in $(seq 0 $last_pod) + [[ 0 -eq 2 ]] + wait_pod some-name-rs0-0 + local pod=some-name-rs0-0 + set +o xtrace waiting for pod/some-name-rs0-0 to be ready.OK + for i in $(seq 0 $last_pod) + [[ 1 -eq 2 ]] + wait_pod some-name-rs0-1 + local pod=some-name-rs0-1 + set +o xtrace waiting for pod/some-name-rs0-1 to be ready.OK + for i in $(seq 0 $last_pod) + [[ 2 -eq 2 ]] ++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].arbiter.enabled}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.IV83kn1PdJ +++ mktemp ++ local LAST_ERR=/tmp/tmp.Cf22csrkNF ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].arbiter.enabled}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.IV83kn1PdJ ++ cat /tmp/tmp.Cf22csrkNF ++ rm /tmp/tmp.IV83kn1PdJ /tmp/tmp.Cf22csrkNF ++ return 0 + [[ '' == \t\r\u\e ]] + wait_pod some-name-rs0-2 + local pod=some-name-rs0-2 + set +o xtrace waiting for pod/some-name-rs0-2 to be ready.OK ++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].nonvoting.enabled}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.r2Ku2IwLAH +++ mktemp ++ local LAST_ERR=/tmp/tmp.tSDLUr7YV4 ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].nonvoting.enabled}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.r2Ku2IwLAH ++ cat /tmp/tmp.tSDLUr7YV4 ++ rm /tmp/tmp.r2Ku2IwLAH /tmp/tmp.tSDLUr7YV4 ++ return 0 + [[ '' == \t\r\u\e ]] ++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].hidden.enabled}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.Y19KL3aMzD +++ mktemp ++ local LAST_ERR=/tmp/tmp.G0mbyH1MaK ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].hidden.enabled}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.Y19KL3aMzD ++ cat /tmp/tmp.G0mbyH1MaK ++ rm /tmp/tmp.Y19KL3aMzD /tmp/tmp.G0mbyH1MaK ++ return 0 + [[ '' == \t\r\u\e ]] + sleep 10 + [[ true == \t\r\u\e ]] + set +x Waiting for cluster readyness................................. + wait_cluster_consistency some-name + local cluster_name=some-name + local wait_time=32 + retry=0 + sleep 7 + echo -n 'waiting for cluster readyness' waiting for cluster readyness++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.7g4KwponhY +++ mktemp ++ local LAST_ERR=/tmp/tmp.ZlGU3ah3Bm ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb some-name -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.7g4KwponhY ++ cat /tmp/tmp.ZlGU3ah3Bm ++ rm /tmp/tmp.7g4KwponhY /tmp/tmp.ZlGU3ah3Bm ++ return 0 + [[ ready == \r\e\a\d\y ]] + echo .OK .OK + kubectl delete psmdb some-name perconaservermongodb.psmdb.percona.com "some-name" deleted from recover-no-primary-29251 namespace + kubectl delete pvc --all persistentvolumeclaim "mongod-data-some-name-rs0-0" deleted from recover-no-primary-29251 namespace persistentvolumeclaim "mongod-data-some-name-rs0-1" deleted from recover-no-primary-29251 namespace persistentvolumeclaim "mongod-data-some-name-rs0-2" deleted from recover-no-primary-29251 namespace + desc 'testing exposed two replsets sharded cluster' + set +o xtrace ----------------------------------------------------------------------------------- testing exposed two replsets sharded cluster ----------------------------------------------------------------------------------- + test_exposed_sharded_cluster + apply_cluster /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2156/e2e-tests/recover-no-primary/conf/some-name-sharded.yml + '[' -z '' ']' + cat_config /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2156/e2e-tests/recover-no-primary/conf/some-name-sharded.yml + kubectl_bin apply -f - + cat /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2156/e2e-tests/recover-no-primary/conf/some-name-sharded.yml + yq eval '(.spec | select(.image == null)).image = "perconalab/percona-server-mongodb-operator:main-mongod8.0"' + yq eval '(.spec | select(has("pmm"))).pmm.image = "percona/pmm-client:2.44.1-1"' + yq eval '(.spec | select(has("initImage"))).initImage = "perconalab/percona-server-mongodb-operator:PR-2156-df65e35b"' ++ mktemp + yq eval '(.spec | select(has("backup"))).backup.image = "perconalab/percona-server-mongodb-operator:main-backup"' + /usr/sbin/sed -e s/NAME_SPACE/recover-no-primary-29251/g + yq eval '.spec.upgradeOptions.apply="Never"' + local LAST_OUT=/tmp/tmp.2CYtqpVeoR ++ mktemp + local LAST_ERR=/tmp/tmp.4ZO1qZC3zM + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.2CYtqpVeoR perconaservermongodb.psmdb.percona.com/some-name created + cat /tmp/tmp.4ZO1qZC3zM + rm /tmp/tmp.2CYtqpVeoR /tmp/tmp.4ZO1qZC3zM + return 0 + wait_for_running some-name-rs0 3 + local name=some-name-rs0 + let last_pod=2 + local check_cluster_readyness=true + set_debug + [[ 1 == 1 ]] + set -o xtrace + local rs_name=rs0 + local cluster_name=some-name ++ seq 0 2 + for i in $(seq 0 $last_pod) + [[ 0 -eq 2 ]] + wait_pod some-name-rs0-0 + local pod=some-name-rs0-0 + set +o xtrace waiting for pod/some-name-rs0-0 to be ready...........OK + for i in $(seq 0 $last_pod) + [[ 1 -eq 2 ]] + wait_pod some-name-rs0-1 + local pod=some-name-rs0-1 + set +o xtrace waiting for pod/some-name-rs0-1 to be ready............OK + for i in $(seq 0 $last_pod) + [[ 2 -eq 2 ]] ++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].arbiter.enabled}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.BXkiZpyghW +++ mktemp ++ local LAST_ERR=/tmp/tmp.XDI2wKBQ3G ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].arbiter.enabled}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.BXkiZpyghW ++ cat /tmp/tmp.XDI2wKBQ3G ++ rm /tmp/tmp.BXkiZpyghW /tmp/tmp.XDI2wKBQ3G ++ return 0 + [[ '' == \t\r\u\e ]] + wait_pod some-name-rs0-2 + local pod=some-name-rs0-2 + set +o xtrace waiting for pod/some-name-rs0-2 to be ready............OK ++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].nonvoting.enabled}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.giarSstkH1 +++ mktemp ++ local LAST_ERR=/tmp/tmp.sHDJHesAjt ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].nonvoting.enabled}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.giarSstkH1 ++ cat /tmp/tmp.sHDJHesAjt ++ rm /tmp/tmp.giarSstkH1 /tmp/tmp.sHDJHesAjt ++ return 0 + [[ '' == \t\r\u\e ]] ++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].hidden.enabled}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.YRXhfcwWB6 +++ mktemp ++ local LAST_ERR=/tmp/tmp.2tGvCenlHJ ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].hidden.enabled}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.YRXhfcwWB6 ++ cat /tmp/tmp.2tGvCenlHJ ++ rm /tmp/tmp.YRXhfcwWB6 /tmp/tmp.2tGvCenlHJ ++ return 0 + [[ '' == \t\r\u\e ]] + sleep 10 + [[ true == \t\r\u\e ]] + set +x Waiting for cluster readyness.............................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................2025-12-22T16:34:58.220Z DEBUG Replset status is not healthy {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"recover-no-primary-29251"}, "namespace": "recover-no-primary-29251", "name": "some-name", "reconcileID": "58e0ff00-aec2-4456-94e1-381256d2fe8b", "replset": "rs1", "status": "initializing"} 2025-12-22T16:34:59.620Z DEBUG Cluster is not ready {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"recover-no-primary-29251"}, "namespace": "recover-no-primary-29251", "name": "some-name", "reconcileID": "58e0ff00-aec2-4456-94e1-381256d2fe8b", "upgradeInProgress": false, "replsetsReady": 3, "clusterState": "initializing"} 2025-12-22T16:35:05.229Z DEBUG Replset is not ready {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"recover-no-primary-29251"}, "namespace": "recover-no-primary-29251", "name": "some-name", "reconcileID": "304e6fcb-6c2c-4325-bea4-308a481d09e7", "liveMembers": 2, "pods": 3} 2025-12-22T16:35:05.232Z DEBUG Replset status is not healthy {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"recover-no-primary-29251"}, "namespace": "recover-no-primary-29251", "name": "some-name", "reconcileID": "304e6fcb-6c2c-4325-bea4-308a481d09e7", "replset": "rs1", "status": "initializing"} 2025-12-22T16:35:06.625Z DEBUG Cluster is not ready {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"recover-no-primary-29251"}, "namespace": "recover-no-primary-29251", "name": "some-name", "reconcileID": "304e6fcb-6c2c-4325-bea4-308a481d09e7", "upgradeInProgress": false, "replsetsReady": 3, "clusterState": "initializing"} 2025-12-22T16:35:11.776Z DEBUG Replset is not ready {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"recover-no-primary-29251"}, "namespace": "recover-no-primary-29251", "name": "some-name", "reconcileID": "7faa2c9b-250a-4cb5-a854-416262363db3", "liveMembers": 2, "pods": 3} 2025-12-22T16:35:11.779Z DEBUG Replset status is not healthy {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"recover-no-primary-29251"}, "namespace": "recover-no-primary-29251", "name": "some-name", "reconcileID": "7faa2c9b-250a-4cb5-a854-416262363db3", "replset": "rs1", "status": "initializing"} 2025-12-22T16:35:13.234Z DEBUG Cluster is not ready {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"recover-no-primary-29251"}, "namespace": "recover-no-primary-29251", "name": "some-name", "reconcileID": "7faa2c9b-250a-4cb5-a854-416262363db3", "upgradeInProgress": false, "replsetsReady": 3, "clusterState": "initializing"} 2025-12-22T16:35:17.938Z DEBUG Replset is not ready {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"recover-no-primary-29251"}, "namespace": "recover-no-primary-29251", "name": "some-name", "reconcileID": "5808e048-ed20-41a5-8861-208d2986c1aa", "liveMembers": 2, "pods": 3} 2025-12-22T16:35:18.130Z DEBUG Replset status is not healthy {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"recover-no-primary-29251"}, "namespace": "recover-no-primary-29251", "name": "some-name", "reconcileID": "5808e048-ed20-41a5-8861-208d2986c1aa", "replset": "rs1", "status": "initializing"} 2025-12-22T16:35:19.651Z DEBUG Cluster is not ready {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"recover-no-primary-29251"}, "namespace": "recover-no-primary-29251", "name": "some-name", "reconcileID": "5808e048-ed20-41a5-8861-208d2986c1aa", "upgradeInProgress": false, "replsetsReady": 3, "clusterState": "initializing"} 2025-12-22T16:35:24.028Z DEBUG Replset is not ready {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"recover-no-primary-29251"}, "namespace": "recover-no-primary-29251", "name": "some-name", "reconcileID": "bff19666-608e-47e9-b4b3-5a6aa8b3171e", "liveMembers": 2, "pods": 3} 2025-12-22T16:35:24.125Z DEBUG Replset status is not healthy {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"recover-no-primary-29251"}, "namespace": "recover-no-primary-29251", "name": "some-name", "reconcileID": "bff19666-608e-47e9-b4b3-5a6aa8b3171e", "replset": "rs1", "status": "initializing"} 2025-12-22T16:35:25.543Z DEBUG Cluster is not ready {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"recover-no-primary-29251"}, "namespace": "recover-no-primary-29251", "name": "some-name", "reconcileID": "bff19666-608e-47e9-b4b3-5a6aa8b3171e", "upgradeInProgress": false, "replsetsReady": 3, "clusterState": "initializing"} 2025-12-22T16:35:38.434Z DEBUG Replset is not ready {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"recover-no-primary-29251"}, "namespace": "recover-no-primary-29251", "name": "some-name", "reconcileID": "f993fabc-bf54-43a8-9b0d-d8d2a3cab748", "liveMembers": 2, "pods": 3} 2025-12-22T16:35:38.436Z DEBUG Replset status is not healthy {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"recover-no-primary-29251"}, "namespace": "recover-no-primary-29251", "name": "some-name", "reconcileID": "f993fabc-bf54-43a8-9b0d-d8d2a3cab748", "replset": "rs1", "status": "initializing"} 2025-12-22T16:35:39.860Z DEBUG Cluster is not ready {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"recover-no-primary-29251"}, "namespace": "recover-no-primary-29251", "name": "some-name", "reconcileID": "f993fabc-bf54-43a8-9b0d-d8d2a3cab748", "upgradeInProgress": false, "replsetsReady": 3, "clusterState": "initializing"} 2025-12-22T16:35:44.694Z DEBUG Replset is not ready {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"recover-no-primary-29251"}, "namespace": "recover-no-primary-29251", "name": "some-name", "reconcileID": "57c1135e-5819-4687-9949-1717d1c6ea1f", "liveMembers": 2, "pods": 3} 2025-12-22T16:35:44.777Z DEBUG Replset status is not healthy {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"recover-no-primary-29251"}, "namespace": "recover-no-primary-29251", "name": "some-name", "reconcileID": "57c1135e-5819-4687-9949-1717d1c6ea1f", "replset": "rs1", "status": "initializing"} 2025-12-22T16:35:46.179Z DEBUG Cluster is not ready {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"recover-no-primary-29251"}, "namespace": "recover-no-primary-29251", "name": "some-name", "reconcileID": "57c1135e-5819-4687-9949-1717d1c6ea1f", "upgradeInProgress": false, "replsetsReady": 3, "clusterState": "initializing"} 2025-12-22T16:35:50.712Z DEBUG Replset is not ready {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"recover-no-primary-29251"}, "namespace": "recover-no-primary-29251", "name": "some-name", "reconcileID": "6a8970ba-a9df-49b6-9443-fd5d111f69ba", "liveMembers": 2, "pods": 3} 2025-12-22T16:35:50.714Z DEBUG Replset status is not healthy {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"recover-no-primary-29251"}, "namespace": "recover-no-primary-29251", "name": "some-name", "reconcileID": "6a8970ba-a9df-49b6-9443-fd5d111f69ba", "replset": "rs1", "status": "initializing"} 2025-12-22T16:35:52.103Z DEBUG Cluster is not ready {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"recover-no-primary-29251"}, "namespace": "recover-no-primary-29251", "name": "some-name", "reconcileID": "6a8970ba-a9df-49b6-9443-fd5d111f69ba", "upgradeInProgress": false, "replsetsReady": 3, "clusterState": "initializing"} 2025-12-22T16:35:57.059Z DEBUG Replset is not ready {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"recover-no-primary-29251"}, "namespace": "recover-no-primary-29251", "name": "some-name", "reconcileID": "f65f3cb6-440b-4e77-b28b-11ad08de6732", "liveMembers": 2, "pods": 3} 2025-12-22T16:35:57.077Z DEBUG Replset status is not healthy {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"recover-no-primary-29251"}, "namespace": "recover-no-primary-29251", "name": "some-name", "reconcileID": "f65f3cb6-440b-4e77-b28b-11ad08de6732", "replset": "rs1", "status": "initializing"} 2025-12-22T16:35:58.465Z DEBUG Cluster is not ready {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"recover-no-primary-29251"}, "namespace": "recover-no-primary-29251", "name": "some-name", "reconcileID": "f65f3cb6-440b-4e77-b28b-11ad08de6732", "upgradeInProgress": false, "replsetsReady": 3, "clusterState": "initializing"} 2025-12-22T16:36:03.687Z DEBUG Replset is not ready {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"recover-no-primary-29251"}, "namespace": "recover-no-primary-29251", "name": "some-name", "reconcileID": "a71427fc-f90b-4f5c-aced-7473ddc42388", "liveMembers": 2, "pods": 3} 2025-12-22T16:36:03.781Z DEBUG Replset status is not healthy {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"recover-no-primary-29251"}, "namespace": "recover-no-primary-29251", "name": "some-name", "reconcileID": "a71427fc-f90b-4f5c-aced-7473ddc42388", "replset": "rs1", "status": "initializing"} 2025-12-22T16:36:05.330Z DEBUG Cluster is not ready {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"recover-no-primary-29251"}, "namespace": "recover-no-primary-29251", "name": "some-name", "reconcileID": "a71427fc-f90b-4f5c-aced-7473ddc42388", "upgradeInProgress": false, "replsetsReady": 3, "clusterState": "initializing"} 2025-12-22T16:36:10.030Z DEBUG Replset is not ready {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"recover-no-primary-29251"}, "namespace": "recover-no-primary-29251", "name": "some-name", "reconcileID": "405b35b4-8fb3-4033-b548-fa990ea7218b", "liveMembers": 2, "pods": 3} 2025-12-22T16:36:10.032Z DEBUG Replset status is not healthy {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"recover-no-primary-29251"}, "namespace": "recover-no-primary-29251", "name": "some-name", "reconcileID": "405b35b4-8fb3-4033-b548-fa990ea7218b", "replset": "rs1", "status": "initializing"} 2025-12-22T16:36:11.441Z DEBUG Cluster is not ready {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"recover-no-primary-29251"}, "namespace": "recover-no-primary-29251", "name": "some-name", "reconcileID": "405b35b4-8fb3-4033-b548-fa990ea7218b", "upgradeInProgress": false, "replsetsReady": 3, "clusterState": "initializing"} 2025-12-22T16:36:18.436Z DEBUG Replset is not ready {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"recover-no-primary-29251"}, "namespace": "recover-no-primary-29251", "name": "some-name", "reconcileID": "de8afdb3-7850-4c17-aecd-d7e5d318c5d8", "liveMembers": 2, "pods": 3} 2025-12-22T16:36:18.622Z DEBUG Replset status is not healthy {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"recover-no-primary-29251"}, "namespace": "recover-no-primary-29251", "name": "some-name", "reconcileID": "de8afdb3-7850-4c17-aecd-d7e5d318c5d8", "replset": "rs1", "status": "initializing"} 2025-12-22T16:36:20.261Z DEBUG Cluster is not ready {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"recover-no-primary-29251"}, "namespace": "recover-no-primary-29251", "name": "some-name", "reconcileID": "de8afdb3-7850-4c17-aecd-d7e5d318c5d8", "upgradeInProgress": false, "replsetsReady": 3, "clusterState": "initializing"} 2025-12-22T16:36:25.125Z DEBUG Replset is not ready {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"recover-no-primary-29251"}, "namespace": "recover-no-primary-29251", "name": "some-name", "reconcileID": "4c86349b-e7e8-4fbb-823a-ebfc9ad3feb7", "liveMembers": 2, "pods": 3} 2025-12-22T16:36:25.335Z DEBUG Replset status is not healthy {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"recover-no-primary-29251"}, "namespace": "recover-no-primary-29251", "name": "some-name", "reconcileID": "4c86349b-e7e8-4fbb-823a-ebfc9ad3feb7", "replset": "rs1", "status": "initializing"} 2025-12-22T16:36:26.732Z DEBUG Cluster is not ready {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"recover-no-primary-29251"}, "namespace": "recover-no-primary-29251", "name": "some-name", "reconcileID": "4c86349b-e7e8-4fbb-823a-ebfc9ad3feb7", "upgradeInProgress": false, "replsetsReady": 3, "clusterState": "initializing"} 2025-12-22T16:36:31.671Z DEBUG Replset is not ready {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"recover-no-primary-29251"}, "namespace": "recover-no-primary-29251", "name": "some-name", "reconcileID": "c6dd61cc-8d1d-4f3f-862c-f6e0bebaefad", "liveMembers": 2, "pods": 3} 2025-12-22T16:36:31.675Z DEBUG Replset status is not healthy {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"recover-no-primary-29251"}, "namespace": "recover-no-primary-29251", "name": "some-name", "reconcileID": "c6dd61cc-8d1d-4f3f-862c-f6e0bebaefad", "replset": "rs1", "status": "initializing"} 2025-12-22T16:36:33.097Z DEBUG Cluster is not ready {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"recover-no-primary-29251"}, "namespace": "recover-no-primary-29251", "name": "some-name", "reconcileID": "c6dd61cc-8d1d-4f3f-862c-f6e0bebaefad", "upgradeInProgress": false, "replsetsReady": 3, "clusterState": "initializing"} 2025-12-22T16:36:38.520Z DEBUG Replset is not ready {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"recover-no-primary-29251"}, "namespace": "recover-no-primary-29251", "name": "some-name", "reconcileID": "10ac2059-9157-45b2-94a7-03d4c757e267", "liveMembers": 2, "pods": 3} 2025-12-22T16:36:38.531Z DEBUG Replset status is not healthy {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"recover-no-primary-29251"}, "namespace": "recover-no-primary-29251", "name": "some-name", "reconcileID": "10ac2059-9157-45b2-94a7-03d4c757e267", "replset": "rs1", "status": "initializing"} 2025-12-22T16:36:39.925Z DEBUG Cluster is not ready {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"recover-no-primary-29251"}, "namespace": "recover-no-primary-29251", "name": "some-name", "reconcileID": "10ac2059-9157-45b2-94a7-03d4c757e267", "upgradeInProgress": false, "replsetsReady": 3, "clusterState": "initializing"} 2025-12-22T16:36:45.026Z DEBUG Replset is not ready {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"recover-no-primary-29251"}, "namespace": "recover-no-primary-29251", "name": "some-name", "reconcileID": "05219f21-b684-4656-a81b-9c5903b04096", "liveMembers": 2, "pods": 3} 2025-12-22T16:36:45.323Z DEBUG Replset status is not healthy {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"recover-no-primary-29251"}, "namespace": "recover-no-primary-29251", "name": "some-name", "reconcileID": "05219f21-b684-4656-a81b-9c5903b04096", "replset": "rs1", "status": "initializing"} 2025-12-22T16:36:46.740Z DEBUG Cluster is not ready {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"recover-no-primary-29251"}, "namespace": "recover-no-primary-29251", "name": "some-name", "reconcileID": "05219f21-b684-4656-a81b-9c5903b04096", "upgradeInProgress": false, "replsetsReady": 3, "clusterState": "initializing"} 2025-12-22T16:36:58.461Z DEBUG Replset is not ready {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"recover-no-primary-29251"}, "namespace": "recover-no-primary-29251", "name": "some-name", "reconcileID": "bbac47a8-d4f9-4332-8fe3-31cfd0fd71cb", "liveMembers": 2, "pods": 3} 2025-12-22T16:36:58.463Z DEBUG Replset status is not healthy {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"recover-no-primary-29251"}, "namespace": "recover-no-primary-29251", "name": "some-name", "reconcileID": "bbac47a8-d4f9-4332-8fe3-31cfd0fd71cb", "replset": "rs1", "status": "initializing"} 2025-12-22T16:36:59.858Z DEBUG Cluster is not ready {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"recover-no-primary-29251"}, "namespace": "recover-no-primary-29251", "name": "some-name", "reconcileID": "bbac47a8-d4f9-4332-8fe3-31cfd0fd71cb", "upgradeInProgress": false, "replsetsReady": 3, "clusterState": "initializing"} 2025-12-22T16:37:05.640Z DEBUG Replset is not ready {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"recover-no-primary-29251"}, "namespace": "recover-no-primary-29251", "name": "some-name", "reconcileID": "92eea23e-3dac-44e7-9728-8a5d4078494f", "liveMembers": 2, "pods": 3} 2025-12-22T16:37:05.642Z DEBUG Replset status is not healthy {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"recover-no-primary-29251"}, "namespace": "recover-no-primary-29251", "name": "some-name", "reconcileID": "92eea23e-3dac-44e7-9728-8a5d4078494f", "replset": "rs1", "status": "initializing"} 2025-12-22T16:37:07.050Z DEBUG Cluster is not ready {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"recover-no-primary-29251"}, "namespace": "recover-no-primary-29251", "name": "some-name", "reconcileID": "92eea23e-3dac-44e7-9728-8a5d4078494f", "upgradeInProgress": false, "replsetsReady": 3, "clusterState": "initializing"} 2025-12-22T16:37:12.582Z DEBUG Replset is not ready {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"recover-no-primary-29251"}, "namespace": "recover-no-primary-29251", "name": "some-name", "reconcileID": "be975532-7c88-4605-9e52-0bae2397453a", "liveMembers": 2, "pods": 3} 2025-12-22T16:37:12.584Z DEBUG Replset status is not healthy {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"recover-no-primary-29251"}, "namespace": "recover-no-primary-29251", "name": "some-name", "reconcileID": "be975532-7c88-4605-9e52-0bae2397453a", "replset": "rs1", "status": "initializing"} 2025-12-22T16:37:13.970Z DEBUG Cluster is not ready {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"recover-no-primary-29251"}, "namespace": "recover-no-primary-29251", "name": "some-name", "reconcileID": "be975532-7c88-4605-9e52-0bae2397453a", "upgradeInProgress": false, "replsetsReady": 3, "clusterState": "initializing"} 2025-12-22T16:37:19.378Z DEBUG Replset is not ready {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"recover-no-primary-29251"}, "namespace": "recover-no-primary-29251", "name": "some-name", "reconcileID": "61450cf6-3093-49c4-99aa-f3fff6a1b196", "liveMembers": 2, "pods": 3} 2025-12-22T16:37:19.424Z DEBUG Replset status is not healthy {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"recover-no-primary-29251"}, "namespace": "recover-no-primary-29251", "name": "some-name", "reconcileID": "61450cf6-3093-49c4-99aa-f3fff6a1b196", "replset": "rs1", "status": "initializing"} 2025-12-22T16:37:20.819Z DEBUG Cluster is not ready {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"recover-no-primary-29251"}, "namespace": "recover-no-primary-29251", "name": "some-name", "reconcileID": "61450cf6-3093-49c4-99aa-f3fff6a1b196", "upgradeInProgress": false, "replsetsReady": 3, "clusterState": "initializing"} 2025-12-22T16:37:26.335Z DEBUG Replset is not ready {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"recover-no-primary-29251"}, "namespace": "recover-no-primary-29251", "name": "some-name", "reconcileID": "5e630e3a-a00c-4a5c-9c64-8375504242c9", "liveMembers": 2, "pods": 3} 2025-12-22T16:37:26.426Z DEBUG Replset status is not healthy {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"recover-no-primary-29251"}, "namespace": "recover-no-primary-29251", "name": "some-name", "reconcileID": "5e630e3a-a00c-4a5c-9c64-8375504242c9", "replset": "rs1", "status": "initializing"} 2025-12-22T16:37:27.842Z DEBUG Cluster is not ready {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"recover-no-primary-29251"}, "namespace": "recover-no-primary-29251", "name": "some-name", "reconcileID": "5e630e3a-a00c-4a5c-9c64-8375504242c9", "upgradeInProgress": false, "replsetsReady": 3, "clusterState": "initializing"} 2025-12-22T16:37:32.655Z DEBUG Replset is not ready {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"recover-no-primary-29251"}, "namespace": "recover-no-primary-29251", "name": "some-name", "reconcileID": "9a95d8af-d222-49e9-8cdb-ac2e16005557", "liveMembers": 2, "pods": 3} 2025-12-22T16:37:32.657Z DEBUG Replset status is not healthy {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"recover-no-primary-29251"}, "namespace": "recover-no-primary-29251", "name": "some-name", "reconcileID": "9a95d8af-d222-49e9-8cdb-ac2e16005557", "replset": "rs1", "status": "initializing"} 2025-12-22T16:37:34.432Z DEBUG Cluster is not ready {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"recover-no-primary-29251"}, "namespace": "recover-no-primary-29251", "name": "some-name", "reconcileID": "9a95d8af-d222-49e9-8cdb-ac2e16005557", "upgradeInProgress": false, "replsetsReady": 3, "clusterState": "initializing"} 2025-12-22T16:37:38.764Z DEBUG Replset is not ready {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"recover-no-primary-29251"}, "namespace": "recover-no-primary-29251", "name": "some-name", "reconcileID": "70f6838e-b42e-4311-abd0-cac87a24a559", "liveMembers": 2, "pods": 3} 2025-12-22T16:37:38.766Z DEBUG Replset status is not healthy {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"recover-no-primary-29251"}, "namespace": "recover-no-primary-29251", "name": "some-name", "reconcileID": "70f6838e-b42e-4311-abd0-cac87a24a559", "replset": "rs1", "status": "initializing"} 2025-12-22T16:37:40.160Z DEBUG Cluster is not ready {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"recover-no-primary-29251"}, "namespace": "recover-no-primary-29251", "name": "some-name", "reconcileID": "70f6838e-b42e-4311-abd0-cac87a24a559", "upgradeInProgress": false, "replsetsReady": 3, "clusterState": "initializing"} 2025-12-22T16:37:44.923Z DEBUG Replset is not ready {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"recover-no-primary-29251"}, "namespace": "recover-no-primary-29251", "name": "some-name", "reconcileID": "c502b662-b6a0-4f24-a7ea-0168ad808286", "liveMembers": 2, "pods": 3} 2025-12-22T16:37:44.981Z DEBUG Replset status is not healthy {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"recover-no-primary-29251"}, "namespace": "recover-no-primary-29251", "name": "some-name", "reconcileID": "c502b662-b6a0-4f24-a7ea-0168ad808286", "replset": "rs1", "status": "initializing"} 2025-12-22T16:37:46.396Z DEBUG Cluster is not ready {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"recover-no-primary-29251"}, "namespace": "recover-no-primary-29251", "name": "some-name", "reconcileID": "c502b662-b6a0-4f24-a7ea-0168ad808286", "upgradeInProgress": false, "replsetsReady": 3, "clusterState": "initializing"} 2025-12-22T16:37:52.122Z DEBUG Replset is not ready {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"recover-no-primary-29251"}, "namespace": "recover-no-primary-29251", "name": "some-name", "reconcileID": "bef732ee-6aa0-447b-a81b-84db9568ffcb", "liveMembers": 2, "pods": 3} 2025-12-22T16:37:52.126Z DEBUG Replset status is not healthy {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"recover-no-primary-29251"}, "namespace": "recover-no-primary-29251", "name": "some-name", "reconcileID": "bef732ee-6aa0-447b-a81b-84db9568ffcb", "replset": "rs1", "status": "initializing"} 2025-12-22T16:37:53.511Z DEBUG Cluster is not ready {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"recover-no-primary-29251"}, "namespace": "recover-no-primary-29251", "name": "some-name", "reconcileID": "bef732ee-6aa0-447b-a81b-84db9568ffcb", "upgradeInProgress": false, "replsetsReady": 3, "clusterState": "initializing"} 2025-12-22T16:37:59.326Z DEBUG Replset is not ready {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"recover-no-primary-29251"}, "namespace": "recover-no-primary-29251", "name": "some-name", "reconcileID": "96fc32fb-a490-4f6e-83d5-3bdb1a771316", "liveMembers": 2, "pods": 3} 2025-12-22T16:37:59.426Z DEBUG Replset status is not healthy {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"recover-no-primary-29251"}, "namespace": "recover-no-primary-29251", "name": "some-name", "reconcileID": "96fc32fb-a490-4f6e-83d5-3bdb1a771316", "replset": "rs1", "status": "initializing"} 2025-12-22T16:38:00.822Z DEBUG Cluster is not ready {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"recover-no-primary-29251"}, "namespace": "recover-no-primary-29251", "name": "some-name", "reconcileID": "96fc32fb-a490-4f6e-83d5-3bdb1a771316", "upgradeInProgress": false, "replsetsReady": 3, "clusterState": "initializing"} 2025-12-22T16:38:06.736Z DEBUG Replset is not ready {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"recover-no-primary-29251"}, "namespace": "recover-no-primary-29251", "name": "some-name", "reconcileID": "bdb98231-0aab-44ac-82c0-c7c8e6267092", "liveMembers": 2, "pods": 3} 2025-12-22T16:38:06.937Z DEBUG Replset status is not healthy {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"recover-no-primary-29251"}, "namespace": "recover-no-primary-29251", "name": "some-name", "reconcileID": "bdb98231-0aab-44ac-82c0-c7c8e6267092", "replset": "rs1", "status": "initializing"} 2025-12-22T16:38:08.393Z DEBUG Cluster is not ready {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"recover-no-primary-29251"}, "namespace": "recover-no-primary-29251", "name": "some-name", "reconcileID": "bdb98231-0aab-44ac-82c0-c7c8e6267092", "upgradeInProgress": false, "replsetsReady": 3, "clusterState": "initializing"} 2025-12-22T16:38:13.250Z DEBUG Replset is not ready {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"recover-no-primary-29251"}, "namespace": "recover-no-primary-29251", "name": "some-name", "reconcileID": "0c07b27a-41bd-4169-a9ca-7e4d1c65b87c", "liveMembers": 2, "pods": 3} 2025-12-22T16:38:13.252Z DEBUG Replset status is not healthy {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"recover-no-primary-29251"}, "namespace": "recover-no-primary-29251", "name": "some-name", "reconcileID": "0c07b27a-41bd-4169-a9ca-7e4d1c65b87c", "replset": "rs1", "status": "initializing"} 2025-12-22T16:38:14.663Z DEBUG Cluster is not ready {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"recover-no-primary-29251"}, "namespace": "recover-no-primary-29251", "name": "some-name", "reconcileID": "0c07b27a-41bd-4169-a9ca-7e4d1c65b87c", "upgradeInProgress": false, "replsetsReady": 3, "clusterState": "initializing"} 2025-12-22T16:38:19.435Z DEBUG Replset is not ready {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"recover-no-primary-29251"}, "namespace": "recover-no-primary-29251", "name": "some-name", "reconcileID": "99ffdd47-42c6-4016-8f12-3915fd9c4a95", "liveMembers": 2, "pods": 3} 2025-12-22T16:38:19.437Z DEBUG Replset status is not healthy {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"recover-no-primary-29251"}, "namespace": "recover-no-primary-29251", "name": "some-name", "reconcileID": "99ffdd47-42c6-4016-8f12-3915fd9c4a95", "replset": "rs1", "status": "initializing"} 2025-12-22T16:38:20.866Z DEBUG Cluster is not ready {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"recover-no-primary-29251"}, "namespace": "recover-no-primary-29251", "name": "some-name", "reconcileID": "99ffdd47-42c6-4016-8f12-3915fd9c4a95", "upgradeInProgress": false, "replsetsReady": 3, "clusterState": "initializing"} 2025-12-22T16:38:26.723Z DEBUG Replset is not ready {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"recover-no-primary-29251"}, "namespace": "recover-no-primary-29251", "name": "some-name", "reconcileID": "e87abf86-6800-4871-ad4c-e83eb057fd0b", "liveMembers": 2, "pods": 3} 2025-12-22T16:38:26.735Z DEBUG Replset status is not healthy {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"recover-no-primary-29251"}, "namespace": "recover-no-primary-29251", "name": "some-name", "reconcileID": "e87abf86-6800-4871-ad4c-e83eb057fd0b", "replset": "rs1", "status": "initializing"} 2025-12-22T16:38:28.133Z DEBUG Cluster is not ready {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"recover-no-primary-29251"}, "namespace": "recover-no-primary-29251", "name": "some-name", "reconcileID": "e87abf86-6800-4871-ad4c-e83eb057fd0b", "upgradeInProgress": false, "replsetsReady": 3, "clusterState": "initializing"} 2025-12-22T16:38:33.740Z DEBUG Replset is not ready {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"recover-no-primary-29251"}, "namespace": "recover-no-primary-29251", "name": "some-name", "reconcileID": "4c9942b9-a2e5-441b-863f-4d66c08a7aa3", "liveMembers": 2, "pods": 3} 2025-12-22T16:38:33.836Z DEBUG Replset status is not healthy {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"recover-no-primary-29251"}, "namespace": "recover-no-primary-29251", "name": "some-name", "reconcileID": "4c9942b9-a2e5-441b-863f-4d66c08a7aa3", "replset": "rs1", "status": "initializing"} 2025-12-22T16:38:35.479Z DEBUG Cluster is not ready {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"recover-no-primary-29251"}, "namespace": "recover-no-primary-29251", "name": "some-name", "reconcileID": "4c9942b9-a2e5-441b-863f-4d66c08a7aa3", "upgradeInProgress": false, "replsetsReady": 3, "clusterState": "initializing"} 2025-12-22T16:38:40.032Z DEBUG Replset is not ready {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"recover-no-primary-29251"}, "namespace": "recover-no-primary-29251", "name": "some-name", "reconcileID": "095050f0-5401-423e-a1f3-df7b5f28b0f1", "liveMembers": 2, "pods": 3} 2025-12-22T16:38:40.034Z DEBUG Replset status is not healthy {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"recover-no-primary-29251"}, "namespace": "recover-no-primary-29251", "name": "some-name", "reconcileID": "095050f0-5401-423e-a1f3-df7b5f28b0f1", "replset": "rs1", "status": "initializing"} 2025-12-22T16:38:41.465Z DEBUG Cluster is not ready {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"recover-no-primary-29251"}, "namespace": "recover-no-primary-29251", "name": "some-name", "reconcileID": "095050f0-5401-423e-a1f3-df7b5f28b0f1", "upgradeInProgress": false, "replsetsReady": 3, "clusterState": "initializing"} 2025-12-22T16:38:47.944Z DEBUG Replset is not ready {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"recover-no-primary-29251"}, "namespace": "recover-no-primary-29251", "name": "some-name", "reconcileID": "56b21834-a536-430b-9e27-cc5c2351535a", "liveMembers": 2, "pods": 3} 2025-12-22T16:38:47.973Z DEBUG Replset status is not healthy {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"recover-no-primary-29251"}, "namespace": "recover-no-primary-29251", "name": "some-name", "reconcileID": "56b21834-a536-430b-9e27-cc5c2351535a", "replset": "rs1", "status": "initializing"} 2025-12-22T16:38:49.472Z DEBUG Cluster is not ready {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"recover-no-primary-29251"}, "namespace": "recover-no-primary-29251", "name": "some-name", "reconcileID": "56b21834-a536-430b-9e27-cc5c2351535a", "upgradeInProgress": false, "replsetsReady": 3, "clusterState": "initializing"} 2025-12-22T16:38:54.659Z DEBUG Replset is not ready {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"recover-no-primary-29251"}, "namespace": "recover-no-primary-29251", "name": "some-name", "reconcileID": "ff7a31e2-6dfa-4a09-aa3f-a67fed6dfa03", "liveMembers": 2, "pods": 3} 2025-12-22T16:38:54.726Z DEBUG Replset status is not healthy {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"recover-no-primary-29251"}, "namespace": "recover-no-primary-29251", "name": "some-name", "reconcileID": "ff7a31e2-6dfa-4a09-aa3f-a67fed6dfa03", "replset": "rs1", "status": "initializing"} 2025-12-22T16:38:56.157Z DEBUG Cluster is not ready {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"recover-no-primary-29251"}, "namespace": "recover-no-primary-29251", "name": "some-name", "reconcileID": "ff7a31e2-6dfa-4a09-aa3f-a67fed6dfa03", "upgradeInProgress": false, "replsetsReady": 3, "clusterState": "initializing"} 2025-12-22T16:39:01.528Z DEBUG Replset is not ready {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"recover-no-primary-29251"}, "namespace": "recover-no-primary-29251", "name": "some-name", "reconcileID": "e45100a9-101c-43d7-8e67-28e471c84b65", "liveMembers": 2, "pods": 3} 2025-12-22T16:39:01.620Z DEBUG Replset status is not healthy {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"recover-no-primary-29251"}, "namespace": "recover-no-primary-29251", "name": "some-name", "reconcileID": "e45100a9-101c-43d7-8e67-28e471c84b65", "replset": "rs1", "status": "initializing"} 2025-12-22T16:39:03.013Z DEBUG Cluster is not ready {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"recover-no-primary-29251"}, "namespace": "recover-no-primary-29251", "name": "some-name", "reconcileID": "e45100a9-101c-43d7-8e67-28e471c84b65", "upgradeInProgress": false, "replsetsReady": 3, "clusterState": "initializing"} 2025-12-22T16:39:08.434Z DEBUG Replset is not ready {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"recover-no-primary-29251"}, "namespace": "recover-no-primary-29251", "name": "some-name", "reconcileID": "adf0866b-2f0a-4ff9-9fcc-bd25aeaf05b0", "liveMembers": 2, "pods": 3} 2025-12-22T16:39:08.527Z DEBUG Replset status is not healthy {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"recover-no-primary-29251"}, "namespace": "recover-no-primary-29251", "name": "some-name", "reconcileID": "adf0866b-2f0a-4ff9-9fcc-bd25aeaf05b0", "replset": "rs1", "status": "initializing"} 2025-12-22T16:39:09.930Z DEBUG Cluster is not ready {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"recover-no-primary-29251"}, "namespace": "recover-no-primary-29251", "name": "some-name", "reconcileID": "adf0866b-2f0a-4ff9-9fcc-bd25aeaf05b0", "upgradeInProgress": false, "replsetsReady": 3, "clusterState": "initializing"} 2025-12-22T16:39:16.732Z DEBUG Replset is not ready {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"recover-no-primary-29251"}, "namespace": "recover-no-primary-29251", "name": "some-name", "reconcileID": "bc309d41-a2cd-47fd-8964-692b94d41f67", "liveMembers": 2, "pods": 3} 2025-12-22T16:39:16.926Z DEBUG Replset status is not healthy {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"recover-no-primary-29251"}, "namespace": "recover-no-primary-29251", "name": "some-name", "reconcileID": "bc309d41-a2cd-47fd-8964-692b94d41f67", "replset": "rs1", "status": "initializing"} 2025-12-22T16:39:18.445Z DEBUG Cluster is not ready {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"recover-no-primary-29251"}, "namespace": "recover-no-primary-29251", "name": "some-name", "reconcileID": "bc309d41-a2cd-47fd-8964-692b94d41f67", "upgradeInProgress": false, "replsetsReady": 3, "clusterState": "initializing"} 2025-12-22T16:39:23.534Z DEBUG Replset is not ready {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"recover-no-primary-29251"}, "namespace": "recover-no-primary-29251", "name": "some-name", "reconcileID": "9ffaa1bf-3480-4a82-b9c6-b745597b7ab5", "liveMembers": 2, "pods": 3} 2025-12-22T16:39:23.576Z DEBUG Replset status is not healthy {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"recover-no-primary-29251"}, "namespace": "recover-no-primary-29251", "name": "some-name", "reconcileID": "9ffaa1bf-3480-4a82-b9c6-b745597b7ab5", "replset": "rs1", "status": "initializing"} 2025-12-22T16:39:24.968Z DEBUG Cluster is not ready {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"recover-no-primary-29251"}, "namespace": "recover-no-primary-29251", "name": "some-name", "reconcileID": "9ffaa1bf-3480-4a82-b9c6-b745597b7ab5", "upgradeInProgress": false, "replsetsReady": 3, "clusterState": "initializing"} 2025-12-22T16:39:30.177Z DEBUG Replset is not ready {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"recover-no-primary-29251"}, "namespace": "recover-no-primary-29251", "name": "some-name", "reconcileID": "e8fd71db-6e61-42d5-909b-d2c5b659e4bb", "liveMembers": 2, "pods": 3} 2025-12-22T16:39:30.181Z DEBUG Replset status is not healthy {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"recover-no-primary-29251"}, "namespace": "recover-no-primary-29251", "name": "some-name", "reconcileID": "e8fd71db-6e61-42d5-909b-d2c5b659e4bb", "replset": "rs1", "status": "initializing"} 2025-12-22T16:39:31.572Z DEBUG Cluster is not ready {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"recover-no-primary-29251"}, "namespace": "recover-no-primary-29251", "name": "some-name", "reconcileID": "e8fd71db-6e61-42d5-909b-d2c5b659e4bb", "upgradeInProgress": false, "replsetsReady": 3, "clusterState": "initializing"} 2025-12-22T16:39:37.524Z DEBUG Replset is not ready {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"recover-no-primary-29251"}, "namespace": "recover-no-primary-29251", "name": "some-name", "reconcileID": "d771fada-a28f-4dca-af0f-1092fd65e85f", "liveMembers": 2, "pods": 3} 2025-12-22T16:39:37.528Z DEBUG Replset status is not healthy {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"recover-no-primary-29251"}, "namespace": "recover-no-primary-29251", "name": "some-name", "reconcileID": "d771fada-a28f-4dca-af0f-1092fd65e85f", "replset": "rs1", "status": "initializing"} 2025-12-22T16:39:39.021Z DEBUG Cluster is not ready {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"recover-no-primary-29251"}, "namespace": "recover-no-primary-29251", "name": "some-name", "reconcileID": "d771fada-a28f-4dca-af0f-1092fd65e85f", "upgradeInProgress": false, "replsetsReady": 3, "clusterState": "initializing"} 2025-12-22T16:39:48.224Z DEBUG Replset is not ready {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"recover-no-primary-29251"}, "namespace": "recover-no-primary-29251", "name": "some-name", "reconcileID": "50289ab0-6458-4427-81e5-f65bafbafc56", "liveMembers": 2, "pods": 3} 2025-12-22T16:39:48.325Z DEBUG Replset status is not healthy {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"recover-no-primary-29251"}, "namespace": "recover-no-primary-29251", "name": "some-name", "reconcileID": "50289ab0-6458-4427-81e5-f65bafbafc56", "replset": "rs1", "status": "initializing"} 2025-12-22T16:39:49.756Z DEBUG Cluster is not ready {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"recover-no-primary-29251"}, "namespace": "recover-no-primary-29251", "name": "some-name", "reconcileID": "50289ab0-6458-4427-81e5-f65bafbafc56", "upgradeInProgress": false, "replsetsReady": 3, "clusterState": "initializing"} 2025-12-22T16:39:55.044Z DEBUG Replset is not ready {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"recover-no-primary-29251"}, "namespace": "recover-no-primary-29251", "name": "some-name", "reconcileID": "e5bd70c0-ff4c-43d3-8b98-64b1f0d71975", "liveMembers": 2, "pods": 3} 2025-12-22T16:39:55.126Z DEBUG Replset status is not healthy {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"recover-no-primary-29251"}, "namespace": "recover-no-primary-29251", "name": "some-name", "reconcileID": "e5bd70c0-ff4c-43d3-8b98-64b1f0d71975", "replset": "rs1", "status": "initializing"} 2025-12-22T16:39:56.526Z DEBUG Cluster is not ready {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"recover-no-primary-29251"}, "namespace": "recover-no-primary-29251", "name": "some-name", "reconcileID": "e5bd70c0-ff4c-43d3-8b98-64b1f0d71975", "upgradeInProgress": false, "replsetsReady": 3, "clusterState": "initializing"} 2025-12-22T16:40:01.752Z DEBUG Replset is not ready {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"recover-no-primary-29251"}, "namespace": "recover-no-primary-29251", "name": "some-name", "reconcileID": "81427312-0f42-4ed7-8f34-2fd0afdab986", "liveMembers": 2, "pods": 3} 2025-12-22T16:40:01.776Z DEBUG Replset status is not healthy {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"recover-no-primary-29251"}, "namespace": "recover-no-primary-29251", "name": "some-name", "reconcileID": "81427312-0f42-4ed7-8f34-2fd0afdab986", "replset": "rs1", "status": "initializing"} 2025-12-22T16:40:03.172Z DEBUG Cluster is not ready {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"recover-no-primary-29251"}, "namespace": "recover-no-primary-29251", "name": "some-name", "reconcileID": "81427312-0f42-4ed7-8f34-2fd0afdab986", "upgradeInProgress": false, "replsetsReady": 3, "clusterState": "initializing"} 2025-12-22T16:40:08.750Z DEBUG Replset is not ready {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"recover-no-primary-29251"}, "namespace": "recover-no-primary-29251", "name": "some-name", "reconcileID": "cf1379e7-8afd-4e34-97f9-38465727a7c4", "liveMembers": 2, "pods": 3} 2025-12-22T16:40:08.752Z DEBUG Replset status is not healthy {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"recover-no-primary-29251"}, "namespace": "recover-no-primary-29251", "name": "some-name", "reconcileID": "cf1379e7-8afd-4e34-97f9-38465727a7c4", "replset": "rs1", "status": "initializing"} 2025-12-22T16:40:10.127Z DEBUG Cluster is not ready {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"recover-no-primary-29251"}, "namespace": "recover-no-primary-29251", "name": "some-name", "reconcileID": "cf1379e7-8afd-4e34-97f9-38465727a7c4", "upgradeInProgress": false, "replsetsReady": 3, "clusterState": "initializing"} 2025-12-22T16:40:15.258Z DEBUG Replset is not ready {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"recover-no-primary-29251"}, "namespace": "recover-no-primary-29251", "name": "some-name", "reconcileID": "f8c2eebb-319a-4c7a-9f09-28f3d8fa14b4", "liveMembers": 2, "pods": 3} 2025-12-22T16:40:15.275Z DEBUG Replset status is not healthy {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"recover-no-primary-29251"}, "namespace": "recover-no-primary-29251", "name": "some-name", "reconcileID": "f8c2eebb-319a-4c7a-9f09-28f3d8fa14b4", "replset": "rs1", "status": "initializing"} 2025-12-22T16:40:16.696Z DEBUG Cluster is not ready {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"recover-no-primary-29251"}, "namespace": "recover-no-primary-29251", "name": "some-name", "reconcileID": "f8c2eebb-319a-4c7a-9f09-28f3d8fa14b4", "upgradeInProgress": false, "replsetsReady": 3, "clusterState": "initializing"} 2025-12-22T16:40:22.775Z DEBUG Replset is not ready {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"recover-no-primary-29251"}, "namespace": "recover-no-primary-29251", "name": "some-name", "reconcileID": "7eb0b796-7850-4c3c-9df1-0f40cf46697a", "liveMembers": 2, "pods": 3} 2025-12-22T16:40:22.779Z DEBUG Replset status is not healthy {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"recover-no-primary-29251"}, "namespace": "recover-no-primary-29251", "name": "some-name", "reconcileID": "7eb0b796-7850-4c3c-9df1-0f40cf46697a", "replset": "rs1", "status": "initializing"} 2025-12-22T16:40:24.187Z DEBUG Cluster is not ready {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"recover-no-primary-29251"}, "namespace": "recover-no-primary-29251", "name": "some-name", "reconcileID": "7eb0b796-7850-4c3c-9df1-0f40cf46697a", "upgradeInProgress": false, "replsetsReady": 3, "clusterState": "initializing"} 2025-12-22T16:40:29.118Z DEBUG Replset is not ready {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"recover-no-primary-29251"}, "namespace": "recover-no-primary-29251", "name": "some-name", "reconcileID": "63e3475d-c210-4d68-88a7-b97140b45b7b", "liveMembers": 2, "pods": 3} 2025-12-22T16:40:29.121Z DEBUG Replset status is not healthy {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"recover-no-primary-29251"}, "namespace": "recover-no-primary-29251", "name": "some-name", "reconcileID": "63e3475d-c210-4d68-88a7-b97140b45b7b", "replset": "rs1", "status": "initializing"} 2025-12-22T16:40:30.533Z DEBUG Cluster is not ready {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"recover-no-primary-29251"}, "namespace": "recover-no-primary-29251", "name": "some-name", "reconcileID": "63e3475d-c210-4d68-88a7-b97140b45b7b", "upgradeInProgress": false, "replsetsReady": 3, "clusterState": "initializing"} 2025-12-22T16:40:35.233Z DEBUG Replset is not ready {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"recover-no-primary-29251"}, "namespace": "recover-no-primary-29251", "name": "some-name", "reconcileID": "9a7be529-e413-4e1a-b8d9-cf0d70b3af5b", "liveMembers": 2, "pods": 3} 2025-12-22T16:40:35.235Z DEBUG Replset status is not healthy {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"recover-no-primary-29251"}, "namespace": "recover-no-primary-29251", "name": "some-name", "reconcileID": "9a7be529-e413-4e1a-b8d9-cf0d70b3af5b", "replset": "rs1", "status": "initializing"} 2025-12-22T16:40:36.631Z DEBUG Cluster is not ready {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"recover-no-primary-29251"}, "namespace": "recover-no-primary-29251", "name": "some-name", "reconcileID": "9a7be529-e413-4e1a-b8d9-cf0d70b3af5b", "upgradeInProgress": false, "replsetsReady": 3, "clusterState": "initializing"} 2025-12-22T16:40:41.051Z DEBUG Replset is not ready {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"recover-no-primary-29251"}, "namespace": "recover-no-primary-29251", "name": "some-name", "reconcileID": "5562c32f-d741-4cc9-bd16-f3b6ffd47291", "liveMembers": 2, "pods": 3} 2025-12-22T16:40:41.053Z DEBUG Replset status is not healthy {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"recover-no-primary-29251"}, "namespace": "recover-no-primary-29251", "name": "some-name", "reconcileID": "5562c32f-d741-4cc9-bd16-f3b6ffd47291", "replset": "rs1", "status": "initializing"} 2025-12-22T16:40:42.468Z DEBUG Cluster is not ready {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"recover-no-primary-29251"}, "namespace": "recover-no-primary-29251", "name": "some-name", "reconcileID": "5562c32f-d741-4cc9-bd16-f3b6ffd47291", "upgradeInProgress": false, "replsetsReady": 3, "clusterState": "initializing"} 2025-12-22T16:40:48.553Z DEBUG Replset is not ready {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"recover-no-primary-29251"}, "namespace": "recover-no-primary-29251", "name": "some-name", "reconcileID": "fc75b12f-c357-49fe-8049-d70ab1c026f7", "liveMembers": 2, "pods": 3} 2025-12-22T16:40:48.555Z DEBUG Replset status is not healthy {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"recover-no-primary-29251"}, "namespace": "recover-no-primary-29251", "name": "some-name", "reconcileID": "fc75b12f-c357-49fe-8049-d70ab1c026f7", "replset": "rs1", "status": "initializing"} 2025-12-22T16:40:50.049Z DEBUG Cluster is not ready {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"recover-no-primary-29251"}, "namespace": "recover-no-primary-29251", "name": "some-name", "reconcileID": "fc75b12f-c357-49fe-8049-d70ab1c026f7", "upgradeInProgress": false, "replsetsReady": 3, "clusterState": "initializing"} 2025-12-22T16:40:55.625Z DEBUG Replset is not ready {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"recover-no-primary-29251"}, "namespace": "recover-no-primary-29251", "name": "some-name", "reconcileID": "df60e51f-f599-4ecf-b916-ccf15768c823", "liveMembers": 2, "pods": 3} 2025-12-22T16:40:55.679Z DEBUG Replset status is not healthy {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"recover-no-primary-29251"}, "namespace": "recover-no-primary-29251", "name": "some-name", "reconcileID": "df60e51f-f599-4ecf-b916-ccf15768c823", "replset": "rs1", "status": "initializing"} 2025-12-22T16:40:57.071Z DEBUG Cluster is not ready {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"recover-no-primary-29251"}, "namespace": "recover-no-primary-29251", "name": "some-name", "reconcileID": "df60e51f-f599-4ecf-b916-ccf15768c823", "upgradeInProgress": false, "replsetsReady": 3, "clusterState": "initializing"} 2025-12-22T16:41:02.476Z DEBUG Replset is not ready {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"recover-no-primary-29251"}, "namespace": "recover-no-primary-29251", "name": "some-name", "reconcileID": "2a1f4281-9c4d-464b-958b-9722fd52ef6d", "liveMembers": 2, "pods": 3} 2025-12-22T16:41:02.534Z DEBUG Replset status is not healthy {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"recover-no-primary-29251"}, "namespace": "recover-no-primary-29251", "name": "some-name", "reconcileID": "2a1f4281-9c4d-464b-958b-9722fd52ef6d", "replset": "rs1", "status": "initializing"} 2025-12-22T16:41:04.219Z DEBUG Cluster is not ready {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"recover-no-primary-29251"}, "namespace": "recover-no-primary-29251", "name": "some-name", "reconcileID": "2a1f4281-9c4d-464b-958b-9722fd52ef6d", "upgradeInProgress": false, "replsetsReady": 3, "clusterState": "initializing"} 2025-12-22T16:41:08.659Z DEBUG Replset is not ready {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"recover-no-primary-29251"}, "namespace": "recover-no-primary-29251", "name": "some-name", "reconcileID": "adf199a9-a74d-41d3-bfd6-0854ff8d5e05", "liveMembers": 2, "pods": 3} 2025-12-22T16:41:08.675Z DEBUG Replset status is not healthy {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"recover-no-primary-29251"}, "namespace": "recover-no-primary-29251", "name": "some-name", "reconcileID": "adf199a9-a74d-41d3-bfd6-0854ff8d5e05", "replset": "rs1", "status": "initializing"} 2025-12-22T16:41:10.083Z DEBUG Cluster is not ready {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"recover-no-primary-29251"}, "namespace": "recover-no-primary-29251", "name": "some-name", "reconcileID": "adf199a9-a74d-41d3-bfd6-0854ff8d5e05", "upgradeInProgress": false, "replsetsReady": 3, "clusterState": "initializing"} 2025-12-22T16:41:14.717Z DEBUG Replset is not ready {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"recover-no-primary-29251"}, "namespace": "recover-no-primary-29251", "name": "some-name", "reconcileID": "e6a9aa69-6bf3-450a-aa9f-18c1a3543bac", "liveMembers": 2, "pods": 3} 2025-12-22T16:41:14.724Z DEBUG Replset status is not healthy {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"recover-no-primary-29251"}, "namespace": "recover-no-primary-29251", "name": "some-name", "reconcileID": "e6a9aa69-6bf3-450a-aa9f-18c1a3543bac", "replset": "rs1", "status": "initializing"} 2025-12-22T16:41:16.130Z DEBUG Cluster is not ready {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"recover-no-primary-29251"}, "namespace": "recover-no-primary-29251", "name": "some-name", "reconcileID": "e6a9aa69-6bf3-450a-aa9f-18c1a3543bac", "upgradeInProgress": false, "replsetsReady": 3, "clusterState": "initializing"} 2025-12-22T16:41:21.059Z DEBUG Replset is not ready {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"recover-no-primary-29251"}, "namespace": "recover-no-primary-29251", "name": "some-name", "reconcileID": "e0091a86-4096-4aff-844f-151a7b6adf82", "liveMembers": 2, "pods": 3} 2025-12-22T16:41:21.433Z DEBUG Replset status is not healthy {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"recover-no-primary-29251"}, "namespace": "recover-no-primary-29251", "name": "some-name", "reconcileID": "e0091a86-4096-4aff-844f-151a7b6adf82", "replset": "rs1", "status": "initializing"} 2025-12-22T16:41:22.835Z DEBUG Cluster is not ready {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"recover-no-primary-29251"}, "namespace": "recover-no-primary-29251", "name": "some-name", "reconcileID": "e0091a86-4096-4aff-844f-151a7b6adf82", "upgradeInProgress": false, "replsetsReady": 3, "clusterState": "initializing"} 2025-12-22T16:41:27.864Z DEBUG Replset is not ready {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"recover-no-primary-29251"}, "namespace": "recover-no-primary-29251", "name": "some-name", "reconcileID": "360ce320-2564-43c6-87b0-694829b72576", "liveMembers": 2, "pods": 3} 2025-12-22T16:41:27.925Z DEBUG Replset status is not healthy {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"recover-no-primary-29251"}, "namespace": "recover-no-primary-29251", "name": "some-name", "reconcileID": "360ce320-2564-43c6-87b0-694829b72576", "replset": "rs1", "status": "initializing"} 2025-12-22T16:41:29.362Z DEBUG Cluster is not ready {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"recover-no-primary-29251"}, "namespace": "recover-no-primary-29251", "name": "some-name", "reconcileID": "360ce320-2564-43c6-87b0-694829b72576", "upgradeInProgress": false, "replsetsReady": 3, "clusterState": "initializing"} 2025-12-22T16:41:35.138Z DEBUG Replset is not ready {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"recover-no-primary-29251"}, "namespace": "recover-no-primary-29251", "name": "some-name", "reconcileID": "1aa80a66-b8c6-4c69-98f4-978764295d4e", "liveMembers": 2, "pods": 3} 2025-12-22T16:41:35.319Z DEBUG Replset status is not healthy {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"recover-no-primary-29251"}, "namespace": "recover-no-primary-29251", "name": "some-name", "reconcileID": "1aa80a66-b8c6-4c69-98f4-978764295d4e", "replset": "rs1", "status": "initializing"} 2025-12-22T16:41:38.029Z DEBUG Cluster is not ready {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"recover-no-primary-29251"}, "namespace": "recover-no-primary-29251", "name": "some-name", "reconcileID": "1aa80a66-b8c6-4c69-98f4-978764295d4e", "upgradeInProgress": false, "replsetsReady": 3, "clusterState": "initializing"} 2025-12-22T16:41:43.921Z DEBUG Replset is not ready {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"recover-no-primary-29251"}, "namespace": "recover-no-primary-29251", "name": "some-name", "reconcileID": "47548cf2-9ef1-4913-8a06-57f0e6935d12", "liveMembers": 2, "pods": 3} 2025-12-22T16:41:43.925Z DEBUG Replset status is not healthy {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"recover-no-primary-29251"}, "namespace": "recover-no-primary-29251", "name": "some-name", "reconcileID": "47548cf2-9ef1-4913-8a06-57f0e6935d12", "replset": "rs1", "status": "initializing"} 2025-12-22T16:41:45.352Z DEBUG Cluster is not ready {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"recover-no-primary-29251"}, "namespace": "recover-no-primary-29251", "name": "some-name", "reconcileID": "47548cf2-9ef1-4913-8a06-57f0e6935d12", "upgradeInProgress": false, "replsetsReady": 3, "clusterState": "initializing"} 2025-12-22T16:41:50.082Z DEBUG Replset is not ready {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"recover-no-primary-29251"}, "namespace": "recover-no-primary-29251", "name": "some-name", "reconcileID": "97ea8ead-056d-4105-a086-87fe2879a848", "liveMembers": 2, "pods": 3} 2025-12-22T16:41:50.122Z DEBUG Replset status is not healthy {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"recover-no-primary-29251"}, "namespace": "recover-no-primary-29251", "name": "some-name", "reconcileID": "97ea8ead-056d-4105-a086-87fe2879a848", "replset": "rs1", "status": "initializing"} 2025-12-22T16:41:51.563Z DEBUG Cluster is not ready {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"recover-no-primary-29251"}, "namespace": "recover-no-primary-29251", "name": "some-name", "reconcileID": "97ea8ead-056d-4105-a086-87fe2879a848", "upgradeInProgress": false, "replsetsReady": 3, "clusterState": "initializing"} 2025-12-22T16:41:56.482Z DEBUG Replset is not ready {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"recover-no-primary-29251"}, "namespace": "recover-no-primary-29251", "name": "some-name", "reconcileID": "8a084718-0fb4-49df-8ec4-8140d234c762", "liveMembers": 2, "pods": 3} 2025-12-22T16:41:56.485Z DEBUG Replset status is not healthy {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"recover-no-primary-29251"}, "namespace": "recover-no-primary-29251", "name": "some-name", "reconcileID": "8a084718-0fb4-49df-8ec4-8140d234c762", "replset": "rs1", "status": "initializing"} 2025-12-22T16:41:57.881Z DEBUG Cluster is not ready {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"recover-no-primary-29251"}, "namespace": "recover-no-primary-29251", "name": "some-name", "reconcileID": "8a084718-0fb4-49df-8ec4-8140d234c762", "upgradeInProgress": false, "replsetsReady": 3, "clusterState": "initializing"} 2025-12-22T16:42:03.274Z DEBUG Replset is not ready {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"recover-no-primary-29251"}, "namespace": "recover-no-primary-29251", "name": "some-name", "reconcileID": "f9b583d2-1722-47e5-80db-88ee166c89b9", "liveMembers": 2, "pods": 3} 2025-12-22T16:42:03.377Z DEBUG Replset status is not healthy {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"recover-no-primary-29251"}, "namespace": "recover-no-primary-29251", "name": "some-name", "reconcileID": "f9b583d2-1722-47e5-80db-88ee166c89b9", "replset": "rs1", "status": "initializing"} 2025-12-22T16:42:05.147Z DEBUG Cluster is not ready {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"recover-no-primary-29251"}, "namespace": "recover-no-primary-29251", "name": "some-name", "reconcileID": "f9b583d2-1722-47e5-80db-88ee166c89b9", "upgradeInProgress": false, "replsetsReady": 3, "clusterState": "initializing"} 2025-12-22T16:42:08.929Z DEBUG Replset is not ready {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"recover-no-primary-29251"}, "namespace": "recover-no-primary-29251", "name": "some-name", "reconcileID": "e1ebd67a-96a8-42cf-9dbb-f087002219da", "liveMembers": 2, "pods": 3} 2025-12-22T16:42:08.936Z DEBUG Replset status is not healthy {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"recover-no-primary-29251"}, "namespace": "recover-no-primary-29251", "name": "some-name", "reconcileID": "e1ebd67a-96a8-42cf-9dbb-f087002219da", "replset": "rs1", "status": "initializing"} 2025-12-22T16:42:10.350Z DEBUG Cluster is not ready {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"recover-no-primary-29251"}, "namespace": "recover-no-primary-29251", "name": "some-name", "reconcileID": "e1ebd67a-96a8-42cf-9dbb-f087002219da", "upgradeInProgress": false, "replsetsReady": 3, "clusterState": "initializing"} 2025-12-22T16:42:16.329Z DEBUG Replset is not ready {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"recover-no-primary-29251"}, "namespace": "recover-no-primary-29251", "name": "some-name", "reconcileID": "a7a16c41-e8b8-460c-ad31-c29a484b43d0", "liveMembers": 2, "pods": 3} 2025-12-22T16:42:16.331Z DEBUG Replset status is not healthy {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"recover-no-primary-29251"}, "namespace": "recover-no-primary-29251", "name": "some-name", "reconcileID": "a7a16c41-e8b8-460c-ad31-c29a484b43d0", "replset": "rs1", "status": "initializing"} 2025-12-22T16:42:17.730Z DEBUG Cluster is not ready {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"recover-no-primary-29251"}, "namespace": "recover-no-primary-29251", "name": "some-name", "reconcileID": "a7a16c41-e8b8-460c-ad31-c29a484b43d0", "upgradeInProgress": false, "replsetsReady": 3, "clusterState": "initializing"} 2025-12-22T16:42:22.675Z DEBUG Replset is not ready {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"recover-no-primary-29251"}, "namespace": "recover-no-primary-29251", "name": "some-name", "reconcileID": "2197c9ea-68ac-4716-8f87-1392d7eb04f5", "liveMembers": 2, "pods": 3} 2025-12-22T16:42:22.728Z DEBUG Replset status is not healthy {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"recover-no-primary-29251"}, "namespace": "recover-no-primary-29251", "name": "some-name", "reconcileID": "2197c9ea-68ac-4716-8f87-1392d7eb04f5", "replset": "rs1", "status": "initializing"} 2025-12-22T16:42:24.114Z DEBUG Cluster is not ready {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"recover-no-primary-29251"}, "namespace": "recover-no-primary-29251", "name": "some-name", "reconcileID": "2197c9ea-68ac-4716-8f87-1392d7eb04f5", "upgradeInProgress": false, "replsetsReady": 3, "clusterState": "initializing"} 2025-12-22T16:42:31.982Z DEBUG Replset is not ready {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"recover-no-primary-29251"}, "namespace": "recover-no-primary-29251", "name": "some-name", "reconcileID": "dbb97663-4044-4c2e-b16e-470a83f17f32", "liveMembers": 2, "pods": 3} 2025-12-22T16:42:32.082Z DEBUG Replset status is not healthy {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"recover-no-primary-29251"}, "namespace": "recover-no-primary-29251", "name": "some-name", "reconcileID": "dbb97663-4044-4c2e-b16e-470a83f17f32", "replset": "rs1", "status": "initializing"} 2025-12-22T16:42:34.416Z DEBUG Cluster is not ready {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"recover-no-primary-29251"}, "namespace": "recover-no-primary-29251", "name": "some-name", "reconcileID": "dbb97663-4044-4c2e-b16e-470a83f17f32", "upgradeInProgress": false, "replsetsReady": 3, "clusterState": "initializing"} 2025-12-22T16:42:44.079Z DEBUG Replset is not ready {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"recover-no-primary-29251"}, "namespace": "recover-no-primary-29251", "name": "some-name", "reconcileID": "756a0ac0-fd5b-4a02-9329-fc62d1105853", "liveMembers": 2, "pods": 3} 2025-12-22T16:42:44.128Z DEBUG Replset status is not healthy {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"recover-no-primary-29251"}, "namespace": "recover-no-primary-29251", "name": "some-name", "reconcileID": "756a0ac0-fd5b-4a02-9329-fc62d1105853", "replset": "rs1", "status": "initializing"} 2025-12-22T16:42:45.515Z DEBUG Cluster is not ready {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"recover-no-primary-29251"}, "namespace": "recover-no-primary-29251", "name": "some-name", "reconcileID": "756a0ac0-fd5b-4a02-9329-fc62d1105853", "upgradeInProgress": false, "replsetsReady": 3, "clusterState": "initializing"} 2025-12-22T16:42:50.658Z DEBUG Replset is not ready {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"recover-no-primary-29251"}, "namespace": "recover-no-primary-29251", "name": "some-name", "reconcileID": "f2a30145-e236-4e0e-a3a9-0e980c030f20", "liveMembers": 2, "pods": 3} 2025-12-22T16:42:50.730Z DEBUG Replset status is not healthy {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"recover-no-primary-29251"}, "namespace": "recover-no-primary-29251", "name": "some-name", "reconcileID": "f2a30145-e236-4e0e-a3a9-0e980c030f20", "replset": "rs1", "status": "initializing"} 2025-12-22T16:42:52.187Z DEBUG Cluster is not ready {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"recover-no-primary-29251"}, "namespace": "recover-no-primary-29251", "name": "some-name", "reconcileID": "f2a30145-e236-4e0e-a3a9-0e980c030f20", "upgradeInProgress": false, "replsetsReady": 3, "clusterState": "initializing"} Waiting timeout has been reached. Exiting...