Log: /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2277/e2e-tests/logs/data-at-rest-encryption.log Warning: version difference between client (1.35) and server (1.32) exceeds the supported minor version skew of +/-1 Warning: version difference between client (1.35) and server (1.32) exceeds the supported minor version skew of +/-1 Warning: version difference between client (1.35) and server (1.32) exceeds the supported minor version skew of +/-1 + create_infra data-at-rest-encryption-4238 + local ns=data-at-rest-encryption-4238 + [[ 1 == 1 ]] + delete_crd + desc 'get and delete old CRDs and RBAC' + set +o xtrace ----------------------------------------------------------------------------------- get and delete old CRDs and RBAC ----------------------------------------------------------------------------------- + kubectl_bin delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2277/deploy/crd.yaml --ignore-not-found --wait=false ++ mktemp + local LAST_OUT=/tmp/tmp.q39fi2TCpi ++ mktemp + local LAST_ERR=/tmp/tmp.ObvQH1EGq7 + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2277/deploy/crd.yaml --ignore-not-found --wait=false + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.q39fi2TCpi customresourcedefinition.apiextensions.k8s.io "perconaservermongodbbackups.psmdb.percona.com" deleted customresourcedefinition.apiextensions.k8s.io "perconaservermongodbrestores.psmdb.percona.com" deleted customresourcedefinition.apiextensions.k8s.io "perconaservermongodbs.psmdb.percona.com" deleted + cat /tmp/tmp.ObvQH1EGq7 + rm /tmp/tmp.q39fi2TCpi /tmp/tmp.ObvQH1EGq7 + return 0 ++ yq eval .metadata.name /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2277/deploy/crd.yaml ++ grep -v '\-\-\-' grep: warning: stray \ before - grep: warning: stray \ before - + for crd_name in $(yq eval '.metadata.name' "${src_dir}/deploy/crd.yaml" | grep -v '\-\-\-') + kubectl get perconaservermongodbbackups.psmdb.percona.com --all-namespaces -o wide + grep -v NAMESPACE + xargs -L 1 sh -xc 'kubectl patch perconaservermongodbbackups.psmdb.percona.com -n $0 $1 --type=merge -p "{\"metadata\":{\"finalizers\":[]}}"' + kubectl patch perconaservermongodbbackups.psmdb.percona.com -n data-at-rest-encryption-3048 backup-gcp-cs --type=merge -p '{"metadata":{"finalizers":[]}}' perconaservermongodbbackup.psmdb.percona.com/backup-gcp-cs patched + kubectl patch perconaservermongodbbackups.psmdb.percona.com -n data-at-rest-encryption-3048 backup-minio --type=merge -p '{"metadata":{"finalizers":[]}}' perconaservermongodbbackup.psmdb.percona.com/backup-minio patched + kubectl_bin wait --for=delete crd perconaservermongodbbackups.psmdb.percona.com ++ mktemp + local LAST_OUT=/tmp/tmp.paIm2iQpTA ++ mktemp + local LAST_ERR=/tmp/tmp.91YRhB8A8l + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl wait --for=delete crd perconaservermongodbbackups.psmdb.percona.com + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.paIm2iQpTA customresourcedefinition.apiextensions.k8s.io/perconaservermongodbbackups.psmdb.percona.com condition met + cat /tmp/tmp.91YRhB8A8l + rm /tmp/tmp.paIm2iQpTA /tmp/tmp.91YRhB8A8l + return 0 + for crd_name in $(yq eval '.metadata.name' "${src_dir}/deploy/crd.yaml" | grep -v '\-\-\-') + kubectl get perconaservermongodbrestores.psmdb.percona.com --all-namespaces -o wide + grep -v NAMESPACE + xargs -L 1 sh -xc 'kubectl patch perconaservermongodbrestores.psmdb.percona.com -n $0 $1 --type=merge -p "{\"metadata\":{\"finalizers\":[]}}"' error: the server doesn't have a resource type "perconaservermongodbrestores" + kubectl patch perconaservermongodbrestores.psmdb.percona.com -n sh --type=merge -p '{"metadata":{"finalizers":[]}}' error: the server doesn't have a resource type "perconaservermongodbrestores" + : + kubectl_bin wait --for=delete crd perconaservermongodbrestores.psmdb.percona.com ++ mktemp + local LAST_OUT=/tmp/tmp.yCGHpIgwuC ++ mktemp + local LAST_ERR=/tmp/tmp.wn0RHe4NG7 + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl wait --for=delete crd perconaservermongodbrestores.psmdb.percona.com + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.yCGHpIgwuC + cat /tmp/tmp.wn0RHe4NG7 + rm /tmp/tmp.yCGHpIgwuC /tmp/tmp.wn0RHe4NG7 + return 0 + for crd_name in $(yq eval '.metadata.name' "${src_dir}/deploy/crd.yaml" | grep -v '\-\-\-') + kubectl get perconaservermongodbs.psmdb.percona.com --all-namespaces -o wide + grep -v NAMESPACE + xargs -L 1 sh -xc 'kubectl patch perconaservermongodbs.psmdb.percona.com -n $0 $1 --type=merge -p "{\"metadata\":{\"finalizers\":[]}}"' error: the server doesn't have a resource type "perconaservermongodbs" + kubectl patch perconaservermongodbs.psmdb.percona.com -n sh --type=merge -p '{"metadata":{"finalizers":[]}}' error: the server doesn't have a resource type "perconaservermongodbs" + : + kubectl_bin wait --for=delete crd perconaservermongodbs.psmdb.percona.com ++ mktemp + local LAST_OUT=/tmp/tmp.52YqSsruws ++ mktemp + local LAST_ERR=/tmp/tmp.EZAoG1B8GY + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl wait --for=delete crd perconaservermongodbs.psmdb.percona.com + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.52YqSsruws + cat /tmp/tmp.EZAoG1B8GY + rm /tmp/tmp.52YqSsruws /tmp/tmp.EZAoG1B8GY + return 0 + local rbac_yaml=rbac.yaml + '[' -n psmdb-operator ']' + rbac_yaml=cw-rbac.yaml + kubectl_bin delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2277/deploy/cw-rbac.yaml --ignore-not-found ++ mktemp + local LAST_OUT=/tmp/tmp.sXbulZ5Hum ++ mktemp + local LAST_ERR=/tmp/tmp.cQectJC1b9 + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2277/deploy/cw-rbac.yaml --ignore-not-found + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.sXbulZ5Hum clusterrole.rbac.authorization.k8s.io "percona-server-mongodb-operator" deleted clusterrolebinding.rbac.authorization.k8s.io "service-account-percona-server-mongodb-operator" deleted + cat /tmp/tmp.cQectJC1b9 + rm /tmp/tmp.sXbulZ5Hum /tmp/tmp.cQectJC1b9 + return 0 + check_crd_for_deletion PR-2277-0a1aa642f + local git_tag=PR-2277-0a1aa642f ++ curl -s https://raw.githubusercontent.com/percona/percona-server-mongodb-operator/PR-2277-0a1aa642f/deploy/crd.yaml ++ yq eval .metadata.name ++ /usr/sbin/sed s/---//g ++ /usr/sbin/sed ':a;N;$!ba;s/\n/ /g' + for crd_name in $(curl -s https://raw.githubusercontent.com/percona/percona-server-mongodb-operator/${git_tag}/deploy/crd.yaml | yq eval '.metadata.name' | $sed 's/---//g' | $sed ':a;N;$!ba;s/\n/ /g') ++ kubectl_bin get crd/null -o 'jsonpath={.status.conditions[-1].type}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.Tow4OY3wcf +++ mktemp ++ local LAST_ERR=/tmp/tmp.garizMfsrX ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get crd/null -o 'jsonpath={.status.conditions[-1].type}' ++ exit_status=1 ++ set -e ++ '[' 1 '!=' 0 -a -n 1 ']' ++ cat /tmp/tmp.Tow4OY3wcf ++ cat /tmp/tmp.garizMfsrX Error from server (NotFound): customresourcedefinitions.apiextensions.k8s.io "null" not found ++ sleep 0 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get crd/null -o 'jsonpath={.status.conditions[-1].type}' ++ exit_status=1 ++ set -e ++ '[' 1 '!=' 0 -a -n 1 ']' ++ cat /tmp/tmp.Tow4OY3wcf ++ cat /tmp/tmp.garizMfsrX Error from server (NotFound): customresourcedefinitions.apiextensions.k8s.io "null" not found ++ sleep 4 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get crd/null -o 'jsonpath={.status.conditions[-1].type}' ++ exit_status=1 ++ set -e ++ '[' 1 '!=' 0 -a -n 1 ']' ++ cat /tmp/tmp.Tow4OY3wcf ++ cat /tmp/tmp.garizMfsrX Error from server (NotFound): customresourcedefinitions.apiextensions.k8s.io "null" not found ++ sleep 8 ++ cat /tmp/tmp.Tow4OY3wcf ++ cat /tmp/tmp.garizMfsrX Error from server (NotFound): customresourcedefinitions.apiextensions.k8s.io "null" not found ++ rm /tmp/tmp.Tow4OY3wcf /tmp/tmp.garizMfsrX ++ return 1 + [[ '' == \T\e\r\m\i\n\a\t\i\n\g ]] + '[' -n psmdb-operator ']' + create_namespace psmdb-operator + local namespace=psmdb-operator + local skip_clean_namespace= + [[ 1 == 1 ]] + [[ -z '' ]] + destroy_chaos_mesh ++ helm list --all-namespaces --filter chaos-mesh ++ tail -n1 ++ awk '-F ' '{print $2}' ++ sed s/NAMESPACE// + local chaos_mesh_ns= + desc 'destroy chaos-mesh' + set +o xtrace ----------------------------------------------------------------------------------- destroy chaos-mesh ----------------------------------------------------------------------------------- + '[' -n '' ']' ++ kubectl get MutatingWebhookConfiguration ++ grep chaos-mesh ++ awk '{print $1}' + timeout 30 kubectl delete MutatingWebhookConfiguration error: resource(s) were provided, but no name was specified + : ++ grep chaos-mesh ++ kubectl get ValidatingWebhookConfiguration ++ awk '{print $1}' + timeout 30 kubectl delete ValidatingWebhookConfiguration error: resource(s) were provided, but no name was specified + : ++ kubectl get ValidatingWebhookConfiguration ++ grep validate-auth ++ awk '{print $1}' + timeout 30 kubectl delete ValidatingWebhookConfiguration error: resource(s) were provided, but no name was specified + : ++ kubectl api-resources ++ grep chaos-mesh ++ awk '{print $1}' ++ kubectl get crd ++ grep chaos-mesh.org ++ awk '{print $1}' + timeout 30 kubectl delete crd error: resource(s) were provided, but no name was specified + : ++ kubectl get clusterrolebinding ++ grep chaos-mesh ++ awk '{print $1}' + timeout 30 kubectl delete clusterrolebinding error: resource(s) were provided, but no name was specified + : ++ kubectl get clusterrole ++ grep chaos-mesh ++ awk '{print $1}' + timeout 30 kubectl delete clusterrole error: resource(s) were provided, but no name was specified + : + desc 'cleaned up all old namespaces' + set +o xtrace ----------------------------------------------------------------------------------- cleaned up all old namespaces ----------------------------------------------------------------------------------- + kubectl_bin get ns + grep -E -v '^kube-|^default|Terminating|psmdb-operator|openshift|^gke-|^gmp-|^NAME' ++ mktemp + '[' -n '' ']' + xargs kubectl delete ns + desc 'cleaned up old namespaces psmdb-operator' + set +o xtrace ----------------------------------------------------------------------------------- cleaned up old namespaces psmdb-operator ----------------------------------------------------------------------------------- + kubectl_bin delete namespace psmdb-operator --ignore-not-found ++ mktemp + awk '{print$1}' + local LAST_OUT=/tmp/tmp.5tJ2nZu0fp ++ mktemp + local LAST_OUT=/tmp/tmp.RnXc2cA6oP ++ mktemp + local LAST_ERR=/tmp/tmp.5hxLaLk2Dx + local exit_status=0 + local timeout=4 + local LAST_ERR=/tmp/tmp.ofpprVn0zJ + local exit_status=0 + local timeout=4 ++ seq 0 2 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl get ns + for i in $(seq 0 2) + set +e + kubectl delete namespace psmdb-operator --ignore-not-found + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.5tJ2nZu0fp + cat /tmp/tmp.5hxLaLk2Dx + rm /tmp/tmp.5tJ2nZu0fp /tmp/tmp.5hxLaLk2Dx + return 0 namespace "data-at-rest-encryption-3048" deleted + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.RnXc2cA6oP namespace "psmdb-operator" deleted + cat /tmp/tmp.ofpprVn0zJ + rm /tmp/tmp.RnXc2cA6oP /tmp/tmp.ofpprVn0zJ + return 0 + kubectl_bin wait --for=delete namespace psmdb-operator ++ mktemp + local LAST_OUT=/tmp/tmp.O5jOSVzZ8Z ++ mktemp + local LAST_ERR=/tmp/tmp.xZYlYssGi9 + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl wait --for=delete namespace psmdb-operator + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.O5jOSVzZ8Z + cat /tmp/tmp.xZYlYssGi9 + rm /tmp/tmp.O5jOSVzZ8Z /tmp/tmp.xZYlYssGi9 + return 0 + desc 'create namespace psmdb-operator' + set +o xtrace ----------------------------------------------------------------------------------- create namespace psmdb-operator ----------------------------------------------------------------------------------- + kubectl_bin create namespace psmdb-operator ++ mktemp + local LAST_OUT=/tmp/tmp.83mqJeTyVe ++ mktemp + local LAST_ERR=/tmp/tmp.8rbcDhDzAc + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl create namespace psmdb-operator + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.83mqJeTyVe namespace/psmdb-operator created + cat /tmp/tmp.8rbcDhDzAc + rm /tmp/tmp.83mqJeTyVe /tmp/tmp.8rbcDhDzAc + return 0 + set_kube_ctx psmdb-operator + local namespace=psmdb-operator ++ kubectl_bin config current-context +++ mktemp ++ local LAST_OUT=/tmp/tmp.6tq8785yPH +++ mktemp ++ local LAST_ERR=/tmp/tmp.boJlYUKTNv ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl config current-context ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.6tq8785yPH ++ cat /tmp/tmp.boJlYUKTNv ++ rm /tmp/tmp.6tq8785yPH /tmp/tmp.boJlYUKTNv ++ return 0 + kubectl_bin config set-context gke_cloud-dev-112233_us-central1-a_jen-psmdb-2277-0a1aa642f-2-cluster8 --namespace=psmdb-operator ++ mktemp + local LAST_OUT=/tmp/tmp.ZZkqztOLwr ++ mktemp + local LAST_ERR=/tmp/tmp.VUQMjD6ehW + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl config set-context gke_cloud-dev-112233_us-central1-a_jen-psmdb-2277-0a1aa642f-2-cluster8 --namespace=psmdb-operator + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.ZZkqztOLwr Context "gke_cloud-dev-112233_us-central1-a_jen-psmdb-2277-0a1aa642f-2-cluster8" modified. + cat /tmp/tmp.VUQMjD6ehW + rm /tmp/tmp.ZZkqztOLwr /tmp/tmp.VUQMjD6ehW + return 0 + deploy_operator + desc 'start PSMDB operator: docker.io/perconalab/percona-server-mongodb-operator:PR-2277-0a1aa642f' + set +o xtrace ----------------------------------------------------------------------------------- start PSMDB operator: docker.io/perconalab/percona-server-mongodb-operator:PR-2277-0a1aa642f ----------------------------------------------------------------------------------- + local cr_file + '[' -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2277/e2e-tests/data-at-rest-encryption/conf/crd.yaml ']' + cr_file=/mnt/jenkins/workspace/cloud-psmdb-operator_PR-2277/deploy/crd.yaml + kubectl_bin apply --server-side --force-conflicts -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2277/deploy/crd.yaml ++ mktemp + local LAST_OUT=/tmp/tmp.GYMSaB8ewu ++ mktemp + local LAST_ERR=/tmp/tmp.G1BT0czO4H + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl apply --server-side --force-conflicts -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2277/deploy/crd.yaml + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.GYMSaB8ewu customresourcedefinition.apiextensions.k8s.io/perconaservermongodbbackups.psmdb.percona.com serverside-applied customresourcedefinition.apiextensions.k8s.io/perconaservermongodbrestores.psmdb.percona.com serverside-applied customresourcedefinition.apiextensions.k8s.io/perconaservermongodbs.psmdb.percona.com serverside-applied + cat /tmp/tmp.G1BT0czO4H + rm /tmp/tmp.GYMSaB8ewu /tmp/tmp.G1BT0czO4H + return 0 + '[' -n psmdb-operator ']' + apply_rbac cw-rbac + local operator_namespace=psmdb-operator + local rbac=cw-rbac + sed -e 's^namespace: .*^namespace: psmdb-operator^' + cat /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2277/deploy/cw-rbac.yaml + kubectl_bin apply -n psmdb-operator -f - ++ mktemp + local LAST_OUT=/tmp/tmp.tsN61CUTUY ++ mktemp + local LAST_ERR=/tmp/tmp.ltdwvcVPHx + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl apply -n psmdb-operator -f - + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.tsN61CUTUY clusterrole.rbac.authorization.k8s.io/percona-server-mongodb-operator created serviceaccount/percona-server-mongodb-operator created clusterrolebinding.rbac.authorization.k8s.io/service-account-percona-server-mongodb-operator created + cat /tmp/tmp.ltdwvcVPHx + rm /tmp/tmp.tsN61CUTUY /tmp/tmp.ltdwvcVPHx + return 0 + yq eval ' (.spec.template.spec.containers[].image = "docker.io/perconalab/percona-server-mongodb-operator:PR-2277-0a1aa642f") | ((.. | select(.[] == "DISABLE_TELEMETRY")) |= .value="true") | ((.. | select(.[] == "LOG_LEVEL")) |= .value="DEBUG")' /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2277/deploy/cw-operator.yaml + kubectl_bin apply -n psmdb-operator -f - ++ mktemp + local LAST_OUT=/tmp/tmp.1vckbMLQza ++ mktemp + local LAST_ERR=/tmp/tmp.hke91zXAYf + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl apply -n psmdb-operator -f - + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.1vckbMLQza deployment.apps/percona-server-mongodb-operator created + cat /tmp/tmp.hke91zXAYf + rm /tmp/tmp.1vckbMLQza /tmp/tmp.hke91zXAYf + return 0 + sleep 20 ++ get_operator_pod ++ kubectl_bin get pods --selector=name=percona-server-mongodb-operator -o 'jsonpath={.items[].metadata.name}' -n psmdb-operator +++ mktemp ++ local LAST_OUT=/tmp/tmp.v7eHqBdBzX +++ mktemp ++ local LAST_ERR=/tmp/tmp.qSUzkn2sVQ ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pods --selector=name=percona-server-mongodb-operator -o 'jsonpath={.items[].metadata.name}' -n psmdb-operator ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.v7eHqBdBzX ++ cat /tmp/tmp.qSUzkn2sVQ ++ rm /tmp/tmp.v7eHqBdBzX /tmp/tmp.qSUzkn2sVQ ++ return 0 + wait_operator_pod percona-server-mongodb-operator-5b5f6fcd94-wmj5f + local pod=percona-server-mongodb-operator-5b5f6fcd94-wmj5f + set +o xtrace waiting for pod/percona-server-mongodb-operator-5b5f6fcd94-wmj5f to be ready.OK + echo 'Print operator info from log' Print operator info from log + grep 'Manager starting up' ++ get_operator_pod ++ kubectl_bin get pods --selector=name=percona-server-mongodb-operator -o 'jsonpath={.items[].metadata.name}' -n psmdb-operator +++ mktemp ++ local LAST_OUT=/tmp/tmp.HFrWpvZ0ax +++ mktemp ++ local LAST_ERR=/tmp/tmp.nwtErxm1Pb ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pods --selector=name=percona-server-mongodb-operator -o 'jsonpath={.items[].metadata.name}' -n psmdb-operator ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.HFrWpvZ0ax ++ cat /tmp/tmp.nwtErxm1Pb ++ rm /tmp/tmp.HFrWpvZ0ax /tmp/tmp.nwtErxm1Pb ++ return 0 + kubectl_bin logs -n psmdb-operator percona-server-mongodb-operator-5b5f6fcd94-wmj5f ++ mktemp + local LAST_OUT=/tmp/tmp.q6r2SJD5Sz ++ mktemp + local LAST_ERR=/tmp/tmp.ekXWnZ247J + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl logs -n psmdb-operator percona-server-mongodb-operator-5b5f6fcd94-wmj5f + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.q6r2SJD5Sz + cat /tmp/tmp.ekXWnZ247J + rm /tmp/tmp.q6r2SJD5Sz /tmp/tmp.ekXWnZ247J + return 0 2026-03-09T09:08:10.924Z INFO setup Manager starting up {"gitCommit": "0a1aa642fab677b3a3eb9b1dee5f81fc42e34249", "gitBranch": "PR-2277-0a1aa642f", "buildTime": "", "goVersion": "go1.25.8", "os": "linux", "arch": "amd64"} + create_namespace data-at-rest-encryption-4238 + local namespace=data-at-rest-encryption-4238 + local skip_clean_namespace= + [[ 1 == 1 ]] + [[ -z '' ]] + destroy_chaos_mesh ++ helm list --all-namespaces --filter chaos-mesh ++ tail -n1 ++ awk '-F ' '{print $2}' ++ sed s/NAMESPACE// + local chaos_mesh_ns= + desc 'destroy chaos-mesh' + set +o xtrace ----------------------------------------------------------------------------------- destroy chaos-mesh ----------------------------------------------------------------------------------- + '[' -n '' ']' ++ kubectl get MutatingWebhookConfiguration ++ grep chaos-mesh ++ awk '{print $1}' + timeout 30 kubectl delete MutatingWebhookConfiguration error: resource(s) were provided, but no name was specified + : ++ kubectl get ValidatingWebhookConfiguration ++ grep chaos-mesh ++ awk '{print $1}' + timeout 30 kubectl delete ValidatingWebhookConfiguration error: resource(s) were provided, but no name was specified + : ++ kubectl get ValidatingWebhookConfiguration ++ grep validate-auth ++ awk '{print $1}' + timeout 30 kubectl delete ValidatingWebhookConfiguration error: resource(s) were provided, but no name was specified + : ++ kubectl api-resources ++ awk '{print $1}' ++ grep chaos-mesh ++ kubectl get crd ++ grep chaos-mesh.org ++ awk '{print $1}' + timeout 30 kubectl delete crd error: resource(s) were provided, but no name was specified + : ++ kubectl get clusterrolebinding ++ grep chaos-mesh ++ awk '{print $1}' + timeout 30 kubectl delete clusterrolebinding error: resource(s) were provided, but no name was specified + : ++ kubectl get clusterrole ++ grep chaos-mesh ++ awk '{print $1}' + timeout 30 kubectl delete clusterrole error: resource(s) were provided, but no name was specified + : + desc 'cleaned up all old namespaces' + set +o xtrace ----------------------------------------------------------------------------------- cleaned up all old namespaces ----------------------------------------------------------------------------------- + kubectl_bin get ns + grep -E -v '^kube-|^default|Terminating|psmdb-operator|openshift|^gke-|^gmp-|^NAME' + awk '{print$1}' + '[' -n '' ']' + xargs kubectl delete ns + desc 'cleaned up old namespaces data-at-rest-encryption-4238' ++ mktemp + set +o xtrace ----------------------------------------------------------------------------------- cleaned up old namespaces data-at-rest-encryption-4238 ----------------------------------------------------------------------------------- + kubectl_bin delete namespace data-at-rest-encryption-4238 --ignore-not-found + local LAST_OUT=/tmp/tmp.X0gFu7rUtm ++ mktemp ++ mktemp + local LAST_OUT=/tmp/tmp.fp4jjFAn8W + local LAST_ERR=/tmp/tmp.qhLjWzforZ + local exit_status=0 + local timeout=4 ++ seq 0 2 ++ mktemp + for i in $(seq 0 2) + set +e + kubectl get ns + local LAST_ERR=/tmp/tmp.gfgmGTrhPu + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl delete namespace data-at-rest-encryption-4238 --ignore-not-found + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.X0gFu7rUtm + cat /tmp/tmp.qhLjWzforZ + rm /tmp/tmp.X0gFu7rUtm /tmp/tmp.qhLjWzforZ + return 0 error: resource(s) were provided, but no name was specified + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.fp4jjFAn8W + cat /tmp/tmp.gfgmGTrhPu + rm /tmp/tmp.fp4jjFAn8W /tmp/tmp.gfgmGTrhPu + return 0 + kubectl_bin wait --for=delete namespace data-at-rest-encryption-4238 ++ mktemp + local LAST_OUT=/tmp/tmp.FnGGWT7Zmi ++ mktemp + local LAST_ERR=/tmp/tmp.YAWbCChyhF + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl wait --for=delete namespace data-at-rest-encryption-4238 + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.FnGGWT7Zmi + cat /tmp/tmp.YAWbCChyhF + rm /tmp/tmp.FnGGWT7Zmi /tmp/tmp.YAWbCChyhF + return 0 + desc 'create namespace data-at-rest-encryption-4238' + set +o xtrace ----------------------------------------------------------------------------------- create namespace data-at-rest-encryption-4238 ----------------------------------------------------------------------------------- + kubectl_bin create namespace data-at-rest-encryption-4238 ++ mktemp + local LAST_OUT=/tmp/tmp.VT9FQIbFRr ++ mktemp + local LAST_ERR=/tmp/tmp.QdHa7qpW4d + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl create namespace data-at-rest-encryption-4238 + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.VT9FQIbFRr namespace/data-at-rest-encryption-4238 created + cat /tmp/tmp.QdHa7qpW4d + rm /tmp/tmp.VT9FQIbFRr /tmp/tmp.QdHa7qpW4d + return 0 + set_kube_ctx data-at-rest-encryption-4238 + local namespace=data-at-rest-encryption-4238 ++ kubectl_bin config current-context +++ mktemp ++ local LAST_OUT=/tmp/tmp.I2HW1RqvWf +++ mktemp ++ local LAST_ERR=/tmp/tmp.Gs4hBrHQaW ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl config current-context ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.I2HW1RqvWf ++ cat /tmp/tmp.Gs4hBrHQaW ++ rm /tmp/tmp.I2HW1RqvWf /tmp/tmp.Gs4hBrHQaW ++ return 0 + kubectl_bin config set-context gke_cloud-dev-112233_us-central1-a_jen-psmdb-2277-0a1aa642f-2-cluster8 --namespace=data-at-rest-encryption-4238 ++ mktemp + local LAST_OUT=/tmp/tmp.0jC6IVqAys ++ mktemp + local LAST_ERR=/tmp/tmp.TCj7XcmrpM + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl config set-context gke_cloud-dev-112233_us-central1-a_jen-psmdb-2277-0a1aa642f-2-cluster8 --namespace=data-at-rest-encryption-4238 + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.0jC6IVqAys Context "gke_cloud-dev-112233_us-central1-a_jen-psmdb-2277-0a1aa642f-2-cluster8" modified. + cat /tmp/tmp.TCj7XcmrpM + rm /tmp/tmp.0jC6IVqAys /tmp/tmp.TCj7XcmrpM + return 0 + deploy_vault + local name=vault-service + [[ 0 -gt 0 ]] + desc 'install Vault vault-service' + set +o xtrace ----------------------------------------------------------------------------------- install Vault vault-service ----------------------------------------------------------------------------------- + helm uninstall vault-service Error: uninstall: Release not loaded: vault-service: release: not found + : + helm repo remove hashicorp "hashicorp" has been removed from your repositories + helm repo add hashicorp https://helm.releases.hashicorp.com "hashicorp" has been added to your repositories + destroy_vault vault-service + local name=vault-service + local vault_ns ++ helm list --all-namespaces --filter vault-service ++ tail -n1 ++ awk '-F ' '{print $2}' ++ sed s/NAMESPACE// + vault_ns= + desc 'destroy vault' + set +o xtrace ----------------------------------------------------------------------------------- destroy vault ----------------------------------------------------------------------------------- ++ grep vault ++ kubectl api-resources ++ awk '{print $1}' + '[' -n '' ']' ++ kubectl get clusterrolebinding -l app.kubernetes.io/instance=vault-service -o 'jsonpath={range .items[*]}{.metadata.name}{" "}{end}' + timeout 30 kubectl delete clusterrolebinding vault-service-agent-injector-binding vault-service-server-binding clusterrolebinding.rbac.authorization.k8s.io "vault-service-agent-injector-binding" deleted clusterrolebinding.rbac.authorization.k8s.io "vault-service-server-binding" deleted ++ kubectl get clusterrole -l app.kubernetes.io/instance=vault-service -o 'jsonpath={range .items[*]}{.metadata.name}{" "}{end}' + timeout 30 kubectl delete clusterrole vault-service-agent-injector-clusterrole clusterrole.rbac.authorization.k8s.io "vault-service-agent-injector-clusterrole" deleted ++ kubectl get mutatingwebhookconfiguration -l app.kubernetes.io/instance=vault-service -o 'jsonpath={range .items[*]}{.metadata.name}{" "}{end}' + timeout 30 kubectl delete mutatingwebhookconfiguration vault-service-agent-injector-cfg mutatingwebhookconfiguration.admissionregistration.k8s.io "vault-service-agent-injector-cfg" deleted + [[ -n '' ]] + retry 10 60 helm install vault-service hashicorp/vault --disable-openapi-validation --set dataStorage.enabled=false + local max=10 + local delay=60 + shift 2 + local n=1 + helm install vault-service hashicorp/vault --disable-openapi-validation --set dataStorage.enabled=false NAME: vault-service LAST DEPLOYED: Mon Mar 9 09:08:52 2026 NAMESPACE: data-at-rest-encryption-4238 STATUS: deployed REVISION: 1 NOTES: Thank you for installing HashiCorp Vault! Now that you have deployed Vault, you should look over the docs on using Vault with Kubernetes available here: https://developer.hashicorp.com/vault/docs Your release is named vault-service. To learn more about the release, try: $ helm status vault-service $ helm get manifest vault-service + kubectl_bin get pod/vault-service-0 -o 'jsonpath={.status.phase}' + grep Running + sleep 1 + kubectl_bin get pod/vault-service-0 -o 'jsonpath={.status.phase}' + grep Running + sleep 1 + kubectl_bin get pod/vault-service-0 -o 'jsonpath={.status.phase}' + grep Running + sleep 1 + kubectl_bin get pod/vault-service-0 -o 'jsonpath={.status.phase}' + grep Running + sleep 1 + kubectl_bin get pod/vault-service-0 -o 'jsonpath={.status.phase}' + grep Running + sleep 1 + kubectl_bin get pod/vault-service-0 -o 'jsonpath={.status.phase}' + grep Running + sleep 1 + kubectl_bin get pod/vault-service-0 -o 'jsonpath={.status.phase}' + grep Running + sleep 1 + kubectl_bin get pod/vault-service-0 -o 'jsonpath={.status.phase}' + grep Running + sleep 1 + kubectl_bin get pod/vault-service-0 -o 'jsonpath={.status.phase}' + grep Running Running + sleep 5 + kubectl_bin exec pod/vault-service-0 -- vault operator init -key-shares=1 -key-threshold=1 -format=json ++ mktemp + local LAST_OUT=/tmp/tmp.TPeczcAqih ++ mktemp + local LAST_ERR=/tmp/tmp.HxjTI5ODmU + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl exec pod/vault-service-0 -- vault operator init -key-shares=1 -key-threshold=1 -format=json + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.TPeczcAqih + cat /tmp/tmp.HxjTI5ODmU + rm /tmp/tmp.TPeczcAqih /tmp/tmp.HxjTI5ODmU + return 0 + local unsealKey ++ jq -r '.unseal_keys_b64[]' + unsealKey=QWaTAHrGyCcfTjaxupHV/vGsIxonNNAoURUIfHjxwys= + local token ++ jq -r .root_token + token=hvs.HsKOW4UlEB5n8Ivo85DbMi0V + kubectl_bin exec pod/vault-service-0 -- vault operator unseal QWaTAHrGyCcfTjaxupHV/vGsIxonNNAoURUIfHjxwys= ++ mktemp + local LAST_OUT=/tmp/tmp.DevSlk8euT ++ mktemp + local LAST_ERR=/tmp/tmp.IlASqjJW9v + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl exec pod/vault-service-0 -- vault operator unseal QWaTAHrGyCcfTjaxupHV/vGsIxonNNAoURUIfHjxwys= + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.DevSlk8euT Key Value --- ----- Seal Type shamir Initialized true Sealed false Total Shares 1 Threshold 1 Version 1.21.2 Build Date 2026-01-06T08:33:05Z Storage Type file Cluster Name vault-cluster-40008715 Cluster ID 1bc38353-ad4b-93ab-98c7-a71710810888 HA Enabled false + cat /tmp/tmp.IlASqjJW9v + rm /tmp/tmp.DevSlk8euT /tmp/tmp.IlASqjJW9v + return 0 + kubectl_bin exec -it pod/vault-service-0 -- sh ++ mktemp + local LAST_OUT=/tmp/tmp.2hZmm1GNfD ++ mktemp + local LAST_ERR=/tmp/tmp.lpHRVn2IRK + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl exec -it pod/vault-service-0 -- sh + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.2hZmm1GNfD Success! You are now authenticated. The token information displayed below is already stored in the token helper. You do NOT need to run "vault login" again. Future Vault requests will automatically use this token. Key Value --- ----- token hvs.HsKOW4UlEB5n8Ivo85DbMi0V token_accessor eMnOEtVgdSCSuTmrWod7OnXz token_duration ∞ token_renewable false token_policies ["root"] identity_policies [] policies ["root"] Success! Enabled the kv-v2 secrets engine at: secret/ + cat /tmp/tmp.lpHRVn2IRK Unable to use a TTY - input is not a terminal or the right kind of file + rm /tmp/tmp.2hZmm1GNfD /tmp/tmp.lpHRVn2IRK + return 0 + kubectl_bin create secret generic vault-secret --from-literal=token=hvs.HsKOW4UlEB5n8Ivo85DbMi0V ++ mktemp + local LAST_OUT=/tmp/tmp.Ogos57waC8 ++ mktemp + local LAST_ERR=/tmp/tmp.sf7HLQXDvT + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl create secret generic vault-secret --from-literal=token=hvs.HsKOW4UlEB5n8Ivo85DbMi0V + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.Ogos57waC8 secret/vault-secret created + cat /tmp/tmp.sf7HLQXDvT + rm /tmp/tmp.Ogos57waC8 /tmp/tmp.sf7HLQXDvT + return 0 + deploy_minio + local cert_secret= + local service_name=minio-service + desc 'install MinIO: minio-service' + set +o xtrace ----------------------------------------------------------------------------------- install MinIO: minio-service ----------------------------------------------------------------------------------- + helm uninstall minio-service + : + helm repo remove minio "minio" has been removed from your repositories + helm repo add minio https://charts.min.io/ "minio" has been added to your repositories + local endpoint=http://minio-service:9000 + minio_args=('--version' '5.4.0' '--set' 'replicas=1' '--set' 'mode=standalone' '--set' 'resources.requests.memory=256Mi' '--set' 'rootUser=rootuser' '--set' 'rootPassword=rootpass123' '--set' 'users[0].accessKey=some-access-key' '--set' 'users[0].secretKey=some-secret-key' '--set' 'users[0].policy=consoleAdmin' '--set' 'service.type=ClusterIP' '--set' 'configPathmc=/tmp/' '--set' 'securityContext.enabled=false' '--set' 'persistence.size=2G' '--set' 'fullnameOverride=minio-service' '--set' 'serviceAccount.create=true' '--set' 'serviceAccount.name=minio-service-sa') + local minio_args + [[ -n '' ]] + retry 10 60 helm install minio-service --version 5.4.0 --set replicas=1 --set mode=standalone --set resources.requests.memory=256Mi --set rootUser=rootuser --set rootPassword=rootpass123 --set 'users[0].accessKey=some-access-key' --set 'users[0].secretKey=some-secret-key' --set 'users[0].policy=consoleAdmin' --set service.type=ClusterIP --set configPathmc=/tmp/ --set securityContext.enabled=false --set persistence.size=2G --set fullnameOverride=minio-service --set serviceAccount.create=true --set serviceAccount.name=minio-service-sa minio/minio + local max=10 + local delay=60 + shift 2 + local n=1 + helm install minio-service --version 5.4.0 --set replicas=1 --set mode=standalone --set resources.requests.memory=256Mi --set rootUser=rootuser --set rootPassword=rootpass123 --set 'users[0].accessKey=some-access-key' --set 'users[0].secretKey=some-secret-key' --set 'users[0].policy=consoleAdmin' --set service.type=ClusterIP --set configPathmc=/tmp/ --set securityContext.enabled=false --set persistence.size=2G --set fullnameOverride=minio-service --set serviceAccount.create=true --set serviceAccount.name=minio-service-sa minio/minio NAME: minio-service LAST DEPLOYED: Mon Mar 9 09:09:27 2026 NAMESPACE: data-at-rest-encryption-4238 STATUS: deployed REVISION: 1 TEST SUITE: None NOTES: MinIO can be accessed via port 9000 on the following DNS name from within your cluster: minio-service.data-at-rest-encryption-4238.cluster.local To access MinIO from localhost, run the below commands: 1. export POD_NAME=$(kubectl get pods --namespace data-at-rest-encryption-4238 -l "release=minio-service" -o jsonpath="{.items[0].metadata.name}") 2. kubectl port-forward $POD_NAME 9000 --namespace data-at-rest-encryption-4238 Read more about port forwarding here: http://kubernetes.io/docs/user-guide/kubectl/kubectl_port-forward/ You can now access MinIO server on http://localhost:9000. Follow the below steps to connect to MinIO server with mc client: 1. Download the MinIO mc client - https://min.io/docs/minio/linux/reference/minio-mc.html#quickstart 2. export MC_HOST_minio-service-local=http://$(kubectl get secret --namespace data-at-rest-encryption-4238 minio-service -o jsonpath="{.data.rootUser}" | base64 --decode):$(kubectl get secret --namespace data-at-rest-encryption-4238 minio-service -o jsonpath="{.data.rootPassword}" | base64 --decode)@localhost:9000 3. mc ls minio-service-local ++ kubectl_bin get pods --selector=release=minio-service -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.2XLeuts1eb +++ mktemp ++ local LAST_ERR=/tmp/tmp.t2NMWoMXQP ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pods --selector=release=minio-service -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.2XLeuts1eb ++ cat /tmp/tmp.t2NMWoMXQP ++ rm /tmp/tmp.2XLeuts1eb /tmp/tmp.t2NMWoMXQP ++ return 0 + local MINIO_POD=minio-service-6d5f646cdc-gxwpr + wait_pod minio-service-6d5f646cdc-gxwpr + local pod=minio-service-6d5f646cdc-gxwpr + set +o xtrace waiting for pod/minio-service-6d5f646cdc-gxwpr to be ready.OK + '[' -n psmdb-operator ']' + kubectl_bin create svc -n psmdb-operator externalname minio-service --external-name=minio-service.data-at-rest-encryption-4238.svc.cluster.local --tcp=9000 service/minio-service created + create_minio_bucket operator-testing http://minio-service:9000 + local bucket=operator-testing + local endpoint=http://minio-service:9000 + kubectl_bin run -i --rm aws-cli --image=docker.io/perconalab/awscli --restart=Never -- bash -c 'AWS_ACCESS_KEY_ID=some-access-key AWS_SECRET_ACCESS_KEY=some-secret-key AWS_DEFAULT_REGION=us-east-1 /usr/bin/aws --no-verify-ssl --endpoint-url http://minio-service:9000 s3 mb s3://operator-testing' ++ mktemp + local LAST_OUT=/tmp/tmp.nSMQSOj8Ez ++ mktemp + local LAST_ERR=/tmp/tmp.Wmj3G2Y2Q4 + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl run -i --rm aws-cli --image=docker.io/perconalab/awscli --restart=Never -- bash -c 'AWS_ACCESS_KEY_ID=some-access-key AWS_SECRET_ACCESS_KEY=some-secret-key AWS_DEFAULT_REGION=us-east-1 /usr/bin/aws --no-verify-ssl --endpoint-url http://minio-service:9000 s3 mb s3://operator-testing' + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.nSMQSOj8Ez pod "aws-cli" deleted from data-at-rest-encryption-4238 namespace + cat /tmp/tmp.Wmj3G2Y2Q4 All commands and output from this session will be recorded in container logs, including credentials and sensitive information passed through the command prompt. If you don't see a command prompt, try pressing enter. + rm /tmp/tmp.nSMQSOj8Ez /tmp/tmp.Wmj3G2Y2Q4 + return 0 + apply_s3_storage_secrets + desc 'create secrets for cloud storages' + set +o xtrace ----------------------------------------------------------------------------------- create secrets for cloud storages ----------------------------------------------------------------------------------- + '[' -z '' ']' + kubectl_bin apply -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2277/e2e-tests/conf/minio-secret.yml -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2277/e2e-tests/conf/cloud-secret.yml ++ mktemp + local LAST_OUT=/tmp/tmp.ZbDFhyTP21 ++ mktemp + local LAST_ERR=/tmp/tmp.O7SQy2faes + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl apply -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2277/e2e-tests/conf/minio-secret.yml -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2277/e2e-tests/conf/cloud-secret.yml + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.ZbDFhyTP21 secret/minio-secret created secret/aws-s3-secret created secret/gcp-cs-secret created secret/azure-secret created secret/gcp-cs-sa-key-secret created + cat /tmp/tmp.O7SQy2faes + rm /tmp/tmp.ZbDFhyTP21 /tmp/tmp.O7SQy2faes + return 0 + desc 'create secrets and start client' + set +o xtrace ----------------------------------------------------------------------------------- create secrets and start client ----------------------------------------------------------------------------------- + kubectl_bin apply -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2277/e2e-tests/conf/secrets.yml -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2277/e2e-tests/conf/client.yml ++ mktemp + local LAST_OUT=/tmp/tmp.hJUVj8KqWU ++ mktemp + local LAST_ERR=/tmp/tmp.uf8cutK6Sb + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl apply -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2277/e2e-tests/conf/secrets.yml -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2277/e2e-tests/conf/client.yml + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.hJUVj8KqWU secret/some-users created deployment.apps/psmdb-client created + cat /tmp/tmp.uf8cutK6Sb + rm /tmp/tmp.hJUVj8KqWU /tmp/tmp.uf8cutK6Sb + return 0 + cluster=some-name + desc 'create PSMDB cluster some-name' + set +o xtrace ----------------------------------------------------------------------------------- create PSMDB cluster some-name ----------------------------------------------------------------------------------- + apply_cluster /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2277/e2e-tests/data-at-rest-encryption/conf/some-name.yml + '[' -z '' ']' + cat_config /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2277/e2e-tests/data-at-rest-encryption/conf/some-name.yml + kubectl_bin apply -f - + cat /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2277/e2e-tests/data-at-rest-encryption/conf/some-name.yml + yq eval '(.spec | select(.image == null)).image = "docker.io/perconalab/percona-server-mongodb-operator:main-mongod8.0"' + yq eval '(.spec | select(has("pmm"))).pmm.image = "docker.io/percona/pmm-client:2.44.1-1"' + yq eval '(.spec | select(has("backup"))).backup.image = "docker.io/perconalab/percona-server-mongodb-operator:main-backup"' ++ mktemp + yq eval '.spec.upgradeOptions.apply="Never"' + yq eval '(.spec | select(has("initImage"))).initImage = "docker.io/perconalab/percona-server-mongodb-operator:PR-2277-0a1aa642f"' + /usr/sbin/sed -e s/NAME_SPACE/data-at-rest-encryption-4238/g + local LAST_OUT=/tmp/tmp.yeHhs1MrMf ++ mktemp + local LAST_ERR=/tmp/tmp.04b7fMGChl + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.yeHhs1MrMf perconaservermongodb.psmdb.percona.com/some-name created + cat /tmp/tmp.04b7fMGChl + rm /tmp/tmp.yeHhs1MrMf /tmp/tmp.04b7fMGChl + return 0 + desc 'check if all Pods started' + set +o xtrace ----------------------------------------------------------------------------------- check if all Pods started ----------------------------------------------------------------------------------- + wait_for_running some-name-rs0 3 false + local name=some-name-rs0 + let last_pod=2 + local check_cluster_readyness=false + set_debug + [[ 1 == 1 ]] + set -o xtrace + local rs_name=rs0 + local cluster_name=some-name ++ seq 0 2 + for i in $(seq 0 $last_pod) + [[ 0 -eq 2 ]] + wait_pod some-name-rs0-0 + local pod=some-name-rs0-0 + set +o xtrace waiting for pod/some-name-rs0-0 to be ready........................................OK + for i in $(seq 0 $last_pod) + [[ 1 -eq 2 ]] + wait_pod some-name-rs0-1 + local pod=some-name-rs0-1 + set +o xtrace waiting for pod/some-name-rs0-1 to be ready......OK + for i in $(seq 0 $last_pod) + [[ 2 -eq 2 ]] ++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].arbiter.enabled}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.9f1osBS1WZ +++ mktemp ++ local LAST_ERR=/tmp/tmp.EroRPw7nF4 ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].arbiter.enabled}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.9f1osBS1WZ ++ cat /tmp/tmp.EroRPw7nF4 ++ rm /tmp/tmp.9f1osBS1WZ /tmp/tmp.EroRPw7nF4 ++ return 0 + [[ '' == \t\r\u\e ]] + wait_pod some-name-rs0-2 + local pod=some-name-rs0-2 + set +o xtrace waiting for pod/some-name-rs0-2 to be ready....................OK ++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].nonvoting.enabled}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.IBL1ylPZBh +++ mktemp ++ local LAST_ERR=/tmp/tmp.R9iYEmgvTp ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].nonvoting.enabled}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.IBL1ylPZBh ++ cat /tmp/tmp.R9iYEmgvTp ++ rm /tmp/tmp.IBL1ylPZBh /tmp/tmp.R9iYEmgvTp ++ return 0 + [[ '' == \t\r\u\e ]] ++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].hidden.enabled}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.gVuMdfiMeg +++ mktemp ++ local LAST_ERR=/tmp/tmp.2MUit2pImH ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].hidden.enabled}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.gVuMdfiMeg ++ cat /tmp/tmp.2MUit2pImH ++ rm /tmp/tmp.gVuMdfiMeg /tmp/tmp.2MUit2pImH ++ return 0 + [[ '' == \t\r\u\e ]] + sleep 10 + [[ false == \t\r\u\e ]] + wait_for_running some-name-cfg 3 false + local name=some-name-cfg + let last_pod=2 + local check_cluster_readyness=false + set_debug + [[ 1 == 1 ]] + set -o xtrace + local rs_name=cfg + local cluster_name=some-name ++ seq 0 2 + for i in $(seq 0 $last_pod) + [[ 0 -eq 2 ]] + wait_pod some-name-cfg-0 + local pod=some-name-cfg-0 + set +o xtrace waiting for pod/some-name-cfg-0 to be ready.OK + for i in $(seq 0 $last_pod) + [[ 1 -eq 2 ]] + wait_pod some-name-cfg-1 + local pod=some-name-cfg-1 + set +o xtrace waiting for pod/some-name-cfg-1 to be ready.OK + for i in $(seq 0 $last_pod) + [[ 2 -eq 2 ]] ++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].arbiter.enabled}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.XDXCDvub7E +++ mktemp ++ local LAST_ERR=/tmp/tmp.QGVF6nbcjw ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].arbiter.enabled}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.XDXCDvub7E ++ cat /tmp/tmp.QGVF6nbcjw ++ rm /tmp/tmp.XDXCDvub7E /tmp/tmp.QGVF6nbcjw ++ return 0 + [[ '' == \t\r\u\e ]] + wait_pod some-name-cfg-2 + local pod=some-name-cfg-2 + set +o xtrace waiting for pod/some-name-cfg-2 to be ready.OK ++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].nonvoting.enabled}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.bwcEx2qa1F +++ mktemp ++ local LAST_ERR=/tmp/tmp.ia4FNMhIqV ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].nonvoting.enabled}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.bwcEx2qa1F ++ cat /tmp/tmp.ia4FNMhIqV ++ rm /tmp/tmp.bwcEx2qa1F /tmp/tmp.ia4FNMhIqV ++ return 0 + [[ '' == \t\r\u\e ]] ++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].hidden.enabled}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.ZkR2oe6Yi7 +++ mktemp ++ local LAST_ERR=/tmp/tmp.HcCe0OY6ed ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].hidden.enabled}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.ZkR2oe6Yi7 ++ cat /tmp/tmp.HcCe0OY6ed ++ rm /tmp/tmp.ZkR2oe6Yi7 /tmp/tmp.HcCe0OY6ed ++ return 0 + [[ '' == \t\r\u\e ]] + sleep 10 + [[ false == \t\r\u\e ]] + wait_for_running some-name-mongos 3 + local name=some-name-mongos + let last_pod=2 + local check_cluster_readyness=true + set_debug + [[ 1 == 1 ]] + set -o xtrace + local rs_name=mongos + local cluster_name=some-name ++ seq 0 2 + for i in $(seq 0 $last_pod) + [[ 0 -eq 2 ]] + wait_pod some-name-mongos-0 + local pod=some-name-mongos-0 + set +o xtrace waiting for pod/some-name-mongos-0 to be ready.OK + for i in $(seq 0 $last_pod) + [[ 1 -eq 2 ]] + wait_pod some-name-mongos-1 + local pod=some-name-mongos-1 + set +o xtrace waiting for pod/some-name-mongos-1 to be ready.OK + for i in $(seq 0 $last_pod) + [[ 2 -eq 2 ]] ++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="mongos")].arbiter.enabled}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.X9hlpuPeZT +++ mktemp ++ local LAST_ERR=/tmp/tmp.Z648OTPYe3 ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="mongos")].arbiter.enabled}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.X9hlpuPeZT ++ cat /tmp/tmp.Z648OTPYe3 ++ rm /tmp/tmp.X9hlpuPeZT /tmp/tmp.Z648OTPYe3 ++ return 0 + [[ '' == \t\r\u\e ]] + wait_pod some-name-mongos-2 + local pod=some-name-mongos-2 + set +o xtrace waiting for pod/some-name-mongos-2 to be ready.OK ++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="mongos")].nonvoting.enabled}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.txysrnqwH0 +++ mktemp ++ local LAST_ERR=/tmp/tmp.wQj7hGV9wY ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="mongos")].nonvoting.enabled}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.txysrnqwH0 ++ cat /tmp/tmp.wQj7hGV9wY ++ rm /tmp/tmp.txysrnqwH0 /tmp/tmp.wQj7hGV9wY ++ return 0 + [[ '' == \t\r\u\e ]] ++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="mongos")].hidden.enabled}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.jsgtH35EUN +++ mktemp ++ local LAST_ERR=/tmp/tmp.PPu9CSbUBg ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="mongos")].hidden.enabled}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.jsgtH35EUN ++ cat /tmp/tmp.PPu9CSbUBg ++ rm /tmp/tmp.jsgtH35EUN /tmp/tmp.PPu9CSbUBg ++ return 0 + [[ '' == \t\r\u\e ]] + sleep 10 + [[ true == \t\r\u\e ]] + set +x Waiting for cluster readyness + desc 'check if service and statefulset created with expected config' + set +o xtrace ----------------------------------------------------------------------------------- check if service and statefulset created with expected config ----------------------------------------------------------------------------------- + compare_kubectl statefulset/some-name-rs0 + local resource=statefulset/some-name-rs0 + local postfix= + local skip_generation_check= + local expected_result=/mnt/jenkins/workspace/cloud-psmdb-operator_PR-2277/e2e-tests/data-at-rest-encryption/compare/statefulset_some-name-rs0.yml + local new_result=/tmp/tmp.OiUpSmpLTj/statefulset_some-name-rs0.yml + '[' -n '' -a -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2277/e2e-tests/data-at-rest-encryption/compare/statefulset_some-name-rs0-oc.yml ']' + kubectl_bin get -o yaml statefulset/some-name-rs0 + yq eval ' del(.metadata.ownerReferences[].apiVersion) | del(.metadata.managedFields) | ++ mktemp del(.. | select(has("creationTimestamp")).creationTimestamp) | del(.. | select(has("namespace")).namespace) | del(.. | select(has("uid")).uid) | del(.metadata.resourceVersion) | del(.spec.template.spec.containers[].env[] | select(.name == "NAMESPACE")) | del(.metadata.selfLink) | del(.metadata.annotations."cloud.google.com/neg") | del(.metadata.annotations."kubectl.kubernetes.io/last-applied-configuration") | del(.. | select(has("image")).image) | del(.. | select(has("clusterIP")).clusterIP) | del(.. | select(has("clusterIPs")).clusterIPs) | del(.. | select(has("dataSource")).dataSource) | del(.. | select(has("procMount")).procMount) | del(.. | select(has("storageClassName")).storageClassName) | del(.. | select(has("finalizers")).finalizers) | del(.. | select(has("kubernetes.io/pvc-protection"))."kubernetes.io/pvc-protection") | del(.. | select(has("volumeName")).volumeName) | del(.. | select(has("volume.beta.kubernetes.io/storage-provisioner"))."volume.beta.kubernetes.io/storage-provisioner") | del(.. | select(has("volume.kubernetes.io/storage-provisioner"))."volume.kubernetes.io/storage-provisioner") | del(.spec.volumeMode) | del(.. | select(has("volume.kubernetes.io/selected-node"))."volume.kubernetes.io/selected-node") | del(.. | select(has("percona.com/last-config-hash"))."percona.com/last-config-hash") | del(.. | select(has("percona.com/configuration-hash"))."percona.com/configuration-hash") | del(.. | select(has("percona.com/ssl-hash"))."percona.com/ssl-hash") | del(.. | select(has("percona.com/ssl-internal-hash"))."percona.com/ssl-internal-hash") | del(.spec.volumeClaimTemplates[].spec.volumeMode | select(. == "Filesystem")) | del(.. | select(has("healthCheckNodePort")).healthCheckNodePort) | del(.. | select(has("nodePort")).nodePort) | del(.status) | (.. | select(tag == "!!str")) |= sub("data-at-rest-encryption-4238", "NAME_SPACE") | del(.spec.volumeClaimTemplates[].apiVersion) | del(.spec.volumeClaimTemplates[].kind) | del(.spec.ipFamilies) | del(.spec.ipFamilyPolicy) | (.. | select(. == "extensions/v1beta1")) = "apps/v1" | (.. | select(. == "batch/v1beta1")) = "batch/v1" ' - + local LAST_OUT=/tmp/tmp.BhdRGqbltD ++ mktemp + local LAST_ERR=/tmp/tmp.2UIbA5x9pE + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl get -o yaml statefulset/some-name-rs0 + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.BhdRGqbltD + cat /tmp/tmp.2UIbA5x9pE + rm /tmp/tmp.BhdRGqbltD /tmp/tmp.2UIbA5x9pE + return 0 + yq -i eval 'del(.spec.persistentVolumeClaimRetentionPolicy)' /tmp/tmp.OiUpSmpLTj/statefulset_some-name-rs0.yml + version_gt 1.22 ++ echo '1.32 >= 1.22' ++ bc -l + '[' 1 -eq 1 ']' + return 0 + yq -i eval 'del(.spec.internalTrafficPolicy)' /tmp/tmp.OiUpSmpLTj/statefulset_some-name-rs0.yml + yq -i eval 'del(.spec.allocateLoadBalancerNodePorts)' /tmp/tmp.OiUpSmpLTj/statefulset_some-name-rs0.yml + [[ /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2277/e2e-tests/data-at-rest-encryption/compare/statefulset_some-name-rs0.yml == */cronjob* ]] + '[' -n '' ']' + [[ 0 -eq 0 ]] + diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2277/e2e-tests/data-at-rest-encryption/compare/statefulset_some-name-rs0.yml /tmp/tmp.OiUpSmpLTj/statefulset_some-name-rs0.yml + log 'compare_kubectl: statefulset/some-name-rs0 OK' + set +o xtrace [2026-03-09T09:13:48+0000] compare_kubectl: statefulset/some-name-rs0 OK + compare_kubectl statefulset/some-name-cfg + local resource=statefulset/some-name-cfg + local postfix= + local skip_generation_check= + local expected_result=/mnt/jenkins/workspace/cloud-psmdb-operator_PR-2277/e2e-tests/data-at-rest-encryption/compare/statefulset_some-name-cfg.yml + local new_result=/tmp/tmp.OiUpSmpLTj/statefulset_some-name-cfg.yml + '[' -n '' -a -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2277/e2e-tests/data-at-rest-encryption/compare/statefulset_some-name-cfg-oc.yml ']' + kubectl_bin get -o yaml statefulset/some-name-cfg + yq eval ' del(.metadata.ownerReferences[].apiVersion) | del(.metadata.managedFields) | del(.. | select(has("creationTimestamp")).creationTimestamp) | del(.. | select(has("namespace")).namespace) | del(.. | select(has("uid")).uid) | del(.metadata.resourceVersion) | del(.spec.template.spec.containers[].env[] | select(.name == "NAMESPACE")) | del(.metadata.selfLink) | del(.metadata.annotations."cloud.google.com/neg") | del(.metadata.annotations."kubectl.kubernetes.io/last-applied-configuration") | del(.. | select(has("image")).image) | del(.. | select(has("clusterIP")).clusterIP) | del(.. | select(has("clusterIPs")).clusterIPs) | del(.. | select(has("dataSource")).dataSource) | del(.. | select(has("procMount")).procMount) | del(.. | select(has("storageClassName")).storageClassName) | del(.. | select(has("finalizers")).finalizers) | del(.. | select(has("kubernetes.io/pvc-protection"))."kubernetes.io/pvc-protection") | del(.. | select(has("volumeName")).volumeName) | del(.. | select(has("volume.beta.kubernetes.io/storage-provisioner"))."volume.beta.kubernetes.io/storage-provisioner") | del(.. | select(has("volume.kubernetes.io/storage-provisioner"))."volume.kubernetes.io/storage-provisioner") | del(.spec.volumeMode) | del(.. | select(has("volume.kubernetes.io/selected-node"))."volume.kubernetes.io/selected-node") | del(.. | select(has("percona.com/last-config-hash"))."percona.com/last-config-hash") | del(.. | select(has("percona.com/configuration-hash"))."percona.com/configuration-hash") | del(.. | select(has("percona.com/ssl-hash"))."percona.com/ssl-hash") | del(.. | select(has("percona.com/ssl-internal-hash"))."percona.com/ssl-internal-hash") | del(.spec.volumeClaimTemplates[].spec.volumeMode | select(. == "Filesystem")) | del(.. | select(has("healthCheckNodePort")).healthCheckNodePort) | del(.. | select(has("nodePort")).nodePort) | del(.status) | (.. | select(tag == "!!str")) |= sub("data-at-rest-encryption-4238", "NAME_SPACE") | del(.spec.volumeClaimTemplates[].apiVersion) | del(.spec.volumeClaimTemplates[].kind) | del(.spec.ipFamilies) | del(.spec.ipFamilyPolicy) | (.. | select(. == "extensions/v1beta1")) = "apps/v1" | (.. | select(. == "batch/v1beta1")) = "batch/v1" ' - ++ mktemp + local LAST_OUT=/tmp/tmp.yskLlXoYdc ++ mktemp + local LAST_ERR=/tmp/tmp.7j1TkV5h8w + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl get -o yaml statefulset/some-name-cfg + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.yskLlXoYdc + cat /tmp/tmp.7j1TkV5h8w + rm /tmp/tmp.yskLlXoYdc /tmp/tmp.7j1TkV5h8w + return 0 + yq -i eval 'del(.spec.persistentVolumeClaimRetentionPolicy)' /tmp/tmp.OiUpSmpLTj/statefulset_some-name-cfg.yml + version_gt 1.22 ++ echo '1.32 >= 1.22' ++ bc -l + '[' 1 -eq 1 ']' + return 0 + yq -i eval 'del(.spec.internalTrafficPolicy)' /tmp/tmp.OiUpSmpLTj/statefulset_some-name-cfg.yml + yq -i eval 'del(.spec.allocateLoadBalancerNodePorts)' /tmp/tmp.OiUpSmpLTj/statefulset_some-name-cfg.yml + [[ /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2277/e2e-tests/data-at-rest-encryption/compare/statefulset_some-name-cfg.yml == */cronjob* ]] + '[' -n '' ']' + [[ 0 -eq 0 ]] + diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2277/e2e-tests/data-at-rest-encryption/compare/statefulset_some-name-cfg.yml /tmp/tmp.OiUpSmpLTj/statefulset_some-name-cfg.yml + log 'compare_kubectl: statefulset/some-name-cfg OK' + set +o xtrace [2026-03-09T09:13:49+0000] compare_kubectl: statefulset/some-name-cfg OK + desc 'create user' + set +o xtrace ----------------------------------------------------------------------------------- create user ----------------------------------------------------------------------------------- + run_mongos 'db.createUser({user:"myApp",pwd:"myPass",roles:[{db:"myApp",role:"readWrite"}]})' userAdmin:userAdmin123456@some-name-mongos.data-at-rest-encryption-4238 + local 'command=db.createUser({user:"myApp",pwd:"myPass",roles:[{db:"myApp",role:"readWrite"}]})' + local uri=userAdmin:userAdmin123456@some-name-mongos.data-at-rest-encryption-4238 + local driver=mongodb + local suffix=.svc.cluster.local + local mongo_flag= + local port=27017 + local mongo_bin=mongo ++ echo .svc.cluster.local ++ awk -F: '{print $2}' + suffix_port= + [[ -z '' ]] + suffix=.svc.cluster.local:27017 ++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.6JZiSd2h45 +++ mktemp ++ local LAST_ERR=/tmp/tmp.1zg1MtZyMd ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.6JZiSd2h45 ++ cat /tmp/tmp.1zg1MtZyMd ++ rm /tmp/tmp.6JZiSd2h45 /tmp/tmp.1zg1MtZyMd ++ return 0 + local client_container=psmdb-client-bb8b97679-d9fpq + kubectl_bin exec psmdb-client-bb8b97679-d9fpq -- bash -c 'printf '\''db.createUser({user:"myApp",pwd:"myPass",roles:[{db:"myApp",role:"readWrite"}]})\n'\'' | mongo mongodb://userAdmin:userAdmin123456@some-name-mongos.data-at-rest-encryption-4238.svc.cluster.local:27017/admin ' ++ mktemp + local LAST_OUT=/tmp/tmp.9L9C2ssYnM ++ mktemp + local LAST_ERR=/tmp/tmp.etcqqzqQNi + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl exec psmdb-client-bb8b97679-d9fpq -- bash -c 'printf '\''db.createUser({user:"myApp",pwd:"myPass",roles:[{db:"myApp",role:"readWrite"}]})\n'\'' | mongo mongodb://userAdmin:userAdmin123456@some-name-mongos.data-at-rest-encryption-4238.svc.cluster.local:27017/admin ' + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.9L9C2ssYnM Percona Server for MongoDB shell version v4.4.29-28 connecting to: mongodb://some-name-mongos.data-at-rest-encryption-4238.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb Implicit session: session { "id" : UUID("6e0c3bd5-c725-4b78-9c56-9f579a7d1acb") } Percona Server for MongoDB server version: v8.0.19-7 WARNING: shell and server versions do not match Successfully added user: { "user" : "myApp", "roles" : [ { "db" : "myApp", "role" : "readWrite" } ] } bye + cat /tmp/tmp.etcqqzqQNi + rm /tmp/tmp.9L9C2ssYnM /tmp/tmp.etcqqzqQNi + return 0 + sleep 2 + desc 'write data, read it' + set +o xtrace ----------------------------------------------------------------------------------- write data, read it ----------------------------------------------------------------------------------- + run_mongos 'use myApp\n db.test.insert({ x: 100500 })' myApp:myPass@some-name-mongos.data-at-rest-encryption-4238 + local 'command=use myApp\n db.test.insert({ x: 100500 })' + local uri=myApp:myPass@some-name-mongos.data-at-rest-encryption-4238 + local driver=mongodb + local suffix=.svc.cluster.local + local mongo_flag= + local port=27017 + local mongo_bin=mongo ++ echo .svc.cluster.local ++ awk -F: '{print $2}' + suffix_port= + [[ -z '' ]] + suffix=.svc.cluster.local:27017 ++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.z6ynaTw7ih +++ mktemp ++ local LAST_ERR=/tmp/tmp.zP7JCdIHDF ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.z6ynaTw7ih ++ cat /tmp/tmp.zP7JCdIHDF ++ rm /tmp/tmp.z6ynaTw7ih /tmp/tmp.zP7JCdIHDF ++ return 0 + local client_container=psmdb-client-bb8b97679-d9fpq + kubectl_bin exec psmdb-client-bb8b97679-d9fpq -- bash -c 'printf '\''use myApp\n db.test.insert({ x: 100500 })\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.data-at-rest-encryption-4238.svc.cluster.local:27017/admin ' ++ mktemp + local LAST_OUT=/tmp/tmp.ff3lSGa6jn ++ mktemp + local LAST_ERR=/tmp/tmp.gi6Q6VdXwf + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl exec psmdb-client-bb8b97679-d9fpq -- bash -c 'printf '\''use myApp\n db.test.insert({ x: 100500 })\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.data-at-rest-encryption-4238.svc.cluster.local:27017/admin ' + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.ff3lSGa6jn Percona Server for MongoDB shell version v4.4.29-28 connecting to: mongodb://some-name-mongos.data-at-rest-encryption-4238.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb Implicit session: session { "id" : UUID("919a35c8-266b-4500-8ee2-d19caf7c9498") } Percona Server for MongoDB server version: v8.0.19-7 WARNING: shell and server versions do not match switched to db myApp WriteResult({ "nInserted" : 1 }) bye + cat /tmp/tmp.gi6Q6VdXwf + rm /tmp/tmp.ff3lSGa6jn /tmp/tmp.gi6Q6VdXwf + return 0 + compare_mongos_cmd find myApp:myPass@some-name-mongos.data-at-rest-encryption-4238 + local command=find + local uri=myApp:myPass@some-name-mongos.data-at-rest-encryption-4238 + local postfix= + local suffix= + local database=myApp + local collection=test + local port=27017 + local tls=false + [[ false == \t\r\u\e ]] + mongos_command=run_mongos + log 'running db.test.command() in myApp' + set +o xtrace [2026-03-09T09:13:56+0000] running db.test.command() in myApp + run_mongos 'use myApp\n db.test.find()' myApp:myPass@some-name-mongos.data-at-rest-encryption-4238 mongodb '' '' 27017 + /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/' + grep -E -v 'I NETWORK|W NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|Started a new thread for the timer service' + local 'command=use myApp\n db.test.find()' + local uri=myApp:myPass@some-name-mongos.data-at-rest-encryption-4238 + local driver=mongodb + local suffix=.svc.cluster.local + local mongo_flag= + local port=27017 + local mongo_bin=mongo ++ echo .svc.cluster.local ++ awk -F: '{print $2}' + suffix_port= + [[ -z '' ]] + suffix=.svc.cluster.local:27017 ++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.yJWVkUjvTf +++ mktemp ++ local LAST_ERR=/tmp/tmp.PGOKD30mBS ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.yJWVkUjvTf ++ cat /tmp/tmp.PGOKD30mBS ++ rm /tmp/tmp.yJWVkUjvTf /tmp/tmp.PGOKD30mBS ++ return 0 + local client_container=psmdb-client-bb8b97679-d9fpq + kubectl_bin exec psmdb-client-bb8b97679-d9fpq -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.data-at-rest-encryption-4238.svc.cluster.local:27017/admin ' ++ mktemp + local LAST_OUT=/tmp/tmp.vzKW1jOaNr ++ mktemp + local LAST_ERR=/tmp/tmp.KoRvj2C7VX + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl exec psmdb-client-bb8b97679-d9fpq -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.data-at-rest-encryption-4238.svc.cluster.local:27017/admin ' + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.vzKW1jOaNr + cat /tmp/tmp.KoRvj2C7VX + rm /tmp/tmp.vzKW1jOaNr /tmp/tmp.KoRvj2C7VX + return 0 + [[ 0 -eq 0 ]] + diff /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2277/e2e-tests/data-at-rest-encryption/compare/find.json /tmp/tmp.OiUpSmpLTj/find + desc 'run backups' + set +o xtrace ----------------------------------------------------------------------------------- run backups ----------------------------------------------------------------------------------- + backup_name_minio=backup-minio + backup_name_gcp=backup-gcp-cs + desc 'run minio backup backup-minio' + set +o xtrace ----------------------------------------------------------------------------------- run minio backup backup-minio ----------------------------------------------------------------------------------- + run_backup minio + local storage=minio + local backup_name=backup-minio + local type=logical + log 'running backup backup-minio' + set +o xtrace [2026-03-09T09:13:58+0000] running backup backup-minio + yq eval '.metadata.name = "backup-minio" | .spec.storageName = "minio" | .spec.type = "logical"' /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2277/e2e-tests/data-at-rest-encryption/conf/backup-minio.yml + kubectl_bin apply -f - ++ mktemp + local LAST_OUT=/tmp/tmp.O9W942yS4v ++ mktemp + local LAST_ERR=/tmp/tmp.gHZgTCcUNS + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.O9W942yS4v perconaservermongodbbackup.psmdb.percona.com/backup-minio created + cat /tmp/tmp.gHZgTCcUNS + rm /tmp/tmp.O9W942yS4v /tmp/tmp.gHZgTCcUNS + return 0 + wait_backup backup-minio + local backup_name=backup-minio + local target_state=ready + set +o xtrace waiting for backup-minio to reach ready state...............OK + '[' -z '' ']' + desc 'run minio backup backup-gcp-cs' + set +o xtrace ----------------------------------------------------------------------------------- run minio backup backup-gcp-cs ----------------------------------------------------------------------------------- + run_backup gcp-cs + local storage=gcp-cs + local backup_name=backup-gcp-cs + local type=logical + log 'running backup backup-gcp-cs' + set +o xtrace [2026-03-09T09:14:28+0000] running backup backup-gcp-cs + yq eval '.metadata.name = "backup-gcp-cs" | .spec.storageName = "gcp-cs" | .spec.type = "logical"' /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2277/e2e-tests/data-at-rest-encryption/conf/backup-gcp-cs.yml + kubectl_bin apply -f - ++ mktemp + local LAST_OUT=/tmp/tmp.kuOUHiW7SH ++ mktemp + local LAST_ERR=/tmp/tmp.EXAXZJAORg + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.kuOUHiW7SH perconaservermongodbbackup.psmdb.percona.com/backup-gcp-cs created + cat /tmp/tmp.EXAXZJAORg + rm /tmp/tmp.kuOUHiW7SH /tmp/tmp.EXAXZJAORg + return 0 + wait_backup backup-gcp-cs + local backup_name=backup-gcp-cs + local target_state=ready + set +o xtrace waiting for backup-gcp-cs to reach ready state...............................................................................................................................................................................................................................................................sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler /go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:479 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem /go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:438 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1 /go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:313 2026-03-09T09:19:36.397Z DEBUG Replset status is not healthy {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"data-at-rest-encryption-4238"}, "namespace": "data-at-rest-encryption-4238", "name": "some-name", "reconcileID": "0be34405-c5cb-4744-8fdf-9928cec24bdc", "replset": "rs0", "status": "error"} 2026-03-09T09:19:36.452Z ERROR Reconciler error {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"data-at-rest-encryption-4238"}, "namespace": "data-at-rest-encryption-4238", "name": "some-name", "reconcileID": "0be34405-c5cb-4744-8fdf-9928cec24bdc", "error": "reconcile statefulsets: failed to get mongos connection: ping mongo: server selection error: server selection timeout, current topology: { Type: Unknown, Servers: [{ Addr: some-name-mongos.data-at-rest-encryption-4238.svc.cluster.local:27017, Type: Unknown, Last error: dial tcp 34.118.237.26:27017: connect: connection refused }, ] }", "errorVerbose": "failed to get mongos connection: ping mongo: server selection error: server selection timeout, current topology: { Type: Unknown, Servers: [{ Addr: some-name-mongos.data-at-rest-encryption-4238.svc.cluster.local:27017, Type: Unknown, Last error: dial tcp 34.118.237.26:27017: connect: connection refused }, ] }\nreconcile statefulsets\ngithub.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodb.(*ReconcilePerconaServerMongoDB).Reconcile\n\t/go/src/github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodb/psmdb_controller.go:450\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:222\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:479\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:438\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:313\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1693"} sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler /go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:495 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem /go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:438 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1 /go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:313 2026-03-09T09:19:53.298Z ERROR FULL CLUSTER CRASH {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"data-at-rest-encryption-4238"}, "namespace": "data-at-rest-encryption-4238", "name": "some-name", "reconcileID": "f220d578-3bb9-4e71-a603-48f02192992e", "error": "ping mongo: server selection error: server selection timeout, current topology: { Type: ReplicaSetNoPrimary, Servers: [{ Addr: some-name-cfg-0.some-name-cfg.data-at-rest-encryption-4238.svc.cluster.local:27017, Type: Unknown, Last error: dial tcp: lookup some-name-cfg-0.some-name-cfg.data-at-rest-encryption-4238.svc.cluster.local on 34.118.224.10:53: no such host }, { Addr: some-name-cfg-1.some-name-cfg.data-at-rest-encryption-4238.svc.cluster.local:27017, Type: RSSecondary, Tag sets: zone=us-central1-a,podName=some-name-cfg-1,nodeName=gke-jen-psmdb-2277-0a1aa-default-pool-256e0b3f-8kt9,serviceName=some-name,region=us-central1, Average RTT: 2416839 }, { Addr: some-name-cfg-2.some-name-cfg.data-at-rest-encryption-4238.svc.cluster.local:27017, Type: Unknown, Last error: dial tcp: lookup some-name-cfg-2.some-name-cfg.data-at-rest-encryption-4238.svc.cluster.local on 34.118.224.10:53: no such host }, ] }", "errorVerbose": "server selection error: server selection timeout, current topology: { Type: ReplicaSetNoPrimary, Servers: [{ Addr: some-name-cfg-0.some-name-cfg.data-at-rest-encryption-4238.svc.cluster.local:27017, Type: Unknown, Last error: dial tcp: lookup some-name-cfg-0.some-name-cfg.data-at-rest-encryption-4238.svc.cluster.local on 34.118.224.10:53: no such host }, { Addr: some-name-cfg-1.some-name-cfg.data-at-rest-encryption-4238.svc.cluster.local:27017, Type: RSSecondary, Tag sets: zone=us-central1-a,podName=some-name-cfg-1,nodeName=gke-jen-psmdb-2277-0a1aa-default-pool-256e0b3f-8kt9,serviceName=some-name,region=us-central1, Average RTT: 2416839 }, { Addr: some-name-cfg-2.some-name-cfg.data-at-rest-encryption-4238.svc.cluster.local:27017, Type: Unknown, Last error: dial tcp: lookup some-name-cfg-2.some-name-cfg.data-at-rest-encryption-4238.svc.cluster.local on 34.118.224.10:53: no such host }, ] }\nping mongo\ngithub.com/percona/percona-server-mongodb-operator/pkg/psmdb/mongo.Dial\n\t/go/src/github.com/percona/percona-server-mongodb-operator/pkg/psmdb/mongo/mongo.go:126\ngithub.com/percona/percona-server-mongodb-operator/pkg/psmdb.MongoClient\n\t/go/src/github.com/percona/percona-server-mongodb-operator/pkg/psmdb/client.go:62\ngithub.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodb.(*mongoClientProvider).Mongo\n\t/go/src/github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodb/connections.go:38\ngithub.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodb.(*ReconcilePerconaServerMongoDB).mongoClientWithRole\n\t/go/src/github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodb/connections.go:60\ngithub.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodb.(*ReconcilePerconaServerMongoDB).reconcileCluster\n\t/go/src/github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodb/mgo.go:94\ngithub.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodb.(*ReconcilePerconaServerMongoDB).reconcileReplsets\n\t/go/src/github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodb/psmdb_controller.go:590\ngithub.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodb.(*ReconcilePerconaServerMongoDB).Reconcile\n\t/go/src/github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodb/psmdb_controller.go:448\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:222\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:479\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:438\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:313\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1693"} github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodb.(*ReconcilePerconaServerMongoDB).reconcileCluster /go/src/github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodb/mgo.go:101 github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodb.(*ReconcilePerconaServerMongoDB).reconcileReplsets /go/src/github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodb/psmdb_controller.go:590 github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodb.(*ReconcilePerconaServerMongoDB).Reconcile /go/src/github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodb/psmdb_controller.go:448 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile /go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:222 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler /go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:479 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem /go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:438 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1 /go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:313 2026-03-09T09:19:53.298Z INFO handleReplicaSetNoPrimary Connecting to pod {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"data-at-rest-encryption-4238"}, "namespace": "data-at-rest-encryption-4238", "name": "some-name", "reconcileID": "f220d578-3bb9-4e71-a603-48f02192992e", "pod": "some-name-cfg-1", "user": "clusterAdmin"} 2026-03-09T09:19:53.387Z DEBUG Running replSetReconfig config {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"data-at-rest-encryption-4238"}, "namespace": "data-at-rest-encryption-4238", "name": "some-name", "reconcileID": "f220d578-3bb9-4e71-a603-48f02192992e", "cfg": {"_id":"cfg","version":662749,"members":[{"_id":0,"host":"some-name-cfg-0.some-name-cfg.data-at-rest-encryption-4238.svc.cluster.local:27017","arbiterOnly":false,"buildIndexes":true,"hidden":false,"priority":2,"tags":{"nodeName":"gke-jen-psmdb-2277-0a1aa-default-pool-256e0b3f-6n8n","podName":"some-name-cfg-0","region":"us-central1","serviceName":"some-name","zone":"us-central1-a"},"secondaryDelaySecs":0,"votes":1},{"_id":1,"host":"some-name-cfg-1.some-name-cfg.data-at-rest-encryption-4238.svc.cluster.local:27017","arbiterOnly":false,"buildIndexes":true,"hidden":false,"priority":2,"tags":{"nodeName":"gke-jen-psmdb-2277-0a1aa-default-pool-256e0b3f-8kt9","podName":"some-name-cfg-1","region":"us-central1","serviceName":"some-name","zone":"us-central1-a"},"secondaryDelaySecs":0,"votes":1},{"_id":2,"host":"some-name-cfg-2.some-name-cfg.data-at-rest-encryption-4238.svc.cluster.local:27017","arbiterOnly":false,"buildIndexes":true,"hidden":false,"priority":2,"tags":{"nodeName":"gke-jen-psmdb-2277-0a1aa-default-pool-256e0b3f-6n8n","podName":"some-name-cfg-2","region":"us-central1","serviceName":"some-name","zone":"us-central1-a"},"secondaryDelaySecs":0,"votes":1}],"configsvr":true,"protocolVersion":1,"settings":{"chainingAllowed":true,"heartbeatIntervalMillis":2000,"heartbeatTimeoutSecs":10,"electionTimeoutMillis":10000,"catchUpTimeoutMillis":-1,"getLastErrorDefaults":{"w":1,"wtimeout":0},"replicaSetId":"69ae8ec416a2c419847896a0"},"writeConcernMajorityJournalDefault":true}} 2026-03-09T09:19:53.392Z DEBUG Replset status is not healthy {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"data-at-rest-encryption-4238"}, "namespace": "data-at-rest-encryption-4238", "name": "some-name", "reconcileID": "f220d578-3bb9-4e71-a603-48f02192992e", "replset": "cfg", "status": "error"} 2026-03-09T09:20:03.846Z ERROR failed to reconcile cluster {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"data-at-rest-encryption-4238"}, "namespace": "data-at-rest-encryption-4238", "name": "some-name", "reconcileID": "f220d578-3bb9-4e71-a603-48f02192992e", "replset": "rs0", "error": "failed to get mongos connection: ping mongo: server selection error: server selection timeout, current topology: { Type: Unknown, Servers: [{ Addr: some-name-mongos.data-at-rest-encryption-4238.svc.cluster.local:27017, Type: Unknown, Last error: dial tcp 34.118.237.26:27017: connect: connection refused }, ] }", "errorVerbose": "server selection error: server selection timeout, current topology: { Type: Unknown, Servers: [{ Addr: some-name-mongos.data-at-rest-encryption-4238.svc.cluster.local:27017, Type: Unknown, Last error: dial tcp 34.118.237.26:27017: connect: connection refused }, ] }\nping mongo\ngithub.com/percona/percona-server-mongodb-operator/pkg/psmdb/mongo.Dial\n\t/go/src/github.com/percona/percona-server-mongodb-operator/pkg/psmdb/mongo/mongo.go:126\ngithub.com/percona/percona-server-mongodb-operator/pkg/psmdb.MongosClient\n\t/go/src/github.com/percona/percona-server-mongodb-operator/pkg/psmdb/client.go:85\ngithub.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodb.(*mongoClientProvider).Mongos\n\t/go/src/github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodb/connections.go:47\ngithub.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodb.(*ReconcilePerconaServerMongoDB).mongosClientWithRole\n\t/go/src/github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodb/connections.go:64\ngithub.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodb.(*ReconcilePerconaServerMongoDB).reconcileCluster\n\t/go/src/github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodb/mgo.go:196\ngithub.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodb.(*ReconcilePerconaServerMongoDB).reconcileReplsets\n\t/go/src/github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodb/psmdb_controller.go:590\ngithub.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodb.(*ReconcilePerconaServerMongoDB).Reconcile\n\t/go/src/github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodb/psmdb_controller.go:448\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:222\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:479\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:438\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:313\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1693\nfailed to get mongos connection\ngithub.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodb.(*ReconcilePerconaServerMongoDB).reconcileCluster\n\t/go/src/github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodb/mgo.go:198\ngithub.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodb.(*ReconcilePerconaServerMongoDB).reconcileReplsets\n\t/go/src/github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodb/psmdb_controller.go:590\ngithub.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodb.(*ReconcilePerconaServerMongoDB).Reconcile\n\t/go/src/github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodb/psmdb_controller.go:448\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:222\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:479\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:438\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:313\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1693"} github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodb.(*ReconcilePerconaServerMongoDB).reconcileReplsets /go/src/github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodb/psmdb_controller.go:592 github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodb.(*ReconcilePerconaServerMongoDB).Reconcile /go/src/github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodb/psmdb_controller.go:448 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile /go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:222 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler /go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:479 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem /go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:438 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1 /go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:313 2026-03-09T09:20:03.846Z DEBUG Replset status is not healthy {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"data-at-rest-encryption-4238"}, "namespace": "data-at-rest-encryption-4238", "name": "some-name", "reconcileID": "f220d578-3bb9-4e71-a603-48f02192992e", "replset": "rs0", "status": "error"} 2026-03-09T09:20:03.938Z ERROR Reconciler error {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"data-at-rest-encryption-4238"}, "namespace": "data-at-rest-encryption-4238", "name": "some-name", "reconcileID": "f220d578-3bb9-4e71-a603-48f02192992e", "error": "reconcile statefulsets: failed to get mongos connection: ping mongo: server selection error: server selection timeout, current topology: { Type: Unknown, Servers: [{ Addr: some-name-mongos.data-at-rest-encryption-4238.svc.cluster.local:27017, Type: Unknown, Last error: dial tcp 34.118.237.26:27017: connect: connection refused }, ] }", "errorVerbose": "failed to get mongos connection: ping mongo: server selection error: server selection timeout, current topology: { Type: Unknown, Servers: [{ Addr: some-name-mongos.data-at-rest-encryption-4238.svc.cluster.local:27017, Type: Unknown, Last error: dial tcp 34.118.237.26:27017: connect: connection refused }, ] }\nreconcile statefulsets\ngithub.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodb.(*ReconcilePerconaServerMongoDB).Reconcile\n\t/go/src/github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodb/psmdb_controller.go:450\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:222\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:479\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:438\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:313\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1693"} sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler /go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:495 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem /go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:438 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1 /go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:313 2026-03-09T09:20:26.124Z ERROR FULL CLUSTER CRASH {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"data-at-rest-encryption-4238"}, "namespace": "data-at-rest-encryption-4238", "name": "some-name", "reconcileID": "5f51db6e-08e0-462f-a632-7c101da4fe68", "error": "ping mongo: server selection error: server selection timeout, current topology: { Type: ReplicaSetNoPrimary, Servers: [{ Addr: some-name-cfg-0.some-name-cfg.data-at-rest-encryption-4238.svc.cluster.local:27017, Type: Unknown, Last error: dial tcp: lookup some-name-cfg-0.some-name-cfg.data-at-rest-encryption-4238.svc.cluster.local on 34.118.224.10:53: no such host }, { Addr: some-name-cfg-1.some-name-cfg.data-at-rest-encryption-4238.svc.cluster.local:27017, Type: RSSecondary, Tag sets: zone=us-central1-a,serviceName=some-name,region=us-central1,nodeName=gke-jen-psmdb-2277-0a1aa-default-pool-256e0b3f-8kt9,podName=some-name-cfg-1, Average RTT: 6008707 }, { Addr: some-name-cfg-2.some-name-cfg.data-at-rest-encryption-4238.svc.cluster.local:27017, Type: Unknown, Last error: dial tcp: lookup some-name-cfg-2.some-name-cfg.data-at-rest-encryption-4238.svc.cluster.local on 34.118.224.10:53: no such host }, ] }", "errorVerbose": "server selection error: server selection timeout, current topology: { Type: ReplicaSetNoPrimary, Servers: [{ Addr: some-name-cfg-0.some-name-cfg.data-at-rest-encryption-4238.svc.cluster.local:27017, Type: Unknown, Last error: dial tcp: lookup some-name-cfg-0.some-name-cfg.data-at-rest-encryption-4238.svc.cluster.local on 34.118.224.10:53: no such host }, { Addr: some-name-cfg-1.some-name-cfg.data-at-rest-encryption-4238.svc.cluster.local:27017, Type: RSSecondary, Tag sets: zone=us-central1-a,serviceName=some-name,region=us-central1,nodeName=gke-jen-psmdb-2277-0a1aa-default-pool-256e0b3f-8kt9,podName=some-name-cfg-1, Average RTT: 6008707 }, { Addr: some-name-cfg-2.some-name-cfg.data-at-rest-encryption-4238.svc.cluster.local:27017, Type: Unknown, Last error: dial tcp: lookup some-name-cfg-2.some-name-cfg.data-at-rest-encryption-4238.svc.cluster.local on 34.118.224.10:53: no such host }, ] }\nping mongo\ngithub.com/percona/percona-server-mongodb-operator/pkg/psmdb/mongo.Dial\n\t/go/src/github.com/percona/percona-server-mongodb-operator/pkg/psmdb/mongo/mongo.go:126\ngithub.com/percona/percona-server-mongodb-operator/pkg/psmdb.MongoClient\n\t/go/src/github.com/percona/percona-server-mongodb-operator/pkg/psmdb/client.go:62\ngithub.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodb.(*mongoClientProvider).Mongo\n\t/go/src/github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodb/connections.go:38\ngithub.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodb.(*ReconcilePerconaServerMongoDB).mongoClientWithRole\n\t/go/src/github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodb/connections.go:60\ngithub.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodb.(*ReconcilePerconaServerMongoDB).reconcileCluster\n\t/go/src/github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodb/mgo.go:94\ngithub.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodb.(*ReconcilePerconaServerMongoDB).reconcileReplsets\n\t/go/src/github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodb/psmdb_controller.go:590\ngithub.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodb.(*ReconcilePerconaServerMongoDB).Reconcile\n\t/go/src/github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodb/psmdb_controller.go:448\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:222\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:479\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:438\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:313\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1693"} github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodb.(*ReconcilePerconaServerMongoDB).reconcileCluster /go/src/github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodb/mgo.go:101 github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodb.(*ReconcilePerconaServerMongoDB).reconcileReplsets /go/src/github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodb/psmdb_controller.go:590 github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodb.(*ReconcilePerconaServerMongoDB).Reconcile /go/src/github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodb/psmdb_controller.go:448 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile /go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:222 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler /go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:479 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem /go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:438 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1 /go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:313 2026-03-09T09:20:26.124Z INFO handleReplicaSetNoPrimary Connecting to pod {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"data-at-rest-encryption-4238"}, "namespace": "data-at-rest-encryption-4238", "name": "some-name", "reconcileID": "5f51db6e-08e0-462f-a632-7c101da4fe68", "pod": "some-name-cfg-1", "user": "clusterAdmin"} 2026-03-09T09:20:26.211Z DEBUG Running replSetReconfig config {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"data-at-rest-encryption-4238"}, "namespace": "data-at-rest-encryption-4238", "name": "some-name", "reconcileID": "5f51db6e-08e0-462f-a632-7c101da4fe68", "cfg": {"_id":"cfg","version":732415,"members":[{"_id":0,"host":"some-name-cfg-0.some-name-cfg.data-at-rest-encryption-4238.svc.cluster.local:27017","arbiterOnly":false,"buildIndexes":true,"hidden":false,"priority":2,"tags":{"nodeName":"gke-jen-psmdb-2277-0a1aa-default-pool-256e0b3f-6n8n","podName":"some-name-cfg-0","region":"us-central1","serviceName":"some-name","zone":"us-central1-a"},"secondaryDelaySecs":0,"votes":1},{"_id":1,"host":"some-name-cfg-1.some-name-cfg.data-at-rest-encryption-4238.svc.cluster.local:27017","arbiterOnly":false,"buildIndexes":true,"hidden":false,"priority":2,"tags":{"nodeName":"gke-jen-psmdb-2277-0a1aa-default-pool-256e0b3f-8kt9","podName":"some-name-cfg-1","region":"us-central1","serviceName":"some-name","zone":"us-central1-a"},"secondaryDelaySecs":0,"votes":1},{"_id":2,"host":"some-name-cfg-2.some-name-cfg.data-at-rest-encryption-4238.svc.cluster.local:27017","arbiterOnly":false,"buildIndexes":true,"hidden":false,"priority":2,"tags":{"nodeName":"gke-jen-psmdb-2277-0a1aa-default-pool-256e0b3f-6n8n","podName":"some-name-cfg-2","region":"us-central1","serviceName":"some-name","zone":"us-central1-a"},"secondaryDelaySecs":0,"votes":1}],"configsvr":true,"protocolVersion":1,"settings":{"chainingAllowed":true,"heartbeatIntervalMillis":2000,"heartbeatTimeoutSecs":10,"electionTimeoutMillis":10000,"catchUpTimeoutMillis":-1,"getLastErrorDefaults":{"w":1,"wtimeout":0},"replicaSetId":"69ae8ec416a2c419847896a0"},"writeConcernMajorityJournalDefault":true}} 2026-03-09T09:20:26.217Z DEBUG Replset status is not healthy {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"data-at-rest-encryption-4238"}, "namespace": "data-at-rest-encryption-4238", "name": "some-name", "reconcileID": "5f51db6e-08e0-462f-a632-7c101da4fe68", "replset": "cfg", "status": "error"} 2026-03-09T09:20:36.707Z ERROR failed to reconcile cluster {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"data-at-rest-encryption-4238"}, "namespace": "data-at-rest-encryption-4238", "name": "some-name", "reconcileID": "5f51db6e-08e0-462f-a632-7c101da4fe68", "replset": "rs0", "error": "failed to get mongos connection: ping mongo: server selection error: server selection timeout, current topology: { Type: Unknown, Servers: [{ Addr: some-name-mongos.data-at-rest-encryption-4238.svc.cluster.local:27017, Type: Unknown, Last error: dial tcp 34.118.237.26:27017: connect: connection refused }, ] }", "errorVerbose": "server selection error: server selection timeout, current topology: { Type: Unknown, Servers: [{ Addr: some-name-mongos.data-at-rest-encryption-4238.svc.cluster.local:27017, Type: Unknown, Last error: dial tcp 34.118.237.26:27017: connect: connection refused }, ] }\nping mongo\ngithub.com/percona/percona-server-mongodb-operator/pkg/psmdb/mongo.Dial\n\t/go/src/github.com/percona/percona-server-mongodb-operator/pkg/psmdb/mongo/mongo.go:126\ngithub.com/percona/percona-server-mongodb-operator/pkg/psmdb.MongosClient\n\t/go/src/github.com/percona/percona-server-mongodb-operator/pkg/psmdb/client.go:85\ngithub.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodb.(*mongoClientProvider).Mongos\n\t/go/src/github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodb/connections.go:47\ngithub.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodb.(*ReconcilePerconaServerMongoDB).mongosClientWithRole\n\t/go/src/github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodb/connections.go:64\ngithub.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodb.(*ReconcilePerconaServerMongoDB).reconcileCluster\n\t/go/src/github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodb/mgo.go:196\ngithub.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodb.(*ReconcilePerconaServerMongoDB).reconcileReplsets\n\t/go/src/github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodb/psmdb_controller.go:590\ngithub.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodb.(*ReconcilePerconaServerMongoDB).Reconcile\n\t/go/src/github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodb/psmdb_controller.go:448\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:222\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:479\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:438\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:313\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1693\nfailed to get mongos connection\ngithub.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodb.(*ReconcilePerconaServerMongoDB).reconcileCluster\n\t/go/src/github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodb/mgo.go:198\ngithub.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodb.(*ReconcilePerconaServerMongoDB).reconcileReplsets\n\t/go/src/github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodb/psmdb_controller.go:590\ngithub.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodb.(*ReconcilePerconaServerMongoDB).Reconcile\n\t/go/src/github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodb/psmdb_controller.go:448\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:222\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:479\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:438\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:313\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1693"} github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodb.(*ReconcilePerconaServerMongoDB).reconcileReplsets /go/src/github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodb/psmdb_controller.go:592 github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodb.(*ReconcilePerconaServerMongoDB).Reconcile /go/src/github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodb/psmdb_controller.go:448 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile /go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:222 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler /go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:479 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem /go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:438 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1 /go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:313 2026-03-09T09:20:36.708Z DEBUG Replset status is not healthy {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"data-at-rest-encryption-4238"}, "namespace": "data-at-rest-encryption-4238", "name": "some-name", "reconcileID": "5f51db6e-08e0-462f-a632-7c101da4fe68", "replset": "rs0", "status": "error"} 2026-03-09T09:20:36.770Z ERROR Reconciler error {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"data-at-rest-encryption-4238"}, "namespace": "data-at-rest-encryption-4238", "name": "some-name", "reconcileID": "5f51db6e-08e0-462f-a632-7c101da4fe68", "error": "reconcile statefulsets: failed to get mongos connection: ping mongo: server selection error: server selection timeout, current topology: { Type: Unknown, Servers: [{ Addr: some-name-mongos.data-at-rest-encryption-4238.svc.cluster.local:27017, Type: Unknown, Last error: dial tcp 34.118.237.26:27017: connect: connection refused }, ] }", "errorVerbose": "failed to get mongos connection: ping mongo: server selection error: server selection timeout, current topology: { Type: Unknown, Servers: [{ Addr: some-name-mongos.data-at-rest-encryption-4238.svc.cluster.local:27017, Type: Unknown, Last error: dial tcp 34.118.237.26:27017: connect: connection refused }, ] }\nreconcile statefulsets\ngithub.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodb.(*ReconcilePerconaServerMongoDB).Reconcile\n\t/go/src/github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodb/psmdb_controller.go:450\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:222\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:479\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:438\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:313\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1693"} sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler /go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:495 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem /go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:438 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1 /go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:313 2026-03-09T09:21:08.913Z ERROR FULL CLUSTER CRASH {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"data-at-rest-encryption-4238"}, "namespace": "data-at-rest-encryption-4238", "name": "some-name", "reconcileID": "9bd2b0a1-6e27-47b8-a0c5-dee1aaa5d4a9", "error": "ping mongo: server selection error: server selection timeout, current topology: { Type: ReplicaSetNoPrimary, Servers: [{ Addr: some-name-cfg-0.some-name-cfg.data-at-rest-encryption-4238.svc.cluster.local:27017, Type: Unknown, Last error: dial tcp: lookup some-name-cfg-0.some-name-cfg.data-at-rest-encryption-4238.svc.cluster.local on 34.118.224.10:53: no such host }, { Addr: some-name-cfg-1.some-name-cfg.data-at-rest-encryption-4238.svc.cluster.local:27017, Type: RSSecondary, Tag sets: podName=some-name-cfg-1,serviceName=some-name,region=us-central1,nodeName=gke-jen-psmdb-2277-0a1aa-default-pool-256e0b3f-8kt9,zone=us-central1-a, Average RTT: 2255360 }, { Addr: some-name-cfg-2.some-name-cfg.data-at-rest-encryption-4238.svc.cluster.local:27017, Type: Unknown, Last error: dial tcp: lookup some-name-cfg-2.some-name-cfg.data-at-rest-encryption-4238.svc.cluster.local on 34.118.224.10:53: no such host }, ] }", "errorVerbose": "server selection error: server selection timeout, current topology: { Type: ReplicaSetNoPrimary, Servers: [{ Addr: some-name-cfg-0.some-name-cfg.data-at-rest-encryption-4238.svc.cluster.local:27017, Type: Unknown, Last error: dial tcp: lookup some-name-cfg-0.some-name-cfg.data-at-rest-encryption-4238.svc.cluster.local on 34.118.224.10:53: no such host }, { Addr: some-name-cfg-1.some-name-cfg.data-at-rest-encryption-4238.svc.cluster.local:27017, Type: RSSecondary, Tag sets: podName=some-name-cfg-1,serviceName=some-name,region=us-central1,nodeName=gke-jen-psmdb-2277-0a1aa-default-pool-256e0b3f-8kt9,zone=us-central1-a, Average RTT: 2255360 }, { Addr: some-name-cfg-2.some-name-cfg.data-at-rest-encryption-4238.svc.cluster.local:27017, Type: Unknown, Last error: dial tcp: lookup some-name-cfg-2.some-name-cfg.data-at-rest-encryption-4238.svc.cluster.local on 34.118.224.10:53: no such host }, ] }\nping mongo\ngithub.com/percona/percona-server-mongodb-operator/pkg/psmdb/mongo.Dial\n\t/go/src/github.com/percona/percona-server-mongodb-operator/pkg/psmdb/mongo/mongo.go:126\ngithub.com/percona/percona-server-mongodb-operator/pkg/psmdb.MongoClient\n\t/go/src/github.com/percona/percona-server-mongodb-operator/pkg/psmdb/client.go:62\ngithub.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodb.(*mongoClientProvider).Mongo\n\t/go/src/github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodb/connections.go:38\ngithub.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodb.(*ReconcilePerconaServerMongoDB).mongoClientWithRole\n\t/go/src/github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodb/connections.go:60\ngithub.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodb.(*ReconcilePerconaServerMongoDB).reconcileCluster\n\t/go/src/github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodb/mgo.go:94\ngithub.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodb.(*ReconcilePerconaServerMongoDB).reconcileReplsets\n\t/go/src/github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodb/psmdb_controller.go:590\ngithub.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodb.(*ReconcilePerconaServerMongoDB).Reconcile\n\t/go/src/github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodb/psmdb_controller.go:448\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:222\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:479\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:438\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:313\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1693"} github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodb.(*ReconcilePerconaServerMongoDB).reconcileCluster /go/src/github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodb/mgo.go:101 github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodb.(*ReconcilePerconaServerMongoDB).reconcileReplsets /go/src/github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodb/psmdb_controller.go:590 github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodb.(*ReconcilePerconaServerMongoDB).Reconcile /go/src/github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodb/psmdb_controller.go:448 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile /go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:222 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler /go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:479 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem /go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:438 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1 /go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:313 2026-03-09T09:21:08.913Z INFO handleReplicaSetNoPrimary Connecting to pod {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"data-at-rest-encryption-4238"}, "namespace": "data-at-rest-encryption-4238", "name": "some-name", "reconcileID": "9bd2b0a1-6e27-47b8-a0c5-dee1aaa5d4a9", "pod": "some-name-cfg-1", "user": "clusterAdmin"} 2026-03-09T09:21:08.980Z DEBUG Running replSetReconfig config {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"data-at-rest-encryption-4238"}, "namespace": "data-at-rest-encryption-4238", "name": "some-name", "reconcileID": "9bd2b0a1-6e27-47b8-a0c5-dee1aaa5d4a9", "cfg": {"_id":"cfg","version":749630,"members":[{"_id":0,"host":"some-name-cfg-0.some-name-cfg.data-at-rest-encryption-4238.svc.cluster.local:27017","arbiterOnly":false,"buildIndexes":true,"hidden":false,"priority":2,"tags":{"nodeName":"gke-jen-psmdb-2277-0a1aa-default-pool-256e0b3f-6n8n","podName":"some-name-cfg-0","region":"us-central1","serviceName":"some-name","zone":"us-central1-a"},"secondaryDelaySecs":0,"votes":1},{"_id":1,"host":"some-name-cfg-1.some-name-cfg.data-at-rest-encryption-4238.svc.cluster.local:27017","arbiterOnly":false,"buildIndexes":true,"hidden":false,"priority":2,"tags":{"nodeName":"gke-jen-psmdb-2277-0a1aa-default-pool-256e0b3f-8kt9","podName":"some-name-cfg-1","region":"us-central1","serviceName":"some-name","zone":"us-central1-a"},"secondaryDelaySecs":0,"votes":1},{"_id":2,"host":"some-name-cfg-2.some-name-cfg.data-at-rest-encryption-4238.svc.cluster.local:27017","arbiterOnly":false,"buildIndexes":true,"hidden":false,"priority":2,"tags":{"nodeName":"gke-jen-psmdb-2277-0a1aa-default-pool-256e0b3f-6n8n","podName":"some-name-cfg-2","region":"us-central1","serviceName":"some-name","zone":"us-central1-a"},"secondaryDelaySecs":0,"votes":1}],"configsvr":true,"protocolVersion":1,"settings":{"chainingAllowed":true,"heartbeatIntervalMillis":2000,"heartbeatTimeoutSecs":10,"electionTimeoutMillis":10000,"catchUpTimeoutMillis":-1,"getLastErrorDefaults":{"w":1,"wtimeout":0},"replicaSetId":"69ae8ec416a2c419847896a0"},"writeConcernMajorityJournalDefault":true}} 2026-03-09T09:21:08.984Z DEBUG Replset status is not healthy {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"data-at-rest-encryption-4238"}, "namespace": "data-at-rest-encryption-4238", "name": "some-name", "reconcileID": "9bd2b0a1-6e27-47b8-a0c5-dee1aaa5d4a9", "replset": "cfg", "status": "error"} 2026-03-09T09:21:19.419Z ERROR failed to reconcile cluster {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"data-at-rest-encryption-4238"}, "namespace": "data-at-rest-encryption-4238", "name": "some-name", "reconcileID": "9bd2b0a1-6e27-47b8-a0c5-dee1aaa5d4a9", "replset": "rs0", "error": "failed to get mongos connection: ping mongo: server selection error: server selection timeout, current topology: { Type: Unknown, Servers: [{ Addr: some-name-mongos.data-at-rest-encryption-4238.svc.cluster.local:27017, Type: Unknown, Last error: dial tcp 34.118.237.26:27017: connect: connection refused }, ] }", "errorVerbose": "server selection error: server selection timeout, current topology: { Type: Unknown, Servers: [{ Addr: some-name-mongos.data-at-rest-encryption-4238.svc.cluster.local:27017, Type: Unknown, Last error: dial tcp 34.118.237.26:27017: connect: connection refused }, ] }\nping mongo\ngithub.com/percona/percona-server-mongodb-operator/pkg/psmdb/mongo.Dial\n\t/go/src/github.com/percona/percona-server-mongodb-operator/pkg/psmdb/mongo/mongo.go:126\ngithub.com/percona/percona-server-mongodb-operator/pkg/psmdb.MongosClient\n\t/go/src/github.com/percona/percona-server-mongodb-operator/pkg/psmdb/client.go:85\ngithub.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodb.(*mongoClientProvider).Mongos\n\t/go/src/github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodb/connections.go:47\ngithub.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodb.(*ReconcilePerconaServerMongoDB).mongosClientWithRole\n\t/go/src/github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodb/connections.go:64\ngithub.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodb.(*ReconcilePerconaServerMongoDB).reconcileCluster\n\t/go/src/github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodb/mgo.go:196\ngithub.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodb.(*ReconcilePerconaServerMongoDB).reconcileReplsets\n\t/go/src/github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodb/psmdb_controller.go:590\ngithub.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodb.(*ReconcilePerconaServerMongoDB).Reconcile\n\t/go/src/github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodb/psmdb_controller.go:448\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:222\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:479\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:438\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:313\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1693\nfailed to get mongos connection\ngithub.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodb.(*ReconcilePerconaServerMongoDB).reconcileCluster\n\t/go/src/github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodb/mgo.go:198\ngithub.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodb.(*ReconcilePerconaServerMongoDB).reconcileReplsets\n\t/go/src/github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodb/psmdb_controller.go:590\ngithub.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodb.(*ReconcilePerconaServerMongoDB).Reconcile\n\t/go/src/github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodb/psmdb_controller.go:448\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:222\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:479\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:438\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:313\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1693"} github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodb.(*ReconcilePerconaServerMongoDB).reconcileReplsets /go/src/github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodb/psmdb_controller.go:592 github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodb.(*ReconcilePerconaServerMongoDB).Reconcile /go/src/github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodb/psmdb_controller.go:448 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile /go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:222 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler /go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:479 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem /go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:438 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1 /go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:313 2026-03-09T09:21:19.420Z DEBUG Replset status is not healthy {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"data-at-rest-encryption-4238"}, "namespace": "data-at-rest-encryption-4238", "name": "some-name", "reconcileID": "9bd2b0a1-6e27-47b8-a0c5-dee1aaa5d4a9", "replset": "rs0", "status": "error"} 2026-03-09T09:21:19.454Z ERROR Reconciler error {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"data-at-rest-encryption-4238"}, "namespace": "data-at-rest-encryption-4238", "name": "some-name", "reconcileID": "9bd2b0a1-6e27-47b8-a0c5-dee1aaa5d4a9", "error": "reconcile statefulsets: failed to get mongos connection: ping mongo: server selection error: server selection timeout, current topology: { Type: Unknown, Servers: [{ Addr: some-name-mongos.data-at-rest-encryption-4238.svc.cluster.local:27017, Type: Unknown, Last error: dial tcp 34.118.237.26:27017: connect: connection refused }, ] }", "errorVerbose": "failed to get mongos connection: ping mongo: server selection error: server selection timeout, current topology: { Type: Unknown, Servers: [{ Addr: some-name-mongos.data-at-rest-encryption-4238.svc.cluster.local:27017, Type: Unknown, Last error: dial tcp 34.118.237.26:27017: connect: connection refused }, ] }\nreconcile statefulsets\ngithub.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodb.(*ReconcilePerconaServerMongoDB).Reconcile\n\t/go/src/github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodb/psmdb_controller.go:450\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:222\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:479\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:438\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:313\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1693"} sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler /go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:495 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem /go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:438 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1 /go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:313 2026-03-09T09:22:12.300Z ERROR FULL CLUSTER CRASH {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"data-at-rest-encryption-4238"}, "namespace": "data-at-rest-encryption-4238", "name": "some-name", "reconcileID": "520fe50d-bd65-4631-9180-d814060182a7", "error": "ping mongo: server selection error: server selection timeout, current topology: { Type: ReplicaSetNoPrimary, Servers: [{ Addr: some-name-cfg-0.some-name-cfg.data-at-rest-encryption-4238.svc.cluster.local:27017, Type: Unknown, Last error: dial tcp: lookup some-name-cfg-0.some-name-cfg.data-at-rest-encryption-4238.svc.cluster.local on 34.118.224.10:53: no such host }, { Addr: some-name-cfg-1.some-name-cfg.data-at-rest-encryption-4238.svc.cluster.local:27017, Type: RSSecondary, Tag sets: serviceName=some-name,zone=us-central1-a,region=us-central1,nodeName=gke-jen-psmdb-2277-0a1aa-default-pool-256e0b3f-8kt9,podName=some-name-cfg-1, Average RTT: 1975467 }, { Addr: some-name-cfg-2.some-name-cfg.data-at-rest-encryption-4238.svc.cluster.local:27017, Type: Unknown, Last error: dial tcp: lookup some-name-cfg-2.some-name-cfg.data-at-rest-encryption-4238.svc.cluster.local on 34.118.224.10:53: no such host }, ] }", "errorVerbose": "server selection error: server selection timeout, current topology: { Type: ReplicaSetNoPrimary, Servers: [{ Addr: some-name-cfg-0.some-name-cfg.data-at-rest-encryption-4238.svc.cluster.local:27017, Type: Unknown, Last error: dial tcp: lookup some-name-cfg-0.some-name-cfg.data-at-rest-encryption-4238.svc.cluster.local on 34.118.224.10:53: no such host }, { Addr: some-name-cfg-1.some-name-cfg.data-at-rest-encryption-4238.svc.cluster.local:27017, Type: RSSecondary, Tag sets: serviceName=some-name,zone=us-central1-a,region=us-central1,nodeName=gke-jen-psmdb-2277-0a1aa-default-pool-256e0b3f-8kt9,podName=some-name-cfg-1, Average RTT: 1975467 }, { Addr: some-name-cfg-2.some-name-cfg.data-at-rest-encryption-4238.svc.cluster.local:27017, Type: Unknown, Last error: dial tcp: lookup some-name-cfg-2.some-name-cfg.data-at-rest-encryption-4238.svc.cluster.local on 34.118.224.10:53: no such host }, ] }\nping mongo\ngithub.com/percona/percona-server-mongodb-operator/pkg/psmdb/mongo.Dial\n\t/go/src/github.com/percona/percona-server-mongodb-operator/pkg/psmdb/mongo/mongo.go:126\ngithub.com/percona/percona-server-mongodb-operator/pkg/psmdb.MongoClient\n\t/go/src/github.com/percona/percona-server-mongodb-operator/pkg/psmdb/client.go:62\ngithub.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodb.(*mongoClientProvider).Mongo\n\t/go/src/github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodb/connections.go:38\ngithub.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodb.(*ReconcilePerconaServerMongoDB).mongoClientWithRole\n\t/go/src/github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodb/connections.go:60\ngithub.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodb.(*ReconcilePerconaServerMongoDB).reconcileCluster\n\t/go/src/github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodb/mgo.go:94\ngithub.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodb.(*ReconcilePerconaServerMongoDB).reconcileReplsets\n\t/go/src/github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodb/psmdb_controller.go:590\ngithub.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodb.(*ReconcilePerconaServerMongoDB).Reconcile\n\t/go/src/github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodb/psmdb_controller.go:448\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:222\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:479\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:438\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:313\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1693"} github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodb.(*ReconcilePerconaServerMongoDB).reconcileCluster /go/src/github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodb/mgo.go:101 github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodb.(*ReconcilePerconaServerMongoDB).reconcileReplsets /go/src/github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodb/psmdb_controller.go:590 github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodb.(*ReconcilePerconaServerMongoDB).Reconcile /go/src/github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodb/psmdb_controller.go:448 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile /go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:222 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler /go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:479 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem /go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:438 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1 /go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:313 2026-03-09T09:22:12.301Z INFO handleReplicaSetNoPrimary Connecting to pod {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"data-at-rest-encryption-4238"}, "namespace": "data-at-rest-encryption-4238", "name": "some-name", "reconcileID": "520fe50d-bd65-4631-9180-d814060182a7", "pod": "some-name-cfg-1", "user": "clusterAdmin"} 2026-03-09T09:22:12.361Z DEBUG Running replSetReconfig config {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"data-at-rest-encryption-4238"}, "namespace": "data-at-rest-encryption-4238", "name": "some-name", "reconcileID": "520fe50d-bd65-4631-9180-d814060182a7", "cfg": {"_id":"cfg","version":812424,"members":[{"_id":0,"host":"some-name-cfg-0.some-name-cfg.data-at-rest-encryption-4238.svc.cluster.local:27017","arbiterOnly":false,"buildIndexes":true,"hidden":false,"priority":2,"tags":{"nodeName":"gke-jen-psmdb-2277-0a1aa-default-pool-256e0b3f-6n8n","podName":"some-name-cfg-0","region":"us-central1","serviceName":"some-name","zone":"us-central1-a"},"secondaryDelaySecs":0,"votes":1},{"_id":1,"host":"some-name-cfg-1.some-name-cfg.data-at-rest-encryption-4238.svc.cluster.local:27017","arbiterOnly":false,"buildIndexes":true,"hidden":false,"priority":2,"tags":{"nodeName":"gke-jen-psmdb-2277-0a1aa-default-pool-256e0b3f-8kt9","podName":"some-name-cfg-1","region":"us-central1","serviceName":"some-name","zone":"us-central1-a"},"secondaryDelaySecs":0,"votes":1},{"_id":2,"host":"some-name-cfg-2.some-name-cfg.data-at-rest-encryption-4238.svc.cluster.local:27017","arbiterOnly":false,"buildIndexes":true,"hidden":false,"priority":2,"tags":{"nodeName":"gke-jen-psmdb-2277-0a1aa-default-pool-256e0b3f-6n8n","podName":"some-name-cfg-2","region":"us-central1","serviceName":"some-name","zone":"us-central1-a"},"secondaryDelaySecs":0,"votes":1}],"configsvr":true,"protocolVersion":1,"settings":{"chainingAllowed":true,"heartbeatIntervalMillis":2000,"heartbeatTimeoutSecs":10,"electionTimeoutMillis":10000,"catchUpTimeoutMillis":-1,"getLastErrorDefaults":{"w":1,"wtimeout":0},"replicaSetId":"69ae8ec416a2c419847896a0"},"writeConcernMajorityJournalDefault":true}} 2026-03-09T09:22:12.367Z DEBUG Replset status is not healthy {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"data-at-rest-encryption-4238"}, "namespace": "data-at-rest-encryption-4238", "name": "some-name", "reconcileID": "520fe50d-bd65-4631-9180-d814060182a7", "replset": "cfg", "status": "error"} 2026-03-09T09:22:22.796Z ERROR failed to reconcile cluster {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"data-at-rest-encryption-4238"}, "namespace": "data-at-rest-encryption-4238", "name": "some-name", "reconcileID": "520fe50d-bd65-4631-9180-d814060182a7", "replset": "rs0", "error": "failed to get mongos connection: ping mongo: server selection error: server selection timeout, current topology: { Type: Unknown, Servers: [{ Addr: some-name-mongos.data-at-rest-encryption-4238.svc.cluster.local:27017, Type: Unknown, Last error: dial tcp 34.118.237.26:27017: connect: connection refused }, ] }", "errorVerbose": "server selection error: server selection timeout, current topology: { Type: Unknown, Servers: [{ Addr: some-name-mongos.data-at-rest-encryption-4238.svc.cluster.local:27017, Type: Unknown, Last error: dial tcp 34.118.237.26:27017: connect: connection refused }, ] }\nping mongo\ngithub.com/percona/percona-server-mongodb-operator/pkg/psmdb/mongo.Dial\n\t/go/src/github.com/percona/percona-server-mongodb-operator/pkg/psmdb/mongo/mongo.go:126\ngithub.com/percona/percona-server-mongodb-operator/pkg/psmdb.MongosClient\n\t/go/src/github.com/percona/percona-server-mongodb-operator/pkg/psmdb/client.go:85\ngithub.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodb.(*mongoClientProvider).Mongos\n\t/go/src/github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodb/connections.go:47\ngithub.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodb.(*ReconcilePerconaServerMongoDB).mongosClientWithRole\n\t/go/src/github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodb/connections.go:64\ngithub.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodb.(*ReconcilePerconaServerMongoDB).reconcileCluster\n\t/go/src/github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodb/mgo.go:196\ngithub.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodb.(*ReconcilePerconaServerMongoDB).reconcileReplsets\n\t/go/src/github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodb/psmdb_controller.go:590\ngithub.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodb.(*ReconcilePerconaServerMongoDB).Reconcile\n\t/go/src/github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodb/psmdb_controller.go:448\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:222\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:479\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:438\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:313\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1693\nfailed to get mongos connection\ngithub.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodb.(*ReconcilePerconaServerMongoDB).reconcileCluster\n\t/go/src/github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodb/mgo.go:198\ngithub.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodb.(*ReconcilePerconaServerMongoDB).reconcileReplsets\n\t/go/src/github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodb/psmdb_controller.go:590\ngithub.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodb.(*ReconcilePerconaServerMongoDB).Reconcile\n\t/go/src/github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodb/psmdb_controller.go:448\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:222\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:479\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:438\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:313\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1693"} github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodb.(*ReconcilePerconaServerMongoDB).reconcileReplsets /go/src/github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodb/psmdb_controller.go:592 github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodb.(*ReconcilePerconaServerMongoDB).Reconcile /go/src/github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodb/psmdb_controller.go:448 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile /go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:222 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler /go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:479 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem /go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:438 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1 /go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:313 2026-03-09T09:22:22.796Z DEBUG Replset status is not healthy {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"data-at-rest-encryption-4238"}, "namespace": "data-at-rest-encryption-4238", "name": "some-name", "reconcileID": "520fe50d-bd65-4631-9180-d814060182a7", "replset": "rs0", "status": "error"} 2026-03-09T09:22:22.838Z ERROR Reconciler error {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"data-at-rest-encryption-4238"}, "namespace": "data-at-rest-encryption-4238", "name": "some-name", "reconcileID": "520fe50d-bd65-4631-9180-d814060182a7", "error": "reconcile statefulsets: failed to get mongos connection: ping mongo: server selection error: server selection timeout, current topology: { Type: Unknown, Servers: [{ Addr: some-name-mongos.data-at-rest-encryption-4238.svc.cluster.local:27017, Type: Unknown, Last error: dial tcp 34.118.237.26:27017: connect: connection refused }, ] }", "errorVerbose": "failed to get mongos connection: ping mongo: server selection error: server selection timeout, current topology: { Type: Unknown, Servers: [{ Addr: some-name-mongos.data-at-rest-encryption-4238.svc.cluster.local:27017, Type: Unknown, Last error: dial tcp 34.118.237.26:27017: connect: connection refused }, ] }\nreconcile statefulsets\ngithub.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodb.(*ReconcilePerconaServerMongoDB).Reconcile\n\t/go/src/github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodb/psmdb_controller.go:450\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:222\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:479\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:438\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:313\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1693"} sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler /go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:495 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem /go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:438 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1 /go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:313 2026-03-09T09:22:48.184Z ERROR failed to make backup {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs","namespace":"data-at-rest-encryption-4238"}, "namespace": "data-at-rest-encryption-4238", "name": "backup-gcp-cs", "reconcileID": "45aa8b23-9aaa-4ab9-ae0b-35f99e96be2c", "backup": "backup-gcp-cs", "error": "create backup object: create pbm object: create PBM connection to some-name-rs0-0.some-name-rs0.data-at-rest-encryption-4238.svc.cluster.local:27017,some-name-rs0-1.some-name-rs0.data-at-rest-encryption-4238.svc.cluster.local:27017,some-name-rs0-2.some-name-rs0.data-at-rest-encryption-4238.svc.cluster.local:27017: create mongo connection to configsvr: ping: server selection error: server selection timeout, current topology: { Type: ReplicaSetNoPrimary, Servers: [{ Addr: some-name-cfg-0.some-name-cfg.data-at-rest-encryption-4238.svc.cluster.local:27017, Type: Unknown, Last error: dial tcp: lookup some-name-cfg-0.some-name-cfg.data-at-rest-encryption-4238.svc.cluster.local on 34.118.224.10:53: no such host }, { Addr: some-name-cfg-1.some-name-cfg.data-at-rest-encryption-4238.svc.cluster.local:27017, Type: RSSecondary, Tag sets: serviceName=some-name,region=us-central1,nodeName=gke-jen-psmdb-2277-0a1aa-default-pool-256e0b3f-8kt9,podName=some-name-cfg-1,zone=us-central1-a, Average RTT: 2150470 }, { Addr: some-name-cfg-2.some-name-cfg.data-at-rest-encryption-4238.svc.cluster.local:27017, Type: Unknown, Last error: dial tcp: lookup some-name-cfg-2.some-name-cfg.data-at-rest-encryption-4238.svc.cluster.local on 34.118.224.10:53: no such host }, ] }", "errorVerbose": "server selection error: server selection timeout, current topology: { Type: ReplicaSetNoPrimary, Servers: [{ Addr: some-name-cfg-0.some-name-cfg.data-at-rest-encryption-4238.svc.cluster.local:27017, Type: Unknown, Last error: dial tcp: lookup some-name-cfg-0.some-name-cfg.data-at-rest-encryption-4238.svc.cluster.local on 34.118.224.10:53: no such host }, { Addr: some-name-cfg-1.some-name-cfg.data-at-rest-encryption-4238.svc.cluster.local:27017, Type: RSSecondary, Tag sets: serviceName=some-name,region=us-central1,nodeName=gke-jen-psmdb-2277-0a1aa-default-pool-256e0b3f-8kt9,podName=some-name-cfg-1,zone=us-central1-a, Average RTT: 2150470 }, { Addr: some-name-cfg-2.some-name-cfg.data-at-rest-encryption-4238.svc.cluster.local:27017, Type: Unknown, Last error: dial tcp: lookup some-name-cfg-2.some-name-cfg.data-at-rest-encryption-4238.svc.cluster.local on 34.118.224.10:53: no such host }, ] }\nping\ncreate mongo connection to configsvr\ncreate PBM connection to some-name-rs0-0.some-name-rs0.data-at-rest-encryption-4238.svc.cluster.local:27017,some-name-rs0-1.some-name-rs0.data-at-rest-encryption-4238.svc.cluster.local:27017,some-name-rs0-2.some-name-rs0.data-at-rest-encryption-4238.svc.cluster.local:27017\ngithub.com/percona/percona-server-mongodb-operator/pkg/psmdb/backup.NewPBM\n\t/go/src/github.com/percona/percona-server-mongodb-operator/pkg/psmdb/backup/pbm.go:220\ngithub.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodbbackup.(*ReconcilePerconaServerMongoDBBackup).newBackup\n\t/go/src/github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodbbackup/backup.go:47\ngithub.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodbbackup.(*ReconcilePerconaServerMongoDBBackup).Reconcile.func3\n\t/go/src/github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodbbackup/psmdb_backup_controller.go:203\nk8s.io/client-go/util/retry.OnError.func1\n\t/go/pkg/mod/k8s.io/client-go@v0.35.2/util/retry/util.go:51\nk8s.io/apimachinery/pkg/util/wait.runConditionWithCrashProtection\n\t/go/pkg/mod/k8s.io/apimachinery@v0.35.2/pkg/util/wait/wait.go:150\nk8s.io/apimachinery/pkg/util/wait.ExponentialBackoff\n\t/go/pkg/mod/k8s.io/apimachinery@v0.35.2/pkg/util/wait/backoff.go:477\nk8s.io/client-go/util/retry.OnError\n\t/go/pkg/mod/k8s.io/client-go@v0.35.2/util/retry/util.go:50\ngithub.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodbbackup.(*ReconcilePerconaServerMongoDBBackup).Reconcile\n\t/go/src/github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodbbackup/psmdb_backup_controller.go:201\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:222\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:479\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:438\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:313\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1693\ncreate pbm object\ngithub.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodbbackup.(*ReconcilePerconaServerMongoDBBackup).newBackup\n\t/go/src/github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodbbackup/backup.go:49\ngithub.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodbbackup.(*ReconcilePerconaServerMongoDBBackup).Reconcile.func3\n\t/go/src/github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodbbackup/psmdb_backup_controller.go:203\nk8s.io/client-go/util/retry.OnError.func1\n\t/go/pkg/mod/k8s.io/client-go@v0.35.2/util/retry/util.go:51\nk8s.io/apimachinery/pkg/util/wait.runConditionWithCrashProtection\n\t/go/pkg/mod/k8s.io/apimachinery@v0.35.2/pkg/util/wait/wait.go:150\nk8s.io/apimachinery/pkg/util/wait.ExponentialBackoff\n\t/go/pkg/mod/k8s.io/apimachinery@v0.35.2/pkg/util/wait/backoff.go:477\nk8s.io/client-go/util/retry.OnError\n\t/go/pkg/mod/k8s.io/client-go@v0.35.2/util/retry/util.go:50\ngithub.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodbbackup.(*ReconcilePerconaServerMongoDBBackup).Reconcile\n\t/go/src/github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodbbackup/psmdb_backup_controller.go:201\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:222\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:479\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:438\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:313\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1693\ncreate backup object\ngithub.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodbbackup.(*ReconcilePerconaServerMongoDBBackup).Reconcile.func3\n\t/go/src/github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodbbackup/psmdb_backup_controller.go:205\nk8s.io/client-go/util/retry.OnError.func1\n\t/go/pkg/mod/k8s.io/client-go@v0.35.2/util/retry/util.go:51\nk8s.io/apimachinery/pkg/util/wait.runConditionWithCrashProtection\n\t/go/pkg/mod/k8s.io/apimachinery@v0.35.2/pkg/util/wait/wait.go:150\nk8s.io/apimachinery/pkg/util/wait.ExponentialBackoff\n\t/go/pkg/mod/k8s.io/apimachinery@v0.35.2/pkg/util/wait/backoff.go:477\nk8s.io/client-go/util/retry.OnError\n\t/go/pkg/mod/k8s.io/client-go@v0.35.2/util/retry/util.go:50\ngithub.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodbbackup.(*ReconcilePerconaServerMongoDBBackup).Reconcile\n\t/go/src/github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodbbackup/psmdb_backup_controller.go:201\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:222\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:479\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:438\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:313\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1693"} github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodbbackup.(*ReconcilePerconaServerMongoDBBackup).Reconcile.func1 /go/src/github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodbbackup/psmdb_backup_controller.go:147 github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodbbackup.(*ReconcilePerconaServerMongoDBBackup).Reconcile /go/src/github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodbbackup/psmdb_backup_controller.go:228 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile /go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:222 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler /go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:479 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem /go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:438 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1 /go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:313 2026-03-09T09:22:48.184Z INFO Backup state changed {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs","namespace":"data-at-rest-encryption-4238"}, "namespace": "data-at-rest-encryption-4238", "name": "backup-gcp-cs", "reconcileID": "45aa8b23-9aaa-4ab9-ae0b-35f99e96be2c", "previous": "requested", "current": "error"} 2026-03-09T09:22:48.211Z INFO Releasing backup lock {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs","namespace":"data-at-rest-encryption-4238"}, "namespace": "data-at-rest-encryption-4238", "name": "backup-gcp-cs", "reconcileID": "45aa8b23-9aaa-4ab9-ae0b-35f99e96be2c", "lease": "psmdb-some-name-backup-lock"} 2026-03-09T09:22:48.346Z DEBUG Reconcile finished {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs","namespace":"data-at-rest-encryption-4238"}, "namespace": "data-at-rest-encryption-4238", "name": "backup-gcp-cs", "reconcileID": "45aa8b23-9aaa-4ab9-ae0b-35f99e96be2c"} 2026-03-09T09:22:48.347Z INFO Warning: Reconciler returned both a result with either RequeueAfter or Requeue set and a non-nil error. RequeueAfter and Requeue will always be ignored if the error is non-nil. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs","namespace":"data-at-rest-encryption-4238"}, "namespace": "data-at-rest-encryption-4238", "name": "backup-gcp-cs", "reconcileID": "45aa8b23-9aaa-4ab9-ae0b-35f99e96be2c"} 2026-03-09T09:22:48.347Z ERROR Reconciler error {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs","namespace":"data-at-rest-encryption-4238"}, "namespace": "data-at-rest-encryption-4238", "name": "backup-gcp-cs", "reconcileID": "45aa8b23-9aaa-4ab9-ae0b-35f99e96be2c", "error": "create backup object: create pbm object: create PBM connection to some-name-rs0-0.some-name-rs0.data-at-rest-encryption-4238.svc.cluster.local:27017,some-name-rs0-1.some-name-rs0.data-at-rest-encryption-4238.svc.cluster.local:27017,some-name-rs0-2.some-name-rs0.data-at-rest-encryption-4238.svc.cluster.local:27017: create mongo connection to configsvr: ping: server selection error: server selection timeout, current topology: { Type: ReplicaSetNoPrimary, Servers: [{ Addr: some-name-cfg-0.some-name-cfg.data-at-rest-encryption-4238.svc.cluster.local:27017, Type: Unknown, Last error: dial tcp: lookup some-name-cfg-0.some-name-cfg.data-at-rest-encryption-4238.svc.cluster.local on 34.118.224.10:53: no such host }, { Addr: some-name-cfg-1.some-name-cfg.data-at-rest-encryption-4238.svc.cluster.local:27017, Type: RSSecondary, Tag sets: serviceName=some-name,region=us-central1,nodeName=gke-jen-psmdb-2277-0a1aa-default-pool-256e0b3f-8kt9,podName=some-name-cfg-1,zone=us-central1-a, Average RTT: 2150470 }, { Addr: some-name-cfg-2.some-name-cfg.data-at-rest-encryption-4238.svc.cluster.local:27017, Type: Unknown, Last error: dial tcp: lookup some-name-cfg-2.some-name-cfg.data-at-rest-encryption-4238.svc.cluster.local on 34.118.224.10:53: no such host }, ] }", "errorVerbose": "server selection error: server selection timeout, current topology: { Type: ReplicaSetNoPrimary, Servers: [{ Addr: some-name-cfg-0.some-name-cfg.data-at-rest-encryption-4238.svc.cluster.local:27017, Type: Unknown, Last error: dial tcp: lookup some-name-cfg-0.some-name-cfg.data-at-rest-encryption-4238.svc.cluster.local on 34.118.224.10:53: no such host }, { Addr: some-name-cfg-1.some-name-cfg.data-at-rest-encryption-4238.svc.cluster.local:27017, Type: RSSecondary, Tag sets: serviceName=some-name,region=us-central1,nodeName=gke-jen-psmdb-2277-0a1aa-default-pool-256e0b3f-8kt9,podName=some-name-cfg-1,zone=us-central1-a, Average RTT: 2150470 }, { Addr: some-name-cfg-2.some-name-cfg.data-at-rest-encryption-4238.svc.cluster.local:27017, Type: Unknown, Last error: dial tcp: lookup some-name-cfg-2.some-name-cfg.data-at-rest-encryption-4238.svc.cluster.local on 34.118.224.10:53: no such host }, ] }\nping\ncreate mongo connection to configsvr\ncreate PBM connection to some-name-rs0-0.some-name-rs0.data-at-rest-encryption-4238.svc.cluster.local:27017,some-name-rs0-1.some-name-rs0.data-at-rest-encryption-4238.svc.cluster.local:27017,some-name-rs0-2.some-name-rs0.data-at-rest-encryption-4238.svc.cluster.local:27017\ngithub.com/percona/percona-server-mongodb-operator/pkg/psmdb/backup.NewPBM\n\t/go/src/github.com/percona/percona-server-mongodb-operator/pkg/psmdb/backup/pbm.go:220\ngithub.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodbbackup.(*ReconcilePerconaServerMongoDBBackup).newBackup\n\t/go/src/github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodbbackup/backup.go:47\ngithub.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodbbackup.(*ReconcilePerconaServerMongoDBBackup).Reconcile.func3\n\t/go/src/github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodbbackup/psmdb_backup_controller.go:203\nk8s.io/client-go/util/retry.OnError.func1\n\t/go/pkg/mod/k8s.io/client-go@v0.35.2/util/retry/util.go:51\nk8s.io/apimachinery/pkg/util/wait.runConditionWithCrashProtection\n\t/go/pkg/mod/k8s.io/apimachinery@v0.35.2/pkg/util/wait/wait.go:150\nk8s.io/apimachinery/pkg/util/wait.ExponentialBackoff\n\t/go/pkg/mod/k8s.io/apimachinery@v0.35.2/pkg/util/wait/backoff.go:477\nk8s.io/client-go/util/retry.OnError\n\t/go/pkg/mod/k8s.io/client-go@v0.35.2/util/retry/util.go:50\ngithub.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodbbackup.(*ReconcilePerconaServerMongoDBBackup).Reconcile\n\t/go/src/github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodbbackup/psmdb_backup_controller.go:201\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:222\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:479\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:438\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:313\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1693\ncreate pbm object\ngithub.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodbbackup.(*ReconcilePerconaServerMongoDBBackup).newBackup\n\t/go/src/github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodbbackup/backup.go:49\ngithub.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodbbackup.(*ReconcilePerconaServerMongoDBBackup).Reconcile.func3\n\t/go/src/github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodbbackup/psmdb_backup_controller.go:203\nk8s.io/client-go/util/retry.OnError.func1\n\t/go/pkg/mod/k8s.io/client-go@v0.35.2/util/retry/util.go:51\nk8s.io/apimachinery/pkg/util/wait.runConditionWithCrashProtection\n\t/go/pkg/mod/k8s.io/apimachinery@v0.35.2/pkg/util/wait/wait.go:150\nk8s.io/apimachinery/pkg/util/wait.ExponentialBackoff\n\t/go/pkg/mod/k8s.io/apimachinery@v0.35.2/pkg/util/wait/backoff.go:477\nk8s.io/client-go/util/retry.OnError\n\t/go/pkg/mod/k8s.io/client-go@v0.35.2/util/retry/util.go:50\ngithub.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodbbackup.(*ReconcilePerconaServerMongoDBBackup).Reconcile\n\t/go/src/github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodbbackup/psmdb_backup_controller.go:201\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:222\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:479\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:438\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:313\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1693\ncreate backup object\ngithub.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodbbackup.(*ReconcilePerconaServerMongoDBBackup).Reconcile.func3\n\t/go/src/github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodbbackup/psmdb_backup_controller.go:205\nk8s.io/client-go/util/retry.OnError.func1\n\t/go/pkg/mod/k8s.io/client-go@v0.35.2/util/retry/util.go:51\nk8s.io/apimachinery/pkg/util/wait.runConditionWithCrashProtection\n\t/go/pkg/mod/k8s.io/apimachinery@v0.35.2/pkg/util/wait/wait.go:150\nk8s.io/apimachinery/pkg/util/wait.ExponentialBackoff\n\t/go/pkg/mod/k8s.io/apimachinery@v0.35.2/pkg/util/wait/backoff.go:477\nk8s.io/client-go/util/retry.OnError\n\t/go/pkg/mod/k8s.io/client-go@v0.35.2/util/retry/util.go:50\ngithub.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodbbackup.(*ReconcilePerconaServerMongoDBBackup).Reconcile\n\t/go/src/github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodbbackup/psmdb_backup_controller.go:201\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:222\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:479\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:438\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:313\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1693"} sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler /go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:495 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem /go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:438 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1 /go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:313 2026-03-09T09:22:48.347Z DEBUG Reconciling {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs","namespace":"data-at-rest-encryption-4238"}, "namespace": "data-at-rest-encryption-4238", "name": "backup-gcp-cs", "reconcileID": "762517e3-6f7b-4085-849b-22fb8424410a"} 2026-03-09T09:22:48.360Z DEBUG Got object from API server {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs","namespace":"data-at-rest-encryption-4238"}, "namespace": "data-at-rest-encryption-4238", "name": "backup-gcp-cs", "reconcileID": "762517e3-6f7b-4085-849b-22fb8424410a", "state": "error"} 2026-03-09T09:22:48.360Z DEBUG Reconcile finished {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs","namespace":"data-at-rest-encryption-4238"}, "namespace": "data-at-rest-encryption-4238", "name": "backup-gcp-cs", "reconcileID": "762517e3-6f7b-4085-849b-22fb8424410a"} 2026-03-09T09:22:48.360Z DEBUG Reconciling {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio","namespace":"data-at-rest-encryption-4238"}, "namespace": "data-at-rest-encryption-4238", "name": "backup-minio", "reconcileID": "1cc84e85-04db-4e2c-b382-c001611c7229"} 2026-03-09T09:22:48.375Z DEBUG Got object from API server {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio","namespace":"data-at-rest-encryption-4238"}, "namespace": "data-at-rest-encryption-4238", "name": "backup-minio", "reconcileID": "1cc84e85-04db-4e2c-b382-c001611c7229", "state": "ready"} 2026-03-09T09:22:48.375Z DEBUG Reconcile finished {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio","namespace":"data-at-rest-encryption-4238"}, "namespace": "data-at-rest-encryption-4238", "name": "backup-minio", "reconcileID": "1cc84e85-04db-4e2c-b382-c001611c7229"} NAME CLUSTER STORAGE DESTINATION TYPE SIZE STATUS COMPLETED AGE backup-gcp-cs some-name gcp-cs s3://operator-testing/psmdb/2026-03-09T09:14:30Z error 8m22s backup-minio some-name minio s3://operator-testing/2026-03-09T09:14:01Z logical 91.43KB ready 8m24s 8m52s Name: backup-gcp-cs Namespace: data-at-rest-encryption-4238 Labels: Annotations: API Version: psmdb.percona.com/v1 Kind: PerconaServerMongoDBBackup Metadata: Creation Timestamp: 2026-03-09T09:14:30Z Finalizers: internal.percona.com/release-lock Generation: 1 Resource Version: 1773048168202959001 UID: bb1a2b8d-2b36-4e81-95d3-b559aa096c42 Spec: Cluster Name: some-name Starting Deadline Seconds: 120 Storage Name: gcp-cs Type: logical Status: Destination: s3://operator-testing/psmdb/2026-03-09T09:14:30Z Error: create backup object: create pbm object: create PBM connection to some-name-rs0-0.some-name-rs0.data-at-rest-encryption-4238.svc.cluster.local:27017,some-name-rs0-1.some-name-rs0.data-at-rest-encryption-4238.svc.cluster.local:27017,some-name-rs0-2.some-name-rs0.data-at-rest-encryption-4238.svc.cluster.local:27017: create mongo connection to configsvr: ping: server selection error: server selection timeout, current topology: { Type: ReplicaSetNoPrimary, Servers: [{ Addr: some-name-cfg-0.some-name-cfg.data-at-rest-encryption-4238.svc.cluster.local:27017, Type: Unknown, Last error: dial tcp: lookup some-name-cfg-0.some-name-cfg.data-at-rest-encryption-4238.svc.cluster.local on 34.118.224.10:53: no such host }, { Addr: some-name-cfg-1.some-name-cfg.data-at-rest-encryption-4238.svc.cluster.local:27017, Type: RSSecondary, Tag sets: serviceName=some-name,region=us-central1,nodeName=gke-jen-psmdb-2277-0a1aa-default-pool-256e0b3f-8kt9,podName=some-name-cfg-1,zone=us-central1-a, Average RTT: 2150470 }, { Addr: some-name-cfg-2.some-name-cfg.data-at-rest-encryption-4238.svc.cluster.local:27017, Type: Unknown, Last error: dial tcp: lookup some-name-cfg-2.some-name-cfg.data-at-rest-encryption-4238.svc.cluster.local on 34.118.224.10:53: no such host }, ] } Last Transition: 2026-03-09T09:14:30Z Pbm Name: 2026-03-09T09:14:30Z Replset Names: cfg rs0 s3: Bucket: operator-testing Credentials Secret: gcp-cs-secret Endpoint URL: https://storage.googleapis.com Prefix: psmdb Region: us-east-1 Server Side Encryption: State: error Storage Name: gcp-cs Events: Backup object psmdb-backup/backup-gcp-cs is in error state.