++ echo 'Log: /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2221/e2e-tests/logs/custom-replset-name.log' Log: /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2221/e2e-tests/logs/custom-replset-name.log ++ '[' -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2221/e2e-tests/conf/cloud-secret.yml ']' ++ SKIP_BACKUPS_TO_AWS_GCP_AZURE= ++ oc get projects ++ kubectl get nodes ++ grep '^minikube' +++ kubectl version -o json +++ jq -r .serverVersion.gitVersion +++ grep -eks- Warning: version difference between client (1.35) and server (1.32) exceeds the supported minor version skew of +/-1 ++ '[' ']' ++ EKS=0 +++ kubectl version -o json +++ grep gke +++ jq -r .serverVersion.gitVersion Warning: version difference between client (1.35) and server (1.32) exceeds the supported minor version skew of +/-1 ++ '[' v1.32.12-gke.1127000 ']' ++ GKE=1 +++ kubectl version -o json +++ jq -r '.serverVersion.major + "." + .serverVersion.minor' +++ /usr/sbin/sed -r 's/[^0-9.]+//g' Warning: version difference between client (1.35) and server (1.32) exceeds the supported minor version skew of +/-1 ++ KUBE_VERSION=1.32 + create_infra custom-replset-name-20645 + local ns=custom-replset-name-20645 + [[ 1 == 1 ]] + delete_crd + desc 'get and delete old CRDs and RBAC' + set +o xtrace ----------------------------------------------------------------------------------- get and delete old CRDs and RBAC ----------------------------------------------------------------------------------- grep: warning: stray \ before - grep: warning: stray \ before - error: the server doesn't have a resource type "perconaservermongodbbackups" + kubectl patch perconaservermongodbbackups.psmdb.percona.com -n sh --type=merge -p '{"metadata":{"finalizers":[]}}' error: the server doesn't have a resource type "perconaservermongodbbackups" error: the server doesn't have a resource type "perconaservermongodbrestores" + kubectl patch perconaservermongodbrestores.psmdb.percona.com -n sh --type=merge -p '{"metadata":{"finalizers":[]}}' error: the server doesn't have a resource type "perconaservermongodbrestores" error: the server doesn't have a resource type "perconaservermongodbs" + kubectl patch perconaservermongodbs.psmdb.percona.com -n sh --type=merge -p '{"metadata":{"finalizers":[]}}' error: the server doesn't have a resource type "perconaservermongodbs" Error from server (NotFound): customresourcedefinitions.apiextensions.k8s.io "null" not found Error from server (NotFound): customresourcedefinitions.apiextensions.k8s.io "null" not found Error from server (NotFound): customresourcedefinitions.apiextensions.k8s.io "null" not found Error from server (NotFound): customresourcedefinitions.apiextensions.k8s.io "null" not found ----------------------------------------------------------------------------------- destroy chaos-mesh ----------------------------------------------------------------------------------- error: resource(s) were provided, but no name was specified error: resource(s) were provided, but no name was specified error: resource(s) were provided, but no name was specified error: resource(s) were provided, but no name was specified error: resource(s) were provided, but no name was specified error: resource(s) were provided, but no name was specified ----------------------------------------------------------------------------------- cleaned up all old namespaces ----------------------------------------------------------------------------------- ----------------------------------------------------------------------------------- cleaned up old namespaces psmdb-operator ----------------------------------------------------------------------------------- error: resource(s) were provided, but no name was specified ----------------------------------------------------------------------------------- create namespace psmdb-operator ----------------------------------------------------------------------------------- namespace/psmdb-operator created Context "gke_cloud-dev-112233_us-central1-a_jen-psmdb-2221-448d41c2-6-cluster4" modified. ----------------------------------------------------------------------------------- start PSMDB operator: docker.io/perconalab/percona-server-mongodb-operator:PR-2221-448d41c2 ----------------------------------------------------------------------------------- customresourcedefinition.apiextensions.k8s.io/perconaservermongodbbackups.psmdb.percona.com serverside-applied customresourcedefinition.apiextensions.k8s.io/perconaservermongodbrestores.psmdb.percona.com serverside-applied customresourcedefinition.apiextensions.k8s.io/perconaservermongodbs.psmdb.percona.com serverside-applied clusterrole.rbac.authorization.k8s.io/percona-server-mongodb-operator created serviceaccount/percona-server-mongodb-operator created clusterrolebinding.rbac.authorization.k8s.io/service-account-percona-server-mongodb-operator created deployment.apps/percona-server-mongodb-operator created waiting for pod/percona-server-mongodb-operator-84785df65d-q22f4 to be ready.OK Print operator info from log 2026-03-10T16:43:50.828Z INFO setup Manager starting up {"gitCommit": "448d41c2c03308c0cc714f5ae30e554dba76e889", "gitBranch": "PR-2221-448d41c2", "buildTime": "", "goVersion": "go1.25.8", "os": "linux", "arch": "amd64"} ----------------------------------------------------------------------------------- destroy chaos-mesh ----------------------------------------------------------------------------------- error: resource(s) were provided, but no name was specified error: resource(s) were provided, but no name was specified error: resource(s) were provided, but no name was specified error: resource(s) were provided, but no name was specified error: resource(s) were provided, but no name was specified error: resource(s) were provided, but no name was specified ----------------------------------------------------------------------------------- cleaned up all old namespaces ----------------------------------------------------------------------------------- ----------------------------------------------------------------------------------- cleaned up old namespaces custom-replset-name-20645 ----------------------------------------------------------------------------------- error: resource(s) were provided, but no name was specified ----------------------------------------------------------------------------------- create namespace custom-replset-name-20645 ----------------------------------------------------------------------------------- namespace/custom-replset-name-20645 created Context "gke_cloud-dev-112233_us-central1-a_jen-psmdb-2221-448d41c2-6-cluster4" modified. ----------------------------------------------------------------------------------- create secrets for cloud storages ----------------------------------------------------------------------------------- secret/minio-secret created secret/aws-s3-secret created secret/gcp-cs-secret created secret/azure-secret created secret/gcp-cs-sa-key-secret created ----------------------------------------------------------------------------------- install MinIO: minio-service ----------------------------------------------------------------------------------- "minio" has been removed from your repositories "minio" has been added to your repositories NAME: minio-service LAST DEPLOYED: Tue Mar 10 16:44:28 2026 NAMESPACE: custom-replset-name-20645 STATUS: deployed REVISION: 1 TEST SUITE: None NOTES: MinIO can be accessed via port 9000 on the following DNS name from within your cluster: minio-service.custom-replset-name-20645.cluster.local To access MinIO from localhost, run the below commands: 1. export POD_NAME=$(kubectl get pods --namespace custom-replset-name-20645 -l "release=minio-service" -o jsonpath="{.items[0].metadata.name}") 2. kubectl port-forward $POD_NAME 9000 --namespace custom-replset-name-20645 Read more about port forwarding here: http://kubernetes.io/docs/user-guide/kubectl/kubectl_port-forward/ You can now access MinIO server on http://localhost:9000. Follow the below steps to connect to MinIO server with mc client: 1. Download the MinIO mc client - https://min.io/docs/minio/linux/reference/minio-mc.html#quickstart 2. export MC_HOST_minio-service-local=http://$(kubectl get secret --namespace custom-replset-name-20645 minio-service -o jsonpath="{.data.rootUser}" | base64 --decode):$(kubectl get secret --namespace custom-replset-name-20645 minio-service -o jsonpath="{.data.rootPassword}" | base64 --decode)@localhost:9000 3. mc ls minio-service-local waiting for pod/minio-service-6d5f646cdc-nzxl8 to be ready.OK service/minio-service created pod "aws-cli" deleted from custom-replset-name-20645 namespace All commands and output from this session will be recorded in container logs, including credentials and sensitive information passed through the command prompt. If you don't see a command prompt, try pressing enter. secret/some-users created deployment.apps/psmdb-client created secret/minio-secret unchanged ----------------------------------------------------------------------------------- create first PSMDB cluster ----------------------------------------------------------------------------------- perconaservermongodb.psmdb.percona.com/some-name created waiting for cluster readyness.........................OK ----------------------------------------------------------------------------------- write data, read from all ----------------------------------------------------------------------------------- Percona Server for MongoDB shell version v4.4.29-28 connecting to: mongodb://some-name-mongos.custom-replset-name-20645.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb Implicit session: session { "id" : UUID("1d5096d8-d5e7-4846-8cd1-bf9826dfd76b") } Percona Server for MongoDB server version: v8.0.19-7 WARNING: shell and server versions do not match Successfully added user: { "user" : "myApp", "roles" : [ { "db" : "myApp", "role" : "readWrite" } ] } bye Percona Server for MongoDB shell version v4.4.29-28 connecting to: mongodb://some-name-mongos.custom-replset-name-20645.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb Implicit session: session { "id" : UUID("af9ba0e4-12d0-49e5-bd16-e07742a30db3") } Percona Server for MongoDB server version: v8.0.19-7 WARNING: shell and server versions do not match switched to db myApp WriteResult({ "nInserted" : 1 }) bye waiting for pbm-agent to be ready in some-name-shard1-0...2026-03-10T16:49:19.000+0000 I listening for the commands waiting for pbm-agent to be ready in some-name-shard1-1...2026-03-10T16:49:23.000+0000 I listening for the commands waiting for pbm-agent to be ready in some-name-shard1-2...2026-03-10T16:49:35.000+0000 I listening for the commands waiting for pbm-agent to be ready in some-name-shard3-0...2026-03-10T16:49:20.000+0000 I listening for the commands waiting for pbm-agent to be ready in some-name-shard3-1...2026-03-10T16:49:24.000+0000 I listening for the commands waiting for pbm-agent to be ready in some-name-shard3-2...2026-03-10T16:49:36.000+0000 I listening for the commands waiting for pbm-agent to be ready in some-name-shard5-0...2026-03-10T16:49:22.000+0000 I listening for the commands waiting for pbm-agent to be ready in some-name-shard5-1...2026-03-10T16:49:25.000+0000 I listening for the commands waiting for pbm-agent to be ready in some-name-shard5-2...2026-03-10T16:49:37.000+0000 I listening for the commands waiting for pbm-agent to be ready in some-name-shard7-0...2026-03-10T16:49:22.000+0000 I listening for the commands waiting for pbm-agent to be ready in some-name-shard7-1...2026-03-10T16:49:26.000+0000 I listening for the commands waiting for pbm-agent to be ready in some-name-shard7-2...2026-03-10T16:49:37.000+0000 I listening for the commands ----------------------------------------------------------------------------------- run backups ----------------------------------------------------------------------------------- [2026-03-10T16:50:12+0000] running backup backup-minio perconaservermongodbbackup.psmdb.percona.com/backup-minio created waiting for backup-minio to reach ready state.....................................................................................2026-03-10T16:51:18.175Z INFO setup Manager starting up {"gitCommit": "448d41c2c03308c0cc714f5ae30e554dba76e889", "gitBranch": "PR-2221-448d41c2", "buildTime": "", "goVersion": "go1.25.8", "os": "linux", "arch": "amd64"} 2026-03-10T16:51:18.232Z INFO setup Multi-cluster services (MCS) are not available: failed to discover API resources {"error": "unable to retrieve the complete list of server APIs: metrics.k8s.io/v1beta1: stale GroupVersion discovery: metrics.k8s.io/v1beta1"} 2026-03-10T16:51:18.237Z INFO server version {"platform": "kubernetes", "version": "v1.32.12-gke.1127000"} 2026-03-10T16:51:18.258Z INFO starting server {"name": "health probe", "addr": "[::]:8081"} I0310 16:51:18.258344 1 leaderelection.go:258] "Attempting to acquire leader lease..." lock="psmdb-operator/08db0feb.percona.com" 2026-03-10T16:51:18.258Z INFO controller-runtime.metrics Starting metrics server 2026-03-10T16:51:18.258Z INFO controller-runtime.metrics Serving metrics server {"bindAddress": ":8080", "secure": false} I0310 16:51:33.540206 1 leaderelection.go:272] "Successfully acquired lease" lock="psmdb-operator/08db0feb.percona.com" 2026-03-10T16:51:33.540Z DEBUG events percona-server-mongodb-operator-84785df65d-lzzxr_7b8707d8-de9a-4eaf-a147-3b40a3b5b380 became leader {"type": "Normal", "object": {"kind":"Lease","namespace":"psmdb-operator","name":"08db0feb.percona.com","uid":"e252e0e9-8127-4ab8-86b6-e7f1d00e7df8","apiVersion":"coordination.k8s.io/v1","resourceVersion":"1773161493534351015"}, "reason": "LeaderElection"} 2026-03-10T16:51:33.540Z INFO Starting EventSource {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "source": "kind source: *v1.PerconaServerMongoDB"} 2026-03-10T16:51:33.541Z INFO Starting EventSource {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "source": "kind source: *v1.Pod"} 2026-03-10T16:51:33.541Z INFO Starting EventSource {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "source": "kind source: *v1.Pod"} 2026-03-10T16:51:33.541Z INFO Starting EventSource {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "source": "kind source: *v1.PerconaServerMongoDBRestore"} 2026-03-10T16:51:33.541Z INFO Starting EventSource {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "source": "kind source: *v1.PerconaServerMongoDBBackup"} 2026-03-10T16:51:33.653Z INFO Starting Controller {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB"} 2026-03-10T16:51:33.654Z INFO Starting workers {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "worker count": 1} 2026-03-10T16:51:33.754Z INFO Starting Controller {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup"} 2026-03-10T16:51:33.754Z INFO Starting workers {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "worker count": 1} 2026-03-10T16:51:33.754Z DEBUG Reconciling {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio","namespace":"custom-replset-name-20645"}, "namespace": "custom-replset-name-20645", "name": "backup-minio", "reconcileID": "085afa5a-1f83-413a-a8f6-5b199ffa1a65"} 2026-03-10T16:51:33.754Z INFO Starting Controller {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore"} 2026-03-10T16:51:33.755Z INFO Starting workers {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "worker count": 1} 2026-03-10T16:51:33.764Z DEBUG Got object from API server {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio","namespace":"custom-replset-name-20645"}, "namespace": "custom-replset-name-20645", "name": "backup-minio", "reconcileID": "085afa5a-1f83-413a-a8f6-5b199ffa1a65", "state": "running"} 2026-03-10T16:51:46.853Z ERROR FULL CLUSTER CRASH {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"custom-replset-name-20645"}, "namespace": "custom-replset-name-20645", "name": "some-name", "reconcileID": "5a570422-e904-48c5-a61a-07115c8cb22b", "error": "ping mongo: server selection error: server selection timeout, current topology: { Type: ReplicaSetNoPrimary, Servers: [{ Addr: some-name-cfg-0.some-name-cfg.custom-replset-name-20645.svc.cluster.local:27017, Type: Unknown }, { Addr: some-name-cfg-1.some-name-cfg.custom-replset-name-20645.svc.cluster.local:27017, Type: Unknown }, { Addr: some-name-cfg-2.some-name-cfg.custom-replset-name-20645.svc.cluster.local:27017, Type: Unknown }, ] }", "errorVerbose": "server selection error: server selection timeout, current topology: { Type: ReplicaSetNoPrimary, Servers: [{ Addr: some-name-cfg-0.some-name-cfg.custom-replset-name-20645.svc.cluster.local:27017, Type: Unknown }, { Addr: some-name-cfg-1.some-name-cfg.custom-replset-name-20645.svc.cluster.local:27017, Type: Unknown }, { Addr: some-name-cfg-2.some-name-cfg.custom-replset-name-20645.svc.cluster.local:27017, Type: Unknown }, ] }\nping mongo\ngithub.com/percona/percona-server-mongodb-operator/pkg/psmdb/mongo.Dial\n\t/go/src/github.com/percona/percona-server-mongodb-operator/pkg/psmdb/mongo/mongo.go:126\ngithub.com/percona/percona-server-mongodb-operator/pkg/psmdb.MongoClient\n\t/go/src/github.com/percona/percona-server-mongodb-operator/pkg/psmdb/client.go:62\ngithub.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodb.(*mongoClientProvider).Mongo\n\t/go/src/github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodb/connections.go:38\ngithub.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodb.(*ReconcilePerconaServerMongoDB).mongoClientWithRole\n\t/go/src/github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodb/connections.go:60\ngithub.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodb.(*ReconcilePerconaServerMongoDB).reconcileCluster\n\t/go/src/github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodb/mgo.go:94\ngithub.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodb.(*ReconcilePerconaServerMongoDB).reconcileReplsets\n\t/go/src/github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodb/psmdb_controller.go:616\ngithub.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodb.(*ReconcilePerconaServerMongoDB).Reconcile\n\t/go/src/github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodb/psmdb_controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:222\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:479\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:438\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:313\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1693"} github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodb.(*ReconcilePerconaServerMongoDB).reconcileCluster /go/src/github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodb/mgo.go:101 github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodb.(*ReconcilePerconaServerMongoDB).reconcileReplsets /go/src/github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodb/psmdb_controller.go:616 github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodb.(*ReconcilePerconaServerMongoDB).Reconcile /go/src/github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodb/psmdb_controller.go:474 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile /go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:222 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler /go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:479 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem /go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:438 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1 /go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:313 2026-03-10T16:51:46.854Z INFO handleReplicaSetNoPrimary Connecting to pod {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"custom-replset-name-20645"}, "namespace": "custom-replset-name-20645", "name": "some-name", "reconcileID": "5a570422-e904-48c5-a61a-07115c8cb22b", "pod": "some-name-cfg-1", "user": "clusterAdmin"} 2026-03-10T16:51:56.873Z ERROR failed to reconcile cluster {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"custom-replset-name-20645"}, "namespace": "custom-replset-name-20645", "name": "some-name", "reconcileID": "5a570422-e904-48c5-a61a-07115c8cb22b", "replset": "cfg", "error": "handle ReplicaSetNoPrimary: get standalone mongo client: ping mongo: timed out while checking out a connection from connection pool: context deadline exceeded; total connections: 1, maxPoolSize: 100, idle connections: 0, wait duration: 9.989427786s", "errorVerbose": "timed out while checking out a connection from connection pool: context deadline exceeded; total connections: 1, maxPoolSize: 100, idle connections: 0, wait duration: 9.989427786s\nping mongo\ngithub.com/percona/percona-server-mongodb-operator/pkg/psmdb/mongo.Dial\n\t/go/src/github.com/percona/percona-server-mongodb-operator/pkg/psmdb/mongo/mongo.go:126\ngithub.com/percona/percona-server-mongodb-operator/pkg/psmdb.StandaloneClient\n\t/go/src/github.com/percona/percona-server-mongodb-operator/pkg/psmdb/client.go:105\ngithub.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodb.(*mongoClientProvider).Standalone\n\t/go/src/github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodb/connections.go:56\ngithub.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodb.(*ReconcilePerconaServerMongoDB).standaloneClientWithRole\n\t/go/src/github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodb/connections.go:72\ngithub.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodb.(*ReconcilePerconaServerMongoDB).handleReplicaSetNoPrimary\n\t/go/src/github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodb/mgo.go:785\ngithub.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodb.(*ReconcilePerconaServerMongoDB).reconcileCluster\n\t/go/src/github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodb/mgo.go:103\ngithub.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodb.(*ReconcilePerconaServerMongoDB).reconcileReplsets\n\t/go/src/github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodb/psmdb_controller.go:616\ngithub.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodb.(*ReconcilePerconaServerMongoDB).Reconcile\n\t/go/src/github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodb/psmdb_controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:222\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:479\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:438\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:313\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1693\nget standalone mongo client\ngithub.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodb.(*ReconcilePerconaServerMongoDB).handleReplicaSetNoPrimary\n\t/go/src/github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodb/mgo.go:787\ngithub.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodb.(*ReconcilePerconaServerMongoDB).reconcileCluster\n\t/go/src/github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodb/mgo.go:103\ngithub.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodb.(*ReconcilePerconaServerMongoDB).reconcileReplsets\n\t/go/src/github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodb/psmdb_controller.go:616\ngithub.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodb.(*ReconcilePerconaServerMongoDB).Reconcile\n\t/go/src/github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodb/psmdb_controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:222\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:479\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:438\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:313\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1693\nhandle ReplicaSetNoPrimary\ngithub.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodb.(*ReconcilePerconaServerMongoDB).reconcileCluster\n\t/go/src/github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodb/mgo.go:105\ngithub.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodb.(*ReconcilePerconaServerMongoDB).reconcileReplsets\n\t/go/src/github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodb/psmdb_controller.go:616\ngithub.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodb.(*ReconcilePerconaServerMongoDB).Reconcile\n\t/go/src/github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodb/psmdb_controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:222\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:479\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:438\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:313\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1693"} github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodb.(*ReconcilePerconaServerMongoDB).reconcileReplsets /go/src/github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodb/psmdb_controller.go:618 github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodb.(*ReconcilePerconaServerMongoDB).Reconcile /go/src/github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodb/psmdb_controller.go:474 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile /go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:222 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler /go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:479 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem /go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:438 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1 /go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:313 2026-03-10T16:51:56.873Z DEBUG Replset status is not healthy {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"custom-replset-name-20645"}, "namespace": "custom-replset-name-20645", "name": "some-name", "reconcileID": "5a570422-e904-48c5-a61a-07115c8cb22b", "replset": "cfg", "status": "error"} 2026-03-10T16:52:07.038Z ERROR FULL CLUSTER CRASH {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"custom-replset-name-20645"}, "namespace": "custom-replset-name-20645", "name": "some-name", "reconcileID": "5a570422-e904-48c5-a61a-07115c8cb22b", "error": "ping mongo: server selection error: server selection timeout, current topology: { Type: ReplicaSetNoPrimary, Servers: [{ Addr: some-name-shard1-0.some-name-shard1.custom-replset-name-20645.svc.cluster.local:27017, Type: Unknown }, { Addr: some-name-shard1-1.some-name-shard1.custom-replset-name-20645.svc.cluster.local:27017, Type: Unknown }, { Addr: some-name-shard1-2.some-name-shard1.custom-replset-name-20645.svc.cluster.local:27017, Type: Unknown }, ] }", "errorVerbose": "server selection error: server selection timeout, current topology: { Type: ReplicaSetNoPrimary, Servers: [{ Addr: some-name-shard1-0.some-name-shard1.custom-replset-name-20645.svc.cluster.local:27017, Type: Unknown }, { Addr: some-name-shard1-1.some-name-shard1.custom-replset-name-20645.svc.cluster.local:27017, Type: Unknown }, { Addr: some-name-shard1-2.some-name-shard1.custom-replset-name-20645.svc.cluster.local:27017, Type: Unknown }, ] }\nping mongo\ngithub.com/percona/percona-server-mongodb-operator/pkg/psmdb/mongo.Dial\n\t/go/src/github.com/percona/percona-server-mongodb-operator/pkg/psmdb/mongo/mongo.go:126\ngithub.com/percona/percona-server-mongodb-operator/pkg/psmdb.MongoClient\n\t/go/src/github.com/percona/percona-server-mongodb-operator/pkg/psmdb/client.go:62\ngithub.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodb.(*mongoClientProvider).Mongo\n\t/go/src/github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodb/connections.go:38\ngithub.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodb.(*ReconcilePerconaServerMongoDB).mongoClientWithRole\n\t/go/src/github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodb/connections.go:60\ngithub.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodb.(*ReconcilePerconaServerMongoDB).reconcileCluster\n\t/go/src/github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodb/mgo.go:94\ngithub.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodb.(*ReconcilePerconaServerMongoDB).reconcileReplsets\n\t/go/src/github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodb/psmdb_controller.go:616\ngithub.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodb.(*ReconcilePerconaServerMongoDB).Reconcile\n\t/go/src/github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodb/psmdb_controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:222\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:479\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:438\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:313\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1693"} github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodb.(*ReconcilePerconaServerMongoDB).reconcileCluster /go/src/github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodb/mgo.go:101 github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodb.(*ReconcilePerconaServerMongoDB).reconcileReplsets /go/src/github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodb/psmdb_controller.go:616 github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodb.(*ReconcilePerconaServerMongoDB).Reconcile /go/src/github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodb/psmdb_controller.go:474 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile /go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:222 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler /go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:479 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem /go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:438 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1 /go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:313 2026-03-10T16:52:07.039Z INFO handleReplicaSetNoPrimary Connecting to pod {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"custom-replset-name-20645"}, "namespace": "custom-replset-name-20645", "name": "some-name", "reconcileID": "5a570422-e904-48c5-a61a-07115c8cb22b", "pod": "some-name-shard1-0", "user": "clusterAdmin"} 2026-03-10T16:52:12.109Z DEBUG Running replSetReconfig config {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"custom-replset-name-20645"}, "namespace": "custom-replset-name-20645", "name": "some-name", "reconcileID": "5a570422-e904-48c5-a61a-07115c8cb22b", "cfg": {"_id":"shard1","version":7,"members":[{"_id":0,"host":"some-name-shard1-0.some-name-shard1.custom-replset-name-20645.svc.cluster.local:27017","arbiterOnly":false,"buildIndexes":true,"hidden":false,"priority":2,"tags":{"nodeName":"gke-jen-psmdb-2221-448d4-default-pool-99ee3404-2txw","podName":"some-name-shard1-0","region":"us-central1","serviceName":"some-name","zone":"us-central1-a"},"secondaryDelaySecs":0,"votes":1},{"_id":1,"host":"some-name-shard1-1.some-name-shard1.custom-replset-name-20645.svc.cluster.local:27017","arbiterOnly":false,"buildIndexes":true,"hidden":false,"priority":2,"tags":{"nodeName":"gke-jen-psmdb-2221-448d4-default-pool-99ee3404-tf85","podName":"some-name-shard1-1","region":"us-central1","serviceName":"some-name","zone":"us-central1-a"},"secondaryDelaySecs":0,"votes":1},{"_id":2,"host":"some-name-shard1-2.some-name-shard1.custom-replset-name-20645.svc.cluster.local:27017","arbiterOnly":false,"buildIndexes":true,"hidden":false,"priority":2,"tags":{"nodeName":"gke-jen-psmdb-2221-448d4-default-pool-99ee3404-tf85","podName":"some-name-shard1-2","region":"us-central1","serviceName":"some-name","zone":"us-central1-a"},"secondaryDelaySecs":0,"votes":1}],"protocolVersion":1,"settings":{"chainingAllowed":true,"heartbeatIntervalMillis":2000,"heartbeatTimeoutSecs":10,"electionTimeoutMillis":10000,"catchUpTimeoutMillis":-1,"getLastErrorDefaults":{"w":1,"wtimeout":0},"replicaSetId":"69b04b52a1deb5441abc8ba7"},"writeConcernMajorityJournalDefault":true}} 2026-03-10T16:52:12.116Z DEBUG Replset status is not healthy {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"custom-replset-name-20645"}, "namespace": "custom-replset-name-20645", "name": "some-name", "reconcileID": "5a570422-e904-48c5-a61a-07115c8cb22b", "replset": "shard1", "status": "error"} 2026-03-10T16:52:22.280Z ERROR FULL CLUSTER CRASH {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"custom-replset-name-20645"}, "namespace": "custom-replset-name-20645", "name": "some-name", "reconcileID": "5a570422-e904-48c5-a61a-07115c8cb22b", "error": "ping mongo: server selection error: server selection timeout, current topology: { Type: ReplicaSetNoPrimary, Servers: [{ Addr: some-name-shard3-0.some-name-shard3.custom-replset-name-20645.svc.cluster.local:27017, Type: RSSecondary, Tag sets: serviceName=some-name,podName=some-name-shard3-0,zone=us-central1-a,region=us-central1,nodeName=gke-jen-psmdb-2221-448d4-default-pool-99ee3404-2txw, Average RTT: 1627537 }, { Addr: some-name-shard3-1.some-name-shard3.custom-replset-name-20645.svc.cluster.local:27017, Type: Unknown, Last error: dial tcp: lookup some-name-shard3-1.some-name-shard3.custom-replset-name-20645.svc.cluster.local on 34.118.224.10:53: no such host }, { Addr: some-name-shard3-2.some-name-shard3.custom-replset-name-20645.svc.cluster.local:27017, Type: Unknown, Last error: dial tcp: lookup some-name-shard3-2.some-name-shard3.custom-replset-name-20645.svc.cluster.local on 34.118.224.10:53: no such host }, ] }", "errorVerbose": "server selection error: server selection timeout, current topology: { Type: ReplicaSetNoPrimary, Servers: [{ Addr: some-name-shard3-0.some-name-shard3.custom-replset-name-20645.svc.cluster.local:27017, Type: RSSecondary, Tag sets: serviceName=some-name,podName=some-name-shard3-0,zone=us-central1-a,region=us-central1,nodeName=gke-jen-psmdb-2221-448d4-default-pool-99ee3404-2txw, Average RTT: 1627537 }, { Addr: some-name-shard3-1.some-name-shard3.custom-replset-name-20645.svc.cluster.local:27017, Type: Unknown, Last error: dial tcp: lookup some-name-shard3-1.some-name-shard3.custom-replset-name-20645.svc.cluster.local on 34.118.224.10:53: no such host }, { Addr: some-name-shard3-2.some-name-shard3.custom-replset-name-20645.svc.cluster.local:27017, Type: Unknown, Last error: dial tcp: lookup some-name-shard3-2.some-name-shard3.custom-replset-name-20645.svc.cluster.local on 34.118.224.10:53: no such host }, ] }\nping mongo\ngithub.com/percona/percona-server-mongodb-operator/pkg/psmdb/mongo.Dial\n\t/go/src/github.com/percona/percona-server-mongodb-operator/pkg/psmdb/mongo/mongo.go:126\ngithub.com/percona/percona-server-mongodb-operator/pkg/psmdb.MongoClient\n\t/go/src/github.com/percona/percona-server-mongodb-operator/pkg/psmdb/client.go:62\ngithub.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodb.(*mongoClientProvider).Mongo\n\t/go/src/github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodb/connections.go:38\ngithub.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodb.(*ReconcilePerconaServerMongoDB).mongoClientWithRole\n\t/go/src/github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodb/connections.go:60\ngithub.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodb.(*ReconcilePerconaServerMongoDB).reconcileCluster\n\t/go/src/github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodb/mgo.go:94\ngithub.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodb.(*ReconcilePerconaServerMongoDB).reconcileReplsets\n\t/go/src/github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodb/psmdb_controller.go:616\ngithub.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodb.(*ReconcilePerconaServerMongoDB).Reconcile\n\t/go/src/github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodb/psmdb_controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:222\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:479\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:438\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:313\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1693"} github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodb.(*ReconcilePerconaServerMongoDB).reconcileCluster /go/src/github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodb/mgo.go:101 github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodb.(*ReconcilePerconaServerMongoDB).reconcileReplsets /go/src/github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodb/psmdb_controller.go:616 github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodb.(*ReconcilePerconaServerMongoDB).Reconcile /go/src/github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodb/psmdb_controller.go:474 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile /go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:222 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler /go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:479 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem /go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:438 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1 /go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:313 2026-03-10T16:52:22.280Z INFO handleReplicaSetNoPrimary Connecting to pod {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"custom-replset-name-20645"}, "namespace": "custom-replset-name-20645", "name": "some-name", "reconcileID": "5a570422-e904-48c5-a61a-07115c8cb22b", "pod": "some-name-shard3-0", "user": "clusterAdmin"} 2026-03-10T16:52:22.498Z DEBUG Running replSetReconfig config {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"custom-replset-name-20645"}, "namespace": "custom-replset-name-20645", "name": "some-name", "reconcileID": "5a570422-e904-48c5-a61a-07115c8cb22b", "cfg": {"_id":"shard3","version":7,"members":[{"_id":0,"host":"some-name-shard3-0.some-name-shard3.custom-replset-name-20645.svc.cluster.local:27017","arbiterOnly":false,"buildIndexes":true,"hidden":false,"priority":2,"tags":{"nodeName":"gke-jen-psmdb-2221-448d4-default-pool-99ee3404-2txw","podName":"some-name-shard3-0","region":"us-central1","serviceName":"some-name","zone":"us-central1-a"},"secondaryDelaySecs":0,"votes":1},{"_id":1,"host":"some-name-shard3-1.some-name-shard3.custom-replset-name-20645.svc.cluster.local:27017","arbiterOnly":false,"buildIndexes":true,"hidden":false,"priority":2,"tags":{"nodeName":"gke-jen-psmdb-2221-448d4-default-pool-99ee3404-wpld","podName":"some-name-shard3-1","region":"us-central1","serviceName":"some-name","zone":"us-central1-a"},"secondaryDelaySecs":0,"votes":1},{"_id":2,"host":"some-name-shard3-2.some-name-shard3.custom-replset-name-20645.svc.cluster.local:27017","arbiterOnly":false,"buildIndexes":true,"hidden":false,"priority":2,"tags":{"nodeName":"gke-jen-psmdb-2221-448d4-default-pool-99ee3404-tf85","podName":"some-name-shard3-2","region":"us-central1","serviceName":"some-name","zone":"us-central1-a"},"secondaryDelaySecs":0,"votes":1}],"protocolVersion":1,"settings":{"chainingAllowed":true,"heartbeatIntervalMillis":2000,"heartbeatTimeoutSecs":10,"electionTimeoutMillis":10000,"catchUpTimeoutMillis":-1,"getLastErrorDefaults":{"w":1,"wtimeout":0},"replicaSetId":"69b04b718d9c1c741f0ee2a4"},"writeConcernMajorityJournalDefault":true}} 2026-03-10T16:52:22.506Z DEBUG Replset status is not healthy {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"custom-replset-name-20645"}, "namespace": "custom-replset-name-20645", "name": "some-name", "reconcileID": "5a570422-e904-48c5-a61a-07115c8cb22b", "replset": "shard3", "status": "error"} 2026-03-10T16:52:25.041Z INFO Tags changed {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"custom-replset-name-20645"}, "namespace": "custom-replset-name-20645", "name": "some-name", "reconcileID": "5a570422-e904-48c5-a61a-07115c8cb22b", "host": "some-name-shard5-0.some-name-shard5.custom-replset-name-20645.svc.cluster.local:27017", "old": {"nodeName":"gke-jen-psmdb-2221-448d4-default-pool-99ee3404-tf85","podName":"some-name-shard5-0","region":"us-central1","serviceName":"some-name","zone":"us-central1-a"}, "new": {"nodeName":"gke-jen-psmdb-2221-448d4-default-pool-99ee3404-2txw","podName":"some-name-shard5-0","region":"us-central1","serviceName":"some-name","zone":"us-central1-a"}} 2026-03-10T16:52:25.042Z INFO Fixing member configurations {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"custom-replset-name-20645"}, "namespace": "custom-replset-name-20645", "name": "some-name", "reconcileID": "5a570422-e904-48c5-a61a-07115c8cb22b", "replset": "shard5"} 2026-03-10T16:52:25.042Z DEBUG Running replSetReconfig config {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"custom-replset-name-20645"}, "namespace": "custom-replset-name-20645", "name": "some-name", "reconcileID": "5a570422-e904-48c5-a61a-07115c8cb22b", "cfg": {"_id":"shard5","version":8,"members":[{"_id":0,"host":"some-name-shard5-0.some-name-shard5.custom-replset-name-20645.svc.cluster.local:27017","arbiterOnly":false,"buildIndexes":true,"hidden":false,"priority":2,"tags":{"nodeName":"gke-jen-psmdb-2221-448d4-default-pool-99ee3404-2txw","podName":"some-name-shard5-0","region":"us-central1","serviceName":"some-name","zone":"us-central1-a"},"secondaryDelaySecs":0,"votes":1},{"_id":1,"host":"some-name-shard5-1.some-name-shard5.custom-replset-name-20645.svc.cluster.local:27017","arbiterOnly":false,"buildIndexes":true,"hidden":false,"priority":2,"tags":{"nodeName":"gke-jen-psmdb-2221-448d4-default-pool-99ee3404-2txw","podName":"some-name-shard5-1","region":"us-central1","serviceName":"some-name","zone":"us-central1-a"},"secondaryDelaySecs":0,"votes":1},{"_id":2,"host":"some-name-shard5-2.some-name-shard5.custom-replset-name-20645.svc.cluster.local:27017","arbiterOnly":false,"buildIndexes":true,"hidden":false,"priority":2,"tags":{"nodeName":"gke-jen-psmdb-2221-448d4-default-pool-99ee3404-2txw","podName":"some-name-shard5-2","region":"us-central1","serviceName":"some-name","zone":"us-central1-a"},"secondaryDelaySecs":0,"votes":1}],"protocolVersion":1,"settings":{"chainingAllowed":true,"heartbeatIntervalMillis":2000,"heartbeatTimeoutSecs":10,"electionTimeoutMillis":10000,"catchUpTimeoutMillis":-1,"getLastErrorDefaults":{"w":1,"wtimeout":0},"replicaSetId":"69b04b1d4b21fa2d340b4429"},"writeConcernMajorityJournalDefault":true}} 2026-03-10T16:52:25.420Z DEBUG Replset is not ready {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"custom-replset-name-20645"}, "namespace": "custom-replset-name-20645", "name": "some-name", "reconcileID": "5a570422-e904-48c5-a61a-07115c8cb22b", "liveMembers": 2, "pods": 3} 2026-03-10T16:52:25.427Z DEBUG Replset status is not healthy {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"custom-replset-name-20645"}, "namespace": "custom-replset-name-20645", "name": "some-name", "reconcileID": "5a570422-e904-48c5-a61a-07115c8cb22b", "replset": "shard5", "status": "initializing"} 2026-03-10T16:52:35.666Z ERROR FULL CLUSTER CRASH {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"custom-replset-name-20645"}, "namespace": "custom-replset-name-20645", "name": "some-name", "reconcileID": "5a570422-e904-48c5-a61a-07115c8cb22b", "error": "ping mongo: server selection error: server selection timeout, current topology: { Type: ReplicaSetNoPrimary, Servers: [{ Addr: some-name-shard7-0.some-name-shard7.custom-replset-name-20645.svc.cluster.local:27017, Type: RSSecondary, Tag sets: region=us-central1,podName=some-name-shard7-0,nodeName=gke-jen-psmdb-2221-448d4-default-pool-99ee3404-2txw,serviceName=some-name,zone=us-central1-a, Average RTT: 7430147 }, { Addr: some-name-shard7-1.some-name-shard7.custom-replset-name-20645.svc.cluster.local:27017, Type: RSSecondary, Tag sets: zone=us-central1-a,nodeName=gke-jen-psmdb-2221-448d4-default-pool-99ee3404-tf85,podName=some-name-shard7-1,serviceName=some-name,region=us-central1, Average RTT: 1533901 }, { Addr: some-name-shard7-2.some-name-shard7.custom-replset-name-20645.svc.cluster.local:27017, Type: Unknown, Last error: dial tcp: lookup some-name-shard7-2.some-name-shard7.custom-replset-name-20645.svc.cluster.local on 34.118.224.10:53: no such host }, ] }", "errorVerbose": "server selection error: server selection timeout, current topology: { Type: ReplicaSetNoPrimary, Servers: [{ Addr: some-name-shard7-0.some-name-shard7.custom-replset-name-20645.svc.cluster.local:27017, Type: RSSecondary, Tag sets: region=us-central1,podName=some-name-shard7-0,nodeName=gke-jen-psmdb-2221-448d4-default-pool-99ee3404-2txw,serviceName=some-name,zone=us-central1-a, Average RTT: 7430147 }, { Addr: some-name-shard7-1.some-name-shard7.custom-replset-name-20645.svc.cluster.local:27017, Type: RSSecondary, Tag sets: zone=us-central1-a,nodeName=gke-jen-psmdb-2221-448d4-default-pool-99ee3404-tf85,podName=some-name-shard7-1,serviceName=some-name,region=us-central1, Average RTT: 1533901 }, { Addr: some-name-shard7-2.some-name-shard7.custom-replset-name-20645.svc.cluster.local:27017, Type: Unknown, Last error: dial tcp: lookup some-name-shard7-2.some-name-shard7.custom-replset-name-20645.svc.cluster.local on 34.118.224.10:53: no such host }, ] }\nping mongo\ngithub.com/percona/percona-server-mongodb-operator/pkg/psmdb/mongo.Dial\n\t/go/src/github.com/percona/percona-server-mongodb-operator/pkg/psmdb/mongo/mongo.go:126\ngithub.com/percona/percona-server-mongodb-operator/pkg/psmdb.MongoClient\n\t/go/src/github.com/percona/percona-server-mongodb-operator/pkg/psmdb/client.go:62\ngithub.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodb.(*mongoClientProvider).Mongo\n\t/go/src/github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodb/connections.go:38\ngithub.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodb.(*ReconcilePerconaServerMongoDB).mongoClientWithRole\n\t/go/src/github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodb/connections.go:60\ngithub.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodb.(*ReconcilePerconaServerMongoDB).reconcileCluster\n\t/go/src/github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodb/mgo.go:94\ngithub.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodb.(*ReconcilePerconaServerMongoDB).reconcileReplsets\n\t/go/src/github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodb/psmdb_controller.go:616\ngithub.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodb.(*ReconcilePerconaServerMongoDB).Reconcile\n\t/go/src/github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodb/psmdb_controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:222\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:479\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:438\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:313\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1693"} github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodb.(*ReconcilePerconaServerMongoDB).reconcileCluster /go/src/github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodb/mgo.go:101 github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodb.(*ReconcilePerconaServerMongoDB).reconcileReplsets /go/src/github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodb/psmdb_controller.go:616 github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodb.(*ReconcilePerconaServerMongoDB).Reconcile /go/src/github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodb/psmdb_controller.go:474 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile /go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:222 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler /go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:479 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem /go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:438 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1 /go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:313 2026-03-10T16:52:35.666Z INFO handleReplicaSetNoPrimary Connecting to pod {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"custom-replset-name-20645"}, "namespace": "custom-replset-name-20645", "name": "some-name", "reconcileID": "5a570422-e904-48c5-a61a-07115c8cb22b", "pod": "some-name-shard7-0", "user": "clusterAdmin"} 2026-03-10T16:52:35.720Z DEBUG Running replSetReconfig config {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"custom-replset-name-20645"}, "namespace": "custom-replset-name-20645", "name": "some-name", "reconcileID": "5a570422-e904-48c5-a61a-07115c8cb22b", "cfg": {"_id":"shard7","version":7,"members":[{"_id":0,"host":"some-name-shard7-0.some-name-shard7.custom-replset-name-20645.svc.cluster.local:27017","arbiterOnly":false,"buildIndexes":true,"hidden":false,"priority":2,"tags":{"nodeName":"gke-jen-psmdb-2221-448d4-default-pool-99ee3404-2txw","podName":"some-name-shard7-0","region":"us-central1","serviceName":"some-name","zone":"us-central1-a"},"secondaryDelaySecs":0,"votes":1},{"_id":1,"host":"some-name-shard7-1.some-name-shard7.custom-replset-name-20645.svc.cluster.local:27017","arbiterOnly":false,"buildIndexes":true,"hidden":false,"priority":2,"tags":{"nodeName":"gke-jen-psmdb-2221-448d4-default-pool-99ee3404-tf85","podName":"some-name-shard7-1","region":"us-central1","serviceName":"some-name","zone":"us-central1-a"},"secondaryDelaySecs":0,"votes":1},{"_id":2,"host":"some-name-shard7-2.some-name-shard7.custom-replset-name-20645.svc.cluster.local:27017","arbiterOnly":false,"buildIndexes":true,"hidden":false,"priority":2,"tags":{"nodeName":"gke-jen-psmdb-2221-448d4-default-pool-99ee3404-tf85","podName":"some-name-shard7-2","region":"us-central1","serviceName":"some-name","zone":"us-central1-a"},"secondaryDelaySecs":0,"votes":1}],"protocolVersion":1,"settings":{"chainingAllowed":true,"heartbeatIntervalMillis":2000,"heartbeatTimeoutSecs":10,"electionTimeoutMillis":10000,"catchUpTimeoutMillis":-1,"getLastErrorDefaults":{"w":1,"wtimeout":0},"replicaSetId":"69b04b314d442c2c9f0febdd"},"writeConcernMajorityJournalDefault":true}} 2026-03-10T16:52:35.726Z DEBUG Replset status is not healthy {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"custom-replset-name-20645"}, "namespace": "custom-replset-name-20645", "name": "some-name", "reconcileID": "5a570422-e904-48c5-a61a-07115c8cb22b", "replset": "shard7", "status": "error"} 2026-03-10T16:52:35.798Z ERROR Reconciler error {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"custom-replset-name-20645"}, "namespace": "custom-replset-name-20645", "name": "some-name", "reconcileID": "5a570422-e904-48c5-a61a-07115c8cb22b", "error": "reconcile statefulsets: handle ReplicaSetNoPrimary: get standalone mongo client: ping mongo: timed out while checking out a connection from connection pool: context deadline exceeded; total connections: 1, maxPoolSize: 100, idle connections: 0, wait duration: 9.989427786s", "errorVerbose": "handle ReplicaSetNoPrimary: get standalone mongo client: ping mongo: timed out while checking out a connection from connection pool: context deadline exceeded; total connections: 1, maxPoolSize: 100, idle connections: 0, wait duration: 9.989427786s\nreconcile statefulsets\ngithub.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodb.(*ReconcilePerconaServerMongoDB).Reconcile\n\t/go/src/github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodb/psmdb_controller.go:476\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:222\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:479\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:438\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:313\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1693"} sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler /go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:495 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem /go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:438 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1 /go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:313 2026-03-10T16:52:39.461Z INFO Tags changed {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"custom-replset-name-20645"}, "namespace": "custom-replset-name-20645", "name": "some-name", "reconcileID": "55cc94ce-9852-44d3-8d98-457d82efe522", "host": "some-name-cfg-0.some-name-cfg.custom-replset-name-20645.svc.cluster.local:27017", "old": {"nodeName":"gke-jen-psmdb-2221-448d4-default-pool-99ee3404-wpld","podName":"some-name-cfg-0","region":"us-central1","serviceName":"some-name","zone":"us-central1-a"}, "new": {"nodeName":"gke-jen-psmdb-2221-448d4-default-pool-99ee3404-2txw","podName":"some-name-cfg-0","region":"us-central1","serviceName":"some-name","zone":"us-central1-a"}} 2026-03-10T16:52:39.461Z INFO Fixing member configurations {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"custom-replset-name-20645"}, "namespace": "custom-replset-name-20645", "name": "some-name", "reconcileID": "55cc94ce-9852-44d3-8d98-457d82efe522", "replset": "cfg"} 2026-03-10T16:52:39.461Z DEBUG Running replSetReconfig config {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"custom-replset-name-20645"}, "namespace": "custom-replset-name-20645", "name": "some-name", "reconcileID": "55cc94ce-9852-44d3-8d98-457d82efe522", "cfg": {"_id":"csReplSet","version":9,"members":[{"_id":0,"host":"some-name-cfg-0.some-name-cfg.custom-replset-name-20645.svc.cluster.local:27017","arbiterOnly":false,"buildIndexes":true,"hidden":false,"priority":2,"tags":{"nodeName":"gke-jen-psmdb-2221-448d4-default-pool-99ee3404-2txw","podName":"some-name-cfg-0","region":"us-central1","serviceName":"some-name","zone":"us-central1-a"},"secondaryDelaySecs":0,"votes":1},{"_id":1,"host":"some-name-cfg-1.some-name-cfg.custom-replset-name-20645.svc.cluster.local:27017","arbiterOnly":false,"buildIndexes":true,"hidden":false,"priority":2,"tags":{"nodeName":"gke-jen-psmdb-2221-448d4-default-pool-99ee3404-2txw","podName":"some-name-cfg-1","region":"us-central1","serviceName":"some-name","zone":"us-central1-a"},"secondaryDelaySecs":0,"votes":1},{"_id":2,"host":"some-name-cfg-2.some-name-cfg.custom-replset-name-20645.svc.cluster.local:27017","arbiterOnly":false,"buildIndexes":true,"hidden":false,"priority":2,"tags":{"nodeName":"gke-jen-psmdb-2221-448d4-default-pool-99ee3404-2txw","podName":"some-name-cfg-2","region":"us-central1","serviceName":"some-name","zone":"us-central1-a"},"secondaryDelaySecs":0,"votes":1}],"configsvr":true,"protocolVersion":1,"settings":{"chainingAllowed":true,"heartbeatIntervalMillis":2000,"heartbeatTimeoutSecs":10,"electionTimeoutMillis":10000,"catchUpTimeoutMillis":-1,"getLastErrorDefaults":{"w":1,"wtimeout":0},"replicaSetId":"69b04b43301a62a52f55034f"},"writeConcernMajorityJournalDefault":true}} 2026-03-10T16:52:39.522Z DEBUG Replset is not ready {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"custom-replset-name-20645"}, "namespace": "custom-replset-name-20645", "name": "some-name", "reconcileID": "55cc94ce-9852-44d3-8d98-457d82efe522", "liveMembers": 2, "pods": 3} 2026-03-10T16:52:39.525Z DEBUG Replset status is not healthy {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"custom-replset-name-20645"}, "namespace": "custom-replset-name-20645", "name": "some-name", "reconcileID": "55cc94ce-9852-44d3-8d98-457d82efe522", "replset": "cfg", "status": "initializing"} 2026-03-10T16:52:39.572Z INFO Waiting for the pods {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"custom-replset-name-20645"}, "namespace": "custom-replset-name-20645", "name": "some-name", "reconcileID": "55cc94ce-9852-44d3-8d98-457d82efe522", "replset": "shard1", "size": 3, "pods": 2} 2026-03-10T16:52:39.572Z DEBUG Replset status is not healthy {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"custom-replset-name-20645"}, "namespace": "custom-replset-name-20645", "name": "some-name", "reconcileID": "55cc94ce-9852-44d3-8d98-457d82efe522", "replset": "shard1", "status": "initializing"} 2026-03-10T16:52:45.728Z INFO Tags changed {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"custom-replset-name-20645"}, "namespace": "custom-replset-name-20645", "name": "some-name", "reconcileID": "55cc94ce-9852-44d3-8d98-457d82efe522", "host": "some-name-shard3-1.some-name-shard3.custom-replset-name-20645.svc.cluster.local:27017", "old": {"nodeName":"gke-jen-psmdb-2221-448d4-default-pool-99ee3404-wpld","podName":"some-name-shard3-1","region":"us-central1","serviceName":"some-name","zone":"us-central1-a"}, "new": {"nodeName":"gke-jen-psmdb-2221-448d4-default-pool-99ee3404-2txw","podName":"some-name-shard3-1","region":"us-central1","serviceName":"some-name","zone":"us-central1-a"}} 2026-03-10T16:52:45.728Z INFO Tags changed {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"custom-replset-name-20645"}, "namespace": "custom-replset-name-20645", "name": "some-name", "reconcileID": "55cc94ce-9852-44d3-8d98-457d82efe522", "host": "some-name-shard3-2.some-name-shard3.custom-replset-name-20645.svc.cluster.local:27017", "old": {"nodeName":"gke-jen-psmdb-2221-448d4-default-pool-99ee3404-tf85","podName":"some-name-shard3-2","region":"us-central1","serviceName":"some-name","zone":"us-central1-a"}, "new": {"nodeName":"gke-jen-psmdb-2221-448d4-default-pool-99ee3404-2txw","podName":"some-name-shard3-2","region":"us-central1","serviceName":"some-name","zone":"us-central1-a"}} 2026-03-10T16:52:45.728Z INFO Fixing member configurations {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"custom-replset-name-20645"}, "namespace": "custom-replset-name-20645", "name": "some-name", "reconcileID": "55cc94ce-9852-44d3-8d98-457d82efe522", "replset": "shard3"} 2026-03-10T16:52:45.728Z DEBUG Running replSetReconfig config {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"custom-replset-name-20645"}, "namespace": "custom-replset-name-20645", "name": "some-name", "reconcileID": "55cc94ce-9852-44d3-8d98-457d82efe522", "cfg": {"_id":"shard3","version":67178,"members":[{"_id":0,"host":"some-name-shard3-0.some-name-shard3.custom-replset-name-20645.svc.cluster.local:27017","arbiterOnly":false,"buildIndexes":true,"hidden":false,"priority":2,"tags":{"nodeName":"gke-jen-psmdb-2221-448d4-default-pool-99ee3404-2txw","podName":"some-name-shard3-0","region":"us-central1","serviceName":"some-name","zone":"us-central1-a"},"secondaryDelaySecs":0,"votes":1},{"_id":1,"host":"some-name-shard3-1.some-name-shard3.custom-replset-name-20645.svc.cluster.local:27017","arbiterOnly":false,"buildIndexes":true,"hidden":false,"priority":2,"tags":{"nodeName":"gke-jen-psmdb-2221-448d4-default-pool-99ee3404-2txw","podName":"some-name-shard3-1","region":"us-central1","serviceName":"some-name","zone":"us-central1-a"},"secondaryDelaySecs":0,"votes":1},{"_id":2,"host":"some-name-shard3-2.some-name-shard3.custom-replset-name-20645.svc.cluster.local:27017","arbiterOnly":false,"buildIndexes":true,"hidden":false,"priority":2,"tags":{"nodeName":"gke-jen-psmdb-2221-448d4-default-pool-99ee3404-2txw","podName":"some-name-shard3-2","region":"us-central1","serviceName":"some-name","zone":"us-central1-a"},"secondaryDelaySecs":0,"votes":1}],"protocolVersion":1,"settings":{"chainingAllowed":true,"heartbeatIntervalMillis":2000,"heartbeatTimeoutSecs":10,"electionTimeoutMillis":10000,"catchUpTimeoutMillis":-1,"getLastErrorDefaults":{"w":1,"wtimeout":0},"replicaSetId":"69b04b718d9c1c741f0ee2a4"},"writeConcernMajorityJournalDefault":true}} 2026-03-10T16:52:45.799Z DEBUG Replset is not ready {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"custom-replset-name-20645"}, "namespace": "custom-replset-name-20645", "name": "some-name", "reconcileID": "55cc94ce-9852-44d3-8d98-457d82efe522", "liveMembers": 2, "pods": 3} 2026-03-10T16:52:45.805Z DEBUG Replset status is not healthy {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"custom-replset-name-20645"}, "namespace": "custom-replset-name-20645", "name": "some-name", "reconcileID": "55cc94ce-9852-44d3-8d98-457d82efe522", "replset": "shard3", "status": "initializing"} 2026-03-10T16:52:47.870Z INFO Tags changed {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"custom-replset-name-20645"}, "namespace": "custom-replset-name-20645", "name": "some-name", "reconcileID": "55cc94ce-9852-44d3-8d98-457d82efe522", "host": "some-name-shard7-1.some-name-shard7.custom-replset-name-20645.svc.cluster.local:27017", "old": {"nodeName":"gke-jen-psmdb-2221-448d4-default-pool-99ee3404-tf85","podName":"some-name-shard7-1","region":"us-central1","serviceName":"some-name","zone":"us-central1-a"}, "new": {"nodeName":"gke-jen-psmdb-2221-448d4-default-pool-99ee3404-2txw","podName":"some-name-shard7-1","region":"us-central1","serviceName":"some-name","zone":"us-central1-a"}} 2026-03-10T16:52:47.870Z INFO Fixing member configurations {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"custom-replset-name-20645"}, "namespace": "custom-replset-name-20645", "name": "some-name", "reconcileID": "55cc94ce-9852-44d3-8d98-457d82efe522", "replset": "shard7"} 2026-03-10T16:52:47.870Z DEBUG Running replSetReconfig config {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"custom-replset-name-20645"}, "namespace": "custom-replset-name-20645", "name": "some-name", "reconcileID": "55cc94ce-9852-44d3-8d98-457d82efe522", "cfg": {"_id":"shard7","version":15743,"members":[{"_id":0,"host":"some-name-shard7-0.some-name-shard7.custom-replset-name-20645.svc.cluster.local:27017","arbiterOnly":false,"buildIndexes":true,"hidden":false,"priority":2,"tags":{"nodeName":"gke-jen-psmdb-2221-448d4-default-pool-99ee3404-2txw","podName":"some-name-shard7-0","region":"us-central1","serviceName":"some-name","zone":"us-central1-a"},"secondaryDelaySecs":0,"votes":1},{"_id":1,"host":"some-name-shard7-1.some-name-shard7.custom-replset-name-20645.svc.cluster.local:27017","arbiterOnly":false,"buildIndexes":true,"hidden":false,"priority":2,"tags":{"nodeName":"gke-jen-psmdb-2221-448d4-default-pool-99ee3404-2txw","podName":"some-name-shard7-1","region":"us-central1","serviceName":"some-name","zone":"us-central1-a"},"secondaryDelaySecs":0,"votes":1},{"_id":2,"host":"some-name-shard7-2.some-name-shard7.custom-replset-name-20645.svc.cluster.local:27017","arbiterOnly":false,"buildIndexes":true,"hidden":false,"priority":2,"tags":{"nodeName":"gke-jen-psmdb-2221-448d4-default-pool-99ee3404-tf85","podName":"some-name-shard7-2","region":"us-central1","serviceName":"some-name","zone":"us-central1-a"},"secondaryDelaySecs":0,"votes":1}],"protocolVersion":1,"settings":{"chainingAllowed":true,"heartbeatIntervalMillis":2000,"heartbeatTimeoutSecs":10,"electionTimeoutMillis":10000,"catchUpTimeoutMillis":-1,"getLastErrorDefaults":{"w":1,"wtimeout":0},"replicaSetId":"69b04b314d442c2c9f0febdd"},"writeConcernMajorityJournalDefault":true}} 2026-03-10T16:52:48.611Z DEBUG Replset is not ready {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"custom-replset-name-20645"}, "namespace": "custom-replset-name-20645", "name": "some-name", "reconcileID": "55cc94ce-9852-44d3-8d98-457d82efe522", "liveMembers": 2, "pods": 3} 2026-03-10T16:52:48.615Z DEBUG Replset status is not healthy {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"custom-replset-name-20645"}, "namespace": "custom-replset-name-20645", "name": "some-name", "reconcileID": "55cc94ce-9852-44d3-8d98-457d82efe522", "replset": "shard7", "status": "initializing"} 2026-03-10T16:52:52.190Z DEBUG Cluster is not ready {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"custom-replset-name-20645"}, "namespace": "custom-replset-name-20645", "name": "some-name", "reconcileID": "55cc94ce-9852-44d3-8d98-457d82efe522", "pbmStatus": "ready", "upgradeInProgress": false, "replsetsReady": 2, "clusterState": "initializing"} 2026-03-10T16:52:52.190Z INFO Cluster state changed {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"custom-replset-name-20645"}, "namespace": "custom-replset-name-20645", "name": "some-name", "reconcileID": "55cc94ce-9852-44d3-8d98-457d82efe522", "previous": "error", "current": "initializing"} 2026-03-10T16:52:56.552Z DEBUG Replset status is not healthy {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"custom-replset-name-20645"}, "namespace": "custom-replset-name-20645", "name": "some-name", "reconcileID": "19350272-ab89-473e-bb96-784eb30cbe9c", "replset": "shard1", "status": "initializing"} 2026-03-10T16:52:56.930Z DEBUG Replset status is not healthy {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"custom-replset-name-20645"}, "namespace": "custom-replset-name-20645", "name": "some-name", "reconcileID": "19350272-ab89-473e-bb96-784eb30cbe9c", "replset": "shard3", "status": "initializing"} 2026-03-10T16:52:57.852Z DEBUG Replset status is not healthy {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"custom-replset-name-20645"}, "namespace": "custom-replset-name-20645", "name": "some-name", "reconcileID": "19350272-ab89-473e-bb96-784eb30cbe9c", "replset": "shard7", "status": "initializing"} 2026-03-10T16:53:01.340Z DEBUG Cluster is not ready {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"custom-replset-name-20645"}, "namespace": "custom-replset-name-20645", "name": "some-name", "reconcileID": "19350272-ab89-473e-bb96-784eb30cbe9c", "pbmStatus": "ready", "upgradeInProgress": false, "replsetsReady": 2, "clusterState": "initializing"} 2026-03-10T16:53:04.356Z DEBUG checking if backup is allowed {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio","namespace":"custom-replset-name-20645"}, "namespace": "custom-replset-name-20645", "name": "backup-minio", "reconcileID": "085afa5a-1f83-413a-a8f6-5b199ffa1a65", "cluster": "some-name", "namespace": "custom-replset-name-20645"} 2026-03-10T16:53:04.356Z DEBUG Checking for active jobs {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio","namespace":"custom-replset-name-20645"}, "namespace": "custom-replset-name-20645", "name": "backup-minio", "reconcileID": "085afa5a-1f83-413a-a8f6-5b199ffa1a65", "currentJob": {"Name":"backup-minio","Type":0}} 2026-03-10T16:53:04.556Z INFO Acquiring the backup lock {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio","namespace":"custom-replset-name-20645"}, "namespace": "custom-replset-name-20645", "name": "backup-minio", "reconcileID": "085afa5a-1f83-413a-a8f6-5b199ffa1a65"} 2026-03-10T16:53:04.659Z DEBUG backupStatus Got backup meta {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio","namespace":"custom-replset-name-20645"}, "namespace": "custom-replset-name-20645", "name": "backup-minio", "reconcileID": "085afa5a-1f83-413a-a8f6-5b199ffa1a65", "backup": "backup-minio", "pbmName": "2026-03-10T16:50:15Z", "meta": {"type":"logical","opid":"69b04bc785c80da200f25c06","name":"2026-03-10T16:50:15Z","shardRemap":{"csReplSet":"config"},"replsets":[{"name":"shard1","backup_name":"2026-03-10T16:50:15Z/shard1/metadata.json","oplog_name":"2026-03-10T16:50:15Z/shard1/oplog","start_ts":1773161416,"status":"dumpDone","size":0,"size_uncompressed":0,"last_transition_ts":1773161420,"first_write_ts":{"T":1773161408,"I":3},"last_write_ts":{"T":0,"I":0},"node":"some-name-shard1-2.some-name-shard1.custom-replset-name-20645.svc.cluster.local:27017","conditions":[{"timestamp":1773161416,"status":"running"},{"timestamp":1773161420,"status":"dumpDone"}],"pbm_version":"2.13.0","mongo_version":"8.0.19-7"},{"name":"shard7","backup_name":"2026-03-10T16:50:15Z/shard7/metadata.json","oplog_name":"2026-03-10T16:50:15Z/shard7/oplog","start_ts":1773161416,"status":"dumpDone","size":0,"size_uncompressed":0,"last_transition_ts":1773161420,"first_write_ts":{"T":1773161415,"I":5},"last_write_ts":{"T":0,"I":0},"node":"some-name-shard7-1.some-name-shard7.custom-replset-name-20645.svc.cluster.local:27017","conditions":[{"timestamp":1773161416,"status":"running"},{"timestamp":1773161420,"status":"dumpDone"}],"pbm_version":"2.13.0","mongo_version":"8.0.19-7"},{"name":"shard5","backup_name":"2026-03-10T16:50:15Z/shard5/metadata.json","oplog_name":"2026-03-10T16:50:15Z/shard5/oplog","start_ts":1773161416,"status":"error","size":0,"size_uncompressed":0,"last_transition_ts":1773161426,"first_write_ts":{"T":1773161415,"I":3},"last_write_ts":{"T":0,"I":0},"node":"some-name-shard5-2.some-name-shard5.custom-replset-name-20645.svc.cluster.local:27017","error":"oplog: write data: save during split-merge mw: upload to S3: operation error S3: PutObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio-service:9000/operator-testing/2026-03-10T16%3A50%3A15Z/shard5/oplog/20260310165015-3.20260310165020-8.gz?x-id=PutObject\": dial tcp 34.118.232.130:9000: connect: connection refused.","conditions":[{"timestamp":1773161416,"status":"running"},{"timestamp":1773161420,"status":"dumpDone"},{"timestamp":1773161426,"status":"error","error":"oplog: write data: save during split-merge mw: upload to S3: operation error S3: PutObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio-service:9000/operator-testing/2026-03-10T16%3A50%3A15Z/shard5/oplog/20260310165015-3.20260310165020-8.gz?x-id=PutObject\": dial tcp 34.118.232.130:9000: connect: connection refused."}],"pbm_version":"2.13.0","mongo_version":"8.0.19-7"},{"name":"shard3","backup_name":"2026-03-10T16:50:15Z/shard3/metadata.json","oplog_name":"2026-03-10T16:50:15Z/shard3/oplog","start_ts":1773161416,"status":"error","size":0,"size_uncompressed":0,"last_transition_ts":1773161444,"first_write_ts":{"T":1773161409,"I":2},"last_write_ts":{"T":0,"I":0},"node":"some-name-shard3-1.some-name-shard3.custom-replset-name-20645.svc.cluster.local:27017","error":"oplog: write data: save during split-merge mw: upload to S3: operation error S3: PutObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio-service:9000/operator-testing/2026-03-10T16%3A50%3A15Z/shard3/oplog/20260310165009-2.20260310165020-11.gz?x-id=PutObject\": dial tcp 34.118.232.130:9000: connect: connection refused.","conditions":[{"timestamp":1773161416,"status":"running"},{"timestamp":1773161420,"status":"dumpDone"},{"timestamp":1773161444,"status":"error","error":"oplog: write data: save during split-merge mw: upload to S3: operation error S3: PutObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio-service:9000/operator-testing/2026-03-10T16%3A50%3A15Z/shard3/oplog/20260310165009-2.20260310165020-11.gz?x-id=PutObject\": dial tcp 34.118.232.130:9000: connect: connection refused."}],"pbm_version":"2.13.0","mongo_version":"8.0.19-7"},{"name":"csReplSet","backup_name":"2026-03-10T16:50:15Z/csReplSet/metadata.json","oplog_name":"2026-03-10T16:50:15Z/csReplSet/oplog","start_ts":1773161416,"status":"error","size":0,"size_uncompressed":0,"iscs":true,"last_transition_ts":1773161532,"first_write_ts":{"T":1773161416,"I":64},"last_write_ts":{"T":1773161420,"I":57},"node":"some-name-cfg-1.some-name-cfg.custom-replset-name-20645.svc.cluster.local:27017","error":"set cluster last write ts: lost shard shard1, last beat ts: 1773161421","conditions":[{"timestamp":1773161416,"status":"running"},{"timestamp":1773161419,"status":"dumpDone"},{"timestamp":1773161532,"status":"error","error":"set cluster last write ts: lost shard shard1, last beat ts: 1773161421"}],"pbm_version":"2.13.0","mongo_version":"8.0.19-7"}],"compression":"gzip","store":{"name":"minio","profile":true,"type":"s3","s3":{"region":"us-east-1","endpointUrl":"http://minio-service:9000/","forcePathStyle":true,"bucket":"operator-testing","credentials":{"access-key-id":"***","secret-access-key":"***","vault":{}},"maxUploadParts":10000,"storageClass":"STANDARD","insecureSkipTLSVerify":false}},"size":0,"size_uncompressed":0,"mongodb_version":"8.0.19-7","fcv":"8.0","start_ts":1773161416,"last_transition_ts":1773161532,"first_write_ts":{"T":1773161416,"I":64},"last_write_ts":{"T":1,"I":1},"hb":{"T":1773161531,"I":4},"status":"error","conditions":[{"timestamp":1773161416,"status":"starting"},{"timestamp":1773161419,"status":"running"},{"timestamp":1773161420,"status":"dumpDone"},{"timestamp":1773161532,"status":"error","error":"set cluster last write ts: lost shard shard1, last beat ts: 1773161421"}],"n":[{"rs":"shard5","n":["some-name-shard5-1.some-name-shard5.custom-replset-name-20645.svc.cluster.local:27017","some-name-shard5-2.some-name-shard5.custom-replset-name-20645.svc.cluster.local:27017"],"ack":"some-name-shard5-2.some-name-shard5.custom-replset-name-20645.svc.cluster.local:27017"},{"rs":"shard1","n":["some-name-shard1-1.some-name-shard1.custom-replset-name-20645.svc.cluster.local:27017","some-name-shard1-2.some-name-shard1.custom-replset-name-20645.svc.cluster.local:27017"],"ack":"some-name-shard1-2.some-name-shard1.custom-replset-name-20645.svc.cluster.local:27017"},{"rs":"shard3","n":["some-name-shard3-1.some-name-shard3.custom-replset-name-20645.svc.cluster.local:27017","some-name-shard3-2.some-name-shard3.custom-replset-name-20645.svc.cluster.local:27017"],"ack":"some-name-shard3-1.some-name-shard3.custom-replset-name-20645.svc.cluster.local:27017"},{"rs":"shard7","n":["some-name-shard7-1.some-name-shard7.custom-replset-name-20645.svc.cluster.local:27017","some-name-shard7-2.some-name-shard7.custom-replset-name-20645.svc.cluster.local:27017"],"ack":"some-name-shard7-1.some-name-shard7.custom-replset-name-20645.svc.cluster.local:27017"},{"rs":"csReplSet","n":["some-name-cfg-1.some-name-cfg.custom-replset-name-20645.svc.cluster.local:27017","some-name-cfg-2.some-name-cfg.custom-replset-name-20645.svc.cluster.local:27017"],"ack":"some-name-cfg-1.some-name-cfg.custom-replset-name-20645.svc.cluster.local:27017"}],"error":"set cluster last write ts: lost shard shard1, last beat ts: 1773161421","pbm_version":"2.13.0","balancer":"full"}} 2026-03-10T16:53:04.663Z INFO Backup state changed {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio","namespace":"custom-replset-name-20645"}, "namespace": "custom-replset-name-20645", "name": "backup-minio", "reconcileID": "085afa5a-1f83-413a-a8f6-5b199ffa1a65", "previous": "running", "current": "error"} 2026-03-10T16:53:04.686Z INFO Releasing backup lock {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio","namespace":"custom-replset-name-20645"}, "namespace": "custom-replset-name-20645", "name": "backup-minio", "reconcileID": "085afa5a-1f83-413a-a8f6-5b199ffa1a65", "lease": "psmdb-some-name-backup-lock"} 2026-03-10T16:53:04.709Z DEBUG Reconcile finished {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio","namespace":"custom-replset-name-20645"}, "namespace": "custom-replset-name-20645", "name": "backup-minio", "reconcileID": "085afa5a-1f83-413a-a8f6-5b199ffa1a65"} 2026-03-10T16:53:04.709Z DEBUG Reconciling {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio","namespace":"custom-replset-name-20645"}, "namespace": "custom-replset-name-20645", "name": "backup-minio", "reconcileID": "dcf4a3d4-206f-4648-af22-d787684ef85d"} 2026-03-10T16:53:04.718Z DEBUG Got object from API server {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio","namespace":"custom-replset-name-20645"}, "namespace": "custom-replset-name-20645", "name": "backup-minio", "reconcileID": "dcf4a3d4-206f-4648-af22-d787684ef85d", "state": "error"} 2026-03-10T16:53:04.718Z DEBUG Reconcile finished {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio","namespace":"custom-replset-name-20645"}, "namespace": "custom-replset-name-20645", "name": "backup-minio", "reconcileID": "dcf4a3d4-206f-4648-af22-d787684ef85d"} 2026-03-10T16:53:05.872Z DEBUG Replset status is not healthy {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"custom-replset-name-20645"}, "namespace": "custom-replset-name-20645", "name": "some-name", "reconcileID": "b255122b-be57-4c51-b3bf-e4ced938af4f", "replset": "shard1", "status": "initializing"} 2026-03-10T16:53:06.328Z DEBUG Replset status is not healthy {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"custom-replset-name-20645"}, "namespace": "custom-replset-name-20645", "name": "some-name", "reconcileID": "b255122b-be57-4c51-b3bf-e4ced938af4f", "replset": "shard3", "status": "initializing"} 2026-03-10T16:53:07.407Z DEBUG Replset status is not healthy {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"custom-replset-name-20645"}, "namespace": "custom-replset-name-20645", "name": "some-name", "reconcileID": "b255122b-be57-4c51-b3bf-e4ced938af4f", "replset": "shard7", "status": "initializing"} NAME CLUSTER STORAGE DESTINATION TYPE SIZE STATUS COMPLETED AGE backup-minio some-name minio s3://operator-testing/2026-03-10T16:50:15Z logical error 2m54s Name: backup-minio Namespace: custom-replset-name-20645 Labels: Annotations: API Version: psmdb.percona.com/v1 Kind: PerconaServerMongoDBBackup Metadata: Creation Timestamp: 2026-03-10T16:50:15Z Finalizers: internal.percona.com/release-lock Generation: 1 Resource Version: 1773161584679695020 UID: 0a9e26ba-9115-47d8-ba59-3067b68404a7 Spec: Cluster Name: some-name Starting Deadline Seconds: 120 Storage Name: minio Type: logical Status: Destination: s3://operator-testing/2026-03-10T16:50:15Z Error: set cluster last write ts: lost shard shard1, last beat ts: 1773161421 Last Transition: 2026-03-10T16:52:12Z Pbm Name: 2026-03-10T16:50:15Z Pbm Pod: some-name-shard1-2.some-name-shard1.custom-replset-name-20645.svc.cluster.local:27017 Pbm Pods: Cs Repl Set: some-name-cfg-1.some-name-cfg.custom-replset-name-20645.svc.cluster.local:27017 shard1: some-name-shard1-2.some-name-shard1.custom-replset-name-20645.svc.cluster.local:27017 shard3: some-name-shard3-1.some-name-shard3.custom-replset-name-20645.svc.cluster.local:27017 shard5: some-name-shard5-2.some-name-shard5.custom-replset-name-20645.svc.cluster.local:27017 shard7: some-name-shard7-1.some-name-shard7.custom-replset-name-20645.svc.cluster.local:27017 Replset Names: cfg shard1 shard3 shard5 shard7 s3: Bucket: operator-testing Credentials Secret: minio-secret Endpoint URL: http://minio-service:9000/ Region: us-east-1 Server Side Encryption: Start: 2026-03-10T16:50:16Z State: error Storage Name: minio Type: logical Events: Backup object psmdb-backup/backup-minio is in error state.