Log: /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1915/e2e-tests/logs/demand-backup.log WARNING: version difference between client (1.33) and server (1.30) exceeds the supported minor version skew of +/-1 WARNING: version difference between client (1.33) and server (1.30) exceeds the supported minor version skew of +/-1 WARNING: version difference between client (1.33) and server (1.30) exceeds the supported minor version skew of +/-1 ----------------------------------------------------------------------------------- get and delete old CRDs and RBAC ----------------------------------------------------------------------------------- error: the server doesn't have a resource type "perconaservermongodbbackups" + kubectl patch perconaservermongodbbackups.psmdb.percona.com -n sh --type=merge -p '{"metadata":{"finalizers":[]}}' error: the server doesn't have a resource type "perconaservermongodbbackups" error: the server doesn't have a resource type "perconaservermongodbrestores" + kubectl patch perconaservermongodbrestores.psmdb.percona.com -n sh --type=merge -p '{"metadata":{"finalizers":[]}}' error: the server doesn't have a resource type "perconaservermongodbrestores" error: the server doesn't have a resource type "perconaservermongodbs" + kubectl patch perconaservermongodbs.psmdb.percona.com -n sh --type=merge -p '{"metadata":{"finalizers":[]}}' error: the server doesn't have a resource type "perconaservermongodbs" Error from server (NotFound): customresourcedefinitions.apiextensions.k8s.io "null" not found Error from server (NotFound): customresourcedefinitions.apiextensions.k8s.io "null" not found Error from server (NotFound): customresourcedefinitions.apiextensions.k8s.io "null" not found Error from server (NotFound): customresourcedefinitions.apiextensions.k8s.io "null" not found ----------------------------------------------------------------------------------- destroy chaos-mesh ----------------------------------------------------------------------------------- error: resource(s) were provided, but no name was specified error: resource(s) were provided, but no name was specified error: resource(s) were provided, but no name was specified error: resource(s) were provided, but no name was specified error: resource(s) were provided, but no name was specified error: resource(s) were provided, but no name was specified ----------------------------------------------------------------------------------- cleaned up all old namespaces ----------------------------------------------------------------------------------- ----------------------------------------------------------------------------------- cleaned up old namespaces psmdb-operator ----------------------------------------------------------------------------------- ----------------------------------------------------------------------------------- create namespace psmdb-operator ----------------------------------------------------------------------------------- namespace "gke-managed-cim" deleted namespace "gke-managed-system" deleted namespace "gmp-public" deleted namespace/psmdb-operator created namespace "gmp-system" deleted Context "gke_cloud-dev-112233_us-central1-a_jen-psmdb-1915-2d829121-4-cluster10" modified. ----------------------------------------------------------------------------------- start PSMDB operator ----------------------------------------------------------------------------------- customresourcedefinition.apiextensions.k8s.io/perconaservermongodbbackups.psmdb.percona.com serverside-applied customresourcedefinition.apiextensions.k8s.io/perconaservermongodbrestores.psmdb.percona.com serverside-applied customresourcedefinition.apiextensions.k8s.io/perconaservermongodbs.psmdb.percona.com serverside-applied clusterrole.rbac.authorization.k8s.io/percona-server-mongodb-operator created serviceaccount/percona-server-mongodb-operator created clusterrolebinding.rbac.authorization.k8s.io/service-account-percona-server-mongodb-operator created deployment.apps/percona-server-mongodb-operator created waiting for pod/percona-server-mongodb-operator-6c96ccdc79-m8j2f to be ready.OK Print operator info from log 2025-05-21T08:59:01.455Z INFO setup Manager starting up {"gitCommit": "2d8291219bbb1a7ad015786c5ebd76d37f8fba63", "gitBranch": "PR-1915-2d829121", "buildTime": "", "goVersion": "go1.24.3", "os": "linux", "arch": "amd64"} ----------------------------------------------------------------------------------- destroy chaos-mesh ----------------------------------------------------------------------------------- error: resource(s) were provided, but no name was specified error: resource(s) were provided, but no name was specified error: resource(s) were provided, but no name was specified error: resource(s) were provided, but no name was specified error: resource(s) were provided, but no name was specified error: resource(s) were provided, but no name was specified ----------------------------------------------------------------------------------- cleaned up all old namespaces ----------------------------------------------------------------------------------- ----------------------------------------------------------------------------------- cleaned up old namespaces demand-backup-1244 ----------------------------------------------------------------------------------- namespace "gke-managed-cim" deleted namespace "gke-managed-system" deleted ----------------------------------------------------------------------------------- create namespace demand-backup-1244 ----------------------------------------------------------------------------------- namespace "gmp-public" deleted namespace "gmp-system" deleted namespace/demand-backup-1244 created Context "gke_cloud-dev-112233_us-central1-a_jen-psmdb-1915-2d829121-4-cluster10" modified. ----------------------------------------------------------------------------------- install Minio ----------------------------------------------------------------------------------- Error: uninstall: Release not loaded: minio-service: release: not found Error: no repositories configured "minio" has been added to your repositories NAME: minio-service LAST DEPLOYED: Wed May 21 08:59:37 2025 NAMESPACE: demand-backup-1244 STATUS: deployed REVISION: 1 TEST SUITE: None NOTES: MinIO can be accessed via port 9000 on the following DNS name from within your cluster: minio-service.demand-backup-1244.cluster.local To access MinIO from localhost, run the below commands: 1. export POD_NAME=$(kubectl get pods --namespace demand-backup-1244 -l "release=minio-service" -o jsonpath="{.items[0].metadata.name}") 2. kubectl port-forward $POD_NAME 9000 --namespace demand-backup-1244 Read more about port forwarding here: http://kubernetes.io/docs/user-guide/kubectl/kubectl_port-forward/ You can now access MinIO server on http://localhost:9000. Follow the below steps to connect to MinIO server with mc client: 1. Download the MinIO mc client - https://min.io/docs/minio/linux/reference/minio-mc.html#quickstart 2. export MC_HOST_minio-service-local=http://$(kubectl get secret --namespace demand-backup-1244 minio-service -o jsonpath="{.data.rootUser}" | base64 --decode):$(kubectl get secret --namespace demand-backup-1244 minio-service -o jsonpath="{.data.rootPassword}" | base64 --decode)@localhost:9000 3. mc ls minio-service-local waiting for pod/minio-service-86dfccd949-5rvp4 to be ready.OK service/minio-service created pod "aws-cli" deleted If you don't see a command prompt, try pressing enter. ----------------------------------------------------------------------------------- create secrets and start client ----------------------------------------------------------------------------------- secret/some-users created deployment.apps/psmdb-client created ----------------------------------------------------------------------------------- create secrets for cloud storages ----------------------------------------------------------------------------------- secret/minio-secret created secret/aws-s3-secret created secret/gcp-cs-secret created secret/azure-secret created ----------------------------------------------------------------------------------- create first PSMDB cluster some-name-rs0 ----------------------------------------------------------------------------------- perconaservermongodb.psmdb.percona.com/some-name created ----------------------------------------------------------------------------------- check if all 3 Pods started ----------------------------------------------------------------------------------- waiting for pod/some-name-rs0-0 to be ready............OK waiting for pod/some-name-rs0-1 to be ready...............OK waiting for pod/some-name-rs0-2 to be ready..............OK Waiting for cluster readyness ----------------------------------------------------------------------------------- check if service and statefulset created with expected config ----------------------------------------------------------------------------------- ----------------------------------------------------------------------------------- create user ----------------------------------------------------------------------------------- Percona Server for MongoDB shell version v4.4.29-28 connecting to: mongodb://some-name-rs0-2.some-name-rs0.demand-backup-1244.svc.cluster.local:27017,some-name-rs0-0.some-name-rs0.demand-backup-1244.svc.cluster.local:27017,some-name-rs0-1.some-name-rs0.demand-backup-1244.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb&replicaSet=rs0&ssl=false Implicit session: session { "id" : UUID("1edd8ca9-1e69-4219-9aa9-eab455826d43") } Percona Server for MongoDB server version: v7.0.18-11 WARNING: shell and server versions do not match Successfully added user: { "user" : "myApp", "roles" : [ { "db" : "myApp", "role" : "readWrite" } ] } bye ----------------------------------------------------------------------------------- write data, read from all ----------------------------------------------------------------------------------- Percona Server for MongoDB shell version v4.4.29-28 connecting to: mongodb://some-name-rs0-0.some-name-rs0.demand-backup-1244.svc.cluster.local:27017,some-name-rs0-1.some-name-rs0.demand-backup-1244.svc.cluster.local:27017,some-name-rs0-2.some-name-rs0.demand-backup-1244.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb&replicaSet=rs0&ssl=false Implicit session: session { "id" : UUID("68a2619f-c1f2-40d6-a167-3b5efbe4a722") } Percona Server for MongoDB server version: v7.0.18-11 WARNING: shell and server versions do not match switched to db myApp WriteResult({ "nInserted" : 1 }) bye Percona Server for MongoDB shell version v4.4.29-28 connecting to: mongodb://some-name-rs0-2.some-name-rs0.demand-backup-1244.svc.cluster.local:27017,some-name-rs0-0.some-name-rs0.demand-backup-1244.svc.cluster.local:27017,some-name-rs0-1.some-name-rs0.demand-backup-1244.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb&replicaSet=rs0&ssl=false Implicit session: session { "id" : UUID("b375d6c3-0d1c-4288-9435-3321672d998e") } Percona Server for MongoDB server version: v7.0.18-11 WARNING: shell and server versions do not match switched to db myApp WriteResult({ "nInserted" : 1 }) bye Percona Server for MongoDB shell version v4.4.29-28 connecting to: mongodb://some-name-rs0-0.some-name-rs0.demand-backup-1244.svc.cluster.local:27017,some-name-rs0-1.some-name-rs0.demand-backup-1244.svc.cluster.local:27017,some-name-rs0-2.some-name-rs0.demand-backup-1244.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb&replicaSet=rs0&ssl=false Implicit session: session { "id" : UUID("63e6c7d9-15fe-45b9-8c0b-4cd74aff6e86") } Percona Server for MongoDB server version: v7.0.18-11 WARNING: shell and server versions do not match switched to db myApp WriteResult({ "nInserted" : 1 }) bye Percona Server for MongoDB shell version v4.4.29-28 connecting to: mongodb://some-name-rs0-1.some-name-rs0.demand-backup-1244.svc.cluster.local:27017,some-name-rs0-2.some-name-rs0.demand-backup-1244.svc.cluster.local:27017,some-name-rs0-0.some-name-rs0.demand-backup-1244.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb&replicaSet=rs0&ssl=false Implicit session: session { "id" : UUID("20187f3f-291a-4880-8c85-cdf2372446a4") } Percona Server for MongoDB server version: v7.0.18-11 WARNING: shell and server versions do not match switched to db myApp Successfully added user: { "user" : "test1user", "roles" : [ ] } bye Percona Server for MongoDB shell version v4.4.29-28 connecting to: mongodb://some-name-rs0-0.some-name-rs0.demand-backup-1244.svc.cluster.local:27017,some-name-rs0-1.some-name-rs0.demand-backup-1244.svc.cluster.local:27017,some-name-rs0-2.some-name-rs0.demand-backup-1244.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb&replicaSet=rs0&ssl=false Implicit session: session { "id" : UUID("4952b50e-1cd2-4341-bec8-e028956436f7") } Percona Server for MongoDB server version: v7.0.18-11 WARNING: shell and server versions do not match switched to db myApp { "role" : "test1role", "privileges" : [ ], "roles" : [ ] } bye [2025-05-21T09:03:03+0000] running db.test.find() in myApp [2025-05-21T09:03:06+0000] running db.test.find() in myApp [2025-05-21T09:03:09+0000] running db.test.find() in myApp waiting for pbm-agent to be ready in some-name-rs0-0...2025-05-21T09:02:23.000+0000 I listening for the commands waiting for pbm-agent to be ready in some-name-rs0-1...2025-05-21T09:02:28.000+0000 I listening for the commands waiting for pbm-agent to be ready in some-name-rs0-2...2025-05-21T09:02:38.000+0000 I listening for the commands ----------------------------------------------------------------------------------- run backups ----------------------------------------------------------------------------------- ----------------------------------------------------------------------------------- run backup backup-aws-s3 ----------------------------------------------------------------------------------- perconaservermongodbbackup.psmdb.percona.com/backup-aws-s3 created ----------------------------------------------------------------------------------- run backup backup-gcp-cs ----------------------------------------------------------------------------------- perconaservermongodbbackup.psmdb.percona.com/backup-gcp-cs created ----------------------------------------------------------------------------------- run backup backup-azure-blob ----------------------------------------------------------------------------------- perconaservermongodbbackup.psmdb.percona.com/backup-azure-blob created ----------------------------------------------------------------------------------- run backup backup-minio ----------------------------------------------------------------------------------- perconaservermongodbbackup.psmdb.percona.com/backup-minio created waiting for backup-aws-s3 to reach ready state. waiting for backup-gcp-cs to reach ready state........... waiting for backup-azure-blob to reach ready state. waiting for backup-minio to reach ready state... ----------------------------------------------------------------------------------- check backup and restore -- aws-s3 ----------------------------------------------------------------------------------- Percona Server for MongoDB shell version v4.4.29-28 connecting to: mongodb://some-name-rs0-1.some-name-rs0.demand-backup-1244.svc.cluster.local:27017,some-name-rs0-2.some-name-rs0.demand-backup-1244.svc.cluster.local:27017,some-name-rs0-0.some-name-rs0.demand-backup-1244.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb&replicaSet=rs0&ssl=false Implicit session: session { "id" : UUID("ecdf9864-a94b-4552-8ed7-09332c038026") } Percona Server for MongoDB server version: v7.0.18-11 WARNING: shell and server versions do not match switched to db myApp WriteResult({ "nInserted" : 1 }) bye [2025-05-21T09:04:15+0000] running db.test.find() in myApp perconaservermongodbrestore.psmdb.percona.com/restore-backup-aws-s3 created Waiting for the psmdb-restore/restore-backup-aws-s3 object to be createdOK Waiting psmdb-restore/restore-backup-aws-s3 to reach state "ready" OK waiting for cluster readyness [2025-05-21T09:04:40+0000] running db.test.find() in myApp [2025-05-21T09:04:42+0000] running db.test.find() in myApp [2025-05-21T09:04:45+0000] running db.test.find() in myApp ----------------------------------------------------------------------------------- check backup and restore -- gcp-cs ----------------------------------------------------------------------------------- Percona Server for MongoDB shell version v4.4.29-28 connecting to: mongodb://some-name-rs0-1.some-name-rs0.demand-backup-1244.svc.cluster.local:27017,some-name-rs0-2.some-name-rs0.demand-backup-1244.svc.cluster.local:27017,some-name-rs0-0.some-name-rs0.demand-backup-1244.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb&replicaSet=rs0&ssl=false Implicit session: session { "id" : UUID("2fd4b682-2cae-4909-9af6-1efb126b1e6b") } Percona Server for MongoDB server version: v7.0.18-11 WARNING: shell and server versions do not match switched to db myApp WriteResult({ "nInserted" : 1 }) bye [2025-05-21T09:04:52+0000] running db.test.find() in myApp perconaservermongodbrestore.psmdb.percona.com/restore-backup-gcp-cs created Waiting for the psmdb-restore/restore-backup-gcp-cs object to be createdOK Waiting psmdb-restore/restore-backup-gcp-cs to reach state "ready" OK waiting for cluster readyness [2025-05-21T09:05:17+0000] running db.test.find() in myApp [2025-05-21T09:05:20+0000] running db.test.find() in myApp [2025-05-21T09:05:22+0000] running db.test.find() in myApp ----------------------------------------------------------------------------------- check backup and restore -- azure-blob ----------------------------------------------------------------------------------- Percona Server for MongoDB shell version v4.4.29-28 connecting to: mongodb://some-name-rs0-1.some-name-rs0.demand-backup-1244.svc.cluster.local:27017,some-name-rs0-2.some-name-rs0.demand-backup-1244.svc.cluster.local:27017,some-name-rs0-0.some-name-rs0.demand-backup-1244.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb&replicaSet=rs0&ssl=false Implicit session: session { "id" : UUID("221f17e1-d1b6-429f-b7a0-5d8cd33d673e") } Percona Server for MongoDB server version: v7.0.18-11 WARNING: shell and server versions do not match switched to db myApp WriteResult({ "nInserted" : 1 }) bye [2025-05-21T09:05:29+0000] running db.test.find() in myApp perconaservermongodbrestore.psmdb.percona.com/restore-backup-azure-blob created Waiting for the psmdb-restore/restore-backup-azure-blob object to be createdOK Waiting psmdb-restore/restore-backup-azure-blob to reach state "ready" OK waiting for cluster readyness [2025-05-21T09:05:52+0000] running db.test.find() in myApp [2025-05-21T09:05:54+0000] running db.test.find() in myApp [2025-05-21T09:05:57+0000] running db.test.find() in myApp ----------------------------------------------------------------------------------- check backup and restore -- minio ----------------------------------------------------------------------------------- 2025-05-21 09:03:59 55 myApp.test.gz Percona Server for MongoDB shell version v4.4.29-28 connecting to: mongodb://some-name-rs0-0.some-name-rs0.demand-backup-1244.svc.cluster.local:27017,some-name-rs0-1.some-name-rs0.demand-backup-1244.svc.cluster.local:27017,some-name-rs0-2.some-name-rs0.demand-backup-1244.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb&replicaSet=rs0&ssl=false Implicit session: session { "id" : UUID("68b857d5-4167-475d-9212-89f4c4b2da80") } Percona Server for MongoDB server version: v7.0.18-11 WARNING: shell and server versions do not match switched to db myApp WriteResult({ "nInserted" : 1 }) bye [2025-05-21T09:06:06+0000] running db.test.find() in myApp perconaservermongodbrestore.psmdb.percona.com/restore-backup-minio created Waiting for the psmdb-restore/restore-backup-minio object to be createdOK Waiting psmdb-restore/restore-backup-minio to reach state "ready" OK waiting for cluster readyness [2025-05-21T09:06:28+0000] running db.test.find() in myApp [2025-05-21T09:06:29+0000] running db.test.find() in myApp [2025-05-21T09:06:30+0000] running db.test.find() in myApp Percona Server for MongoDB shell version v4.4.29-28 connecting to: mongodb://some-name-rs0-2.some-name-rs0.demand-backup-1244.svc.cluster.local:27017,some-name-rs0-0.some-name-rs0.demand-backup-1244.svc.cluster.local:27017,some-name-rs0-1.some-name-rs0.demand-backup-1244.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb&replicaSet=rs0&ssl=false Implicit session: session { "id" : UUID("77cf9a1f-c857-4f67-9737-1a249ac277e6") } Percona Server for MongoDB server version: v7.0.18-11 WARNING: shell and server versions do not match switched to db myApp true bye Percona Server for MongoDB shell version v4.4.29-28 connecting to: mongodb://some-name-rs0-0.some-name-rs0.demand-backup-1244.svc.cluster.local:27017,some-name-rs0-1.some-name-rs0.demand-backup-1244.svc.cluster.local:27017,some-name-rs0-2.some-name-rs0.demand-backup-1244.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb&replicaSet=rs0&ssl=false Implicit session: session { "id" : UUID("f3ea9058-aff4-47d8-84d2-2278b18f063f") } Percona Server for MongoDB server version: v7.0.18-11 WARNING: shell and server versions do not match switched to db myApp true bye ----------------------------------------------------------------------------------- selective restore -- minio ----------------------------------------------------------------------------------- Percona Server for MongoDB shell version v4.4.29-28 connecting to: mongodb://some-name-rs0-1.some-name-rs0.demand-backup-1244.svc.cluster.local:27017,some-name-rs0-2.some-name-rs0.demand-backup-1244.svc.cluster.local:27017,some-name-rs0-0.some-name-rs0.demand-backup-1244.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb&replicaSet=rs0&ssl=false Implicit session: session { "id" : UUID("ae7d4962-9f0e-4daf-b9de-6f88b8e89ff7") } Percona Server for MongoDB server version: v7.0.18-11 WARNING: shell and server versions do not match switched to db myApp true bye perconaservermongodbrestore.psmdb.percona.com/restore-backup-minio-selective created Waiting for the psmdb-restore/restore-backup-minio-selective object to be createdOK Waiting psmdb-restore/restore-backup-minio-selective to reach state "ready" OK waiting for cluster readyness jq: error: myApp/0 is not defined at , line 1: any(.[]; ._id==myApp.test1user) jq: 1 compile error jq: error: myApp/0 is not defined at , line 1: any(.[]; ._id==myApp.test1role) jq: 1 compile error perconaservermongodbrestore.psmdb.percona.com "restore-backup-minio-selective" deleted Percona Server for MongoDB shell version v4.4.29-28 connecting to: mongodb://some-name-rs0-0.some-name-rs0.demand-backup-1244.svc.cluster.local:27017,some-name-rs0-1.some-name-rs0.demand-backup-1244.svc.cluster.local:27017,some-name-rs0-2.some-name-rs0.demand-backup-1244.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb&replicaSet=rs0&ssl=false Implicit session: session { "id" : UUID("f9fe4259-c833-4d09-8d23-945d94510e23") } Percona Server for MongoDB server version: v7.0.18-11 WARNING: shell and server versions do not match switched to db myApp WriteResult({ "nInserted" : 1 }) bye ----------------------------------------------------------------------------------- selective restore with users and roles -- minio ----------------------------------------------------------------------------------- Percona Server for MongoDB shell version v4.4.29-28 connecting to: mongodb://some-name-rs0-0.some-name-rs0.demand-backup-1244.svc.cluster.local:27017,some-name-rs0-1.some-name-rs0.demand-backup-1244.svc.cluster.local:27017,some-name-rs0-2.some-name-rs0.demand-backup-1244.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb&replicaSet=rs0&ssl=false Implicit session: session { "id" : UUID("453238fb-51b5-4f53-a9f5-52935a7d113b") } Percona Server for MongoDB server version: v7.0.18-11 WARNING: shell and server versions do not match switched to db myApp true bye perconaservermongodbrestore.psmdb.percona.com/restore-backup-minio-selective created Waiting for the psmdb-restore/restore-backup-minio-selective object to be createdOK Waiting psmdb-restore/restore-backup-minio-selective to reach state "ready" OK waiting for cluster readyness jq: error: myApp/0 is not defined at , line 1: any(.[]; ._id==myApp.test1user) jq: 1 compile error jq: error: myApp/0 is not defined at , line 1: any(.[]; ._id==myApp.test1role) jq: 1 compile error ----------------------------------------------------------------------------------- restore from backup source, with storageName -- minio ----------------------------------------------------------------------------------- Percona Server for MongoDB shell version v4.4.29-28 connecting to: mongodb://some-name-rs0-0.some-name-rs0.demand-backup-1244.svc.cluster.local:27017,some-name-rs0-1.some-name-rs0.demand-backup-1244.svc.cluster.local:27017,some-name-rs0-2.some-name-rs0.demand-backup-1244.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb&replicaSet=rs0&ssl=false Implicit session: session { "id" : UUID("3da6d0c7-0db2-4834-bffa-f49de1f3d698") } Percona Server for MongoDB server version: v7.0.18-11 WARNING: shell and server versions do not match switched to db myApp WriteResult({ "nInserted" : 1 }) bye [2025-05-21T09:07:43+0000] running db.test.find() in myApp ----------------------------------------------------------------------------------- run restore restore-backup-minio-source-0 from backup backup-minio-source-0 destination is operator-testing/2025-05-21T09:03:56Z ----------------------------------------------------------------------------------- perconaservermongodbrestore.psmdb.percona.com/restore-backup-minio-source-0 created Waiting for the psmdb-restore/restore-backup-minio-source-0 object to be createdOK Waiting psmdb-restore/restore-backup-minio-source-0 to reach state "ready" OK waiting for cluster readyness [2025-05-21T09:08:08+0000] running db.test.find() in myApp [2025-05-21T09:08:10+0000] running db.test.find() in myApp [2025-05-21T09:08:13+0000] running db.test.find() in myApp ----------------------------------------------------------------------------------- restore from backup source, no storageName -- minio ----------------------------------------------------------------------------------- Percona Server for MongoDB shell version v4.4.29-28 connecting to: mongodb://some-name-rs0-1.some-name-rs0.demand-backup-1244.svc.cluster.local:27017,some-name-rs0-2.some-name-rs0.demand-backup-1244.svc.cluster.local:27017,some-name-rs0-0.some-name-rs0.demand-backup-1244.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb&replicaSet=rs0&ssl=false Implicit session: session { "id" : UUID("6b458d47-1642-4d16-a483-b92d33e74b76") } Percona Server for MongoDB server version: v7.0.18-11 WARNING: shell and server versions do not match switched to db myApp WriteResult({ "nInserted" : 1 }) bye [2025-05-21T09:08:18+0000] running db.test.find() in myApp ----------------------------------------------------------------------------------- run restore restore-backup-minio-source-1 from backup backup-minio-source-1 destination is operator-testing/2025-05-21T09:03:56Z ----------------------------------------------------------------------------------- perconaservermongodbrestore.psmdb.percona.com/restore-backup-minio-source-1 created Waiting for the psmdb-restore/restore-backup-minio-source-1 object to be createdOK Waiting psmdb-restore/restore-backup-minio-source-1 to reach state "ready" OK waiting for cluster readyness [2025-05-21T09:08:42+0000] running db.test.find() in myApp [2025-05-21T09:08:45+0000] running db.test.find() in myApp [2025-05-21T09:08:47+0000] running db.test.find() in myApp ----------------------------------------------------------------------------------- delete backup and check if it is removed from bucket -- minio ----------------------------------------------------------------------------------- perconaservermongodbbackup.psmdb.percona.com "backup-aws-s3" deleted perconaservermongodbbackup.psmdb.percona.com "backup-azure-blob" deleted perconaservermongodbbackup.psmdb.percona.com "backup-gcp-cs" deleted perconaservermongodbbackup.psmdb.percona.com "backup-minio" deleted ----------------------------------------------------------------------------------- checking backup deletion without cr ----------------------------------------------------------------------------------- ----------------------------------------------------------------------------------- run backup backup-minio ----------------------------------------------------------------------------------- perconaservermongodbbackup.psmdb.percona.com/backup-minio created ----------------------------------------------------------------------------------- run backup backup-aws-s3 ----------------------------------------------------------------------------------- perconaservermongodbbackup.psmdb.percona.com/backup-aws-s3 created ----------------------------------------------------------------------------------- run backup backup-gcp-cs ----------------------------------------------------------------------------------- perconaservermongodbbackup.psmdb.percona.com/backup-gcp-cs created ----------------------------------------------------------------------------------- run backup backup-azure-blob ----------------------------------------------------------------------------------- perconaservermongodbbackup.psmdb.percona.com/backup-azure-blob created waiting for backup-minio to reach ready state... waiting for backup-aws-s3 to reach ready state...........................................2025-05-21T09:09:29.030Z INFO Acquiring the backup lock {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs","namespace":"demand-backup-1244"}, "namespace": "demand-backup-1244", "name": "backup-gcp-cs", "reconcileID": "d8eef04f-805a-4361-bc52-3fbea0b849bc"} 2025-05-21T09:09:29.030Z INFO Another backup is holding the lock {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs","namespace":"demand-backup-1244"}, "namespace": "demand-backup-1244", "name": "backup-gcp-cs", "reconcileID": "d8eef04f-805a-4361-bc52-3fbea0b849bc", "holder": "backup-aws-s3-af618cd5-f107-4adc-b134-41114fe2cdfe"} 2025-05-21T09:09:29.031Z DEBUG Reconcile finished {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs","namespace":"demand-backup-1244"}, "namespace": "demand-backup-1244", "name": "backup-gcp-cs", "reconcileID": "d8eef04f-805a-4361-bc52-3fbea0b849bc"} 2025-05-21T09:09:30.641Z DEBUG Reconciling {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-azure-blob","namespace":"demand-backup-1244"}, "namespace": "demand-backup-1244", "name": "backup-azure-blob", "reconcileID": "7559020b-ae7d-430a-89d5-cf7f05037f26"} 2025-05-21T09:09:30.674Z DEBUG checking if backup is allowed {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-azure-blob","namespace":"demand-backup-1244"}, "namespace": "demand-backup-1244", "name": "backup-azure-blob", "reconcileID": "7559020b-ae7d-430a-89d5-cf7f05037f26", "cluster": "some-name", "namespace": "demand-backup-1244"} 2025-05-21T09:09:30.674Z DEBUG Checking for active jobs {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-azure-blob","namespace":"demand-backup-1244"}, "namespace": "demand-backup-1244", "name": "backup-azure-blob", "reconcileID": "7559020b-ae7d-430a-89d5-cf7f05037f26", "currentJob": {"Name":"backup-azure-blob","Type":0}} 2025-05-21T09:09:30.712Z INFO Acquiring the backup lock {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-azure-blob","namespace":"demand-backup-1244"}, "namespace": "demand-backup-1244", "name": "backup-azure-blob", "reconcileID": "7559020b-ae7d-430a-89d5-cf7f05037f26"} 2025-05-21T09:09:30.712Z INFO Another backup is holding the lock {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-azure-blob","namespace":"demand-backup-1244"}, "namespace": "demand-backup-1244", "name": "backup-azure-blob", "reconcileID": "7559020b-ae7d-430a-89d5-cf7f05037f26", "holder": "backup-aws-s3-af618cd5-f107-4adc-b134-41114fe2cdfe"} 2025-05-21T09:09:30.713Z DEBUG Reconcile finished {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-azure-blob","namespace":"demand-backup-1244"}, "namespace": "demand-backup-1244", "name": "backup-azure-blob", "reconcileID": "7559020b-ae7d-430a-89d5-cf7f05037f26"} 2025-05-21T09:09:32.485Z DEBUG Reconciling {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-aws-s3","namespace":"demand-backup-1244"}, "namespace": "demand-backup-1244", "name": "backup-aws-s3", "reconcileID": "27287c14-b432-4876-aace-92200bc4b357"} 2025-05-21T09:09:32.518Z DEBUG checking if backup is allowed {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-aws-s3","namespace":"demand-backup-1244"}, "namespace": "demand-backup-1244", "name": "backup-aws-s3", "reconcileID": "27287c14-b432-4876-aace-92200bc4b357", "cluster": "some-name", "namespace": "demand-backup-1244"} 2025-05-21T09:09:32.518Z DEBUG Checking for active jobs {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-aws-s3","namespace":"demand-backup-1244"}, "namespace": "demand-backup-1244", "name": "backup-aws-s3", "reconcileID": "27287c14-b432-4876-aace-92200bc4b357", "currentJob": {"Name":"backup-aws-s3","Type":0}} 2025-05-21T09:09:32.554Z INFO Acquiring the backup lock {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-aws-s3","namespace":"demand-backup-1244"}, "namespace": "demand-backup-1244", "name": "backup-aws-s3", "reconcileID": "27287c14-b432-4876-aace-92200bc4b357"} 2025-05-21T09:09:32.555Z DEBUG backupStatus Got backup meta {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-aws-s3","namespace":"demand-backup-1244"}, "namespace": "demand-backup-1244", "name": "backup-aws-s3", "reconcileID": "27287c14-b432-4876-aace-92200bc4b357", "backup": "backup-aws-s3", "pbmName": "2025-05-21T09:09:12Z", "meta": {"type":"logical","opid":"682d9838b3cfcdcb66ec6493","name":"2025-05-21T09:09:12Z","replsets":[{"name":"rs0","backup_name":"2025-05-21T09:09:12Z/rs0/metadata.json","oplog_name":"2025-05-21T09:09:12Z/rs0/oplog","start_ts":1747818553,"status":"running","last_transition_ts":1747818553,"first_write_ts":{"T":1747818553,"I":4},"last_write_ts":{"T":0,"I":0},"node":"some-name-rs0-2.some-name-rs0.demand-backup-1244.svc.cluster.local:27017","conditions":[{"timestamp":1747818553,"status":"running"}],"pbm_version":"2.9.1","mongo_version":"7.0.18-11"}],"compression":"gzip","store":{"type":"s3","s3":{"region":"us-east-1","endpointUrl":"","forcePathStyle":true,"bucket":"operator-testing","prefix":"psmdb-demand-backup","maxUploadParts":10000,"storageClass":"STANDARD","insecureSkipTLSVerify":false}},"size":0,"mongodb_version":"7.0.18-11","fcv":"7.0","start_ts":1747818552,"last_transition_ts":1747818552,"first_write_ts":{"T":1,"I":1},"last_write_ts":{"T":1,"I":1},"hb":{"T":1747818553,"I":4},"status":"starting","conditions":[{"timestamp":1747818552,"status":"starting"}],"n":[{"rs":"rs0","n":["some-name-rs0-1.some-name-rs0.demand-backup-1244.svc.cluster.local:27017","some-name-rs0-2.some-name-rs0.demand-backup-1244.svc.cluster.local:27017"],"ack":"some-name-rs0-2.some-name-rs0.demand-backup-1244.svc.cluster.local:27017"}],"pbm_version":"2.9.1","balancer":"off"}} 2025-05-21T09:09:32.557Z DEBUG Reconcile finished {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-aws-s3","namespace":"demand-backup-1244"}, "namespace": "demand-backup-1244", "name": "backup-aws-s3", "reconcileID": "27287c14-b432-4876-aace-92200bc4b357"} 2025-05-21T09:09:34.031Z DEBUG Reconciling {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs","namespace":"demand-backup-1244"}, "namespace": "demand-backup-1244", "name": "backup-gcp-cs", "reconcileID": "36ddb398-c479-43cf-9d40-86e28b502abd"} 2025-05-21T09:09:34.078Z DEBUG checking if backup is allowed {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs","namespace":"demand-backup-1244"}, "namespace": "demand-backup-1244", "name": "backup-gcp-cs", "reconcileID": "36ddb398-c479-43cf-9d40-86e28b502abd", "cluster": "some-name", "namespace": "demand-backup-1244"} 2025-05-21T09:09:34.078Z DEBUG Checking for active jobs {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs","namespace":"demand-backup-1244"}, "namespace": "demand-backup-1244", "name": "backup-gcp-cs", "reconcileID": "36ddb398-c479-43cf-9d40-86e28b502abd", "currentJob": {"Name":"backup-gcp-cs","Type":0}} 2025-05-21T09:09:34.113Z INFO Acquiring the backup lock {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs","namespace":"demand-backup-1244"}, "namespace": "demand-backup-1244", "name": "backup-gcp-cs", "reconcileID": "36ddb398-c479-43cf-9d40-86e28b502abd"} 2025-05-21T09:09:34.113Z INFO Another backup is holding the lock {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs","namespace":"demand-backup-1244"}, "namespace": "demand-backup-1244", "name": "backup-gcp-cs", "reconcileID": "36ddb398-c479-43cf-9d40-86e28b502abd", "holder": "backup-aws-s3-af618cd5-f107-4adc-b134-41114fe2cdfe"} 2025-05-21T09:09:34.114Z DEBUG Reconcile finished {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs","namespace":"demand-backup-1244"}, "namespace": "demand-backup-1244", "name": "backup-gcp-cs", "reconcileID": "36ddb398-c479-43cf-9d40-86e28b502abd"} 2025-05-21T09:09:34.315Z INFO Tags changed {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"demand-backup-1244"}, "namespace": "demand-backup-1244", "name": "some-name", "reconcileID": "b94ecbdd-1046-45a2-ac0f-ebcb8d478fb8", "host": "some-name-rs0-2.some-name-rs0.demand-backup-1244.svc.cluster.local:27017", "old": {"nodeName":"gke-jen-psmdb-1915-2d829-default-pool-a7be86c6-s8l8","podName":"some-name-rs0-2","region":"us-central1","serviceName":"some-name","zone":"us-central1-a"}, "new": {"nodeName":"gke-jen-psmdb-1915-2d829-default-pool-a7be86c6-v268","podName":"some-name-rs0-2","region":"us-central1","serviceName":"some-name","zone":"us-central1-a"}} 2025-05-21T09:09:34.315Z INFO Fixing member configurations {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"demand-backup-1244"}, "namespace": "demand-backup-1244", "name": "some-name", "reconcileID": "b94ecbdd-1046-45a2-ac0f-ebcb8d478fb8", "replset": "rs0"} 2025-05-21T09:09:34.315Z DEBUG Running replSetReconfig config {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"demand-backup-1244"}, "namespace": "demand-backup-1244", "name": "some-name", "reconcileID": "b94ecbdd-1046-45a2-ac0f-ebcb8d478fb8", "cfg": {"_id":"rs0","version":8,"members":[{"_id":0,"host":"some-name-rs0-0.some-name-rs0.demand-backup-1244.svc.cluster.local:27017","arbiterOnly":false,"buildIndexes":true,"hidden":false,"priority":2,"tags":{"nodeName":"gke-jen-psmdb-1915-2d829-default-pool-a7be86c6-v268","podName":"some-name-rs0-0","region":"us-central1","serviceName":"some-name","zone":"us-central1-a"},"secondaryDelaySecs":0,"votes":1},{"_id":1,"host":"some-name-rs0-1.some-name-rs0.demand-backup-1244.svc.cluster.local:27017","arbiterOnly":false,"buildIndexes":true,"hidden":false,"priority":2,"tags":{"nodeName":"gke-jen-psmdb-1915-2d829-default-pool-a7be86c6-szf6","podName":"some-name-rs0-1","region":"us-central1","serviceName":"some-name","zone":"us-central1-a"},"secondaryDelaySecs":0,"votes":1},{"_id":2,"host":"some-name-rs0-2.some-name-rs0.demand-backup-1244.svc.cluster.local:27017","arbiterOnly":false,"buildIndexes":true,"hidden":false,"priority":2,"tags":{"nodeName":"gke-jen-psmdb-1915-2d829-default-pool-a7be86c6-v268","podName":"some-name-rs0-2","region":"us-central1","serviceName":"some-name","zone":"us-central1-a"},"secondaryDelaySecs":0,"votes":1}],"protocolVersion":1,"settings":{"chainingAllowed":true,"heartbeatIntervalMillis":2000,"heartbeatTimeoutSecs":10,"electionTimeoutMillis":10000,"catchUpTimeoutMillis":-1,"getLastErrorDefaults":{"w":1,"wtimeout":0},"replicaSetId":"682d969565eebccafea9a458"},"writeConcernMajorityJournalDefault":true}} 2025-05-21T09:09:35.713Z DEBUG Reconciling {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-azure-blob","namespace":"demand-backup-1244"}, "namespace": "demand-backup-1244", "name": "backup-azure-blob", "reconcileID": "34f0c402-f12d-42e3-97c7-8c6546556e74"} 2025-05-21T09:09:35.747Z DEBUG checking if backup is allowed {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-azure-blob","namespace":"demand-backup-1244"}, "namespace": "demand-backup-1244", "name": "backup-azure-blob", "reconcileID": "34f0c402-f12d-42e3-97c7-8c6546556e74", "cluster": "some-name", "namespace": "demand-backup-1244"} 2025-05-21T09:09:35.747Z DEBUG Checking for active jobs {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-azure-blob","namespace":"demand-backup-1244"}, "namespace": "demand-backup-1244", "name": "backup-azure-blob", "reconcileID": "34f0c402-f12d-42e3-97c7-8c6546556e74", "currentJob": {"Name":"backup-azure-blob","Type":0}} 2025-05-21T09:09:35.782Z INFO Acquiring the backup lock {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-azure-blob","namespace":"demand-backup-1244"}, "namespace": "demand-backup-1244", "name": "backup-azure-blob", "reconcileID": "34f0c402-f12d-42e3-97c7-8c6546556e74"} 2025-05-21T09:09:35.783Z INFO Another backup is holding the lock {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-azure-blob","namespace":"demand-backup-1244"}, "namespace": "demand-backup-1244", "name": "backup-azure-blob", "reconcileID": "34f0c402-f12d-42e3-97c7-8c6546556e74", "holder": "backup-aws-s3-af618cd5-f107-4adc-b134-41114fe2cdfe"} 2025-05-21T09:09:35.784Z DEBUG Reconcile finished {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-azure-blob","namespace":"demand-backup-1244"}, "namespace": "demand-backup-1244", "name": "backup-azure-blob", "reconcileID": "34f0c402-f12d-42e3-97c7-8c6546556e74"} 2025-05-21T09:09:37.558Z DEBUG Reconciling {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-aws-s3","namespace":"demand-backup-1244"}, "namespace": "demand-backup-1244", "name": "backup-aws-s3", "reconcileID": "839e09ea-f361-4490-8dd6-82d9b7a0aae5"} 2025-05-21T09:09:37.593Z DEBUG checking if backup is allowed {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-aws-s3","namespace":"demand-backup-1244"}, "namespace": "demand-backup-1244", "name": "backup-aws-s3", "reconcileID": "839e09ea-f361-4490-8dd6-82d9b7a0aae5", "cluster": "some-name", "namespace": "demand-backup-1244"} 2025-05-21T09:09:37.593Z DEBUG Checking for active jobs {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-aws-s3","namespace":"demand-backup-1244"}, "namespace": "demand-backup-1244", "name": "backup-aws-s3", "reconcileID": "839e09ea-f361-4490-8dd6-82d9b7a0aae5", "currentJob": {"Name":"backup-aws-s3","Type":0}} 2025-05-21T09:09:37.627Z INFO Acquiring the backup lock {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-aws-s3","namespace":"demand-backup-1244"}, "namespace": "demand-backup-1244", "name": "backup-aws-s3", "reconcileID": "839e09ea-f361-4490-8dd6-82d9b7a0aae5"} 2025-05-21T09:09:37.628Z DEBUG backupStatus Got backup meta {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-aws-s3","namespace":"demand-backup-1244"}, "namespace": "demand-backup-1244", "name": "backup-aws-s3", "reconcileID": "839e09ea-f361-4490-8dd6-82d9b7a0aae5", "backup": "backup-aws-s3", "pbmName": "2025-05-21T09:09:12Z", "meta": {"type":"logical","opid":"682d9838b3cfcdcb66ec6493","name":"2025-05-21T09:09:12Z","replsets":[{"name":"rs0","backup_name":"2025-05-21T09:09:12Z/rs0/metadata.json","oplog_name":"2025-05-21T09:09:12Z/rs0/oplog","start_ts":1747818553,"status":"running","last_transition_ts":1747818553,"first_write_ts":{"T":1747818553,"I":4},"last_write_ts":{"T":0,"I":0},"node":"some-name-rs0-2.some-name-rs0.demand-backup-1244.svc.cluster.local:27017","conditions":[{"timestamp":1747818553,"status":"running"}],"pbm_version":"2.9.1","mongo_version":"7.0.18-11"}],"compression":"gzip","store":{"type":"s3","s3":{"region":"us-east-1","endpointUrl":"","forcePathStyle":true,"bucket":"operator-testing","prefix":"psmdb-demand-backup","maxUploadParts":10000,"storageClass":"STANDARD","insecureSkipTLSVerify":false}},"size":0,"mongodb_version":"7.0.18-11","fcv":"7.0","start_ts":1747818552,"last_transition_ts":1747818552,"first_write_ts":{"T":1,"I":1},"last_write_ts":{"T":1,"I":1},"hb":{"T":1747818553,"I":4},"status":"starting","conditions":[{"timestamp":1747818552,"status":"starting"}],"n":[{"rs":"rs0","n":["some-name-rs0-1.some-name-rs0.demand-backup-1244.svc.cluster.local:27017","some-name-rs0-2.some-name-rs0.demand-backup-1244.svc.cluster.local:27017"],"ack":"some-name-rs0-2.some-name-rs0.demand-backup-1244.svc.cluster.local:27017"}],"pbm_version":"2.9.1","balancer":"off"}} 2025-05-21T09:09:37.630Z DEBUG Reconcile finished {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-aws-s3","namespace":"demand-backup-1244"}, "namespace": "demand-backup-1244", "name": "backup-aws-s3", "reconcileID": "839e09ea-f361-4490-8dd6-82d9b7a0aae5"} 2025-05-21T09:09:39.115Z DEBUG Reconciling {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs","namespace":"demand-backup-1244"}, "namespace": "demand-backup-1244", "name": "backup-gcp-cs", "reconcileID": "f98d7eef-3e8e-4fcf-81e8-82cfdfab7fcd"} 2025-05-21T09:09:39.147Z DEBUG checking if backup is allowed {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs","namespace":"demand-backup-1244"}, "namespace": "demand-backup-1244", "name": "backup-gcp-cs", "reconcileID": "f98d7eef-3e8e-4fcf-81e8-82cfdfab7fcd", "cluster": "some-name", "namespace": "demand-backup-1244"} 2025-05-21T09:09:39.147Z DEBUG Checking for active jobs {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs","namespace":"demand-backup-1244"}, "namespace": "demand-backup-1244", "name": "backup-gcp-cs", "reconcileID": "f98d7eef-3e8e-4fcf-81e8-82cfdfab7fcd", "currentJob": {"Name":"backup-gcp-cs","Type":0}} 2025-05-21T09:09:39.182Z INFO Acquiring the backup lock {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs","namespace":"demand-backup-1244"}, "namespace": "demand-backup-1244", "name": "backup-gcp-cs", "reconcileID": "f98d7eef-3e8e-4fcf-81e8-82cfdfab7fcd"} 2025-05-21T09:09:39.182Z INFO Another backup is holding the lock {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs","namespace":"demand-backup-1244"}, "namespace": "demand-backup-1244", "name": "backup-gcp-cs", "reconcileID": "f98d7eef-3e8e-4fcf-81e8-82cfdfab7fcd", "holder": "backup-aws-s3-af618cd5-f107-4adc-b134-41114fe2cdfe"} 2025-05-21T09:09:39.183Z DEBUG Reconcile finished {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs","namespace":"demand-backup-1244"}, "namespace": "demand-backup-1244", "name": "backup-gcp-cs", "reconcileID": "f98d7eef-3e8e-4fcf-81e8-82cfdfab7fcd"} 2025-05-21T09:09:40.785Z DEBUG Reconciling {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-azure-blob","namespace":"demand-backup-1244"}, "namespace": "demand-backup-1244", "name": "backup-azure-blob", "reconcileID": "617d6085-f9a4-485b-9ddf-4faee1e5b667"} 2025-05-21T09:09:40.819Z DEBUG checking if backup is allowed {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-azure-blob","namespace":"demand-backup-1244"}, "namespace": "demand-backup-1244", "name": "backup-azure-blob", "reconcileID": "617d6085-f9a4-485b-9ddf-4faee1e5b667", "cluster": "some-name", "namespace": "demand-backup-1244"} 2025-05-21T09:09:40.819Z DEBUG Checking for active jobs {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-azure-blob","namespace":"demand-backup-1244"}, "namespace": "demand-backup-1244", "name": "backup-azure-blob", "reconcileID": "617d6085-f9a4-485b-9ddf-4faee1e5b667", "currentJob": {"Name":"backup-azure-blob","Type":0}} 2025-05-21T09:09:40.851Z INFO Acquiring the backup lock {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-azure-blob","namespace":"demand-backup-1244"}, "namespace": "demand-backup-1244", "name": "backup-azure-blob", "reconcileID": "617d6085-f9a4-485b-9ddf-4faee1e5b667"} 2025-05-21T09:09:40.851Z INFO Another backup is holding the lock {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-azure-blob","namespace":"demand-backup-1244"}, "namespace": "demand-backup-1244", "name": "backup-azure-blob", "reconcileID": "617d6085-f9a4-485b-9ddf-4faee1e5b667", "holder": "backup-aws-s3-af618cd5-f107-4adc-b134-41114fe2cdfe"} 2025-05-21T09:09:40.852Z DEBUG Reconcile finished {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-azure-blob","namespace":"demand-backup-1244"}, "namespace": "demand-backup-1244", "name": "backup-azure-blob", "reconcileID": "617d6085-f9a4-485b-9ddf-4faee1e5b667"} 2025-05-21T09:09:42.631Z DEBUG Reconciling {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-aws-s3","namespace":"demand-backup-1244"}, "namespace": "demand-backup-1244", "name": "backup-aws-s3", "reconcileID": "e2bcd8eb-de65-4f2c-872c-31177272f26c"} 2025-05-21T09:09:42.664Z DEBUG checking if backup is allowed {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-aws-s3","namespace":"demand-backup-1244"}, "namespace": "demand-backup-1244", "name": "backup-aws-s3", "reconcileID": "e2bcd8eb-de65-4f2c-872c-31177272f26c", "cluster": "some-name", "namespace": "demand-backup-1244"} 2025-05-21T09:09:42.664Z DEBUG Checking for active jobs {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-aws-s3","namespace":"demand-backup-1244"}, "namespace": "demand-backup-1244", "name": "backup-aws-s3", "reconcileID": "e2bcd8eb-de65-4f2c-872c-31177272f26c", "currentJob": {"Name":"backup-aws-s3","Type":0}} 2025-05-21T09:09:42.698Z INFO Acquiring the backup lock {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-aws-s3","namespace":"demand-backup-1244"}, "namespace": "demand-backup-1244", "name": "backup-aws-s3", "reconcileID": "e2bcd8eb-de65-4f2c-872c-31177272f26c"} 2025-05-21T09:09:42.699Z DEBUG backupStatus Got backup meta {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-aws-s3","namespace":"demand-backup-1244"}, "namespace": "demand-backup-1244", "name": "backup-aws-s3", "reconcileID": "e2bcd8eb-de65-4f2c-872c-31177272f26c", "backup": "backup-aws-s3", "pbmName": "2025-05-21T09:09:12Z", "meta": {"type":"logical","opid":"682d9838b3cfcdcb66ec6493","name":"2025-05-21T09:09:12Z","replsets":[{"name":"rs0","backup_name":"2025-05-21T09:09:12Z/rs0/metadata.json","oplog_name":"2025-05-21T09:09:12Z/rs0/oplog","start_ts":1747818553,"status":"running","last_transition_ts":1747818553,"first_write_ts":{"T":1747818553,"I":4},"last_write_ts":{"T":0,"I":0},"node":"some-name-rs0-2.some-name-rs0.demand-backup-1244.svc.cluster.local:27017","conditions":[{"timestamp":1747818553,"status":"running"}],"pbm_version":"2.9.1","mongo_version":"7.0.18-11"}],"compression":"gzip","store":{"type":"s3","s3":{"region":"us-east-1","endpointUrl":"","forcePathStyle":true,"bucket":"operator-testing","prefix":"psmdb-demand-backup","maxUploadParts":10000,"storageClass":"STANDARD","insecureSkipTLSVerify":false}},"size":0,"mongodb_version":"7.0.18-11","fcv":"7.0","start_ts":1747818552,"last_transition_ts":1747818552,"first_write_ts":{"T":1,"I":1},"last_write_ts":{"T":1,"I":1},"hb":{"T":1747818553,"I":4},"status":"starting","conditions":[{"timestamp":1747818552,"status":"starting"}],"n":[{"rs":"rs0","n":["some-name-rs0-1.some-name-rs0.demand-backup-1244.svc.cluster.local:27017","some-name-rs0-2.some-name-rs0.demand-backup-1244.svc.cluster.local:27017"],"ack":"some-name-rs0-2.some-name-rs0.demand-backup-1244.svc.cluster.local:27017"}],"pbm_version":"2.9.1","balancer":"off"}} 2025-05-21T09:09:42.702Z DEBUG Reconcile finished {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-aws-s3","namespace":"demand-backup-1244"}, "namespace": "demand-backup-1244", "name": "backup-aws-s3", "reconcileID": "e2bcd8eb-de65-4f2c-872c-31177272f26c"} 2025-05-21T09:09:44.184Z DEBUG Reconciling {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs","namespace":"demand-backup-1244"}, "namespace": "demand-backup-1244", "name": "backup-gcp-cs", "reconcileID": "e2bdc862-1637-4cbc-83f0-bd7f42ceb806"} 2025-05-21T09:09:44.328Z DEBUG Replset is not ready {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"demand-backup-1244"}, "namespace": "demand-backup-1244", "name": "some-name", "reconcileID": "b94ecbdd-1046-45a2-ac0f-ebcb8d478fb8", "liveMembers": 2, "pods": 3} 2025-05-21T09:10:14.503Z ERROR Reconciler error {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"demand-backup-1244"}, "namespace": "demand-backup-1244", "name": "some-name", "reconcileID": "b94ecbdd-1046-45a2-ac0f-ebcb8d478fb8", "error": "reconcile PBM: update PiTR config: create pbm object: create PBM connection to some-name-rs0-0.some-name-rs0.demand-backup-1244.svc.cluster.local:27017,some-name-rs0-1.some-name-rs0.demand-backup-1244.svc.cluster.local:27017,some-name-rs0-2.some-name-rs0.demand-backup-1244.svc.cluster.local:27017: create mongo connection: ping: server selection error: server selection timeout, current topology: { Type: ReplicaSetNoPrimary, Servers: [{ Addr: some-name-rs0-0.some-name-rs0.demand-backup-1244.svc.cluster.local:27017, Type: Unknown, Last error: dial tcp: lookup some-name-rs0-0.some-name-rs0.demand-backup-1244.svc.cluster.local: i/o timeout }, { Addr: some-name-rs0-1.some-name-rs0.demand-backup-1244.svc.cluster.local:27017, Type: RSSecondary, Tag sets: serviceName=some-name,nodeName=gke-jen-psmdb-1915-2d829-default-pool-a7be86c6-szf6,podName=some-name-rs0-1,region=us-central1,zone=us-central1-a, Average RTT: 1272339 }, { Addr: some-name-rs0-2.some-name-rs0.demand-backup-1244.svc.cluster.local:27017, Type: Unknown, Last error: dial tcp: lookup some-name-rs0-2.some-name-rs0.demand-backup-1244.svc.cluster.local: i/o timeout }, ] }", "errorVerbose": "server selection error: server selection timeout, current topology: { Type: ReplicaSetNoPrimary, Servers: [{ Addr: some-name-rs0-0.some-name-rs0.demand-backup-1244.svc.cluster.local:27017, Type: Unknown, Last error: dial tcp: lookup some-name-rs0-0.some-name-rs0.demand-backup-1244.svc.cluster.local: i/o timeout }, { Addr: some-name-rs0-1.some-name-rs0.demand-backup-1244.svc.cluster.local:27017, Type: RSSecondary, Tag sets: serviceName=some-name,nodeName=gke-jen-psmdb-1915-2d829-default-pool-a7be86c6-szf6,podName=some-name-rs0-1,region=us-central1,zone=us-central1-a, Average RTT: 1272339 }, { Addr: some-name-rs0-2.some-name-rs0.demand-backup-1244.svc.cluster.local:27017, Type: Unknown, Last error: dial tcp: lookup some-name-rs0-2.some-name-rs0.demand-backup-1244.svc.cluster.local: i/o timeout }, ] }\nping\ncreate mongo connection\ncreate PBM connection to some-name-rs0-0.some-name-rs0.demand-backup-1244.svc.cluster.local:27017,some-name-rs0-1.some-name-rs0.demand-backup-1244.svc.cluster.local:27017,some-name-rs0-2.some-name-rs0.demand-backup-1244.svc.cluster.local:27017\ngithub.com/percona/percona-server-mongodb-operator/pkg/psmdb/backup.NewPBM\n\t/go/src/github.com/percona/percona-server-mongodb-operator/pkg/psmdb/backup/pbm.go:212\ngithub.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodb.(*ReconcilePerconaServerMongoDB).reconcilePiTRConfig\n\t/go/src/github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodb/pbm.go:290\ngithub.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodb.(*ReconcilePerconaServerMongoDB).reconcilePBM\n\t/go/src/github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodb/pbm.go:34\ngithub.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodb.(*ReconcilePerconaServerMongoDB).Reconcile\n\t/go/src/github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodb/psmdb_controller.go:462\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.20.4/pkg/internal/controller/controller.go:119\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.20.4/pkg/internal/controller/controller.go:334\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.20.4/pkg/internal/controller/controller.go:294\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.20.4/pkg/internal/controller/controller.go:255\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1700\ncreate pbm object\ngithub.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodb.(*ReconcilePerconaServerMongoDB).reconcilePiTRConfig\n\t/go/src/github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodb/pbm.go:292\ngithub.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodb.(*ReconcilePerconaServerMongoDB).reconcilePBM\n\t/go/src/github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodb/pbm.go:34\ngithub.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodb.(*ReconcilePerconaServerMongoDB).Reconcile\n\t/go/src/github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodb/psmdb_controller.go:462\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.20.4/pkg/internal/controller/controller.go:119\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.20.4/pkg/internal/controller/controller.go:334\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.20.4/pkg/internal/controller/controller.go:294\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.20.4/pkg/internal/controller/controller.go:255\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1700\nupdate PiTR config\ngithub.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodb.(*ReconcilePerconaServerMongoDB).reconcilePBM\n\t/go/src/github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodb/pbm.go:35\ngithub.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodb.(*ReconcilePerconaServerMongoDB).Reconcile\n\t/go/src/github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodb/psmdb_controller.go:462\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.20.4/pkg/internal/controller/controller.go:119\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.20.4/pkg/internal/controller/controller.go:334\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.20.4/pkg/internal/controller/controller.go:294\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.20.4/pkg/internal/controller/controller.go:255\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1700\nreconcile PBM\ngithub.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodb.(*ReconcilePerconaServerMongoDB).Reconcile\n\t/go/src/github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodb/psmdb_controller.go:464\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.20.4/pkg/internal/controller/controller.go:119\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.20.4/pkg/internal/controller/controller.go:334\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.20.4/pkg/internal/controller/controller.go:294\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.20.4/pkg/internal/controller/controller.go:255\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1700"} sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler /go/pkg/mod/sigs.k8s.io/controller-runtime@v0.20.4/pkg/internal/controller/controller.go:347 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem /go/pkg/mod/sigs.k8s.io/controller-runtime@v0.20.4/pkg/internal/controller/controller.go:294 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2 /go/pkg/mod/sigs.k8s.io/controller-runtime@v0.20.4/pkg/internal/controller/controller.go:255 2025-05-21T09:10:24.897Z ERROR FULL CLUSTER CRASH {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"demand-backup-1244"}, "namespace": "demand-backup-1244", "name": "some-name", "reconcileID": "07a8e349-5afd-41cf-9632-2f92561fcde9", "error": "ping mongo: server selection error: server selection timeout, current topology: { Type: ReplicaSetNoPrimary, Servers: [{ Addr: some-name-rs0-0.some-name-rs0.demand-backup-1244.svc.cluster.local:27017, Type: Unknown }, { Addr: some-name-rs0-1.some-name-rs0.demand-backup-1244.svc.cluster.local:27017, Type: RSSecondary, Tag sets: serviceName=some-name,zone=us-central1-a,podName=some-name-rs0-1,nodeName=gke-jen-psmdb-1915-2d829-default-pool-a7be86c6-szf6,region=us-central1, Average RTT: 1446666 }, { Addr: some-name-rs0-2.some-name-rs0.demand-backup-1244.svc.cluster.local:27017, Type: Unknown, Last error: dial tcp: lookup some-name-rs0-2.some-name-rs0.demand-backup-1244.svc.cluster.local on 34.118.224.10:53: no such host }, ] }", "errorVerbose": "server selection error: server selection timeout, current topology: { Type: ReplicaSetNoPrimary, Servers: [{ Addr: some-name-rs0-0.some-name-rs0.demand-backup-1244.svc.cluster.local:27017, Type: Unknown }, { Addr: some-name-rs0-1.some-name-rs0.demand-backup-1244.svc.cluster.local:27017, Type: RSSecondary, Tag sets: serviceName=some-name,zone=us-central1-a,podName=some-name-rs0-1,nodeName=gke-jen-psmdb-1915-2d829-default-pool-a7be86c6-szf6,region=us-central1, Average RTT: 1446666 }, { Addr: some-name-rs0-2.some-name-rs0.demand-backup-1244.svc.cluster.local:27017, Type: Unknown, Last error: dial tcp: lookup some-name-rs0-2.some-name-rs0.demand-backup-1244.svc.cluster.local on 34.118.224.10:53: no such host }, ] }\nping mongo\ngithub.com/percona/percona-server-mongodb-operator/pkg/psmdb/mongo.Dial\n\t/go/src/github.com/percona/percona-server-mongodb-operator/pkg/psmdb/mongo/mongo.go:123\ngithub.com/percona/percona-server-mongodb-operator/pkg/psmdb.MongoClient\n\t/go/src/github.com/percona/percona-server-mongodb-operator/pkg/psmdb/client.go:62\ngithub.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodb.(*mongoClientProvider).Mongo\n\t/go/src/github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodb/connections.go:38\ngithub.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodb.(*ReconcilePerconaServerMongoDB).mongoClientWithRole\n\t/go/src/github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodb/connections.go:60\ngithub.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodb.(*ReconcilePerconaServerMongoDB).reconcileCluster\n\t/go/src/github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodb/mgo.go:98\ngithub.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodb.(*ReconcilePerconaServerMongoDB).reconcileReplsets\n\t/go/src/github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodb/psmdb_controller.go:546\ngithub.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodb.(*ReconcilePerconaServerMongoDB).Reconcile\n\t/go/src/github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodb/psmdb_controller.go:420\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.20.4/pkg/internal/controller/controller.go:119\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.20.4/pkg/internal/controller/controller.go:334\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.20.4/pkg/internal/controller/controller.go:294\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.20.4/pkg/internal/controller/controller.go:255\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1700"} github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodb.(*ReconcilePerconaServerMongoDB).reconcileCluster /go/src/github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodb/mgo.go:105 github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodb.(*ReconcilePerconaServerMongoDB).reconcileReplsets /go/src/github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodb/psmdb_controller.go:546 github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodb.(*ReconcilePerconaServerMongoDB).Reconcile /go/src/github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodb/psmdb_controller.go:420 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile /go/pkg/mod/sigs.k8s.io/controller-runtime@v0.20.4/pkg/internal/controller/controller.go:119 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler /go/pkg/mod/sigs.k8s.io/controller-runtime@v0.20.4/pkg/internal/controller/controller.go:334 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem /go/pkg/mod/sigs.k8s.io/controller-runtime@v0.20.4/pkg/internal/controller/controller.go:294 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2 /go/pkg/mod/sigs.k8s.io/controller-runtime@v0.20.4/pkg/internal/controller/controller.go:255 2025-05-21T09:10:24.898Z INFO handleReplicaSetNoPrimary Connecting to pod {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"demand-backup-1244"}, "namespace": "demand-backup-1244", "name": "some-name", "reconcileID": "07a8e349-5afd-41cf-9632-2f92561fcde9", "pod": "some-name-rs0-0", "user": "clusterAdmin"} 2025-05-21T09:10:34.910Z ERROR failed to reconcile cluster {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"demand-backup-1244"}, "namespace": "demand-backup-1244", "name": "some-name", "reconcileID": "07a8e349-5afd-41cf-9632-2f92561fcde9", "replset": "rs0", "error": "handle ReplicaSetNoPrimary: get standalone mongo client: ping mongo: server selection error: server selection timeout, current topology: { Type: Single, Servers: [{ Addr: some-name-rs0-0.some-name-rs0.demand-backup-1244.svc.cluster.local:27017, Type: Unknown, Last error: dial tcp: lookup some-name-rs0-0.some-name-rs0.demand-backup-1244.svc.cluster.local on 34.118.224.10:53: read udp 10.35.98.8:40021->34.118.224.10:53: i/o timeout }, ] }", "errorVerbose": "server selection error: server selection timeout, current topology: { Type: Single, Servers: [{ Addr: some-name-rs0-0.some-name-rs0.demand-backup-1244.svc.cluster.local:27017, Type: Unknown, Last error: dial tcp: lookup some-name-rs0-0.some-name-rs0.demand-backup-1244.svc.cluster.local on 34.118.224.10:53: read udp 10.35.98.8:40021->34.118.224.10:53: i/o timeout }, ] }\nping mongo\ngithub.com/percona/percona-server-mongodb-operator/pkg/psmdb/mongo.Dial\n\t/go/src/github.com/percona/percona-server-mongodb-operator/pkg/psmdb/mongo/mongo.go:123\ngithub.com/percona/percona-server-mongodb-operator/pkg/psmdb.StandaloneClient\n\t/go/src/github.com/percona/percona-server-mongodb-operator/pkg/psmdb/client.go:105\ngithub.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodb.(*mongoClientProvider).Standalone\n\t/go/src/github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodb/connections.go:56\ngithub.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodb.(*ReconcilePerconaServerMongoDB).standaloneClientWithRole\n\t/go/src/github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodb/connections.go:72\ngithub.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodb.(*ReconcilePerconaServerMongoDB).handleReplicaSetNoPrimary\n\t/go/src/github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodb/mgo.go:760\ngithub.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodb.(*ReconcilePerconaServerMongoDB).reconcileCluster\n\t/go/src/github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodb/mgo.go:107\ngithub.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodb.(*ReconcilePerconaServerMongoDB).reconcileReplsets\n\t/go/src/github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodb/psmdb_controller.go:546\ngithub.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodb.(*ReconcilePerconaServerMongoDB).Reconcile\n\t/go/src/github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodb/psmdb_controller.go:420\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.20.4/pkg/internal/controller/controller.go:119\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.20.4/pkg/internal/controller/controller.go:334\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.20.4/pkg/internal/controller/controller.go:294\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.20.4/pkg/internal/controller/controller.go:255\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1700\nget standalone mongo client\ngithub.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodb.(*ReconcilePerconaServerMongoDB).handleReplicaSetNoPrimary\n\t/go/src/github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodb/mgo.go:762\ngithub.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodb.(*ReconcilePerconaServerMongoDB).reconcileCluster\n\t/go/src/github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodb/mgo.go:107\ngithub.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodb.(*ReconcilePerconaServerMongoDB).reconcileReplsets\n\t/go/src/github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodb/psmdb_controller.go:546\ngithub.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodb.(*ReconcilePerconaServerMongoDB).Reconcile\n\t/go/src/github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodb/psmdb_controller.go:420\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.20.4/pkg/internal/controller/controller.go:119\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.20.4/pkg/internal/controller/controller.go:334\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.20.4/pkg/internal/controller/controller.go:294\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.20.4/pkg/internal/controller/controller.go:255\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1700\nhandle ReplicaSetNoPrimary\ngithub.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodb.(*ReconcilePerconaServerMongoDB).reconcileCluster\n\t/go/src/github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodb/mgo.go:109\ngithub.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodb.(*ReconcilePerconaServerMongoDB).reconcileReplsets\n\t/go/src/github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodb/psmdb_controller.go:546\ngithub.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodb.(*ReconcilePerconaServerMongoDB).Reconcile\n\t/go/src/github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodb/psmdb_controller.go:420\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.20.4/pkg/internal/controller/controller.go:119\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.20.4/pkg/internal/controller/controller.go:334\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.20.4/pkg/internal/controller/controller.go:294\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.20.4/pkg/internal/controller/controller.go:255\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1700"} github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodb.(*ReconcilePerconaServerMongoDB).reconcileReplsets /go/src/github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodb/psmdb_controller.go:548 github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodb.(*ReconcilePerconaServerMongoDB).Reconcile /go/src/github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodb/psmdb_controller.go:420 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile /go/pkg/mod/sigs.k8s.io/controller-runtime@v0.20.4/pkg/internal/controller/controller.go:119 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler /go/pkg/mod/sigs.k8s.io/controller-runtime@v0.20.4/pkg/internal/controller/controller.go:334 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem /go/pkg/mod/sigs.k8s.io/controller-runtime@v0.20.4/pkg/internal/controller/controller.go:294 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2 /go/pkg/mod/sigs.k8s.io/controller-runtime@v0.20.4/pkg/internal/controller/controller.go:255 2025-05-21T09:10:34.947Z ERROR Reconciler error {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"demand-backup-1244"}, "namespace": "demand-backup-1244", "name": "some-name", "reconcileID": "07a8e349-5afd-41cf-9632-2f92561fcde9", "error": "reconcile statefulsets: handle ReplicaSetNoPrimary: get standalone mongo client: ping mongo: server selection error: server selection timeout, current topology: { Type: Single, Servers: [{ Addr: some-name-rs0-0.some-name-rs0.demand-backup-1244.svc.cluster.local:27017, Type: Unknown, Last error: dial tcp: lookup some-name-rs0-0.some-name-rs0.demand-backup-1244.svc.cluster.local on 34.118.224.10:53: read udp 10.35.98.8:40021->34.118.224.10:53: i/o timeout }, ] }", "errorVerbose": "handle ReplicaSetNoPrimary: get standalone mongo client: ping mongo: server selection error: server selection timeout, current topology: { Type: Single, Servers: [{ Addr: some-name-rs0-0.some-name-rs0.demand-backup-1244.svc.cluster.local:27017, Type: Unknown, Last error: dial tcp: lookup some-name-rs0-0.some-name-rs0.demand-backup-1244.svc.cluster.local on 34.118.224.10:53: read udp 10.35.98.8:40021->34.118.224.10:53: i/o timeout }, ] }\nreconcile statefulsets\ngithub.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodb.(*ReconcilePerconaServerMongoDB).Reconcile\n\t/go/src/github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodb/psmdb_controller.go:422\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.20.4/pkg/internal/controller/controller.go:119\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.20.4/pkg/internal/controller/controller.go:334\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.20.4/pkg/internal/controller/controller.go:294\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.20.4/pkg/internal/controller/controller.go:255\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1700"} sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler /go/pkg/mod/sigs.k8s.io/controller-runtime@v0.20.4/pkg/internal/controller/controller.go:347 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem /go/pkg/mod/sigs.k8s.io/controller-runtime@v0.20.4/pkg/internal/controller/controller.go:294 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2 /go/pkg/mod/sigs.k8s.io/controller-runtime@v0.20.4/pkg/internal/controller/controller.go:255 2025-05-21T09:10:45.298Z ERROR FULL CLUSTER CRASH {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"demand-backup-1244"}, "namespace": "demand-backup-1244", "name": "some-name", "reconcileID": "43305cc5-1518-44c4-811a-c3153d924f12", "error": "ping mongo: server selection error: server selection timeout, current topology: { Type: ReplicaSetNoPrimary, Servers: [{ Addr: some-name-rs0-0.some-name-rs0.demand-backup-1244.svc.cluster.local:27017, Type: Unknown }, { Addr: some-name-rs0-1.some-name-rs0.demand-backup-1244.svc.cluster.local:27017, Type: Unknown }, { Addr: some-name-rs0-2.some-name-rs0.demand-backup-1244.svc.cluster.local:27017, Type: Unknown }, ] }", "errorVerbose": "server selection error: server selection timeout, current topology: { Type: ReplicaSetNoPrimary, Servers: [{ Addr: some-name-rs0-0.some-name-rs0.demand-backup-1244.svc.cluster.local:27017, Type: Unknown }, { Addr: some-name-rs0-1.some-name-rs0.demand-backup-1244.svc.cluster.local:27017, Type: Unknown }, { Addr: some-name-rs0-2.some-name-rs0.demand-backup-1244.svc.cluster.local:27017, Type: Unknown }, ] }\nping mongo\ngithub.com/percona/percona-server-mongodb-operator/pkg/psmdb/mongo.Dial\n\t/go/src/github.com/percona/percona-server-mongodb-operator/pkg/psmdb/mongo/mongo.go:123\ngithub.com/percona/percona-server-mongodb-operator/pkg/psmdb.MongoClient\n\t/go/src/github.com/percona/percona-server-mongodb-operator/pkg/psmdb/client.go:62\ngithub.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodb.(*mongoClientProvider).Mongo\n\t/go/src/github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodb/connections.go:38\ngithub.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodb.(*ReconcilePerconaServerMongoDB).mongoClientWithRole\n\t/go/src/github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodb/connections.go:60\ngithub.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodb.(*ReconcilePerconaServerMongoDB).reconcileCluster\n\t/go/src/github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodb/mgo.go:98\ngithub.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodb.(*ReconcilePerconaServerMongoDB).reconcileReplsets\n\t/go/src/github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodb/psmdb_controller.go:546\ngithub.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodb.(*ReconcilePerconaServerMongoDB).Reconcile\n\t/go/src/github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodb/psmdb_controller.go:420\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.20.4/pkg/internal/controller/controller.go:119\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.20.4/pkg/internal/controller/controller.go:334\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.20.4/pkg/internal/controller/controller.go:294\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.20.4/pkg/internal/controller/controller.go:255\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1700"} github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodb.(*ReconcilePerconaServerMongoDB).reconcileCluster /go/src/github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodb/mgo.go:105 github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodb.(*ReconcilePerconaServerMongoDB).reconcileReplsets /go/src/github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodb/psmdb_controller.go:546 github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodb.(*ReconcilePerconaServerMongoDB).Reconcile /go/src/github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodb/psmdb_controller.go:420 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile /go/pkg/mod/sigs.k8s.io/controller-runtime@v0.20.4/pkg/internal/controller/controller.go:119 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler /go/pkg/mod/sigs.k8s.io/controller-runtime@v0.20.4/pkg/internal/controller/controller.go:334 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem /go/pkg/mod/sigs.k8s.io/controller-runtime@v0.20.4/pkg/internal/controller/controller.go:294 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2 /go/pkg/mod/sigs.k8s.io/controller-runtime@v0.20.4/pkg/internal/controller/controller.go:255 2025-05-21T09:10:45.298Z INFO handleReplicaSetNoPrimary Connecting to pod {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"demand-backup-1244"}, "namespace": "demand-backup-1244", "name": "some-name", "reconcileID": "43305cc5-1518-44c4-811a-c3153d924f12", "pod": "some-name-rs0-0", "user": "clusterAdmin"} 2025-05-21T09:10:49.248Z DEBUG Running replSetReconfig config {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"demand-backup-1244"}, "namespace": "demand-backup-1244", "name": "some-name", "reconcileID": "43305cc5-1518-44c4-811a-c3153d924f12", "cfg": {"_id":"rs0","version":8,"members":[{"_id":0,"host":"some-name-rs0-0.some-name-rs0.demand-backup-1244.svc.cluster.local:27017","arbiterOnly":false,"buildIndexes":true,"hidden":false,"priority":2,"tags":{"nodeName":"gke-jen-psmdb-1915-2d829-default-pool-a7be86c6-v268","podName":"some-name-rs0-0","region":"us-central1","serviceName":"some-name","zone":"us-central1-a"},"secondaryDelaySecs":0,"votes":1},{"_id":1,"host":"some-name-rs0-1.some-name-rs0.demand-backup-1244.svc.cluster.local:27017","arbiterOnly":false,"buildIndexes":true,"hidden":false,"priority":2,"tags":{"nodeName":"gke-jen-psmdb-1915-2d829-default-pool-a7be86c6-szf6","podName":"some-name-rs0-1","region":"us-central1","serviceName":"some-name","zone":"us-central1-a"},"secondaryDelaySecs":0,"votes":1},{"_id":2,"host":"some-name-rs0-2.some-name-rs0.demand-backup-1244.svc.cluster.local:27017","arbiterOnly":false,"buildIndexes":true,"hidden":false,"priority":2,"tags":{"nodeName":"gke-jen-psmdb-1915-2d829-default-pool-a7be86c6-v268","podName":"some-name-rs0-2","region":"us-central1","serviceName":"some-name","zone":"us-central1-a"},"secondaryDelaySecs":0,"votes":1}],"protocolVersion":1,"settings":{"chainingAllowed":true,"heartbeatIntervalMillis":2000,"heartbeatTimeoutSecs":10,"electionTimeoutMillis":10000,"catchUpTimeoutMillis":-1,"getLastErrorDefaults":{"w":1,"wtimeout":0},"replicaSetId":"682d969565eebccafea9a458"},"writeConcernMajorityJournalDefault":true}} 2025-05-21T09:10:49.682Z DEBUG Cluster is not ready {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"demand-backup-1244"}, "namespace": "demand-backup-1244", "name": "some-name", "reconcileID": "43305cc5-1518-44c4-811a-c3153d924f12", "upgradeInProgress": false, "replsetsReady": 0, "clusterState": "error"} 2025-05-21T09:10:49.682Z INFO Cluster state changed {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"demand-backup-1244"}, "namespace": "demand-backup-1244", "name": "some-name", "reconcileID": "43305cc5-1518-44c4-811a-c3153d924f12", "previous": "error", "current": "initializing"} 2025-05-21T09:10:49.720Z DEBUG checking if backup is allowed {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs","namespace":"demand-backup-1244"}, "namespace": "demand-backup-1244", "name": "backup-gcp-cs", "reconcileID": "e2bdc862-1637-4cbc-83f0-bd7f42ceb806", "cluster": "some-name", "namespace": "demand-backup-1244"} 2025-05-21T09:10:49.720Z DEBUG Checking for active jobs {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs","namespace":"demand-backup-1244"}, "namespace": "demand-backup-1244", "name": "backup-gcp-cs", "reconcileID": "e2bdc862-1637-4cbc-83f0-bd7f42ceb806", "currentJob": {"Name":"backup-gcp-cs","Type":0}} 2025-05-21T09:10:52.254Z DEBUG Replset is not ready {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"demand-backup-1244"}, "namespace": "demand-backup-1244", "name": "some-name", "reconcileID": "316d74ac-d837-42ee-b27b-d497c45c4339", "liveMembers": 2, "pods": 3} 2025-05-21T09:10:52.482Z DEBUG Cluster is not ready {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"demand-backup-1244"}, "namespace": "demand-backup-1244", "name": "some-name", "reconcileID": "316d74ac-d837-42ee-b27b-d497c45c4339", "upgradeInProgress": false, "replsetsReady": 0, "clusterState": "initializing"} 2025-05-21T09:10:53.259Z DEBUG Replset is not ready {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"demand-backup-1244"}, "namespace": "demand-backup-1244", "name": "some-name", "reconcileID": "bf288340-baeb-4fd9-aa14-f44c9f54abee", "liveMembers": 2, "pods": 3} 2025-05-21T09:10:53.388Z DEBUG Cluster is not ready {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"demand-backup-1244"}, "namespace": "demand-backup-1244", "name": "some-name", "reconcileID": "bf288340-baeb-4fd9-aa14-f44c9f54abee", "upgradeInProgress": false, "replsetsReady": 0, "clusterState": "initializing"} 2025-05-21T09:10:53.388Z INFO Acquiring the backup lock {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs","namespace":"demand-backup-1244"}, "namespace": "demand-backup-1244", "name": "backup-gcp-cs", "reconcileID": "e2bdc862-1637-4cbc-83f0-bd7f42ceb806"} 2025-05-21T09:10:53.388Z INFO Another backup is holding the lock {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs","namespace":"demand-backup-1244"}, "namespace": "demand-backup-1244", "name": "backup-gcp-cs", "reconcileID": "e2bdc862-1637-4cbc-83f0-bd7f42ceb806", "holder": "backup-aws-s3-af618cd5-f107-4adc-b134-41114fe2cdfe"} 2025-05-21T09:10:53.391Z DEBUG Reconcile finished {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs","namespace":"demand-backup-1244"}, "namespace": "demand-backup-1244", "name": "backup-gcp-cs", "reconcileID": "e2bdc862-1637-4cbc-83f0-bd7f42ceb806"} 2025-05-21T09:10:53.391Z DEBUG Reconciling {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-azure-blob","namespace":"demand-backup-1244"}, "namespace": "demand-backup-1244", "name": "backup-azure-blob", "reconcileID": "21610a64-c3c8-446f-bfaa-2f7a38002188"} 2025-05-21T09:10:53.434Z DEBUG checking if backup is allowed {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-azure-blob","namespace":"demand-backup-1244"}, "namespace": "demand-backup-1244", "name": "backup-azure-blob", "reconcileID": "21610a64-c3c8-446f-bfaa-2f7a38002188", "cluster": "some-name", "namespace": "demand-backup-1244"} 2025-05-21T09:10:53.435Z ERROR failed to make backup {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-azure-blob","namespace":"demand-backup-1244"}, "namespace": "demand-backup-1244", "name": "backup-azure-blob", "reconcileID": "21610a64-c3c8-446f-bfaa-2f7a38002188", "backup": "backup-azure-blob", "error": "failed to run backup: spec.unsafeFlags.backupIfUnhealthy must be true to run backup on cluster with status initializing", "errorVerbose": "spec.unsafeFlags.backupIfUnhealthy must be true to run backup on cluster with status initializing\ngithub.com/percona/percona-server-mongodb-operator/pkg/apis/psmdb/v1.(*PerconaServerMongoDB).CanBackup\n\t/go/src/github.com/percona/percona-server-mongodb-operator/pkg/apis/psmdb/v1/psmdb_types.go:1258\ngithub.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodbbackup.(*ReconcilePerconaServerMongoDBBackup).reconcile\n\t/go/src/github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodbbackup/perconaservermongodbbackup_controller.go:232\ngithub.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodbbackup.(*ReconcilePerconaServerMongoDBBackup).Reconcile\n\t/go/src/github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodbbackup/perconaservermongodbbackup_controller.go:210\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.20.4/pkg/internal/controller/controller.go:119\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.20.4/pkg/internal/controller/controller.go:334\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.20.4/pkg/internal/controller/controller.go:294\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.20.4/pkg/internal/controller/controller.go:255\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1700\nfailed to run backup\ngithub.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodbbackup.(*ReconcilePerconaServerMongoDBBackup).reconcile\n\t/go/src/github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodbbackup/perconaservermongodbbackup_controller.go:233\ngithub.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodbbackup.(*ReconcilePerconaServerMongoDBBackup).Reconcile\n\t/go/src/github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodbbackup/perconaservermongodbbackup_controller.go:210\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.20.4/pkg/internal/controller/controller.go:119\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.20.4/pkg/internal/controller/controller.go:334\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.20.4/pkg/internal/controller/controller.go:294\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.20.4/pkg/internal/controller/controller.go:255\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1700"} github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodbbackup.(*ReconcilePerconaServerMongoDBBackup).Reconcile.func1 /go/src/github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodbbackup/perconaservermongodbbackup_controller.go:139 github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodbbackup.(*ReconcilePerconaServerMongoDBBackup).Reconcile /go/src/github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodbbackup/perconaservermongodbbackup_controller.go:212 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile /go/pkg/mod/sigs.k8s.io/controller-runtime@v0.20.4/pkg/internal/controller/controller.go:119 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler /go/pkg/mod/sigs.k8s.io/controller-runtime@v0.20.4/pkg/internal/controller/controller.go:334 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem /go/pkg/mod/sigs.k8s.io/controller-runtime@v0.20.4/pkg/internal/controller/controller.go:294 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2 /go/pkg/mod/sigs.k8s.io/controller-runtime@v0.20.4/pkg/internal/controller/controller.go:255 2025-05-21T09:10:53.436Z INFO Backup state changed {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-azure-blob","namespace":"demand-backup-1244"}, "namespace": "demand-backup-1244", "name": "backup-azure-blob", "reconcileID": "21610a64-c3c8-446f-bfaa-2f7a38002188", "previous": "waiting", "current": "error"} 2025-05-21T09:10:53.447Z INFO Releasing backup lock {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-azure-blob","namespace":"demand-backup-1244"}, "namespace": "demand-backup-1244", "name": "backup-azure-blob", "reconcileID": "21610a64-c3c8-446f-bfaa-2f7a38002188", "lease": "psmdb-some-name-backup-lock"} 2025-05-21T09:10:53.447Z ERROR failed to release the lock {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-azure-blob","namespace":"demand-backup-1244"}, "namespace": "demand-backup-1244", "name": "backup-azure-blob", "reconcileID": "21610a64-c3c8-446f-bfaa-2f7a38002188", "error": "not the holder", "errorVerbose": "not the holder\ngithub.com/percona/percona-server-mongodb-operator/pkg/k8s.init\n\t:1\nruntime.doInit1\n\t/usr/local/go/src/runtime/proc.go:7353\nruntime.doInit\n\t/usr/local/go/src/runtime/proc.go:7320\nruntime.main\n\t/usr/local/go/src/runtime/proc.go:254\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1700"} github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodbbackup.(*ReconcilePerconaServerMongoDBBackup).Reconcile.func1 /go/src/github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodbbackup/perconaservermongodbbackup_controller.go:155 github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodbbackup.(*ReconcilePerconaServerMongoDBBackup).Reconcile /go/src/github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodbbackup/perconaservermongodbbackup_controller.go:212 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile /go/pkg/mod/sigs.k8s.io/controller-runtime@v0.20.4/pkg/internal/controller/controller.go:119 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler /go/pkg/mod/sigs.k8s.io/controller-runtime@v0.20.4/pkg/internal/controller/controller.go:334 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem /go/pkg/mod/sigs.k8s.io/controller-runtime@v0.20.4/pkg/internal/controller/controller.go:294 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2 /go/pkg/mod/sigs.k8s.io/controller-runtime@v0.20.4/pkg/internal/controller/controller.go:255 2025-05-21T09:10:53.448Z DEBUG Reconcile finished {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-azure-blob","namespace":"demand-backup-1244"}, "namespace": "demand-backup-1244", "name": "backup-azure-blob", "reconcileID": "21610a64-c3c8-446f-bfaa-2f7a38002188"} 2025-05-21T09:10:53.448Z INFO Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-azure-blob","namespace":"demand-backup-1244"}, "namespace": "demand-backup-1244", "name": "backup-azure-blob", "reconcileID": "21610a64-c3c8-446f-bfaa-2f7a38002188"} 2025-05-21T09:10:53.448Z ERROR Reconciler error {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-azure-blob","namespace":"demand-backup-1244"}, "namespace": "demand-backup-1244", "name": "backup-azure-blob", "reconcileID": "21610a64-c3c8-446f-bfaa-2f7a38002188", "error": "reconcile backup: failed to run backup: spec.unsafeFlags.backupIfUnhealthy must be true to run backup on cluster with status initializing", "errorVerbose": "spec.unsafeFlags.backupIfUnhealthy must be true to run backup on cluster with status initializing\ngithub.com/percona/percona-server-mongodb-operator/pkg/apis/psmdb/v1.(*PerconaServerMongoDB).CanBackup\n\t/go/src/github.com/percona/percona-server-mongodb-operator/pkg/apis/psmdb/v1/psmdb_types.go:1258\ngithub.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodbbackup.(*ReconcilePerconaServerMongoDBBackup).reconcile\n\t/go/src/github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodbbackup/perconaservermongodbbackup_controller.go:232\ngithub.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodbbackup.(*ReconcilePerconaServerMongoDBBackup).Reconcile\n\t/go/src/github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodbbackup/perconaservermongodbbackup_controller.go:210\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.20.4/pkg/internal/controller/controller.go:119\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.20.4/pkg/internal/controller/controller.go:334\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.20.4/pkg/internal/controller/controller.go:294\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.20.4/pkg/internal/controller/controller.go:255\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1700\nfailed to run backup\ngithub.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodbbackup.(*ReconcilePerconaServerMongoDBBackup).reconcile\n\t/go/src/github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodbbackup/perconaservermongodbbackup_controller.go:233\ngithub.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodbbackup.(*ReconcilePerconaServerMongoDBBackup).Reconcile\n\t/go/src/github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodbbackup/perconaservermongodbbackup_controller.go:210\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.20.4/pkg/internal/controller/controller.go:119\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.20.4/pkg/internal/controller/controller.go:334\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.20.4/pkg/internal/controller/controller.go:294\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.20.4/pkg/internal/controller/controller.go:255\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1700\nreconcile backup\ngithub.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodbbackup.(*ReconcilePerconaServerMongoDBBackup).Reconcile\n\t/go/src/github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodbbackup/perconaservermongodbbackup_controller.go:212\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.20.4/pkg/internal/controller/controller.go:119\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.20.4/pkg/internal/controller/controller.go:334\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.20.4/pkg/internal/controller/controller.go:294\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.20.4/pkg/internal/controller/controller.go:255\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1700"} sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler /go/pkg/mod/sigs.k8s.io/controller-runtime@v0.20.4/pkg/internal/controller/controller.go:347 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem /go/pkg/mod/sigs.k8s.io/controller-runtime@v0.20.4/pkg/internal/controller/controller.go:294 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2 /go/pkg/mod/sigs.k8s.io/controller-runtime@v0.20.4/pkg/internal/controller/controller.go:255 2025-05-21T09:10:53.448Z DEBUG Reconciling {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-aws-s3","namespace":"demand-backup-1244"}, "namespace": "demand-backup-1244", "name": "backup-aws-s3", "reconcileID": "4076d553-da26-41f3-b705-d99424b8cd80"} 2025-05-21T09:10:53.498Z DEBUG checking if backup is allowed {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-aws-s3","namespace":"demand-backup-1244"}, "namespace": "demand-backup-1244", "name": "backup-aws-s3", "reconcileID": "4076d553-da26-41f3-b705-d99424b8cd80", "cluster": "some-name", "namespace": "demand-backup-1244"} 2025-05-21T09:10:53.500Z ERROR failed to make backup {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-aws-s3","namespace":"demand-backup-1244"}, "namespace": "demand-backup-1244", "name": "backup-aws-s3", "reconcileID": "4076d553-da26-41f3-b705-d99424b8cd80", "backup": "backup-aws-s3", "error": "failed to run backup: spec.unsafeFlags.backupIfUnhealthy must be true to run backup on cluster with status initializing", "errorVerbose": "spec.unsafeFlags.backupIfUnhealthy must be true to run backup on cluster with status initializing\ngithub.com/percona/percona-server-mongodb-operator/pkg/apis/psmdb/v1.(*PerconaServerMongoDB).CanBackup\n\t/go/src/github.com/percona/percona-server-mongodb-operator/pkg/apis/psmdb/v1/psmdb_types.go:1258\ngithub.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodbbackup.(*ReconcilePerconaServerMongoDBBackup).reconcile\n\t/go/src/github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodbbackup/perconaservermongodbbackup_controller.go:232\ngithub.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodbbackup.(*ReconcilePerconaServerMongoDBBackup).Reconcile\n\t/go/src/github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodbbackup/perconaservermongodbbackup_controller.go:210\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.20.4/pkg/internal/controller/controller.go:119\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.20.4/pkg/internal/controller/controller.go:334\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.20.4/pkg/internal/controller/controller.go:294\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.20.4/pkg/internal/controller/controller.go:255\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1700\nfailed to run backup\ngithub.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodbbackup.(*ReconcilePerconaServerMongoDBBackup).reconcile\n\t/go/src/github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodbbackup/perconaservermongodbbackup_controller.go:233\ngithub.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodbbackup.(*ReconcilePerconaServerMongoDBBackup).Reconcile\n\t/go/src/github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodbbackup/perconaservermongodbbackup_controller.go:210\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.20.4/pkg/internal/controller/controller.go:119\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.20.4/pkg/internal/controller/controller.go:334\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.20.4/pkg/internal/controller/controller.go:294\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.20.4/pkg/internal/controller/controller.go:255\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1700"} github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodbbackup.(*ReconcilePerconaServerMongoDBBackup).Reconcile.func1 /go/src/github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodbbackup/perconaservermongodbbackup_controller.go:139 github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodbbackup.(*ReconcilePerconaServerMongoDBBackup).Reconcile /go/src/github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodbbackup/perconaservermongodbbackup_controller.go:212 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile /go/pkg/mod/sigs.k8s.io/controller-runtime@v0.20.4/pkg/internal/controller/controller.go:119 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler /go/pkg/mod/sigs.k8s.io/controller-runtime@v0.20.4/pkg/internal/controller/controller.go:334 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem /go/pkg/mod/sigs.k8s.io/controller-runtime@v0.20.4/pkg/internal/controller/controller.go:294 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2 /go/pkg/mod/sigs.k8s.io/controller-runtime@v0.20.4/pkg/internal/controller/controller.go:255 2025-05-21T09:10:53.500Z INFO Backup state changed {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-aws-s3","namespace":"demand-backup-1244"}, "namespace": "demand-backup-1244", "name": "backup-aws-s3", "reconcileID": "4076d553-da26-41f3-b705-d99424b8cd80", "previous": "requested", "current": "error"} 2025-05-21T09:10:53.515Z INFO Releasing backup lock {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-aws-s3","namespace":"demand-backup-1244"}, "namespace": "demand-backup-1244", "name": "backup-aws-s3", "reconcileID": "4076d553-da26-41f3-b705-d99424b8cd80", "lease": "psmdb-some-name-backup-lock"} 2025-05-21T09:10:53.525Z DEBUG Reconcile finished {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-aws-s3","namespace":"demand-backup-1244"}, "namespace": "demand-backup-1244", "name": "backup-aws-s3", "reconcileID": "4076d553-da26-41f3-b705-d99424b8cd80"} 2025-05-21T09:10:53.525Z INFO Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-aws-s3","namespace":"demand-backup-1244"}, "namespace": "demand-backup-1244", "name": "backup-aws-s3", "reconcileID": "4076d553-da26-41f3-b705-d99424b8cd80"} 2025-05-21T09:10:53.525Z ERROR Reconciler error {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-aws-s3","namespace":"demand-backup-1244"}, "namespace": "demand-backup-1244", "name": "backup-aws-s3", "reconcileID": "4076d553-da26-41f3-b705-d99424b8cd80", "error": "reconcile backup: failed to run backup: spec.unsafeFlags.backupIfUnhealthy must be true to run backup on cluster with status initializing", "errorVerbose": "spec.unsafeFlags.backupIfUnhealthy must be true to run backup on cluster with status initializing\ngithub.com/percona/percona-server-mongodb-operator/pkg/apis/psmdb/v1.(*PerconaServerMongoDB).CanBackup\n\t/go/src/github.com/percona/percona-server-mongodb-operator/pkg/apis/psmdb/v1/psmdb_types.go:1258\ngithub.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodbbackup.(*ReconcilePerconaServerMongoDBBackup).reconcile\n\t/go/src/github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodbbackup/perconaservermongodbbackup_controller.go:232\ngithub.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodbbackup.(*ReconcilePerconaServerMongoDBBackup).Reconcile\n\t/go/src/github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodbbackup/perconaservermongodbbackup_controller.go:210\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.20.4/pkg/internal/controller/controller.go:119\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.20.4/pkg/internal/controller/controller.go:334\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.20.4/pkg/internal/controller/controller.go:294\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.20.4/pkg/internal/controller/controller.go:255\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1700\nfailed to run backup\ngithub.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodbbackup.(*ReconcilePerconaServerMongoDBBackup).reconcile\n\t/go/src/github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodbbackup/perconaservermongodbbackup_controller.go:233\ngithub.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodbbackup.(*ReconcilePerconaServerMongoDBBackup).Reconcile\n\t/go/src/github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodbbackup/perconaservermongodbbackup_controller.go:210\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.20.4/pkg/internal/controller/controller.go:119\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.20.4/pkg/internal/controller/controller.go:334\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.20.4/pkg/internal/controller/controller.go:294\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.20.4/pkg/internal/controller/controller.go:255\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1700\nreconcile backup\ngithub.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodbbackup.(*ReconcilePerconaServerMongoDBBackup).Reconcile\n\t/go/src/github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodbbackup/perconaservermongodbbackup_controller.go:212\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.20.4/pkg/internal/controller/controller.go:119\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.20.4/pkg/internal/controller/controller.go:334\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.20.4/pkg/internal/controller/controller.go:294\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.20.4/pkg/internal/controller/controller.go:255\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1700"} sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler /go/pkg/mod/sigs.k8s.io/controller-runtime@v0.20.4/pkg/internal/controller/controller.go:347 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem /go/pkg/mod/sigs.k8s.io/controller-runtime@v0.20.4/pkg/internal/controller/controller.go:294 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2 /go/pkg/mod/sigs.k8s.io/controller-runtime@v0.20.4/pkg/internal/controller/controller.go:255 2025-05-21T09:10:53.525Z DEBUG Reconciling {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-azure-blob","namespace":"demand-backup-1244"}, "namespace": "demand-backup-1244", "name": "backup-azure-blob", "reconcileID": "bb1ab511-65ad-4d45-aa41-4285686e9f67"} 2025-05-21T09:10:53.525Z DEBUG Reconcile finished {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-azure-blob","namespace":"demand-backup-1244"}, "namespace": "demand-backup-1244", "name": "backup-azure-blob", "reconcileID": "bb1ab511-65ad-4d45-aa41-4285686e9f67"} 2025-05-21T09:10:53.526Z DEBUG Reconciling {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-aws-s3","namespace":"demand-backup-1244"}, "namespace": "demand-backup-1244", "name": "backup-aws-s3", "reconcileID": "3ab53999-8f7b-42e3-83f5-be5c22b49840"} 2025-05-21T09:10:53.526Z DEBUG Reconcile finished {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-aws-s3","namespace":"demand-backup-1244"}, "namespace": "demand-backup-1244", "name": "backup-aws-s3", "reconcileID": "3ab53999-8f7b-42e3-83f5-be5c22b49840"} 2025-05-21T09:10:53.531Z DEBUG Reconciling {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-aws-s3","namespace":"demand-backup-1244"}, "namespace": "demand-backup-1244", "name": "backup-aws-s3", "reconcileID": "a30ad39e-3f64-40cf-b8d6-a3659fd18ffe"} 2025-05-21T09:10:53.531Z DEBUG Reconcile finished {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-aws-s3","namespace":"demand-backup-1244"}, "namespace": "demand-backup-1244", "name": "backup-aws-s3", "reconcileID": "a30ad39e-3f64-40cf-b8d6-a3659fd18ffe"} 2025-05-21T09:10:55.421Z DEBUG Replset is not ready {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"demand-backup-1244"}, "namespace": "demand-backup-1244", "name": "some-name", "reconcileID": "6109bfcc-bc66-4cae-8ef2-64333522c29a", "liveMembers": 2, "pods": 3} 2025-05-21T09:10:55.520Z DEBUG Cluster is not ready {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"demand-backup-1244"}, "namespace": "demand-backup-1244", "name": "some-name", "reconcileID": "6109bfcc-bc66-4cae-8ef2-64333522c29a", "upgradeInProgress": false, "replsetsReady": 0, "clusterState": "initializing"} NAME CLUSTER STORAGE DESTINATION TYPE STATUS COMPLETED AGE backup-aws-s3 some-name aws-s3 s3://operator-testing/psmdb-demand-backup/2025-05-21T09:09:12Z error 119s backup-azure-blob some-name azure-blob error 115s backup-gcp-cs some-name gcp-cs error 117s backup-minio some-name minio s3://operator-testing/2025-05-21T09:09:00Z logical ready 113s 2m Backup object psmdb-backup/backup-aws-s3 is in error state. something went wrong with operator or kubernetes cluster