Log: /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1904/e2e-tests/logs/demand-backup-fs.log WARNING: version difference between client (1.33) and server (1.30) exceeds the supported minor version skew of +/-1 WARNING: version difference between client (1.33) and server (1.30) exceeds the supported minor version skew of +/-1 WARNING: version difference between client (1.33) and server (1.30) exceeds the supported minor version skew of +/-1 ----------------------------------------------------------------------------------- get and delete old CRDs and RBAC ----------------------------------------------------------------------------------- error: the server doesn't have a resource type "perconaservermongodbbackups" + kubectl patch perconaservermongodbbackups.psmdb.percona.com -n sh --type=merge -p '{"metadata":{"finalizers":[]}}' error: the server doesn't have a resource type "perconaservermongodbbackups" error: the server doesn't have a resource type "perconaservermongodbrestores" + kubectl patch perconaservermongodbrestores.psmdb.percona.com -n sh --type=merge -p '{"metadata":{"finalizers":[]}}' error: the server doesn't have a resource type "perconaservermongodbrestores" error: the server doesn't have a resource type "perconaservermongodbs" + kubectl patch perconaservermongodbs.psmdb.percona.com -n sh --type=merge -p '{"metadata":{"finalizers":[]}}' error: the server doesn't have a resource type "perconaservermongodbs" Error from server (NotFound): customresourcedefinitions.apiextensions.k8s.io "null" not found Error from server (NotFound): customresourcedefinitions.apiextensions.k8s.io "null" not found Error from server (NotFound): customresourcedefinitions.apiextensions.k8s.io "null" not found Error from server (NotFound): customresourcedefinitions.apiextensions.k8s.io "null" not found ----------------------------------------------------------------------------------- destroy chaos-mesh ----------------------------------------------------------------------------------- error: resource(s) were provided, but no name was specified error: resource(s) were provided, but no name was specified error: resource(s) were provided, but no name was specified error: resource(s) were provided, but no name was specified error: resource(s) were provided, but no name was specified error: resource(s) were provided, but no name was specified ----------------------------------------------------------------------------------- cleaned up all old namespaces ----------------------------------------------------------------------------------- ----------------------------------------------------------------------------------- cleaned up old namespaces psmdb-operator ----------------------------------------------------------------------------------- namespace "gke-managed-cim" deleted ----------------------------------------------------------------------------------- create namespace psmdb-operator ----------------------------------------------------------------------------------- namespace "gke-managed-system" deleted namespace "gmp-public" deleted namespace "gmp-system" deleted namespace/psmdb-operator created Context "gke_cloud-dev-112233_us-central1-a_jen-psmdb-1904-a4e779d3-11-cluster4" modified. ----------------------------------------------------------------------------------- start PSMDB operator ----------------------------------------------------------------------------------- customresourcedefinition.apiextensions.k8s.io/perconaservermongodbbackups.psmdb.percona.com serverside-applied customresourcedefinition.apiextensions.k8s.io/perconaservermongodbrestores.psmdb.percona.com serverside-applied customresourcedefinition.apiextensions.k8s.io/perconaservermongodbs.psmdb.percona.com serverside-applied clusterrole.rbac.authorization.k8s.io/percona-server-mongodb-operator created serviceaccount/percona-server-mongodb-operator created clusterrolebinding.rbac.authorization.k8s.io/service-account-percona-server-mongodb-operator created deployment.apps/percona-server-mongodb-operator created waiting for pod/percona-server-mongodb-operator-85c5b67d9d-dtdpf to be ready.OK Print operator info from log 2025-05-14T15:57:19.350Z INFO setup Manager starting up {"gitCommit": "a4e779d35a323625e9652ede4a65877e643ad719", "gitBranch": "PR-1904-a4e779d3", "buildTime": "", "goVersion": "go1.24.3", "os": "linux", "arch": "amd64"} ----------------------------------------------------------------------------------- destroy chaos-mesh ----------------------------------------------------------------------------------- error: resource(s) were provided, but no name was specified error: resource(s) were provided, but no name was specified error: resource(s) were provided, but no name was specified error: resource(s) were provided, but no name was specified error: resource(s) were provided, but no name was specified error: resource(s) were provided, but no name was specified ----------------------------------------------------------------------------------- cleaned up all old namespaces ----------------------------------------------------------------------------------- ----------------------------------------------------------------------------------- cleaned up old namespaces demand-backup-fs-15947 ----------------------------------------------------------------------------------- namespace "gke-managed-cim" deleted namespace "gke-managed-system" deleted namespace "gmp-public" deleted namespace "gmp-system" deleted ----------------------------------------------------------------------------------- create namespace demand-backup-fs-15947 ----------------------------------------------------------------------------------- namespace/demand-backup-fs-15947 created Context "gke_cloud-dev-112233_us-central1-a_jen-psmdb-1904-a4e779d3-11-cluster4" modified. Error from server (NotFound): namespaces "storage" not found Error from server (NotFound): namespaces "storage" not found Error from server (NotFound): namespaces "storage" not found Error from server (NotFound): namespaces "storage" not found [2025-05-14T15:57:58+0000] deploying NFS server namespace/storage created persistentvolumeclaim/nfs-pvc created deployment.apps/nfs-server created service/nfs-service created [2025-05-14T15:58:11+0000] Waiting for nfs-server-d6d48c54d-qvn74 to start Running [2025-05-14T15:58:13+0000] Waiting for nfs-server-d6d48c54d-qvn74 to start Running [2025-05-14T15:58:16+0000] Waiting for nfs-server-d6d48c54d-qvn74 to start Running [2025-05-14T15:58:18+0000] Waiting for nfs-server-d6d48c54d-qvn74 to start Running [2025-05-14T15:58:20+0000] Waiting for nfs-server-d6d48c54d-qvn74 to start Running [2025-05-14T15:58:23+0000] Waiting for nfs-server-d6d48c54d-qvn74 to start Running [2025-05-14T15:58:30+0000] creating secrets and start client secret/some-users created deployment.apps/psmdb-client created [2025-05-14T15:58:32+0000] creating PSMDB cluster some-name perconaservermongodb.psmdb.percona.com/some-name created [2025-05-14T15:58:36+0000] wait for all 3 pods to start waiting for pod/some-name-rs0-0 to be ready............OK waiting for pod/some-name-rs0-1 to be ready.........OK waiting for pod/some-name-rs0-2 to be ready...........OK Waiting for cluster readyness [2025-05-14T16:00:38+0000] checking if statefulset created with expected config [2025-05-14T16:00:40+0000] creating user Percona Server for MongoDB shell version v4.4.29-28 connecting to: mongodb://some-name-rs0-0.some-name-rs0.demand-backup-fs-15947.svc.cluster.local:27017,some-name-rs0-1.some-name-rs0.demand-backup-fs-15947.svc.cluster.local:27017,some-name-rs0-2.some-name-rs0.demand-backup-fs-15947.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb&replicaSet=rs0&ssl=false Implicit session: session { "id" : UUID("02359a15-e1c2-4b63-9016-99581de2a912") } Percona Server for MongoDB server version: v8.0.8-3 WARNING: shell and server versions do not match Successfully added user: { "user" : "myApp", "roles" : [ { "db" : "myApp", "role" : "readWrite" } ] } bye [2025-05-14T16:00:45+0000] write initial data Percona Server for MongoDB shell version v4.4.29-28 connecting to: mongodb://some-name-rs0-0.some-name-rs0.demand-backup-fs-15947.svc.cluster.local:27017,some-name-rs0-1.some-name-rs0.demand-backup-fs-15947.svc.cluster.local:27017,some-name-rs0-2.some-name-rs0.demand-backup-fs-15947.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb&replicaSet=rs0&ssl=false Implicit session: session { "id" : UUID("8d1d7f4d-6abf-42aa-87fc-f5be558f4103") } Percona Server for MongoDB server version: v8.0.8-3 WARNING: shell and server versions do not match switched to db myApp WriteResult({ "nInserted" : 1 }) bye [2025-05-14T16:00:50+0000] running db.test.find() in myApp waiting for pbm-agent to be ready in some-name-rs0-0...2025-05-14T16:00:27.000+0000 I listening for the commands waiting for pbm-agent to be ready in some-name-rs0-1...2025-05-14T16:00:31.000+0000 I listening for the commands waiting for pbm-agent to be ready in some-name-rs0-2...2025-05-14T16:00:37.000+0000 I listening for the commands ----------------------------------------------------------------------------------- CASE 1: Logical backup and restore ----------------------------------------------------------------------------------- ----------------------------------------------------------------------------------- run backup backup-nfs-logical ----------------------------------------------------------------------------------- perconaservermongodbbackup.psmdb.percona.com/backup-nfs-logical created waiting for backup-nfs-logical to reach ready state..... Percona Server for MongoDB shell version v4.4.29-28 connecting to: mongodb://some-name-rs0-0.some-name-rs0.demand-backup-fs-15947.svc.cluster.local:27017,some-name-rs0-1.some-name-rs0.demand-backup-fs-15947.svc.cluster.local:27017,some-name-rs0-2.some-name-rs0.demand-backup-fs-15947.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb&replicaSet=rs0&ssl=false Implicit session: session { "id" : UUID("5d20ad40-21c7-4632-b93d-5f2b1180ef25") } Percona Server for MongoDB server version: v8.0.8-3 WARNING: shell and server versions do not match switched to db myApp WriteResult({ "nInserted" : 1 }) bye [2025-05-14T16:01:12+0000] running db.test.find() in myApp perconaservermongodbrestore.psmdb.percona.com/restore-backup-nfs-logical created Waiting for the psmdb-restore/restore-backup-nfs-logical object to be createdOK Waiting psmdb-restore/restore-backup-nfs-logical to reach state "ready" OK waiting for cluster readyness [2025-05-14T16:01:37+0000] running db.test.find() in myApp ----------------------------------------------------------------------------------- CASE 2: Logical backup and PiTR ----------------------------------------------------------------------------------- ----------------------------------------------------------------------------------- run backup backup-nfs-logical-pitr ----------------------------------------------------------------------------------- perconaservermongodbbackup.psmdb.percona.com/backup-nfs-logical-pitr created waiting for backup-nfs-logical-pitr to reach ready state.... Percona Server for MongoDB shell version v4.4.29-28 connecting to: mongodb://some-name-rs0-0.some-name-rs0.demand-backup-fs-15947.svc.cluster.local:27017,some-name-rs0-1.some-name-rs0.demand-backup-fs-15947.svc.cluster.local:27017,some-name-rs0-2.some-name-rs0.demand-backup-fs-15947.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb&replicaSet=rs0&ssl=false Implicit session: session { "id" : UUID("835cf246-015d-4e87-a16e-50e6143d4598") } Percona Server for MongoDB server version: v8.0.8-3 WARNING: shell and server versions do not match switched to db myApp WriteResult({ "nInserted" : 1 }) bye [2025-05-14T16:01:58+0000] running db.test.find() in myApp /usr/bin/date: invalid date ‘@null’ [2025-05-14T16:02:08+0000] Waiting for last oplog chunk () to be greater than last write (2025-05-14 16:01:50) /usr/bin/date: invalid date ‘@null’ [2025-05-14T16:02:20+0000] Waiting for last oplog chunk () to be greater than last write (2025-05-14 16:01:50) /usr/bin/date: invalid date ‘@null’ [2025-05-14T16:02:32+0000] Waiting for last oplog chunk () to be greater than last write (2025-05-14 16:01:50) /usr/bin/date: invalid date ‘@null’ [2025-05-14T16:02:45+0000] Waiting for last oplog chunk () to be greater than last write (2025-05-14 16:01:50) /usr/bin/date: invalid date ‘@null’ [2025-05-14T16:02:57+0000] Waiting for last oplog chunk () to be greater than last write (2025-05-14 16:01:50) /usr/bin/date: invalid date ‘@null’ [2025-05-14T16:03:10+0000] Waiting for last oplog chunk () to be greater than last write (2025-05-14 16:01:50) [2025-05-14T16:03:23+0000] Waiting for last oplog chunk (2025-05-14 16:03:09) to be greater than last write (2025-05-14 16:01:50) [2025-05-14T16:03:35+0000] dropping test collection Percona Server for MongoDB shell version v4.4.29-28 connecting to: mongodb://some-name-rs0-0.some-name-rs0.demand-backup-fs-15947.svc.cluster.local:27017,some-name-rs0-1.some-name-rs0.demand-backup-fs-15947.svc.cluster.local:27017,some-name-rs0-2.some-name-rs0.demand-backup-fs-15947.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb&replicaSet=rs0&ssl=false Implicit session: session { "id" : UUID("8f4e2d59-2aa1-414b-9271-903164839c80") } Percona Server for MongoDB server version: v8.0.8-3 WARNING: shell and server versions do not match switched to db myApp true bye [2025-05-14T16:03:38+0000] checking pitr... backup: backup-nfs-logical-pitr target: 2025-05-14 16:03:09 perconaservermongodbrestore.psmdb.percona.com/restore-backup-nfs-logical-pitr created Waiting for the psmdb-restore/restore-backup-nfs-logical-pitr object to be createdOK Waiting psmdb-restore/restore-backup-nfs-logical-pitr to reach state "ready" OK waiting for cluster readyness [2025-05-14T16:04:14+0000] running db.test.find() in myApp error: Internal error occurred: error sending request: Post "https://10.211.0.85:10250/exec/demand-backup-fs-15947/psmdb-client-66f577db5f-vjq8h/psmdb-client?command=bash&command=-c&command=printf+%27use+myApp%5Cn+db.test.find%28%29%5Cn%27+%7C+mongo+mongodb%3A%2F%2FmyApp%3AmyPass%40some-name-rs0.demand-backup-fs-15947.svc.cluster.local%2Fadmin%3Fssl%3Dfalse%5C%26replicaSet%3Drs0+&error=1&output=1": No agent available error: Internal error occurred: error sending request: Post "https://10.211.0.85:10250/exec/demand-backup-fs-15947/psmdb-client-66f577db5f-vjq8h/psmdb-client?command=bash&command=-c&command=printf+%27use+myApp%5Cn+db.test.find%28%29%5Cn%27+%7C+mongo+mongodb%3A%2F%2FmyApp%3AmyPass%40some-name-rs0.demand-backup-fs-15947.svc.cluster.local%2Fadmin%3Fssl%3Dfalse%5C%26replicaSet%3Drs0+&error=1&output=1": No agent available command terminated with exit code 1 command terminated with exit code 1 --- /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1904/e2e-tests/demand-backup-fs/compare/find-3rd.json 2025-05-14 15:29:38.647696530 +0000 +++ /tmp/tmp.5U4bn14O1B/find-3rd 2025-05-14 16:04:48.100302986 +0000 @@ -1,4 +1,12 @@ -switched to db myApp -{ "_id" : , "x" : 100500 } -{ "_id" : , "x" : 100502 } -bye +{"t":{"$date":"2025-05-14T16:04:40.030Z"},"s":"I", "c":"NETWORK", "id":4333208, "ctx":"ReplicaSetMonitor-TaskExecutor","msg":"RSM host selection timeout","attr":{"replicaSet":"rs0","error":"FailedToSatisfyReadPreference: Could not find host matching read preference { mode: \"primary\", tags: [ {} ] } for set rs0"}} +Error: Could not find host matching read preference { mode: "primary", tags: [ {} ] } for set rs0 : +connect@src/mongo/shell/mongo.js:374:17 +@(connect):2:6 +exception: connect failed +exiting with code 1 +{"t":{"$date":"2025-05-14T16:04:40.030Z"},"s":"I", "c":"NETWORK", "id":4333208, "ctx":"ReplicaSetMonitor-TaskExecutor","msg":"RSM host selection timeout","attr":{"replicaSet":"rs0","error":"FailedToSatisfyReadPreference: Could not find host matching read preference { mode: \"primary\", tags: [ {} ] } for set rs0"}} +Error: Could not find host matching read preference { mode: "primary", tags: [ {} ] } for set rs0 : +connect@src/mongo/shell/mongo.js:374:17 +@(connect):2:6 +exception: connect failed +exiting with code 1