++ echo 'Log: /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1352/e2e-tests/logs/demand-backup-sharded.log'
Log: /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1352/e2e-tests/logs/demand-backup-sharded.log
++ '[' -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1352/e2e-tests/conf/cloud-secret.yml ']'
++ SKIP_BACKUPS_TO_AWS_GCP_AZURE=
++ oc get projects
++ kubectl get nodes
++ grep '^minikube'
+++ kubectl version -o json
+++ jq -r .serverVersion.gitVersion
+++ grep '\-eks\-'
++ '[' ']'
++ EKS=0
+++ kubectl version -o json
+++ jq -r .serverVersion.gitVersion
+++ grep gke
++ '[' v1.24.17-gke.200 ']'
++ GKE=1
+++ kubectl version -o json
+++ jq -r '.serverVersion.major + "." + .serverVersion.minor'
+++ /usr/bin/sed -r 's/[^0-9.]+//g'
++ KUBE_VERSION=1.24
+ set_debug
+ [[ 0 == 1 ]]
+ set +o xtrace


-----------------------------------------------------------------------------------
get and delete old CRDs and RBAC
-----------------------------------------------------------------------------------

error: the server doesn't have a resource type "perconaservermongodbbackups"
+ kubectl patch perconaservermongodbbackups.psmdb.percona.com -n sh --type=merge -p '{"metadata":{"finalizers":[]}}'
error: the server doesn't have a resource type "perconaservermongodbbackups"
error: the server doesn't have a resource type "perconaservermongodbrestores"
+ kubectl patch perconaservermongodbrestores.psmdb.percona.com -n sh --type=merge -p '{"metadata":{"finalizers":[]}}'
error: the server doesn't have a resource type "perconaservermongodbrestores"
error: the server doesn't have a resource type "perconaservermongodbs"
+ kubectl patch perconaservermongodbs.psmdb.percona.com -n sh --type=merge -p '{"metadata":{"finalizers":[]}}'
error: the server doesn't have a resource type "perconaservermongodbs"
Error from server (NotFound): customresourcedefinitions.apiextensions.k8s.io "null" not found
Error from server (NotFound): customresourcedefinitions.apiextensions.k8s.io "null" not found
Error from server (NotFound): customresourcedefinitions.apiextensions.k8s.io "null" not found
Error from server (NotFound): customresourcedefinitions.apiextensions.k8s.io "null" not found


-----------------------------------------------------------------------------------
destroy chaos-mesh
-----------------------------------------------------------------------------------

error: resource(s) were provided, but no name was specified
error: resource(s) were provided, but no name was specified
error: resource(s) were provided, but no name was specified
error: resource(s) were provided, but no name was specified
error: resource(s) were provided, but no name was specified
error: resource(s) were provided, but no name was specified


-----------------------------------------------------------------------------------
cleaned up all old namespaces
-----------------------------------------------------------------------------------



-----------------------------------------------------------------------------------
cleaned up old namespaces demand-backup-sharded-26392
-----------------------------------------------------------------------------------

error: resource(s) were provided, but no name was specified


-----------------------------------------------------------------------------------
create namespace demand-backup-sharded-26392
-----------------------------------------------------------------------------------

namespace/demand-backup-sharded-26392 created
Context "gke_cloud-dev-112233_us-central1-a_jen-psmdb-1352-9793733e-5-cluster1" modified.


-----------------------------------------------------------------------------------
start PSMDB operator
-----------------------------------------------------------------------------------

customresourcedefinition.apiextensions.k8s.io/perconaservermongodbbackups.psmdb.percona.com serverside-applied
customresourcedefinition.apiextensions.k8s.io/perconaservermongodbrestores.psmdb.percona.com serverside-applied
customresourcedefinition.apiextensions.k8s.io/perconaservermongodbs.psmdb.percona.com serverside-applied
role.rbac.authorization.k8s.io/percona-server-mongodb-operator created
serviceaccount/percona-server-mongodb-operator created
rolebinding.rbac.authorization.k8s.io/service-account-percona-server-mongodb-operator created
deployment.apps/percona-server-mongodb-operator created
waiting for pod/percona-server-mongodb-operator-6b6bcb669-6cf88 to be ready.OK


-----------------------------------------------------------------------------------
install Minio
-----------------------------------------------------------------------------------

Error: uninstall: Release not loaded: minio-service: release: not found
"minio" has been removed from your repositories
"minio" has been added to your repositories
NAME: minio-service
LAST DEPLOYED: Thu Oct  5 13:36:37 2023
NAMESPACE: demand-backup-sharded-26392
STATUS: deployed
REVISION: 1
TEST SUITE: None
NOTES:
Minio can be accessed via port 9000 on the following DNS name from within your cluster:
minio-service.demand-backup-sharded-26392.svc.cluster.local

To access Minio from localhost, run the below commands:

  1. export POD_NAME=$(kubectl get pods --namespace demand-backup-sharded-26392 -l "release=minio-service" -o jsonpath="{.items[0].metadata.name}")

  2. kubectl port-forward $POD_NAME 9000 --namespace demand-backup-sharded-26392

Read more about port forwarding here: http://kubernetes.io/docs/user-guide/kubectl/kubectl_port-forward/

You can now access Minio server on http://localhost:9000. Follow the below steps to connect to Minio server with mc client:

  1. Download the Minio mc client - https://docs.minio.io/docs/minio-client-quickstart-guide

  2. Get the ACCESS_KEY=$(kubectl get secret minio-service -o jsonpath="{.data.accesskey}" | base64 --decode) and the SECRET_KEY=$(kubectl get secret minio-service -o jsonpath="{.data.secretkey}" | base64 --decode)

  3. mc alias set minio-service-local http://localhost:9000 "$ACCESS_KEY" "$SECRET_KEY" --api s3v4

  4. mc ls minio-service-local

Alternately, you can use your browser or the Minio SDK to access the server - https://docs.minio.io/categories/17
waiting for pod/minio-service-5fc4dc898-8nhvn to be ready......OK
pod "aws-cli" deleted
If you don't see a command prompt, try pressing enter.


-----------------------------------------------------------------------------------
create first PSMDB cluster
-----------------------------------------------------------------------------------

secret/some-users created
deployment.apps/psmdb-client created


-----------------------------------------------------------------------------------
create secrets for cloud storages
-----------------------------------------------------------------------------------

secret/minio-secret created
secret/aws-s3-secret created
secret/gcp-cs-secret created
secret/azure-secret created
runtimeclass.node.k8s.io/container-rc created
perconaservermongodb.psmdb.percona.com/some-name created


-----------------------------------------------------------------------------------
check if all 3 Pods started
-----------------------------------------------------------------------------------

waiting for pod/some-name-rs0-0 to be ready............OK
waiting for pod/some-name-rs0-1 to be ready........OK
waiting for pod/some-name-rs0-2 to be ready.....OK
Waiting for cluster readyness................
waiting for pod/some-name-cfg-0 to be ready.OK
waiting for pod/some-name-cfg-1 to be ready.OK
waiting for pod/some-name-cfg-2 to be ready.OK
waiting for pod/some-name-mongos-0 to be ready.OK
waiting for pod/some-name-mongos-1 to be ready.OK
waiting for pod/some-name-mongos-2 to be ready.OK
Waiting for cluster readyness


-----------------------------------------------------------------------------------
check if service and statefulset created with expected config
-----------------------------------------------------------------------------------

secret/some-name-mongos created
waiting for pod/some-name-mongos-0 to be ready.OK
waiting for pod/some-name-mongos-1 to be ready.OK
waiting for pod/some-name-mongos-2 to be ready.OK
Waiting for cluster readyness.................................................................................


-----------------------------------------------------------------------------------
write data, read from all
-----------------------------------------------------------------------------------

Percona Server for MongoDB shell version v4.4.24-23
connecting to: mongodb://some-name-mongos.demand-backup-sharded-26392.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb
Implicit session: session { "id" : UUID("8a99a807-5557-43da-b071-204850ce8aba") }
Percona Server for MongoDB server version: v5.0.20-17
WARNING: shell and server versions do not match
Successfully added user: {
	"user" : "myApp",
	"roles" : [
		{
			"db" : "myApp",
			"role" : "readWrite"
		},
		{
			"db" : "myApp1",
			"role" : "readWrite"
		},
		{
			"db" : "myApp2",
			"role" : "readWrite"
		}
	]
}
bye
Percona Server for MongoDB shell version v4.4.24-23
connecting to: mongodb://some-name-mongos.demand-backup-sharded-26392.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb
Implicit session: session { "id" : UUID("b1c4a0c8-8db2-4856-94de-b4eb3773fcb8") }
Percona Server for MongoDB server version: v5.0.20-17
WARNING: shell and server versions do not match
{
	"ok" : 1,
	"$clusterTime" : {
		"clusterTime" : Timestamp(1696513461, 4),
		"signature" : {
			"hash" : BinData(0,"iuvF6d8cKCYDznk2EayphY0DdG0="),
			"keyId" : NumberLong("7286468427764465682")
		}
	},
	"operationTime" : Timestamp(1696513461, 2)
}
bye
Percona Server for MongoDB shell version v4.4.24-23
connecting to: mongodb://some-name-mongos.demand-backup-sharded-26392.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb
Implicit session: session { "id" : UUID("37db9ca4-55ee-452c-a8cf-3e5966802eec") }
Percona Server for MongoDB server version: v5.0.20-17
WARNING: shell and server versions do not match
{
	"ok" : 1,
	"$clusterTime" : {
		"clusterTime" : Timestamp(1696513464, 33),
		"signature" : {
			"hash" : BinData(0,"/d4Bs2xcVHhphLR4phDHBqwMA8s="),
			"keyId" : NumberLong("7286468427764465682")
		}
	},
	"operationTime" : Timestamp(1696513464, 20)
}
bye
Percona Server for MongoDB shell version v4.4.24-23
connecting to: mongodb://some-name-mongos.demand-backup-sharded-26392.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb
Implicit session: session { "id" : UUID("e0208b8f-c01e-490f-b741-4c27f433194b") }
Percona Server for MongoDB server version: v5.0.20-17
WARNING: shell and server versions do not match
{
	"ok" : 1,
	"$clusterTime" : {
		"clusterTime" : Timestamp(1696513468, 1),
		"signature" : {
			"hash" : BinData(0,"rV1yoAI0Z0xoBQNW75ugYhrtKIs="),
			"keyId" : NumberLong("7286468427764465682")
		}
	},
	"operationTime" : Timestamp(1696513467, 39)
}
bye
Percona Server for MongoDB shell version v4.4.24-23
connecting to: mongodb://some-name-mongos.demand-backup-sharded-26392.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb
Implicit session: session { "id" : UUID("087abade-296f-4121-aec5-f9cf75dd102a") }
Percona Server for MongoDB server version: v5.0.20-17
WARNING: shell and server versions do not match
switched to db myApp
WriteResult({ "nInserted" : 1 })
bye
Percona Server for MongoDB shell version v4.4.24-23
connecting to: mongodb://some-name-mongos.demand-backup-sharded-26392.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb
Implicit session: session { "id" : UUID("7bf9d12c-4739-41d3-9a78-d9039d031c82") }
Percona Server for MongoDB server version: v5.0.20-17
WARNING: shell and server versions do not match
switched to db myApp1
WriteResult({ "nInserted" : 1 })
bye
Percona Server for MongoDB shell version v4.4.24-23
connecting to: mongodb://some-name-mongos.demand-backup-sharded-26392.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb
Implicit session: session { "id" : UUID("7f5cb0b0-819e-4f71-a584-5c478cd43f5a") }
Percona Server for MongoDB server version: v5.0.20-17
WARNING: shell and server versions do not match
switched to db myApp2
WriteResult({ "nInserted" : 1 })
bye
some-name-rs0-0
some-name-rs0-1
some-name-rs0-2
some-name-rs1-0
some-name-rs1-1
some-name-rs1-2
some-name-rs2-0
some-name-rs2-1
some-name-rs2-2


-----------------------------------------------------------------------------------
run backups
-----------------------------------------------------------------------------------



-----------------------------------------------------------------------------------
run backup backup-minio
-----------------------------------------------------------------------------------

perconaservermongodbbackup.psmdb.percona.com/backup-minio created


-----------------------------------------------------------------------------------
run backup backup-aws-s3
-----------------------------------------------------------------------------------

perconaservermongodbbackup.psmdb.percona.com/backup-aws-s3 created


-----------------------------------------------------------------------------------
run backup backup-gcp-cs
-----------------------------------------------------------------------------------

perconaservermongodbbackup.psmdb.percona.com/backup-gcp-cs created


-----------------------------------------------------------------------------------
run backup backup-azure-blob
-----------------------------------------------------------------------------------

perconaservermongodbbackup.psmdb.percona.com/backup-azure-blob created
backup-aws-s3..............................
backup-gcp-cs..............
backup-azure-blob................
backup-minio.


-----------------------------------------------------------------------------------
check backup and restore -- aws-s3
-----------------------------------------------------------------------------------

Percona Server for MongoDB shell version v4.4.24-23
connecting to: mongodb://some-name-mongos.demand-backup-sharded-26392.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb
Implicit session: session { "id" : UUID("6bbb0146-cd2e-4e6b-b37d-bcce2d485d52") }
Percona Server for MongoDB server version: v5.0.20-17
WARNING: shell and server versions do not match
switched to db myApp
WriteResult({ "nInserted" : 1 })
bye
Percona Server for MongoDB shell version v4.4.24-23
connecting to: mongodb://some-name-mongos.demand-backup-sharded-26392.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb
Implicit session: session { "id" : UUID("7dd1ac94-9909-4589-805a-c58f7d353202") }
Percona Server for MongoDB server version: v5.0.20-17
WARNING: shell and server versions do not match
switched to db myApp1
WriteResult({ "nInserted" : 1 })
bye
Percona Server for MongoDB shell version v4.4.24-23
connecting to: mongodb://some-name-mongos.demand-backup-sharded-26392.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb
Implicit session: session { "id" : UUID("52195e81-b1aa-45e5-93dd-2ddef4a79dd5") }
Percona Server for MongoDB server version: v5.0.20-17
WARNING: shell and server versions do not match
switched to db myApp2
WriteResult({ "nInserted" : 1 })
bye
perconaservermongodbrestore.psmdb.percona.com/restore-backup-aws-s3 created
waiting psmdb-restore/backup-aws-s3 to reach ready state.............
+ '[' 1 -eq 1 ']'
+ wait_cluster_consistency some-name
+ local cluster_name=some-name
+ local wait_time=32
+ retry=0
+ sleep 7
+ echo -n 'waiting for cluster readyness'
waiting for cluster readyness++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.o77Hh1LKYQ
+++ mktemp
++ local LAST_ERR=/tmp/tmp.gjzInFeRbT
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 0 ']'
++ break
++ cat /tmp/tmp.o77Hh1LKYQ
++ cat /tmp/tmp.gjzInFeRbT
++ rm /tmp/tmp.o77Hh1LKYQ /tmp/tmp.gjzInFeRbT
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 1 -ge 32 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.Wl2ghdCHM3
+++ mktemp
++ local LAST_ERR=/tmp/tmp.GtbuCtsp6R
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 0 ']'
++ break
++ cat /tmp/tmp.Wl2ghdCHM3
++ cat /tmp/tmp.GtbuCtsp6R
++ rm /tmp/tmp.Wl2ghdCHM3 /tmp/tmp.GtbuCtsp6R
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 2 -ge 32 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.mbwEzEnXXg
+++ mktemp
++ local LAST_ERR=/tmp/tmp.Dhq9sRl99c
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 0 ']'
++ break
++ cat /tmp/tmp.mbwEzEnXXg
++ cat /tmp/tmp.Dhq9sRl99c
++ rm /tmp/tmp.mbwEzEnXXg /tmp/tmp.Dhq9sRl99c
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 3 -ge 32 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.I6YZoKyb2V
+++ mktemp
++ local LAST_ERR=/tmp/tmp.SwzBMCCW36
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 0 ']'
++ break
++ cat /tmp/tmp.I6YZoKyb2V
++ cat /tmp/tmp.SwzBMCCW36
++ rm /tmp/tmp.I6YZoKyb2V /tmp/tmp.SwzBMCCW36
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 4 -ge 32 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.c5R7eakB8e
+++ mktemp
++ local LAST_ERR=/tmp/tmp.3z4h13RK3V
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 0 ']'
++ break
++ cat /tmp/tmp.c5R7eakB8e
++ cat /tmp/tmp.3z4h13RK3V
++ rm /tmp/tmp.c5R7eakB8e /tmp/tmp.3z4h13RK3V
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 5 -ge 32 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.iAVMmYJ17p
+++ mktemp
++ local LAST_ERR=/tmp/tmp.P7bHxM0KqN
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 0 ']'
++ break
++ cat /tmp/tmp.iAVMmYJ17p
++ cat /tmp/tmp.P7bHxM0KqN
++ rm /tmp/tmp.iAVMmYJ17p /tmp/tmp.P7bHxM0KqN
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 6 -ge 32 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.Vr9ymzp3ov
+++ mktemp
++ local LAST_ERR=/tmp/tmp.O0AeymAN6A
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 0 ']'
++ break
++ cat /tmp/tmp.Vr9ymzp3ov
++ cat /tmp/tmp.O0AeymAN6A
++ rm /tmp/tmp.Vr9ymzp3ov /tmp/tmp.O0AeymAN6A
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 7 -ge 32 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.DoQHutg3no
+++ mktemp
++ local LAST_ERR=/tmp/tmp.sLBT1HgfJE
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 0 ']'
++ break
++ cat /tmp/tmp.DoQHutg3no
++ cat /tmp/tmp.sLBT1HgfJE
++ rm /tmp/tmp.DoQHutg3no /tmp/tmp.sLBT1HgfJE
++ return 0
+ [[ ready == \r\e\a\d\y ]]
+ check_data
+ local postfix=
++ seq 0 2
+ for i in '$(seq 0 2)'
+ compare_mongos_cmd find myApp:myPass@some-name-mongos.demand-backup-sharded-26392 '' .svc.cluster.local myApp test
+ local command=find
+ local uri=myApp:myPass@some-name-mongos.demand-backup-sharded-26392
+ local postfix=
+ local suffix=.svc.cluster.local
+ local database=myApp
+ local collection=test
+ egrep -v 'I NETWORK|W NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match'
+ /usr/bin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
+ run_mongos 'use myApp\n db.test.find()' myApp:myPass@some-name-mongos.demand-backup-sharded-26392 mongodb .svc.cluster.local
+ local 'command=use myApp\n db.test.find()'
+ local uri=myApp:myPass@some-name-mongos.demand-backup-sharded-26392
+ local driver=mongodb
+ local suffix=.svc.cluster.local
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.YMn9IjzCG2
+++ mktemp
++ local LAST_ERR=/tmp/tmp.6eDbEK0oeu
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 0 ']'
++ break
++ cat /tmp/tmp.YMn9IjzCG2
++ cat /tmp/tmp.6eDbEK0oeu
++ rm /tmp/tmp.YMn9IjzCG2 /tmp/tmp.6eDbEK0oeu
++ return 0
+ local client_container=psmdb-client-9b7b5fdcc-ls88q
+ local mongo_flag=
+ kubectl_bin exec psmdb-client-9b7b5fdcc-ls88q -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-26392.svc.cluster.local/admin '
++ mktemp
+ local LAST_OUT=/tmp/tmp.kMISWQTgv4
++ mktemp
+ local LAST_ERR=/tmp/tmp.2vUxv9ZVmG
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl exec psmdb-client-9b7b5fdcc-ls88q -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-26392.svc.cluster.local/admin '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 0 ']'
+ break
+ cat /tmp/tmp.kMISWQTgv4
+ cat /tmp/tmp.2vUxv9ZVmG
+ rm /tmp/tmp.kMISWQTgv4 /tmp/tmp.2vUxv9ZVmG
+ return 0
+ diff /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1352/e2e-tests/demand-backup-sharded/compare/find.json /tmp/tmp.YDY0m4Yus9/find
+ for i in '$(seq 0 2)'
+ compare_mongos_cmd find myApp:myPass@some-name-mongos.demand-backup-sharded-26392 1 .svc.cluster.local myApp1 test
+ local command=find
+ local uri=myApp:myPass@some-name-mongos.demand-backup-sharded-26392
+ local postfix=1
+ local suffix=.svc.cluster.local
+ local database=myApp1
+ local collection=test
+ run_mongos 'use myApp1\n db.test.find()' myApp:myPass@some-name-mongos.demand-backup-sharded-26392 mongodb .svc.cluster.local
+ local 'command=use myApp1\n db.test.find()'
+ local uri=myApp:myPass@some-name-mongos.demand-backup-sharded-26392
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ egrep -v 'I NETWORK|W NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match'
+ /usr/bin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.anOSweYX0O
+++ mktemp
++ local LAST_ERR=/tmp/tmp.EQWGJpqTX9
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 0 ']'
++ break
++ cat /tmp/tmp.anOSweYX0O
++ cat /tmp/tmp.EQWGJpqTX9
++ rm /tmp/tmp.anOSweYX0O /tmp/tmp.EQWGJpqTX9
++ return 0
+ local client_container=psmdb-client-9b7b5fdcc-ls88q
+ local mongo_flag=
+ kubectl_bin exec psmdb-client-9b7b5fdcc-ls88q -- bash -c 'printf '\''use myApp1\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-26392.svc.cluster.local/admin '
++ mktemp
+ local LAST_OUT=/tmp/tmp.kQzlB9dJH4
++ mktemp
+ local LAST_ERR=/tmp/tmp.lFp5e6iLNe
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl exec psmdb-client-9b7b5fdcc-ls88q -- bash -c 'printf '\''use myApp1\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-26392.svc.cluster.local/admin '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 0 ']'
+ break
+ cat /tmp/tmp.kQzlB9dJH4
+ cat /tmp/tmp.lFp5e6iLNe
+ rm /tmp/tmp.kQzlB9dJH4 /tmp/tmp.lFp5e6iLNe
+ return 0
+ diff /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1352/e2e-tests/demand-backup-sharded/compare/find1.json /tmp/tmp.YDY0m4Yus9/find1
+ for i in '$(seq 0 2)'
+ compare_mongos_cmd find myApp:myPass@some-name-mongos.demand-backup-sharded-26392 2 .svc.cluster.local myApp2 test
+ local command=find
+ local uri=myApp:myPass@some-name-mongos.demand-backup-sharded-26392
+ local postfix=2
+ local suffix=.svc.cluster.local
+ local database=myApp2
+ local collection=test
+ run_mongos 'use myApp2\n db.test.find()' myApp:myPass@some-name-mongos.demand-backup-sharded-26392 mongodb .svc.cluster.local
+ local 'command=use myApp2\n db.test.find()'
+ local uri=myApp:myPass@some-name-mongos.demand-backup-sharded-26392
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ /usr/bin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
+ egrep -v 'I NETWORK|W NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match'
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.BInXpnddHc
+++ mktemp
++ local LAST_ERR=/tmp/tmp.zSUsfkKZb3
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 0 ']'
++ break
++ cat /tmp/tmp.BInXpnddHc
++ cat /tmp/tmp.zSUsfkKZb3
++ rm /tmp/tmp.BInXpnddHc /tmp/tmp.zSUsfkKZb3
++ return 0
+ local client_container=psmdb-client-9b7b5fdcc-ls88q
+ local mongo_flag=
+ kubectl_bin exec psmdb-client-9b7b5fdcc-ls88q -- bash -c 'printf '\''use myApp2\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-26392.svc.cluster.local/admin '
++ mktemp
+ local LAST_OUT=/tmp/tmp.NjR8DiWmFH
++ mktemp
+ local LAST_ERR=/tmp/tmp.fxY2rsyRbH
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl exec psmdb-client-9b7b5fdcc-ls88q -- bash -c 'printf '\''use myApp2\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-26392.svc.cluster.local/admin '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 0 ']'
+ break
+ cat /tmp/tmp.NjR8DiWmFH
+ cat /tmp/tmp.fxY2rsyRbH
+ rm /tmp/tmp.NjR8DiWmFH /tmp/tmp.fxY2rsyRbH
+ return 0
+ diff /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1352/e2e-tests/demand-backup-sharded/compare/find2.json /tmp/tmp.YDY0m4Yus9/find2
+ desc 'check backup and restore -- gcp-cs'
+ set +o xtrace


-----------------------------------------------------------------------------------
check backup and restore -- gcp-cs
-----------------------------------------------------------------------------------

Percona Server for MongoDB shell version v4.4.24-23
connecting to: mongodb://some-name-mongos.demand-backup-sharded-26392.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb
Implicit session: session { "id" : UUID("2a58c187-8af9-4b2f-938d-e678696c44be") }
Percona Server for MongoDB server version: v5.0.20-17
WARNING: shell and server versions do not match
switched to db myApp
WriteResult({ "nInserted" : 1 })
bye
Percona Server for MongoDB shell version v4.4.24-23
connecting to: mongodb://some-name-mongos.demand-backup-sharded-26392.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb
Implicit session: session { "id" : UUID("9417ab83-9771-4a26-aae2-1e0c952c6c17") }
Percona Server for MongoDB server version: v5.0.20-17
WARNING: shell and server versions do not match
switched to db myApp1
WriteResult({ "nInserted" : 1 })
bye
Percona Server for MongoDB shell version v4.4.24-23
connecting to: mongodb://some-name-mongos.demand-backup-sharded-26392.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb
Implicit session: session { "id" : UUID("7233147a-8a3c-44da-970e-e72aa6dca214") }
Percona Server for MongoDB server version: v5.0.20-17
WARNING: shell and server versions do not match
switched to db myApp2
WriteResult({ "nInserted" : 1 })
bye
perconaservermongodbrestore.psmdb.percona.com/restore-backup-gcp-cs created
waiting psmdb-restore/backup-gcp-cs to reach ready state.............
+ '[' 1 -eq 1 ']'
+ wait_cluster_consistency some-name
+ local cluster_name=some-name
+ local wait_time=32
+ retry=0
+ sleep 7
+ echo -n 'waiting for cluster readyness'
waiting for cluster readyness++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.UOwcMTNq62
+++ mktemp
++ local LAST_ERR=/tmp/tmp.gxGxfEekVa
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 0 ']'
++ break
++ cat /tmp/tmp.UOwcMTNq62
++ cat /tmp/tmp.gxGxfEekVa
++ rm /tmp/tmp.UOwcMTNq62 /tmp/tmp.gxGxfEekVa
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 1 -ge 32 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.17J1g4Rdim
+++ mktemp
++ local LAST_ERR=/tmp/tmp.WFBeSDW4lN
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 0 ']'
++ break
++ cat /tmp/tmp.17J1g4Rdim
++ cat /tmp/tmp.WFBeSDW4lN
++ rm /tmp/tmp.17J1g4Rdim /tmp/tmp.WFBeSDW4lN
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 2 -ge 32 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.YVQYyMqNHO
+++ mktemp
++ local LAST_ERR=/tmp/tmp.jlzoIr196D
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 0 ']'
++ break
++ cat /tmp/tmp.YVQYyMqNHO
++ cat /tmp/tmp.jlzoIr196D
++ rm /tmp/tmp.YVQYyMqNHO /tmp/tmp.jlzoIr196D
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 3 -ge 32 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.6OJezhdnLj
+++ mktemp
++ local LAST_ERR=/tmp/tmp.obzNEB5ppI
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 0 ']'
++ break
++ cat /tmp/tmp.6OJezhdnLj
++ cat /tmp/tmp.obzNEB5ppI
++ rm /tmp/tmp.6OJezhdnLj /tmp/tmp.obzNEB5ppI
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 4 -ge 32 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.oKURpB6CWv
+++ mktemp
++ local LAST_ERR=/tmp/tmp.XnvQ9jKgaT
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 0 ']'
++ break
++ cat /tmp/tmp.oKURpB6CWv
++ cat /tmp/tmp.XnvQ9jKgaT
++ rm /tmp/tmp.oKURpB6CWv /tmp/tmp.XnvQ9jKgaT
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 5 -ge 32 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.oj4BDvkUGL
+++ mktemp
++ local LAST_ERR=/tmp/tmp.XtWaNyMvZ0
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 0 ']'
++ break
++ cat /tmp/tmp.oj4BDvkUGL
++ cat /tmp/tmp.XtWaNyMvZ0
++ rm /tmp/tmp.oj4BDvkUGL /tmp/tmp.XtWaNyMvZ0
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 6 -ge 32 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.jajyhWm9h6
+++ mktemp
++ local LAST_ERR=/tmp/tmp.9B8BM9c6xf
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 0 ']'
++ break
++ cat /tmp/tmp.jajyhWm9h6
++ cat /tmp/tmp.9B8BM9c6xf
++ rm /tmp/tmp.jajyhWm9h6 /tmp/tmp.9B8BM9c6xf
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 7 -ge 32 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.yOrhnH4Uf1
+++ mktemp
++ local LAST_ERR=/tmp/tmp.uoP79bOvJg
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 0 ']'
++ break
++ cat /tmp/tmp.yOrhnH4Uf1
++ cat /tmp/tmp.uoP79bOvJg
++ rm /tmp/tmp.yOrhnH4Uf1 /tmp/tmp.uoP79bOvJg
++ return 0
+ [[ ready == \r\e\a\d\y ]]
+ check_data
+ local postfix=
++ seq 0 2
+ for i in '$(seq 0 2)'
+ compare_mongos_cmd find myApp:myPass@some-name-mongos.demand-backup-sharded-26392 '' .svc.cluster.local myApp test
+ local command=find
+ local uri=myApp:myPass@some-name-mongos.demand-backup-sharded-26392
+ local postfix=
+ local suffix=.svc.cluster.local
+ local database=myApp
+ local collection=test
+ run_mongos 'use myApp\n db.test.find()' myApp:myPass@some-name-mongos.demand-backup-sharded-26392 mongodb .svc.cluster.local
+ egrep -v 'I NETWORK|W NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match'
+ local 'command=use myApp\n db.test.find()'
+ local uri=myApp:myPass@some-name-mongos.demand-backup-sharded-26392
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ /usr/bin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.QdKYclDUwV
+++ mktemp
++ local LAST_ERR=/tmp/tmp.r9wFY0daHL
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 0 ']'
++ break
++ cat /tmp/tmp.QdKYclDUwV
++ cat /tmp/tmp.r9wFY0daHL
++ rm /tmp/tmp.QdKYclDUwV /tmp/tmp.r9wFY0daHL
++ return 0
+ local client_container=psmdb-client-9b7b5fdcc-ls88q
+ local mongo_flag=
+ kubectl_bin exec psmdb-client-9b7b5fdcc-ls88q -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-26392.svc.cluster.local/admin '
++ mktemp
+ local LAST_OUT=/tmp/tmp.KESQlqpDkk
++ mktemp
+ local LAST_ERR=/tmp/tmp.eC2f70ZkLL
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl exec psmdb-client-9b7b5fdcc-ls88q -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-26392.svc.cluster.local/admin '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 0 ']'
+ break
+ cat /tmp/tmp.KESQlqpDkk
+ cat /tmp/tmp.eC2f70ZkLL
+ rm /tmp/tmp.KESQlqpDkk /tmp/tmp.eC2f70ZkLL
+ return 0
+ diff /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1352/e2e-tests/demand-backup-sharded/compare/find.json /tmp/tmp.YDY0m4Yus9/find
+ for i in '$(seq 0 2)'
+ compare_mongos_cmd find myApp:myPass@some-name-mongos.demand-backup-sharded-26392 1 .svc.cluster.local myApp1 test
+ local command=find
+ local uri=myApp:myPass@some-name-mongos.demand-backup-sharded-26392
+ local postfix=1
+ local suffix=.svc.cluster.local
+ local database=myApp1
+ local collection=test
+ run_mongos 'use myApp1\n db.test.find()' myApp:myPass@some-name-mongos.demand-backup-sharded-26392 mongodb .svc.cluster.local
+ local 'command=use myApp1\n db.test.find()'
+ egrep -v 'I NETWORK|W NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match'
+ local uri=myApp:myPass@some-name-mongos.demand-backup-sharded-26392
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ /usr/bin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.Jyk7ZswHdN
+++ mktemp
++ local LAST_ERR=/tmp/tmp.m71Lxm52xZ
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 0 ']'
++ break
++ cat /tmp/tmp.Jyk7ZswHdN
++ cat /tmp/tmp.m71Lxm52xZ
++ rm /tmp/tmp.Jyk7ZswHdN /tmp/tmp.m71Lxm52xZ
++ return 0
+ local client_container=psmdb-client-9b7b5fdcc-ls88q
+ local mongo_flag=
+ kubectl_bin exec psmdb-client-9b7b5fdcc-ls88q -- bash -c 'printf '\''use myApp1\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-26392.svc.cluster.local/admin '
++ mktemp
+ local LAST_OUT=/tmp/tmp.c3Ayf1ngGL
++ mktemp
+ local LAST_ERR=/tmp/tmp.mMelts1FY6
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl exec psmdb-client-9b7b5fdcc-ls88q -- bash -c 'printf '\''use myApp1\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-26392.svc.cluster.local/admin '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 0 ']'
+ break
+ cat /tmp/tmp.c3Ayf1ngGL
+ cat /tmp/tmp.mMelts1FY6
+ rm /tmp/tmp.c3Ayf1ngGL /tmp/tmp.mMelts1FY6
+ return 0
+ diff /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1352/e2e-tests/demand-backup-sharded/compare/find1.json /tmp/tmp.YDY0m4Yus9/find1
+ for i in '$(seq 0 2)'
+ compare_mongos_cmd find myApp:myPass@some-name-mongos.demand-backup-sharded-26392 2 .svc.cluster.local myApp2 test
+ local command=find
+ local uri=myApp:myPass@some-name-mongos.demand-backup-sharded-26392
+ local postfix=2
+ local suffix=.svc.cluster.local
+ local database=myApp2
+ local collection=test
+ run_mongos 'use myApp2\n db.test.find()' myApp:myPass@some-name-mongos.demand-backup-sharded-26392 mongodb .svc.cluster.local
+ local 'command=use myApp2\n db.test.find()'
+ local uri=myApp:myPass@some-name-mongos.demand-backup-sharded-26392
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ egrep -v 'I NETWORK|W NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match'
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.pFSPzKmrYA
+ /usr/bin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
+++ mktemp
++ local LAST_ERR=/tmp/tmp.Ggjprrr5ba
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 0 ']'
++ break
++ cat /tmp/tmp.pFSPzKmrYA
++ cat /tmp/tmp.Ggjprrr5ba
++ rm /tmp/tmp.pFSPzKmrYA /tmp/tmp.Ggjprrr5ba
++ return 0
+ local client_container=psmdb-client-9b7b5fdcc-ls88q
+ local mongo_flag=
+ kubectl_bin exec psmdb-client-9b7b5fdcc-ls88q -- bash -c 'printf '\''use myApp2\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-26392.svc.cluster.local/admin '
++ mktemp
+ local LAST_OUT=/tmp/tmp.0oWb4sjZQf
++ mktemp
+ local LAST_ERR=/tmp/tmp.p2PESvSAS6
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl exec psmdb-client-9b7b5fdcc-ls88q -- bash -c 'printf '\''use myApp2\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-26392.svc.cluster.local/admin '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 0 ']'
+ break
+ cat /tmp/tmp.0oWb4sjZQf
+ cat /tmp/tmp.p2PESvSAS6
+ rm /tmp/tmp.0oWb4sjZQf /tmp/tmp.p2PESvSAS6
+ return 0
+ diff /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1352/e2e-tests/demand-backup-sharded/compare/find2.json /tmp/tmp.YDY0m4Yus9/find2
+ desc 'check backup and restore -- azure-blob'
+ set +o xtrace


-----------------------------------------------------------------------------------
check backup and restore -- azure-blob
-----------------------------------------------------------------------------------

Percona Server for MongoDB shell version v4.4.24-23
connecting to: mongodb://some-name-mongos.demand-backup-sharded-26392.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb
Implicit session: session { "id" : UUID("834306be-75cf-4cc7-abd7-48bf553df833") }
Percona Server for MongoDB server version: v5.0.20-17
WARNING: shell and server versions do not match
switched to db myApp
WriteResult({ "nInserted" : 1 })
bye
Percona Server for MongoDB shell version v4.4.24-23
connecting to: mongodb://some-name-mongos.demand-backup-sharded-26392.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb
Implicit session: session { "id" : UUID("2128ca1d-4eb5-4dda-985b-90874dd9befb") }
Percona Server for MongoDB server version: v5.0.20-17
WARNING: shell and server versions do not match
switched to db myApp1
WriteResult({ "nInserted" : 1 })
bye
Percona Server for MongoDB shell version v4.4.24-23
connecting to: mongodb://some-name-mongos.demand-backup-sharded-26392.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb
Implicit session: session { "id" : UUID("3e006647-b871-4240-b075-fb5f4e96427b") }
Percona Server for MongoDB server version: v5.0.20-17
WARNING: shell and server versions do not match
switched to db myApp2
WriteResult({ "nInserted" : 1 })
bye
perconaservermongodbrestore.psmdb.percona.com/restore-backup-azure-blob created
waiting psmdb-restore/backup-azure-blob to reach ready state.............
+ '[' 1 -eq 1 ']'
+ wait_cluster_consistency some-name
+ local cluster_name=some-name
+ local wait_time=32
+ retry=0
+ sleep 7
+ echo -n 'waiting for cluster readyness'
waiting for cluster readyness++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.4xgv2fE3hB
+++ mktemp
++ local LAST_ERR=/tmp/tmp.ixQ3jmkscT
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 0 ']'
++ break
++ cat /tmp/tmp.4xgv2fE3hB
++ cat /tmp/tmp.ixQ3jmkscT
++ rm /tmp/tmp.4xgv2fE3hB /tmp/tmp.ixQ3jmkscT
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 1 -ge 32 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.RUujgS8K8J
+++ mktemp
++ local LAST_ERR=/tmp/tmp.5yV34wM3uh
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 0 ']'
++ break
++ cat /tmp/tmp.RUujgS8K8J
++ cat /tmp/tmp.5yV34wM3uh
++ rm /tmp/tmp.RUujgS8K8J /tmp/tmp.5yV34wM3uh
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 2 -ge 32 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.zgKbrHiUQx
+++ mktemp
++ local LAST_ERR=/tmp/tmp.TKYF6RCijp
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 0 ']'
++ break
++ cat /tmp/tmp.zgKbrHiUQx
++ cat /tmp/tmp.TKYF6RCijp
++ rm /tmp/tmp.zgKbrHiUQx /tmp/tmp.TKYF6RCijp
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 3 -ge 32 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.07GQlDIBHx
+++ mktemp
++ local LAST_ERR=/tmp/tmp.wTHy9jmIZX
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 0 ']'
++ break
++ cat /tmp/tmp.07GQlDIBHx
++ cat /tmp/tmp.wTHy9jmIZX
++ rm /tmp/tmp.07GQlDIBHx /tmp/tmp.wTHy9jmIZX
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 4 -ge 32 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.PRF8vMGq3G
+++ mktemp
++ local LAST_ERR=/tmp/tmp.2D4Dbw1cE0
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 0 ']'
++ break
++ cat /tmp/tmp.PRF8vMGq3G
++ cat /tmp/tmp.2D4Dbw1cE0
++ rm /tmp/tmp.PRF8vMGq3G /tmp/tmp.2D4Dbw1cE0
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 5 -ge 32 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.nU0kK6qFpK
+++ mktemp
++ local LAST_ERR=/tmp/tmp.L3DNgL6aRM
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 0 ']'
++ break
++ cat /tmp/tmp.nU0kK6qFpK
++ cat /tmp/tmp.L3DNgL6aRM
++ rm /tmp/tmp.nU0kK6qFpK /tmp/tmp.L3DNgL6aRM
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 6 -ge 32 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.aUfxnAG23x
+++ mktemp
++ local LAST_ERR=/tmp/tmp.IXVpHLr6nq
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 0 ']'
++ break
++ cat /tmp/tmp.aUfxnAG23x
++ cat /tmp/tmp.IXVpHLr6nq
++ rm /tmp/tmp.aUfxnAG23x /tmp/tmp.IXVpHLr6nq
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 7 -ge 32 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.CLMXth889r
+++ mktemp
++ local LAST_ERR=/tmp/tmp.qBfAFBpfzQ
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 0 ']'
++ break
++ cat /tmp/tmp.CLMXth889r
++ cat /tmp/tmp.qBfAFBpfzQ
++ rm /tmp/tmp.CLMXth889r /tmp/tmp.qBfAFBpfzQ
++ return 0
+ [[ ready == \r\e\a\d\y ]]
+ check_data
+ local postfix=
++ seq 0 2
+ for i in '$(seq 0 2)'
+ compare_mongos_cmd find myApp:myPass@some-name-mongos.demand-backup-sharded-26392 '' .svc.cluster.local myApp test
+ local command=find
+ local uri=myApp:myPass@some-name-mongos.demand-backup-sharded-26392
+ local postfix=
+ local suffix=.svc.cluster.local
+ local database=myApp
+ local collection=test
+ run_mongos 'use myApp\n db.test.find()' myApp:myPass@some-name-mongos.demand-backup-sharded-26392 mongodb .svc.cluster.local
+ local 'command=use myApp\n db.test.find()'
+ local uri=myApp:myPass@some-name-mongos.demand-backup-sharded-26392
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ egrep -v 'I NETWORK|W NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match'
+ /usr/bin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.Xf576hgJN6
+++ mktemp
++ local LAST_ERR=/tmp/tmp.8iyxyVtuyf
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 0 ']'
++ break
++ cat /tmp/tmp.Xf576hgJN6
++ cat /tmp/tmp.8iyxyVtuyf
++ rm /tmp/tmp.Xf576hgJN6 /tmp/tmp.8iyxyVtuyf
++ return 0
+ local client_container=psmdb-client-9b7b5fdcc-ls88q
+ local mongo_flag=
+ kubectl_bin exec psmdb-client-9b7b5fdcc-ls88q -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-26392.svc.cluster.local/admin '
++ mktemp
+ local LAST_OUT=/tmp/tmp.eKPcqKd0Ii
++ mktemp
+ local LAST_ERR=/tmp/tmp.OPYDX7UCk4
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl exec psmdb-client-9b7b5fdcc-ls88q -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-26392.svc.cluster.local/admin '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 0 ']'
+ break
+ cat /tmp/tmp.eKPcqKd0Ii
+ cat /tmp/tmp.OPYDX7UCk4
+ rm /tmp/tmp.eKPcqKd0Ii /tmp/tmp.OPYDX7UCk4
+ return 0
+ diff /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1352/e2e-tests/demand-backup-sharded/compare/find.json /tmp/tmp.YDY0m4Yus9/find
+ for i in '$(seq 0 2)'
+ compare_mongos_cmd find myApp:myPass@some-name-mongos.demand-backup-sharded-26392 1 .svc.cluster.local myApp1 test
+ local command=find
+ local uri=myApp:myPass@some-name-mongos.demand-backup-sharded-26392
+ local postfix=1
+ local suffix=.svc.cluster.local
+ local database=myApp1
+ local collection=test
+ run_mongos 'use myApp1\n db.test.find()' myApp:myPass@some-name-mongos.demand-backup-sharded-26392 mongodb .svc.cluster.local
+ local 'command=use myApp1\n db.test.find()'
+ local uri=myApp:myPass@some-name-mongos.demand-backup-sharded-26392
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ egrep -v 'I NETWORK|W NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match'
+ /usr/bin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.QfEQW8wlbT
+++ mktemp
++ local LAST_ERR=/tmp/tmp.mAAJZu5xTz
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 0 ']'
++ break
++ cat /tmp/tmp.QfEQW8wlbT
++ cat /tmp/tmp.mAAJZu5xTz
++ rm /tmp/tmp.QfEQW8wlbT /tmp/tmp.mAAJZu5xTz
++ return 0
+ local client_container=psmdb-client-9b7b5fdcc-ls88q
+ local mongo_flag=
+ kubectl_bin exec psmdb-client-9b7b5fdcc-ls88q -- bash -c 'printf '\''use myApp1\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-26392.svc.cluster.local/admin '
++ mktemp
+ local LAST_OUT=/tmp/tmp.Q1mJzzKOvD
++ mktemp
+ local LAST_ERR=/tmp/tmp.9W6zJWXJU7
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl exec psmdb-client-9b7b5fdcc-ls88q -- bash -c 'printf '\''use myApp1\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-26392.svc.cluster.local/admin '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 0 ']'
+ break
+ cat /tmp/tmp.Q1mJzzKOvD
+ cat /tmp/tmp.9W6zJWXJU7
+ rm /tmp/tmp.Q1mJzzKOvD /tmp/tmp.9W6zJWXJU7
+ return 0
+ diff /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1352/e2e-tests/demand-backup-sharded/compare/find1.json /tmp/tmp.YDY0m4Yus9/find1
+ for i in '$(seq 0 2)'
+ compare_mongos_cmd find myApp:myPass@some-name-mongos.demand-backup-sharded-26392 2 .svc.cluster.local myApp2 test
+ local command=find
+ local uri=myApp:myPass@some-name-mongos.demand-backup-sharded-26392
+ local postfix=2
+ local suffix=.svc.cluster.local
+ local database=myApp2
+ local collection=test
+ run_mongos 'use myApp2\n db.test.find()' myApp:myPass@some-name-mongos.demand-backup-sharded-26392 mongodb .svc.cluster.local
+ local 'command=use myApp2\n db.test.find()'
+ local uri=myApp:myPass@some-name-mongos.demand-backup-sharded-26392
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ egrep -v 'I NETWORK|W NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match'
+ /usr/bin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.WAPnGRU1va
+++ mktemp
++ local LAST_ERR=/tmp/tmp.jEkDaMjfKp
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 0 ']'
++ break
++ cat /tmp/tmp.WAPnGRU1va
++ cat /tmp/tmp.jEkDaMjfKp
++ rm /tmp/tmp.WAPnGRU1va /tmp/tmp.jEkDaMjfKp
++ return 0
+ local client_container=psmdb-client-9b7b5fdcc-ls88q
+ local mongo_flag=
+ kubectl_bin exec psmdb-client-9b7b5fdcc-ls88q -- bash -c 'printf '\''use myApp2\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-26392.svc.cluster.local/admin '
++ mktemp
+ local LAST_OUT=/tmp/tmp.RNW8qWhmVm
++ mktemp
+ local LAST_ERR=/tmp/tmp.LBZ1BY75ie
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl exec psmdb-client-9b7b5fdcc-ls88q -- bash -c 'printf '\''use myApp2\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-26392.svc.cluster.local/admin '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 0 ']'
+ break
+ cat /tmp/tmp.RNW8qWhmVm
+ cat /tmp/tmp.LBZ1BY75ie
+ rm /tmp/tmp.RNW8qWhmVm /tmp/tmp.LBZ1BY75ie
+ return 0
+ diff /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1352/e2e-tests/demand-backup-sharded/compare/find2.json /tmp/tmp.YDY0m4Yus9/find2
+ desc 'check backup and restore -- minio'
+ set +o xtrace


-----------------------------------------------------------------------------------
check backup and restore -- minio
-----------------------------------------------------------------------------------

If you don't see a command prompt, try pressing enter.
Error attaching, falling back to logs: unable to upgrade connection: container aws-cli not found in pod aws-cli_demand-backup-sharded-26392
2023-10-05 13:45:22         55 myApp.test.gz
Percona Server for MongoDB shell version v4.4.24-23
connecting to: mongodb://some-name-mongos.demand-backup-sharded-26392.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb
Implicit session: session { "id" : UUID("d9fad3ad-7f5b-4201-ba47-bc469111524b") }
Percona Server for MongoDB server version: v5.0.20-17
WARNING: shell and server versions do not match
switched to db myApp
WriteResult({ "nInserted" : 1 })
bye
Percona Server for MongoDB shell version v4.4.24-23
connecting to: mongodb://some-name-mongos.demand-backup-sharded-26392.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb
Implicit session: session { "id" : UUID("9443ee08-59ae-48c5-963c-cf3ae96510f2") }
Percona Server for MongoDB server version: v5.0.20-17
WARNING: shell and server versions do not match
switched to db myApp1
WriteResult({ "nInserted" : 1 })
bye
Percona Server for MongoDB shell version v4.4.24-23
connecting to: mongodb://some-name-mongos.demand-backup-sharded-26392.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb
Implicit session: session { "id" : UUID("fd3f20ef-2484-44f5-93cb-c3ccc2ab397c") }
Percona Server for MongoDB server version: v5.0.20-17
WARNING: shell and server versions do not match
switched to db myApp2
WriteResult({ "nInserted" : 1 })
bye
perconaservermongodbrestore.psmdb.percona.com/restore-backup-minio created
waiting psmdb-restore/backup-minio to reach ready state..........
+ '[' 1 -eq 1 ']'
+ wait_cluster_consistency some-name
+ local cluster_name=some-name
+ local wait_time=32
+ retry=0
+ sleep 7
+ echo -n 'waiting for cluster readyness'
waiting for cluster readyness++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.EAsuMvfqnU
+++ mktemp
++ local LAST_ERR=/tmp/tmp.FUM4ccDLu9
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 0 ']'
++ break
++ cat /tmp/tmp.EAsuMvfqnU
++ cat /tmp/tmp.FUM4ccDLu9
++ rm /tmp/tmp.EAsuMvfqnU /tmp/tmp.FUM4ccDLu9
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 1 -ge 32 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.Bvp44krg4x
+++ mktemp
++ local LAST_ERR=/tmp/tmp.NZqzKxjTd9
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 0 ']'
++ break
++ cat /tmp/tmp.Bvp44krg4x
++ cat /tmp/tmp.NZqzKxjTd9
++ rm /tmp/tmp.Bvp44krg4x /tmp/tmp.NZqzKxjTd9
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 2 -ge 32 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.Jx3UOFcHir
+++ mktemp
++ local LAST_ERR=/tmp/tmp.R75nsU3fkm
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 0 ']'
++ break
++ cat /tmp/tmp.Jx3UOFcHir
++ cat /tmp/tmp.R75nsU3fkm
++ rm /tmp/tmp.Jx3UOFcHir /tmp/tmp.R75nsU3fkm
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 3 -ge 32 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.vQTdGbhC6Q
+++ mktemp
++ local LAST_ERR=/tmp/tmp.reqvp9MAkk
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 0 ']'
++ break
++ cat /tmp/tmp.vQTdGbhC6Q
++ cat /tmp/tmp.reqvp9MAkk
++ rm /tmp/tmp.vQTdGbhC6Q /tmp/tmp.reqvp9MAkk
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 4 -ge 32 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.pLZRDc4WWV
+++ mktemp
++ local LAST_ERR=/tmp/tmp.2RlAsB6622
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 0 ']'
++ break
++ cat /tmp/tmp.pLZRDc4WWV
++ cat /tmp/tmp.2RlAsB6622
++ rm /tmp/tmp.pLZRDc4WWV /tmp/tmp.2RlAsB6622
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 5 -ge 32 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.GVM9Eiqghr
+++ mktemp
++ local LAST_ERR=/tmp/tmp.Zv3btGzPmV
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 0 ']'
++ break
++ cat /tmp/tmp.GVM9Eiqghr
++ cat /tmp/tmp.Zv3btGzPmV
++ rm /tmp/tmp.GVM9Eiqghr /tmp/tmp.Zv3btGzPmV
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 6 -ge 32 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.Dk1n3LCSOu
+++ mktemp
++ local LAST_ERR=/tmp/tmp.L0oBwGaRBt
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 0 ']'
++ break
++ cat /tmp/tmp.Dk1n3LCSOu
++ cat /tmp/tmp.L0oBwGaRBt
++ rm /tmp/tmp.Dk1n3LCSOu /tmp/tmp.L0oBwGaRBt
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 7 -ge 32 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.IsI40oshCc
+++ mktemp
++ local LAST_ERR=/tmp/tmp.uH9mXXaxnj
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 0 ']'
++ break
++ cat /tmp/tmp.IsI40oshCc
++ cat /tmp/tmp.uH9mXXaxnj
++ rm /tmp/tmp.IsI40oshCc /tmp/tmp.uH9mXXaxnj
++ return 0
+ [[ ready == \r\e\a\d\y ]]
+ check_data
+ local postfix=
++ seq 0 2
+ for i in '$(seq 0 2)'
+ compare_mongos_cmd find myApp:myPass@some-name-mongos.demand-backup-sharded-26392 '' .svc.cluster.local myApp test
+ local command=find
+ local uri=myApp:myPass@some-name-mongos.demand-backup-sharded-26392
+ local postfix=
+ local suffix=.svc.cluster.local
+ local database=myApp
+ local collection=test
+ run_mongos 'use myApp\n db.test.find()' myApp:myPass@some-name-mongos.demand-backup-sharded-26392 mongodb .svc.cluster.local
+ local 'command=use myApp\n db.test.find()'
+ local uri=myApp:myPass@some-name-mongos.demand-backup-sharded-26392
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ egrep -v 'I NETWORK|W NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match'
+ /usr/bin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.WuMfsHoX68
+++ mktemp
++ local LAST_ERR=/tmp/tmp.zKU6Zp9ABX
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 0 ']'
++ break
++ cat /tmp/tmp.WuMfsHoX68
++ cat /tmp/tmp.zKU6Zp9ABX
++ rm /tmp/tmp.WuMfsHoX68 /tmp/tmp.zKU6Zp9ABX
++ return 0
+ local client_container=psmdb-client-9b7b5fdcc-ls88q
+ local mongo_flag=
+ kubectl_bin exec psmdb-client-9b7b5fdcc-ls88q -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-26392.svc.cluster.local/admin '
++ mktemp
+ local LAST_OUT=/tmp/tmp.V7wGM68nYZ
++ mktemp
+ local LAST_ERR=/tmp/tmp.rqIluhcmLh
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl exec psmdb-client-9b7b5fdcc-ls88q -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-26392.svc.cluster.local/admin '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 0 ']'
+ break
+ cat /tmp/tmp.V7wGM68nYZ
+ cat /tmp/tmp.rqIluhcmLh
+ rm /tmp/tmp.V7wGM68nYZ /tmp/tmp.rqIluhcmLh
+ return 0
+ diff /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1352/e2e-tests/demand-backup-sharded/compare/find.json /tmp/tmp.YDY0m4Yus9/find
+ for i in '$(seq 0 2)'
+ compare_mongos_cmd find myApp:myPass@some-name-mongos.demand-backup-sharded-26392 1 .svc.cluster.local myApp1 test
+ local command=find
+ local uri=myApp:myPass@some-name-mongos.demand-backup-sharded-26392
+ local postfix=1
+ local suffix=.svc.cluster.local
+ local database=myApp1
+ local collection=test
+ egrep -v 'I NETWORK|W NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match'
+ run_mongos 'use myApp1\n db.test.find()' myApp:myPass@some-name-mongos.demand-backup-sharded-26392 mongodb .svc.cluster.local
+ local 'command=use myApp1\n db.test.find()'
+ local uri=myApp:myPass@some-name-mongos.demand-backup-sharded-26392
+ local driver=mongodb
+ local suffix=.svc.cluster.local
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+ /usr/bin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.Fhpu0TK6XU
+++ mktemp
++ local LAST_ERR=/tmp/tmp.UjviwEDKLP
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 0 ']'
++ break
++ cat /tmp/tmp.Fhpu0TK6XU
++ cat /tmp/tmp.UjviwEDKLP
++ rm /tmp/tmp.Fhpu0TK6XU /tmp/tmp.UjviwEDKLP
++ return 0
+ local client_container=psmdb-client-9b7b5fdcc-ls88q
+ local mongo_flag=
+ kubectl_bin exec psmdb-client-9b7b5fdcc-ls88q -- bash -c 'printf '\''use myApp1\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-26392.svc.cluster.local/admin '
++ mktemp
+ local LAST_OUT=/tmp/tmp.DZR75YQ3pq
++ mktemp
+ local LAST_ERR=/tmp/tmp.wHPXnfClDT
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl exec psmdb-client-9b7b5fdcc-ls88q -- bash -c 'printf '\''use myApp1\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-26392.svc.cluster.local/admin '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 0 ']'
+ break
+ cat /tmp/tmp.DZR75YQ3pq
+ cat /tmp/tmp.wHPXnfClDT
+ rm /tmp/tmp.DZR75YQ3pq /tmp/tmp.wHPXnfClDT
+ return 0
+ diff /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1352/e2e-tests/demand-backup-sharded/compare/find1.json /tmp/tmp.YDY0m4Yus9/find1
+ for i in '$(seq 0 2)'
+ compare_mongos_cmd find myApp:myPass@some-name-mongos.demand-backup-sharded-26392 2 .svc.cluster.local myApp2 test
+ local command=find
+ local uri=myApp:myPass@some-name-mongos.demand-backup-sharded-26392
+ local postfix=2
+ local suffix=.svc.cluster.local
+ local database=myApp2
+ local collection=test
+ run_mongos 'use myApp2\n db.test.find()' myApp:myPass@some-name-mongos.demand-backup-sharded-26392 mongodb .svc.cluster.local
+ local 'command=use myApp2\n db.test.find()'
+ local uri=myApp:myPass@some-name-mongos.demand-backup-sharded-26392
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ /usr/bin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+ egrep -v 'I NETWORK|W NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.9ZVX1WHyBp
+++ mktemp
++ local LAST_ERR=/tmp/tmp.NvZ7jVzH4y
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 0 ']'
++ break
++ cat /tmp/tmp.9ZVX1WHyBp
++ cat /tmp/tmp.NvZ7jVzH4y
++ rm /tmp/tmp.9ZVX1WHyBp /tmp/tmp.NvZ7jVzH4y
++ return 0
+ local client_container=psmdb-client-9b7b5fdcc-ls88q
+ local mongo_flag=
+ kubectl_bin exec psmdb-client-9b7b5fdcc-ls88q -- bash -c 'printf '\''use myApp2\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-26392.svc.cluster.local/admin '
++ mktemp
+ local LAST_OUT=/tmp/tmp.w0u7GJshXV
++ mktemp
+ local LAST_ERR=/tmp/tmp.7UpUbhL5vr
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl exec psmdb-client-9b7b5fdcc-ls88q -- bash -c 'printf '\''use myApp2\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-26392.svc.cluster.local/admin '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 0 ']'
+ break
+ cat /tmp/tmp.w0u7GJshXV
+ cat /tmp/tmp.7UpUbhL5vr
+ rm /tmp/tmp.w0u7GJshXV /tmp/tmp.7UpUbhL5vr
+ return 0
+ diff /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1352/e2e-tests/demand-backup-sharded/compare/find2.json /tmp/tmp.YDY0m4Yus9/find2
+ desc 'delete backup and check if it is removed from bucket -- minio'
+ set +o xtrace


-----------------------------------------------------------------------------------
delete backup and check if it is removed from bucket -- minio
-----------------------------------------------------------------------------------

perconaservermongodbbackup.psmdb.percona.com "backup-aws-s3" deleted
perconaservermongodbbackup.psmdb.percona.com "backup-azure-blob" deleted
perconaservermongodbbackup.psmdb.percona.com "backup-gcp-cs" deleted
perconaservermongodbbackup.psmdb.percona.com "backup-minio" deleted
If you don't see a command prompt, try pressing enter.
Error attaching, falling back to logs: Internal error occurred: error attaching to container: container is in CONTAINER_EXITED state


-----------------------------------------------------------------------------------
check for passwords leak
-----------------------------------------------------------------------------------

secrets=YmFja3VwMTIzNDU2 Y2x1c3RlckFkbWluMTIzNDU2 Y2x1c3Rlck1vbml0b3IxMjM0NTY= ZGF0YWJhc2VBZG1pbjEyMzQ1Ng== dXNlckFkbWluMTIzNDU2 YmFja3VwMTIzNDU2 Y2x1c3RlckFkbWluMTIzNDU2 Y2x1c3Rlck1vbml0b3IxMjM0NTY= ZGF0YWJhc2VBZG1pbjEyMzQ1Ng== dXNlckFkbWluMTIzNDU2
passwords=backup123456 clusterAdmin123456 clusterMonitor123456 databaseAdmin123456 userAdmin123456 backup123456 clusterAdmin123456 clusterMonitor123456 databaseAdmin123456 userAdmin123456 YmFja3VwMTIzNDU2 Y2x1c3RlckFkbWluMTIzNDU2 Y2x1c3Rlck1vbml0b3IxMjM0NTY= ZGF0YWJhc2VBZG1pbjEyMzQ1Ng== dXNlckFkbWluMTIzNDU2 YmFja3VwMTIzNDU2 Y2x1c3RlckFkbWluMTIzNDU2 Y2x1c3Rlck1vbml0b3IxMjM0NTY= ZGF0YWJhc2VBZG1pbjEyMzQ1Ng== dXNlckFkbWluMTIzNDU2
pods=minio-service-5fc4dc898-8nhvn percona-server-mongodb-operator-6b6bcb669-6cf88 psmdb-client-9b7b5fdcc-ls88q some-name-cfg-0 some-name-cfg-1 some-name-cfg-2 some-name-mongos-0 some-name-mongos-1 some-name-mongos-2 some-name-rs0-0 some-name-rs0-1 some-name-rs0-2 some-name-rs1-0 some-name-rs1-1 some-name-rs1-2 some-name-rs2-0 some-name-rs2-1 some-name-rs2-2
logs saved in: /tmp/tmp.vH0S3yHpzI/logs_output-minio-service-5fc4dc898-8nhvn-minio.txt

logs saved in: /tmp/tmp.vH0S3yHpzI/logs_output-percona-server-mongodb-operator-6b6bcb669-6cf88-percona-server-mongodb-operator.txt

logs saved in: /tmp/tmp.vH0S3yHpzI/logs_output-psmdb-client-9b7b5fdcc-ls88q-psmdb-client.txt

logs saved in: /tmp/tmp.vH0S3yHpzI/logs_output-some-name-cfg-0-mongod.txt
logs saved in: /tmp/tmp.vH0S3yHpzI/logs_output-some-name-cfg-0-cfg-sidecar-1.txt
logs saved in: /tmp/tmp.vH0S3yHpzI/logs_output-some-name-cfg-0-backup-agent.txt

logs saved in: /tmp/tmp.vH0S3yHpzI/logs_output-some-name-cfg-1-mongod.txt
logs saved in: /tmp/tmp.vH0S3yHpzI/logs_output-some-name-cfg-1-cfg-sidecar-1.txt
logs saved in: /tmp/tmp.vH0S3yHpzI/logs_output-some-name-cfg-1-backup-agent.txt

logs saved in: /tmp/tmp.vH0S3yHpzI/logs_output-some-name-cfg-2-mongod.txt
logs saved in: /tmp/tmp.vH0S3yHpzI/logs_output-some-name-cfg-2-cfg-sidecar-1.txt
logs saved in: /tmp/tmp.vH0S3yHpzI/logs_output-some-name-cfg-2-backup-agent.txt

logs saved in: /tmp/tmp.vH0S3yHpzI/logs_output-some-name-mongos-0-mongos.txt
logs saved in: /tmp/tmp.vH0S3yHpzI/logs_output-some-name-mongos-0-mongos-sidecar-1.txt

logs saved in: /tmp/tmp.vH0S3yHpzI/logs_output-some-name-mongos-1-mongos.txt
logs saved in: /tmp/tmp.vH0S3yHpzI/logs_output-some-name-mongos-1-mongos-sidecar-1.txt

logs saved in: /tmp/tmp.vH0S3yHpzI/logs_output-some-name-mongos-2-mongos.txt
logs saved in: /tmp/tmp.vH0S3yHpzI/logs_output-some-name-mongos-2-mongos-sidecar-1.txt

logs saved in: /tmp/tmp.vH0S3yHpzI/logs_output-some-name-rs0-0-mongod.txt
logs saved in: /tmp/tmp.vH0S3yHpzI/logs_output-some-name-rs0-0-backup-agent.txt

logs saved in: /tmp/tmp.vH0S3yHpzI/logs_output-some-name-rs0-1-mongod.txt
logs saved in: /tmp/tmp.vH0S3yHpzI/logs_output-some-name-rs0-1-backup-agent.txt

logs saved in: /tmp/tmp.vH0S3yHpzI/logs_output-some-name-rs0-2-mongod.txt
logs saved in: /tmp/tmp.vH0S3yHpzI/logs_output-some-name-rs0-2-backup-agent.txt

logs saved in: /tmp/tmp.vH0S3yHpzI/logs_output-some-name-rs1-0-mongod.txt
logs saved in: /tmp/tmp.vH0S3yHpzI/logs_output-some-name-rs1-0-backup-agent.txt

logs saved in: /tmp/tmp.vH0S3yHpzI/logs_output-some-name-rs1-1-mongod.txt
logs saved in: /tmp/tmp.vH0S3yHpzI/logs_output-some-name-rs1-1-backup-agent.txt

logs saved in: /tmp/tmp.vH0S3yHpzI/logs_output-some-name-rs1-2-mongod.txt
logs saved in: /tmp/tmp.vH0S3yHpzI/logs_output-some-name-rs1-2-backup-agent.txt

logs saved in: /tmp/tmp.vH0S3yHpzI/logs_output-some-name-rs2-0-mongod.txt
logs saved in: /tmp/tmp.vH0S3yHpzI/logs_output-some-name-rs2-0-rs-sidecar-1.txt
logs saved in: /tmp/tmp.vH0S3yHpzI/logs_output-some-name-rs2-0-backup-agent.txt

logs saved in: /tmp/tmp.vH0S3yHpzI/logs_output-some-name-rs2-1-mongod.txt
logs saved in: /tmp/tmp.vH0S3yHpzI/logs_output-some-name-rs2-1-rs-sidecar-1.txt
logs saved in: /tmp/tmp.vH0S3yHpzI/logs_output-some-name-rs2-1-backup-agent.txt

logs saved in: /tmp/tmp.vH0S3yHpzI/logs_output-some-name-rs2-2-mongod.txt
logs saved in: /tmp/tmp.vH0S3yHpzI/logs_output-some-name-rs2-2-rs-sidecar-1.txt
logs saved in: /tmp/tmp.vH0S3yHpzI/logs_output-some-name-rs2-2-backup-agent.txt

runtimeclass.node.k8s.io "container-rc" deleted


-----------------------------------------------------------------------------------
destroy cluster/operator and all other resources
-----------------------------------------------------------------------------------



-----------------------------------------------------------------------------------
get and delete old CRDs and RBAC
-----------------------------------------------------------------------------------

customresourcedefinition.apiextensions.k8s.io "perconaservermongodbbackups.psmdb.percona.com" deleted
customresourcedefinition.apiextensions.k8s.io "perconaservermongodbrestores.psmdb.percona.com" deleted
customresourcedefinition.apiextensions.k8s.io "perconaservermongodbs.psmdb.percona.com" deleted
error: the server doesn't have a resource type "perconaservermongodbbackups"
+ kubectl patch perconaservermongodbbackups.psmdb.percona.com -n sh --type=merge -p '{"metadata":{"finalizers":[]}}'
error: the server doesn't have a resource type "perconaservermongodbbackups"
error: the server doesn't have a resource type "perconaservermongodbrestores"
+ kubectl patch perconaservermongodbrestores.psmdb.percona.com -n sh --type=merge -p '{"metadata":{"finalizers":[]}}'
error: the server doesn't have a resource type "perconaservermongodbrestores"
error: the server doesn't have a resource type "perconaservermongodbs"
+ kubectl patch perconaservermongodbs.psmdb.percona.com -n sh --type=merge -p '{"metadata":{"finalizers":[]}}'
error: the server doesn't have a resource type "perconaservermongodbs"
role.rbac.authorization.k8s.io "percona-server-mongodb-operator" deleted
serviceaccount "percona-server-mongodb-operator" deleted
rolebinding.rbac.authorization.k8s.io "service-account-percona-server-mongodb-operator" deleted