++ echo 'Log: /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1377/e2e-tests/logs/demand-backup-sharded.log'
Log: /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1377/e2e-tests/logs/demand-backup-sharded.log
++ '[' -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1377/e2e-tests/conf/cloud-secret.yml ']'
++ SKIP_BACKUPS_TO_AWS_GCP_AZURE=
++ oc get projects
++ kubectl get nodes
++ grep '^minikube'
+++ grep '\-eks\-'
+++ kubectl version -o json
+++ jq -r .serverVersion.gitVersion
WARNING: version difference between client (1.29) and server (1.25) exceeds the supported minor version skew of +/-1
++ '[' ']'
++ EKS=0
+++ kubectl version -o json
+++ jq -r .serverVersion.gitVersion
+++ grep gke
WARNING: version difference between client (1.29) and server (1.25) exceeds the supported minor version skew of +/-1
++ '[' v1.25.16-gke.1268000 ']'
++ GKE=1
+++ kubectl version -o json
+++ jq -r '.serverVersion.major + "." + .serverVersion.minor'
+++ /usr/bin/sed -r 's/[^0-9.]+//g'
WARNING: version difference between client (1.29) and server (1.25) exceeds the supported minor version skew of +/-1
++ KUBE_VERSION=1.25
+ set_debug
+ [[ 0 == 1 ]]
+ set +o xtrace


-----------------------------------------------------------------------------------
get and delete old CRDs and RBAC
-----------------------------------------------------------------------------------

error: the server doesn't have a resource type "perconaservermongodbbackups"
+ kubectl patch perconaservermongodbbackups.psmdb.percona.com -n sh --type=merge -p '{"metadata":{"finalizers":[]}}'
error: the server doesn't have a resource type "perconaservermongodbbackups"
error: the server doesn't have a resource type "perconaservermongodbrestores"
+ kubectl patch perconaservermongodbrestores.psmdb.percona.com -n sh --type=merge -p '{"metadata":{"finalizers":[]}}'
error: the server doesn't have a resource type "perconaservermongodbrestores"
error: the server doesn't have a resource type "perconaservermongodbs"
+ kubectl patch perconaservermongodbs.psmdb.percona.com -n sh --type=merge -p '{"metadata":{"finalizers":[]}}'
error: the server doesn't have a resource type "perconaservermongodbs"
Error from server (NotFound): customresourcedefinitions.apiextensions.k8s.io "null" not found
Error from server (NotFound): customresourcedefinitions.apiextensions.k8s.io "null" not found
Error from server (NotFound): customresourcedefinitions.apiextensions.k8s.io "null" not found
Error from server (NotFound): customresourcedefinitions.apiextensions.k8s.io "null" not found


-----------------------------------------------------------------------------------
destroy chaos-mesh
-----------------------------------------------------------------------------------

error: resource(s) were provided, but no name was specified
error: resource(s) were provided, but no name was specified
error: resource(s) were provided, but no name was specified
error: resource(s) were provided, but no name was specified
error: resource(s) were provided, but no name was specified
error: resource(s) were provided, but no name was specified


-----------------------------------------------------------------------------------
cleaned up all old namespaces
-----------------------------------------------------------------------------------



-----------------------------------------------------------------------------------
cleaned up old namespaces psmdb-operator
-----------------------------------------------------------------------------------

error: resource(s) were provided, but no name was specified


-----------------------------------------------------------------------------------
create namespace psmdb-operator
-----------------------------------------------------------------------------------

namespace/psmdb-operator created
Context "gke_cloud-dev-112233_us-central1-a_jen-psmdb-1377-eeba1539-6-cluster2" modified.


-----------------------------------------------------------------------------------
start PSMDB operator
-----------------------------------------------------------------------------------

customresourcedefinition.apiextensions.k8s.io/perconaservermongodbbackups.psmdb.percona.com serverside-applied
customresourcedefinition.apiextensions.k8s.io/perconaservermongodbrestores.psmdb.percona.com serverside-applied
customresourcedefinition.apiextensions.k8s.io/perconaservermongodbs.psmdb.percona.com serverside-applied
clusterrole.rbac.authorization.k8s.io/percona-server-mongodb-operator created
serviceaccount/percona-server-mongodb-operator created
clusterrolebinding.rbac.authorization.k8s.io/service-account-percona-server-mongodb-operator created
deployment.apps/percona-server-mongodb-operator created
waiting for pod/percona-server-mongodb-operator-d6f9576f6-dbvch to be ready.OK


-----------------------------------------------------------------------------------
destroy chaos-mesh
-----------------------------------------------------------------------------------

error: resource(s) were provided, but no name was specified
error: resource(s) were provided, but no name was specified
error: resource(s) were provided, but no name was specified
error: resource(s) were provided, but no name was specified
error: resource(s) were provided, but no name was specified
error: resource(s) were provided, but no name was specified


-----------------------------------------------------------------------------------
cleaned up all old namespaces
-----------------------------------------------------------------------------------



-----------------------------------------------------------------------------------
cleaned up old namespaces demand-backup-sharded-21215
-----------------------------------------------------------------------------------

error: resource(s) were provided, but no name was specified


-----------------------------------------------------------------------------------
create namespace demand-backup-sharded-21215
-----------------------------------------------------------------------------------

namespace/demand-backup-sharded-21215 created
Context "gke_cloud-dev-112233_us-central1-a_jen-psmdb-1377-eeba1539-6-cluster2" modified.


-----------------------------------------------------------------------------------
install Minio
-----------------------------------------------------------------------------------

Error: uninstall: Release not loaded: minio-service: release: not found
"minio" has been removed from your repositories
"minio" has been added to your repositories
NAME: minio-service
LAST DEPLOYED: Sat Jan 27 16:19:21 2024
NAMESPACE: demand-backup-sharded-21215
STATUS: deployed
REVISION: 1
TEST SUITE: None
NOTES:
MinIO can be accessed via port 9000 on the following DNS name from within your cluster:
minio-service.demand-backup-sharded-21215.svc.cluster.local

To access MinIO from localhost, run the below commands:

  1. export POD_NAME=$(kubectl get pods --namespace demand-backup-sharded-21215 -l "release=minio-service" -o jsonpath="{.items[0].metadata.name}")

  2. kubectl port-forward $POD_NAME 9000 --namespace demand-backup-sharded-21215

Read more about port forwarding here: http://kubernetes.io/docs/user-guide/kubectl/kubectl_port-forward/

You can now access MinIO server on http://localhost:9000. Follow the below steps to connect to MinIO server with mc client:

  1. Download the MinIO mc client - https://min.io/docs/minio/linux/reference/minio-mc.html#quickstart

  2. export MC_HOST_minio-service-local=http://$(kubectl get secret --namespace demand-backup-sharded-21215 minio-service -o jsonpath="{.data.rootUser}" | base64 --decode):$(kubectl get secret --namespace demand-backup-sharded-21215 minio-service -o jsonpath="{.data.rootPassword}" | base64 --decode)@localhost:9000

  3. mc ls minio-service-local
waiting for pod/minio-service-75dd45bdcd-snvkg to be ready.OK
service/minio-service created
make_bucket: operator-testing
pod "aws-cli" deleted
If you don't see a command prompt, try pressing enter.
warning: couldn't attach to pod/aws-cli, falling back to streaming logs: unable to upgrade connection: container aws-cli not found in pod aws-cli_demand-backup-sharded-21215


-----------------------------------------------------------------------------------
create first PSMDB cluster
-----------------------------------------------------------------------------------

secret/some-users created
deployment.apps/psmdb-client created


-----------------------------------------------------------------------------------
create secrets for cloud storages
-----------------------------------------------------------------------------------

secret/minio-secret created
secret/aws-s3-secret created
secret/gcp-cs-secret created
secret/azure-secret created
runtimeclass.node.k8s.io/container-rc unchanged
perconaservermongodb.psmdb.percona.com/some-name created


-----------------------------------------------------------------------------------
check if all 3 Pods started
-----------------------------------------------------------------------------------

waiting for pod/some-name-rs0-0 to be ready.......OK
waiting for pod/some-name-rs0-1 to be ready.......OK
waiting for pod/some-name-rs0-2 to be ready.....OK
Waiting for cluster readyness...............................
waiting for pod/some-name-cfg-0 to be ready.OK
waiting for pod/some-name-cfg-1 to be ready.OK
waiting for pod/some-name-cfg-2 to be ready.OK
waiting for pod/some-name-mongos-0 to be ready.OK
waiting for pod/some-name-mongos-1 to be ready.OK
waiting for pod/some-name-mongos-2 to be ready.OK
Waiting for cluster readyness


-----------------------------------------------------------------------------------
check if service and statefulset created with expected config
-----------------------------------------------------------------------------------

secret/some-name-mongos created
waiting for pod/some-name-mongos-0 to be ready.OK
waiting for pod/some-name-mongos-1 to be ready.OK
waiting for pod/some-name-mongos-2 to be ready.OK
Waiting for cluster readyness..................................................................................


-----------------------------------------------------------------------------------
write data, read from all
-----------------------------------------------------------------------------------

Percona Server for MongoDB shell version v4.4.27-26
connecting to: mongodb://some-name-mongos.demand-backup-sharded-21215.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb
Implicit session: session { "id" : UUID("07cec164-2317-4040-8c08-c43c8503664b") }
Percona Server for MongoDB server version: v5.0.23-20
WARNING: shell and server versions do not match
Successfully added user: {
	"user" : "myApp",
	"roles" : [
		{
			"db" : "myApp",
			"role" : "readWrite"
		},
		{
			"db" : "myApp1",
			"role" : "readWrite"
		},
		{
			"db" : "myApp2",
			"role" : "readWrite"
		}
	]
}
bye
Percona Server for MongoDB shell version v4.4.27-26
connecting to: mongodb://some-name-mongos.demand-backup-sharded-21215.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb
Implicit session: session { "id" : UUID("3201c476-68cc-4986-92a9-e6af50950cfd") }
Percona Server for MongoDB server version: v5.0.23-20
WARNING: shell and server versions do not match
{
	"ok" : 1,
	"$clusterTime" : {
		"clusterTime" : Timestamp(1706372869, 1),
		"signature" : {
			"hash" : BinData(0,"UM7exDotv64TNLdNPCAkDJzvRG4="),
			"keyId" : NumberLong("7328814004984348677")
		}
	},
	"operationTime" : Timestamp(1706372868, 49)
}
bye
Percona Server for MongoDB shell version v4.4.27-26
connecting to: mongodb://some-name-mongos.demand-backup-sharded-21215.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb
Implicit session: session { "id" : UUID("ac3312ef-bef5-4cb4-8aa7-a6dbe3a1492c") }
Percona Server for MongoDB server version: v5.0.23-20
WARNING: shell and server versions do not match
{
	"ok" : 1,
	"$clusterTime" : {
		"clusterTime" : Timestamp(1706372873, 4),
		"signature" : {
			"hash" : BinData(0,"q1eN+Sd2naCh4ECR13Kf8ENYqVM="),
			"keyId" : NumberLong("7328814004984348677")
		}
	},
	"operationTime" : Timestamp(1706372873, 1)
}
bye
Percona Server for MongoDB shell version v4.4.27-26
connecting to: mongodb://some-name-mongos.demand-backup-sharded-21215.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb
Implicit session: session { "id" : UUID("b2838f2e-c665-4fbe-96ff-b5888170b5b9") }
Percona Server for MongoDB server version: v5.0.23-20
WARNING: shell and server versions do not match
{
	"ok" : 1,
	"$clusterTime" : {
		"clusterTime" : Timestamp(1706372876, 10),
		"signature" : {
			"hash" : BinData(0,"E7HL6G6ZjLwcac5ard/HmdPT0eY="),
			"keyId" : NumberLong("7328814004984348677")
		}
	},
	"operationTime" : Timestamp(1706372876, 7)
}
bye
Percona Server for MongoDB shell version v4.4.27-26
connecting to: mongodb://some-name-mongos.demand-backup-sharded-21215.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb
Implicit session: session { "id" : UUID("ed25a217-0746-4048-b3f7-8a0c73d93735") }
Percona Server for MongoDB server version: v5.0.23-20
WARNING: shell and server versions do not match
switched to db myApp
WriteResult({ "nInserted" : 1 })
bye
Percona Server for MongoDB shell version v4.4.27-26
connecting to: mongodb://some-name-mongos.demand-backup-sharded-21215.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb
Implicit session: session { "id" : UUID("142e6ba4-c68c-46c7-945c-bc13141928ba") }
Percona Server for MongoDB server version: v5.0.23-20
WARNING: shell and server versions do not match
switched to db myApp1
WriteResult({ "nInserted" : 1 })
bye
Percona Server for MongoDB shell version v4.4.27-26
connecting to: mongodb://some-name-mongos.demand-backup-sharded-21215.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb
Implicit session: session { "id" : UUID("bcf706fc-2f85-4b34-8a22-db0bfe0a5fb4") }
Percona Server for MongoDB server version: v5.0.23-20
WARNING: shell and server versions do not match
switched to db myApp2
WriteResult({ "nInserted" : 1 })
bye
some-name-rs0-0
some-name-rs0-1
some-name-rs0-2
some-name-rs1-0
some-name-rs1-1
some-name-rs1-2
some-name-rs2-0
some-name-rs2-1
some-name-rs2-2


-----------------------------------------------------------------------------------
run backups
-----------------------------------------------------------------------------------



-----------------------------------------------------------------------------------
run backup backup-minio
-----------------------------------------------------------------------------------

perconaservermongodbbackup.psmdb.percona.com/backup-minio created


-----------------------------------------------------------------------------------
run backup backup-aws-s3
-----------------------------------------------------------------------------------

perconaservermongodbbackup.psmdb.percona.com/backup-aws-s3 created


-----------------------------------------------------------------------------------
run backup backup-gcp-cs
-----------------------------------------------------------------------------------

perconaservermongodbbackup.psmdb.percona.com/backup-gcp-cs created


-----------------------------------------------------------------------------------
run backup backup-azure-blob
-----------------------------------------------------------------------------------

perconaservermongodbbackup.psmdb.percona.com/backup-azure-blob created
backup-aws-s3............................
backup-gcp-cs...............
backup-azure-blob...............
backup-minio.


-----------------------------------------------------------------------------------
check backup and restore -- aws-s3
-----------------------------------------------------------------------------------

Percona Server for MongoDB shell version v4.4.27-26
connecting to: mongodb://some-name-mongos.demand-backup-sharded-21215.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb
Implicit session: session { "id" : UUID("26ae0a29-8a2c-4d45-82e8-00d24d9c06b4") }
Percona Server for MongoDB server version: v5.0.23-20
WARNING: shell and server versions do not match
switched to db myApp
WriteResult({ "nInserted" : 1 })
bye
Percona Server for MongoDB shell version v4.4.27-26
connecting to: mongodb://some-name-mongos.demand-backup-sharded-21215.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb
Implicit session: session { "id" : UUID("6ba14989-9ab0-4916-a767-041da1c6c6ee") }
Percona Server for MongoDB server version: v5.0.23-20
WARNING: shell and server versions do not match
switched to db myApp1
WriteResult({ "nInserted" : 1 })
bye
Percona Server for MongoDB shell version v4.4.27-26
connecting to: mongodb://some-name-mongos.demand-backup-sharded-21215.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb
Implicit session: session { "id" : UUID("f73f1720-bc83-47e2-a139-b88e86c87ae4") }
Percona Server for MongoDB server version: v5.0.23-20
WARNING: shell and server versions do not match
switched to db myApp2
WriteResult({ "nInserted" : 1 })
bye
perconaservermongodbrestore.psmdb.percona.com/restore-backup-aws-s3 created
waiting psmdb-restore/backup-aws-s3 to reach ready state............
+ '[' 1 -eq 1 ']'
+ wait_cluster_consistency some-name
+ local cluster_name=some-name
+ local wait_time=32
+ retry=0
+ sleep 7
+ echo -n 'waiting for cluster readyness'
waiting for cluster readyness++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.u4Y8aT4xcT
+++ mktemp
++ local LAST_ERR=/tmp/tmp.J8To5UKyuG
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 0 ']'
++ break
++ cat /tmp/tmp.u4Y8aT4xcT
++ cat /tmp/tmp.J8To5UKyuG
++ rm /tmp/tmp.u4Y8aT4xcT /tmp/tmp.J8To5UKyuG
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 1 -ge 32 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.ST23eVAy73
+++ mktemp
++ local LAST_ERR=/tmp/tmp.zMLAfzN5BY
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 0 ']'
++ break
++ cat /tmp/tmp.ST23eVAy73
++ cat /tmp/tmp.zMLAfzN5BY
++ rm /tmp/tmp.ST23eVAy73 /tmp/tmp.zMLAfzN5BY
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 2 -ge 32 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.YBV3x3WwDw
+++ mktemp
++ local LAST_ERR=/tmp/tmp.O8uX6IONls
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 0 ']'
++ break
++ cat /tmp/tmp.YBV3x3WwDw
++ cat /tmp/tmp.O8uX6IONls
++ rm /tmp/tmp.YBV3x3WwDw /tmp/tmp.O8uX6IONls
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 3 -ge 32 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.TODA219MaY
+++ mktemp
++ local LAST_ERR=/tmp/tmp.2G0OAowebr
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 0 ']'
++ break
++ cat /tmp/tmp.TODA219MaY
++ cat /tmp/tmp.2G0OAowebr
++ rm /tmp/tmp.TODA219MaY /tmp/tmp.2G0OAowebr
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 4 -ge 32 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.vmy2ispOA7
+++ mktemp
++ local LAST_ERR=/tmp/tmp.AvFexb50DC
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 0 ']'
++ break
++ cat /tmp/tmp.vmy2ispOA7
++ cat /tmp/tmp.AvFexb50DC
++ rm /tmp/tmp.vmy2ispOA7 /tmp/tmp.AvFexb50DC
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 5 -ge 32 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.GfCBozB5lV
+++ mktemp
++ local LAST_ERR=/tmp/tmp.pmYQh4TRMC
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 0 ']'
++ break
++ cat /tmp/tmp.GfCBozB5lV
++ cat /tmp/tmp.pmYQh4TRMC
++ rm /tmp/tmp.GfCBozB5lV /tmp/tmp.pmYQh4TRMC
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 6 -ge 32 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.6qTLZBHiez
+++ mktemp
++ local LAST_ERR=/tmp/tmp.RPid2ngL3Y
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 0 ']'
++ break
++ cat /tmp/tmp.6qTLZBHiez
++ cat /tmp/tmp.RPid2ngL3Y
++ rm /tmp/tmp.6qTLZBHiez /tmp/tmp.RPid2ngL3Y
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 7 -ge 32 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.uWjzF8qj2r
+++ mktemp
++ local LAST_ERR=/tmp/tmp.Jom1kpvxv1
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 0 ']'
++ break
++ cat /tmp/tmp.uWjzF8qj2r
++ cat /tmp/tmp.Jom1kpvxv1
++ rm /tmp/tmp.uWjzF8qj2r /tmp/tmp.Jom1kpvxv1
++ return 0
+ [[ ready == \r\e\a\d\y ]]
+ check_data
+ local postfix=
++ seq 0 2
+ for i in '$(seq 0 2)'
+ compare_mongos_cmd find myApp:myPass@some-name-mongos.demand-backup-sharded-21215 '' .svc.cluster.local myApp test
+ local command=find
+ local uri=myApp:myPass@some-name-mongos.demand-backup-sharded-21215
+ local postfix=
+ local suffix=.svc.cluster.local
+ local database=myApp
+ local collection=test
+ egrep -v 'I NETWORK|W NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:'
+ run_mongos 'use myApp\n db.test.find()' myApp:myPass@some-name-mongos.demand-backup-sharded-21215 mongodb .svc.cluster.local
+ local 'command=use myApp\n db.test.find()'
+ local uri=myApp:myPass@some-name-mongos.demand-backup-sharded-21215
+ local driver=mongodb
+ local suffix=.svc.cluster.local
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
+ /usr/bin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
++ local LAST_OUT=/tmp/tmp.E234bzQh4p
+++ mktemp
++ local LAST_ERR=/tmp/tmp.MFBQKyagrT
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 0 ']'
++ break
++ cat /tmp/tmp.E234bzQh4p
++ cat /tmp/tmp.MFBQKyagrT
++ rm /tmp/tmp.E234bzQh4p /tmp/tmp.MFBQKyagrT
++ return 0
+ local client_container=psmdb-client-6b9c964f46-h2gfr
+ local mongo_flag=
+ kubectl_bin exec psmdb-client-6b9c964f46-h2gfr -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-21215.svc.cluster.local/admin '
++ mktemp
+ local LAST_OUT=/tmp/tmp.PNK4yTujJn
++ mktemp
+ local LAST_ERR=/tmp/tmp.Y9ZJ9PpEN7
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl exec psmdb-client-6b9c964f46-h2gfr -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-21215.svc.cluster.local/admin '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 0 ']'
+ break
+ cat /tmp/tmp.PNK4yTujJn
+ cat /tmp/tmp.Y9ZJ9PpEN7
+ rm /tmp/tmp.PNK4yTujJn /tmp/tmp.Y9ZJ9PpEN7
+ return 0
+ diff /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1377/e2e-tests/demand-backup-sharded/compare/find.json /tmp/tmp.geCbJnr5r4/find
+ for i in '$(seq 0 2)'
+ compare_mongos_cmd find myApp:myPass@some-name-mongos.demand-backup-sharded-21215 1 .svc.cluster.local myApp1 test
+ local command=find
+ local uri=myApp:myPass@some-name-mongos.demand-backup-sharded-21215
+ local postfix=1
+ local suffix=.svc.cluster.local
+ local database=myApp1
+ local collection=test
+ /usr/bin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
+ egrep -v 'I NETWORK|W NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:'
+ run_mongos 'use myApp1\n db.test.find()' myApp:myPass@some-name-mongos.demand-backup-sharded-21215 mongodb .svc.cluster.local
+ local 'command=use myApp1\n db.test.find()'
+ local uri=myApp:myPass@some-name-mongos.demand-backup-sharded-21215
+ local driver=mongodb
+ local suffix=.svc.cluster.local
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.aHEnKgqX2O
+++ mktemp
++ local LAST_ERR=/tmp/tmp.ik41VBnrZ6
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 0 ']'
++ break
++ cat /tmp/tmp.aHEnKgqX2O
++ cat /tmp/tmp.ik41VBnrZ6
++ rm /tmp/tmp.aHEnKgqX2O /tmp/tmp.ik41VBnrZ6
++ return 0
+ local client_container=psmdb-client-6b9c964f46-h2gfr
+ local mongo_flag=
+ kubectl_bin exec psmdb-client-6b9c964f46-h2gfr -- bash -c 'printf '\''use myApp1\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-21215.svc.cluster.local/admin '
++ mktemp
+ local LAST_OUT=/tmp/tmp.QPTWr1Z8aG
++ mktemp
+ local LAST_ERR=/tmp/tmp.zNBGeGXY9f
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl exec psmdb-client-6b9c964f46-h2gfr -- bash -c 'printf '\''use myApp1\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-21215.svc.cluster.local/admin '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 0 ']'
+ break
+ cat /tmp/tmp.QPTWr1Z8aG
+ cat /tmp/tmp.zNBGeGXY9f
+ rm /tmp/tmp.QPTWr1Z8aG /tmp/tmp.zNBGeGXY9f
+ return 0
+ diff /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1377/e2e-tests/demand-backup-sharded/compare/find1.json /tmp/tmp.geCbJnr5r4/find1
+ for i in '$(seq 0 2)'
+ compare_mongos_cmd find myApp:myPass@some-name-mongos.demand-backup-sharded-21215 2 .svc.cluster.local myApp2 test
+ local command=find
+ local uri=myApp:myPass@some-name-mongos.demand-backup-sharded-21215
+ local postfix=2
+ local suffix=.svc.cluster.local
+ local database=myApp2
+ local collection=test
+ egrep -v 'I NETWORK|W NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:'
+ /usr/bin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
+ run_mongos 'use myApp2\n db.test.find()' myApp:myPass@some-name-mongos.demand-backup-sharded-21215 mongodb .svc.cluster.local
+ local 'command=use myApp2\n db.test.find()'
+ local uri=myApp:myPass@some-name-mongos.demand-backup-sharded-21215
+ local driver=mongodb
+ local suffix=.svc.cluster.local
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.3DGLMfEs7i
+++ mktemp
++ local LAST_ERR=/tmp/tmp.5LjK1W2VJR
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 0 ']'
++ break
++ cat /tmp/tmp.3DGLMfEs7i
++ cat /tmp/tmp.5LjK1W2VJR
++ rm /tmp/tmp.3DGLMfEs7i /tmp/tmp.5LjK1W2VJR
++ return 0
+ local client_container=psmdb-client-6b9c964f46-h2gfr
+ local mongo_flag=
+ kubectl_bin exec psmdb-client-6b9c964f46-h2gfr -- bash -c 'printf '\''use myApp2\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-21215.svc.cluster.local/admin '
++ mktemp
+ local LAST_OUT=/tmp/tmp.SePwJf0JpQ
++ mktemp
+ local LAST_ERR=/tmp/tmp.fnwFjOdWUv
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl exec psmdb-client-6b9c964f46-h2gfr -- bash -c 'printf '\''use myApp2\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-21215.svc.cluster.local/admin '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 0 ']'
+ break
+ cat /tmp/tmp.SePwJf0JpQ
+ cat /tmp/tmp.fnwFjOdWUv
+ rm /tmp/tmp.SePwJf0JpQ /tmp/tmp.fnwFjOdWUv
+ return 0
+ diff /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1377/e2e-tests/demand-backup-sharded/compare/find2.json /tmp/tmp.geCbJnr5r4/find2
+ desc 'check backup and restore -- gcp-cs'
+ set +o xtrace


-----------------------------------------------------------------------------------
check backup and restore -- gcp-cs
-----------------------------------------------------------------------------------

Percona Server for MongoDB shell version v4.4.27-26
connecting to: mongodb://some-name-mongos.demand-backup-sharded-21215.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb
Implicit session: session { "id" : UUID("3e33b835-483a-4efd-a54d-53cb021b5140") }
Percona Server for MongoDB server version: v5.0.23-20
WARNING: shell and server versions do not match
switched to db myApp
WriteResult({ "nInserted" : 1 })
bye
Percona Server for MongoDB shell version v4.4.27-26
connecting to: mongodb://some-name-mongos.demand-backup-sharded-21215.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb
Implicit session: session { "id" : UUID("40f024a4-e979-4760-82b7-f8183401d7f5") }
Percona Server for MongoDB server version: v5.0.23-20
WARNING: shell and server versions do not match
switched to db myApp1
WriteResult({ "nInserted" : 1 })
bye
Percona Server for MongoDB shell version v4.4.27-26
connecting to: mongodb://some-name-mongos.demand-backup-sharded-21215.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb
Implicit session: session { "id" : UUID("9adad592-719f-4dcc-9efb-37891df27e60") }
Percona Server for MongoDB server version: v5.0.23-20
WARNING: shell and server versions do not match
switched to db myApp2
WriteResult({ "nInserted" : 1 })
bye
perconaservermongodbrestore.psmdb.percona.com/restore-backup-gcp-cs created
waiting psmdb-restore/backup-gcp-cs to reach ready state...........
+ '[' 1 -eq 1 ']'
+ wait_cluster_consistency some-name
+ local cluster_name=some-name
+ local wait_time=32
+ retry=0
+ sleep 7
+ echo -n 'waiting for cluster readyness'
waiting for cluster readyness++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.CiwIXDcw81
+++ mktemp
++ local LAST_ERR=/tmp/tmp.hx6Sj4GML5
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 0 ']'
++ break
++ cat /tmp/tmp.CiwIXDcw81
++ cat /tmp/tmp.hx6Sj4GML5
++ rm /tmp/tmp.CiwIXDcw81 /tmp/tmp.hx6Sj4GML5
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 1 -ge 32 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.q9F3VTQ2OL
+++ mktemp
++ local LAST_ERR=/tmp/tmp.LNYBIYGerD
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 0 ']'
++ break
++ cat /tmp/tmp.q9F3VTQ2OL
++ cat /tmp/tmp.LNYBIYGerD
++ rm /tmp/tmp.q9F3VTQ2OL /tmp/tmp.LNYBIYGerD
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 2 -ge 32 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.QqNC4PdwGu
+++ mktemp
++ local LAST_ERR=/tmp/tmp.tZ2vcMc94r
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 0 ']'
++ break
++ cat /tmp/tmp.QqNC4PdwGu
++ cat /tmp/tmp.tZ2vcMc94r
++ rm /tmp/tmp.QqNC4PdwGu /tmp/tmp.tZ2vcMc94r
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 3 -ge 32 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.zQf5RxEGyG
+++ mktemp
++ local LAST_ERR=/tmp/tmp.IVIVQ6ZfEK
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 0 ']'
++ break
++ cat /tmp/tmp.zQf5RxEGyG
++ cat /tmp/tmp.IVIVQ6ZfEK
++ rm /tmp/tmp.zQf5RxEGyG /tmp/tmp.IVIVQ6ZfEK
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 4 -ge 32 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.xZqZrMFaOh
+++ mktemp
++ local LAST_ERR=/tmp/tmp.qnAjBf5nwi
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 0 ']'
++ break
++ cat /tmp/tmp.xZqZrMFaOh
++ cat /tmp/tmp.qnAjBf5nwi
++ rm /tmp/tmp.xZqZrMFaOh /tmp/tmp.qnAjBf5nwi
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 5 -ge 32 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.7MhpTeFRBq
+++ mktemp
++ local LAST_ERR=/tmp/tmp.1mCCAZbyCX
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 0 ']'
++ break
++ cat /tmp/tmp.7MhpTeFRBq
++ cat /tmp/tmp.1mCCAZbyCX
++ rm /tmp/tmp.7MhpTeFRBq /tmp/tmp.1mCCAZbyCX
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 6 -ge 32 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.TJQoebwrOE
+++ mktemp
++ local LAST_ERR=/tmp/tmp.0JTeMdkD2P
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 0 ']'
++ break
++ cat /tmp/tmp.TJQoebwrOE
++ cat /tmp/tmp.0JTeMdkD2P
++ rm /tmp/tmp.TJQoebwrOE /tmp/tmp.0JTeMdkD2P
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 7 -ge 32 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.AiWaChqdqO
+++ mktemp
++ local LAST_ERR=/tmp/tmp.eP2vRza5To
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 0 ']'
++ break
++ cat /tmp/tmp.AiWaChqdqO
++ cat /tmp/tmp.eP2vRza5To
++ rm /tmp/tmp.AiWaChqdqO /tmp/tmp.eP2vRza5To
++ return 0
+ [[ ready == \r\e\a\d\y ]]
+ check_data
+ local postfix=
++ seq 0 2
+ for i in '$(seq 0 2)'
+ compare_mongos_cmd find myApp:myPass@some-name-mongos.demand-backup-sharded-21215 '' .svc.cluster.local myApp test
+ local command=find
+ local uri=myApp:myPass@some-name-mongos.demand-backup-sharded-21215
+ local postfix=
+ local suffix=.svc.cluster.local
+ local database=myApp
+ local collection=test
+ run_mongos 'use myApp\n db.test.find()' myApp:myPass@some-name-mongos.demand-backup-sharded-21215 mongodb .svc.cluster.local
+ local 'command=use myApp\n db.test.find()'
+ local uri=myApp:myPass@some-name-mongos.demand-backup-sharded-21215
+ local driver=mongodb
+ local suffix=.svc.cluster.local
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
+ /usr/bin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
+ egrep -v 'I NETWORK|W NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:'
++ local LAST_OUT=/tmp/tmp.kCPO2mZysT
+++ mktemp
++ local LAST_ERR=/tmp/tmp.AS8x2WdHvG
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 0 ']'
++ break
++ cat /tmp/tmp.kCPO2mZysT
++ cat /tmp/tmp.AS8x2WdHvG
++ rm /tmp/tmp.kCPO2mZysT /tmp/tmp.AS8x2WdHvG
++ return 0
+ local client_container=psmdb-client-6b9c964f46-h2gfr
+ local mongo_flag=
+ kubectl_bin exec psmdb-client-6b9c964f46-h2gfr -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-21215.svc.cluster.local/admin '
++ mktemp
+ local LAST_OUT=/tmp/tmp.ftUvFhzWxT
++ mktemp
+ local LAST_ERR=/tmp/tmp.VHufabynnr
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl exec psmdb-client-6b9c964f46-h2gfr -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-21215.svc.cluster.local/admin '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 0 ']'
+ break
+ cat /tmp/tmp.ftUvFhzWxT
+ cat /tmp/tmp.VHufabynnr
+ rm /tmp/tmp.ftUvFhzWxT /tmp/tmp.VHufabynnr
+ return 0
+ diff /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1377/e2e-tests/demand-backup-sharded/compare/find.json /tmp/tmp.geCbJnr5r4/find
+ for i in '$(seq 0 2)'
+ compare_mongos_cmd find myApp:myPass@some-name-mongos.demand-backup-sharded-21215 1 .svc.cluster.local myApp1 test
+ local command=find
+ local uri=myApp:myPass@some-name-mongos.demand-backup-sharded-21215
+ local postfix=1
+ local suffix=.svc.cluster.local
+ local database=myApp1
+ local collection=test
+ run_mongos 'use myApp1\n db.test.find()' myApp:myPass@some-name-mongos.demand-backup-sharded-21215 mongodb .svc.cluster.local
+ local 'command=use myApp1\n db.test.find()'
+ local uri=myApp:myPass@some-name-mongos.demand-backup-sharded-21215
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ egrep -v 'I NETWORK|W NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:'
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.cKyw4DuLTu
+++ mktemp
+ /usr/bin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
++ local LAST_ERR=/tmp/tmp.RG7MlAUsgR
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 0 ']'
++ break
++ cat /tmp/tmp.cKyw4DuLTu
++ cat /tmp/tmp.RG7MlAUsgR
++ rm /tmp/tmp.cKyw4DuLTu /tmp/tmp.RG7MlAUsgR
++ return 0
+ local client_container=psmdb-client-6b9c964f46-h2gfr
+ local mongo_flag=
+ kubectl_bin exec psmdb-client-6b9c964f46-h2gfr -- bash -c 'printf '\''use myApp1\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-21215.svc.cluster.local/admin '
++ mktemp
+ local LAST_OUT=/tmp/tmp.8ECvHs81j9
++ mktemp
+ local LAST_ERR=/tmp/tmp.00JVLbQnxZ
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl exec psmdb-client-6b9c964f46-h2gfr -- bash -c 'printf '\''use myApp1\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-21215.svc.cluster.local/admin '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 0 ']'
+ break
+ cat /tmp/tmp.8ECvHs81j9
+ cat /tmp/tmp.00JVLbQnxZ
+ rm /tmp/tmp.8ECvHs81j9 /tmp/tmp.00JVLbQnxZ
+ return 0
+ diff /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1377/e2e-tests/demand-backup-sharded/compare/find1.json /tmp/tmp.geCbJnr5r4/find1
+ for i in '$(seq 0 2)'
+ compare_mongos_cmd find myApp:myPass@some-name-mongos.demand-backup-sharded-21215 2 .svc.cluster.local myApp2 test
+ local command=find
+ local uri=myApp:myPass@some-name-mongos.demand-backup-sharded-21215
+ local postfix=2
+ local suffix=.svc.cluster.local
+ local database=myApp2
+ local collection=test
+ egrep -v 'I NETWORK|W NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:'
+ run_mongos 'use myApp2\n db.test.find()' myApp:myPass@some-name-mongos.demand-backup-sharded-21215 mongodb .svc.cluster.local
+ local 'command=use myApp2\n db.test.find()'
+ local uri=myApp:myPass@some-name-mongos.demand-backup-sharded-21215
+ local driver=mongodb
+ local suffix=.svc.cluster.local
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+ /usr/bin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.vyyls76vbP
+++ mktemp
++ local LAST_ERR=/tmp/tmp.Kqd7p4FfG7
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 0 ']'
++ break
++ cat /tmp/tmp.vyyls76vbP
++ cat /tmp/tmp.Kqd7p4FfG7
++ rm /tmp/tmp.vyyls76vbP /tmp/tmp.Kqd7p4FfG7
++ return 0
+ local client_container=psmdb-client-6b9c964f46-h2gfr
+ local mongo_flag=
+ kubectl_bin exec psmdb-client-6b9c964f46-h2gfr -- bash -c 'printf '\''use myApp2\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-21215.svc.cluster.local/admin '
++ mktemp
+ local LAST_OUT=/tmp/tmp.Y7gre7lFTx
++ mktemp
+ local LAST_ERR=/tmp/tmp.5rhghH2Xvy
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl exec psmdb-client-6b9c964f46-h2gfr -- bash -c 'printf '\''use myApp2\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-21215.svc.cluster.local/admin '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 0 ']'
+ break
+ cat /tmp/tmp.Y7gre7lFTx
+ cat /tmp/tmp.5rhghH2Xvy
+ rm /tmp/tmp.Y7gre7lFTx /tmp/tmp.5rhghH2Xvy
+ return 0
+ diff /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1377/e2e-tests/demand-backup-sharded/compare/find2.json /tmp/tmp.geCbJnr5r4/find2
+ desc 'check backup and restore -- azure-blob'
+ set +o xtrace


-----------------------------------------------------------------------------------
check backup and restore -- azure-blob
-----------------------------------------------------------------------------------

Percona Server for MongoDB shell version v4.4.27-26
connecting to: mongodb://some-name-mongos.demand-backup-sharded-21215.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb
Implicit session: session { "id" : UUID("3dcfc0a2-90fc-4188-8b34-a2ea255d23ba") }
Percona Server for MongoDB server version: v5.0.23-20
WARNING: shell and server versions do not match
switched to db myApp
WriteResult({ "nInserted" : 1 })
bye
Percona Server for MongoDB shell version v4.4.27-26
connecting to: mongodb://some-name-mongos.demand-backup-sharded-21215.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb
Implicit session: session { "id" : UUID("5caeba54-66b9-4319-8009-bc16bbafc47c") }
Percona Server for MongoDB server version: v5.0.23-20
WARNING: shell and server versions do not match
switched to db myApp1
WriteResult({ "nInserted" : 1 })
bye
Percona Server for MongoDB shell version v4.4.27-26
connecting to: mongodb://some-name-mongos.demand-backup-sharded-21215.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb
Implicit session: session { "id" : UUID("5d1ce07e-3197-437c-98bc-8ccc5968e9d5") }
Percona Server for MongoDB server version: v5.0.23-20
WARNING: shell and server versions do not match
switched to db myApp2
WriteResult({ "nInserted" : 1 })
bye
perconaservermongodbrestore.psmdb.percona.com/restore-backup-azure-blob created
waiting psmdb-restore/backup-azure-blob to reach ready state...........
+ '[' 1 -eq 1 ']'
+ wait_cluster_consistency some-name
+ local cluster_name=some-name
+ local wait_time=32
+ retry=0
+ sleep 7
+ echo -n 'waiting for cluster readyness'
waiting for cluster readyness++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.nrd30pcwrB
+++ mktemp
++ local LAST_ERR=/tmp/tmp.t91OP1Kd3o
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 0 ']'
++ break
++ cat /tmp/tmp.nrd30pcwrB
++ cat /tmp/tmp.t91OP1Kd3o
++ rm /tmp/tmp.nrd30pcwrB /tmp/tmp.t91OP1Kd3o
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 1 -ge 32 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.u7uolmOFT9
+++ mktemp
++ local LAST_ERR=/tmp/tmp.vcnHfHwfpU
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 0 ']'
++ break
++ cat /tmp/tmp.u7uolmOFT9
++ cat /tmp/tmp.vcnHfHwfpU
++ rm /tmp/tmp.u7uolmOFT9 /tmp/tmp.vcnHfHwfpU
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 2 -ge 32 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.WPSHGy44LU
+++ mktemp
++ local LAST_ERR=/tmp/tmp.9wvWIUblBA
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 0 ']'
++ break
++ cat /tmp/tmp.WPSHGy44LU
++ cat /tmp/tmp.9wvWIUblBA
++ rm /tmp/tmp.WPSHGy44LU /tmp/tmp.9wvWIUblBA
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 3 -ge 32 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.0Uj5YfEBVK
+++ mktemp
++ local LAST_ERR=/tmp/tmp.3PkIWjS5wC
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 0 ']'
++ break
++ cat /tmp/tmp.0Uj5YfEBVK
++ cat /tmp/tmp.3PkIWjS5wC
++ rm /tmp/tmp.0Uj5YfEBVK /tmp/tmp.3PkIWjS5wC
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 4 -ge 32 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.RiHfmretYw
+++ mktemp
++ local LAST_ERR=/tmp/tmp.5vNk5UM5di
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 0 ']'
++ break
++ cat /tmp/tmp.RiHfmretYw
++ cat /tmp/tmp.5vNk5UM5di
++ rm /tmp/tmp.RiHfmretYw /tmp/tmp.5vNk5UM5di
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 5 -ge 32 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.neNpWJzOfB
+++ mktemp
++ local LAST_ERR=/tmp/tmp.31YGf2YziX
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 0 ']'
++ break
++ cat /tmp/tmp.neNpWJzOfB
++ cat /tmp/tmp.31YGf2YziX
++ rm /tmp/tmp.neNpWJzOfB /tmp/tmp.31YGf2YziX
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 6 -ge 32 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.JwzNqOoK9L
+++ mktemp
++ local LAST_ERR=/tmp/tmp.XtQnyU2rfl
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 0 ']'
++ break
++ cat /tmp/tmp.JwzNqOoK9L
++ cat /tmp/tmp.XtQnyU2rfl
++ rm /tmp/tmp.JwzNqOoK9L /tmp/tmp.XtQnyU2rfl
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 7 -ge 32 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.fK3tWk8SRc
+++ mktemp
++ local LAST_ERR=/tmp/tmp.PM8CaY2F4K
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 0 ']'
++ break
++ cat /tmp/tmp.fK3tWk8SRc
++ cat /tmp/tmp.PM8CaY2F4K
++ rm /tmp/tmp.fK3tWk8SRc /tmp/tmp.PM8CaY2F4K
++ return 0
+ [[ ready == \r\e\a\d\y ]]
+ check_data
+ local postfix=
++ seq 0 2
+ for i in '$(seq 0 2)'
+ compare_mongos_cmd find myApp:myPass@some-name-mongos.demand-backup-sharded-21215 '' .svc.cluster.local myApp test
+ local command=find
+ local uri=myApp:myPass@some-name-mongos.demand-backup-sharded-21215
+ local postfix=
+ local suffix=.svc.cluster.local
+ local database=myApp
+ local collection=test
+ run_mongos 'use myApp\n db.test.find()' myApp:myPass@some-name-mongos.demand-backup-sharded-21215 mongodb .svc.cluster.local
+ local 'command=use myApp\n db.test.find()'
+ local uri=myApp:myPass@some-name-mongos.demand-backup-sharded-21215
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ /usr/bin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
+ egrep -v 'I NETWORK|W NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:'
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.5PNmXhzzJp
+++ mktemp
++ local LAST_ERR=/tmp/tmp.VtpMy1A0gj
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 0 ']'
++ break
++ cat /tmp/tmp.5PNmXhzzJp
++ cat /tmp/tmp.VtpMy1A0gj
++ rm /tmp/tmp.5PNmXhzzJp /tmp/tmp.VtpMy1A0gj
++ return 0
+ local client_container=psmdb-client-6b9c964f46-h2gfr
+ local mongo_flag=
+ kubectl_bin exec psmdb-client-6b9c964f46-h2gfr -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-21215.svc.cluster.local/admin '
++ mktemp
+ local LAST_OUT=/tmp/tmp.v5F8EkSzzC
++ mktemp
+ local LAST_ERR=/tmp/tmp.AObigeDgsq
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl exec psmdb-client-6b9c964f46-h2gfr -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-21215.svc.cluster.local/admin '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 0 ']'
+ break
+ cat /tmp/tmp.v5F8EkSzzC
+ cat /tmp/tmp.AObigeDgsq
+ rm /tmp/tmp.v5F8EkSzzC /tmp/tmp.AObigeDgsq
+ return 0
+ diff /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1377/e2e-tests/demand-backup-sharded/compare/find.json /tmp/tmp.geCbJnr5r4/find
+ for i in '$(seq 0 2)'
+ compare_mongos_cmd find myApp:myPass@some-name-mongos.demand-backup-sharded-21215 1 .svc.cluster.local myApp1 test
+ local command=find
+ local uri=myApp:myPass@some-name-mongos.demand-backup-sharded-21215
+ local postfix=1
+ local suffix=.svc.cluster.local
+ local database=myApp1
+ local collection=test
+ run_mongos 'use myApp1\n db.test.find()' myApp:myPass@some-name-mongos.demand-backup-sharded-21215 mongodb .svc.cluster.local
+ local 'command=use myApp1\n db.test.find()'
+ local uri=myApp:myPass@some-name-mongos.demand-backup-sharded-21215
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ /usr/bin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
+ egrep -v 'I NETWORK|W NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:'
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.XkbjmAosUz
+++ mktemp
++ local LAST_ERR=/tmp/tmp.5gsZmTpRfN
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 0 ']'
++ break
++ cat /tmp/tmp.XkbjmAosUz
++ cat /tmp/tmp.5gsZmTpRfN
++ rm /tmp/tmp.XkbjmAosUz /tmp/tmp.5gsZmTpRfN
++ return 0
+ local client_container=psmdb-client-6b9c964f46-h2gfr
+ local mongo_flag=
+ kubectl_bin exec psmdb-client-6b9c964f46-h2gfr -- bash -c 'printf '\''use myApp1\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-21215.svc.cluster.local/admin '
++ mktemp
+ local LAST_OUT=/tmp/tmp.qtBsWkQCTq
++ mktemp
+ local LAST_ERR=/tmp/tmp.2kyHAANSUf
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl exec psmdb-client-6b9c964f46-h2gfr -- bash -c 'printf '\''use myApp1\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-21215.svc.cluster.local/admin '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 0 ']'
+ break
+ cat /tmp/tmp.qtBsWkQCTq
+ cat /tmp/tmp.2kyHAANSUf
+ rm /tmp/tmp.qtBsWkQCTq /tmp/tmp.2kyHAANSUf
+ return 0
+ diff /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1377/e2e-tests/demand-backup-sharded/compare/find1.json /tmp/tmp.geCbJnr5r4/find1
+ for i in '$(seq 0 2)'
+ compare_mongos_cmd find myApp:myPass@some-name-mongos.demand-backup-sharded-21215 2 .svc.cluster.local myApp2 test
+ local command=find
+ local uri=myApp:myPass@some-name-mongos.demand-backup-sharded-21215
+ local postfix=2
+ local suffix=.svc.cluster.local
+ local database=myApp2
+ local collection=test
+ run_mongos 'use myApp2\n db.test.find()' myApp:myPass@some-name-mongos.demand-backup-sharded-21215 mongodb .svc.cluster.local
+ egrep -v 'I NETWORK|W NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:'
+ local 'command=use myApp2\n db.test.find()'
+ local uri=myApp:myPass@some-name-mongos.demand-backup-sharded-21215
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ /usr/bin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.g9v35OHHFB
+++ mktemp
++ local LAST_ERR=/tmp/tmp.6tc7ZHgTwS
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 0 ']'
++ break
++ cat /tmp/tmp.g9v35OHHFB
++ cat /tmp/tmp.6tc7ZHgTwS
++ rm /tmp/tmp.g9v35OHHFB /tmp/tmp.6tc7ZHgTwS
++ return 0
+ local client_container=psmdb-client-6b9c964f46-h2gfr
+ local mongo_flag=
+ kubectl_bin exec psmdb-client-6b9c964f46-h2gfr -- bash -c 'printf '\''use myApp2\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-21215.svc.cluster.local/admin '
++ mktemp
+ local LAST_OUT=/tmp/tmp.cWvPIRPIeZ
++ mktemp
+ local LAST_ERR=/tmp/tmp.MekG5m93yV
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl exec psmdb-client-6b9c964f46-h2gfr -- bash -c 'printf '\''use myApp2\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-21215.svc.cluster.local/admin '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 0 ']'
+ break
+ cat /tmp/tmp.cWvPIRPIeZ
+ cat /tmp/tmp.MekG5m93yV
+ rm /tmp/tmp.cWvPIRPIeZ /tmp/tmp.MekG5m93yV
+ return 0
+ diff /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1377/e2e-tests/demand-backup-sharded/compare/find2.json /tmp/tmp.geCbJnr5r4/find2
+ desc 'check backup and restore -- minio'
+ set +o xtrace


-----------------------------------------------------------------------------------
check backup and restore -- minio
-----------------------------------------------------------------------------------

2024-01-27 16:28:49         55 myApp.test.gz
Percona Server for MongoDB shell version v4.4.27-26
connecting to: mongodb://some-name-mongos.demand-backup-sharded-21215.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb
Implicit session: session { "id" : UUID("477064af-9e8b-4848-baf5-114c76279171") }
Percona Server for MongoDB server version: v5.0.23-20
WARNING: shell and server versions do not match
switched to db myApp
WriteResult({ "nInserted" : 1 })
bye
Percona Server for MongoDB shell version v4.4.27-26
connecting to: mongodb://some-name-mongos.demand-backup-sharded-21215.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb
Implicit session: session { "id" : UUID("7b9276b1-6d53-4f73-aada-0a906afd8835") }
Percona Server for MongoDB server version: v5.0.23-20
WARNING: shell and server versions do not match
switched to db myApp1
WriteResult({ "nInserted" : 1 })
bye
Percona Server for MongoDB shell version v4.4.27-26
connecting to: mongodb://some-name-mongos.demand-backup-sharded-21215.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb
Implicit session: session { "id" : UUID("9b617d36-e3b6-4a2c-9c91-78467cf50fa0") }
Percona Server for MongoDB server version: v5.0.23-20
WARNING: shell and server versions do not match
switched to db myApp2
WriteResult({ "nInserted" : 1 })
bye
perconaservermongodbrestore.psmdb.percona.com/restore-backup-minio created
waiting psmdb-restore/backup-minio to reach ready state..........
+ '[' 1 -eq 1 ']'
+ wait_cluster_consistency some-name
+ local cluster_name=some-name
+ local wait_time=32
+ retry=0
+ sleep 7
+ echo -n 'waiting for cluster readyness'
waiting for cluster readyness++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.hOIysk5ycL
+++ mktemp
++ local LAST_ERR=/tmp/tmp.MjTfAqFyex
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 0 ']'
++ break
++ cat /tmp/tmp.hOIysk5ycL
++ cat /tmp/tmp.MjTfAqFyex
++ rm /tmp/tmp.hOIysk5ycL /tmp/tmp.MjTfAqFyex
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 1 -ge 32 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.5XZbfd3e2E
+++ mktemp
++ local LAST_ERR=/tmp/tmp.IzuaRfLQ0Z
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 0 ']'
++ break
++ cat /tmp/tmp.5XZbfd3e2E
++ cat /tmp/tmp.IzuaRfLQ0Z
++ rm /tmp/tmp.5XZbfd3e2E /tmp/tmp.IzuaRfLQ0Z
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 2 -ge 32 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.mGyraVXN41
+++ mktemp
++ local LAST_ERR=/tmp/tmp.ycDV3OI5k4
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 0 ']'
++ break
++ cat /tmp/tmp.mGyraVXN41
++ cat /tmp/tmp.ycDV3OI5k4
++ rm /tmp/tmp.mGyraVXN41 /tmp/tmp.ycDV3OI5k4
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 3 -ge 32 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.wTlkaSs01Y
+++ mktemp
++ local LAST_ERR=/tmp/tmp.J5QymR94mC
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 0 ']'
++ break
++ cat /tmp/tmp.wTlkaSs01Y
++ cat /tmp/tmp.J5QymR94mC
++ rm /tmp/tmp.wTlkaSs01Y /tmp/tmp.J5QymR94mC
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 4 -ge 32 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.3g5VSQuZK2
+++ mktemp
++ local LAST_ERR=/tmp/tmp.ja2LyC3Bj4
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 0 ']'
++ break
++ cat /tmp/tmp.3g5VSQuZK2
++ cat /tmp/tmp.ja2LyC3Bj4
++ rm /tmp/tmp.3g5VSQuZK2 /tmp/tmp.ja2LyC3Bj4
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 5 -ge 32 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.J9IlGBjl20
+++ mktemp
++ local LAST_ERR=/tmp/tmp.kHW2HTBdhO
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 0 ']'
++ break
++ cat /tmp/tmp.J9IlGBjl20
++ cat /tmp/tmp.kHW2HTBdhO
++ rm /tmp/tmp.J9IlGBjl20 /tmp/tmp.kHW2HTBdhO
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 6 -ge 32 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.Gh36XkqyCn
+++ mktemp
++ local LAST_ERR=/tmp/tmp.TIFgmHxjh2
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 0 ']'
++ break
++ cat /tmp/tmp.Gh36XkqyCn
++ cat /tmp/tmp.TIFgmHxjh2
++ rm /tmp/tmp.Gh36XkqyCn /tmp/tmp.TIFgmHxjh2
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 7 -ge 32 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.riAp1sCi1v
+++ mktemp
++ local LAST_ERR=/tmp/tmp.uNX3SmfoHs
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 0 ']'
++ break
++ cat /tmp/tmp.riAp1sCi1v
++ cat /tmp/tmp.uNX3SmfoHs
++ rm /tmp/tmp.riAp1sCi1v /tmp/tmp.uNX3SmfoHs
++ return 0
+ [[ ready == \r\e\a\d\y ]]
+ check_data
+ local postfix=
++ seq 0 2
+ for i in '$(seq 0 2)'
+ compare_mongos_cmd find myApp:myPass@some-name-mongos.demand-backup-sharded-21215 '' .svc.cluster.local myApp test
+ local command=find
+ local uri=myApp:myPass@some-name-mongos.demand-backup-sharded-21215
+ local postfix=
+ local suffix=.svc.cluster.local
+ local database=myApp
+ local collection=test
+ run_mongos 'use myApp\n db.test.find()' myApp:myPass@some-name-mongos.demand-backup-sharded-21215 mongodb .svc.cluster.local
+ local 'command=use myApp\n db.test.find()'
+ local uri=myApp:myPass@some-name-mongos.demand-backup-sharded-21215
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ egrep -v 'I NETWORK|W NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:'
+ /usr/bin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.x0pwmVLQDi
+++ mktemp
++ local LAST_ERR=/tmp/tmp.BqlPC2GYaX
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 0 ']'
++ break
++ cat /tmp/tmp.x0pwmVLQDi
++ cat /tmp/tmp.BqlPC2GYaX
++ rm /tmp/tmp.x0pwmVLQDi /tmp/tmp.BqlPC2GYaX
++ return 0
+ local client_container=psmdb-client-6b9c964f46-h2gfr
+ local mongo_flag=
+ kubectl_bin exec psmdb-client-6b9c964f46-h2gfr -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-21215.svc.cluster.local/admin '
++ mktemp
+ local LAST_OUT=/tmp/tmp.WZkuZOEH2Q
++ mktemp
+ local LAST_ERR=/tmp/tmp.ibI7rFAiwI
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl exec psmdb-client-6b9c964f46-h2gfr -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-21215.svc.cluster.local/admin '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 0 ']'
+ break
+ cat /tmp/tmp.WZkuZOEH2Q
+ cat /tmp/tmp.ibI7rFAiwI
+ rm /tmp/tmp.WZkuZOEH2Q /tmp/tmp.ibI7rFAiwI
+ return 0
+ diff /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1377/e2e-tests/demand-backup-sharded/compare/find.json /tmp/tmp.geCbJnr5r4/find
+ for i in '$(seq 0 2)'
+ compare_mongos_cmd find myApp:myPass@some-name-mongos.demand-backup-sharded-21215 1 .svc.cluster.local myApp1 test
+ local command=find
+ local uri=myApp:myPass@some-name-mongos.demand-backup-sharded-21215
+ local postfix=1
+ local suffix=.svc.cluster.local
+ local database=myApp1
+ local collection=test
+ run_mongos 'use myApp1\n db.test.find()' myApp:myPass@some-name-mongos.demand-backup-sharded-21215 mongodb .svc.cluster.local
+ local 'command=use myApp1\n db.test.find()'
+ local uri=myApp:myPass@some-name-mongos.demand-backup-sharded-21215
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ egrep -v 'I NETWORK|W NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:'
+ /usr/bin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.OX7Ds0Xe0C
+++ mktemp
++ local LAST_ERR=/tmp/tmp.mElk3K8jzn
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 0 ']'
++ break
++ cat /tmp/tmp.OX7Ds0Xe0C
++ cat /tmp/tmp.mElk3K8jzn
++ rm /tmp/tmp.OX7Ds0Xe0C /tmp/tmp.mElk3K8jzn
++ return 0
+ local client_container=psmdb-client-6b9c964f46-h2gfr
+ local mongo_flag=
+ kubectl_bin exec psmdb-client-6b9c964f46-h2gfr -- bash -c 'printf '\''use myApp1\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-21215.svc.cluster.local/admin '
++ mktemp
+ local LAST_OUT=/tmp/tmp.fkMhbD0udk
++ mktemp
+ local LAST_ERR=/tmp/tmp.JnSH8VVphW
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl exec psmdb-client-6b9c964f46-h2gfr -- bash -c 'printf '\''use myApp1\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-21215.svc.cluster.local/admin '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 0 ']'
+ break
+ cat /tmp/tmp.fkMhbD0udk
+ cat /tmp/tmp.JnSH8VVphW
+ rm /tmp/tmp.fkMhbD0udk /tmp/tmp.JnSH8VVphW
+ return 0
+ diff /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1377/e2e-tests/demand-backup-sharded/compare/find1.json /tmp/tmp.geCbJnr5r4/find1
+ for i in '$(seq 0 2)'
+ compare_mongos_cmd find myApp:myPass@some-name-mongos.demand-backup-sharded-21215 2 .svc.cluster.local myApp2 test
+ local command=find
+ local uri=myApp:myPass@some-name-mongos.demand-backup-sharded-21215
+ local postfix=2
+ local suffix=.svc.cluster.local
+ local database=myApp2
+ local collection=test
+ run_mongos 'use myApp2\n db.test.find()' myApp:myPass@some-name-mongos.demand-backup-sharded-21215 mongodb .svc.cluster.local
+ local 'command=use myApp2\n db.test.find()'
+ local uri=myApp:myPass@some-name-mongos.demand-backup-sharded-21215
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ egrep -v 'I NETWORK|W NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:'
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
+ /usr/bin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
++ local LAST_OUT=/tmp/tmp.XX3ZzMteVz
+++ mktemp
++ local LAST_ERR=/tmp/tmp.KwxldoF62J
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 0 ']'
++ break
++ cat /tmp/tmp.XX3ZzMteVz
++ cat /tmp/tmp.KwxldoF62J
++ rm /tmp/tmp.XX3ZzMteVz /tmp/tmp.KwxldoF62J
++ return 0
+ local client_container=psmdb-client-6b9c964f46-h2gfr
+ local mongo_flag=
+ kubectl_bin exec psmdb-client-6b9c964f46-h2gfr -- bash -c 'printf '\''use myApp2\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-21215.svc.cluster.local/admin '
++ mktemp
+ local LAST_OUT=/tmp/tmp.P7dqeRLXSw
++ mktemp
+ local LAST_ERR=/tmp/tmp.ZvQNQcnjuW
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl exec psmdb-client-6b9c964f46-h2gfr -- bash -c 'printf '\''use myApp2\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-21215.svc.cluster.local/admin '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 0 ']'
+ break
+ cat /tmp/tmp.P7dqeRLXSw
+ cat /tmp/tmp.ZvQNQcnjuW
+ rm /tmp/tmp.P7dqeRLXSw /tmp/tmp.ZvQNQcnjuW
+ return 0
+ diff /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1377/e2e-tests/demand-backup-sharded/compare/find2.json /tmp/tmp.geCbJnr5r4/find2
+ desc 'delete backup and check if it is removed from bucket -- minio'
+ set +o xtrace


-----------------------------------------------------------------------------------
delete backup and check if it is removed from bucket -- minio
-----------------------------------------------------------------------------------

perconaservermongodbbackup.psmdb.percona.com "backup-aws-s3" deleted
perconaservermongodbbackup.psmdb.percona.com "backup-azure-blob" deleted
perconaservermongodbbackup.psmdb.percona.com "backup-gcp-cs" deleted
perconaservermongodbbackup.psmdb.percona.com "backup-minio" deleted
If you don't see a command prompt, try pressing enter.
warning: couldn't attach to pod/aws-cli, falling back to streaming logs: Internal error occurred: error attaching to container: container is in CONTAINER_EXITED state


-----------------------------------------------------------------------------------
check for passwords leak
-----------------------------------------------------------------------------------

secrets=YmFja3VwMTIzNDU2 Y2x1c3RlckFkbWluMTIzNDU2 Y2x1c3Rlck1vbml0b3IxMjM0NTY= ZGF0YWJhc2VBZG1pbjEyMzQ1Ng== dXNlckFkbWluMTIzNDU2 YmFja3VwMTIzNDU2 Y2x1c3RlckFkbWluMTIzNDU2 Y2x1c3Rlck1vbml0b3IxMjM0NTY= ZGF0YWJhc2VBZG1pbjEyMzQ1Ng== dXNlckFkbWluMTIzNDU2
passwords=backup123456 clusterAdmin123456 clusterMonitor123456 databaseAdmin123456 userAdmin123456 backup123456 clusterAdmin123456 clusterMonitor123456 databaseAdmin123456 userAdmin123456 YmFja3VwMTIzNDU2 Y2x1c3RlckFkbWluMTIzNDU2 Y2x1c3Rlck1vbml0b3IxMjM0NTY= ZGF0YWJhc2VBZG1pbjEyMzQ1Ng== dXNlckFkbWluMTIzNDU2 YmFja3VwMTIzNDU2 Y2x1c3RlckFkbWluMTIzNDU2 Y2x1c3Rlck1vbml0b3IxMjM0NTY= ZGF0YWJhc2VBZG1pbjEyMzQ1Ng== dXNlckFkbWluMTIzNDU2
pods=minio-service-75dd45bdcd-snvkg psmdb-client-6b9c964f46-h2gfr some-name-cfg-0 some-name-cfg-1 some-name-cfg-2 some-name-mongos-0 some-name-mongos-1 some-name-mongos-2 some-name-rs0-0 some-name-rs0-1 some-name-rs0-2 some-name-rs1-0 some-name-rs1-1 some-name-rs1-2 some-name-rs2-0 some-name-rs2-1 some-name-rs2-2
logs saved in: /tmp/tmp.geCbJnr5r4/logs_output-minio-service-75dd45bdcd-snvkg-minio.txt

logs saved in: /tmp/tmp.geCbJnr5r4/logs_output-psmdb-client-6b9c964f46-h2gfr-psmdb-client.txt

logs saved in: /tmp/tmp.geCbJnr5r4/logs_output-some-name-cfg-0-mongod.txt
logs saved in: /tmp/tmp.geCbJnr5r4/logs_output-some-name-cfg-0-cfg-sidecar-1.txt
logs saved in: /tmp/tmp.geCbJnr5r4/logs_output-some-name-cfg-0-backup-agent.txt

logs saved in: /tmp/tmp.geCbJnr5r4/logs_output-some-name-cfg-1-mongod.txt
logs saved in: /tmp/tmp.geCbJnr5r4/logs_output-some-name-cfg-1-cfg-sidecar-1.txt
logs saved in: /tmp/tmp.geCbJnr5r4/logs_output-some-name-cfg-1-backup-agent.txt

logs saved in: /tmp/tmp.geCbJnr5r4/logs_output-some-name-cfg-2-mongod.txt
logs saved in: /tmp/tmp.geCbJnr5r4/logs_output-some-name-cfg-2-cfg-sidecar-1.txt
logs saved in: /tmp/tmp.geCbJnr5r4/logs_output-some-name-cfg-2-backup-agent.txt

logs saved in: /tmp/tmp.geCbJnr5r4/logs_output-some-name-mongos-0-mongos.txt
logs saved in: /tmp/tmp.geCbJnr5r4/logs_output-some-name-mongos-0-mongos-sidecar-1.txt

logs saved in: /tmp/tmp.geCbJnr5r4/logs_output-some-name-mongos-1-mongos.txt
logs saved in: /tmp/tmp.geCbJnr5r4/logs_output-some-name-mongos-1-mongos-sidecar-1.txt

logs saved in: /tmp/tmp.geCbJnr5r4/logs_output-some-name-mongos-2-mongos.txt
logs saved in: /tmp/tmp.geCbJnr5r4/logs_output-some-name-mongos-2-mongos-sidecar-1.txt

logs saved in: /tmp/tmp.geCbJnr5r4/logs_output-some-name-rs0-0-mongod.txt
logs saved in: /tmp/tmp.geCbJnr5r4/logs_output-some-name-rs0-0-backup-agent.txt

logs saved in: /tmp/tmp.geCbJnr5r4/logs_output-some-name-rs0-1-mongod.txt
logs saved in: /tmp/tmp.geCbJnr5r4/logs_output-some-name-rs0-1-backup-agent.txt

logs saved in: /tmp/tmp.geCbJnr5r4/logs_output-some-name-rs0-2-mongod.txt
logs saved in: /tmp/tmp.geCbJnr5r4/logs_output-some-name-rs0-2-backup-agent.txt

logs saved in: /tmp/tmp.geCbJnr5r4/logs_output-some-name-rs1-0-mongod.txt
logs saved in: /tmp/tmp.geCbJnr5r4/logs_output-some-name-rs1-0-backup-agent.txt

logs saved in: /tmp/tmp.geCbJnr5r4/logs_output-some-name-rs1-1-mongod.txt
logs saved in: /tmp/tmp.geCbJnr5r4/logs_output-some-name-rs1-1-backup-agent.txt

logs saved in: /tmp/tmp.geCbJnr5r4/logs_output-some-name-rs1-2-mongod.txt
logs saved in: /tmp/tmp.geCbJnr5r4/logs_output-some-name-rs1-2-backup-agent.txt

logs saved in: /tmp/tmp.geCbJnr5r4/logs_output-some-name-rs2-0-mongod.txt
logs saved in: /tmp/tmp.geCbJnr5r4/logs_output-some-name-rs2-0-rs-sidecar-1.txt
logs saved in: /tmp/tmp.geCbJnr5r4/logs_output-some-name-rs2-0-backup-agent.txt

logs saved in: /tmp/tmp.geCbJnr5r4/logs_output-some-name-rs2-1-mongod.txt
logs saved in: /tmp/tmp.geCbJnr5r4/logs_output-some-name-rs2-1-rs-sidecar-1.txt
logs saved in: /tmp/tmp.geCbJnr5r4/logs_output-some-name-rs2-1-backup-agent.txt

logs saved in: /tmp/tmp.geCbJnr5r4/logs_output-some-name-rs2-2-mongod.txt
logs saved in: /tmp/tmp.geCbJnr5r4/logs_output-some-name-rs2-2-rs-sidecar-1.txt
logs saved in: /tmp/tmp.geCbJnr5r4/logs_output-some-name-rs2-2-backup-agent.txt

logs saved in: /tmp/tmp.geCbJnr5r4/logs_output-percona-server-mongodb-operator-d6f9576f6-dbvch-percona-server-mongodb-operator.txt

runtimeclass.node.k8s.io "container-rc" deleted


-----------------------------------------------------------------------------------
destroy cluster/operator and all other resources
-----------------------------------------------------------------------------------



-----------------------------------------------------------------------------------
get and delete old CRDs and RBAC
-----------------------------------------------------------------------------------

customresourcedefinition.apiextensions.k8s.io "perconaservermongodbbackups.psmdb.percona.com" deleted
customresourcedefinition.apiextensions.k8s.io "perconaservermongodbrestores.psmdb.percona.com" deleted
customresourcedefinition.apiextensions.k8s.io "perconaservermongodbs.psmdb.percona.com" deleted
error: the server doesn't have a resource type "perconaservermongodbbackups"
+ kubectl patch perconaservermongodbbackups.psmdb.percona.com -n sh --type=merge -p '{"metadata":{"finalizers":[]}}'
error: the server doesn't have a resource type "perconaservermongodbbackups"
error: the server doesn't have a resource type "perconaservermongodbrestores"
+ kubectl patch perconaservermongodbrestores.psmdb.percona.com -n sh --type=merge -p '{"metadata":{"finalizers":[]}}'
error: the server doesn't have a resource type "perconaservermongodbrestores"
error: the server doesn't have a resource type "perconaservermongodbs"
+ kubectl patch perconaservermongodbs.psmdb.percona.com -n sh --type=merge -p '{"metadata":{"finalizers":[]}}'
error: the server doesn't have a resource type "perconaservermongodbs"
clusterrole.rbac.authorization.k8s.io "percona-server-mongodb-operator" deleted
clusterrolebinding.rbac.authorization.k8s.io "service-account-percona-server-mongodb-operator" deleted