++ echo 'Log: /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1543/e2e-tests/logs/demand-backup-sharded.log'
Log: /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1543/e2e-tests/logs/demand-backup-sharded.log
++ '[' -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1543/e2e-tests/conf/cloud-secret.yml ']'
++ SKIP_BACKUPS_TO_AWS_GCP_AZURE=
++ oc get projects
++ kubectl get nodes
++ grep '^minikube'
+++ kubectl version -o json
+++ jq -r .serverVersion.gitVersion
+++ grep '\-eks\-'
WARNING: version difference between client (1.30) and server (1.26) exceeds the supported minor version skew of +/-1
++ '[' ']'
++ EKS=0
+++ kubectl version -o json
+++ grep gke
+++ jq -r .serverVersion.gitVersion
WARNING: version difference between client (1.30) and server (1.26) exceeds the supported minor version skew of +/-1
++ '[' v1.26.15-gke.1243000 ']'
++ GKE=1
+++ kubectl version -o json
+++ jq -r '.serverVersion.major + "." + .serverVersion.minor'
+++ /usr/bin/sed -r 's/[^0-9.]+//g'
WARNING: version difference between client (1.30) and server (1.26) exceeds the supported minor version skew of +/-1
++ KUBE_VERSION=1.26
+ set_debug
+ [[ 0 == 1 ]]
+ set +o xtrace


-----------------------------------------------------------------------------------
get and delete old CRDs and RBAC
-----------------------------------------------------------------------------------

error: the server doesn't have a resource type "perconaservermongodbbackups"
+ kubectl patch perconaservermongodbbackups.psmdb.percona.com -n sh --type=merge -p '{"metadata":{"finalizers":[]}}'
error: the server doesn't have a resource type "perconaservermongodbbackups"
error: the server doesn't have a resource type "perconaservermongodbrestores"
+ kubectl patch perconaservermongodbrestores.psmdb.percona.com -n sh --type=merge -p '{"metadata":{"finalizers":[]}}'
error: the server doesn't have a resource type "perconaservermongodbrestores"
error: the server doesn't have a resource type "perconaservermongodbs"
+ kubectl patch perconaservermongodbs.psmdb.percona.com -n sh --type=merge -p '{"metadata":{"finalizers":[]}}'
error: the server doesn't have a resource type "perconaservermongodbs"
Error from server (NotFound): customresourcedefinitions.apiextensions.k8s.io "null" not found
Error from server (NotFound): customresourcedefinitions.apiextensions.k8s.io "null" not found
Error from server (NotFound): customresourcedefinitions.apiextensions.k8s.io "null" not found
Error from server (NotFound): customresourcedefinitions.apiextensions.k8s.io "null" not found


-----------------------------------------------------------------------------------
destroy chaos-mesh
-----------------------------------------------------------------------------------

error: resource(s) were provided, but no name was specified
error: resource(s) were provided, but no name was specified
error: resource(s) were provided, but no name was specified
error: resource(s) were provided, but no name was specified
error: resource(s) were provided, but no name was specified
error: resource(s) were provided, but no name was specified


-----------------------------------------------------------------------------------
cleaned up all old namespaces
-----------------------------------------------------------------------------------



-----------------------------------------------------------------------------------
cleaned up old namespaces psmdb-operator
-----------------------------------------------------------------------------------

error: resource(s) were provided, but no name was specified


-----------------------------------------------------------------------------------
create namespace psmdb-operator
-----------------------------------------------------------------------------------

namespace/psmdb-operator created
Context "gke_cloud-dev-112233_us-central1-a_jen-psmdb-1543-c8a64a78-9-cluster2" modified.


-----------------------------------------------------------------------------------
start PSMDB operator
-----------------------------------------------------------------------------------

customresourcedefinition.apiextensions.k8s.io/perconaservermongodbbackups.psmdb.percona.com serverside-applied
customresourcedefinition.apiextensions.k8s.io/perconaservermongodbrestores.psmdb.percona.com serverside-applied
customresourcedefinition.apiextensions.k8s.io/perconaservermongodbs.psmdb.percona.com serverside-applied
clusterrole.rbac.authorization.k8s.io/percona-server-mongodb-operator created
serviceaccount/percona-server-mongodb-operator created
clusterrolebinding.rbac.authorization.k8s.io/service-account-percona-server-mongodb-operator created
deployment.apps/percona-server-mongodb-operator created
waiting for pod/percona-server-mongodb-operator-fdfdd955d-5m8kh to be ready.OK


-----------------------------------------------------------------------------------
destroy chaos-mesh
-----------------------------------------------------------------------------------

error: resource(s) were provided, but no name was specified
error: resource(s) were provided, but no name was specified
error: resource(s) were provided, but no name was specified
error: resource(s) were provided, but no name was specified
error: resource(s) were provided, but no name was specified
error: resource(s) were provided, but no name was specified


-----------------------------------------------------------------------------------
cleaned up all old namespaces
-----------------------------------------------------------------------------------



-----------------------------------------------------------------------------------
cleaned up old namespaces demand-backup-sharded-1695
-----------------------------------------------------------------------------------

error: resource(s) were provided, but no name was specified


-----------------------------------------------------------------------------------
create namespace demand-backup-sharded-1695
-----------------------------------------------------------------------------------

namespace/demand-backup-sharded-1695 created
Context "gke_cloud-dev-112233_us-central1-a_jen-psmdb-1543-c8a64a78-9-cluster2" modified.


-----------------------------------------------------------------------------------
install Minio
-----------------------------------------------------------------------------------

Error: uninstall: Release not loaded: minio-service: release: not found
"minio" has been removed from your repositories
"minio" has been added to your repositories
NAME: minio-service
LAST DEPLOYED: Tue May  7 11:06:13 2024
NAMESPACE: demand-backup-sharded-1695
STATUS: deployed
REVISION: 1
TEST SUITE: None
NOTES:
MinIO can be accessed via port 9000 on the following DNS name from within your cluster:
minio-service.demand-backup-sharded-1695.svc.cluster.local

To access MinIO from localhost, run the below commands:

  1. export POD_NAME=$(kubectl get pods --namespace demand-backup-sharded-1695 -l "release=minio-service" -o jsonpath="{.items[0].metadata.name}")

  2. kubectl port-forward $POD_NAME 9000 --namespace demand-backup-sharded-1695

Read more about port forwarding here: http://kubernetes.io/docs/user-guide/kubectl/kubectl_port-forward/

You can now access MinIO server on http://localhost:9000. Follow the below steps to connect to MinIO server with mc client:

  1. Download the MinIO mc client - https://min.io/docs/minio/linux/reference/minio-mc.html#quickstart

  2. export MC_HOST_minio-service-local=http://$(kubectl get secret --namespace demand-backup-sharded-1695 minio-service -o jsonpath="{.data.rootUser}" | base64 --decode):$(kubectl get secret --namespace demand-backup-sharded-1695 minio-service -o jsonpath="{.data.rootPassword}" | base64 --decode)@localhost:9000

  3. mc ls minio-service-local
waiting for pod/minio-service-57dd49b-lrchv to be ready.OK
service/minio-service created
pod "aws-cli" deleted
If you don't see a command prompt, try pressing enter.


-----------------------------------------------------------------------------------
create first PSMDB cluster
-----------------------------------------------------------------------------------

secret/some-users created
deployment.apps/psmdb-client created


-----------------------------------------------------------------------------------
create secrets for cloud storages
-----------------------------------------------------------------------------------

secret/minio-secret created
secret/aws-s3-secret created
secret/gcp-cs-secret created
secret/azure-secret created
runtimeclass.node.k8s.io/container-rc unchanged
perconaservermongodb.psmdb.percona.com/some-name created


-----------------------------------------------------------------------------------
check if all 3 Pods started
-----------------------------------------------------------------------------------

waiting for pod/some-name-rs0-0 to be ready..........OK
waiting for pod/some-name-rs0-1 to be ready..........OK
waiting for pod/some-name-rs0-2 to be ready.......OK
Waiting for cluster readyness.......................................
waiting for pod/some-name-cfg-0 to be ready.OK
waiting for pod/some-name-cfg-1 to be ready.OK
waiting for pod/some-name-cfg-2 to be ready.OK
waiting for pod/some-name-mongos-0 to be ready.OK
waiting for pod/some-name-mongos-1 to be ready.OK
waiting for pod/some-name-mongos-2 to be ready.OK
Waiting for cluster readyness


-----------------------------------------------------------------------------------
check if service and statefulset created with expected config
-----------------------------------------------------------------------------------

secret/some-name-mongos created
waiting for pod/some-name-mongos-0 to be ready.OK
waiting for pod/some-name-mongos-1 to be ready.OK
waiting for pod/some-name-mongos-2 to be ready.OK
Waiting for cluster readyness...............................................................................................


-----------------------------------------------------------------------------------
write data, read from all
-----------------------------------------------------------------------------------

Percona Server for MongoDB shell version v4.4.29-28
connecting to: mongodb://some-name-mongos.demand-backup-sharded-1695.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb
Implicit session: session { "id" : UUID("abece51b-f53f-442e-8f86-f63584cc4123") }
Percona Server for MongoDB server version: v7.0.8-5
WARNING: shell and server versions do not match
Successfully added user: {
	"user" : "myApp",
	"roles" : [
		{
			"db" : "myApp",
			"role" : "readWrite"
		},
		{
			"db" : "myApp1",
			"role" : "readWrite"
		},
		{
			"db" : "myApp2",
			"role" : "readWrite"
		}
	]
}
bye
Percona Server for MongoDB shell version v4.4.29-28
connecting to: mongodb://some-name-mongos.demand-backup-sharded-1695.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb
Implicit session: session { "id" : UUID("d7d1cffc-bd4a-4feb-ae4c-1b92ae5ee161") }
Percona Server for MongoDB server version: v7.0.8-5
WARNING: shell and server versions do not match
{
	"ok" : 1,
	"$clusterTime" : {
		"clusterTime" : Timestamp(1715080509, 12),
		"signature" : {
			"hash" : BinData(0,"Xd7etOnNHozKjTMCkJqHmceQgBQ="),
			"keyId" : NumberLong("7366212926635507735")
		}
	},
	"operationTime" : Timestamp(1715080509, 6)
}
bye
Percona Server for MongoDB shell version v4.4.29-28
connecting to: mongodb://some-name-mongos.demand-backup-sharded-1695.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb
Implicit session: session { "id" : UUID("ae304a60-4cc1-475a-bbd3-54d5d5194060") }
Percona Server for MongoDB server version: v7.0.8-5
WARNING: shell and server versions do not match
{
	"ok" : 1,
	"$clusterTime" : {
		"clusterTime" : Timestamp(1715080513, 7),
		"signature" : {
			"hash" : BinData(0,"jae/mqfsBPmhPduWiquwf9bS+y4="),
			"keyId" : NumberLong("7366212926635507735")
		}
	},
	"operationTime" : Timestamp(1715080513, 2)
}
bye
Percona Server for MongoDB shell version v4.4.29-28
connecting to: mongodb://some-name-mongos.demand-backup-sharded-1695.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb
Implicit session: session { "id" : UUID("127ca9b2-6140-454f-9bd7-55004a5d4809") }
Percona Server for MongoDB server version: v7.0.8-5
WARNING: shell and server versions do not match
{
	"ok" : 1,
	"$clusterTime" : {
		"clusterTime" : Timestamp(1715080517, 7),
		"signature" : {
			"hash" : BinData(0,"Hjbm4GTEuH+JAlBK1d9fAiD6+IU="),
			"keyId" : NumberLong("7366212926635507735")
		}
	},
	"operationTime" : Timestamp(1715080517, 2)
}
bye
Percona Server for MongoDB shell version v4.4.29-28
connecting to: mongodb://some-name-mongos.demand-backup-sharded-1695.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb
Implicit session: session { "id" : UUID("a336d809-bf51-45f9-b234-ede2f33f3d05") }
Percona Server for MongoDB server version: v7.0.8-5
WARNING: shell and server versions do not match
switched to db myApp
WriteResult({ "nInserted" : 1 })
bye
Percona Server for MongoDB shell version v4.4.29-28
connecting to: mongodb://some-name-mongos.demand-backup-sharded-1695.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb
Implicit session: session { "id" : UUID("0fbee3af-9b50-4965-994f-04c5e147123c") }
Percona Server for MongoDB server version: v7.0.8-5
WARNING: shell and server versions do not match
switched to db myApp1
WriteResult({ "nInserted" : 1 })
bye
Percona Server for MongoDB shell version v4.4.29-28
connecting to: mongodb://some-name-mongos.demand-backup-sharded-1695.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb
Implicit session: session { "id" : UUID("24f3dd23-0fd4-4c25-9cea-fbd80677642c") }
Percona Server for MongoDB server version: v7.0.8-5
WARNING: shell and server versions do not match
switched to db myApp2
WriteResult({ "nInserted" : 1 })
bye
some-name-rs0-0
some-name-rs0-1
some-name-rs0-2
some-name-rs1-0
some-name-rs1-1
some-name-rs1-2
some-name-rs2-0
some-name-rs2-1
some-name-rs2-2


-----------------------------------------------------------------------------------
run backups
-----------------------------------------------------------------------------------



-----------------------------------------------------------------------------------
run backup backup-minio
-----------------------------------------------------------------------------------

perconaservermongodbbackup.psmdb.percona.com/backup-minio created


-----------------------------------------------------------------------------------
run backup backup-aws-s3
-----------------------------------------------------------------------------------

perconaservermongodbbackup.psmdb.percona.com/backup-aws-s3 created


-----------------------------------------------------------------------------------
run backup backup-gcp-cs
-----------------------------------------------------------------------------------

perconaservermongodbbackup.psmdb.percona.com/backup-gcp-cs created


-----------------------------------------------------------------------------------
run backup backup-azure-blob
-----------------------------------------------------------------------------------

perconaservermongodbbackup.psmdb.percona.com/backup-azure-blob created
backup-aws-s3...........................................
backup-gcp-cs...............
backup-azure-blob......................................
backup-minio.


-----------------------------------------------------------------------------------
check backup and restore -- aws-s3
-----------------------------------------------------------------------------------

Percona Server for MongoDB shell version v4.4.29-28
connecting to: mongodb://some-name-mongos.demand-backup-sharded-1695.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb
Implicit session: session { "id" : UUID("452d0e81-946e-4882-bd04-bd1d25ad22ba") }
Percona Server for MongoDB server version: v7.0.8-5
WARNING: shell and server versions do not match
switched to db myApp
WriteResult({ "nInserted" : 1 })
bye
Percona Server for MongoDB shell version v4.4.29-28
connecting to: mongodb://some-name-mongos.demand-backup-sharded-1695.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb
Implicit session: session { "id" : UUID("15e47c57-badd-48c4-8a84-48c983a1cdf3") }
Percona Server for MongoDB server version: v7.0.8-5
WARNING: shell and server versions do not match
switched to db myApp1
WriteResult({ "nInserted" : 1 })
bye
Percona Server for MongoDB shell version v4.4.29-28
connecting to: mongodb://some-name-mongos.demand-backup-sharded-1695.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb
Implicit session: session { "id" : UUID("8623ea6d-4a10-4696-8ca2-8d1a05089a79") }
Percona Server for MongoDB server version: v7.0.8-5
WARNING: shell and server versions do not match
switched to db myApp2
WriteResult({ "nInserted" : 1 })
bye
perconaservermongodbrestore.psmdb.percona.com/restore-backup-aws-s3 created
waiting psmdb-restore/backup-aws-s3 to reach ready state.............
+ '[' 1 -eq 1 ']'
+ wait_cluster_consistency some-name
+ local cluster_name=some-name
+ local wait_time=32
+ retry=0
+ sleep 7
+ echo -n 'waiting for cluster readyness'
waiting for cluster readyness++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.h875jgYOXc
+++ mktemp
++ local LAST_ERR=/tmp/tmp.SGeCylhkxU
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 0 ']'
++ break
++ cat /tmp/tmp.h875jgYOXc
++ cat /tmp/tmp.SGeCylhkxU
++ rm /tmp/tmp.h875jgYOXc /tmp/tmp.SGeCylhkxU
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 1 -ge 32 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.swlUueR9eo
+++ mktemp
++ local LAST_ERR=/tmp/tmp.b07ZL6agNb
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 0 ']'
++ break
++ cat /tmp/tmp.swlUueR9eo
++ cat /tmp/tmp.b07ZL6agNb
++ rm /tmp/tmp.swlUueR9eo /tmp/tmp.b07ZL6agNb
++ return 0
+ [[ error == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 2 -ge 32 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.f6ZvKv1rFH
+++ mktemp
++ local LAST_ERR=/tmp/tmp.Td43bXIYyc
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 0 ']'
++ break
++ cat /tmp/tmp.f6ZvKv1rFH
++ cat /tmp/tmp.Td43bXIYyc
++ rm /tmp/tmp.f6ZvKv1rFH /tmp/tmp.Td43bXIYyc
++ return 0
+ [[ error == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 3 -ge 32 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.UvBOcNzkDy
+++ mktemp
++ local LAST_ERR=/tmp/tmp.sUeAGUi7p4
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 0 ']'
++ break
++ cat /tmp/tmp.UvBOcNzkDy
++ cat /tmp/tmp.sUeAGUi7p4
++ rm /tmp/tmp.UvBOcNzkDy /tmp/tmp.sUeAGUi7p4
++ return 0
+ [[ error == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 4 -ge 32 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.cWr4hMPb7O
+++ mktemp
++ local LAST_ERR=/tmp/tmp.7CfJyjWXwi
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 0 ']'
++ break
++ cat /tmp/tmp.cWr4hMPb7O
++ cat /tmp/tmp.7CfJyjWXwi
++ rm /tmp/tmp.cWr4hMPb7O /tmp/tmp.7CfJyjWXwi
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 5 -ge 32 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.vOOC24RzHF
+++ mktemp
++ local LAST_ERR=/tmp/tmp.ELZXsZ7LEK
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 0 ']'
++ break
++ cat /tmp/tmp.vOOC24RzHF
++ cat /tmp/tmp.ELZXsZ7LEK
++ rm /tmp/tmp.vOOC24RzHF /tmp/tmp.ELZXsZ7LEK
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 6 -ge 32 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.KgVDZLYTJ8
+++ mktemp
++ local LAST_ERR=/tmp/tmp.szEnSXRunT
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 0 ']'
++ break
++ cat /tmp/tmp.KgVDZLYTJ8
++ cat /tmp/tmp.szEnSXRunT
++ rm /tmp/tmp.KgVDZLYTJ8 /tmp/tmp.szEnSXRunT
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 7 -ge 32 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.tcqNDHyl0q
+++ mktemp
++ local LAST_ERR=/tmp/tmp.1HQuYISjqf
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 0 ']'
++ break
++ cat /tmp/tmp.tcqNDHyl0q
++ cat /tmp/tmp.1HQuYISjqf
++ rm /tmp/tmp.tcqNDHyl0q /tmp/tmp.1HQuYISjqf
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 8 -ge 32 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.Xuu9QTpaT2
+++ mktemp
++ local LAST_ERR=/tmp/tmp.mk4Y5JqPnd
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 0 ']'
++ break
++ cat /tmp/tmp.Xuu9QTpaT2
++ cat /tmp/tmp.mk4Y5JqPnd
++ rm /tmp/tmp.Xuu9QTpaT2 /tmp/tmp.mk4Y5JqPnd
++ return 0
+ [[ ready == \r\e\a\d\y ]]
+ check_data
+ local postfix=
++ seq 0 2
+ for i in '$(seq 0 2)'
+ compare_mongos_cmd find myApp:myPass@some-name-mongos.demand-backup-sharded-1695 '' .svc.cluster.local myApp test
+ local command=find
+ local uri=myApp:myPass@some-name-mongos.demand-backup-sharded-1695
+ local postfix=
+ local suffix=.svc.cluster.local
+ local database=myApp
+ local collection=test
+ run_mongos 'use myApp\n db.test.find()' myApp:myPass@some-name-mongos.demand-backup-sharded-1695 mongodb .svc.cluster.local
+ local 'command=use myApp\n db.test.find()'
+ local uri=myApp:myPass@some-name-mongos.demand-backup-sharded-1695
+ local driver=mongodb
+ local suffix=.svc.cluster.local
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
+ /usr/bin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
+ egrep -v 'I NETWORK|W NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:'
++ local LAST_OUT=/tmp/tmp.q2UGz2Eb0H
+++ mktemp
++ local LAST_ERR=/tmp/tmp.aFnkbT9R4z
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 0 ']'
++ break
++ cat /tmp/tmp.q2UGz2Eb0H
++ cat /tmp/tmp.aFnkbT9R4z
++ rm /tmp/tmp.q2UGz2Eb0H /tmp/tmp.aFnkbT9R4z
++ return 0
+ local client_container=psmdb-client-7469665986-brptm
+ local mongo_flag=
+ kubectl_bin exec psmdb-client-7469665986-brptm -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-1695.svc.cluster.local/admin '
++ mktemp
+ local LAST_OUT=/tmp/tmp.Gg2h3pgXiO
++ mktemp
+ local LAST_ERR=/tmp/tmp.OADhK3z6hR
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl exec psmdb-client-7469665986-brptm -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-1695.svc.cluster.local/admin '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 0 ']'
+ break
+ cat /tmp/tmp.Gg2h3pgXiO
+ cat /tmp/tmp.OADhK3z6hR
+ rm /tmp/tmp.Gg2h3pgXiO /tmp/tmp.OADhK3z6hR
+ return 0
+ diff /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1543/e2e-tests/demand-backup-sharded/compare/find.json /tmp/tmp.VmM91SP9mO/find
+ for i in '$(seq 0 2)'
+ compare_mongos_cmd find myApp:myPass@some-name-mongos.demand-backup-sharded-1695 1 .svc.cluster.local myApp1 test
+ local command=find
+ local uri=myApp:myPass@some-name-mongos.demand-backup-sharded-1695
+ local postfix=1
+ local suffix=.svc.cluster.local
+ local database=myApp1
+ local collection=test
+ run_mongos 'use myApp1\n db.test.find()' myApp:myPass@some-name-mongos.demand-backup-sharded-1695 mongodb .svc.cluster.local
+ local 'command=use myApp1\n db.test.find()'
+ /usr/bin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
+ local uri=myApp:myPass@some-name-mongos.demand-backup-sharded-1695
+ local driver=mongodb
+ local suffix=.svc.cluster.local
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.ct0QPUzwhj
+++ mktemp
++ local LAST_ERR=/tmp/tmp.Fr6jFZTrdW
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+ egrep -v 'I NETWORK|W NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 0 ']'
++ break
++ cat /tmp/tmp.ct0QPUzwhj
++ cat /tmp/tmp.Fr6jFZTrdW
++ rm /tmp/tmp.ct0QPUzwhj /tmp/tmp.Fr6jFZTrdW
++ return 0
+ local client_container=psmdb-client-7469665986-brptm
+ local mongo_flag=
+ kubectl_bin exec psmdb-client-7469665986-brptm -- bash -c 'printf '\''use myApp1\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-1695.svc.cluster.local/admin '
++ mktemp
+ local LAST_OUT=/tmp/tmp.ZkcH33FzLv
++ mktemp
+ local LAST_ERR=/tmp/tmp.Ym3N8Z4yb2
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl exec psmdb-client-7469665986-brptm -- bash -c 'printf '\''use myApp1\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-1695.svc.cluster.local/admin '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 0 ']'
+ break
+ cat /tmp/tmp.ZkcH33FzLv
+ cat /tmp/tmp.Ym3N8Z4yb2
+ rm /tmp/tmp.ZkcH33FzLv /tmp/tmp.Ym3N8Z4yb2
+ return 0
+ diff /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1543/e2e-tests/demand-backup-sharded/compare/find1.json /tmp/tmp.VmM91SP9mO/find1
+ for i in '$(seq 0 2)'
+ compare_mongos_cmd find myApp:myPass@some-name-mongos.demand-backup-sharded-1695 2 .svc.cluster.local myApp2 test
+ local command=find
+ local uri=myApp:myPass@some-name-mongos.demand-backup-sharded-1695
+ local postfix=2
+ local suffix=.svc.cluster.local
+ local database=myApp2
+ local collection=test
+ egrep -v 'I NETWORK|W NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:'
+ /usr/bin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
+ run_mongos 'use myApp2\n db.test.find()' myApp:myPass@some-name-mongos.demand-backup-sharded-1695 mongodb .svc.cluster.local
+ local 'command=use myApp2\n db.test.find()'
+ local uri=myApp:myPass@some-name-mongos.demand-backup-sharded-1695
+ local driver=mongodb
+ local suffix=.svc.cluster.local
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.zZO1N3Bx1j
+++ mktemp
++ local LAST_ERR=/tmp/tmp.mpa2elnVT6
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 0 ']'
++ break
++ cat /tmp/tmp.zZO1N3Bx1j
++ cat /tmp/tmp.mpa2elnVT6
++ rm /tmp/tmp.zZO1N3Bx1j /tmp/tmp.mpa2elnVT6
++ return 0
+ local client_container=psmdb-client-7469665986-brptm
+ local mongo_flag=
+ kubectl_bin exec psmdb-client-7469665986-brptm -- bash -c 'printf '\''use myApp2\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-1695.svc.cluster.local/admin '
++ mktemp
+ local LAST_OUT=/tmp/tmp.qguZOgUmsB
++ mktemp
+ local LAST_ERR=/tmp/tmp.mV3lUHFO0H
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl exec psmdb-client-7469665986-brptm -- bash -c 'printf '\''use myApp2\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-1695.svc.cluster.local/admin '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 0 ']'
+ break
+ cat /tmp/tmp.qguZOgUmsB
+ cat /tmp/tmp.mV3lUHFO0H
+ rm /tmp/tmp.qguZOgUmsB /tmp/tmp.mV3lUHFO0H
+ return 0
+ diff /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1543/e2e-tests/demand-backup-sharded/compare/find2.json /tmp/tmp.VmM91SP9mO/find2
+ desc 'check backup and restore -- gcp-cs'
+ set +o xtrace


-----------------------------------------------------------------------------------
check backup and restore -- gcp-cs
-----------------------------------------------------------------------------------

Percona Server for MongoDB shell version v4.4.29-28
connecting to: mongodb://some-name-mongos.demand-backup-sharded-1695.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb
Implicit session: session { "id" : UUID("d577c6ce-5f0a-43ce-9a34-473de8c4a43d") }
Percona Server for MongoDB server version: v7.0.8-5
WARNING: shell and server versions do not match
switched to db myApp
WriteResult({ "nInserted" : 1 })
bye
Percona Server for MongoDB shell version v4.4.29-28
connecting to: mongodb://some-name-mongos.demand-backup-sharded-1695.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb
Implicit session: session { "id" : UUID("4455f8c2-5789-4523-9933-00c7a3cae85f") }
Percona Server for MongoDB server version: v7.0.8-5
WARNING: shell and server versions do not match
switched to db myApp1
WriteResult({ "nInserted" : 1 })
bye
Percona Server for MongoDB shell version v4.4.29-28
connecting to: mongodb://some-name-mongos.demand-backup-sharded-1695.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb
Implicit session: session { "id" : UUID("73190d8b-e22e-4c14-b221-2ce0ffb30635") }
Percona Server for MongoDB server version: v7.0.8-5
WARNING: shell and server versions do not match
switched to db myApp2
WriteResult({ "nInserted" : 1 })
bye
perconaservermongodbrestore.psmdb.percona.com/restore-backup-gcp-cs created
waiting psmdb-restore/backup-gcp-cs to reach ready state............
+ '[' 1 -eq 1 ']'
+ wait_cluster_consistency some-name
+ local cluster_name=some-name
+ local wait_time=32
+ retry=0
+ sleep 7
+ echo -n 'waiting for cluster readyness'
waiting for cluster readyness++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.jDRjEglgI2
+++ mktemp
++ local LAST_ERR=/tmp/tmp.yQJcvQ1j1l
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 0 ']'
++ break
++ cat /tmp/tmp.jDRjEglgI2
++ cat /tmp/tmp.yQJcvQ1j1l
++ rm /tmp/tmp.jDRjEglgI2 /tmp/tmp.yQJcvQ1j1l
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 1 -ge 32 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.ffLwiiFEYz
+++ mktemp
++ local LAST_ERR=/tmp/tmp.MG3UR7sMNZ
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 0 ']'
++ break
++ cat /tmp/tmp.ffLwiiFEYz
++ cat /tmp/tmp.MG3UR7sMNZ
++ rm /tmp/tmp.ffLwiiFEYz /tmp/tmp.MG3UR7sMNZ
++ return 0
+ [[ error == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 2 -ge 32 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.ciThjxEP3Q
+++ mktemp
++ local LAST_ERR=/tmp/tmp.PvNH4yB83o
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 0 ']'
++ break
++ cat /tmp/tmp.ciThjxEP3Q
++ cat /tmp/tmp.PvNH4yB83o
++ rm /tmp/tmp.ciThjxEP3Q /tmp/tmp.PvNH4yB83o
++ return 0
+ [[ error == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 3 -ge 32 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.W1U6Q5Hahv
+++ mktemp
++ local LAST_ERR=/tmp/tmp.xmWac7EghY
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 0 ']'
++ break
++ cat /tmp/tmp.W1U6Q5Hahv
++ cat /tmp/tmp.xmWac7EghY
++ rm /tmp/tmp.W1U6Q5Hahv /tmp/tmp.xmWac7EghY
++ return 0
+ [[ error == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 4 -ge 32 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.Lc1u5s9PFT
+++ mktemp
++ local LAST_ERR=/tmp/tmp.cMvMBanLWk
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 0 ']'
++ break
++ cat /tmp/tmp.Lc1u5s9PFT
++ cat /tmp/tmp.cMvMBanLWk
++ rm /tmp/tmp.Lc1u5s9PFT /tmp/tmp.cMvMBanLWk
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 5 -ge 32 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.NAKvUNO18X
+++ mktemp
++ local LAST_ERR=/tmp/tmp.I3jNyadIuy
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 0 ']'
++ break
++ cat /tmp/tmp.NAKvUNO18X
++ cat /tmp/tmp.I3jNyadIuy
++ rm /tmp/tmp.NAKvUNO18X /tmp/tmp.I3jNyadIuy
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 6 -ge 32 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.g9VQYaS99a
+++ mktemp
++ local LAST_ERR=/tmp/tmp.lyq5JGdneh
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 0 ']'
++ break
++ cat /tmp/tmp.g9VQYaS99a
++ cat /tmp/tmp.lyq5JGdneh
++ rm /tmp/tmp.g9VQYaS99a /tmp/tmp.lyq5JGdneh
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 7 -ge 32 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.KDA7TRFf9F
+++ mktemp
++ local LAST_ERR=/tmp/tmp.8QmGA28uWZ
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 0 ']'
++ break
++ cat /tmp/tmp.KDA7TRFf9F
++ cat /tmp/tmp.8QmGA28uWZ
++ rm /tmp/tmp.KDA7TRFf9F /tmp/tmp.8QmGA28uWZ
++ return 0
+ [[ ready == \r\e\a\d\y ]]
+ check_data
+ local postfix=
++ seq 0 2
+ for i in '$(seq 0 2)'
+ compare_mongos_cmd find myApp:myPass@some-name-mongos.demand-backup-sharded-1695 '' .svc.cluster.local myApp test
+ local command=find
+ local uri=myApp:myPass@some-name-mongos.demand-backup-sharded-1695
+ local postfix=
+ local suffix=.svc.cluster.local
+ local database=myApp
+ local collection=test
+ run_mongos 'use myApp\n db.test.find()' myApp:myPass@some-name-mongos.demand-backup-sharded-1695 mongodb .svc.cluster.local
+ local 'command=use myApp\n db.test.find()'
+ local uri=myApp:myPass@some-name-mongos.demand-backup-sharded-1695
+ local driver=mongodb
+ local suffix=.svc.cluster.local
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
+ egrep -v 'I NETWORK|W NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:'
+ /usr/bin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
++ local LAST_OUT=/tmp/tmp.3ImitC6p2L
+++ mktemp
++ local LAST_ERR=/tmp/tmp.4Mn6NHq1gh
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 0 ']'
++ break
++ cat /tmp/tmp.3ImitC6p2L
++ cat /tmp/tmp.4Mn6NHq1gh
++ rm /tmp/tmp.3ImitC6p2L /tmp/tmp.4Mn6NHq1gh
++ return 0
+ local client_container=psmdb-client-7469665986-brptm
+ local mongo_flag=
+ kubectl_bin exec psmdb-client-7469665986-brptm -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-1695.svc.cluster.local/admin '
++ mktemp
+ local LAST_OUT=/tmp/tmp.85FF24MpAo
++ mktemp
+ local LAST_ERR=/tmp/tmp.t2FUO1c6Qk
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl exec psmdb-client-7469665986-brptm -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-1695.svc.cluster.local/admin '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 0 ']'
+ break
+ cat /tmp/tmp.85FF24MpAo
+ cat /tmp/tmp.t2FUO1c6Qk
+ rm /tmp/tmp.85FF24MpAo /tmp/tmp.t2FUO1c6Qk
+ return 0
+ diff /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1543/e2e-tests/demand-backup-sharded/compare/find.json /tmp/tmp.VmM91SP9mO/find
+ for i in '$(seq 0 2)'
+ compare_mongos_cmd find myApp:myPass@some-name-mongos.demand-backup-sharded-1695 1 .svc.cluster.local myApp1 test
+ local command=find
+ local uri=myApp:myPass@some-name-mongos.demand-backup-sharded-1695
+ local postfix=1
+ local suffix=.svc.cluster.local
+ local database=myApp1
+ local collection=test
+ /usr/bin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
+ run_mongos 'use myApp1\n db.test.find()' myApp:myPass@some-name-mongos.demand-backup-sharded-1695 mongodb .svc.cluster.local
+ local 'command=use myApp1\n db.test.find()'
+ local uri=myApp:myPass@some-name-mongos.demand-backup-sharded-1695
+ local driver=mongodb
+ local suffix=.svc.cluster.local
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
+ egrep -v 'I NETWORK|W NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:'
++ local LAST_OUT=/tmp/tmp.SCAU1wLgZI
+++ mktemp
++ local LAST_ERR=/tmp/tmp.EV7VXrZrMZ
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 0 ']'
++ break
++ cat /tmp/tmp.SCAU1wLgZI
++ cat /tmp/tmp.EV7VXrZrMZ
++ rm /tmp/tmp.SCAU1wLgZI /tmp/tmp.EV7VXrZrMZ
++ return 0
+ local client_container=psmdb-client-7469665986-brptm
+ local mongo_flag=
+ kubectl_bin exec psmdb-client-7469665986-brptm -- bash -c 'printf '\''use myApp1\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-1695.svc.cluster.local/admin '
++ mktemp
+ local LAST_OUT=/tmp/tmp.tMWicIjym2
++ mktemp
+ local LAST_ERR=/tmp/tmp.UDCSrhp5rL
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl exec psmdb-client-7469665986-brptm -- bash -c 'printf '\''use myApp1\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-1695.svc.cluster.local/admin '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 0 ']'
+ break
+ cat /tmp/tmp.tMWicIjym2
+ cat /tmp/tmp.UDCSrhp5rL
+ rm /tmp/tmp.tMWicIjym2 /tmp/tmp.UDCSrhp5rL
+ return 0
+ diff /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1543/e2e-tests/demand-backup-sharded/compare/find1.json /tmp/tmp.VmM91SP9mO/find1
+ for i in '$(seq 0 2)'
+ compare_mongos_cmd find myApp:myPass@some-name-mongos.demand-backup-sharded-1695 2 .svc.cluster.local myApp2 test
+ local command=find
+ local uri=myApp:myPass@some-name-mongos.demand-backup-sharded-1695
+ local postfix=2
+ local suffix=.svc.cluster.local
+ local database=myApp2
+ local collection=test
+ egrep -v 'I NETWORK|W NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:'
+ /usr/bin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
+ run_mongos 'use myApp2\n db.test.find()' myApp:myPass@some-name-mongos.demand-backup-sharded-1695 mongodb .svc.cluster.local
+ local 'command=use myApp2\n db.test.find()'
+ local uri=myApp:myPass@some-name-mongos.demand-backup-sharded-1695
+ local driver=mongodb
+ local suffix=.svc.cluster.local
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.B150Qlrsi2
+++ mktemp
++ local LAST_ERR=/tmp/tmp.QeYKPwnKrh
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 0 ']'
++ break
++ cat /tmp/tmp.B150Qlrsi2
++ cat /tmp/tmp.QeYKPwnKrh
++ rm /tmp/tmp.B150Qlrsi2 /tmp/tmp.QeYKPwnKrh
++ return 0
+ local client_container=psmdb-client-7469665986-brptm
+ local mongo_flag=
+ kubectl_bin exec psmdb-client-7469665986-brptm -- bash -c 'printf '\''use myApp2\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-1695.svc.cluster.local/admin '
++ mktemp
+ local LAST_OUT=/tmp/tmp.R8aioA4lcn
++ mktemp
+ local LAST_ERR=/tmp/tmp.Vl4bEl6n8W
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl exec psmdb-client-7469665986-brptm -- bash -c 'printf '\''use myApp2\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-1695.svc.cluster.local/admin '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 0 ']'
+ break
+ cat /tmp/tmp.R8aioA4lcn
+ cat /tmp/tmp.Vl4bEl6n8W
+ rm /tmp/tmp.R8aioA4lcn /tmp/tmp.Vl4bEl6n8W
+ return 0
+ diff /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1543/e2e-tests/demand-backup-sharded/compare/find2.json /tmp/tmp.VmM91SP9mO/find2
+ desc 'check backup and restore -- azure-blob'
+ set +o xtrace


-----------------------------------------------------------------------------------
check backup and restore -- azure-blob
-----------------------------------------------------------------------------------

Percona Server for MongoDB shell version v4.4.29-28
connecting to: mongodb://some-name-mongos.demand-backup-sharded-1695.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb
Implicit session: session { "id" : UUID("36b1a91f-859f-490f-a4e8-7e21f6cf2908") }
Percona Server for MongoDB server version: v7.0.8-5
WARNING: shell and server versions do not match
switched to db myApp
WriteResult({ "nInserted" : 1 })
bye
Percona Server for MongoDB shell version v4.4.29-28
connecting to: mongodb://some-name-mongos.demand-backup-sharded-1695.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb
Implicit session: session { "id" : UUID("ae483465-9914-4f5d-a758-a8532f4325f6") }
Percona Server for MongoDB server version: v7.0.8-5
WARNING: shell and server versions do not match
switched to db myApp1
WriteResult({ "nInserted" : 1 })
bye
Percona Server for MongoDB shell version v4.4.29-28
connecting to: mongodb://some-name-mongos.demand-backup-sharded-1695.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb
Implicit session: session { "id" : UUID("e301e3e4-889e-43c9-9027-15e59ea27a3f") }
Percona Server for MongoDB server version: v7.0.8-5
WARNING: shell and server versions do not match
switched to db myApp2
WriteResult({ "nInserted" : 1 })
bye
perconaservermongodbrestore.psmdb.percona.com/restore-backup-azure-blob created
waiting psmdb-restore/backup-azure-blob to reach ready state............
+ '[' 1 -eq 1 ']'
+ wait_cluster_consistency some-name
+ local cluster_name=some-name
+ local wait_time=32
+ retry=0
+ sleep 7
+ echo -n 'waiting for cluster readyness'
waiting for cluster readyness++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.xN3C4MjLmk
+++ mktemp
++ local LAST_ERR=/tmp/tmp.E2ePlGhmYe
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 0 ']'
++ break
++ cat /tmp/tmp.xN3C4MjLmk
++ cat /tmp/tmp.E2ePlGhmYe
++ rm /tmp/tmp.xN3C4MjLmk /tmp/tmp.E2ePlGhmYe
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 1 -ge 32 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.vJe9WiaGB5
+++ mktemp
++ local LAST_ERR=/tmp/tmp.7bhieOvNrB
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 0 ']'
++ break
++ cat /tmp/tmp.vJe9WiaGB5
++ cat /tmp/tmp.7bhieOvNrB
++ rm /tmp/tmp.vJe9WiaGB5 /tmp/tmp.7bhieOvNrB
++ return 0
+ [[ error == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 2 -ge 32 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.E8F7xCqRIA
+++ mktemp
++ local LAST_ERR=/tmp/tmp.wmGsqU7DGO
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 0 ']'
++ break
++ cat /tmp/tmp.E8F7xCqRIA
++ cat /tmp/tmp.wmGsqU7DGO
++ rm /tmp/tmp.E8F7xCqRIA /tmp/tmp.wmGsqU7DGO
++ return 0
+ [[ error == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 3 -ge 32 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.XbJkRjoEV6
+++ mktemp
++ local LAST_ERR=/tmp/tmp.IHV6IxQMSg
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 0 ']'
++ break
++ cat /tmp/tmp.XbJkRjoEV6
++ cat /tmp/tmp.IHV6IxQMSg
++ rm /tmp/tmp.XbJkRjoEV6 /tmp/tmp.IHV6IxQMSg
++ return 0
+ [[ error == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 4 -ge 32 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.KDIc8QxUy7
+++ mktemp
++ local LAST_ERR=/tmp/tmp.wcruuyy30T
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 0 ']'
++ break
++ cat /tmp/tmp.KDIc8QxUy7
++ cat /tmp/tmp.wcruuyy30T
++ rm /tmp/tmp.KDIc8QxUy7 /tmp/tmp.wcruuyy30T
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 5 -ge 32 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.6ZMTTZUXnk
+++ mktemp
++ local LAST_ERR=/tmp/tmp.lL7137ZsS6
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 0 ']'
++ break
++ cat /tmp/tmp.6ZMTTZUXnk
++ cat /tmp/tmp.lL7137ZsS6
++ rm /tmp/tmp.6ZMTTZUXnk /tmp/tmp.lL7137ZsS6
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 6 -ge 32 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.WxiowY3xL5
+++ mktemp
++ local LAST_ERR=/tmp/tmp.G7L0W8LwGA
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 0 ']'
++ break
++ cat /tmp/tmp.WxiowY3xL5
++ cat /tmp/tmp.G7L0W8LwGA
++ rm /tmp/tmp.WxiowY3xL5 /tmp/tmp.G7L0W8LwGA
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 7 -ge 32 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.umGfnp7Csc
+++ mktemp
++ local LAST_ERR=/tmp/tmp.gGrflMBMwd
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 0 ']'
++ break
++ cat /tmp/tmp.umGfnp7Csc
++ cat /tmp/tmp.gGrflMBMwd
++ rm /tmp/tmp.umGfnp7Csc /tmp/tmp.gGrflMBMwd
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 8 -ge 32 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.V137TTn59d
+++ mktemp
++ local LAST_ERR=/tmp/tmp.KjE6aRF01z
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 0 ']'
++ break
++ cat /tmp/tmp.V137TTn59d
++ cat /tmp/tmp.KjE6aRF01z
++ rm /tmp/tmp.V137TTn59d /tmp/tmp.KjE6aRF01z
++ return 0
+ [[ ready == \r\e\a\d\y ]]
+ check_data
+ local postfix=
++ seq 0 2
+ for i in '$(seq 0 2)'
+ compare_mongos_cmd find myApp:myPass@some-name-mongos.demand-backup-sharded-1695 '' .svc.cluster.local myApp test
+ local command=find
+ local uri=myApp:myPass@some-name-mongos.demand-backup-sharded-1695
+ local postfix=
+ local suffix=.svc.cluster.local
+ local database=myApp
+ local collection=test
+ run_mongos 'use myApp\n db.test.find()' myApp:myPass@some-name-mongos.demand-backup-sharded-1695 mongodb .svc.cluster.local
+ local 'command=use myApp\n db.test.find()'
+ local uri=myApp:myPass@some-name-mongos.demand-backup-sharded-1695
+ local driver=mongodb
+ local suffix=.svc.cluster.local
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
+ /usr/bin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
++ local LAST_OUT=/tmp/tmp.NZvjjABEEc
+ egrep -v 'I NETWORK|W NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:'
+++ mktemp
++ local LAST_ERR=/tmp/tmp.LYwA0aW2de
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 0 ']'
++ break
++ cat /tmp/tmp.NZvjjABEEc
++ cat /tmp/tmp.LYwA0aW2de
++ rm /tmp/tmp.NZvjjABEEc /tmp/tmp.LYwA0aW2de
++ return 0
+ local client_container=psmdb-client-7469665986-brptm
+ local mongo_flag=
+ kubectl_bin exec psmdb-client-7469665986-brptm -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-1695.svc.cluster.local/admin '
++ mktemp
+ local LAST_OUT=/tmp/tmp.L9u85DuVN7
++ mktemp
+ local LAST_ERR=/tmp/tmp.lAhbFHSWIJ
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl exec psmdb-client-7469665986-brptm -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-1695.svc.cluster.local/admin '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 0 ']'
+ break
+ cat /tmp/tmp.L9u85DuVN7
+ cat /tmp/tmp.lAhbFHSWIJ
+ rm /tmp/tmp.L9u85DuVN7 /tmp/tmp.lAhbFHSWIJ
+ return 0
+ diff /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1543/e2e-tests/demand-backup-sharded/compare/find.json /tmp/tmp.VmM91SP9mO/find
+ for i in '$(seq 0 2)'
+ compare_mongos_cmd find myApp:myPass@some-name-mongos.demand-backup-sharded-1695 1 .svc.cluster.local myApp1 test
+ local command=find
+ local uri=myApp:myPass@some-name-mongos.demand-backup-sharded-1695
+ local postfix=1
+ local suffix=.svc.cluster.local
+ local database=myApp1
+ local collection=test
+ run_mongos 'use myApp1\n db.test.find()' myApp:myPass@some-name-mongos.demand-backup-sharded-1695 mongodb .svc.cluster.local
+ local 'command=use myApp1\n db.test.find()'
+ egrep -v 'I NETWORK|W NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:'
+ local uri=myApp:myPass@some-name-mongos.demand-backup-sharded-1695
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ /usr/bin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.289R8OZzO0
+++ mktemp
++ local LAST_ERR=/tmp/tmp.QIbbFUzaFv
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 0 ']'
++ break
++ cat /tmp/tmp.289R8OZzO0
++ cat /tmp/tmp.QIbbFUzaFv
++ rm /tmp/tmp.289R8OZzO0 /tmp/tmp.QIbbFUzaFv
++ return 0
+ local client_container=psmdb-client-7469665986-brptm
+ local mongo_flag=
+ kubectl_bin exec psmdb-client-7469665986-brptm -- bash -c 'printf '\''use myApp1\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-1695.svc.cluster.local/admin '
++ mktemp
+ local LAST_OUT=/tmp/tmp.pS5VyIhLAo
++ mktemp
+ local LAST_ERR=/tmp/tmp.UkxpceGqVQ
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl exec psmdb-client-7469665986-brptm -- bash -c 'printf '\''use myApp1\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-1695.svc.cluster.local/admin '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 0 ']'
+ break
+ cat /tmp/tmp.pS5VyIhLAo
+ cat /tmp/tmp.UkxpceGqVQ
+ rm /tmp/tmp.pS5VyIhLAo /tmp/tmp.UkxpceGqVQ
+ return 0
+ diff /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1543/e2e-tests/demand-backup-sharded/compare/find1.json /tmp/tmp.VmM91SP9mO/find1
+ for i in '$(seq 0 2)'
+ compare_mongos_cmd find myApp:myPass@some-name-mongos.demand-backup-sharded-1695 2 .svc.cluster.local myApp2 test
+ local command=find
+ local uri=myApp:myPass@some-name-mongos.demand-backup-sharded-1695
+ local postfix=2
+ local suffix=.svc.cluster.local
+ local database=myApp2
+ local collection=test
+ run_mongos 'use myApp2\n db.test.find()' myApp:myPass@some-name-mongos.demand-backup-sharded-1695 mongodb .svc.cluster.local
+ local 'command=use myApp2\n db.test.find()'
+ egrep -v 'I NETWORK|W NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:'
+ local uri=myApp:myPass@some-name-mongos.demand-backup-sharded-1695
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ /usr/bin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.1Xh1Udi8KZ
+++ mktemp
++ local LAST_ERR=/tmp/tmp.5yoY3eGWA6
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 0 ']'
++ break
++ cat /tmp/tmp.1Xh1Udi8KZ
++ cat /tmp/tmp.5yoY3eGWA6
++ rm /tmp/tmp.1Xh1Udi8KZ /tmp/tmp.5yoY3eGWA6
++ return 0
+ local client_container=psmdb-client-7469665986-brptm
+ local mongo_flag=
+ kubectl_bin exec psmdb-client-7469665986-brptm -- bash -c 'printf '\''use myApp2\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-1695.svc.cluster.local/admin '
++ mktemp
+ local LAST_OUT=/tmp/tmp.x6FSeKoXzF
++ mktemp
+ local LAST_ERR=/tmp/tmp.nRyUB3r8eH
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl exec psmdb-client-7469665986-brptm -- bash -c 'printf '\''use myApp2\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-1695.svc.cluster.local/admin '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 0 ']'
+ break
+ cat /tmp/tmp.x6FSeKoXzF
+ cat /tmp/tmp.nRyUB3r8eH
+ rm /tmp/tmp.x6FSeKoXzF /tmp/tmp.nRyUB3r8eH
+ return 0
+ diff /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1543/e2e-tests/demand-backup-sharded/compare/find2.json /tmp/tmp.VmM91SP9mO/find2
+ desc 'check backup and restore -- minio'
+ set +o xtrace


-----------------------------------------------------------------------------------
check backup and restore -- minio
-----------------------------------------------------------------------------------

2024-05-07 11:16:12         55 myApp.test.gz
Percona Server for MongoDB shell version v4.4.29-28
connecting to: mongodb://some-name-mongos.demand-backup-sharded-1695.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb
Implicit session: session { "id" : UUID("70349a3b-ff4b-4b09-b49f-8911de1ef13a") }
Percona Server for MongoDB server version: v7.0.8-5
WARNING: shell and server versions do not match
switched to db myApp
WriteResult({ "nInserted" : 1 })
bye
Percona Server for MongoDB shell version v4.4.29-28
connecting to: mongodb://some-name-mongos.demand-backup-sharded-1695.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb
Implicit session: session { "id" : UUID("9aadeb02-9ff4-447e-9b54-93fd085d5f39") }
Percona Server for MongoDB server version: v7.0.8-5
WARNING: shell and server versions do not match
switched to db myApp1
WriteResult({ "nInserted" : 1 })
bye
Percona Server for MongoDB shell version v4.4.29-28
connecting to: mongodb://some-name-mongos.demand-backup-sharded-1695.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb
Implicit session: session { "id" : UUID("627e155a-c932-4427-b721-06b493579668") }
Percona Server for MongoDB server version: v7.0.8-5
WARNING: shell and server versions do not match
switched to db myApp2
WriteResult({ "nInserted" : 1 })
bye
perconaservermongodbrestore.psmdb.percona.com/restore-backup-minio created
waiting psmdb-restore/backup-minio to reach ready state...........
+ '[' 1 -eq 1 ']'
+ wait_cluster_consistency some-name
+ local cluster_name=some-name
+ local wait_time=32
+ retry=0
+ sleep 7
+ echo -n 'waiting for cluster readyness'
waiting for cluster readyness++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.Kq5bdPqwMU
+++ mktemp
++ local LAST_ERR=/tmp/tmp.Pc227UhZJC
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 0 ']'
++ break
++ cat /tmp/tmp.Kq5bdPqwMU
++ cat /tmp/tmp.Pc227UhZJC
++ rm /tmp/tmp.Kq5bdPqwMU /tmp/tmp.Pc227UhZJC
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 1 -ge 32 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.hvDYSfaM8u
+++ mktemp
++ local LAST_ERR=/tmp/tmp.tDb8Chss6b
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 0 ']'
++ break
++ cat /tmp/tmp.hvDYSfaM8u
++ cat /tmp/tmp.tDb8Chss6b
++ rm /tmp/tmp.hvDYSfaM8u /tmp/tmp.tDb8Chss6b
++ return 0
+ [[ error == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 2 -ge 32 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.Ua687oGWUN
+++ mktemp
++ local LAST_ERR=/tmp/tmp.IpnHSc1SCn
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 0 ']'
++ break
++ cat /tmp/tmp.Ua687oGWUN
++ cat /tmp/tmp.IpnHSc1SCn
++ rm /tmp/tmp.Ua687oGWUN /tmp/tmp.IpnHSc1SCn
++ return 0
+ [[ error == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 3 -ge 32 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.7lxfTc45AC
+++ mktemp
++ local LAST_ERR=/tmp/tmp.JyJlP71VhZ
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 0 ']'
++ break
++ cat /tmp/tmp.7lxfTc45AC
++ cat /tmp/tmp.JyJlP71VhZ
++ rm /tmp/tmp.7lxfTc45AC /tmp/tmp.JyJlP71VhZ
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 4 -ge 32 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.p65wwoSJkf
+++ mktemp
++ local LAST_ERR=/tmp/tmp.53qxQ3DQfo
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 0 ']'
++ break
++ cat /tmp/tmp.p65wwoSJkf
++ cat /tmp/tmp.53qxQ3DQfo
++ rm /tmp/tmp.p65wwoSJkf /tmp/tmp.53qxQ3DQfo
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 5 -ge 32 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.1DNWf8oR3f
+++ mktemp
++ local LAST_ERR=/tmp/tmp.v0QvY0ONI4
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 0 ']'
++ break
++ cat /tmp/tmp.1DNWf8oR3f
++ cat /tmp/tmp.v0QvY0ONI4
++ rm /tmp/tmp.1DNWf8oR3f /tmp/tmp.v0QvY0ONI4
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 6 -ge 32 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.DJ8JBQblIl
+++ mktemp
++ local LAST_ERR=/tmp/tmp.jGz9Hs0HR0
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 0 ']'
++ break
++ cat /tmp/tmp.DJ8JBQblIl
++ cat /tmp/tmp.jGz9Hs0HR0
++ rm /tmp/tmp.DJ8JBQblIl /tmp/tmp.jGz9Hs0HR0
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 7 -ge 32 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.da29rA193p
+++ mktemp
++ local LAST_ERR=/tmp/tmp.5BRiDCDyFf
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 0 ']'
++ break
++ cat /tmp/tmp.da29rA193p
++ cat /tmp/tmp.5BRiDCDyFf
++ rm /tmp/tmp.da29rA193p /tmp/tmp.5BRiDCDyFf
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 8 -ge 32 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.iuLjNvERf0
+++ mktemp
++ local LAST_ERR=/tmp/tmp.kwt9aonGK4
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 0 ']'
++ break
++ cat /tmp/tmp.iuLjNvERf0
++ cat /tmp/tmp.kwt9aonGK4
++ rm /tmp/tmp.iuLjNvERf0 /tmp/tmp.kwt9aonGK4
++ return 0
+ [[ ready == \r\e\a\d\y ]]
+ check_data
+ local postfix=
++ seq 0 2
+ for i in '$(seq 0 2)'
+ compare_mongos_cmd find myApp:myPass@some-name-mongos.demand-backup-sharded-1695 '' .svc.cluster.local myApp test
+ local command=find
+ local uri=myApp:myPass@some-name-mongos.demand-backup-sharded-1695
+ local postfix=
+ local suffix=.svc.cluster.local
+ local database=myApp
+ local collection=test
+ run_mongos 'use myApp\n db.test.find()' myApp:myPass@some-name-mongos.demand-backup-sharded-1695 mongodb .svc.cluster.local
+ local 'command=use myApp\n db.test.find()'
+ local uri=myApp:myPass@some-name-mongos.demand-backup-sharded-1695
+ local driver=mongodb
+ egrep -v 'I NETWORK|W NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:'
+ local suffix=.svc.cluster.local
+ /usr/bin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.GtgjuCQVa6
+++ mktemp
++ local LAST_ERR=/tmp/tmp.cLqHdtJ3p7
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 0 ']'
++ break
++ cat /tmp/tmp.GtgjuCQVa6
++ cat /tmp/tmp.cLqHdtJ3p7
++ rm /tmp/tmp.GtgjuCQVa6 /tmp/tmp.cLqHdtJ3p7
++ return 0
+ local client_container=psmdb-client-7469665986-brptm
+ local mongo_flag=
+ kubectl_bin exec psmdb-client-7469665986-brptm -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-1695.svc.cluster.local/admin '
++ mktemp
+ local LAST_OUT=/tmp/tmp.UuzyYRVb6w
++ mktemp
+ local LAST_ERR=/tmp/tmp.88IsL29Rce
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl exec psmdb-client-7469665986-brptm -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-1695.svc.cluster.local/admin '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 0 ']'
+ break
+ cat /tmp/tmp.UuzyYRVb6w
+ cat /tmp/tmp.88IsL29Rce
+ rm /tmp/tmp.UuzyYRVb6w /tmp/tmp.88IsL29Rce
+ return 0
+ diff /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1543/e2e-tests/demand-backup-sharded/compare/find.json /tmp/tmp.VmM91SP9mO/find
+ for i in '$(seq 0 2)'
+ compare_mongos_cmd find myApp:myPass@some-name-mongos.demand-backup-sharded-1695 1 .svc.cluster.local myApp1 test
+ local command=find
+ local uri=myApp:myPass@some-name-mongos.demand-backup-sharded-1695
+ local postfix=1
+ local suffix=.svc.cluster.local
+ local database=myApp1
+ local collection=test
+ run_mongos 'use myApp1\n db.test.find()' myApp:myPass@some-name-mongos.demand-backup-sharded-1695 mongodb .svc.cluster.local
+ egrep -v 'I NETWORK|W NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:'
+ local 'command=use myApp1\n db.test.find()'
+ local uri=myApp:myPass@some-name-mongos.demand-backup-sharded-1695
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ /usr/bin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.GVhrXjIfse
+++ mktemp
++ local LAST_ERR=/tmp/tmp.Kz5lpoQcdS
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 0 ']'
++ break
++ cat /tmp/tmp.GVhrXjIfse
++ cat /tmp/tmp.Kz5lpoQcdS
++ rm /tmp/tmp.GVhrXjIfse /tmp/tmp.Kz5lpoQcdS
++ return 0
+ local client_container=psmdb-client-7469665986-brptm
+ local mongo_flag=
+ kubectl_bin exec psmdb-client-7469665986-brptm -- bash -c 'printf '\''use myApp1\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-1695.svc.cluster.local/admin '
++ mktemp
+ local LAST_OUT=/tmp/tmp.VijhCfUFuZ
++ mktemp
+ local LAST_ERR=/tmp/tmp.av4ukBKOsz
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl exec psmdb-client-7469665986-brptm -- bash -c 'printf '\''use myApp1\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-1695.svc.cluster.local/admin '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 0 ']'
+ break
+ cat /tmp/tmp.VijhCfUFuZ
+ cat /tmp/tmp.av4ukBKOsz
+ rm /tmp/tmp.VijhCfUFuZ /tmp/tmp.av4ukBKOsz
+ return 0
+ diff /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1543/e2e-tests/demand-backup-sharded/compare/find1.json /tmp/tmp.VmM91SP9mO/find1
+ for i in '$(seq 0 2)'
+ compare_mongos_cmd find myApp:myPass@some-name-mongos.demand-backup-sharded-1695 2 .svc.cluster.local myApp2 test
+ local command=find
+ local uri=myApp:myPass@some-name-mongos.demand-backup-sharded-1695
+ local postfix=2
+ local suffix=.svc.cluster.local
+ local database=myApp2
+ local collection=test
+ run_mongos 'use myApp2\n db.test.find()' myApp:myPass@some-name-mongos.demand-backup-sharded-1695 mongodb .svc.cluster.local
+ egrep -v 'I NETWORK|W NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:'
+ local 'command=use myApp2\n db.test.find()'
+ local uri=myApp:myPass@some-name-mongos.demand-backup-sharded-1695
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ /usr/bin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.YxezC4HwaA
+++ mktemp
++ local LAST_ERR=/tmp/tmp.HkL4Evw3zM
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 0 ']'
++ break
++ cat /tmp/tmp.YxezC4HwaA
++ cat /tmp/tmp.HkL4Evw3zM
++ rm /tmp/tmp.YxezC4HwaA /tmp/tmp.HkL4Evw3zM
++ return 0
+ local client_container=psmdb-client-7469665986-brptm
+ local mongo_flag=
+ kubectl_bin exec psmdb-client-7469665986-brptm -- bash -c 'printf '\''use myApp2\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-1695.svc.cluster.local/admin '
++ mktemp
+ local LAST_OUT=/tmp/tmp.nWSeFYmaP4
++ mktemp
+ local LAST_ERR=/tmp/tmp.WTW6U1mfpM
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl exec psmdb-client-7469665986-brptm -- bash -c 'printf '\''use myApp2\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-1695.svc.cluster.local/admin '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 0 ']'
+ break
+ cat /tmp/tmp.nWSeFYmaP4
+ cat /tmp/tmp.WTW6U1mfpM
+ rm /tmp/tmp.nWSeFYmaP4 /tmp/tmp.WTW6U1mfpM
+ return 0
+ diff /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1543/e2e-tests/demand-backup-sharded/compare/find2.json /tmp/tmp.VmM91SP9mO/find2
+ desc 'delete backup and check if it is removed from bucket -- minio'
+ set +o xtrace


-----------------------------------------------------------------------------------
delete backup and check if it is removed from bucket -- minio
-----------------------------------------------------------------------------------

perconaservermongodbbackup.psmdb.percona.com "backup-aws-s3" deleted
perconaservermongodbbackup.psmdb.percona.com "backup-azure-blob" deleted
perconaservermongodbbackup.psmdb.percona.com "backup-gcp-cs" deleted
perconaservermongodbbackup.psmdb.percona.com "backup-minio" deleted


-----------------------------------------------------------------------------------
check for passwords leak
-----------------------------------------------------------------------------------

secrets=YmFja3VwMTIzNDU2 Y2x1c3RlckFkbWluMTIzNDU2 Y2x1c3Rlck1vbml0b3IxMjM0NTY= ZGF0YWJhc2VBZG1pbjEyMzQ1Ng== dXNlckFkbWluMTIzNDU2 YmFja3VwMTIzNDU2 Y2x1c3RlckFkbWluMTIzNDU2 Y2x1c3Rlck1vbml0b3IxMjM0NTY= ZGF0YWJhc2VBZG1pbjEyMzQ1Ng== dXNlckFkbWluMTIzNDU2
passwords=backup123456 clusterAdmin123456 clusterMonitor123456 databaseAdmin123456 userAdmin123456 backup123456 clusterAdmin123456 clusterMonitor123456 databaseAdmin123456 userAdmin123456 YmFja3VwMTIzNDU2 Y2x1c3RlckFkbWluMTIzNDU2 Y2x1c3Rlck1vbml0b3IxMjM0NTY= ZGF0YWJhc2VBZG1pbjEyMzQ1Ng== dXNlckFkbWluMTIzNDU2 YmFja3VwMTIzNDU2 Y2x1c3RlckFkbWluMTIzNDU2 Y2x1c3Rlck1vbml0b3IxMjM0NTY= ZGF0YWJhc2VBZG1pbjEyMzQ1Ng== dXNlckFkbWluMTIzNDU2
pods=minio-service-57dd49b-lrchv psmdb-client-7469665986-brptm some-name-cfg-0 some-name-cfg-1 some-name-cfg-2 some-name-mongos-0 some-name-mongos-1 some-name-mongos-2 some-name-rs0-0 some-name-rs0-1 some-name-rs0-2 some-name-rs1-0 some-name-rs1-1 some-name-rs1-2 some-name-rs2-0 some-name-rs2-1 some-name-rs2-2
logs saved in: /tmp/tmp.VmM91SP9mO/logs_output-minio-service-57dd49b-lrchv-minio.txt

logs saved in: /tmp/tmp.VmM91SP9mO/logs_output-psmdb-client-7469665986-brptm-psmdb-client.txt

logs saved in: /tmp/tmp.VmM91SP9mO/logs_output-some-name-cfg-0-mongod.txt
logs saved in: /tmp/tmp.VmM91SP9mO/logs_output-some-name-cfg-0-cfg-sidecar-1.txt
logs saved in: /tmp/tmp.VmM91SP9mO/logs_output-some-name-cfg-0-backup-agent.txt

logs saved in: /tmp/tmp.VmM91SP9mO/logs_output-some-name-cfg-1-mongod.txt
logs saved in: /tmp/tmp.VmM91SP9mO/logs_output-some-name-cfg-1-cfg-sidecar-1.txt
logs saved in: /tmp/tmp.VmM91SP9mO/logs_output-some-name-cfg-1-backup-agent.txt

logs saved in: /tmp/tmp.VmM91SP9mO/logs_output-some-name-cfg-2-mongod.txt
logs saved in: /tmp/tmp.VmM91SP9mO/logs_output-some-name-cfg-2-cfg-sidecar-1.txt
logs saved in: /tmp/tmp.VmM91SP9mO/logs_output-some-name-cfg-2-backup-agent.txt

logs saved in: /tmp/tmp.VmM91SP9mO/logs_output-some-name-mongos-0-mongos.txt
logs saved in: /tmp/tmp.VmM91SP9mO/logs_output-some-name-mongos-0-mongos-sidecar-1.txt

logs saved in: /tmp/tmp.VmM91SP9mO/logs_output-some-name-mongos-1-mongos.txt
logs saved in: /tmp/tmp.VmM91SP9mO/logs_output-some-name-mongos-1-mongos-sidecar-1.txt

logs saved in: /tmp/tmp.VmM91SP9mO/logs_output-some-name-mongos-2-mongos.txt
logs saved in: /tmp/tmp.VmM91SP9mO/logs_output-some-name-mongos-2-mongos-sidecar-1.txt

logs saved in: /tmp/tmp.VmM91SP9mO/logs_output-some-name-rs0-0-mongod.txt
logs saved in: /tmp/tmp.VmM91SP9mO/logs_output-some-name-rs0-0-backup-agent.txt

logs saved in: /tmp/tmp.VmM91SP9mO/logs_output-some-name-rs0-1-mongod.txt
logs saved in: /tmp/tmp.VmM91SP9mO/logs_output-some-name-rs0-1-backup-agent.txt

logs saved in: /tmp/tmp.VmM91SP9mO/logs_output-some-name-rs0-2-mongod.txt
logs saved in: /tmp/tmp.VmM91SP9mO/logs_output-some-name-rs0-2-backup-agent.txt

logs saved in: /tmp/tmp.VmM91SP9mO/logs_output-some-name-rs1-0-mongod.txt
logs saved in: /tmp/tmp.VmM91SP9mO/logs_output-some-name-rs1-0-backup-agent.txt

logs saved in: /tmp/tmp.VmM91SP9mO/logs_output-some-name-rs1-1-mongod.txt
logs saved in: /tmp/tmp.VmM91SP9mO/logs_output-some-name-rs1-1-backup-agent.txt

logs saved in: /tmp/tmp.VmM91SP9mO/logs_output-some-name-rs1-2-mongod.txt
logs saved in: /tmp/tmp.VmM91SP9mO/logs_output-some-name-rs1-2-backup-agent.txt

logs saved in: /tmp/tmp.VmM91SP9mO/logs_output-some-name-rs2-0-mongod.txt
logs saved in: /tmp/tmp.VmM91SP9mO/logs_output-some-name-rs2-0-rs-sidecar-1.txt
logs saved in: /tmp/tmp.VmM91SP9mO/logs_output-some-name-rs2-0-backup-agent.txt

logs saved in: /tmp/tmp.VmM91SP9mO/logs_output-some-name-rs2-1-mongod.txt
logs saved in: /tmp/tmp.VmM91SP9mO/logs_output-some-name-rs2-1-rs-sidecar-1.txt
logs saved in: /tmp/tmp.VmM91SP9mO/logs_output-some-name-rs2-1-backup-agent.txt

logs saved in: /tmp/tmp.VmM91SP9mO/logs_output-some-name-rs2-2-mongod.txt
logs saved in: /tmp/tmp.VmM91SP9mO/logs_output-some-name-rs2-2-rs-sidecar-1.txt
logs saved in: /tmp/tmp.VmM91SP9mO/logs_output-some-name-rs2-2-backup-agent.txt

logs saved in: /tmp/tmp.VmM91SP9mO/logs_output-percona-server-mongodb-operator-fdfdd955d-5m8kh-percona-server-mongodb-operator.txt

runtimeclass.node.k8s.io "container-rc" deleted


-----------------------------------------------------------------------------------
destroy cluster/operator and all other resources
-----------------------------------------------------------------------------------



-----------------------------------------------------------------------------------
get and delete old CRDs and RBAC
-----------------------------------------------------------------------------------

customresourcedefinition.apiextensions.k8s.io "perconaservermongodbbackups.psmdb.percona.com" deleted
customresourcedefinition.apiextensions.k8s.io "perconaservermongodbrestores.psmdb.percona.com" deleted
customresourcedefinition.apiextensions.k8s.io "perconaservermongodbs.psmdb.percona.com" deleted
error: the server doesn't have a resource type "perconaservermongodbbackups"
+ kubectl patch perconaservermongodbbackups.psmdb.percona.com -n sh --type=merge -p '{"metadata":{"finalizers":[]}}'
error: the server doesn't have a resource type "perconaservermongodbbackups"
error: the server doesn't have a resource type "perconaservermongodbrestores"
+ kubectl patch perconaservermongodbrestores.psmdb.percona.com -n sh --type=merge -p '{"metadata":{"finalizers":[]}}'
E0507 11:31:29.696479   24173 memcache.go:287] couldn't get resource list for psmdb.percona.com/v1-12-0: the server could not find the requested resource
E0507 11:31:29.944382   24173 memcache.go:287] couldn't get resource list for psmdb.percona.com/v1: the server could not find the requested resource
E0507 11:31:29.951755   24173 memcache.go:287] couldn't get resource list for psmdb.percona.com/v1-10-0: the server could not find the requested resource
E0507 11:31:29.954912   24173 memcache.go:287] couldn't get resource list for psmdb.percona.com/v1-11-0: the server could not find the requested resource
error: the server doesn't have a resource type "perconaservermongodbrestores"
error: the server doesn't have a resource type "perconaservermongodbs"
+ kubectl patch perconaservermongodbs.psmdb.percona.com -n sh --type=merge -p '{"metadata":{"finalizers":[]}}'
error: the server doesn't have a resource type "perconaservermongodbs"
clusterrole.rbac.authorization.k8s.io "percona-server-mongodb-operator" deleted
clusterrolebinding.rbac.authorization.k8s.io "service-account-percona-server-mongodb-operator" deleted