++ echo 'Log: /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1544/e2e-tests/logs/demand-backup-sharded.log'
Log: /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1544/e2e-tests/logs/demand-backup-sharded.log
++ '[' -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1544/e2e-tests/conf/cloud-secret.yml ']'
++ SKIP_BACKUPS_TO_AWS_GCP_AZURE=
++ oc get projects
++ kubectl get nodes
++ grep '^minikube'
+++ jq -r .serverVersion.gitVersion
+++ kubectl version -o json
+++ grep '\-eks\-'
WARNING: version difference between client (1.30) and server (1.26) exceeds the supported minor version skew of +/-1
++ '[' ']'
++ EKS=0
+++ grep gke
+++ jq -r .serverVersion.gitVersion
+++ kubectl version -o json
WARNING: version difference between client (1.30) and server (1.26) exceeds the supported minor version skew of +/-1
++ '[' v1.26.15-gke.1243000 ']'
++ GKE=1
+++ kubectl version -o json
+++ jq -r '.serverVersion.major + "." + .serverVersion.minor'
+++ /usr/bin/sed -r 's/[^0-9.]+//g'
WARNING: version difference between client (1.30) and server (1.26) exceeds the supported minor version skew of +/-1
++ KUBE_VERSION=1.26
+ set_debug
+ [[ 0 == 1 ]]
+ set +o xtrace


-----------------------------------------------------------------------------------
get and delete old CRDs and RBAC
-----------------------------------------------------------------------------------

error: the server doesn't have a resource type "perconaservermongodbbackups"
+ kubectl patch perconaservermongodbbackups.psmdb.percona.com -n sh --type=merge -p '{"metadata":{"finalizers":[]}}'
error: the server doesn't have a resource type "perconaservermongodbbackups"
error: the server doesn't have a resource type "perconaservermongodbrestores"
+ kubectl patch perconaservermongodbrestores.psmdb.percona.com -n sh --type=merge -p '{"metadata":{"finalizers":[]}}'
error: the server doesn't have a resource type "perconaservermongodbrestores"
error: the server doesn't have a resource type "perconaservermongodbs"
+ kubectl patch perconaservermongodbs.psmdb.percona.com -n sh --type=merge -p '{"metadata":{"finalizers":[]}}'
error: the server doesn't have a resource type "perconaservermongodbs"
Error from server (NotFound): customresourcedefinitions.apiextensions.k8s.io "null" not found
Error from server (NotFound): customresourcedefinitions.apiextensions.k8s.io "null" not found
Error from server (NotFound): customresourcedefinitions.apiextensions.k8s.io "null" not found
Error from server (NotFound): customresourcedefinitions.apiextensions.k8s.io "null" not found


-----------------------------------------------------------------------------------
destroy chaos-mesh
-----------------------------------------------------------------------------------

error: resource(s) were provided, but no name was specified
error: resource(s) were provided, but no name was specified
error: resource(s) were provided, but no name was specified
error: resource(s) were provided, but no name was specified
error: resource(s) were provided, but no name was specified
error: resource(s) were provided, but no name was specified


-----------------------------------------------------------------------------------
cleaned up all old namespaces
-----------------------------------------------------------------------------------



-----------------------------------------------------------------------------------
cleaned up old namespaces psmdb-operator
-----------------------------------------------------------------------------------

error: resource(s) were provided, but no name was specified


-----------------------------------------------------------------------------------
create namespace psmdb-operator
-----------------------------------------------------------------------------------

namespace/psmdb-operator created
Context "gke_cloud-dev-112233_us-central1-a_jen-psmdb-1544-2c8d07ee-1-cluster2" modified.


-----------------------------------------------------------------------------------
start PSMDB operator
-----------------------------------------------------------------------------------

customresourcedefinition.apiextensions.k8s.io/perconaservermongodbbackups.psmdb.percona.com serverside-applied
customresourcedefinition.apiextensions.k8s.io/perconaservermongodbrestores.psmdb.percona.com serverside-applied
customresourcedefinition.apiextensions.k8s.io/perconaservermongodbs.psmdb.percona.com serverside-applied
clusterrole.rbac.authorization.k8s.io/percona-server-mongodb-operator created
serviceaccount/percona-server-mongodb-operator created
clusterrolebinding.rbac.authorization.k8s.io/service-account-percona-server-mongodb-operator created
deployment.apps/percona-server-mongodb-operator created
waiting for pod/percona-server-mongodb-operator-557db8d66f-gqjqn to be ready.OK


-----------------------------------------------------------------------------------
destroy chaos-mesh
-----------------------------------------------------------------------------------

error: resource(s) were provided, but no name was specified
error: resource(s) were provided, but no name was specified
error: resource(s) were provided, but no name was specified
error: resource(s) were provided, but no name was specified
error: resource(s) were provided, but no name was specified
error: resource(s) were provided, but no name was specified


-----------------------------------------------------------------------------------
cleaned up all old namespaces
-----------------------------------------------------------------------------------



-----------------------------------------------------------------------------------
cleaned up old namespaces demand-backup-sharded-10660
-----------------------------------------------------------------------------------

error: resource(s) were provided, but no name was specified


-----------------------------------------------------------------------------------
create namespace demand-backup-sharded-10660
-----------------------------------------------------------------------------------

namespace/demand-backup-sharded-10660 created
Context "gke_cloud-dev-112233_us-central1-a_jen-psmdb-1544-2c8d07ee-1-cluster2" modified.


-----------------------------------------------------------------------------------
install Minio
-----------------------------------------------------------------------------------

Error: uninstall: Release not loaded: minio-service: release: not found
"minio" has been removed from your repositories
"minio" has been added to your repositories
NAME: minio-service
LAST DEPLOYED: Tue May  7 06:46:41 2024
NAMESPACE: demand-backup-sharded-10660
STATUS: deployed
REVISION: 1
TEST SUITE: None
NOTES:
MinIO can be accessed via port 9000 on the following DNS name from within your cluster:
minio-service.demand-backup-sharded-10660.svc.cluster.local

To access MinIO from localhost, run the below commands:

  1. export POD_NAME=$(kubectl get pods --namespace demand-backup-sharded-10660 -l "release=minio-service" -o jsonpath="{.items[0].metadata.name}")

  2. kubectl port-forward $POD_NAME 9000 --namespace demand-backup-sharded-10660

Read more about port forwarding here: http://kubernetes.io/docs/user-guide/kubectl/kubectl_port-forward/

You can now access MinIO server on http://localhost:9000. Follow the below steps to connect to MinIO server with mc client:

  1. Download the MinIO mc client - https://min.io/docs/minio/linux/reference/minio-mc.html#quickstart

  2. export MC_HOST_minio-service-local=http://$(kubectl get secret --namespace demand-backup-sharded-10660 minio-service -o jsonpath="{.data.rootUser}" | base64 --decode):$(kubectl get secret --namespace demand-backup-sharded-10660 minio-service -o jsonpath="{.data.rootPassword}" | base64 --decode)@localhost:9000

  3. mc ls minio-service-local
waiting for pod/minio-service-57dd49b-vzwzr to be ready.OK
service/minio-service created
make_bucket: operator-testing
pod "aws-cli" deleted
If you don't see a command prompt, try pressing enter.
warning: couldn't attach to pod/aws-cli, falling back to streaming logs: Internal error occurred: error attaching to container: container is in CONTAINER_EXITED state


-----------------------------------------------------------------------------------
create first PSMDB cluster
-----------------------------------------------------------------------------------

secret/some-users created
deployment.apps/psmdb-client created


-----------------------------------------------------------------------------------
create secrets for cloud storages
-----------------------------------------------------------------------------------

secret/minio-secret created
secret/aws-s3-secret created
secret/gcp-cs-secret created
secret/azure-secret created
runtimeclass.node.k8s.io/container-rc unchanged
perconaservermongodb.psmdb.percona.com/some-name created


-----------------------------------------------------------------------------------
check if all 3 Pods started
-----------------------------------------------------------------------------------

waiting for pod/some-name-rs0-0 to be ready............OK
waiting for pod/some-name-rs0-1 to be ready......OK
waiting for pod/some-name-rs0-2 to be ready.....OK
Waiting for cluster readyness..........................................
waiting for pod/some-name-cfg-0 to be ready.OK
waiting for pod/some-name-cfg-1 to be ready.OK
waiting for pod/some-name-cfg-2 to be ready.OK
waiting for pod/some-name-mongos-0 to be ready.OK
waiting for pod/some-name-mongos-1 to be ready.OK
waiting for pod/some-name-mongos-2 to be ready.OK
Waiting for cluster readyness


-----------------------------------------------------------------------------------
check if service and statefulset created with expected config
-----------------------------------------------------------------------------------

secret/some-name-mongos created
waiting for pod/some-name-mongos-0 to be ready.OK
waiting for pod/some-name-mongos-1 to be ready.OK
waiting for pod/some-name-mongos-2 to be ready.OK
Waiting for cluster readyness.................................................................................................


-----------------------------------------------------------------------------------
write data, read from all
-----------------------------------------------------------------------------------

Percona Server for MongoDB shell version v4.4.29-28
connecting to: mongodb://some-name-mongos.demand-backup-sharded-10660.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb
Implicit session: session { "id" : UUID("eb35e043-3dbe-4aee-900f-b1974e81309a") }
Percona Server for MongoDB server version: v7.0.8-5
WARNING: shell and server versions do not match
Successfully added user: {
	"user" : "myApp",
	"roles" : [
		{
			"db" : "myApp",
			"role" : "readWrite"
		},
		{
			"db" : "myApp1",
			"role" : "readWrite"
		},
		{
			"db" : "myApp2",
			"role" : "readWrite"
		}
	]
}
bye
Percona Server for MongoDB shell version v4.4.29-28
connecting to: mongodb://some-name-mongos.demand-backup-sharded-10660.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb
Implicit session: session { "id" : UUID("6ba63291-414f-4299-8efb-129aa3f9dbad") }
Percona Server for MongoDB server version: v7.0.8-5
WARNING: shell and server versions do not match
{
	"ok" : 1,
	"$clusterTime" : {
		"clusterTime" : Timestamp(1715064930, 16),
		"signature" : {
			"hash" : BinData(0,"RegdrSW7UGKnNHsSFkjvwUXo82I="),
			"keyId" : NumberLong("7366146036814839830")
		}
	},
	"operationTime" : Timestamp(1715064930, 10)
}
bye
Percona Server for MongoDB shell version v4.4.29-28
connecting to: mongodb://some-name-mongos.demand-backup-sharded-10660.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb
Implicit session: session { "id" : UUID("ec5f95d4-ee17-4f88-8a45-6c058bef149c") }
Percona Server for MongoDB server version: v7.0.8-5
WARNING: shell and server versions do not match
{
	"ok" : 1,
	"$clusterTime" : {
		"clusterTime" : Timestamp(1715064934, 12),
		"signature" : {
			"hash" : BinData(0,"+hppMlB/GFI1+YWaq9A0x66eu2M="),
			"keyId" : NumberLong("7366146036814839830")
		}
	},
	"operationTime" : Timestamp(1715064934, 7)
}
bye
Percona Server for MongoDB shell version v4.4.29-28
connecting to: mongodb://some-name-mongos.demand-backup-sharded-10660.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb
Implicit session: session { "id" : UUID("1a747dc6-3f37-418b-82e4-c1a177bacfe8") }
Percona Server for MongoDB server version: v7.0.8-5
WARNING: shell and server versions do not match
{
	"ok" : 1,
	"$clusterTime" : {
		"clusterTime" : Timestamp(1715064937, 11),
		"signature" : {
			"hash" : BinData(0,"r/LeXeBSFgGhisxtHah/7jtKktU="),
			"keyId" : NumberLong("7366146036814839830")
		}
	},
	"operationTime" : Timestamp(1715064937, 6)
}
bye
Percona Server for MongoDB shell version v4.4.29-28
connecting to: mongodb://some-name-mongos.demand-backup-sharded-10660.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb
Implicit session: session { "id" : UUID("27d83ff9-2d6b-46ac-af82-e1ceb692f9f0") }
Percona Server for MongoDB server version: v7.0.8-5
WARNING: shell and server versions do not match
switched to db myApp
WriteResult({ "nInserted" : 1 })
bye
Percona Server for MongoDB shell version v4.4.29-28
connecting to: mongodb://some-name-mongos.demand-backup-sharded-10660.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb
Implicit session: session { "id" : UUID("ac683d6f-c346-4c72-97cb-460d29c4469c") }
Percona Server for MongoDB server version: v7.0.8-5
WARNING: shell and server versions do not match
switched to db myApp1
WriteResult({ "nInserted" : 1 })
bye
Percona Server for MongoDB shell version v4.4.29-28
connecting to: mongodb://some-name-mongos.demand-backup-sharded-10660.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb
Implicit session: session { "id" : UUID("fe994343-70ab-436a-8b38-41c1c64386b0") }
Percona Server for MongoDB server version: v7.0.8-5
WARNING: shell and server versions do not match
switched to db myApp2
WriteResult({ "nInserted" : 1 })
bye
some-name-rs0-0
some-name-rs0-1
some-name-rs0-2
some-name-rs1-0
some-name-rs1-1
some-name-rs1-2
some-name-rs2-0
some-name-rs2-1
some-name-rs2-2


-----------------------------------------------------------------------------------
run backups
-----------------------------------------------------------------------------------



-----------------------------------------------------------------------------------
run backup backup-minio
-----------------------------------------------------------------------------------

perconaservermongodbbackup.psmdb.percona.com/backup-minio created


-----------------------------------------------------------------------------------
run backup backup-aws-s3
-----------------------------------------------------------------------------------

perconaservermongodbbackup.psmdb.percona.com/backup-aws-s3 created


-----------------------------------------------------------------------------------
run backup backup-gcp-cs
-----------------------------------------------------------------------------------

perconaservermongodbbackup.psmdb.percona.com/backup-gcp-cs created


-----------------------------------------------------------------------------------
run backup backup-azure-blob
-----------------------------------------------------------------------------------

perconaservermongodbbackup.psmdb.percona.com/backup-azure-blob created
backup-aws-s3.................................
backup-gcp-cs..................................
backup-azure-blob.................
backup-minio.


-----------------------------------------------------------------------------------
check backup and restore -- aws-s3
-----------------------------------------------------------------------------------

Percona Server for MongoDB shell version v4.4.29-28
connecting to: mongodb://some-name-mongos.demand-backup-sharded-10660.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb
Implicit session: session { "id" : UUID("494afdcf-c7cf-45bf-bde9-d363efc6ceaf") }
Percona Server for MongoDB server version: v7.0.8-5
WARNING: shell and server versions do not match
switched to db myApp
WriteResult({ "nInserted" : 1 })
bye
Percona Server for MongoDB shell version v4.4.29-28
connecting to: mongodb://some-name-mongos.demand-backup-sharded-10660.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb
Implicit session: session { "id" : UUID("2970b296-0485-4bb4-88fe-d0c264a24667") }
Percona Server for MongoDB server version: v7.0.8-5
WARNING: shell and server versions do not match
switched to db myApp1
WriteResult({ "nInserted" : 1 })
bye
Percona Server for MongoDB shell version v4.4.29-28
connecting to: mongodb://some-name-mongos.demand-backup-sharded-10660.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb
Implicit session: session { "id" : UUID("5f4e3fd2-910d-4a67-b65c-9ba14224c33d") }
Percona Server for MongoDB server version: v7.0.8-5
WARNING: shell and server versions do not match
switched to db myApp2
WriteResult({ "nInserted" : 1 })
bye
perconaservermongodbrestore.psmdb.percona.com/restore-backup-aws-s3 created
waiting psmdb-restore/backup-aws-s3 to reach ready state.............
+ '[' 1 -eq 1 ']'
+ wait_cluster_consistency some-name
+ local cluster_name=some-name
+ local wait_time=32
+ retry=0
+ sleep 7
+ echo -n 'waiting for cluster readyness'
waiting for cluster readyness++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.1KrI47h9tB
+++ mktemp
++ local LAST_ERR=/tmp/tmp.dM9qssdKOi
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 0 ']'
++ break
++ cat /tmp/tmp.1KrI47h9tB
++ cat /tmp/tmp.dM9qssdKOi
++ rm /tmp/tmp.1KrI47h9tB /tmp/tmp.dM9qssdKOi
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 1 -ge 32 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.y0J6MSKRDm
+++ mktemp
++ local LAST_ERR=/tmp/tmp.TctnzvDgwL
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 0 ']'
++ break
++ cat /tmp/tmp.y0J6MSKRDm
++ cat /tmp/tmp.TctnzvDgwL
++ rm /tmp/tmp.y0J6MSKRDm /tmp/tmp.TctnzvDgwL
++ return 0
+ [[ error == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 2 -ge 32 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.5otI4MQVZ2
+++ mktemp
++ local LAST_ERR=/tmp/tmp.R4pBg9A9Pt
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 0 ']'
++ break
++ cat /tmp/tmp.5otI4MQVZ2
++ cat /tmp/tmp.R4pBg9A9Pt
++ rm /tmp/tmp.5otI4MQVZ2 /tmp/tmp.R4pBg9A9Pt
++ return 0
+ [[ error == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 3 -ge 32 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.FvBgm0dGG9
+++ mktemp
++ local LAST_ERR=/tmp/tmp.5i1DhDMDkK
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 0 ']'
++ break
++ cat /tmp/tmp.FvBgm0dGG9
++ cat /tmp/tmp.5i1DhDMDkK
++ rm /tmp/tmp.FvBgm0dGG9 /tmp/tmp.5i1DhDMDkK
++ return 0
+ [[ error == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 4 -ge 32 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.t8wE4JkrBm
+++ mktemp
++ local LAST_ERR=/tmp/tmp.MUTtEfCK4q
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 0 ']'
++ break
++ cat /tmp/tmp.t8wE4JkrBm
++ cat /tmp/tmp.MUTtEfCK4q
++ rm /tmp/tmp.t8wE4JkrBm /tmp/tmp.MUTtEfCK4q
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 5 -ge 32 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.L9dhGhPXAr
+++ mktemp
++ local LAST_ERR=/tmp/tmp.PNeHUCO53A
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 0 ']'
++ break
++ cat /tmp/tmp.L9dhGhPXAr
++ cat /tmp/tmp.PNeHUCO53A
++ rm /tmp/tmp.L9dhGhPXAr /tmp/tmp.PNeHUCO53A
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 6 -ge 32 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.4Hxv5Ej3ZA
+++ mktemp
++ local LAST_ERR=/tmp/tmp.HfhSN8N2nd
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 0 ']'
++ break
++ cat /tmp/tmp.4Hxv5Ej3ZA
++ cat /tmp/tmp.HfhSN8N2nd
++ rm /tmp/tmp.4Hxv5Ej3ZA /tmp/tmp.HfhSN8N2nd
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 7 -ge 32 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.5XHSXQ1a3i
+++ mktemp
++ local LAST_ERR=/tmp/tmp.u55Kqs3Ig2
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 0 ']'
++ break
++ cat /tmp/tmp.5XHSXQ1a3i
++ cat /tmp/tmp.u55Kqs3Ig2
++ rm /tmp/tmp.5XHSXQ1a3i /tmp/tmp.u55Kqs3Ig2
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 8 -ge 32 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.YSMBvl9ldJ
+++ mktemp
++ local LAST_ERR=/tmp/tmp.spxm3nKXmE
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 0 ']'
++ break
++ cat /tmp/tmp.YSMBvl9ldJ
++ cat /tmp/tmp.spxm3nKXmE
++ rm /tmp/tmp.YSMBvl9ldJ /tmp/tmp.spxm3nKXmE
++ return 0
+ [[ ready == \r\e\a\d\y ]]
+ check_data
+ local postfix=
++ seq 0 2
+ for i in '$(seq 0 2)'
+ compare_mongos_cmd find myApp:myPass@some-name-mongos.demand-backup-sharded-10660 '' .svc.cluster.local myApp test
+ local command=find
+ local uri=myApp:myPass@some-name-mongos.demand-backup-sharded-10660
+ local postfix=
+ local suffix=.svc.cluster.local
+ local database=myApp
+ local collection=test
+ egrep -v 'I NETWORK|W NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:'
+ /usr/bin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
+ run_mongos 'use myApp\n db.test.find()' myApp:myPass@some-name-mongos.demand-backup-sharded-10660 mongodb .svc.cluster.local
+ local 'command=use myApp\n db.test.find()'
+ local uri=myApp:myPass@some-name-mongos.demand-backup-sharded-10660
+ local driver=mongodb
+ local suffix=.svc.cluster.local
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.2iAQ6Ngxby
+++ mktemp
++ local LAST_ERR=/tmp/tmp.jzSujYYnvO
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 0 ']'
++ break
++ cat /tmp/tmp.2iAQ6Ngxby
++ cat /tmp/tmp.jzSujYYnvO
++ rm /tmp/tmp.2iAQ6Ngxby /tmp/tmp.jzSujYYnvO
++ return 0
+ local client_container=psmdb-client-5dc94d5b48-7tmsd
+ local mongo_flag=
+ kubectl_bin exec psmdb-client-5dc94d5b48-7tmsd -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-10660.svc.cluster.local/admin '
++ mktemp
+ local LAST_OUT=/tmp/tmp.9o3S2Tm0tm
++ mktemp
+ local LAST_ERR=/tmp/tmp.fsEY7kxasV
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl exec psmdb-client-5dc94d5b48-7tmsd -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-10660.svc.cluster.local/admin '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 0 ']'
+ break
+ cat /tmp/tmp.9o3S2Tm0tm
+ cat /tmp/tmp.fsEY7kxasV
+ rm /tmp/tmp.9o3S2Tm0tm /tmp/tmp.fsEY7kxasV
+ return 0
+ diff /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1544/e2e-tests/demand-backup-sharded/compare/find.json /tmp/tmp.QarcRDZZGq/find
+ for i in '$(seq 0 2)'
+ compare_mongos_cmd find myApp:myPass@some-name-mongos.demand-backup-sharded-10660 1 .svc.cluster.local myApp1 test
+ local command=find
+ local uri=myApp:myPass@some-name-mongos.demand-backup-sharded-10660
+ local postfix=1
+ local suffix=.svc.cluster.local
+ local database=myApp1
+ local collection=test
+ run_mongos 'use myApp1\n db.test.find()' myApp:myPass@some-name-mongos.demand-backup-sharded-10660 mongodb .svc.cluster.local
+ local 'command=use myApp1\n db.test.find()'
+ local uri=myApp:myPass@some-name-mongos.demand-backup-sharded-10660
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ /usr/bin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
+ egrep -v 'I NETWORK|W NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:'
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.Pa92NSCu6e
+++ mktemp
++ local LAST_ERR=/tmp/tmp.9bef2NsOBy
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 0 ']'
++ break
++ cat /tmp/tmp.Pa92NSCu6e
++ cat /tmp/tmp.9bef2NsOBy
++ rm /tmp/tmp.Pa92NSCu6e /tmp/tmp.9bef2NsOBy
++ return 0
+ local client_container=psmdb-client-5dc94d5b48-7tmsd
+ local mongo_flag=
+ kubectl_bin exec psmdb-client-5dc94d5b48-7tmsd -- bash -c 'printf '\''use myApp1\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-10660.svc.cluster.local/admin '
++ mktemp
+ local LAST_OUT=/tmp/tmp.xfYiAsMoYA
++ mktemp
+ local LAST_ERR=/tmp/tmp.d75BXnTtkN
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl exec psmdb-client-5dc94d5b48-7tmsd -- bash -c 'printf '\''use myApp1\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-10660.svc.cluster.local/admin '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 0 ']'
+ break
+ cat /tmp/tmp.xfYiAsMoYA
+ cat /tmp/tmp.d75BXnTtkN
+ rm /tmp/tmp.xfYiAsMoYA /tmp/tmp.d75BXnTtkN
+ return 0
+ diff /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1544/e2e-tests/demand-backup-sharded/compare/find1.json /tmp/tmp.QarcRDZZGq/find1
+ for i in '$(seq 0 2)'
+ compare_mongos_cmd find myApp:myPass@some-name-mongos.demand-backup-sharded-10660 2 .svc.cluster.local myApp2 test
+ local command=find
+ local uri=myApp:myPass@some-name-mongos.demand-backup-sharded-10660
+ local postfix=2
+ local suffix=.svc.cluster.local
+ local database=myApp2
+ local collection=test
+ egrep -v 'I NETWORK|W NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:'
+ /usr/bin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
+ run_mongos 'use myApp2\n db.test.find()' myApp:myPass@some-name-mongos.demand-backup-sharded-10660 mongodb .svc.cluster.local
+ local 'command=use myApp2\n db.test.find()'
+ local uri=myApp:myPass@some-name-mongos.demand-backup-sharded-10660
+ local driver=mongodb
+ local suffix=.svc.cluster.local
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.UsL9JyKIFz
+++ mktemp
++ local LAST_ERR=/tmp/tmp.SajxQUsQeJ
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 0 ']'
++ break
++ cat /tmp/tmp.UsL9JyKIFz
++ cat /tmp/tmp.SajxQUsQeJ
++ rm /tmp/tmp.UsL9JyKIFz /tmp/tmp.SajxQUsQeJ
++ return 0
+ local client_container=psmdb-client-5dc94d5b48-7tmsd
+ local mongo_flag=
+ kubectl_bin exec psmdb-client-5dc94d5b48-7tmsd -- bash -c 'printf '\''use myApp2\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-10660.svc.cluster.local/admin '
++ mktemp
+ local LAST_OUT=/tmp/tmp.QQlfhFZd07
++ mktemp
+ local LAST_ERR=/tmp/tmp.cEgh2N5WZK
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl exec psmdb-client-5dc94d5b48-7tmsd -- bash -c 'printf '\''use myApp2\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-10660.svc.cluster.local/admin '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 0 ']'
+ break
+ cat /tmp/tmp.QQlfhFZd07
+ cat /tmp/tmp.cEgh2N5WZK
+ rm /tmp/tmp.QQlfhFZd07 /tmp/tmp.cEgh2N5WZK
+ return 0
+ diff /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1544/e2e-tests/demand-backup-sharded/compare/find2.json /tmp/tmp.QarcRDZZGq/find2
+ desc 'check backup and restore -- gcp-cs'
+ set +o xtrace


-----------------------------------------------------------------------------------
check backup and restore -- gcp-cs
-----------------------------------------------------------------------------------

Percona Server for MongoDB shell version v4.4.29-28
connecting to: mongodb://some-name-mongos.demand-backup-sharded-10660.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb
Implicit session: session { "id" : UUID("b1b468cb-ee24-4dab-b996-4c4ac5be4e1a") }
Percona Server for MongoDB server version: v7.0.8-5
WARNING: shell and server versions do not match
switched to db myApp
WriteResult({ "nInserted" : 1 })
bye
Percona Server for MongoDB shell version v4.4.29-28
connecting to: mongodb://some-name-mongos.demand-backup-sharded-10660.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb
Implicit session: session { "id" : UUID("8c2497ec-4235-4703-b5b8-3bffba70ffeb") }
Percona Server for MongoDB server version: v7.0.8-5
WARNING: shell and server versions do not match
switched to db myApp1
WriteResult({ "nInserted" : 1 })
bye
Percona Server for MongoDB shell version v4.4.29-28
connecting to: mongodb://some-name-mongos.demand-backup-sharded-10660.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb
Implicit session: session { "id" : UUID("136e29de-ac93-4be3-845d-e20c2de0c303") }
Percona Server for MongoDB server version: v7.0.8-5
WARNING: shell and server versions do not match
switched to db myApp2
WriteResult({ "nInserted" : 1 })
bye
perconaservermongodbrestore.psmdb.percona.com/restore-backup-gcp-cs created
waiting psmdb-restore/backup-gcp-cs to reach ready state...........
+ '[' 1 -eq 1 ']'
+ wait_cluster_consistency some-name
+ local cluster_name=some-name
+ local wait_time=32
+ retry=0
+ sleep 7
+ echo -n 'waiting for cluster readyness'
waiting for cluster readyness++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.FZ7WIGaFBu
+++ mktemp
++ local LAST_ERR=/tmp/tmp.0SmWiEpqgt
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 0 ']'
++ break
++ cat /tmp/tmp.FZ7WIGaFBu
++ cat /tmp/tmp.0SmWiEpqgt
++ rm /tmp/tmp.FZ7WIGaFBu /tmp/tmp.0SmWiEpqgt
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 1 -ge 32 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.QOVz48dM17
+++ mktemp
++ local LAST_ERR=/tmp/tmp.nhyn2IitrA
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 0 ']'
++ break
++ cat /tmp/tmp.QOVz48dM17
++ cat /tmp/tmp.nhyn2IitrA
++ rm /tmp/tmp.QOVz48dM17 /tmp/tmp.nhyn2IitrA
++ return 0
+ [[ error == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 2 -ge 32 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.pwyZzNG7dS
+++ mktemp
++ local LAST_ERR=/tmp/tmp.Z4UfMYyriT
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 0 ']'
++ break
++ cat /tmp/tmp.pwyZzNG7dS
++ cat /tmp/tmp.Z4UfMYyriT
++ rm /tmp/tmp.pwyZzNG7dS /tmp/tmp.Z4UfMYyriT
++ return 0
+ [[ error == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 3 -ge 32 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.DNRurlzfKV
+++ mktemp
++ local LAST_ERR=/tmp/tmp.i9TiAz0ifG
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 0 ']'
++ break
++ cat /tmp/tmp.DNRurlzfKV
++ cat /tmp/tmp.i9TiAz0ifG
++ rm /tmp/tmp.DNRurlzfKV /tmp/tmp.i9TiAz0ifG
++ return 0
+ [[ error == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 4 -ge 32 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.ITFpA47nfS
+++ mktemp
++ local LAST_ERR=/tmp/tmp.UOsVsjiIlD
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 0 ']'
++ break
++ cat /tmp/tmp.ITFpA47nfS
++ cat /tmp/tmp.UOsVsjiIlD
++ rm /tmp/tmp.ITFpA47nfS /tmp/tmp.UOsVsjiIlD
++ return 0
+ [[ error == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 5 -ge 32 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.7XlQ3bur1A
+++ mktemp
++ local LAST_ERR=/tmp/tmp.ddkGjj8wuc
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 0 ']'
++ break
++ cat /tmp/tmp.7XlQ3bur1A
++ cat /tmp/tmp.ddkGjj8wuc
++ rm /tmp/tmp.7XlQ3bur1A /tmp/tmp.ddkGjj8wuc
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 6 -ge 32 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.nb26bQDYIb
+++ mktemp
++ local LAST_ERR=/tmp/tmp.a4gxopa5GW
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 0 ']'
++ break
++ cat /tmp/tmp.nb26bQDYIb
++ cat /tmp/tmp.a4gxopa5GW
++ rm /tmp/tmp.nb26bQDYIb /tmp/tmp.a4gxopa5GW
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 7 -ge 32 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.yI82cmLcCO
+++ mktemp
++ local LAST_ERR=/tmp/tmp.Mk0KxIclEJ
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 0 ']'
++ break
++ cat /tmp/tmp.yI82cmLcCO
++ cat /tmp/tmp.Mk0KxIclEJ
++ rm /tmp/tmp.yI82cmLcCO /tmp/tmp.Mk0KxIclEJ
++ return 0
+ [[ ready == \r\e\a\d\y ]]
+ check_data
+ local postfix=
++ seq 0 2
+ for i in '$(seq 0 2)'
+ compare_mongos_cmd find myApp:myPass@some-name-mongos.demand-backup-sharded-10660 '' .svc.cluster.local myApp test
+ local command=find
+ local uri=myApp:myPass@some-name-mongos.demand-backup-sharded-10660
+ local postfix=
+ local suffix=.svc.cluster.local
+ local database=myApp
+ local collection=test
+ run_mongos 'use myApp\n db.test.find()' myApp:myPass@some-name-mongos.demand-backup-sharded-10660 mongodb .svc.cluster.local
+ egrep -v 'I NETWORK|W NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:'
+ local 'command=use myApp\n db.test.find()'
+ local uri=myApp:myPass@some-name-mongos.demand-backup-sharded-10660
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ /usr/bin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.fP6trwAY4I
+++ mktemp
++ local LAST_ERR=/tmp/tmp.grzW9aYl88
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 0 ']'
++ break
++ cat /tmp/tmp.fP6trwAY4I
++ cat /tmp/tmp.grzW9aYl88
++ rm /tmp/tmp.fP6trwAY4I /tmp/tmp.grzW9aYl88
++ return 0
+ local client_container=psmdb-client-5dc94d5b48-7tmsd
+ local mongo_flag=
+ kubectl_bin exec psmdb-client-5dc94d5b48-7tmsd -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-10660.svc.cluster.local/admin '
++ mktemp
+ local LAST_OUT=/tmp/tmp.us4KgWRedP
++ mktemp
+ local LAST_ERR=/tmp/tmp.GVcOpC90Po
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl exec psmdb-client-5dc94d5b48-7tmsd -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-10660.svc.cluster.local/admin '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 0 ']'
+ break
+ cat /tmp/tmp.us4KgWRedP
+ cat /tmp/tmp.GVcOpC90Po
+ rm /tmp/tmp.us4KgWRedP /tmp/tmp.GVcOpC90Po
+ return 0
+ diff /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1544/e2e-tests/demand-backup-sharded/compare/find.json /tmp/tmp.QarcRDZZGq/find
+ for i in '$(seq 0 2)'
+ compare_mongos_cmd find myApp:myPass@some-name-mongos.demand-backup-sharded-10660 1 .svc.cluster.local myApp1 test
+ local command=find
+ local uri=myApp:myPass@some-name-mongos.demand-backup-sharded-10660
+ local postfix=1
+ local suffix=.svc.cluster.local
+ local database=myApp1
+ local collection=test
+ egrep -v 'I NETWORK|W NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:'
+ run_mongos 'use myApp1\n db.test.find()' myApp:myPass@some-name-mongos.demand-backup-sharded-10660 mongodb .svc.cluster.local
+ local 'command=use myApp1\n db.test.find()'
+ local uri=myApp:myPass@some-name-mongos.demand-backup-sharded-10660
+ local driver=mongodb
+ local suffix=.svc.cluster.local
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+ /usr/bin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.HbPveogfRy
+++ mktemp
++ local LAST_ERR=/tmp/tmp.ZJtkGxdZBb
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 0 ']'
++ break
++ cat /tmp/tmp.HbPveogfRy
++ cat /tmp/tmp.ZJtkGxdZBb
++ rm /tmp/tmp.HbPveogfRy /tmp/tmp.ZJtkGxdZBb
++ return 0
+ local client_container=psmdb-client-5dc94d5b48-7tmsd
+ local mongo_flag=
+ kubectl_bin exec psmdb-client-5dc94d5b48-7tmsd -- bash -c 'printf '\''use myApp1\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-10660.svc.cluster.local/admin '
++ mktemp
+ local LAST_OUT=/tmp/tmp.0n8EjRMePT
++ mktemp
+ local LAST_ERR=/tmp/tmp.FeM83T1uCd
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl exec psmdb-client-5dc94d5b48-7tmsd -- bash -c 'printf '\''use myApp1\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-10660.svc.cluster.local/admin '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 0 ']'
+ break
+ cat /tmp/tmp.0n8EjRMePT
+ cat /tmp/tmp.FeM83T1uCd
+ rm /tmp/tmp.0n8EjRMePT /tmp/tmp.FeM83T1uCd
+ return 0
+ diff /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1544/e2e-tests/demand-backup-sharded/compare/find1.json /tmp/tmp.QarcRDZZGq/find1
+ for i in '$(seq 0 2)'
+ compare_mongos_cmd find myApp:myPass@some-name-mongos.demand-backup-sharded-10660 2 .svc.cluster.local myApp2 test
+ local command=find
+ local uri=myApp:myPass@some-name-mongos.demand-backup-sharded-10660
+ local postfix=2
+ local suffix=.svc.cluster.local
+ local database=myApp2
+ local collection=test
+ run_mongos 'use myApp2\n db.test.find()' myApp:myPass@some-name-mongos.demand-backup-sharded-10660 mongodb .svc.cluster.local
+ local 'command=use myApp2\n db.test.find()'
+ local uri=myApp:myPass@some-name-mongos.demand-backup-sharded-10660
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ /usr/bin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
+ egrep -v 'I NETWORK|W NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:'
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.QH9sIlQ5Bn
+++ mktemp
++ local LAST_ERR=/tmp/tmp.ILnVMghMfX
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 0 ']'
++ break
++ cat /tmp/tmp.QH9sIlQ5Bn
++ cat /tmp/tmp.ILnVMghMfX
++ rm /tmp/tmp.QH9sIlQ5Bn /tmp/tmp.ILnVMghMfX
++ return 0
+ local client_container=psmdb-client-5dc94d5b48-7tmsd
+ local mongo_flag=
+ kubectl_bin exec psmdb-client-5dc94d5b48-7tmsd -- bash -c 'printf '\''use myApp2\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-10660.svc.cluster.local/admin '
++ mktemp
+ local LAST_OUT=/tmp/tmp.dDFFOvSd80
++ mktemp
+ local LAST_ERR=/tmp/tmp.J1vRsiTuZs
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl exec psmdb-client-5dc94d5b48-7tmsd -- bash -c 'printf '\''use myApp2\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-10660.svc.cluster.local/admin '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 0 ']'
+ break
+ cat /tmp/tmp.dDFFOvSd80
+ cat /tmp/tmp.J1vRsiTuZs
+ rm /tmp/tmp.dDFFOvSd80 /tmp/tmp.J1vRsiTuZs
+ return 0
+ diff /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1544/e2e-tests/demand-backup-sharded/compare/find2.json /tmp/tmp.QarcRDZZGq/find2
+ desc 'check backup and restore -- azure-blob'
+ set +o xtrace


-----------------------------------------------------------------------------------
check backup and restore -- azure-blob
-----------------------------------------------------------------------------------

Percona Server for MongoDB shell version v4.4.29-28
connecting to: mongodb://some-name-mongos.demand-backup-sharded-10660.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb
Implicit session: session { "id" : UUID("563d7c45-9374-4b0e-b7d6-aed79b993e08") }
Percona Server for MongoDB server version: v7.0.8-5
WARNING: shell and server versions do not match
switched to db myApp
WriteResult({ "nInserted" : 1 })
bye
Percona Server for MongoDB shell version v4.4.29-28
connecting to: mongodb://some-name-mongos.demand-backup-sharded-10660.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb
Implicit session: session { "id" : UUID("56902aa1-a628-471a-b8e8-f1b6bc9406cf") }
Percona Server for MongoDB server version: v7.0.8-5
WARNING: shell and server versions do not match
switched to db myApp1
WriteResult({ "nInserted" : 1 })
bye
Percona Server for MongoDB shell version v4.4.29-28
connecting to: mongodb://some-name-mongos.demand-backup-sharded-10660.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb
Implicit session: session { "id" : UUID("8462973b-553a-4bf1-adf6-400499f14314") }
Percona Server for MongoDB server version: v7.0.8-5
WARNING: shell and server versions do not match
switched to db myApp2
WriteResult({ "nInserted" : 1 })
bye
perconaservermongodbrestore.psmdb.percona.com/restore-backup-azure-blob created
waiting psmdb-restore/backup-azure-blob to reach ready state............
+ '[' 1 -eq 1 ']'
+ wait_cluster_consistency some-name
+ local cluster_name=some-name
+ local wait_time=32
+ retry=0
+ sleep 7
+ echo -n 'waiting for cluster readyness'
waiting for cluster readyness++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.PBE6SbqINb
+++ mktemp
++ local LAST_ERR=/tmp/tmp.3JdRVmuLfF
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 0 ']'
++ break
++ cat /tmp/tmp.PBE6SbqINb
++ cat /tmp/tmp.3JdRVmuLfF
++ rm /tmp/tmp.PBE6SbqINb /tmp/tmp.3JdRVmuLfF
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 1 -ge 32 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.puYrhHSlsf
+++ mktemp
++ local LAST_ERR=/tmp/tmp.bFhhVzx203
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 0 ']'
++ break
++ cat /tmp/tmp.puYrhHSlsf
++ cat /tmp/tmp.bFhhVzx203
++ rm /tmp/tmp.puYrhHSlsf /tmp/tmp.bFhhVzx203
++ return 0
+ [[ error == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 2 -ge 32 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.HlzzqwriuC
+++ mktemp
++ local LAST_ERR=/tmp/tmp.1yD8kzoR4x
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 0 ']'
++ break
++ cat /tmp/tmp.HlzzqwriuC
++ cat /tmp/tmp.1yD8kzoR4x
++ rm /tmp/tmp.HlzzqwriuC /tmp/tmp.1yD8kzoR4x
++ return 0
+ [[ error == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 3 -ge 32 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.woOB3XPhQj
+++ mktemp
++ local LAST_ERR=/tmp/tmp.f0MJGSzLlZ
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 0 ']'
++ break
++ cat /tmp/tmp.woOB3XPhQj
++ cat /tmp/tmp.f0MJGSzLlZ
++ rm /tmp/tmp.woOB3XPhQj /tmp/tmp.f0MJGSzLlZ
++ return 0
+ [[ error == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 4 -ge 32 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.J7xO3VKlJF
+++ mktemp
++ local LAST_ERR=/tmp/tmp.owjKAJtWer
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 0 ']'
++ break
++ cat /tmp/tmp.J7xO3VKlJF
++ cat /tmp/tmp.owjKAJtWer
++ rm /tmp/tmp.J7xO3VKlJF /tmp/tmp.owjKAJtWer
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 5 -ge 32 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.M2bpy6eXjq
+++ mktemp
++ local LAST_ERR=/tmp/tmp.S7k56rCEc5
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 0 ']'
++ break
++ cat /tmp/tmp.M2bpy6eXjq
++ cat /tmp/tmp.S7k56rCEc5
++ rm /tmp/tmp.M2bpy6eXjq /tmp/tmp.S7k56rCEc5
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 6 -ge 32 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.DXTvAgcsyU
+++ mktemp
++ local LAST_ERR=/tmp/tmp.qr1rnkTozX
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 0 ']'
++ break
++ cat /tmp/tmp.DXTvAgcsyU
++ cat /tmp/tmp.qr1rnkTozX
++ rm /tmp/tmp.DXTvAgcsyU /tmp/tmp.qr1rnkTozX
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 7 -ge 32 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.F6onfoHwBf
+++ mktemp
++ local LAST_ERR=/tmp/tmp.kppZwGnb7H
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 0 ']'
++ break
++ cat /tmp/tmp.F6onfoHwBf
++ cat /tmp/tmp.kppZwGnb7H
++ rm /tmp/tmp.F6onfoHwBf /tmp/tmp.kppZwGnb7H
++ return 0
+ [[ ready == \r\e\a\d\y ]]
+ check_data
+ local postfix=
++ seq 0 2
+ for i in '$(seq 0 2)'
+ compare_mongos_cmd find myApp:myPass@some-name-mongos.demand-backup-sharded-10660 '' .svc.cluster.local myApp test
+ local command=find
+ local uri=myApp:myPass@some-name-mongos.demand-backup-sharded-10660
+ local postfix=
+ local suffix=.svc.cluster.local
+ local database=myApp
+ local collection=test
+ /usr/bin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
+ run_mongos 'use myApp\n db.test.find()' myApp:myPass@some-name-mongos.demand-backup-sharded-10660 mongodb .svc.cluster.local
+ local 'command=use myApp\n db.test.find()'
+ local uri=myApp:myPass@some-name-mongos.demand-backup-sharded-10660
+ local driver=mongodb
+ local suffix=.svc.cluster.local
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
+ egrep -v 'I NETWORK|W NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:'
++ local LAST_OUT=/tmp/tmp.2xXBIDcdac
+++ mktemp
++ local LAST_ERR=/tmp/tmp.xGPbq0HjSO
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 0 ']'
++ break
++ cat /tmp/tmp.2xXBIDcdac
++ cat /tmp/tmp.xGPbq0HjSO
++ rm /tmp/tmp.2xXBIDcdac /tmp/tmp.xGPbq0HjSO
++ return 0
+ local client_container=psmdb-client-5dc94d5b48-7tmsd
+ local mongo_flag=
+ kubectl_bin exec psmdb-client-5dc94d5b48-7tmsd -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-10660.svc.cluster.local/admin '
++ mktemp
+ local LAST_OUT=/tmp/tmp.P9pVzZool0
++ mktemp
+ local LAST_ERR=/tmp/tmp.2TTrpcFlUB
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl exec psmdb-client-5dc94d5b48-7tmsd -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-10660.svc.cluster.local/admin '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 0 ']'
+ break
+ cat /tmp/tmp.P9pVzZool0
+ cat /tmp/tmp.2TTrpcFlUB
+ rm /tmp/tmp.P9pVzZool0 /tmp/tmp.2TTrpcFlUB
+ return 0
+ diff /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1544/e2e-tests/demand-backup-sharded/compare/find.json /tmp/tmp.QarcRDZZGq/find
+ for i in '$(seq 0 2)'
+ compare_mongos_cmd find myApp:myPass@some-name-mongos.demand-backup-sharded-10660 1 .svc.cluster.local myApp1 test
+ local command=find
+ local uri=myApp:myPass@some-name-mongos.demand-backup-sharded-10660
+ local postfix=1
+ local suffix=.svc.cluster.local
+ local database=myApp1
+ local collection=test
+ egrep -v 'I NETWORK|W NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:'
+ /usr/bin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
+ run_mongos 'use myApp1\n db.test.find()' myApp:myPass@some-name-mongos.demand-backup-sharded-10660 mongodb .svc.cluster.local
+ local 'command=use myApp1\n db.test.find()'
+ local uri=myApp:myPass@some-name-mongos.demand-backup-sharded-10660
+ local driver=mongodb
+ local suffix=.svc.cluster.local
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.yI9FZZhgde
+++ mktemp
++ local LAST_ERR=/tmp/tmp.KkxqjaYeU1
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 0 ']'
++ break
++ cat /tmp/tmp.yI9FZZhgde
++ cat /tmp/tmp.KkxqjaYeU1
++ rm /tmp/tmp.yI9FZZhgde /tmp/tmp.KkxqjaYeU1
++ return 0
+ local client_container=psmdb-client-5dc94d5b48-7tmsd
+ local mongo_flag=
+ kubectl_bin exec psmdb-client-5dc94d5b48-7tmsd -- bash -c 'printf '\''use myApp1\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-10660.svc.cluster.local/admin '
++ mktemp
+ local LAST_OUT=/tmp/tmp.OnDfuzKMjp
++ mktemp
+ local LAST_ERR=/tmp/tmp.0V06sK7Zqq
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl exec psmdb-client-5dc94d5b48-7tmsd -- bash -c 'printf '\''use myApp1\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-10660.svc.cluster.local/admin '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 0 ']'
+ break
+ cat /tmp/tmp.OnDfuzKMjp
+ cat /tmp/tmp.0V06sK7Zqq
+ rm /tmp/tmp.OnDfuzKMjp /tmp/tmp.0V06sK7Zqq
+ return 0
+ diff /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1544/e2e-tests/demand-backup-sharded/compare/find1.json /tmp/tmp.QarcRDZZGq/find1
+ for i in '$(seq 0 2)'
+ compare_mongos_cmd find myApp:myPass@some-name-mongos.demand-backup-sharded-10660 2 .svc.cluster.local myApp2 test
+ local command=find
+ local uri=myApp:myPass@some-name-mongos.demand-backup-sharded-10660
+ local postfix=2
+ local suffix=.svc.cluster.local
+ local database=myApp2
+ local collection=test
+ egrep -v 'I NETWORK|W NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:'
+ run_mongos 'use myApp2\n db.test.find()' myApp:myPass@some-name-mongos.demand-backup-sharded-10660 mongodb .svc.cluster.local
+ /usr/bin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
+ local 'command=use myApp2\n db.test.find()'
+ local uri=myApp:myPass@some-name-mongos.demand-backup-sharded-10660
+ local driver=mongodb
+ local suffix=.svc.cluster.local
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.cFTwT7by8e
+++ mktemp
++ local LAST_ERR=/tmp/tmp.GRWH9BXOHU
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 0 ']'
++ break
++ cat /tmp/tmp.cFTwT7by8e
++ cat /tmp/tmp.GRWH9BXOHU
++ rm /tmp/tmp.cFTwT7by8e /tmp/tmp.GRWH9BXOHU
++ return 0
+ local client_container=psmdb-client-5dc94d5b48-7tmsd
+ local mongo_flag=
+ kubectl_bin exec psmdb-client-5dc94d5b48-7tmsd -- bash -c 'printf '\''use myApp2\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-10660.svc.cluster.local/admin '
++ mktemp
+ local LAST_OUT=/tmp/tmp.uNgKRJbNRV
++ mktemp
+ local LAST_ERR=/tmp/tmp.ZLzDMV8rCO
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl exec psmdb-client-5dc94d5b48-7tmsd -- bash -c 'printf '\''use myApp2\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-10660.svc.cluster.local/admin '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 0 ']'
+ break
+ cat /tmp/tmp.uNgKRJbNRV
+ cat /tmp/tmp.ZLzDMV8rCO
+ rm /tmp/tmp.uNgKRJbNRV /tmp/tmp.ZLzDMV8rCO
+ return 0
+ diff /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1544/e2e-tests/demand-backup-sharded/compare/find2.json /tmp/tmp.QarcRDZZGq/find2
+ desc 'check backup and restore -- minio'
+ set +o xtrace


-----------------------------------------------------------------------------------
check backup and restore -- minio
-----------------------------------------------------------------------------------

2024-05-07 06:56:29         55 myApp.test.gz
Percona Server for MongoDB shell version v4.4.29-28
connecting to: mongodb://some-name-mongos.demand-backup-sharded-10660.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb
Implicit session: session { "id" : UUID("de5c6df3-3a83-4ee1-a0c9-5438ae3d5ae0") }
Percona Server for MongoDB server version: v7.0.8-5
WARNING: shell and server versions do not match
switched to db myApp
WriteResult({ "nInserted" : 1 })
bye
Percona Server for MongoDB shell version v4.4.29-28
connecting to: mongodb://some-name-mongos.demand-backup-sharded-10660.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb
Implicit session: session { "id" : UUID("1396da06-2430-4693-82ae-f1341622f27f") }
Percona Server for MongoDB server version: v7.0.8-5
WARNING: shell and server versions do not match
switched to db myApp1
WriteResult({ "nInserted" : 1 })
bye
Percona Server for MongoDB shell version v4.4.29-28
connecting to: mongodb://some-name-mongos.demand-backup-sharded-10660.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb
Implicit session: session { "id" : UUID("1ae5d49b-7396-4cfd-b48e-2738648ea046") }
Percona Server for MongoDB server version: v7.0.8-5
WARNING: shell and server versions do not match
switched to db myApp2
WriteResult({ "nInserted" : 1 })
bye
perconaservermongodbrestore.psmdb.percona.com/restore-backup-minio created
waiting psmdb-restore/backup-minio to reach ready state...........
+ '[' 1 -eq 1 ']'
+ wait_cluster_consistency some-name
+ local cluster_name=some-name
+ local wait_time=32
+ retry=0
+ sleep 7
+ echo -n 'waiting for cluster readyness'
waiting for cluster readyness++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.pwArAeOqFn
+++ mktemp
++ local LAST_ERR=/tmp/tmp.fQaNx3vjxl
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 0 ']'
++ break
++ cat /tmp/tmp.pwArAeOqFn
++ cat /tmp/tmp.fQaNx3vjxl
++ rm /tmp/tmp.pwArAeOqFn /tmp/tmp.fQaNx3vjxl
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 1 -ge 32 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.eSS6xf0fC1
+++ mktemp
++ local LAST_ERR=/tmp/tmp.ukphpUAVk5
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 0 ']'
++ break
++ cat /tmp/tmp.eSS6xf0fC1
++ cat /tmp/tmp.ukphpUAVk5
++ rm /tmp/tmp.eSS6xf0fC1 /tmp/tmp.ukphpUAVk5
++ return 0
+ [[ error == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 2 -ge 32 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.IUKyKHkjXz
+++ mktemp
++ local LAST_ERR=/tmp/tmp.LzwFyJk5mV
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 0 ']'
++ break
++ cat /tmp/tmp.IUKyKHkjXz
++ cat /tmp/tmp.LzwFyJk5mV
++ rm /tmp/tmp.IUKyKHkjXz /tmp/tmp.LzwFyJk5mV
++ return 0
+ [[ error == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 3 -ge 32 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.G5cOY1lWYa
+++ mktemp
++ local LAST_ERR=/tmp/tmp.YkQ8xB6Ep5
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 0 ']'
++ break
++ cat /tmp/tmp.G5cOY1lWYa
++ cat /tmp/tmp.YkQ8xB6Ep5
++ rm /tmp/tmp.G5cOY1lWYa /tmp/tmp.YkQ8xB6Ep5
++ return 0
+ [[ error == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 4 -ge 32 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.HD4UxKtkGb
+++ mktemp
++ local LAST_ERR=/tmp/tmp.nDuBqFM8I8
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 0 ']'
++ break
++ cat /tmp/tmp.HD4UxKtkGb
++ cat /tmp/tmp.nDuBqFM8I8
++ rm /tmp/tmp.HD4UxKtkGb /tmp/tmp.nDuBqFM8I8
++ return 0
+ [[ error == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 5 -ge 32 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.QpkriQYqlk
+++ mktemp
++ local LAST_ERR=/tmp/tmp.jYEPCLqnJ9
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 0 ']'
++ break
++ cat /tmp/tmp.QpkriQYqlk
++ cat /tmp/tmp.jYEPCLqnJ9
++ rm /tmp/tmp.QpkriQYqlk /tmp/tmp.jYEPCLqnJ9
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 6 -ge 32 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.WVR7vrOKp4
+++ mktemp
++ local LAST_ERR=/tmp/tmp.TbHPU4a3tq
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 0 ']'
++ break
++ cat /tmp/tmp.WVR7vrOKp4
++ cat /tmp/tmp.TbHPU4a3tq
++ rm /tmp/tmp.WVR7vrOKp4 /tmp/tmp.TbHPU4a3tq
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 7 -ge 32 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.66LRzgHF9l
+++ mktemp
++ local LAST_ERR=/tmp/tmp.l23OMcoOJD
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 0 ']'
++ break
++ cat /tmp/tmp.66LRzgHF9l
++ cat /tmp/tmp.l23OMcoOJD
++ rm /tmp/tmp.66LRzgHF9l /tmp/tmp.l23OMcoOJD
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 8 -ge 32 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.NBS5ruIfEm
+++ mktemp
++ local LAST_ERR=/tmp/tmp.K9NFC9db2n
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 0 ']'
++ break
++ cat /tmp/tmp.NBS5ruIfEm
++ cat /tmp/tmp.K9NFC9db2n
++ rm /tmp/tmp.NBS5ruIfEm /tmp/tmp.K9NFC9db2n
++ return 0
+ [[ ready == \r\e\a\d\y ]]
+ check_data
+ local postfix=
++ seq 0 2
+ for i in '$(seq 0 2)'
+ compare_mongos_cmd find myApp:myPass@some-name-mongos.demand-backup-sharded-10660 '' .svc.cluster.local myApp test
+ local command=find
+ local uri=myApp:myPass@some-name-mongos.demand-backup-sharded-10660
+ local postfix=
+ local suffix=.svc.cluster.local
+ local database=myApp
+ local collection=test
+ /usr/bin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
+ run_mongos 'use myApp\n db.test.find()' myApp:myPass@some-name-mongos.demand-backup-sharded-10660 mongodb .svc.cluster.local
+ local 'command=use myApp\n db.test.find()'
+ local uri=myApp:myPass@some-name-mongos.demand-backup-sharded-10660
+ local driver=mongodb
+ local suffix=.svc.cluster.local
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+ egrep -v 'I NETWORK|W NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.TcHKO8NYvj
+++ mktemp
++ local LAST_ERR=/tmp/tmp.76018ZMjX8
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 0 ']'
++ break
++ cat /tmp/tmp.TcHKO8NYvj
++ cat /tmp/tmp.76018ZMjX8
++ rm /tmp/tmp.TcHKO8NYvj /tmp/tmp.76018ZMjX8
++ return 0
+ local client_container=psmdb-client-5dc94d5b48-7tmsd
+ local mongo_flag=
+ kubectl_bin exec psmdb-client-5dc94d5b48-7tmsd -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-10660.svc.cluster.local/admin '
++ mktemp
+ local LAST_OUT=/tmp/tmp.P2wLx3PYr2
++ mktemp
+ local LAST_ERR=/tmp/tmp.uq9rAXQMEq
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl exec psmdb-client-5dc94d5b48-7tmsd -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-10660.svc.cluster.local/admin '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 0 ']'
+ break
+ cat /tmp/tmp.P2wLx3PYr2
+ cat /tmp/tmp.uq9rAXQMEq
+ rm /tmp/tmp.P2wLx3PYr2 /tmp/tmp.uq9rAXQMEq
+ return 0
+ diff /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1544/e2e-tests/demand-backup-sharded/compare/find.json /tmp/tmp.QarcRDZZGq/find
+ for i in '$(seq 0 2)'
+ compare_mongos_cmd find myApp:myPass@some-name-mongos.demand-backup-sharded-10660 1 .svc.cluster.local myApp1 test
+ local command=find
+ local uri=myApp:myPass@some-name-mongos.demand-backup-sharded-10660
+ local postfix=1
+ local suffix=.svc.cluster.local
+ local database=myApp1
+ local collection=test
+ egrep -v 'I NETWORK|W NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:'
+ /usr/bin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
+ run_mongos 'use myApp1\n db.test.find()' myApp:myPass@some-name-mongos.demand-backup-sharded-10660 mongodb .svc.cluster.local
+ local 'command=use myApp1\n db.test.find()'
+ local uri=myApp:myPass@some-name-mongos.demand-backup-sharded-10660
+ local driver=mongodb
+ local suffix=.svc.cluster.local
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.dzfjXDS2QD
+++ mktemp
++ local LAST_ERR=/tmp/tmp.hhtYk3piZ0
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 0 ']'
++ break
++ cat /tmp/tmp.dzfjXDS2QD
++ cat /tmp/tmp.hhtYk3piZ0
++ rm /tmp/tmp.dzfjXDS2QD /tmp/tmp.hhtYk3piZ0
++ return 0
+ local client_container=psmdb-client-5dc94d5b48-7tmsd
+ local mongo_flag=
+ kubectl_bin exec psmdb-client-5dc94d5b48-7tmsd -- bash -c 'printf '\''use myApp1\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-10660.svc.cluster.local/admin '
++ mktemp
+ local LAST_OUT=/tmp/tmp.4ZPaZFaZyI
++ mktemp
+ local LAST_ERR=/tmp/tmp.7CERoBIg2A
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl exec psmdb-client-5dc94d5b48-7tmsd -- bash -c 'printf '\''use myApp1\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-10660.svc.cluster.local/admin '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 0 ']'
+ break
+ cat /tmp/tmp.4ZPaZFaZyI
+ cat /tmp/tmp.7CERoBIg2A
+ rm /tmp/tmp.4ZPaZFaZyI /tmp/tmp.7CERoBIg2A
+ return 0
+ diff /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1544/e2e-tests/demand-backup-sharded/compare/find1.json /tmp/tmp.QarcRDZZGq/find1
+ for i in '$(seq 0 2)'
+ compare_mongos_cmd find myApp:myPass@some-name-mongos.demand-backup-sharded-10660 2 .svc.cluster.local myApp2 test
+ local command=find
+ local uri=myApp:myPass@some-name-mongos.demand-backup-sharded-10660
+ local postfix=2
+ local suffix=.svc.cluster.local
+ local database=myApp2
+ local collection=test
+ run_mongos 'use myApp2\n db.test.find()' myApp:myPass@some-name-mongos.demand-backup-sharded-10660 mongodb .svc.cluster.local
+ local 'command=use myApp2\n db.test.find()'
+ local uri=myApp:myPass@some-name-mongos.demand-backup-sharded-10660
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ egrep -v 'I NETWORK|W NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:'
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+ /usr/bin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.fQ5YTTZAcN
+++ mktemp
++ local LAST_ERR=/tmp/tmp.DYPu3JVfqi
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 0 ']'
++ break
++ cat /tmp/tmp.fQ5YTTZAcN
++ cat /tmp/tmp.DYPu3JVfqi
++ rm /tmp/tmp.fQ5YTTZAcN /tmp/tmp.DYPu3JVfqi
++ return 0
+ local client_container=psmdb-client-5dc94d5b48-7tmsd
+ local mongo_flag=
+ kubectl_bin exec psmdb-client-5dc94d5b48-7tmsd -- bash -c 'printf '\''use myApp2\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-10660.svc.cluster.local/admin '
++ mktemp
+ local LAST_OUT=/tmp/tmp.ibVJkWv5wt
++ mktemp
+ local LAST_ERR=/tmp/tmp.A7lGUcDCS1
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl exec psmdb-client-5dc94d5b48-7tmsd -- bash -c 'printf '\''use myApp2\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-10660.svc.cluster.local/admin '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 0 ']'
+ break
+ cat /tmp/tmp.ibVJkWv5wt
+ cat /tmp/tmp.A7lGUcDCS1
+ rm /tmp/tmp.ibVJkWv5wt /tmp/tmp.A7lGUcDCS1
+ return 0
+ diff /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1544/e2e-tests/demand-backup-sharded/compare/find2.json /tmp/tmp.QarcRDZZGq/find2
+ desc 'delete backup and check if it is removed from bucket -- minio'
+ set +o xtrace


-----------------------------------------------------------------------------------
delete backup and check if it is removed from bucket -- minio
-----------------------------------------------------------------------------------

perconaservermongodbbackup.psmdb.percona.com "backup-aws-s3" deleted
perconaservermongodbbackup.psmdb.percona.com "backup-azure-blob" deleted
perconaservermongodbbackup.psmdb.percona.com "backup-gcp-cs" deleted
perconaservermongodbbackup.psmdb.percona.com "backup-minio" deleted
If you don't see a command prompt, try pressing enter.
warning: couldn't attach to pod/aws-cli, falling back to streaming logs: unable to upgrade connection: container aws-cli not found in pod aws-cli_demand-backup-sharded-10660


-----------------------------------------------------------------------------------
check for passwords leak
-----------------------------------------------------------------------------------

secrets=YmFja3VwMTIzNDU2 Y2x1c3RlckFkbWluMTIzNDU2 Y2x1c3Rlck1vbml0b3IxMjM0NTY= ZGF0YWJhc2VBZG1pbjEyMzQ1Ng== dXNlckFkbWluMTIzNDU2 YmFja3VwMTIzNDU2 Y2x1c3RlckFkbWluMTIzNDU2 Y2x1c3Rlck1vbml0b3IxMjM0NTY= ZGF0YWJhc2VBZG1pbjEyMzQ1Ng== dXNlckFkbWluMTIzNDU2
passwords=backup123456 clusterAdmin123456 clusterMonitor123456 databaseAdmin123456 userAdmin123456 backup123456 clusterAdmin123456 clusterMonitor123456 databaseAdmin123456 userAdmin123456 YmFja3VwMTIzNDU2 Y2x1c3RlckFkbWluMTIzNDU2 Y2x1c3Rlck1vbml0b3IxMjM0NTY= ZGF0YWJhc2VBZG1pbjEyMzQ1Ng== dXNlckFkbWluMTIzNDU2 YmFja3VwMTIzNDU2 Y2x1c3RlckFkbWluMTIzNDU2 Y2x1c3Rlck1vbml0b3IxMjM0NTY= ZGF0YWJhc2VBZG1pbjEyMzQ1Ng== dXNlckFkbWluMTIzNDU2
pods=minio-service-57dd49b-vzwzr psmdb-client-5dc94d5b48-7tmsd some-name-cfg-0 some-name-cfg-1 some-name-cfg-2 some-name-mongos-0 some-name-mongos-1 some-name-mongos-2 some-name-rs0-0 some-name-rs0-1 some-name-rs0-2 some-name-rs1-0 some-name-rs1-1 some-name-rs1-2 some-name-rs2-0 some-name-rs2-1 some-name-rs2-2
logs saved in: /tmp/tmp.QarcRDZZGq/logs_output-minio-service-57dd49b-vzwzr-minio.txt

logs saved in: /tmp/tmp.QarcRDZZGq/logs_output-psmdb-client-5dc94d5b48-7tmsd-psmdb-client.txt

logs saved in: /tmp/tmp.QarcRDZZGq/logs_output-some-name-cfg-0-mongod.txt
logs saved in: /tmp/tmp.QarcRDZZGq/logs_output-some-name-cfg-0-cfg-sidecar-1.txt
logs saved in: /tmp/tmp.QarcRDZZGq/logs_output-some-name-cfg-0-backup-agent.txt

logs saved in: /tmp/tmp.QarcRDZZGq/logs_output-some-name-cfg-1-mongod.txt
logs saved in: /tmp/tmp.QarcRDZZGq/logs_output-some-name-cfg-1-cfg-sidecar-1.txt
logs saved in: /tmp/tmp.QarcRDZZGq/logs_output-some-name-cfg-1-backup-agent.txt

logs saved in: /tmp/tmp.QarcRDZZGq/logs_output-some-name-cfg-2-mongod.txt
logs saved in: /tmp/tmp.QarcRDZZGq/logs_output-some-name-cfg-2-cfg-sidecar-1.txt
logs saved in: /tmp/tmp.QarcRDZZGq/logs_output-some-name-cfg-2-backup-agent.txt

logs saved in: /tmp/tmp.QarcRDZZGq/logs_output-some-name-mongos-0-mongos.txt
logs saved in: /tmp/tmp.QarcRDZZGq/logs_output-some-name-mongos-0-mongos-sidecar-1.txt

logs saved in: /tmp/tmp.QarcRDZZGq/logs_output-some-name-mongos-1-mongos.txt
logs saved in: /tmp/tmp.QarcRDZZGq/logs_output-some-name-mongos-1-mongos-sidecar-1.txt

logs saved in: /tmp/tmp.QarcRDZZGq/logs_output-some-name-mongos-2-mongos.txt
logs saved in: /tmp/tmp.QarcRDZZGq/logs_output-some-name-mongos-2-mongos-sidecar-1.txt

logs saved in: /tmp/tmp.QarcRDZZGq/logs_output-some-name-rs0-0-mongod.txt
logs saved in: /tmp/tmp.QarcRDZZGq/logs_output-some-name-rs0-0-backup-agent.txt

logs saved in: /tmp/tmp.QarcRDZZGq/logs_output-some-name-rs0-1-mongod.txt
logs saved in: /tmp/tmp.QarcRDZZGq/logs_output-some-name-rs0-1-backup-agent.txt

logs saved in: /tmp/tmp.QarcRDZZGq/logs_output-some-name-rs0-2-mongod.txt
logs saved in: /tmp/tmp.QarcRDZZGq/logs_output-some-name-rs0-2-backup-agent.txt

logs saved in: /tmp/tmp.QarcRDZZGq/logs_output-some-name-rs1-0-mongod.txt
logs saved in: /tmp/tmp.QarcRDZZGq/logs_output-some-name-rs1-0-backup-agent.txt

logs saved in: /tmp/tmp.QarcRDZZGq/logs_output-some-name-rs1-1-mongod.txt
logs saved in: /tmp/tmp.QarcRDZZGq/logs_output-some-name-rs1-1-backup-agent.txt

logs saved in: /tmp/tmp.QarcRDZZGq/logs_output-some-name-rs1-2-mongod.txt
logs saved in: /tmp/tmp.QarcRDZZGq/logs_output-some-name-rs1-2-backup-agent.txt

logs saved in: /tmp/tmp.QarcRDZZGq/logs_output-some-name-rs2-0-mongod.txt
logs saved in: /tmp/tmp.QarcRDZZGq/logs_output-some-name-rs2-0-rs-sidecar-1.txt
logs saved in: /tmp/tmp.QarcRDZZGq/logs_output-some-name-rs2-0-backup-agent.txt

logs saved in: /tmp/tmp.QarcRDZZGq/logs_output-some-name-rs2-1-mongod.txt
logs saved in: /tmp/tmp.QarcRDZZGq/logs_output-some-name-rs2-1-rs-sidecar-1.txt
logs saved in: /tmp/tmp.QarcRDZZGq/logs_output-some-name-rs2-1-backup-agent.txt

logs saved in: /tmp/tmp.QarcRDZZGq/logs_output-some-name-rs2-2-mongod.txt
logs saved in: /tmp/tmp.QarcRDZZGq/logs_output-some-name-rs2-2-rs-sidecar-1.txt
logs saved in: /tmp/tmp.QarcRDZZGq/logs_output-some-name-rs2-2-backup-agent.txt

logs saved in: /tmp/tmp.QarcRDZZGq/logs_output-percona-server-mongodb-operator-557db8d66f-gqjqn-percona-server-mongodb-operator.txt

runtimeclass.node.k8s.io "container-rc" deleted


-----------------------------------------------------------------------------------
destroy cluster/operator and all other resources
-----------------------------------------------------------------------------------



-----------------------------------------------------------------------------------
get and delete old CRDs and RBAC
-----------------------------------------------------------------------------------

customresourcedefinition.apiextensions.k8s.io "perconaservermongodbbackups.psmdb.percona.com" deleted
customresourcedefinition.apiextensions.k8s.io "perconaservermongodbrestores.psmdb.percona.com" deleted
customresourcedefinition.apiextensions.k8s.io "perconaservermongodbs.psmdb.percona.com" deleted
error: the server doesn't have a resource type "perconaservermongodbbackups"
+ kubectl patch perconaservermongodbbackups.psmdb.percona.com -n sh --type=merge -p '{"metadata":{"finalizers":[]}}'
error: the server doesn't have a resource type "perconaservermongodbbackups"
error: the server doesn't have a resource type "perconaservermongodbrestores"
+ kubectl patch perconaservermongodbrestores.psmdb.percona.com -n sh --type=merge -p '{"metadata":{"finalizers":[]}}'
error: the server doesn't have a resource type "perconaservermongodbrestores"
error: the server doesn't have a resource type "perconaservermongodbs"
+ kubectl patch perconaservermongodbs.psmdb.percona.com -n sh --type=merge -p '{"metadata":{"finalizers":[]}}'
error: the server doesn't have a resource type "perconaservermongodbs"
clusterrole.rbac.authorization.k8s.io "percona-server-mongodb-operator" deleted
clusterrolebinding.rbac.authorization.k8s.io "service-account-percona-server-mongodb-operator" deleted