++ echo 'Log: /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1539/e2e-tests/logs/demand-backup-sharded.log'
Log: /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1539/e2e-tests/logs/demand-backup-sharded.log
++ '[' -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1539/e2e-tests/conf/cloud-secret.yml ']'
++ SKIP_BACKUPS_TO_AWS_GCP_AZURE=
++ oc get projects
++ grep '^minikube'
++ kubectl get nodes
+++ kubectl version -o json
+++ jq -r .serverVersion.gitVersion
+++ grep '\-eks\-'
WARNING: version difference between client (1.30) and server (1.26) exceeds the supported minor version skew of +/-1
++ '[' ']'
++ EKS=0
+++ grep gke
+++ jq -r .serverVersion.gitVersion
+++ kubectl version -o json
WARNING: version difference between client (1.30) and server (1.26) exceeds the supported minor version skew of +/-1
++ '[' v1.26.15-gke.1191000 ']'
++ GKE=1
+++ kubectl version -o json
+++ /usr/bin/sed -r 's/[^0-9.]+//g'
+++ jq -r '.serverVersion.major + "." + .serverVersion.minor'
WARNING: version difference between client (1.30) and server (1.26) exceeds the supported minor version skew of +/-1
++ KUBE_VERSION=1.26
+ set_debug
+ [[ 0 == 1 ]]
+ set +o xtrace


-----------------------------------------------------------------------------------
get and delete old CRDs and RBAC
-----------------------------------------------------------------------------------

error: the server doesn't have a resource type "perconaservermongodbbackups"
+ kubectl patch perconaservermongodbbackups.psmdb.percona.com -n sh --type=merge -p '{"metadata":{"finalizers":[]}}'
error: the server doesn't have a resource type "perconaservermongodbbackups"
error: the server doesn't have a resource type "perconaservermongodbrestores"
+ kubectl patch perconaservermongodbrestores.psmdb.percona.com -n sh --type=merge -p '{"metadata":{"finalizers":[]}}'
error: the server doesn't have a resource type "perconaservermongodbrestores"
error: the server doesn't have a resource type "perconaservermongodbs"
+ kubectl patch perconaservermongodbs.psmdb.percona.com -n sh --type=merge -p '{"metadata":{"finalizers":[]}}'
error: the server doesn't have a resource type "perconaservermongodbs"
Error from server (NotFound): customresourcedefinitions.apiextensions.k8s.io "null" not found
Error from server (NotFound): customresourcedefinitions.apiextensions.k8s.io "null" not found
Error from server (NotFound): customresourcedefinitions.apiextensions.k8s.io "null" not found
Error from server (NotFound): customresourcedefinitions.apiextensions.k8s.io "null" not found


-----------------------------------------------------------------------------------
destroy chaos-mesh
-----------------------------------------------------------------------------------

error: resource(s) were provided, but no name was specified
error: resource(s) were provided, but no name was specified
error: resource(s) were provided, but no name was specified
error: resource(s) were provided, but no name was specified
error: resource(s) were provided, but no name was specified
error: resource(s) were provided, but no name was specified


-----------------------------------------------------------------------------------
cleaned up all old namespaces
-----------------------------------------------------------------------------------



-----------------------------------------------------------------------------------
cleaned up old namespaces psmdb-operator
-----------------------------------------------------------------------------------

error: resource(s) were provided, but no name was specified


-----------------------------------------------------------------------------------
create namespace psmdb-operator
-----------------------------------------------------------------------------------

namespace/psmdb-operator created
Context "gke_cloud-dev-112233_us-central1-a_jen-psmdb-1539-b60f2a71-1-cluster2" modified.


-----------------------------------------------------------------------------------
start PSMDB operator
-----------------------------------------------------------------------------------

customresourcedefinition.apiextensions.k8s.io/perconaservermongodbbackups.psmdb.percona.com serverside-applied
customresourcedefinition.apiextensions.k8s.io/perconaservermongodbrestores.psmdb.percona.com serverside-applied
customresourcedefinition.apiextensions.k8s.io/perconaservermongodbs.psmdb.percona.com serverside-applied
clusterrole.rbac.authorization.k8s.io/percona-server-mongodb-operator created
serviceaccount/percona-server-mongodb-operator created
clusterrolebinding.rbac.authorization.k8s.io/service-account-percona-server-mongodb-operator created
deployment.apps/percona-server-mongodb-operator created
waiting for pod/percona-server-mongodb-operator-699b5984b6-fz5t9 to be ready.OK


-----------------------------------------------------------------------------------
destroy chaos-mesh
-----------------------------------------------------------------------------------

error: resource(s) were provided, but no name was specified
error: resource(s) were provided, but no name was specified
error: resource(s) were provided, but no name was specified
error: resource(s) were provided, but no name was specified
error: resource(s) were provided, but no name was specified
error: resource(s) were provided, but no name was specified


-----------------------------------------------------------------------------------
cleaned up all old namespaces
-----------------------------------------------------------------------------------



-----------------------------------------------------------------------------------
cleaned up old namespaces demand-backup-sharded-1506
-----------------------------------------------------------------------------------

error: resource(s) were provided, but no name was specified


-----------------------------------------------------------------------------------
create namespace demand-backup-sharded-1506
-----------------------------------------------------------------------------------

namespace/demand-backup-sharded-1506 created
Context "gke_cloud-dev-112233_us-central1-a_jen-psmdb-1539-b60f2a71-1-cluster2" modified.


-----------------------------------------------------------------------------------
install Minio
-----------------------------------------------------------------------------------

Error: uninstall: Release not loaded: minio-service: release: not found
"minio" has been removed from your repositories
"minio" has been added to your repositories
NAME: minio-service
LAST DEPLOYED: Wed May  1 02:37:05 2024
NAMESPACE: demand-backup-sharded-1506
STATUS: deployed
REVISION: 1
TEST SUITE: None
NOTES:
MinIO can be accessed via port 9000 on the following DNS name from within your cluster:
minio-service.demand-backup-sharded-1506.svc.cluster.local

To access MinIO from localhost, run the below commands:

  1. export POD_NAME=$(kubectl get pods --namespace demand-backup-sharded-1506 -l "release=minio-service" -o jsonpath="{.items[0].metadata.name}")

  2. kubectl port-forward $POD_NAME 9000 --namespace demand-backup-sharded-1506

Read more about port forwarding here: http://kubernetes.io/docs/user-guide/kubectl/kubectl_port-forward/

You can now access MinIO server on http://localhost:9000. Follow the below steps to connect to MinIO server with mc client:

  1. Download the MinIO mc client - https://min.io/docs/minio/linux/reference/minio-mc.html#quickstart

  2. export MC_HOST_minio-service-local=http://$(kubectl get secret --namespace demand-backup-sharded-1506 minio-service -o jsonpath="{.data.rootUser}" | base64 --decode):$(kubectl get secret --namespace demand-backup-sharded-1506 minio-service -o jsonpath="{.data.rootPassword}" | base64 --decode)@localhost:9000

  3. mc ls minio-service-local
waiting for pod/minio-service-57dd49b-96zkd to be ready.OK
service/minio-service created
make_bucket: operator-testing
pod "aws-cli" deleted
If you don't see a command prompt, try pressing enter.
warning: couldn't attach to pod/aws-cli, falling back to streaming logs: Internal error occurred: error attaching to container: container is in CONTAINER_EXITED state


-----------------------------------------------------------------------------------
create first PSMDB cluster
-----------------------------------------------------------------------------------

secret/some-users created
deployment.apps/psmdb-client created


-----------------------------------------------------------------------------------
create secrets for cloud storages
-----------------------------------------------------------------------------------

secret/minio-secret created
secret/aws-s3-secret created
secret/gcp-cs-secret created
secret/azure-secret created
runtimeclass.node.k8s.io/container-rc unchanged
perconaservermongodb.psmdb.percona.com/some-name created


-----------------------------------------------------------------------------------
check if all 3 Pods started
-----------------------------------------------------------------------------------

waiting for pod/some-name-rs0-0 to be ready...........OK
waiting for pod/some-name-rs0-1 to be ready........OK
waiting for pod/some-name-rs0-2 to be ready........OK
Waiting for cluster readyness....................................
waiting for pod/some-name-cfg-0 to be ready.OK
waiting for pod/some-name-cfg-1 to be ready.OK
waiting for pod/some-name-cfg-2 to be ready.OK
waiting for pod/some-name-mongos-0 to be ready.OK
waiting for pod/some-name-mongos-1 to be ready.OK
waiting for pod/some-name-mongos-2 to be ready.OK
Waiting for cluster readyness


-----------------------------------------------------------------------------------
check if service and statefulset created with expected config
-----------------------------------------------------------------------------------

secret/some-name-mongos created
waiting for pod/some-name-mongos-0 to be ready.OK
waiting for pod/some-name-mongos-1 to be ready.OK
waiting for pod/some-name-mongos-2 to be ready.OK
Waiting for cluster readyness............................................................................................


-----------------------------------------------------------------------------------
write data, read from all
-----------------------------------------------------------------------------------

Percona Server for MongoDB shell version v4.4.29-28
connecting to: mongodb://some-name-mongos.demand-backup-sharded-1506.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb
Implicit session: session { "id" : UUID("b7967fb1-f661-402d-9b1c-484c1fa00a5d") }
Percona Server for MongoDB server version: v7.0.8-5
WARNING: shell and server versions do not match
Successfully added user: {
	"user" : "myApp",
	"roles" : [
		{
			"db" : "myApp",
			"role" : "readWrite"
		},
		{
			"db" : "myApp1",
			"role" : "readWrite"
		},
		{
			"db" : "myApp2",
			"role" : "readWrite"
		}
	]
}
bye
Percona Server for MongoDB shell version v4.4.29-28
connecting to: mongodb://some-name-mongos.demand-backup-sharded-1506.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb
Implicit session: session { "id" : UUID("0733ae29-1670-4e12-8da3-032bbdc242c2") }
Percona Server for MongoDB server version: v7.0.8-5
WARNING: shell and server versions do not match
{
	"ok" : 1,
	"$clusterTime" : {
		"clusterTime" : Timestamp(1714531534, 13),
		"signature" : {
			"hash" : BinData(0,"Q1aqLI/wcQcU85ZPeH8A0XjdJTg="),
			"keyId" : NumberLong("7363855479216275469")
		}
	},
	"operationTime" : Timestamp(1714531534, 4)
}
bye
Percona Server for MongoDB shell version v4.4.29-28
connecting to: mongodb://some-name-mongos.demand-backup-sharded-1506.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb
Implicit session: session { "id" : UUID("cc73a818-25f7-4254-99ed-9ffa1accd2b2") }
Percona Server for MongoDB server version: v7.0.8-5
WARNING: shell and server versions do not match
{
	"ok" : 1,
	"$clusterTime" : {
		"clusterTime" : Timestamp(1714531538, 7),
		"signature" : {
			"hash" : BinData(0,"bpwU1J1/D53ZOlolersT1zjymws="),
			"keyId" : NumberLong("7363855479216275469")
		}
	},
	"operationTime" : Timestamp(1714531538, 2)
}
bye
Percona Server for MongoDB shell version v4.4.29-28
connecting to: mongodb://some-name-mongos.demand-backup-sharded-1506.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb
Implicit session: session { "id" : UUID("7ca918f5-fd32-48bb-bdd8-f656b56eb96c") }
Percona Server for MongoDB server version: v7.0.8-5
WARNING: shell and server versions do not match
{
	"ok" : 1,
	"$clusterTime" : {
		"clusterTime" : Timestamp(1714531542, 14),
		"signature" : {
			"hash" : BinData(0,"/YVT7vJTSN3CuftEIsn9B96WsnA="),
			"keyId" : NumberLong("7363855479216275469")
		}
	},
	"operationTime" : Timestamp(1714531542, 9)
}
bye
Percona Server for MongoDB shell version v4.4.29-28
connecting to: mongodb://some-name-mongos.demand-backup-sharded-1506.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb
Implicit session: session { "id" : UUID("aadf83e0-a657-46b0-a515-668a1b457ab6") }
Percona Server for MongoDB server version: v7.0.8-5
WARNING: shell and server versions do not match
switched to db myApp
WriteResult({ "nInserted" : 1 })
bye
Percona Server for MongoDB shell version v4.4.29-28
connecting to: mongodb://some-name-mongos.demand-backup-sharded-1506.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb
Implicit session: session { "id" : UUID("44df06d0-9cec-4fd7-afd3-5672ffb60960") }
Percona Server for MongoDB server version: v7.0.8-5
WARNING: shell and server versions do not match
switched to db myApp1
WriteResult({ "nInserted" : 1 })
bye
Percona Server for MongoDB shell version v4.4.29-28
connecting to: mongodb://some-name-mongos.demand-backup-sharded-1506.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb
Implicit session: session { "id" : UUID("fa000ca2-e6b0-4979-97c4-7d7a171a90f8") }
Percona Server for MongoDB server version: v7.0.8-5
WARNING: shell and server versions do not match
switched to db myApp2
WriteResult({ "nInserted" : 1 })
bye
some-name-rs0-0
some-name-rs0-1
some-name-rs0-2
some-name-rs1-0
some-name-rs1-1
some-name-rs1-2
some-name-rs2-0
some-name-rs2-1
some-name-rs2-2


-----------------------------------------------------------------------------------
run backups
-----------------------------------------------------------------------------------



-----------------------------------------------------------------------------------
run backup backup-minio
-----------------------------------------------------------------------------------

perconaservermongodbbackup.psmdb.percona.com/backup-minio created


-----------------------------------------------------------------------------------
run backup backup-aws-s3
-----------------------------------------------------------------------------------

perconaservermongodbbackup.psmdb.percona.com/backup-aws-s3 created


-----------------------------------------------------------------------------------
run backup backup-gcp-cs
-----------------------------------------------------------------------------------

perconaservermongodbbackup.psmdb.percona.com/backup-gcp-cs created


-----------------------------------------------------------------------------------
run backup backup-azure-blob
-----------------------------------------------------------------------------------

perconaservermongodbbackup.psmdb.percona.com/backup-azure-blob created
backup-aws-s3.............................................
backup-gcp-cs................
backup-azure-blob..................
backup-minio.


-----------------------------------------------------------------------------------
check backup and restore -- aws-s3
-----------------------------------------------------------------------------------

Percona Server for MongoDB shell version v4.4.29-28
connecting to: mongodb://some-name-mongos.demand-backup-sharded-1506.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb
Implicit session: session { "id" : UUID("b94825b3-320d-410d-a93b-9b6332be5ed2") }
Percona Server for MongoDB server version: v7.0.8-5
WARNING: shell and server versions do not match
switched to db myApp
WriteResult({ "nInserted" : 1 })
bye
Percona Server for MongoDB shell version v4.4.29-28
connecting to: mongodb://some-name-mongos.demand-backup-sharded-1506.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb
Implicit session: session { "id" : UUID("c5cef77e-0316-47f2-a1eb-4f557fefbeb2") }
Percona Server for MongoDB server version: v7.0.8-5
WARNING: shell and server versions do not match
switched to db myApp1
WriteResult({ "nInserted" : 1 })
bye
Percona Server for MongoDB shell version v4.4.29-28
connecting to: mongodb://some-name-mongos.demand-backup-sharded-1506.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb
Implicit session: session { "id" : UUID("175e004b-b238-456e-b222-44b06cecd343") }
Percona Server for MongoDB server version: v7.0.8-5
WARNING: shell and server versions do not match
switched to db myApp2
WriteResult({ "nInserted" : 1 })
bye
perconaservermongodbrestore.psmdb.percona.com/restore-backup-aws-s3 created
waiting psmdb-restore/backup-aws-s3 to reach ready state............
+ '[' 1 -eq 1 ']'
+ wait_cluster_consistency some-name
+ local cluster_name=some-name
+ local wait_time=32
+ retry=0
+ sleep 7
+ echo -n 'waiting for cluster readyness'
waiting for cluster readyness++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.SLPFYrBOZ8
+++ mktemp
++ local LAST_ERR=/tmp/tmp.bs2nHNEzYd
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 0 ']'
++ break
++ cat /tmp/tmp.SLPFYrBOZ8
++ cat /tmp/tmp.bs2nHNEzYd
++ rm /tmp/tmp.SLPFYrBOZ8 /tmp/tmp.bs2nHNEzYd
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 1 -ge 32 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.zT6md6Oj7D
+++ mktemp
++ local LAST_ERR=/tmp/tmp.fkyuY30zol
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 0 ']'
++ break
++ cat /tmp/tmp.zT6md6Oj7D
++ cat /tmp/tmp.fkyuY30zol
++ rm /tmp/tmp.zT6md6Oj7D /tmp/tmp.fkyuY30zol
++ return 0
+ [[ error == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 2 -ge 32 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.LemLFnGxFw
+++ mktemp
++ local LAST_ERR=/tmp/tmp.C1yDFKchpR
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 0 ']'
++ break
++ cat /tmp/tmp.LemLFnGxFw
++ cat /tmp/tmp.C1yDFKchpR
++ rm /tmp/tmp.LemLFnGxFw /tmp/tmp.C1yDFKchpR
++ return 0
+ [[ error == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 3 -ge 32 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.jcI352mstw
+++ mktemp
++ local LAST_ERR=/tmp/tmp.wPw0lERYGm
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 0 ']'
++ break
++ cat /tmp/tmp.jcI352mstw
++ cat /tmp/tmp.wPw0lERYGm
++ rm /tmp/tmp.jcI352mstw /tmp/tmp.wPw0lERYGm
++ return 0
+ [[ error == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 4 -ge 32 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.oWDW9C4UE3
+++ mktemp
++ local LAST_ERR=/tmp/tmp.ArOQUKMIBz
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 0 ']'
++ break
++ cat /tmp/tmp.oWDW9C4UE3
++ cat /tmp/tmp.ArOQUKMIBz
++ rm /tmp/tmp.oWDW9C4UE3 /tmp/tmp.ArOQUKMIBz
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 5 -ge 32 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.CFIM46eOun
+++ mktemp
++ local LAST_ERR=/tmp/tmp.ZLiSUkj8ul
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 0 ']'
++ break
++ cat /tmp/tmp.CFIM46eOun
++ cat /tmp/tmp.ZLiSUkj8ul
++ rm /tmp/tmp.CFIM46eOun /tmp/tmp.ZLiSUkj8ul
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 6 -ge 32 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.W7YSjsvVBC
+++ mktemp
++ local LAST_ERR=/tmp/tmp.zwqKY2y70U
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 0 ']'
++ break
++ cat /tmp/tmp.W7YSjsvVBC
++ cat /tmp/tmp.zwqKY2y70U
++ rm /tmp/tmp.W7YSjsvVBC /tmp/tmp.zwqKY2y70U
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 7 -ge 32 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.GUakCTiDwq
+++ mktemp
++ local LAST_ERR=/tmp/tmp.AFHrBqwPZw
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 0 ']'
++ break
++ cat /tmp/tmp.GUakCTiDwq
++ cat /tmp/tmp.AFHrBqwPZw
++ rm /tmp/tmp.GUakCTiDwq /tmp/tmp.AFHrBqwPZw
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 8 -ge 32 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.BvsoCekC3p
+++ mktemp
++ local LAST_ERR=/tmp/tmp.oq2DNtKsfe
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 0 ']'
++ break
++ cat /tmp/tmp.BvsoCekC3p
++ cat /tmp/tmp.oq2DNtKsfe
++ rm /tmp/tmp.BvsoCekC3p /tmp/tmp.oq2DNtKsfe
++ return 0
+ [[ ready == \r\e\a\d\y ]]
+ check_data
+ local postfix=
++ seq 0 2
+ for i in '$(seq 0 2)'
+ compare_mongos_cmd find myApp:myPass@some-name-mongos.demand-backup-sharded-1506 '' .svc.cluster.local myApp test
+ local command=find
+ local uri=myApp:myPass@some-name-mongos.demand-backup-sharded-1506
+ local postfix=
+ local suffix=.svc.cluster.local
+ local database=myApp
+ local collection=test
+ run_mongos 'use myApp\n db.test.find()' myApp:myPass@some-name-mongos.demand-backup-sharded-1506 mongodb .svc.cluster.local
+ local 'command=use myApp\n db.test.find()'
+ local uri=myApp:myPass@some-name-mongos.demand-backup-sharded-1506
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ egrep -v 'I NETWORK|W NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:'
+ /usr/bin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.GRQdPlFrIA
+++ mktemp
++ local LAST_ERR=/tmp/tmp.ikVwGRyKfH
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 0 ']'
++ break
++ cat /tmp/tmp.GRQdPlFrIA
++ cat /tmp/tmp.ikVwGRyKfH
++ rm /tmp/tmp.GRQdPlFrIA /tmp/tmp.ikVwGRyKfH
++ return 0
+ local client_container=psmdb-client-5dc94d5b48-7ctb7
+ local mongo_flag=
+ kubectl_bin exec psmdb-client-5dc94d5b48-7ctb7 -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-1506.svc.cluster.local/admin '
++ mktemp
+ local LAST_OUT=/tmp/tmp.EWHEFXSNh1
++ mktemp
+ local LAST_ERR=/tmp/tmp.UrKtG9EBJS
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl exec psmdb-client-5dc94d5b48-7ctb7 -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-1506.svc.cluster.local/admin '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 0 ']'
+ break
+ cat /tmp/tmp.EWHEFXSNh1
+ cat /tmp/tmp.UrKtG9EBJS
+ rm /tmp/tmp.EWHEFXSNh1 /tmp/tmp.UrKtG9EBJS
+ return 0
+ diff /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1539/e2e-tests/demand-backup-sharded/compare/find.json /tmp/tmp.4LAUoxvZTn/find
+ for i in '$(seq 0 2)'
+ compare_mongos_cmd find myApp:myPass@some-name-mongos.demand-backup-sharded-1506 1 .svc.cluster.local myApp1 test
+ local command=find
+ local uri=myApp:myPass@some-name-mongos.demand-backup-sharded-1506
+ local postfix=1
+ local suffix=.svc.cluster.local
+ local database=myApp1
+ local collection=test
+ run_mongos 'use myApp1\n db.test.find()' myApp:myPass@some-name-mongos.demand-backup-sharded-1506 mongodb .svc.cluster.local
+ local 'command=use myApp1\n db.test.find()'
+ local uri=myApp:myPass@some-name-mongos.demand-backup-sharded-1506
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ /usr/bin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
+ egrep -v 'I NETWORK|W NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:'
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.2pfFnU6tRd
+++ mktemp
++ local LAST_ERR=/tmp/tmp.QDW9p4Q1zO
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 0 ']'
++ break
++ cat /tmp/tmp.2pfFnU6tRd
++ cat /tmp/tmp.QDW9p4Q1zO
++ rm /tmp/tmp.2pfFnU6tRd /tmp/tmp.QDW9p4Q1zO
++ return 0
+ local client_container=psmdb-client-5dc94d5b48-7ctb7
+ local mongo_flag=
+ kubectl_bin exec psmdb-client-5dc94d5b48-7ctb7 -- bash -c 'printf '\''use myApp1\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-1506.svc.cluster.local/admin '
++ mktemp
+ local LAST_OUT=/tmp/tmp.ZEJkYTve0j
++ mktemp
+ local LAST_ERR=/tmp/tmp.U36b26sL7W
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl exec psmdb-client-5dc94d5b48-7ctb7 -- bash -c 'printf '\''use myApp1\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-1506.svc.cluster.local/admin '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 0 ']'
+ break
+ cat /tmp/tmp.ZEJkYTve0j
+ cat /tmp/tmp.U36b26sL7W
+ rm /tmp/tmp.ZEJkYTve0j /tmp/tmp.U36b26sL7W
+ return 0
+ diff /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1539/e2e-tests/demand-backup-sharded/compare/find1.json /tmp/tmp.4LAUoxvZTn/find1
+ for i in '$(seq 0 2)'
+ compare_mongos_cmd find myApp:myPass@some-name-mongos.demand-backup-sharded-1506 2 .svc.cluster.local myApp2 test
+ local command=find
+ local uri=myApp:myPass@some-name-mongos.demand-backup-sharded-1506
+ local postfix=2
+ local suffix=.svc.cluster.local
+ local database=myApp2
+ local collection=test
+ /usr/bin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
+ egrep -v 'I NETWORK|W NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:'
+ run_mongos 'use myApp2\n db.test.find()' myApp:myPass@some-name-mongos.demand-backup-sharded-1506 mongodb .svc.cluster.local
+ local 'command=use myApp2\n db.test.find()'
+ local uri=myApp:myPass@some-name-mongos.demand-backup-sharded-1506
+ local driver=mongodb
+ local suffix=.svc.cluster.local
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.3rBeVyIeG0
+++ mktemp
++ local LAST_ERR=/tmp/tmp.GBJjo1sTXC
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 0 ']'
++ break
++ cat /tmp/tmp.3rBeVyIeG0
++ cat /tmp/tmp.GBJjo1sTXC
++ rm /tmp/tmp.3rBeVyIeG0 /tmp/tmp.GBJjo1sTXC
++ return 0
+ local client_container=psmdb-client-5dc94d5b48-7ctb7
+ local mongo_flag=
+ kubectl_bin exec psmdb-client-5dc94d5b48-7ctb7 -- bash -c 'printf '\''use myApp2\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-1506.svc.cluster.local/admin '
++ mktemp
+ local LAST_OUT=/tmp/tmp.qwMwcUMHc0
++ mktemp
+ local LAST_ERR=/tmp/tmp.ACOrjDoYxR
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl exec psmdb-client-5dc94d5b48-7ctb7 -- bash -c 'printf '\''use myApp2\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-1506.svc.cluster.local/admin '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 0 ']'
+ break
+ cat /tmp/tmp.qwMwcUMHc0
+ cat /tmp/tmp.ACOrjDoYxR
+ rm /tmp/tmp.qwMwcUMHc0 /tmp/tmp.ACOrjDoYxR
+ return 0
+ diff /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1539/e2e-tests/demand-backup-sharded/compare/find2.json /tmp/tmp.4LAUoxvZTn/find2
+ desc 'check backup and restore -- gcp-cs'
+ set +o xtrace


-----------------------------------------------------------------------------------
check backup and restore -- gcp-cs
-----------------------------------------------------------------------------------

Percona Server for MongoDB shell version v4.4.29-28
connecting to: mongodb://some-name-mongos.demand-backup-sharded-1506.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb
Implicit session: session { "id" : UUID("1aafaae5-27b7-43f2-bd7b-8250625ef519") }
Percona Server for MongoDB server version: v7.0.8-5
WARNING: shell and server versions do not match
switched to db myApp
WriteResult({ "nInserted" : 1 })
bye
Percona Server for MongoDB shell version v4.4.29-28
connecting to: mongodb://some-name-mongos.demand-backup-sharded-1506.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb
Implicit session: session { "id" : UUID("f946d73c-570f-428c-a365-45a043eddca7") }
Percona Server for MongoDB server version: v7.0.8-5
WARNING: shell and server versions do not match
switched to db myApp1
WriteResult({ "nInserted" : 1 })
bye
Percona Server for MongoDB shell version v4.4.29-28
connecting to: mongodb://some-name-mongos.demand-backup-sharded-1506.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb
Implicit session: session { "id" : UUID("8709b054-6b5a-428d-9d60-3861ac1989b4") }
Percona Server for MongoDB server version: v7.0.8-5
WARNING: shell and server versions do not match
switched to db myApp2
WriteResult({ "nInserted" : 1 })
bye
perconaservermongodbrestore.psmdb.percona.com/restore-backup-gcp-cs created
waiting psmdb-restore/backup-gcp-cs to reach ready state...........
+ '[' 1 -eq 1 ']'
+ wait_cluster_consistency some-name
+ local cluster_name=some-name
+ local wait_time=32
+ retry=0
+ sleep 7
+ echo -n 'waiting for cluster readyness'
waiting for cluster readyness++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.V7lLlrJOgT
+++ mktemp
++ local LAST_ERR=/tmp/tmp.FXraP9LwlN
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 0 ']'
++ break
++ cat /tmp/tmp.V7lLlrJOgT
++ cat /tmp/tmp.FXraP9LwlN
++ rm /tmp/tmp.V7lLlrJOgT /tmp/tmp.FXraP9LwlN
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 1 -ge 32 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.NTDOot1yqS
+++ mktemp
++ local LAST_ERR=/tmp/tmp.DlALSAJ17g
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 0 ']'
++ break
++ cat /tmp/tmp.NTDOot1yqS
++ cat /tmp/tmp.DlALSAJ17g
++ rm /tmp/tmp.NTDOot1yqS /tmp/tmp.DlALSAJ17g
++ return 0
+ [[ error == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 2 -ge 32 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.W1Wo5Zz1fM
+++ mktemp
++ local LAST_ERR=/tmp/tmp.YOcat80yFi
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 0 ']'
++ break
++ cat /tmp/tmp.W1Wo5Zz1fM
++ cat /tmp/tmp.YOcat80yFi
++ rm /tmp/tmp.W1Wo5Zz1fM /tmp/tmp.YOcat80yFi
++ return 0
+ [[ error == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 3 -ge 32 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.zNSmQAL9YD
+++ mktemp
++ local LAST_ERR=/tmp/tmp.yWnmpAJ1LB
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 0 ']'
++ break
++ cat /tmp/tmp.zNSmQAL9YD
++ cat /tmp/tmp.yWnmpAJ1LB
++ rm /tmp/tmp.zNSmQAL9YD /tmp/tmp.yWnmpAJ1LB
++ return 0
+ [[ error == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 4 -ge 32 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.r8IPdo6wuR
+++ mktemp
++ local LAST_ERR=/tmp/tmp.hEwmeJiz4E
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 0 ']'
++ break
++ cat /tmp/tmp.r8IPdo6wuR
++ cat /tmp/tmp.hEwmeJiz4E
++ rm /tmp/tmp.r8IPdo6wuR /tmp/tmp.hEwmeJiz4E
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 5 -ge 32 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.Egt3PJXHvw
+++ mktemp
++ local LAST_ERR=/tmp/tmp.7u3Ic5g2da
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 0 ']'
++ break
++ cat /tmp/tmp.Egt3PJXHvw
++ cat /tmp/tmp.7u3Ic5g2da
++ rm /tmp/tmp.Egt3PJXHvw /tmp/tmp.7u3Ic5g2da
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 6 -ge 32 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.oa2a0KLCSI
+++ mktemp
++ local LAST_ERR=/tmp/tmp.XPxUalEltk
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 0 ']'
++ break
++ cat /tmp/tmp.oa2a0KLCSI
++ cat /tmp/tmp.XPxUalEltk
++ rm /tmp/tmp.oa2a0KLCSI /tmp/tmp.XPxUalEltk
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 7 -ge 32 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.kqQ44pF1s1
+++ mktemp
++ local LAST_ERR=/tmp/tmp.Edv251y4qF
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 0 ']'
++ break
++ cat /tmp/tmp.kqQ44pF1s1
++ cat /tmp/tmp.Edv251y4qF
++ rm /tmp/tmp.kqQ44pF1s1 /tmp/tmp.Edv251y4qF
++ return 0
+ [[ ready == \r\e\a\d\y ]]
+ check_data
+ local postfix=
++ seq 0 2
+ for i in '$(seq 0 2)'
+ compare_mongos_cmd find myApp:myPass@some-name-mongos.demand-backup-sharded-1506 '' .svc.cluster.local myApp test
+ local command=find
+ local uri=myApp:myPass@some-name-mongos.demand-backup-sharded-1506
+ local postfix=
+ local suffix=.svc.cluster.local
+ local database=myApp
+ local collection=test
+ run_mongos 'use myApp\n db.test.find()' myApp:myPass@some-name-mongos.demand-backup-sharded-1506 mongodb .svc.cluster.local
+ local 'command=use myApp\n db.test.find()'
+ local uri=myApp:myPass@some-name-mongos.demand-backup-sharded-1506
+ egrep -v 'I NETWORK|W NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:'
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ /usr/bin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.4ddi8VBMdT
+++ mktemp
++ local LAST_ERR=/tmp/tmp.VXkBSSjAwr
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 0 ']'
++ break
++ cat /tmp/tmp.4ddi8VBMdT
++ cat /tmp/tmp.VXkBSSjAwr
++ rm /tmp/tmp.4ddi8VBMdT /tmp/tmp.VXkBSSjAwr
++ return 0
+ local client_container=psmdb-client-5dc94d5b48-7ctb7
+ local mongo_flag=
+ kubectl_bin exec psmdb-client-5dc94d5b48-7ctb7 -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-1506.svc.cluster.local/admin '
++ mktemp
+ local LAST_OUT=/tmp/tmp.YKA1w4xYyH
++ mktemp
+ local LAST_ERR=/tmp/tmp.TN9CB7MCyd
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl exec psmdb-client-5dc94d5b48-7ctb7 -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-1506.svc.cluster.local/admin '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 0 ']'
+ break
+ cat /tmp/tmp.YKA1w4xYyH
+ cat /tmp/tmp.TN9CB7MCyd
+ rm /tmp/tmp.YKA1w4xYyH /tmp/tmp.TN9CB7MCyd
+ return 0
+ diff /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1539/e2e-tests/demand-backup-sharded/compare/find.json /tmp/tmp.4LAUoxvZTn/find
+ for i in '$(seq 0 2)'
+ compare_mongos_cmd find myApp:myPass@some-name-mongos.demand-backup-sharded-1506 1 .svc.cluster.local myApp1 test
+ local command=find
+ local uri=myApp:myPass@some-name-mongos.demand-backup-sharded-1506
+ local postfix=1
+ local suffix=.svc.cluster.local
+ local database=myApp1
+ local collection=test
+ egrep -v 'I NETWORK|W NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:'
+ run_mongos 'use myApp1\n db.test.find()' myApp:myPass@some-name-mongos.demand-backup-sharded-1506 mongodb .svc.cluster.local
+ local 'command=use myApp1\n db.test.find()'
+ local uri=myApp:myPass@some-name-mongos.demand-backup-sharded-1506
+ local driver=mongodb
+ local suffix=.svc.cluster.local
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.o1yqeWXIvz
+++ mktemp
++ local LAST_ERR=/tmp/tmp.PKncFPauuU
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
+ /usr/bin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 0 ']'
++ break
++ cat /tmp/tmp.o1yqeWXIvz
++ cat /tmp/tmp.PKncFPauuU
++ rm /tmp/tmp.o1yqeWXIvz /tmp/tmp.PKncFPauuU
++ return 0
+ local client_container=psmdb-client-5dc94d5b48-7ctb7
+ local mongo_flag=
+ kubectl_bin exec psmdb-client-5dc94d5b48-7ctb7 -- bash -c 'printf '\''use myApp1\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-1506.svc.cluster.local/admin '
++ mktemp
+ local LAST_OUT=/tmp/tmp.jTBFZAzbjc
++ mktemp
+ local LAST_ERR=/tmp/tmp.0d1b4LAVAt
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl exec psmdb-client-5dc94d5b48-7ctb7 -- bash -c 'printf '\''use myApp1\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-1506.svc.cluster.local/admin '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 0 ']'
+ break
+ cat /tmp/tmp.jTBFZAzbjc
+ cat /tmp/tmp.0d1b4LAVAt
+ rm /tmp/tmp.jTBFZAzbjc /tmp/tmp.0d1b4LAVAt
+ return 0
+ diff /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1539/e2e-tests/demand-backup-sharded/compare/find1.json /tmp/tmp.4LAUoxvZTn/find1
+ for i in '$(seq 0 2)'
+ compare_mongos_cmd find myApp:myPass@some-name-mongos.demand-backup-sharded-1506 2 .svc.cluster.local myApp2 test
+ local command=find
+ local uri=myApp:myPass@some-name-mongos.demand-backup-sharded-1506
+ local postfix=2
+ local suffix=.svc.cluster.local
+ local database=myApp2
+ local collection=test
+ egrep -v 'I NETWORK|W NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:'
+ run_mongos 'use myApp2\n db.test.find()' myApp:myPass@some-name-mongos.demand-backup-sharded-1506 mongodb .svc.cluster.local
+ local 'command=use myApp2\n db.test.find()'
+ local uri=myApp:myPass@some-name-mongos.demand-backup-sharded-1506
+ local driver=mongodb
+ local suffix=.svc.cluster.local
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.utR8BrCocm
+ /usr/bin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
+++ mktemp
++ local LAST_ERR=/tmp/tmp.bIpm63IKjM
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 0 ']'
++ break
++ cat /tmp/tmp.utR8BrCocm
++ cat /tmp/tmp.bIpm63IKjM
++ rm /tmp/tmp.utR8BrCocm /tmp/tmp.bIpm63IKjM
++ return 0
+ local client_container=psmdb-client-5dc94d5b48-7ctb7
+ local mongo_flag=
+ kubectl_bin exec psmdb-client-5dc94d5b48-7ctb7 -- bash -c 'printf '\''use myApp2\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-1506.svc.cluster.local/admin '
++ mktemp
+ local LAST_OUT=/tmp/tmp.Q0ST005CbK
++ mktemp
+ local LAST_ERR=/tmp/tmp.RkCCVmZHoW
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl exec psmdb-client-5dc94d5b48-7ctb7 -- bash -c 'printf '\''use myApp2\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-1506.svc.cluster.local/admin '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 0 ']'
+ break
+ cat /tmp/tmp.Q0ST005CbK
+ cat /tmp/tmp.RkCCVmZHoW
+ rm /tmp/tmp.Q0ST005CbK /tmp/tmp.RkCCVmZHoW
+ return 0
+ diff /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1539/e2e-tests/demand-backup-sharded/compare/find2.json /tmp/tmp.4LAUoxvZTn/find2
+ desc 'check backup and restore -- azure-blob'
+ set +o xtrace


-----------------------------------------------------------------------------------
check backup and restore -- azure-blob
-----------------------------------------------------------------------------------

Percona Server for MongoDB shell version v4.4.29-28
connecting to: mongodb://some-name-mongos.demand-backup-sharded-1506.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb
Implicit session: session { "id" : UUID("c2fcddae-6c34-41bb-b8e5-123072aca241") }
Percona Server for MongoDB server version: v7.0.8-5
WARNING: shell and server versions do not match
switched to db myApp
WriteResult({ "nInserted" : 1 })
bye
Percona Server for MongoDB shell version v4.4.29-28
connecting to: mongodb://some-name-mongos.demand-backup-sharded-1506.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb
Implicit session: session { "id" : UUID("eac0dc4f-5071-412b-85b7-6fcf058273a8") }
Percona Server for MongoDB server version: v7.0.8-5
WARNING: shell and server versions do not match
switched to db myApp1
WriteResult({ "nInserted" : 1 })
bye
Percona Server for MongoDB shell version v4.4.29-28
connecting to: mongodb://some-name-mongos.demand-backup-sharded-1506.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb
Implicit session: session { "id" : UUID("e6592e48-82c8-4bf5-9bab-8b212f66254c") }
Percona Server for MongoDB server version: v7.0.8-5
WARNING: shell and server versions do not match
switched to db myApp2
WriteResult({ "nInserted" : 1 })
bye
perconaservermongodbrestore.psmdb.percona.com/restore-backup-azure-blob created
waiting psmdb-restore/backup-azure-blob to reach ready state...........
+ '[' 1 -eq 1 ']'
+ wait_cluster_consistency some-name
+ local cluster_name=some-name
+ local wait_time=32
+ retry=0
+ sleep 7
+ echo -n 'waiting for cluster readyness'
waiting for cluster readyness++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.Del02KGenn
+++ mktemp
++ local LAST_ERR=/tmp/tmp.cMldqprSiP
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 0 ']'
++ break
++ cat /tmp/tmp.Del02KGenn
++ cat /tmp/tmp.cMldqprSiP
++ rm /tmp/tmp.Del02KGenn /tmp/tmp.cMldqprSiP
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 1 -ge 32 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.59ao8GQk9z
+++ mktemp
++ local LAST_ERR=/tmp/tmp.N2Mc63VupK
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 0 ']'
++ break
++ cat /tmp/tmp.59ao8GQk9z
++ cat /tmp/tmp.N2Mc63VupK
++ rm /tmp/tmp.59ao8GQk9z /tmp/tmp.N2Mc63VupK
++ return 0
+ [[ error == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 2 -ge 32 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.EZXcwsiZ5Z
+++ mktemp
++ local LAST_ERR=/tmp/tmp.xQ83bOXUfG
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 0 ']'
++ break
++ cat /tmp/tmp.EZXcwsiZ5Z
++ cat /tmp/tmp.xQ83bOXUfG
++ rm /tmp/tmp.EZXcwsiZ5Z /tmp/tmp.xQ83bOXUfG
++ return 0
+ [[ error == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 3 -ge 32 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.51XpJ0pjAv
+++ mktemp
++ local LAST_ERR=/tmp/tmp.n5mNl6sxTo
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 0 ']'
++ break
++ cat /tmp/tmp.51XpJ0pjAv
++ cat /tmp/tmp.n5mNl6sxTo
++ rm /tmp/tmp.51XpJ0pjAv /tmp/tmp.n5mNl6sxTo
++ return 0
+ [[ error == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 4 -ge 32 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.rnBjxHVQnj
+++ mktemp
++ local LAST_ERR=/tmp/tmp.E9PUQp01hL
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 0 ']'
++ break
++ cat /tmp/tmp.rnBjxHVQnj
++ cat /tmp/tmp.E9PUQp01hL
++ rm /tmp/tmp.rnBjxHVQnj /tmp/tmp.E9PUQp01hL
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 5 -ge 32 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.JTJKZm36pW
+++ mktemp
++ local LAST_ERR=/tmp/tmp.Hyc10yMKAg
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 0 ']'
++ break
++ cat /tmp/tmp.JTJKZm36pW
++ cat /tmp/tmp.Hyc10yMKAg
++ rm /tmp/tmp.JTJKZm36pW /tmp/tmp.Hyc10yMKAg
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 6 -ge 32 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.6b1P4qg9uV
+++ mktemp
++ local LAST_ERR=/tmp/tmp.zhzfpsGp2u
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 0 ']'
++ break
++ cat /tmp/tmp.6b1P4qg9uV
++ cat /tmp/tmp.zhzfpsGp2u
++ rm /tmp/tmp.6b1P4qg9uV /tmp/tmp.zhzfpsGp2u
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 7 -ge 32 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.QFjqLzJ3nq
+++ mktemp
++ local LAST_ERR=/tmp/tmp.i9xB5Xa0en
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 0 ']'
++ break
++ cat /tmp/tmp.QFjqLzJ3nq
++ cat /tmp/tmp.i9xB5Xa0en
++ rm /tmp/tmp.QFjqLzJ3nq /tmp/tmp.i9xB5Xa0en
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 8 -ge 32 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.guG6HJVwNP
+++ mktemp
++ local LAST_ERR=/tmp/tmp.1hUyYaHd2o
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 0 ']'
++ break
++ cat /tmp/tmp.guG6HJVwNP
++ cat /tmp/tmp.1hUyYaHd2o
++ rm /tmp/tmp.guG6HJVwNP /tmp/tmp.1hUyYaHd2o
++ return 0
+ [[ ready == \r\e\a\d\y ]]
+ check_data
+ local postfix=
++ seq 0 2
+ for i in '$(seq 0 2)'
+ compare_mongos_cmd find myApp:myPass@some-name-mongos.demand-backup-sharded-1506 '' .svc.cluster.local myApp test
+ local command=find
+ local uri=myApp:myPass@some-name-mongos.demand-backup-sharded-1506
+ local postfix=
+ local suffix=.svc.cluster.local
+ local database=myApp
+ local collection=test
+ run_mongos 'use myApp\n db.test.find()' myApp:myPass@some-name-mongos.demand-backup-sharded-1506 mongodb .svc.cluster.local
+ local 'command=use myApp\n db.test.find()'
+ local uri=myApp:myPass@some-name-mongos.demand-backup-sharded-1506
+ local driver=mongodb
+ local suffix=.svc.cluster.local
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.KCpSaC2Mcj
+ egrep -v 'I NETWORK|W NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:'
+++ mktemp
++ local LAST_ERR=/tmp/tmp.k8Sl8LCclk
++ local exit_status=0
++ local timeout=4
+ /usr/bin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 0 ']'
++ break
++ cat /tmp/tmp.KCpSaC2Mcj
++ cat /tmp/tmp.k8Sl8LCclk
++ rm /tmp/tmp.KCpSaC2Mcj /tmp/tmp.k8Sl8LCclk
++ return 0
+ local client_container=psmdb-client-5dc94d5b48-7ctb7
+ local mongo_flag=
+ kubectl_bin exec psmdb-client-5dc94d5b48-7ctb7 -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-1506.svc.cluster.local/admin '
++ mktemp
+ local LAST_OUT=/tmp/tmp.bAfa0RU5YI
++ mktemp
+ local LAST_ERR=/tmp/tmp.FqE6yNuVK1
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl exec psmdb-client-5dc94d5b48-7ctb7 -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-1506.svc.cluster.local/admin '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 0 ']'
+ break
+ cat /tmp/tmp.bAfa0RU5YI
+ cat /tmp/tmp.FqE6yNuVK1
+ rm /tmp/tmp.bAfa0RU5YI /tmp/tmp.FqE6yNuVK1
+ return 0
+ diff /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1539/e2e-tests/demand-backup-sharded/compare/find.json /tmp/tmp.4LAUoxvZTn/find
+ for i in '$(seq 0 2)'
+ compare_mongos_cmd find myApp:myPass@some-name-mongos.demand-backup-sharded-1506 1 .svc.cluster.local myApp1 test
+ local command=find
+ local uri=myApp:myPass@some-name-mongos.demand-backup-sharded-1506
+ local postfix=1
+ local suffix=.svc.cluster.local
+ local database=myApp1
+ local collection=test
+ run_mongos 'use myApp1\n db.test.find()' myApp:myPass@some-name-mongos.demand-backup-sharded-1506 mongodb .svc.cluster.local
+ local 'command=use myApp1\n db.test.find()'
+ local uri=myApp:myPass@some-name-mongos.demand-backup-sharded-1506
+ local driver=mongodb
+ egrep -v 'I NETWORK|W NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:'
+ local suffix=.svc.cluster.local
+ /usr/bin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.H3ye7PsFuq
+++ mktemp
++ local LAST_ERR=/tmp/tmp.vxP436pzfh
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 0 ']'
++ break
++ cat /tmp/tmp.H3ye7PsFuq
++ cat /tmp/tmp.vxP436pzfh
++ rm /tmp/tmp.H3ye7PsFuq /tmp/tmp.vxP436pzfh
++ return 0
+ local client_container=psmdb-client-5dc94d5b48-7ctb7
+ local mongo_flag=
+ kubectl_bin exec psmdb-client-5dc94d5b48-7ctb7 -- bash -c 'printf '\''use myApp1\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-1506.svc.cluster.local/admin '
++ mktemp
+ local LAST_OUT=/tmp/tmp.DsJuL8jt5J
++ mktemp
+ local LAST_ERR=/tmp/tmp.y7FlV4LqB8
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl exec psmdb-client-5dc94d5b48-7ctb7 -- bash -c 'printf '\''use myApp1\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-1506.svc.cluster.local/admin '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 0 ']'
+ break
+ cat /tmp/tmp.DsJuL8jt5J
+ cat /tmp/tmp.y7FlV4LqB8
+ rm /tmp/tmp.DsJuL8jt5J /tmp/tmp.y7FlV4LqB8
+ return 0
+ diff /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1539/e2e-tests/demand-backup-sharded/compare/find1.json /tmp/tmp.4LAUoxvZTn/find1
+ for i in '$(seq 0 2)'
+ compare_mongos_cmd find myApp:myPass@some-name-mongos.demand-backup-sharded-1506 2 .svc.cluster.local myApp2 test
+ local command=find
+ local uri=myApp:myPass@some-name-mongos.demand-backup-sharded-1506
+ local postfix=2
+ local suffix=.svc.cluster.local
+ local database=myApp2
+ local collection=test
+ run_mongos 'use myApp2\n db.test.find()' myApp:myPass@some-name-mongos.demand-backup-sharded-1506 mongodb .svc.cluster.local
+ local 'command=use myApp2\n db.test.find()'
+ local uri=myApp:myPass@some-name-mongos.demand-backup-sharded-1506
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ egrep -v 'I NETWORK|W NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:'
+ /usr/bin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.mg8CsSYeUn
+++ mktemp
++ local LAST_ERR=/tmp/tmp.JIzUpkS3Ae
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 0 ']'
++ break
++ cat /tmp/tmp.mg8CsSYeUn
++ cat /tmp/tmp.JIzUpkS3Ae
++ rm /tmp/tmp.mg8CsSYeUn /tmp/tmp.JIzUpkS3Ae
++ return 0
+ local client_container=psmdb-client-5dc94d5b48-7ctb7
+ local mongo_flag=
+ kubectl_bin exec psmdb-client-5dc94d5b48-7ctb7 -- bash -c 'printf '\''use myApp2\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-1506.svc.cluster.local/admin '
++ mktemp
+ local LAST_OUT=/tmp/tmp.uLgmVqraUk
++ mktemp
+ local LAST_ERR=/tmp/tmp.AXuSclnZ5O
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl exec psmdb-client-5dc94d5b48-7ctb7 -- bash -c 'printf '\''use myApp2\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-1506.svc.cluster.local/admin '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 0 ']'
+ break
+ cat /tmp/tmp.uLgmVqraUk
+ cat /tmp/tmp.AXuSclnZ5O
+ rm /tmp/tmp.uLgmVqraUk /tmp/tmp.AXuSclnZ5O
+ return 0
+ diff /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1539/e2e-tests/demand-backup-sharded/compare/find2.json /tmp/tmp.4LAUoxvZTn/find2
+ desc 'check backup and restore -- minio'
+ set +o xtrace


-----------------------------------------------------------------------------------
check backup and restore -- minio
-----------------------------------------------------------------------------------

2024-05-01 02:46:38         55 myApp.test.gz
Percona Server for MongoDB shell version v4.4.29-28
connecting to: mongodb://some-name-mongos.demand-backup-sharded-1506.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb
Implicit session: session { "id" : UUID("2d12860c-0a00-4126-895b-ea54f9b091d7") }
Percona Server for MongoDB server version: v7.0.8-5
WARNING: shell and server versions do not match
switched to db myApp
WriteResult({ "nInserted" : 1 })
bye
Percona Server for MongoDB shell version v4.4.29-28
connecting to: mongodb://some-name-mongos.demand-backup-sharded-1506.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb
Implicit session: session { "id" : UUID("1935aaed-0f00-4c0d-be2e-546f5d3f822b") }
Percona Server for MongoDB server version: v7.0.8-5
WARNING: shell and server versions do not match
switched to db myApp1
WriteResult({ "nInserted" : 1 })
bye
Percona Server for MongoDB shell version v4.4.29-28
connecting to: mongodb://some-name-mongos.demand-backup-sharded-1506.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb
Implicit session: session { "id" : UUID("bf59dc2d-0ab3-4f49-af8e-1e21411a45ed") }
Percona Server for MongoDB server version: v7.0.8-5
WARNING: shell and server versions do not match
switched to db myApp2
WriteResult({ "nInserted" : 1 })
bye
perconaservermongodbrestore.psmdb.percona.com/restore-backup-minio created
waiting psmdb-restore/backup-minio to reach ready state............
+ '[' 1 -eq 1 ']'
+ wait_cluster_consistency some-name
+ local cluster_name=some-name
+ local wait_time=32
+ retry=0
+ sleep 7
+ echo -n 'waiting for cluster readyness'
waiting for cluster readyness++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.e5ydOw007Y
+++ mktemp
++ local LAST_ERR=/tmp/tmp.ThaNCPnGmx
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 0 ']'
++ break
++ cat /tmp/tmp.e5ydOw007Y
++ cat /tmp/tmp.ThaNCPnGmx
++ rm /tmp/tmp.e5ydOw007Y /tmp/tmp.ThaNCPnGmx
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 1 -ge 32 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.vrekRnDgPs
+++ mktemp
++ local LAST_ERR=/tmp/tmp.TD1uzbFTWq
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 0 ']'
++ break
++ cat /tmp/tmp.vrekRnDgPs
++ cat /tmp/tmp.TD1uzbFTWq
++ rm /tmp/tmp.vrekRnDgPs /tmp/tmp.TD1uzbFTWq
++ return 0
+ [[ error == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 2 -ge 32 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.herHrkQwJI
+++ mktemp
++ local LAST_ERR=/tmp/tmp.ZCJ3MsQpju
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 0 ']'
++ break
++ cat /tmp/tmp.herHrkQwJI
++ cat /tmp/tmp.ZCJ3MsQpju
++ rm /tmp/tmp.herHrkQwJI /tmp/tmp.ZCJ3MsQpju
++ return 0
+ [[ error == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 3 -ge 32 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.l32UsugbnI
+++ mktemp
++ local LAST_ERR=/tmp/tmp.3zAYRthAjv
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 0 ']'
++ break
++ cat /tmp/tmp.l32UsugbnI
++ cat /tmp/tmp.3zAYRthAjv
++ rm /tmp/tmp.l32UsugbnI /tmp/tmp.3zAYRthAjv
++ return 0
+ [[ error == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 4 -ge 32 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.gUZmUdcRk2
+++ mktemp
++ local LAST_ERR=/tmp/tmp.SwAMPLdbSq
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 0 ']'
++ break
++ cat /tmp/tmp.gUZmUdcRk2
++ cat /tmp/tmp.SwAMPLdbSq
++ rm /tmp/tmp.gUZmUdcRk2 /tmp/tmp.SwAMPLdbSq
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 5 -ge 32 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.Qops4fPQbM
+++ mktemp
++ local LAST_ERR=/tmp/tmp.RjmRuNx1iA
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 0 ']'
++ break
++ cat /tmp/tmp.Qops4fPQbM
++ cat /tmp/tmp.RjmRuNx1iA
++ rm /tmp/tmp.Qops4fPQbM /tmp/tmp.RjmRuNx1iA
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 6 -ge 32 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.UentcfnRG9
+++ mktemp
++ local LAST_ERR=/tmp/tmp.rYbULYcwOg
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 0 ']'
++ break
++ cat /tmp/tmp.UentcfnRG9
++ cat /tmp/tmp.rYbULYcwOg
++ rm /tmp/tmp.UentcfnRG9 /tmp/tmp.rYbULYcwOg
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 7 -ge 32 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.wJaB0FMDBn
+++ mktemp
++ local LAST_ERR=/tmp/tmp.KKgGL5QGAF
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 0 ']'
++ break
++ cat /tmp/tmp.wJaB0FMDBn
++ cat /tmp/tmp.KKgGL5QGAF
++ rm /tmp/tmp.wJaB0FMDBn /tmp/tmp.KKgGL5QGAF
++ return 0
+ [[ ready == \r\e\a\d\y ]]
+ check_data
+ local postfix=
++ seq 0 2
+ for i in '$(seq 0 2)'
+ compare_mongos_cmd find myApp:myPass@some-name-mongos.demand-backup-sharded-1506 '' .svc.cluster.local myApp test
+ local command=find
+ local uri=myApp:myPass@some-name-mongos.demand-backup-sharded-1506
+ local postfix=
+ local suffix=.svc.cluster.local
+ local database=myApp
+ local collection=test
+ /usr/bin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
+ egrep -v 'I NETWORK|W NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:'
+ run_mongos 'use myApp\n db.test.find()' myApp:myPass@some-name-mongos.demand-backup-sharded-1506 mongodb .svc.cluster.local
+ local 'command=use myApp\n db.test.find()'
+ local uri=myApp:myPass@some-name-mongos.demand-backup-sharded-1506
+ local driver=mongodb
+ local suffix=.svc.cluster.local
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.rr8mdxbA5h
+++ mktemp
++ local LAST_ERR=/tmp/tmp.bgAPwUjJSa
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 0 ']'
++ break
++ cat /tmp/tmp.rr8mdxbA5h
++ cat /tmp/tmp.bgAPwUjJSa
++ rm /tmp/tmp.rr8mdxbA5h /tmp/tmp.bgAPwUjJSa
++ return 0
+ local client_container=psmdb-client-5dc94d5b48-7ctb7
+ local mongo_flag=
+ kubectl_bin exec psmdb-client-5dc94d5b48-7ctb7 -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-1506.svc.cluster.local/admin '
++ mktemp
+ local LAST_OUT=/tmp/tmp.asodsqBRVZ
++ mktemp
+ local LAST_ERR=/tmp/tmp.U2a44iRQDC
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl exec psmdb-client-5dc94d5b48-7ctb7 -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-1506.svc.cluster.local/admin '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 0 ']'
+ break
+ cat /tmp/tmp.asodsqBRVZ
+ cat /tmp/tmp.U2a44iRQDC
+ rm /tmp/tmp.asodsqBRVZ /tmp/tmp.U2a44iRQDC
+ return 0
+ diff /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1539/e2e-tests/demand-backup-sharded/compare/find.json /tmp/tmp.4LAUoxvZTn/find
+ for i in '$(seq 0 2)'
+ compare_mongos_cmd find myApp:myPass@some-name-mongos.demand-backup-sharded-1506 1 .svc.cluster.local myApp1 test
+ local command=find
+ local uri=myApp:myPass@some-name-mongos.demand-backup-sharded-1506
+ local postfix=1
+ local suffix=.svc.cluster.local
+ local database=myApp1
+ local collection=test
+ run_mongos 'use myApp1\n db.test.find()' myApp:myPass@some-name-mongos.demand-backup-sharded-1506 mongodb .svc.cluster.local
+ local 'command=use myApp1\n db.test.find()'
+ local uri=myApp:myPass@some-name-mongos.demand-backup-sharded-1506
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ /usr/bin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
+ egrep -v 'I NETWORK|W NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:'
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.8VetRkzoRQ
+++ mktemp
++ local LAST_ERR=/tmp/tmp.vrXsYKPEm9
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 0 ']'
++ break
++ cat /tmp/tmp.8VetRkzoRQ
++ cat /tmp/tmp.vrXsYKPEm9
++ rm /tmp/tmp.8VetRkzoRQ /tmp/tmp.vrXsYKPEm9
++ return 0
+ local client_container=psmdb-client-5dc94d5b48-7ctb7
+ local mongo_flag=
+ kubectl_bin exec psmdb-client-5dc94d5b48-7ctb7 -- bash -c 'printf '\''use myApp1\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-1506.svc.cluster.local/admin '
++ mktemp
+ local LAST_OUT=/tmp/tmp.5Bqre2wx1C
++ mktemp
+ local LAST_ERR=/tmp/tmp.l5odPusokB
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl exec psmdb-client-5dc94d5b48-7ctb7 -- bash -c 'printf '\''use myApp1\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-1506.svc.cluster.local/admin '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 0 ']'
+ break
+ cat /tmp/tmp.5Bqre2wx1C
+ cat /tmp/tmp.l5odPusokB
+ rm /tmp/tmp.5Bqre2wx1C /tmp/tmp.l5odPusokB
+ return 0
+ diff /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1539/e2e-tests/demand-backup-sharded/compare/find1.json /tmp/tmp.4LAUoxvZTn/find1
+ for i in '$(seq 0 2)'
+ compare_mongos_cmd find myApp:myPass@some-name-mongos.demand-backup-sharded-1506 2 .svc.cluster.local myApp2 test
+ local command=find
+ local uri=myApp:myPass@some-name-mongos.demand-backup-sharded-1506
+ local postfix=2
+ local suffix=.svc.cluster.local
+ local database=myApp2
+ local collection=test
+ /usr/bin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
+ run_mongos 'use myApp2\n db.test.find()' myApp:myPass@some-name-mongos.demand-backup-sharded-1506 mongodb .svc.cluster.local
+ local 'command=use myApp2\n db.test.find()'
+ local uri=myApp:myPass@some-name-mongos.demand-backup-sharded-1506
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ egrep -v 'I NETWORK|W NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:'
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.IzM6EBbcbi
+++ mktemp
++ local LAST_ERR=/tmp/tmp.k6ZH9ko1Ci
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 0 ']'
++ break
++ cat /tmp/tmp.IzM6EBbcbi
++ cat /tmp/tmp.k6ZH9ko1Ci
++ rm /tmp/tmp.IzM6EBbcbi /tmp/tmp.k6ZH9ko1Ci
++ return 0
+ local client_container=psmdb-client-5dc94d5b48-7ctb7
+ local mongo_flag=
+ kubectl_bin exec psmdb-client-5dc94d5b48-7ctb7 -- bash -c 'printf '\''use myApp2\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-1506.svc.cluster.local/admin '
++ mktemp
+ local LAST_OUT=/tmp/tmp.678Le6H4Kr
++ mktemp
+ local LAST_ERR=/tmp/tmp.b9rXQGrbd1
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl exec psmdb-client-5dc94d5b48-7ctb7 -- bash -c 'printf '\''use myApp2\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-1506.svc.cluster.local/admin '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 0 ']'
+ break
+ cat /tmp/tmp.678Le6H4Kr
+ cat /tmp/tmp.b9rXQGrbd1
+ rm /tmp/tmp.678Le6H4Kr /tmp/tmp.b9rXQGrbd1
+ return 0
+ diff /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1539/e2e-tests/demand-backup-sharded/compare/find2.json /tmp/tmp.4LAUoxvZTn/find2
+ desc 'delete backup and check if it is removed from bucket -- minio'
+ set +o xtrace


-----------------------------------------------------------------------------------
delete backup and check if it is removed from bucket -- minio
-----------------------------------------------------------------------------------

perconaservermongodbbackup.psmdb.percona.com "backup-aws-s3" deleted
perconaservermongodbbackup.psmdb.percona.com "backup-azure-blob" deleted
perconaservermongodbbackup.psmdb.percona.com "backup-gcp-cs" deleted
perconaservermongodbbackup.psmdb.percona.com "backup-minio" deleted
If you don't see a command prompt, try pressing enter.
warning: couldn't attach to pod/aws-cli, falling back to streaming logs: unable to upgrade connection: container aws-cli not found in pod aws-cli_demand-backup-sharded-1506


-----------------------------------------------------------------------------------
check for passwords leak
-----------------------------------------------------------------------------------

secrets=YmFja3VwMTIzNDU2 Y2x1c3RlckFkbWluMTIzNDU2 Y2x1c3Rlck1vbml0b3IxMjM0NTY= ZGF0YWJhc2VBZG1pbjEyMzQ1Ng== dXNlckFkbWluMTIzNDU2 YmFja3VwMTIzNDU2 Y2x1c3RlckFkbWluMTIzNDU2 Y2x1c3Rlck1vbml0b3IxMjM0NTY= ZGF0YWJhc2VBZG1pbjEyMzQ1Ng== dXNlckFkbWluMTIzNDU2
passwords=backup123456 clusterAdmin123456 clusterMonitor123456 databaseAdmin123456 userAdmin123456 backup123456 clusterAdmin123456 clusterMonitor123456 databaseAdmin123456 userAdmin123456 YmFja3VwMTIzNDU2 Y2x1c3RlckFkbWluMTIzNDU2 Y2x1c3Rlck1vbml0b3IxMjM0NTY= ZGF0YWJhc2VBZG1pbjEyMzQ1Ng== dXNlckFkbWluMTIzNDU2 YmFja3VwMTIzNDU2 Y2x1c3RlckFkbWluMTIzNDU2 Y2x1c3Rlck1vbml0b3IxMjM0NTY= ZGF0YWJhc2VBZG1pbjEyMzQ1Ng== dXNlckFkbWluMTIzNDU2
pods=minio-service-57dd49b-96zkd psmdb-client-5dc94d5b48-7ctb7 some-name-cfg-0 some-name-cfg-1 some-name-cfg-2 some-name-mongos-0 some-name-mongos-1 some-name-mongos-2 some-name-rs0-0 some-name-rs0-1 some-name-rs0-2 some-name-rs1-0 some-name-rs1-1 some-name-rs1-2 some-name-rs2-0 some-name-rs2-1 some-name-rs2-2
logs saved in: /tmp/tmp.4LAUoxvZTn/logs_output-minio-service-57dd49b-96zkd-minio.txt

logs saved in: /tmp/tmp.4LAUoxvZTn/logs_output-psmdb-client-5dc94d5b48-7ctb7-psmdb-client.txt

logs saved in: /tmp/tmp.4LAUoxvZTn/logs_output-some-name-cfg-0-mongod.txt
logs saved in: /tmp/tmp.4LAUoxvZTn/logs_output-some-name-cfg-0-cfg-sidecar-1.txt
logs saved in: /tmp/tmp.4LAUoxvZTn/logs_output-some-name-cfg-0-backup-agent.txt

logs saved in: /tmp/tmp.4LAUoxvZTn/logs_output-some-name-cfg-1-mongod.txt
logs saved in: /tmp/tmp.4LAUoxvZTn/logs_output-some-name-cfg-1-cfg-sidecar-1.txt
logs saved in: /tmp/tmp.4LAUoxvZTn/logs_output-some-name-cfg-1-backup-agent.txt

logs saved in: /tmp/tmp.4LAUoxvZTn/logs_output-some-name-cfg-2-mongod.txt
logs saved in: /tmp/tmp.4LAUoxvZTn/logs_output-some-name-cfg-2-cfg-sidecar-1.txt
logs saved in: /tmp/tmp.4LAUoxvZTn/logs_output-some-name-cfg-2-backup-agent.txt

logs saved in: /tmp/tmp.4LAUoxvZTn/logs_output-some-name-mongos-0-mongos.txt
logs saved in: /tmp/tmp.4LAUoxvZTn/logs_output-some-name-mongos-0-mongos-sidecar-1.txt

logs saved in: /tmp/tmp.4LAUoxvZTn/logs_output-some-name-mongos-1-mongos.txt
logs saved in: /tmp/tmp.4LAUoxvZTn/logs_output-some-name-mongos-1-mongos-sidecar-1.txt

logs saved in: /tmp/tmp.4LAUoxvZTn/logs_output-some-name-mongos-2-mongos.txt
logs saved in: /tmp/tmp.4LAUoxvZTn/logs_output-some-name-mongos-2-mongos-sidecar-1.txt

logs saved in: /tmp/tmp.4LAUoxvZTn/logs_output-some-name-rs0-0-mongod.txt
logs saved in: /tmp/tmp.4LAUoxvZTn/logs_output-some-name-rs0-0-backup-agent.txt

logs saved in: /tmp/tmp.4LAUoxvZTn/logs_output-some-name-rs0-1-mongod.txt
logs saved in: /tmp/tmp.4LAUoxvZTn/logs_output-some-name-rs0-1-backup-agent.txt

logs saved in: /tmp/tmp.4LAUoxvZTn/logs_output-some-name-rs0-2-mongod.txt
logs saved in: /tmp/tmp.4LAUoxvZTn/logs_output-some-name-rs0-2-backup-agent.txt

logs saved in: /tmp/tmp.4LAUoxvZTn/logs_output-some-name-rs1-0-mongod.txt
logs saved in: /tmp/tmp.4LAUoxvZTn/logs_output-some-name-rs1-0-backup-agent.txt

logs saved in: /tmp/tmp.4LAUoxvZTn/logs_output-some-name-rs1-1-mongod.txt
logs saved in: /tmp/tmp.4LAUoxvZTn/logs_output-some-name-rs1-1-backup-agent.txt

logs saved in: /tmp/tmp.4LAUoxvZTn/logs_output-some-name-rs1-2-mongod.txt
logs saved in: /tmp/tmp.4LAUoxvZTn/logs_output-some-name-rs1-2-backup-agent.txt

logs saved in: /tmp/tmp.4LAUoxvZTn/logs_output-some-name-rs2-0-mongod.txt
logs saved in: /tmp/tmp.4LAUoxvZTn/logs_output-some-name-rs2-0-rs-sidecar-1.txt
logs saved in: /tmp/tmp.4LAUoxvZTn/logs_output-some-name-rs2-0-backup-agent.txt

logs saved in: /tmp/tmp.4LAUoxvZTn/logs_output-some-name-rs2-1-mongod.txt
logs saved in: /tmp/tmp.4LAUoxvZTn/logs_output-some-name-rs2-1-rs-sidecar-1.txt
logs saved in: /tmp/tmp.4LAUoxvZTn/logs_output-some-name-rs2-1-backup-agent.txt

logs saved in: /tmp/tmp.4LAUoxvZTn/logs_output-some-name-rs2-2-mongod.txt
logs saved in: /tmp/tmp.4LAUoxvZTn/logs_output-some-name-rs2-2-rs-sidecar-1.txt
logs saved in: /tmp/tmp.4LAUoxvZTn/logs_output-some-name-rs2-2-backup-agent.txt

logs saved in: /tmp/tmp.4LAUoxvZTn/logs_output-percona-server-mongodb-operator-699b5984b6-fz5t9-percona-server-mongodb-operator.txt

runtimeclass.node.k8s.io "container-rc" deleted


-----------------------------------------------------------------------------------
destroy cluster/operator and all other resources
-----------------------------------------------------------------------------------



-----------------------------------------------------------------------------------
get and delete old CRDs and RBAC
-----------------------------------------------------------------------------------

customresourcedefinition.apiextensions.k8s.io "perconaservermongodbbackups.psmdb.percona.com" deleted
customresourcedefinition.apiextensions.k8s.io "perconaservermongodbrestores.psmdb.percona.com" deleted
customresourcedefinition.apiextensions.k8s.io "perconaservermongodbs.psmdb.percona.com" deleted
error: the server doesn't have a resource type "perconaservermongodbbackups"
+ kubectl patch perconaservermongodbbackups.psmdb.percona.com -n sh --type=merge -p '{"metadata":{"finalizers":[]}}'
error: the server doesn't have a resource type "perconaservermongodbbackups"
error: the server doesn't have a resource type "perconaservermongodbrestores"
+ kubectl patch perconaservermongodbrestores.psmdb.percona.com -n sh --type=merge -p '{"metadata":{"finalizers":[]}}'
error: the server doesn't have a resource type "perconaservermongodbrestores"
error: the server doesn't have a resource type "perconaservermongodbs"
+ kubectl patch perconaservermongodbs.psmdb.percona.com -n sh --type=merge -p '{"metadata":{"finalizers":[]}}'
error: the server doesn't have a resource type "perconaservermongodbs"
clusterrole.rbac.authorization.k8s.io "percona-server-mongodb-operator" deleted
clusterrolebinding.rbac.authorization.k8s.io "service-account-percona-server-mongodb-operator" deleted