++ echo 'Log: /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1536/e2e-tests/logs/demand-backup-sharded.log'
Log: /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1536/e2e-tests/logs/demand-backup-sharded.log
++ '[' -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1536/e2e-tests/conf/cloud-secret.yml ']'
++ SKIP_BACKUPS_TO_AWS_GCP_AZURE=
++ oc get projects
++ kubectl get nodes
++ grep '^minikube'
+++ kubectl version -o json
+++ jq -r .serverVersion.gitVersion
+++ grep '\-eks\-'
WARNING: version difference between client (1.30) and server (1.26) exceeds the supported minor version skew of +/-1
++ '[' ']'
++ EKS=0
+++ grep gke
+++ kubectl version -o json
+++ jq -r .serverVersion.gitVersion
WARNING: version difference between client (1.30) and server (1.26) exceeds the supported minor version skew of +/-1
++ '[' v1.26.15-gke.1191000 ']'
++ GKE=1
+++ /usr/bin/sed -r 's/[^0-9.]+//g'
+++ jq -r '.serverVersion.major + "." + .serverVersion.minor'
+++ kubectl version -o json
WARNING: version difference between client (1.30) and server (1.26) exceeds the supported minor version skew of +/-1
++ KUBE_VERSION=1.26
+ set_debug
+ [[ 0 == 1 ]]
+ set +o xtrace


-----------------------------------------------------------------------------------
get and delete old CRDs and RBAC
-----------------------------------------------------------------------------------

error: the server doesn't have a resource type "perconaservermongodbbackups"
+ kubectl patch perconaservermongodbbackups.psmdb.percona.com -n sh --type=merge -p '{"metadata":{"finalizers":[]}}'
error: the server doesn't have a resource type "perconaservermongodbbackups"
error: the server doesn't have a resource type "perconaservermongodbrestores"
+ kubectl patch perconaservermongodbrestores.psmdb.percona.com -n sh --type=merge -p '{"metadata":{"finalizers":[]}}'
error: the server doesn't have a resource type "perconaservermongodbrestores"
error: the server doesn't have a resource type "perconaservermongodbs"
+ kubectl patch perconaservermongodbs.psmdb.percona.com -n sh --type=merge -p '{"metadata":{"finalizers":[]}}'
error: the server doesn't have a resource type "perconaservermongodbs"
Error from server (NotFound): customresourcedefinitions.apiextensions.k8s.io "null" not found
Error from server (NotFound): customresourcedefinitions.apiextensions.k8s.io "null" not found
Error from server (NotFound): customresourcedefinitions.apiextensions.k8s.io "null" not found
Error from server (NotFound): customresourcedefinitions.apiextensions.k8s.io "null" not found


-----------------------------------------------------------------------------------
destroy chaos-mesh
-----------------------------------------------------------------------------------

error: resource(s) were provided, but no name was specified
error: resource(s) were provided, but no name was specified
error: resource(s) were provided, but no name was specified
error: resource(s) were provided, but no name was specified
error: resource(s) were provided, but no name was specified
error: resource(s) were provided, but no name was specified


-----------------------------------------------------------------------------------
cleaned up all old namespaces
-----------------------------------------------------------------------------------



-----------------------------------------------------------------------------------
cleaned up old namespaces psmdb-operator
-----------------------------------------------------------------------------------

error: resource(s) were provided, but no name was specified


-----------------------------------------------------------------------------------
create namespace psmdb-operator
-----------------------------------------------------------------------------------

namespace/psmdb-operator created
Context "gke_cloud-dev-112233_us-central1-a_jen-psmdb-1536-4cc529f8-10-cluster2" modified.


-----------------------------------------------------------------------------------
start PSMDB operator
-----------------------------------------------------------------------------------

customresourcedefinition.apiextensions.k8s.io/perconaservermongodbbackups.psmdb.percona.com serverside-applied
customresourcedefinition.apiextensions.k8s.io/perconaservermongodbrestores.psmdb.percona.com serverside-applied
customresourcedefinition.apiextensions.k8s.io/perconaservermongodbs.psmdb.percona.com serverside-applied
clusterrole.rbac.authorization.k8s.io/percona-server-mongodb-operator created
serviceaccount/percona-server-mongodb-operator created
clusterrolebinding.rbac.authorization.k8s.io/service-account-percona-server-mongodb-operator created
deployment.apps/percona-server-mongodb-operator created
waiting for pod/percona-server-mongodb-operator-75c69449b4-jcrdz to be ready.OK


-----------------------------------------------------------------------------------
destroy chaos-mesh
-----------------------------------------------------------------------------------

error: resource(s) were provided, but no name was specified
error: resource(s) were provided, but no name was specified
error: resource(s) were provided, but no name was specified
error: resource(s) were provided, but no name was specified
error: resource(s) were provided, but no name was specified
error: resource(s) were provided, but no name was specified


-----------------------------------------------------------------------------------
cleaned up all old namespaces
-----------------------------------------------------------------------------------



-----------------------------------------------------------------------------------
cleaned up old namespaces demand-backup-sharded-32015
-----------------------------------------------------------------------------------

error: resource(s) were provided, but no name was specified


-----------------------------------------------------------------------------------
create namespace demand-backup-sharded-32015
-----------------------------------------------------------------------------------

namespace/demand-backup-sharded-32015 created
Context "gke_cloud-dev-112233_us-central1-a_jen-psmdb-1536-4cc529f8-10-cluster2" modified.


-----------------------------------------------------------------------------------
install Minio
-----------------------------------------------------------------------------------

Error: uninstall: Release not loaded: minio-service: release: not found
"minio" has been removed from your repositories
"minio" has been added to your repositories
NAME: minio-service
LAST DEPLOYED: Thu May  2 14:23:12 2024
NAMESPACE: demand-backup-sharded-32015
STATUS: deployed
REVISION: 1
TEST SUITE: None
NOTES:
MinIO can be accessed via port 9000 on the following DNS name from within your cluster:
minio-service.demand-backup-sharded-32015.svc.cluster.local

To access MinIO from localhost, run the below commands:

  1. export POD_NAME=$(kubectl get pods --namespace demand-backup-sharded-32015 -l "release=minio-service" -o jsonpath="{.items[0].metadata.name}")

  2. kubectl port-forward $POD_NAME 9000 --namespace demand-backup-sharded-32015

Read more about port forwarding here: http://kubernetes.io/docs/user-guide/kubectl/kubectl_port-forward/

You can now access MinIO server on http://localhost:9000. Follow the below steps to connect to MinIO server with mc client:

  1. Download the MinIO mc client - https://min.io/docs/minio/linux/reference/minio-mc.html#quickstart

  2. export MC_HOST_minio-service-local=http://$(kubectl get secret --namespace demand-backup-sharded-32015 minio-service -o jsonpath="{.data.rootUser}" | base64 --decode):$(kubectl get secret --namespace demand-backup-sharded-32015 minio-service -o jsonpath="{.data.rootPassword}" | base64 --decode)@localhost:9000

  3. mc ls minio-service-local
waiting for pod/minio-service-57dd49b-nqprw to be ready.OK
service/minio-service created
make_bucket: operator-testing
pod "aws-cli" deleted
If you don't see a command prompt, try pressing enter.
warning: couldn't attach to pod/aws-cli, falling back to streaming logs: Internal error occurred: error attaching to container: container is in CONTAINER_EXITED state


-----------------------------------------------------------------------------------
create first PSMDB cluster
-----------------------------------------------------------------------------------

secret/some-users created
deployment.apps/psmdb-client created


-----------------------------------------------------------------------------------
create secrets for cloud storages
-----------------------------------------------------------------------------------

secret/minio-secret created
secret/aws-s3-secret created
secret/gcp-cs-secret created
secret/azure-secret created
runtimeclass.node.k8s.io/container-rc unchanged
perconaservermongodb.psmdb.percona.com/some-name created


-----------------------------------------------------------------------------------
check if all 3 Pods started
-----------------------------------------------------------------------------------

waiting for pod/some-name-rs0-0 to be ready..........OK
waiting for pod/some-name-rs0-1 to be ready.........OK
waiting for pod/some-name-rs0-2 to be ready.........OK
Waiting for cluster readyness............................................................
waiting for pod/some-name-cfg-0 to be ready.OK
waiting for pod/some-name-cfg-1 to be ready.OK
waiting for pod/some-name-cfg-2 to be ready.OK
waiting for pod/some-name-mongos-0 to be ready.OK
waiting for pod/some-name-mongos-1 to be ready.OK
waiting for pod/some-name-mongos-2 to be ready.OK
Waiting for cluster readyness


-----------------------------------------------------------------------------------
check if service and statefulset created with expected config
-----------------------------------------------------------------------------------

secret/some-name-mongos created
waiting for pod/some-name-mongos-0 to be ready.OK
waiting for pod/some-name-mongos-1 to be ready.OK
waiting for pod/some-name-mongos-2 to be ready.OK
Waiting for cluster readyness...........................................................................................


-----------------------------------------------------------------------------------
write data, read from all
-----------------------------------------------------------------------------------

Percona Server for MongoDB shell version v4.4.29-28
connecting to: mongodb://some-name-mongos.demand-backup-sharded-32015.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb
Implicit session: session { "id" : UUID("e932ce32-27a8-4c57-8272-4ab21e21d095") }
Percona Server for MongoDB server version: v7.0.8-5
WARNING: shell and server versions do not match
Successfully added user: {
	"user" : "myApp",
	"roles" : [
		{
			"db" : "myApp",
			"role" : "readWrite"
		},
		{
			"db" : "myApp1",
			"role" : "readWrite"
		},
		{
			"db" : "myApp2",
			"role" : "readWrite"
		}
	]
}
bye
Percona Server for MongoDB shell version v4.4.29-28
connecting to: mongodb://some-name-mongos.demand-backup-sharded-32015.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb
Implicit session: session { "id" : UUID("734331ab-792b-4ba7-a273-839b16e14061") }
Percona Server for MongoDB server version: v7.0.8-5
WARNING: shell and server versions do not match
{
	"ok" : 1,
	"$clusterTime" : {
		"clusterTime" : Timestamp(1714660367, 12),
		"signature" : {
			"hash" : BinData(0,"p2EWvtpSATq4bCP9d0nsdsqhbT4="),
			"keyId" : NumberLong("7364408262982107158")
		}
	},
	"operationTime" : Timestamp(1714660367, 4)
}
bye
Percona Server for MongoDB shell version v4.4.29-28
connecting to: mongodb://some-name-mongos.demand-backup-sharded-32015.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb
Implicit session: session { "id" : UUID("df425ec1-94f2-4bfc-8240-a456cb78b55a") }
Percona Server for MongoDB server version: v7.0.8-5
WARNING: shell and server versions do not match
{
	"ok" : 1,
	"$clusterTime" : {
		"clusterTime" : Timestamp(1714660371, 2),
		"signature" : {
			"hash" : BinData(0,"d4bDOkaFthfHshNp+RSbxdXuk1U="),
			"keyId" : NumberLong("7364408262982107158")
		}
	},
	"operationTime" : Timestamp(1714660370, 3)
}
bye
Percona Server for MongoDB shell version v4.4.29-28
connecting to: mongodb://some-name-mongos.demand-backup-sharded-32015.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb
Implicit session: session { "id" : UUID("5cd3c3e3-05bf-4b91-9e8e-72b605804705") }
Percona Server for MongoDB server version: v7.0.8-5
WARNING: shell and server versions do not match
{
	"ok" : 1,
	"$clusterTime" : {
		"clusterTime" : Timestamp(1714660374, 8),
		"signature" : {
			"hash" : BinData(0,"9bR3elVN+2tfxuL52BJixy8BqkM="),
			"keyId" : NumberLong("7364408262982107158")
		}
	},
	"operationTime" : Timestamp(1714660374, 3)
}
bye
Percona Server for MongoDB shell version v4.4.29-28
connecting to: mongodb://some-name-mongos.demand-backup-sharded-32015.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb
Implicit session: session { "id" : UUID("83ba8d00-2a51-42c4-a46b-ed130f9583b5") }
Percona Server for MongoDB server version: v7.0.8-5
WARNING: shell and server versions do not match
switched to db myApp
WriteResult({ "nInserted" : 1 })
bye
Percona Server for MongoDB shell version v4.4.29-28
connecting to: mongodb://some-name-mongos.demand-backup-sharded-32015.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb
Implicit session: session { "id" : UUID("0626df47-9143-422d-9309-f25b68d6538d") }
Percona Server for MongoDB server version: v7.0.8-5
WARNING: shell and server versions do not match
switched to db myApp1
WriteResult({ "nInserted" : 1 })
bye
Percona Server for MongoDB shell version v4.4.29-28
connecting to: mongodb://some-name-mongos.demand-backup-sharded-32015.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb
Implicit session: session { "id" : UUID("d18a819e-27c9-45c6-a699-9d78986a47bb") }
Percona Server for MongoDB server version: v7.0.8-5
WARNING: shell and server versions do not match
switched to db myApp2
WriteResult({ "nInserted" : 1 })
bye
some-name-rs0-0
some-name-rs0-1
some-name-rs0-2
some-name-rs1-0
some-name-rs1-1
some-name-rs1-2
some-name-rs2-0
some-name-rs2-1
some-name-rs2-2


-----------------------------------------------------------------------------------
run backups
-----------------------------------------------------------------------------------



-----------------------------------------------------------------------------------
run backup backup-minio
-----------------------------------------------------------------------------------

perconaservermongodbbackup.psmdb.percona.com/backup-minio created


-----------------------------------------------------------------------------------
run backup backup-aws-s3
-----------------------------------------------------------------------------------

perconaservermongodbbackup.psmdb.percona.com/backup-aws-s3 created


-----------------------------------------------------------------------------------
run backup backup-gcp-cs
-----------------------------------------------------------------------------------

perconaservermongodbbackup.psmdb.percona.com/backup-gcp-cs created


-----------------------------------------------------------------------------------
run backup backup-azure-blob
-----------------------------------------------------------------------------------

perconaservermongodbbackup.psmdb.percona.com/backup-azure-blob created
backup-aws-s3................................
backup-gcp-cs..................
backup-azure-blob...................................
backup-minio.


-----------------------------------------------------------------------------------
check backup and restore -- aws-s3
-----------------------------------------------------------------------------------

Percona Server for MongoDB shell version v4.4.29-28
connecting to: mongodb://some-name-mongos.demand-backup-sharded-32015.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb
Implicit session: session { "id" : UUID("9d503272-c2cf-45a9-9d8e-98379a621c56") }
Percona Server for MongoDB server version: v7.0.8-5
WARNING: shell and server versions do not match
switched to db myApp
WriteResult({ "nInserted" : 1 })
bye
Percona Server for MongoDB shell version v4.4.29-28
connecting to: mongodb://some-name-mongos.demand-backup-sharded-32015.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb
Implicit session: session { "id" : UUID("32fddd16-e57b-43aa-9def-99c79eed3fbd") }
Percona Server for MongoDB server version: v7.0.8-5
WARNING: shell and server versions do not match
switched to db myApp1
WriteResult({ "nInserted" : 1 })
bye
Percona Server for MongoDB shell version v4.4.29-28
connecting to: mongodb://some-name-mongos.demand-backup-sharded-32015.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb
Implicit session: session { "id" : UUID("31a7da94-e31b-42d1-8c5b-3d9d9207c864") }
Percona Server for MongoDB server version: v7.0.8-5
WARNING: shell and server versions do not match
switched to db myApp2
WriteResult({ "nInserted" : 1 })
bye
perconaservermongodbrestore.psmdb.percona.com/restore-backup-aws-s3 created
waiting psmdb-restore/backup-aws-s3 to reach ready state............
+ '[' 1 -eq 1 ']'
+ wait_cluster_consistency some-name
+ local cluster_name=some-name
+ local wait_time=32
+ retry=0
+ sleep 7
+ echo -n 'waiting for cluster readyness'
waiting for cluster readyness++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.2BE3TiZCn6
+++ mktemp
++ local LAST_ERR=/tmp/tmp.vZHSPdpGzi
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 0 ']'
++ break
++ cat /tmp/tmp.2BE3TiZCn6
++ cat /tmp/tmp.vZHSPdpGzi
++ rm /tmp/tmp.2BE3TiZCn6 /tmp/tmp.vZHSPdpGzi
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 1 -ge 32 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.BYzu2jANps
+++ mktemp
++ local LAST_ERR=/tmp/tmp.jixHnklQ8M
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 0 ']'
++ break
++ cat /tmp/tmp.BYzu2jANps
++ cat /tmp/tmp.jixHnklQ8M
++ rm /tmp/tmp.BYzu2jANps /tmp/tmp.jixHnklQ8M
++ return 0
+ [[ error == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 2 -ge 32 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.yYhz2ulGoh
+++ mktemp
++ local LAST_ERR=/tmp/tmp.Apm59THlxk
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 0 ']'
++ break
++ cat /tmp/tmp.yYhz2ulGoh
++ cat /tmp/tmp.Apm59THlxk
++ rm /tmp/tmp.yYhz2ulGoh /tmp/tmp.Apm59THlxk
++ return 0
+ [[ error == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 3 -ge 32 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.vSYptxVnld
+++ mktemp
++ local LAST_ERR=/tmp/tmp.swHx3NiQsh
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 0 ']'
++ break
++ cat /tmp/tmp.vSYptxVnld
++ cat /tmp/tmp.swHx3NiQsh
++ rm /tmp/tmp.vSYptxVnld /tmp/tmp.swHx3NiQsh
++ return 0
+ [[ error == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 4 -ge 32 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.vJDMtOXPb2
+++ mktemp
++ local LAST_ERR=/tmp/tmp.LogDh16SLM
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 0 ']'
++ break
++ cat /tmp/tmp.vJDMtOXPb2
++ cat /tmp/tmp.LogDh16SLM
++ rm /tmp/tmp.vJDMtOXPb2 /tmp/tmp.LogDh16SLM
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 5 -ge 32 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.xKGsuWL047
+++ mktemp
++ local LAST_ERR=/tmp/tmp.mLmP8tUf8t
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 0 ']'
++ break
++ cat /tmp/tmp.xKGsuWL047
++ cat /tmp/tmp.mLmP8tUf8t
++ rm /tmp/tmp.xKGsuWL047 /tmp/tmp.mLmP8tUf8t
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 6 -ge 32 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.HaPAocrac5
+++ mktemp
++ local LAST_ERR=/tmp/tmp.EDNU0yol62
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 0 ']'
++ break
++ cat /tmp/tmp.HaPAocrac5
++ cat /tmp/tmp.EDNU0yol62
++ rm /tmp/tmp.HaPAocrac5 /tmp/tmp.EDNU0yol62
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 7 -ge 32 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.5y4FAMWDdJ
+++ mktemp
++ local LAST_ERR=/tmp/tmp.7rJkulQhmp
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 0 ']'
++ break
++ cat /tmp/tmp.5y4FAMWDdJ
++ cat /tmp/tmp.7rJkulQhmp
++ rm /tmp/tmp.5y4FAMWDdJ /tmp/tmp.7rJkulQhmp
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 8 -ge 32 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.uR0RPIoTdY
+++ mktemp
++ local LAST_ERR=/tmp/tmp.97CubLiXjR
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 0 ']'
++ break
++ cat /tmp/tmp.uR0RPIoTdY
++ cat /tmp/tmp.97CubLiXjR
++ rm /tmp/tmp.uR0RPIoTdY /tmp/tmp.97CubLiXjR
++ return 0
+ [[ ready == \r\e\a\d\y ]]
+ check_data
+ local postfix=
++ seq 0 2
+ for i in '$(seq 0 2)'
+ compare_mongos_cmd find myApp:myPass@some-name-mongos.demand-backup-sharded-32015 '' .svc.cluster.local myApp test
+ local command=find
+ local uri=myApp:myPass@some-name-mongos.demand-backup-sharded-32015
+ local postfix=
+ local suffix=.svc.cluster.local
+ local database=myApp
+ local collection=test
+ run_mongos 'use myApp\n db.test.find()' myApp:myPass@some-name-mongos.demand-backup-sharded-32015 mongodb .svc.cluster.local
+ local 'command=use myApp\n db.test.find()'
+ local uri=myApp:myPass@some-name-mongos.demand-backup-sharded-32015
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ egrep -v 'I NETWORK|W NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:'
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
+ /usr/bin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
++ local LAST_OUT=/tmp/tmp.zsglZXbizw
+++ mktemp
++ local LAST_ERR=/tmp/tmp.Fg4sGwnFx8
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 0 ']'
++ break
++ cat /tmp/tmp.zsglZXbizw
++ cat /tmp/tmp.Fg4sGwnFx8
++ rm /tmp/tmp.zsglZXbizw /tmp/tmp.Fg4sGwnFx8
++ return 0
+ local client_container=psmdb-client-5dc94d5b48-k2rvq
+ local mongo_flag=
+ kubectl_bin exec psmdb-client-5dc94d5b48-k2rvq -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-32015.svc.cluster.local/admin '
++ mktemp
+ local LAST_OUT=/tmp/tmp.IkDod4jbTm
++ mktemp
+ local LAST_ERR=/tmp/tmp.DAv09iGhBe
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl exec psmdb-client-5dc94d5b48-k2rvq -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-32015.svc.cluster.local/admin '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 0 ']'
+ break
+ cat /tmp/tmp.IkDod4jbTm
+ cat /tmp/tmp.DAv09iGhBe
+ rm /tmp/tmp.IkDod4jbTm /tmp/tmp.DAv09iGhBe
+ return 0
+ diff /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1536/e2e-tests/demand-backup-sharded/compare/find.json /tmp/tmp.2ZOEX1UYL1/find
+ for i in '$(seq 0 2)'
+ compare_mongos_cmd find myApp:myPass@some-name-mongos.demand-backup-sharded-32015 1 .svc.cluster.local myApp1 test
+ local command=find
+ local uri=myApp:myPass@some-name-mongos.demand-backup-sharded-32015
+ local postfix=1
+ local suffix=.svc.cluster.local
+ local database=myApp1
+ local collection=test
+ run_mongos 'use myApp1\n db.test.find()' myApp:myPass@some-name-mongos.demand-backup-sharded-32015 mongodb .svc.cluster.local
+ local 'command=use myApp1\n db.test.find()'
+ local uri=myApp:myPass@some-name-mongos.demand-backup-sharded-32015
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ egrep -v 'I NETWORK|W NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:'
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
+ /usr/bin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
++ local LAST_OUT=/tmp/tmp.csQDtvEcbH
+++ mktemp
++ local LAST_ERR=/tmp/tmp.IeDDiBmDqI
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 0 ']'
++ break
++ cat /tmp/tmp.csQDtvEcbH
++ cat /tmp/tmp.IeDDiBmDqI
++ rm /tmp/tmp.csQDtvEcbH /tmp/tmp.IeDDiBmDqI
++ return 0
+ local client_container=psmdb-client-5dc94d5b48-k2rvq
+ local mongo_flag=
+ kubectl_bin exec psmdb-client-5dc94d5b48-k2rvq -- bash -c 'printf '\''use myApp1\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-32015.svc.cluster.local/admin '
++ mktemp
+ local LAST_OUT=/tmp/tmp.PNE7ph6Qdi
++ mktemp
+ local LAST_ERR=/tmp/tmp.k518BrxlL1
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl exec psmdb-client-5dc94d5b48-k2rvq -- bash -c 'printf '\''use myApp1\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-32015.svc.cluster.local/admin '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 0 ']'
+ break
+ cat /tmp/tmp.PNE7ph6Qdi
+ cat /tmp/tmp.k518BrxlL1
+ rm /tmp/tmp.PNE7ph6Qdi /tmp/tmp.k518BrxlL1
+ return 0
+ diff /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1536/e2e-tests/demand-backup-sharded/compare/find1.json /tmp/tmp.2ZOEX1UYL1/find1
+ for i in '$(seq 0 2)'
+ compare_mongos_cmd find myApp:myPass@some-name-mongos.demand-backup-sharded-32015 2 .svc.cluster.local myApp2 test
+ local command=find
+ local uri=myApp:myPass@some-name-mongos.demand-backup-sharded-32015
+ local postfix=2
+ local suffix=.svc.cluster.local
+ local database=myApp2
+ local collection=test
+ run_mongos 'use myApp2\n db.test.find()' myApp:myPass@some-name-mongos.demand-backup-sharded-32015 mongodb .svc.cluster.local
+ egrep -v 'I NETWORK|W NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:'
+ local 'command=use myApp2\n db.test.find()'
+ local uri=myApp:myPass@some-name-mongos.demand-backup-sharded-32015
+ local driver=mongodb
+ local suffix=.svc.cluster.local
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.losb9gHBnt
+++ mktemp
+ /usr/bin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
++ local LAST_ERR=/tmp/tmp.nnvx7NwY7k
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 0 ']'
++ break
++ cat /tmp/tmp.losb9gHBnt
++ cat /tmp/tmp.nnvx7NwY7k
++ rm /tmp/tmp.losb9gHBnt /tmp/tmp.nnvx7NwY7k
++ return 0
+ local client_container=psmdb-client-5dc94d5b48-k2rvq
+ local mongo_flag=
+ kubectl_bin exec psmdb-client-5dc94d5b48-k2rvq -- bash -c 'printf '\''use myApp2\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-32015.svc.cluster.local/admin '
++ mktemp
+ local LAST_OUT=/tmp/tmp.ain7bay9SG
++ mktemp
+ local LAST_ERR=/tmp/tmp.QIQxhPqqsv
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl exec psmdb-client-5dc94d5b48-k2rvq -- bash -c 'printf '\''use myApp2\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-32015.svc.cluster.local/admin '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 0 ']'
+ break
+ cat /tmp/tmp.ain7bay9SG
+ cat /tmp/tmp.QIQxhPqqsv
+ rm /tmp/tmp.ain7bay9SG /tmp/tmp.QIQxhPqqsv
+ return 0
+ diff /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1536/e2e-tests/demand-backup-sharded/compare/find2.json /tmp/tmp.2ZOEX1UYL1/find2
+ desc 'check backup and restore -- gcp-cs'
+ set +o xtrace


-----------------------------------------------------------------------------------
check backup and restore -- gcp-cs
-----------------------------------------------------------------------------------

Percona Server for MongoDB shell version v4.4.29-28
connecting to: mongodb://some-name-mongos.demand-backup-sharded-32015.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb
Implicit session: session { "id" : UUID("8739f0f3-c49a-4e45-bae3-ec7400a2c6e4") }
Percona Server for MongoDB server version: v7.0.8-5
WARNING: shell and server versions do not match
switched to db myApp
WriteResult({ "nInserted" : 1 })
bye
Percona Server for MongoDB shell version v4.4.29-28
connecting to: mongodb://some-name-mongos.demand-backup-sharded-32015.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb
Implicit session: session { "id" : UUID("47d077d5-3519-43a8-805e-ae79bc79b0b4") }
Percona Server for MongoDB server version: v7.0.8-5
WARNING: shell and server versions do not match
switched to db myApp1
WriteResult({ "nInserted" : 1 })
bye
Percona Server for MongoDB shell version v4.4.29-28
connecting to: mongodb://some-name-mongos.demand-backup-sharded-32015.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb
Implicit session: session { "id" : UUID("71962ed1-f270-43a0-a558-6025a98cc4ff") }
Percona Server for MongoDB server version: v7.0.8-5
WARNING: shell and server versions do not match
switched to db myApp2
WriteResult({ "nInserted" : 1 })
bye
perconaservermongodbrestore.psmdb.percona.com/restore-backup-gcp-cs created
waiting psmdb-restore/backup-gcp-cs to reach ready state............
+ '[' 1 -eq 1 ']'
+ wait_cluster_consistency some-name
+ local cluster_name=some-name
+ local wait_time=32
+ retry=0
+ sleep 7
+ echo -n 'waiting for cluster readyness'
waiting for cluster readyness++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.BRUCKmwIxR
+++ mktemp
++ local LAST_ERR=/tmp/tmp.L75zmyFH4t
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 0 ']'
++ break
++ cat /tmp/tmp.BRUCKmwIxR
++ cat /tmp/tmp.L75zmyFH4t
++ rm /tmp/tmp.BRUCKmwIxR /tmp/tmp.L75zmyFH4t
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 1 -ge 32 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.duVH5gNp0h
+++ mktemp
++ local LAST_ERR=/tmp/tmp.BHmGBMnsIa
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 0 ']'
++ break
++ cat /tmp/tmp.duVH5gNp0h
++ cat /tmp/tmp.BHmGBMnsIa
++ rm /tmp/tmp.duVH5gNp0h /tmp/tmp.BHmGBMnsIa
++ return 0
+ [[ error == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 2 -ge 32 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.EbpedEIvuq
+++ mktemp
++ local LAST_ERR=/tmp/tmp.dvS8OD1XvX
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 0 ']'
++ break
++ cat /tmp/tmp.EbpedEIvuq
++ cat /tmp/tmp.dvS8OD1XvX
++ rm /tmp/tmp.EbpedEIvuq /tmp/tmp.dvS8OD1XvX
++ return 0
+ [[ error == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 3 -ge 32 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.xaxYKBHZKH
+++ mktemp
++ local LAST_ERR=/tmp/tmp.5HK4YiVasT
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 0 ']'
++ break
++ cat /tmp/tmp.xaxYKBHZKH
++ cat /tmp/tmp.5HK4YiVasT
++ rm /tmp/tmp.xaxYKBHZKH /tmp/tmp.5HK4YiVasT
++ return 0
+ [[ error == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 4 -ge 32 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.nyJGWhGLHb
+++ mktemp
++ local LAST_ERR=/tmp/tmp.sTwBB9LcVC
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 0 ']'
++ break
++ cat /tmp/tmp.nyJGWhGLHb
++ cat /tmp/tmp.sTwBB9LcVC
++ rm /tmp/tmp.nyJGWhGLHb /tmp/tmp.sTwBB9LcVC
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 5 -ge 32 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.u9cGxTzgFs
+++ mktemp
++ local LAST_ERR=/tmp/tmp.4Nz1zJTP1L
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 0 ']'
++ break
++ cat /tmp/tmp.u9cGxTzgFs
++ cat /tmp/tmp.4Nz1zJTP1L
++ rm /tmp/tmp.u9cGxTzgFs /tmp/tmp.4Nz1zJTP1L
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 6 -ge 32 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.G3JyO2Xv9P
+++ mktemp
++ local LAST_ERR=/tmp/tmp.5jTJGYtvUK
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 0 ']'
++ break
++ cat /tmp/tmp.G3JyO2Xv9P
++ cat /tmp/tmp.5jTJGYtvUK
++ rm /tmp/tmp.G3JyO2Xv9P /tmp/tmp.5jTJGYtvUK
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 7 -ge 32 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.6ZxBDi4gUL
+++ mktemp
++ local LAST_ERR=/tmp/tmp.fhBsDemyuP
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 0 ']'
++ break
++ cat /tmp/tmp.6ZxBDi4gUL
++ cat /tmp/tmp.fhBsDemyuP
++ rm /tmp/tmp.6ZxBDi4gUL /tmp/tmp.fhBsDemyuP
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 8 -ge 32 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.CwwgihlxK6
+++ mktemp
++ local LAST_ERR=/tmp/tmp.9WpxyHuvXi
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 0 ']'
++ break
++ cat /tmp/tmp.CwwgihlxK6
++ cat /tmp/tmp.9WpxyHuvXi
++ rm /tmp/tmp.CwwgihlxK6 /tmp/tmp.9WpxyHuvXi
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 9 -ge 32 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.aiZcjaSDXI
+++ mktemp
++ local LAST_ERR=/tmp/tmp.6QGVWcBWWF
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 0 ']'
++ break
++ cat /tmp/tmp.aiZcjaSDXI
++ cat /tmp/tmp.6QGVWcBWWF
++ rm /tmp/tmp.aiZcjaSDXI /tmp/tmp.6QGVWcBWWF
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 10 -ge 32 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.bj99sw3lq7
+++ mktemp
++ local LAST_ERR=/tmp/tmp.qu7NKEBMVa
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 0 ']'
++ break
++ cat /tmp/tmp.bj99sw3lq7
++ cat /tmp/tmp.qu7NKEBMVa
++ rm /tmp/tmp.bj99sw3lq7 /tmp/tmp.qu7NKEBMVa
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 11 -ge 32 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.HxHH9rvGNe
+++ mktemp
++ local LAST_ERR=/tmp/tmp.UdK9AyFBOx
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 0 ']'
++ break
++ cat /tmp/tmp.HxHH9rvGNe
++ cat /tmp/tmp.UdK9AyFBOx
++ rm /tmp/tmp.HxHH9rvGNe /tmp/tmp.UdK9AyFBOx
++ return 0
+ [[ ready == \r\e\a\d\y ]]
+ check_data
+ local postfix=
++ seq 0 2
+ for i in '$(seq 0 2)'
+ compare_mongos_cmd find myApp:myPass@some-name-mongos.demand-backup-sharded-32015 '' .svc.cluster.local myApp test
+ local command=find
+ local uri=myApp:myPass@some-name-mongos.demand-backup-sharded-32015
+ local postfix=
+ local suffix=.svc.cluster.local
+ local database=myApp
+ local collection=test
+ run_mongos 'use myApp\n db.test.find()' myApp:myPass@some-name-mongos.demand-backup-sharded-32015 mongodb .svc.cluster.local
+ local 'command=use myApp\n db.test.find()'
+ local uri=myApp:myPass@some-name-mongos.demand-backup-sharded-32015
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ /usr/bin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
+ egrep -v 'I NETWORK|W NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:'
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.PD05vqC6nz
+++ mktemp
++ local LAST_ERR=/tmp/tmp.k4icSE4fxJ
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 0 ']'
++ break
++ cat /tmp/tmp.PD05vqC6nz
++ cat /tmp/tmp.k4icSE4fxJ
++ rm /tmp/tmp.PD05vqC6nz /tmp/tmp.k4icSE4fxJ
++ return 0
+ local client_container=psmdb-client-5dc94d5b48-k2rvq
+ local mongo_flag=
+ kubectl_bin exec psmdb-client-5dc94d5b48-k2rvq -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-32015.svc.cluster.local/admin '
++ mktemp
+ local LAST_OUT=/tmp/tmp.vFRlL9FUW8
++ mktemp
+ local LAST_ERR=/tmp/tmp.FvLjLHsG7d
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl exec psmdb-client-5dc94d5b48-k2rvq -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-32015.svc.cluster.local/admin '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 0 ']'
+ break
+ cat /tmp/tmp.vFRlL9FUW8
+ cat /tmp/tmp.FvLjLHsG7d
+ rm /tmp/tmp.vFRlL9FUW8 /tmp/tmp.FvLjLHsG7d
+ return 0
+ diff /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1536/e2e-tests/demand-backup-sharded/compare/find.json /tmp/tmp.2ZOEX1UYL1/find
+ for i in '$(seq 0 2)'
+ compare_mongos_cmd find myApp:myPass@some-name-mongos.demand-backup-sharded-32015 1 .svc.cluster.local myApp1 test
+ local command=find
+ local uri=myApp:myPass@some-name-mongos.demand-backup-sharded-32015
+ local postfix=1
+ local suffix=.svc.cluster.local
+ local database=myApp1
+ local collection=test
+ egrep -v 'I NETWORK|W NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:'
+ run_mongos 'use myApp1\n db.test.find()' myApp:myPass@some-name-mongos.demand-backup-sharded-32015 mongodb .svc.cluster.local
+ local 'command=use myApp1\n db.test.find()'
+ local uri=myApp:myPass@some-name-mongos.demand-backup-sharded-32015
+ local driver=mongodb
+ local suffix=.svc.cluster.local
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
+ /usr/bin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
++ local LAST_OUT=/tmp/tmp.eJwbgU6GW8
+++ mktemp
++ local LAST_ERR=/tmp/tmp.6CDTGziVys
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 0 ']'
++ break
++ cat /tmp/tmp.eJwbgU6GW8
++ cat /tmp/tmp.6CDTGziVys
++ rm /tmp/tmp.eJwbgU6GW8 /tmp/tmp.6CDTGziVys
++ return 0
+ local client_container=psmdb-client-5dc94d5b48-k2rvq
+ local mongo_flag=
+ kubectl_bin exec psmdb-client-5dc94d5b48-k2rvq -- bash -c 'printf '\''use myApp1\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-32015.svc.cluster.local/admin '
++ mktemp
+ local LAST_OUT=/tmp/tmp.nQ6ulU3rVP
++ mktemp
+ local LAST_ERR=/tmp/tmp.R4oeQ0KPvw
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl exec psmdb-client-5dc94d5b48-k2rvq -- bash -c 'printf '\''use myApp1\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-32015.svc.cluster.local/admin '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 0 ']'
+ break
+ cat /tmp/tmp.nQ6ulU3rVP
+ cat /tmp/tmp.R4oeQ0KPvw
+ rm /tmp/tmp.nQ6ulU3rVP /tmp/tmp.R4oeQ0KPvw
+ return 0
+ diff /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1536/e2e-tests/demand-backup-sharded/compare/find1.json /tmp/tmp.2ZOEX1UYL1/find1
+ for i in '$(seq 0 2)'
+ compare_mongos_cmd find myApp:myPass@some-name-mongos.demand-backup-sharded-32015 2 .svc.cluster.local myApp2 test
+ local command=find
+ local uri=myApp:myPass@some-name-mongos.demand-backup-sharded-32015
+ local postfix=2
+ local suffix=.svc.cluster.local
+ local database=myApp2
+ local collection=test
+ egrep -v 'I NETWORK|W NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:'
+ /usr/bin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
+ run_mongos 'use myApp2\n db.test.find()' myApp:myPass@some-name-mongos.demand-backup-sharded-32015 mongodb .svc.cluster.local
+ local 'command=use myApp2\n db.test.find()'
+ local uri=myApp:myPass@some-name-mongos.demand-backup-sharded-32015
+ local driver=mongodb
+ local suffix=.svc.cluster.local
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.239ewoq0v3
+++ mktemp
++ local LAST_ERR=/tmp/tmp.CkxhtyTewK
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 0 ']'
++ break
++ cat /tmp/tmp.239ewoq0v3
++ cat /tmp/tmp.CkxhtyTewK
++ rm /tmp/tmp.239ewoq0v3 /tmp/tmp.CkxhtyTewK
++ return 0
+ local client_container=psmdb-client-5dc94d5b48-k2rvq
+ local mongo_flag=
+ kubectl_bin exec psmdb-client-5dc94d5b48-k2rvq -- bash -c 'printf '\''use myApp2\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-32015.svc.cluster.local/admin '
++ mktemp
+ local LAST_OUT=/tmp/tmp.pU7fFW4S9c
++ mktemp
+ local LAST_ERR=/tmp/tmp.eTrloKVmJc
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl exec psmdb-client-5dc94d5b48-k2rvq -- bash -c 'printf '\''use myApp2\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-32015.svc.cluster.local/admin '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 0 ']'
+ break
+ cat /tmp/tmp.pU7fFW4S9c
+ cat /tmp/tmp.eTrloKVmJc
+ rm /tmp/tmp.pU7fFW4S9c /tmp/tmp.eTrloKVmJc
+ return 0
+ diff /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1536/e2e-tests/demand-backup-sharded/compare/find2.json /tmp/tmp.2ZOEX1UYL1/find2
+ desc 'check backup and restore -- azure-blob'
+ set +o xtrace


-----------------------------------------------------------------------------------
check backup and restore -- azure-blob
-----------------------------------------------------------------------------------

Percona Server for MongoDB shell version v4.4.29-28
connecting to: mongodb://some-name-mongos.demand-backup-sharded-32015.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb
Implicit session: session { "id" : UUID("b0a0d7ae-e061-4b8b-938e-eee85619f1c7") }
Percona Server for MongoDB server version: v7.0.8-5
WARNING: shell and server versions do not match
switched to db myApp
WriteResult({ "nInserted" : 1 })
bye
Percona Server for MongoDB shell version v4.4.29-28
connecting to: mongodb://some-name-mongos.demand-backup-sharded-32015.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb
Implicit session: session { "id" : UUID("e0ac372d-e59f-4e4f-ae41-1e6d5bab9ffc") }
Percona Server for MongoDB server version: v7.0.8-5
WARNING: shell and server versions do not match
switched to db myApp1
WriteResult({ "nInserted" : 1 })
bye
Percona Server for MongoDB shell version v4.4.29-28
connecting to: mongodb://some-name-mongos.demand-backup-sharded-32015.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb
Implicit session: session { "id" : UUID("ea10137e-2b6f-425f-a34a-00bf72462140") }
Percona Server for MongoDB server version: v7.0.8-5
WARNING: shell and server versions do not match
switched to db myApp2
WriteResult({ "nInserted" : 1 })
bye
perconaservermongodbrestore.psmdb.percona.com/restore-backup-azure-blob created
waiting psmdb-restore/backup-azure-blob to reach ready state............
+ '[' 1 -eq 1 ']'
+ wait_cluster_consistency some-name
+ local cluster_name=some-name
+ local wait_time=32
+ retry=0
+ sleep 7
+ echo -n 'waiting for cluster readyness'
waiting for cluster readyness++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.42mGLkw3Bx
+++ mktemp
++ local LAST_ERR=/tmp/tmp.i1iR4AHMAc
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 0 ']'
++ break
++ cat /tmp/tmp.42mGLkw3Bx
++ cat /tmp/tmp.i1iR4AHMAc
++ rm /tmp/tmp.42mGLkw3Bx /tmp/tmp.i1iR4AHMAc
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 1 -ge 32 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.uClAgqFSCn
+++ mktemp
++ local LAST_ERR=/tmp/tmp.WRQqEnvmxB
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 0 ']'
++ break
++ cat /tmp/tmp.uClAgqFSCn
++ cat /tmp/tmp.WRQqEnvmxB
++ rm /tmp/tmp.uClAgqFSCn /tmp/tmp.WRQqEnvmxB
++ return 0
+ [[ error == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 2 -ge 32 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.Q77S6Gdv0c
+++ mktemp
++ local LAST_ERR=/tmp/tmp.Y23GLmOCGT
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 0 ']'
++ break
++ cat /tmp/tmp.Q77S6Gdv0c
++ cat /tmp/tmp.Y23GLmOCGT
++ rm /tmp/tmp.Q77S6Gdv0c /tmp/tmp.Y23GLmOCGT
++ return 0
+ [[ error == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 3 -ge 32 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.cQbtViKBTg
+++ mktemp
++ local LAST_ERR=/tmp/tmp.PvaaHjeT9t
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 0 ']'
++ break
++ cat /tmp/tmp.cQbtViKBTg
++ cat /tmp/tmp.PvaaHjeT9t
++ rm /tmp/tmp.cQbtViKBTg /tmp/tmp.PvaaHjeT9t
++ return 0
+ [[ error == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 4 -ge 32 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.a5On1Z8WwI
+++ mktemp
++ local LAST_ERR=/tmp/tmp.yxVBYRDjiF
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 0 ']'
++ break
++ cat /tmp/tmp.a5On1Z8WwI
++ cat /tmp/tmp.yxVBYRDjiF
++ rm /tmp/tmp.a5On1Z8WwI /tmp/tmp.yxVBYRDjiF
++ return 0
+ [[ error == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 5 -ge 32 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.cynVe2hDMQ
+++ mktemp
++ local LAST_ERR=/tmp/tmp.Nzt0lqKK7y
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 0 ']'
++ break
++ cat /tmp/tmp.cynVe2hDMQ
++ cat /tmp/tmp.Nzt0lqKK7y
++ rm /tmp/tmp.cynVe2hDMQ /tmp/tmp.Nzt0lqKK7y
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 6 -ge 32 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.b6j4tlKTlO
+++ mktemp
++ local LAST_ERR=/tmp/tmp.DeWRRY6BBr
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 0 ']'
++ break
++ cat /tmp/tmp.b6j4tlKTlO
++ cat /tmp/tmp.DeWRRY6BBr
++ rm /tmp/tmp.b6j4tlKTlO /tmp/tmp.DeWRRY6BBr
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 7 -ge 32 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.GbMMqQxzUz
+++ mktemp
++ local LAST_ERR=/tmp/tmp.NQi9Odl2H3
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 0 ']'
++ break
++ cat /tmp/tmp.GbMMqQxzUz
++ cat /tmp/tmp.NQi9Odl2H3
++ rm /tmp/tmp.GbMMqQxzUz /tmp/tmp.NQi9Odl2H3
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 8 -ge 32 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.Oqptreg4C4
+++ mktemp
++ local LAST_ERR=/tmp/tmp.EEbrNQJXax
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 0 ']'
++ break
++ cat /tmp/tmp.Oqptreg4C4
++ cat /tmp/tmp.EEbrNQJXax
++ rm /tmp/tmp.Oqptreg4C4 /tmp/tmp.EEbrNQJXax
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 9 -ge 32 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.o6qaV6BSY8
+++ mktemp
++ local LAST_ERR=/tmp/tmp.Q7vEGJagvA
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 0 ']'
++ break
++ cat /tmp/tmp.o6qaV6BSY8
++ cat /tmp/tmp.Q7vEGJagvA
++ rm /tmp/tmp.o6qaV6BSY8 /tmp/tmp.Q7vEGJagvA
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 10 -ge 32 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.ajmpSYGoRX
+++ mktemp
++ local LAST_ERR=/tmp/tmp.5ZxWMJygv9
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 0 ']'
++ break
++ cat /tmp/tmp.ajmpSYGoRX
++ cat /tmp/tmp.5ZxWMJygv9
++ rm /tmp/tmp.ajmpSYGoRX /tmp/tmp.5ZxWMJygv9
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 11 -ge 32 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.L2TjzZdObR
+++ mktemp
++ local LAST_ERR=/tmp/tmp.29BRSpHi4A
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 0 ']'
++ break
++ cat /tmp/tmp.L2TjzZdObR
++ cat /tmp/tmp.29BRSpHi4A
++ rm /tmp/tmp.L2TjzZdObR /tmp/tmp.29BRSpHi4A
++ return 0
+ [[ ready == \r\e\a\d\y ]]
+ check_data
+ local postfix=
++ seq 0 2
+ for i in '$(seq 0 2)'
+ compare_mongos_cmd find myApp:myPass@some-name-mongos.demand-backup-sharded-32015 '' .svc.cluster.local myApp test
+ local command=find
+ local uri=myApp:myPass@some-name-mongos.demand-backup-sharded-32015
+ local postfix=
+ local suffix=.svc.cluster.local
+ local database=myApp
+ local collection=test
+ /usr/bin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
+ run_mongos 'use myApp\n db.test.find()' myApp:myPass@some-name-mongos.demand-backup-sharded-32015 mongodb .svc.cluster.local
+ local 'command=use myApp\n db.test.find()'
+ local uri=myApp:myPass@some-name-mongos.demand-backup-sharded-32015
+ local driver=mongodb
+ local suffix=.svc.cluster.local
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+ egrep -v 'I NETWORK|W NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.PGovXdNqur
+++ mktemp
++ local LAST_ERR=/tmp/tmp.02noLlfAdV
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 0 ']'
++ break
++ cat /tmp/tmp.PGovXdNqur
++ cat /tmp/tmp.02noLlfAdV
++ rm /tmp/tmp.PGovXdNqur /tmp/tmp.02noLlfAdV
++ return 0
+ local client_container=psmdb-client-5dc94d5b48-k2rvq
+ local mongo_flag=
+ kubectl_bin exec psmdb-client-5dc94d5b48-k2rvq -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-32015.svc.cluster.local/admin '
++ mktemp
+ local LAST_OUT=/tmp/tmp.aV3pikBnOY
++ mktemp
+ local LAST_ERR=/tmp/tmp.ap7Ha7SM4S
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl exec psmdb-client-5dc94d5b48-k2rvq -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-32015.svc.cluster.local/admin '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 0 ']'
+ break
+ cat /tmp/tmp.aV3pikBnOY
+ cat /tmp/tmp.ap7Ha7SM4S
+ rm /tmp/tmp.aV3pikBnOY /tmp/tmp.ap7Ha7SM4S
+ return 0
+ diff /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1536/e2e-tests/demand-backup-sharded/compare/find.json /tmp/tmp.2ZOEX1UYL1/find
+ for i in '$(seq 0 2)'
+ compare_mongos_cmd find myApp:myPass@some-name-mongos.demand-backup-sharded-32015 1 .svc.cluster.local myApp1 test
+ local command=find
+ local uri=myApp:myPass@some-name-mongos.demand-backup-sharded-32015
+ local postfix=1
+ local suffix=.svc.cluster.local
+ local database=myApp1
+ local collection=test
+ run_mongos 'use myApp1\n db.test.find()' myApp:myPass@some-name-mongos.demand-backup-sharded-32015 mongodb .svc.cluster.local
+ local 'command=use myApp1\n db.test.find()'
+ local uri=myApp:myPass@some-name-mongos.demand-backup-sharded-32015
+ local driver=mongodb
+ local suffix=.svc.cluster.local
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+ /usr/bin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
+ egrep -v 'I NETWORK|W NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.WYCEwIo9we
+++ mktemp
++ local LAST_ERR=/tmp/tmp.t317Z9blw3
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 0 ']'
++ break
++ cat /tmp/tmp.WYCEwIo9we
++ cat /tmp/tmp.t317Z9blw3
++ rm /tmp/tmp.WYCEwIo9we /tmp/tmp.t317Z9blw3
++ return 0
+ local client_container=psmdb-client-5dc94d5b48-k2rvq
+ local mongo_flag=
+ kubectl_bin exec psmdb-client-5dc94d5b48-k2rvq -- bash -c 'printf '\''use myApp1\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-32015.svc.cluster.local/admin '
++ mktemp
+ local LAST_OUT=/tmp/tmp.EKLSaeiV0Y
++ mktemp
+ local LAST_ERR=/tmp/tmp.dyymFjaG7Q
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl exec psmdb-client-5dc94d5b48-k2rvq -- bash -c 'printf '\''use myApp1\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-32015.svc.cluster.local/admin '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 0 ']'
+ break
+ cat /tmp/tmp.EKLSaeiV0Y
+ cat /tmp/tmp.dyymFjaG7Q
+ rm /tmp/tmp.EKLSaeiV0Y /tmp/tmp.dyymFjaG7Q
+ return 0
+ diff /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1536/e2e-tests/demand-backup-sharded/compare/find1.json /tmp/tmp.2ZOEX1UYL1/find1
+ for i in '$(seq 0 2)'
+ compare_mongos_cmd find myApp:myPass@some-name-mongos.demand-backup-sharded-32015 2 .svc.cluster.local myApp2 test
+ local command=find
+ local uri=myApp:myPass@some-name-mongos.demand-backup-sharded-32015
+ local postfix=2
+ local suffix=.svc.cluster.local
+ local database=myApp2
+ local collection=test
+ egrep -v 'I NETWORK|W NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:'
+ run_mongos 'use myApp2\n db.test.find()' myApp:myPass@some-name-mongos.demand-backup-sharded-32015 mongodb .svc.cluster.local
+ /usr/bin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
+ local 'command=use myApp2\n db.test.find()'
+ local uri=myApp:myPass@some-name-mongos.demand-backup-sharded-32015
+ local driver=mongodb
+ local suffix=.svc.cluster.local
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.MyZasL9Rsd
+++ mktemp
++ local LAST_ERR=/tmp/tmp.1nLbtmw2vg
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 0 ']'
++ break
++ cat /tmp/tmp.MyZasL9Rsd
++ cat /tmp/tmp.1nLbtmw2vg
++ rm /tmp/tmp.MyZasL9Rsd /tmp/tmp.1nLbtmw2vg
++ return 0
+ local client_container=psmdb-client-5dc94d5b48-k2rvq
+ local mongo_flag=
+ kubectl_bin exec psmdb-client-5dc94d5b48-k2rvq -- bash -c 'printf '\''use myApp2\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-32015.svc.cluster.local/admin '
++ mktemp
+ local LAST_OUT=/tmp/tmp.NMazaGtDIQ
++ mktemp
+ local LAST_ERR=/tmp/tmp.QjfLvv6SUU
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl exec psmdb-client-5dc94d5b48-k2rvq -- bash -c 'printf '\''use myApp2\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-32015.svc.cluster.local/admin '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 0 ']'
+ break
+ cat /tmp/tmp.NMazaGtDIQ
+ cat /tmp/tmp.QjfLvv6SUU
+ rm /tmp/tmp.NMazaGtDIQ /tmp/tmp.QjfLvv6SUU
+ return 0
+ diff /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1536/e2e-tests/demand-backup-sharded/compare/find2.json /tmp/tmp.2ZOEX1UYL1/find2
+ desc 'check backup and restore -- minio'
+ set +o xtrace


-----------------------------------------------------------------------------------
check backup and restore -- minio
-----------------------------------------------------------------------------------

If you don't see a command prompt, try pressing enter.
warning: couldn't attach to pod/aws-cli, falling back to streaming logs: unable to upgrade connection: container aws-cli not found in pod aws-cli_demand-backup-sharded-32015
2024-05-02 14:33:49         55 myApp.test.gz
Percona Server for MongoDB shell version v4.4.29-28
connecting to: mongodb://some-name-mongos.demand-backup-sharded-32015.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb
Implicit session: session { "id" : UUID("2243e3fb-ff48-452a-8811-c11920558b16") }
Percona Server for MongoDB server version: v7.0.8-5
WARNING: shell and server versions do not match
switched to db myApp
WriteResult({ "nInserted" : 1 })
bye
Percona Server for MongoDB shell version v4.4.29-28
connecting to: mongodb://some-name-mongos.demand-backup-sharded-32015.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb
Implicit session: session { "id" : UUID("f5308711-fa87-4256-bc41-806e8204663a") }
Percona Server for MongoDB server version: v7.0.8-5
WARNING: shell and server versions do not match
switched to db myApp1
WriteResult({ "nInserted" : 1 })
bye
Percona Server for MongoDB shell version v4.4.29-28
connecting to: mongodb://some-name-mongos.demand-backup-sharded-32015.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb
Implicit session: session { "id" : UUID("301cf9e0-9838-4217-a68c-15432b0b1bec") }
Percona Server for MongoDB server version: v7.0.8-5
WARNING: shell and server versions do not match
switched to db myApp2
WriteResult({ "nInserted" : 1 })
bye
perconaservermongodbrestore.psmdb.percona.com/restore-backup-minio created
waiting psmdb-restore/backup-minio to reach ready state...........
+ '[' 1 -eq 1 ']'
+ wait_cluster_consistency some-name
+ local cluster_name=some-name
+ local wait_time=32
+ retry=0
+ sleep 7
+ echo -n 'waiting for cluster readyness'
waiting for cluster readyness++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.eUi3vtfXZM
+++ mktemp
++ local LAST_ERR=/tmp/tmp.qy498ZcyDQ
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 0 ']'
++ break
++ cat /tmp/tmp.eUi3vtfXZM
++ cat /tmp/tmp.qy498ZcyDQ
++ rm /tmp/tmp.eUi3vtfXZM /tmp/tmp.qy498ZcyDQ
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 1 -ge 32 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.05tV2tIbOp
+++ mktemp
++ local LAST_ERR=/tmp/tmp.PJt9t1TfDp
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 0 ']'
++ break
++ cat /tmp/tmp.05tV2tIbOp
++ cat /tmp/tmp.PJt9t1TfDp
++ rm /tmp/tmp.05tV2tIbOp /tmp/tmp.PJt9t1TfDp
++ return 0
+ [[ error == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 2 -ge 32 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.a55DG7mHud
+++ mktemp
++ local LAST_ERR=/tmp/tmp.FSbVYiUGR3
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 0 ']'
++ break
++ cat /tmp/tmp.a55DG7mHud
++ cat /tmp/tmp.FSbVYiUGR3
++ rm /tmp/tmp.a55DG7mHud /tmp/tmp.FSbVYiUGR3
++ return 0
+ [[ error == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 3 -ge 32 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.fUKRLqjGCY
+++ mktemp
++ local LAST_ERR=/tmp/tmp.vro7yTUq6k
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 0 ']'
++ break
++ cat /tmp/tmp.fUKRLqjGCY
++ cat /tmp/tmp.vro7yTUq6k
++ rm /tmp/tmp.fUKRLqjGCY /tmp/tmp.vro7yTUq6k
++ return 0
+ [[ error == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 4 -ge 32 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.vkifInQtD7
+++ mktemp
++ local LAST_ERR=/tmp/tmp.eYl29RxnN3
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 0 ']'
++ break
++ cat /tmp/tmp.vkifInQtD7
++ cat /tmp/tmp.eYl29RxnN3
++ rm /tmp/tmp.vkifInQtD7 /tmp/tmp.eYl29RxnN3
++ return 0
+ [[ error == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 5 -ge 32 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.ZKJ8ePREXm
+++ mktemp
++ local LAST_ERR=/tmp/tmp.qj2FG40DUJ
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 0 ']'
++ break
++ cat /tmp/tmp.ZKJ8ePREXm
++ cat /tmp/tmp.qj2FG40DUJ
++ rm /tmp/tmp.ZKJ8ePREXm /tmp/tmp.qj2FG40DUJ
++ return 0
+ [[ error == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 6 -ge 32 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.aKAN1MnAip
+++ mktemp
++ local LAST_ERR=/tmp/tmp.EOY8VeU4zQ
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 0 ']'
++ break
++ cat /tmp/tmp.aKAN1MnAip
++ cat /tmp/tmp.EOY8VeU4zQ
++ rm /tmp/tmp.aKAN1MnAip /tmp/tmp.EOY8VeU4zQ
++ return 0
+ [[ error == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 7 -ge 32 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.cBN4XJHcKj
+++ mktemp
++ local LAST_ERR=/tmp/tmp.KVQOztTN2b
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 0 ']'
++ break
++ cat /tmp/tmp.cBN4XJHcKj
++ cat /tmp/tmp.KVQOztTN2b
++ rm /tmp/tmp.cBN4XJHcKj /tmp/tmp.KVQOztTN2b
++ return 0
+ [[ error == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 8 -ge 32 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.1y4vNkBavl
+++ mktemp
++ local LAST_ERR=/tmp/tmp.HUlMnuE9xF
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 0 ']'
++ break
++ cat /tmp/tmp.1y4vNkBavl
++ cat /tmp/tmp.HUlMnuE9xF
++ rm /tmp/tmp.1y4vNkBavl /tmp/tmp.HUlMnuE9xF
++ return 0
+ [[ error == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 9 -ge 32 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.sOH1PxXmVM
+++ mktemp
++ local LAST_ERR=/tmp/tmp.QAlQdmpZjw
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 0 ']'
++ break
++ cat /tmp/tmp.sOH1PxXmVM
++ cat /tmp/tmp.QAlQdmpZjw
++ rm /tmp/tmp.sOH1PxXmVM /tmp/tmp.QAlQdmpZjw
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 10 -ge 32 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.BaOne23LR4
+++ mktemp
++ local LAST_ERR=/tmp/tmp.IDErlbsCBc
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 0 ']'
++ break
++ cat /tmp/tmp.BaOne23LR4
++ cat /tmp/tmp.IDErlbsCBc
++ rm /tmp/tmp.BaOne23LR4 /tmp/tmp.IDErlbsCBc
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 11 -ge 32 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.0awEpzHmKp
+++ mktemp
++ local LAST_ERR=/tmp/tmp.zjcgVZrbhw
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 0 ']'
++ break
++ cat /tmp/tmp.0awEpzHmKp
++ cat /tmp/tmp.zjcgVZrbhw
++ rm /tmp/tmp.0awEpzHmKp /tmp/tmp.zjcgVZrbhw
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 12 -ge 32 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.UC98Eno01o
+++ mktemp
++ local LAST_ERR=/tmp/tmp.usv5RfGXRB
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 0 ']'
++ break
++ cat /tmp/tmp.UC98Eno01o
++ cat /tmp/tmp.usv5RfGXRB
++ rm /tmp/tmp.UC98Eno01o /tmp/tmp.usv5RfGXRB
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 13 -ge 32 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.5CDc8wuwo1
+++ mktemp
++ local LAST_ERR=/tmp/tmp.PHHVenqjkt
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 0 ']'
++ break
++ cat /tmp/tmp.5CDc8wuwo1
++ cat /tmp/tmp.PHHVenqjkt
++ rm /tmp/tmp.5CDc8wuwo1 /tmp/tmp.PHHVenqjkt
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 14 -ge 32 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.gfI0MFtHfQ
+++ mktemp
++ local LAST_ERR=/tmp/tmp.rBkteGmpKx
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 0 ']'
++ break
++ cat /tmp/tmp.gfI0MFtHfQ
++ cat /tmp/tmp.rBkteGmpKx
++ rm /tmp/tmp.gfI0MFtHfQ /tmp/tmp.rBkteGmpKx
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 15 -ge 32 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.vismRRYpbS
+++ mktemp
++ local LAST_ERR=/tmp/tmp.EIGtqVI8Cw
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 0 ']'
++ break
++ cat /tmp/tmp.vismRRYpbS
++ cat /tmp/tmp.EIGtqVI8Cw
++ rm /tmp/tmp.vismRRYpbS /tmp/tmp.EIGtqVI8Cw
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 16 -ge 32 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.2FtqU0vcW9
+++ mktemp
++ local LAST_ERR=/tmp/tmp.V90Dl6r7W9
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 0 ']'
++ break
++ cat /tmp/tmp.2FtqU0vcW9
++ cat /tmp/tmp.V90Dl6r7W9
++ rm /tmp/tmp.2FtqU0vcW9 /tmp/tmp.V90Dl6r7W9
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 17 -ge 32 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.Gkxm09KN9z
+++ mktemp
++ local LAST_ERR=/tmp/tmp.kkheDQBg50
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 0 ']'
++ break
++ cat /tmp/tmp.Gkxm09KN9z
++ cat /tmp/tmp.kkheDQBg50
++ rm /tmp/tmp.Gkxm09KN9z /tmp/tmp.kkheDQBg50
++ return 0
+ [[ ready == \r\e\a\d\y ]]
+ check_data
+ local postfix=
++ seq 0 2
+ for i in '$(seq 0 2)'
+ compare_mongos_cmd find myApp:myPass@some-name-mongos.demand-backup-sharded-32015 '' .svc.cluster.local myApp test
+ local command=find
+ local uri=myApp:myPass@some-name-mongos.demand-backup-sharded-32015
+ local postfix=
+ local suffix=.svc.cluster.local
+ local database=myApp
+ local collection=test
+ egrep -v 'I NETWORK|W NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:'
+ run_mongos 'use myApp\n db.test.find()' myApp:myPass@some-name-mongos.demand-backup-sharded-32015 mongodb .svc.cluster.local
+ local 'command=use myApp\n db.test.find()'
+ local uri=myApp:myPass@some-name-mongos.demand-backup-sharded-32015
+ local driver=mongodb
+ local suffix=.svc.cluster.local
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
+ /usr/bin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
++ local LAST_OUT=/tmp/tmp.THgFuDyJju
+++ mktemp
++ local LAST_ERR=/tmp/tmp.K2o9yZv3Xv
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 0 ']'
++ break
++ cat /tmp/tmp.THgFuDyJju
++ cat /tmp/tmp.K2o9yZv3Xv
++ rm /tmp/tmp.THgFuDyJju /tmp/tmp.K2o9yZv3Xv
++ return 0
+ local client_container=psmdb-client-5dc94d5b48-vt7s4
+ local mongo_flag=
+ kubectl_bin exec psmdb-client-5dc94d5b48-vt7s4 -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-32015.svc.cluster.local/admin '
++ mktemp
+ local LAST_OUT=/tmp/tmp.WCm37pUVoH
++ mktemp
+ local LAST_ERR=/tmp/tmp.Wqa9HZ1nTp
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl exec psmdb-client-5dc94d5b48-vt7s4 -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-32015.svc.cluster.local/admin '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 0 ']'
+ break
+ cat /tmp/tmp.WCm37pUVoH
+ cat /tmp/tmp.Wqa9HZ1nTp
+ rm /tmp/tmp.WCm37pUVoH /tmp/tmp.Wqa9HZ1nTp
+ return 0
+ diff /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1536/e2e-tests/demand-backup-sharded/compare/find.json /tmp/tmp.2ZOEX1UYL1/find
+ for i in '$(seq 0 2)'
+ compare_mongos_cmd find myApp:myPass@some-name-mongos.demand-backup-sharded-32015 1 .svc.cluster.local myApp1 test
+ local command=find
+ local uri=myApp:myPass@some-name-mongos.demand-backup-sharded-32015
+ local postfix=1
+ local suffix=.svc.cluster.local
+ local database=myApp1
+ local collection=test
+ run_mongos 'use myApp1\n db.test.find()' myApp:myPass@some-name-mongos.demand-backup-sharded-32015 mongodb .svc.cluster.local
+ egrep -v 'I NETWORK|W NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:'
+ local 'command=use myApp1\n db.test.find()'
+ local uri=myApp:myPass@some-name-mongos.demand-backup-sharded-32015
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ /usr/bin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.bnpN11n445
+++ mktemp
++ local LAST_ERR=/tmp/tmp.C42QzD5MQI
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 0 ']'
++ break
++ cat /tmp/tmp.bnpN11n445
++ cat /tmp/tmp.C42QzD5MQI
++ rm /tmp/tmp.bnpN11n445 /tmp/tmp.C42QzD5MQI
++ return 0
+ local client_container=psmdb-client-5dc94d5b48-vt7s4
+ local mongo_flag=
+ kubectl_bin exec psmdb-client-5dc94d5b48-vt7s4 -- bash -c 'printf '\''use myApp1\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-32015.svc.cluster.local/admin '
++ mktemp
+ local LAST_OUT=/tmp/tmp.7HSX88idgY
++ mktemp
+ local LAST_ERR=/tmp/tmp.89szyQyTD9
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl exec psmdb-client-5dc94d5b48-vt7s4 -- bash -c 'printf '\''use myApp1\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-32015.svc.cluster.local/admin '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 0 ']'
+ break
+ cat /tmp/tmp.7HSX88idgY
+ cat /tmp/tmp.89szyQyTD9
+ rm /tmp/tmp.7HSX88idgY /tmp/tmp.89szyQyTD9
+ return 0
+ diff /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1536/e2e-tests/demand-backup-sharded/compare/find1.json /tmp/tmp.2ZOEX1UYL1/find1
+ for i in '$(seq 0 2)'
+ compare_mongos_cmd find myApp:myPass@some-name-mongos.demand-backup-sharded-32015 2 .svc.cluster.local myApp2 test
+ local command=find
+ local uri=myApp:myPass@some-name-mongos.demand-backup-sharded-32015
+ local postfix=2
+ local suffix=.svc.cluster.local
+ local database=myApp2
+ local collection=test
+ run_mongos 'use myApp2\n db.test.find()' myApp:myPass@some-name-mongos.demand-backup-sharded-32015 mongodb .svc.cluster.local
+ local 'command=use myApp2\n db.test.find()'
+ local uri=myApp:myPass@some-name-mongos.demand-backup-sharded-32015
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ egrep -v 'I NETWORK|W NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:'
+ /usr/bin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.9LcUBDtcrZ
+++ mktemp
++ local LAST_ERR=/tmp/tmp.5u9hqURAeX
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 0 ']'
++ break
++ cat /tmp/tmp.9LcUBDtcrZ
++ cat /tmp/tmp.5u9hqURAeX
++ rm /tmp/tmp.9LcUBDtcrZ /tmp/tmp.5u9hqURAeX
++ return 0
+ local client_container=psmdb-client-5dc94d5b48-vt7s4
+ local mongo_flag=
+ kubectl_bin exec psmdb-client-5dc94d5b48-vt7s4 -- bash -c 'printf '\''use myApp2\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-32015.svc.cluster.local/admin '
++ mktemp
+ local LAST_OUT=/tmp/tmp.eH9PahpV59
++ mktemp
+ local LAST_ERR=/tmp/tmp.cDKHhn7eqc
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl exec psmdb-client-5dc94d5b48-vt7s4 -- bash -c 'printf '\''use myApp2\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-32015.svc.cluster.local/admin '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 0 ']'
+ break
+ cat /tmp/tmp.eH9PahpV59
+ cat /tmp/tmp.cDKHhn7eqc
+ rm /tmp/tmp.eH9PahpV59 /tmp/tmp.cDKHhn7eqc
+ return 0
+ diff /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1536/e2e-tests/demand-backup-sharded/compare/find2.json /tmp/tmp.2ZOEX1UYL1/find2
+ desc 'delete backup and check if it is removed from bucket -- minio'
+ set +o xtrace


-----------------------------------------------------------------------------------
delete backup and check if it is removed from bucket -- minio
-----------------------------------------------------------------------------------

perconaservermongodbbackup.psmdb.percona.com "backup-aws-s3" deleted
perconaservermongodbbackup.psmdb.percona.com "backup-azure-blob" deleted
perconaservermongodbbackup.psmdb.percona.com "backup-gcp-cs" deleted
perconaservermongodbbackup.psmdb.percona.com "backup-minio" deleted
If you don't see a command prompt, try pressing enter.
warning: couldn't attach to pod/aws-cli, falling back to streaming logs: unable to upgrade connection: container aws-cli not found in pod aws-cli_demand-backup-sharded-32015


-----------------------------------------------------------------------------------
check for passwords leak
-----------------------------------------------------------------------------------

secrets=YmFja3VwMTIzNDU2 Y2x1c3RlckFkbWluMTIzNDU2 Y2x1c3Rlck1vbml0b3IxMjM0NTY= ZGF0YWJhc2VBZG1pbjEyMzQ1Ng== dXNlckFkbWluMTIzNDU2 YmFja3VwMTIzNDU2 Y2x1c3RlckFkbWluMTIzNDU2 Y2x1c3Rlck1vbml0b3IxMjM0NTY= ZGF0YWJhc2VBZG1pbjEyMzQ1Ng== dXNlckFkbWluMTIzNDU2
passwords=backup123456 clusterAdmin123456 clusterMonitor123456 databaseAdmin123456 userAdmin123456 backup123456 clusterAdmin123456 clusterMonitor123456 databaseAdmin123456 userAdmin123456 YmFja3VwMTIzNDU2 Y2x1c3RlckFkbWluMTIzNDU2 Y2x1c3Rlck1vbml0b3IxMjM0NTY= ZGF0YWJhc2VBZG1pbjEyMzQ1Ng== dXNlckFkbWluMTIzNDU2 YmFja3VwMTIzNDU2 Y2x1c3RlckFkbWluMTIzNDU2 Y2x1c3Rlck1vbml0b3IxMjM0NTY= ZGF0YWJhc2VBZG1pbjEyMzQ1Ng== dXNlckFkbWluMTIzNDU2
pods=minio-service-57dd49b-8qvml psmdb-client-5dc94d5b48-vt7s4 some-name-cfg-0 some-name-cfg-1 some-name-cfg-2 some-name-mongos-0 some-name-mongos-1 some-name-mongos-2 some-name-rs0-0 some-name-rs0-1 some-name-rs0-2 some-name-rs1-0 some-name-rs1-1 some-name-rs1-2 some-name-rs2-0 some-name-rs2-1 some-name-rs2-2
logs saved in: /tmp/tmp.2ZOEX1UYL1/logs_output-minio-service-57dd49b-8qvml-minio.txt

logs saved in: /tmp/tmp.2ZOEX1UYL1/logs_output-psmdb-client-5dc94d5b48-vt7s4-psmdb-client.txt

logs saved in: /tmp/tmp.2ZOEX1UYL1/logs_output-some-name-cfg-0-mongod.txt
logs saved in: /tmp/tmp.2ZOEX1UYL1/logs_output-some-name-cfg-0-cfg-sidecar-1.txt
logs saved in: /tmp/tmp.2ZOEX1UYL1/logs_output-some-name-cfg-0-backup-agent.txt

logs saved in: /tmp/tmp.2ZOEX1UYL1/logs_output-some-name-cfg-1-mongod.txt
logs saved in: /tmp/tmp.2ZOEX1UYL1/logs_output-some-name-cfg-1-cfg-sidecar-1.txt
logs saved in: /tmp/tmp.2ZOEX1UYL1/logs_output-some-name-cfg-1-backup-agent.txt

logs saved in: /tmp/tmp.2ZOEX1UYL1/logs_output-some-name-cfg-2-mongod.txt
logs saved in: /tmp/tmp.2ZOEX1UYL1/logs_output-some-name-cfg-2-cfg-sidecar-1.txt
logs saved in: /tmp/tmp.2ZOEX1UYL1/logs_output-some-name-cfg-2-backup-agent.txt

logs saved in: /tmp/tmp.2ZOEX1UYL1/logs_output-some-name-mongos-0-mongos.txt
logs saved in: /tmp/tmp.2ZOEX1UYL1/logs_output-some-name-mongos-0-mongos-sidecar-1.txt

logs saved in: /tmp/tmp.2ZOEX1UYL1/logs_output-some-name-mongos-1-mongos.txt
logs saved in: /tmp/tmp.2ZOEX1UYL1/logs_output-some-name-mongos-1-mongos-sidecar-1.txt

logs saved in: /tmp/tmp.2ZOEX1UYL1/logs_output-some-name-mongos-2-mongos.txt
logs saved in: /tmp/tmp.2ZOEX1UYL1/logs_output-some-name-mongos-2-mongos-sidecar-1.txt

logs saved in: /tmp/tmp.2ZOEX1UYL1/logs_output-some-name-rs0-0-mongod.txt
logs saved in: /tmp/tmp.2ZOEX1UYL1/logs_output-some-name-rs0-0-backup-agent.txt

logs saved in: /tmp/tmp.2ZOEX1UYL1/logs_output-some-name-rs0-1-mongod.txt
logs saved in: /tmp/tmp.2ZOEX1UYL1/logs_output-some-name-rs0-1-backup-agent.txt

logs saved in: /tmp/tmp.2ZOEX1UYL1/logs_output-some-name-rs0-2-mongod.txt
logs saved in: /tmp/tmp.2ZOEX1UYL1/logs_output-some-name-rs0-2-backup-agent.txt

logs saved in: /tmp/tmp.2ZOEX1UYL1/logs_output-some-name-rs1-0-mongod.txt
logs saved in: /tmp/tmp.2ZOEX1UYL1/logs_output-some-name-rs1-0-backup-agent.txt

logs saved in: /tmp/tmp.2ZOEX1UYL1/logs_output-some-name-rs1-1-mongod.txt
logs saved in: /tmp/tmp.2ZOEX1UYL1/logs_output-some-name-rs1-1-backup-agent.txt

logs saved in: /tmp/tmp.2ZOEX1UYL1/logs_output-some-name-rs1-2-mongod.txt
logs saved in: /tmp/tmp.2ZOEX1UYL1/logs_output-some-name-rs1-2-backup-agent.txt

logs saved in: /tmp/tmp.2ZOEX1UYL1/logs_output-some-name-rs2-0-mongod.txt
logs saved in: /tmp/tmp.2ZOEX1UYL1/logs_output-some-name-rs2-0-rs-sidecar-1.txt
logs saved in: /tmp/tmp.2ZOEX1UYL1/logs_output-some-name-rs2-0-backup-agent.txt

logs saved in: /tmp/tmp.2ZOEX1UYL1/logs_output-some-name-rs2-1-mongod.txt
logs saved in: /tmp/tmp.2ZOEX1UYL1/logs_output-some-name-rs2-1-rs-sidecar-1.txt
logs saved in: /tmp/tmp.2ZOEX1UYL1/logs_output-some-name-rs2-1-backup-agent.txt

logs saved in: /tmp/tmp.2ZOEX1UYL1/logs_output-some-name-rs2-2-mongod.txt
logs saved in: /tmp/tmp.2ZOEX1UYL1/logs_output-some-name-rs2-2-rs-sidecar-1.txt
logs saved in: /tmp/tmp.2ZOEX1UYL1/logs_output-some-name-rs2-2-backup-agent.txt

logs saved in: /tmp/tmp.2ZOEX1UYL1/logs_output-percona-server-mongodb-operator-75c69449b4-x2kfr-percona-server-mongodb-operator.txt

runtimeclass.node.k8s.io "container-rc" deleted


-----------------------------------------------------------------------------------
destroy cluster/operator and all other resources
-----------------------------------------------------------------------------------



-----------------------------------------------------------------------------------
get and delete old CRDs and RBAC
-----------------------------------------------------------------------------------

customresourcedefinition.apiextensions.k8s.io "perconaservermongodbbackups.psmdb.percona.com" deleted
customresourcedefinition.apiextensions.k8s.io "perconaservermongodbrestores.psmdb.percona.com" deleted
customresourcedefinition.apiextensions.k8s.io "perconaservermongodbs.psmdb.percona.com" deleted
error: the server doesn't have a resource type "perconaservermongodbbackups"
+ kubectl patch perconaservermongodbbackups.psmdb.percona.com -n sh --type=merge -p '{"metadata":{"finalizers":[]}}'
error: the server doesn't have a resource type "perconaservermongodbbackups"
error: the server doesn't have a resource type "perconaservermongodbrestores"
+ kubectl patch perconaservermongodbrestores.psmdb.percona.com -n sh --type=merge -p '{"metadata":{"finalizers":[]}}'
error: the server doesn't have a resource type "perconaservermongodbrestores"
error: the server doesn't have a resource type "perconaservermongodbs"
+ kubectl patch perconaservermongodbs.psmdb.percona.com -n sh --type=merge -p '{"metadata":{"finalizers":[]}}'
error: the server doesn't have a resource type "perconaservermongodbs"
clusterrole.rbac.authorization.k8s.io "percona-server-mongodb-operator" deleted
clusterrolebinding.rbac.authorization.k8s.io "service-account-percona-server-mongodb-operator" deleted