Log: /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1352/e2e-tests/logs/pitr.log
+ main
+ create_infra pitr-27834
+ local ns=pitr-27834
+ delete_crd
+ desc 'get and delete old CRDs and RBAC'
+ set +o xtrace


-----------------------------------------------------------------------------------
get and delete old CRDs and RBAC
-----------------------------------------------------------------------------------

+ kubectl_bin delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1352/deploy/crd.yaml --ignore-not-found --wait=false
++ mktemp
+ local LAST_OUT=/tmp/tmp.y4rHuudeWc
++ mktemp
+ local LAST_ERR=/tmp/tmp.pxjXzEyMen
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1352/deploy/crd.yaml --ignore-not-found --wait=false
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.y4rHuudeWc
customresourcedefinition.apiextensions.k8s.io "perconaservermongodbbackups.psmdb.percona.com" deleted
customresourcedefinition.apiextensions.k8s.io "perconaservermongodbrestores.psmdb.percona.com" deleted
customresourcedefinition.apiextensions.k8s.io "perconaservermongodbs.psmdb.percona.com" deleted
+ cat /tmp/tmp.pxjXzEyMen
+ rm /tmp/tmp.y4rHuudeWc /tmp/tmp.pxjXzEyMen
+ return 0
++ yq eval .metadata.name /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1352/deploy/crd.yaml
++ grep -v '\-\-\-'
+ for crd_name in '$(yq eval '\''.metadata.name'\'' "${src_dir}/deploy/crd.yaml" | grep -v '\''\-\-\-'\'')'
+ kubectl get perconaservermongodbbackups.psmdb.percona.com --all-namespaces -o wide
+ xargs -L 1 sh -xc 'kubectl patch perconaservermongodbbackups.psmdb.percona.com -n $0 $1 --type=merge -p "{\"metadata\":{\"finalizers\":[]}}"'
+ grep -v NAMESPACE
+ kubectl patch perconaservermongodbbackups.psmdb.percona.com -n pitr-2615 backup-minio-0 --type=merge -p '{"metadata":{"finalizers":[]}}'
perconaservermongodbbackup.psmdb.percona.com/backup-minio-0 patched
+ kubectl patch perconaservermongodbbackups.psmdb.percona.com -n pitr-2615 backup-minio-1 --type=merge -p '{"metadata":{"finalizers":[]}}'
perconaservermongodbbackup.psmdb.percona.com/backup-minio-1 patched
+ kubectl patch perconaservermongodbbackups.psmdb.percona.com -n pitr-2615 backup-minio-2 --type=merge -p '{"metadata":{"finalizers":[]}}'
perconaservermongodbbackup.psmdb.percona.com/backup-minio-2 patched
+ kubectl patch perconaservermongodbbackups.psmdb.percona.com -n pitr-2615 backup-minio-3 --type=merge -p '{"metadata":{"finalizers":[]}}'
perconaservermongodbbackup.psmdb.percona.com/backup-minio-3 patched
+ kubectl_bin wait --for=delete crd perconaservermongodbbackups.psmdb.percona.com
++ mktemp
+ local LAST_OUT=/tmp/tmp.uwgKXXKhLS
++ mktemp
+ local LAST_ERR=/tmp/tmp.zd9QD9DLMA
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl wait --for=delete crd perconaservermongodbbackups.psmdb.percona.com
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.uwgKXXKhLS
+ cat /tmp/tmp.zd9QD9DLMA
+ rm /tmp/tmp.uwgKXXKhLS /tmp/tmp.zd9QD9DLMA
+ return 0
+ for crd_name in '$(yq eval '\''.metadata.name'\'' "${src_dir}/deploy/crd.yaml" | grep -v '\''\-\-\-'\'')'
+ xargs -L 1 sh -xc 'kubectl patch perconaservermongodbrestores.psmdb.percona.com -n $0 $1 --type=merge -p "{\"metadata\":{\"finalizers\":[]}}"'
+ grep -v NAMESPACE
+ kubectl get perconaservermongodbrestores.psmdb.percona.com --all-namespaces -o wide
error: the server doesn't have a resource type "perconaservermongodbrestores"
+ kubectl patch perconaservermongodbrestores.psmdb.percona.com -n sh --type=merge -p '{"metadata":{"finalizers":[]}}'
error: the server doesn't have a resource type "perconaservermongodbrestores"
+ :
+ kubectl_bin wait --for=delete crd perconaservermongodbrestores.psmdb.percona.com
++ mktemp
+ local LAST_OUT=/tmp/tmp.gjEErWkDL1
++ mktemp
+ local LAST_ERR=/tmp/tmp.aHGP48tPlc
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl wait --for=delete crd perconaservermongodbrestores.psmdb.percona.com
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.gjEErWkDL1
+ cat /tmp/tmp.aHGP48tPlc
+ rm /tmp/tmp.gjEErWkDL1 /tmp/tmp.aHGP48tPlc
+ return 0
+ for crd_name in '$(yq eval '\''.metadata.name'\'' "${src_dir}/deploy/crd.yaml" | grep -v '\''\-\-\-'\'')'
+ grep -v NAMESPACE
+ kubectl get perconaservermongodbs.psmdb.percona.com --all-namespaces -o wide
+ xargs -L 1 sh -xc 'kubectl patch perconaservermongodbs.psmdb.percona.com -n $0 $1 --type=merge -p "{\"metadata\":{\"finalizers\":[]}}"'
error: the server doesn't have a resource type "perconaservermongodbs"
+ kubectl patch perconaservermongodbs.psmdb.percona.com -n sh --type=merge -p '{"metadata":{"finalizers":[]}}'
error: the server doesn't have a resource type "perconaservermongodbs"
+ :
+ kubectl_bin wait --for=delete crd perconaservermongodbs.psmdb.percona.com
++ mktemp
+ local LAST_OUT=/tmp/tmp.O4VD2vOUjj
++ mktemp
+ local LAST_ERR=/tmp/tmp.rnXHFojKzt
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl wait --for=delete crd perconaservermongodbs.psmdb.percona.com
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.O4VD2vOUjj
+ cat /tmp/tmp.rnXHFojKzt
+ rm /tmp/tmp.O4VD2vOUjj /tmp/tmp.rnXHFojKzt
+ return 0
+ local rbac_yaml=rbac.yaml
+ '[' -n '' ']'
+ kubectl_bin delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1352/deploy/rbac.yaml --ignore-not-found
++ mktemp
+ local LAST_OUT=/tmp/tmp.ig9e96zNeV
++ mktemp
+ local LAST_ERR=/tmp/tmp.NockVeeTQH
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1352/deploy/rbac.yaml --ignore-not-found
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.ig9e96zNeV
role.rbac.authorization.k8s.io "percona-server-mongodb-operator" deleted
serviceaccount "percona-server-mongodb-operator" deleted
rolebinding.rbac.authorization.k8s.io "service-account-percona-server-mongodb-operator" deleted
+ cat /tmp/tmp.NockVeeTQH
+ rm /tmp/tmp.ig9e96zNeV /tmp/tmp.NockVeeTQH
+ return 0
+ check_crd_for_deletion PR-1352-9793733e
+ local git_tag=PR-1352-9793733e
++ /usr/bin/sed ':a;N;$!ba;s/\n/ /g'
++ curl -s https://raw.githubusercontent.com/percona/percona-server-mongodb-operator/PR-1352-9793733e/deploy/crd.yaml
++ yq eval .metadata.name
++ /usr/bin/sed s/---//g
+ for crd_name in '$(curl -s https://raw.githubusercontent.com/percona/percona-server-mongodb-operator/${git_tag}/deploy/crd.yaml | yq eval '\''.metadata.name'\'' | $sed '\''s/---//g'\'' | $sed '\'':a;N;$!ba;s/\n/ /g'\'')'
++ kubectl_bin get crd/null -o 'jsonpath={.status.conditions[-1].type}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.rkMQxLDfBO
+++ mktemp
++ local LAST_ERR=/tmp/tmp.gu422m0K1H
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get crd/null -o 'jsonpath={.status.conditions[-1].type}'
++ exit_status=1
++ set -e
++ '[' 1 '!=' 0 -a -n 1 ']'
++ cat /tmp/tmp.rkMQxLDfBO
++ cat /tmp/tmp.gu422m0K1H
Error from server (NotFound): customresourcedefinitions.apiextensions.k8s.io "null" not found
++ sleep 0
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get crd/null -o 'jsonpath={.status.conditions[-1].type}'
++ exit_status=1
++ set -e
++ '[' 1 '!=' 0 -a -n 1 ']'
++ cat /tmp/tmp.rkMQxLDfBO
++ cat /tmp/tmp.gu422m0K1H
Error from server (NotFound): customresourcedefinitions.apiextensions.k8s.io "null" not found
++ sleep 4
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get crd/null -o 'jsonpath={.status.conditions[-1].type}'
++ exit_status=1
++ set -e
++ '[' 1 '!=' 0 -a -n 1 ']'
++ cat /tmp/tmp.rkMQxLDfBO
++ cat /tmp/tmp.gu422m0K1H
Error from server (NotFound): customresourcedefinitions.apiextensions.k8s.io "null" not found
++ sleep 8
++ cat /tmp/tmp.rkMQxLDfBO
++ cat /tmp/tmp.gu422m0K1H
Error from server (NotFound): customresourcedefinitions.apiextensions.k8s.io "null" not found
++ rm /tmp/tmp.rkMQxLDfBO /tmp/tmp.gu422m0K1H
++ return 1
+ [[ '' == \T\e\r\m\i\n\a\t\i\n\g ]]
+ '[' -n '' ']'
+ create_namespace pitr-27834
+ local namespace=pitr-27834
+ local skip_clean_namespace=
+ [[ 1 == 1 ]]
+ [[ -z '' ]]
+ destroy_chaos_mesh
++ sed s/NAMESPACE//
++ awk '-F ' '{print $2}'
++ helm list --all-namespaces --filter chaos-mesh
++ tail -n1
+ local chaos_mesh_ns=
+ desc 'destroy chaos-mesh'
+ set +o xtrace


-----------------------------------------------------------------------------------
destroy chaos-mesh
-----------------------------------------------------------------------------------

++ awk '{print $1}'
++ grep chaos-mesh
++ kubectl api-resources
+ '[' -n '' ']'
++ kubectl get crd
++ awk '{print $1}'
++ grep chaos-mesh.org
+ timeout 30 kubectl delete crd
error: resource(s) were provided, but no name was specified
+ :
++ awk '{print $1}'
++ kubectl get clusterrolebinding
++ grep chaos-mesh
+ timeout 30 kubectl delete clusterrolebinding
error: resource(s) were provided, but no name was specified
+ :
++ awk '{print $1}'
++ kubectl get clusterrole
++ grep chaos-mesh
+ timeout 30 kubectl delete clusterrole
error: resource(s) were provided, but no name was specified
+ :
++ kubectl get MutatingWebhookConfiguration
++ awk '{print $1}'
++ grep chaos-mesh
+ timeout 30 kubectl delete MutatingWebhookConfiguration
error: resource(s) were provided, but no name was specified
+ :
++ kubectl get ValidatingWebhookConfiguration
++ grep chaos-mesh
++ awk '{print $1}'
+ timeout 30 kubectl delete ValidatingWebhookConfiguration
error: resource(s) were provided, but no name was specified
+ :
++ awk '{print $1}'
++ kubectl get ValidatingWebhookConfiguration
++ grep validate-auth
+ timeout 30 kubectl delete ValidatingWebhookConfiguration
error: resource(s) were provided, but no name was specified
+ :
+ desc 'cleaned up all old namespaces'
+ set +o xtrace


-----------------------------------------------------------------------------------
cleaned up all old namespaces
-----------------------------------------------------------------------------------

+ egrep -v '^kube-|^default|Terminating|psmdb-operator|openshift|gke-mcs|^NAME'
+ awk '{print$1}'
+ '[' -n '' ']'
+ desc 'cleaned up old namespaces pitr-27834'
+ set +o xtrace


-----------------------------------------------------------------------------------
cleaned up old namespaces pitr-27834
-----------------------------------------------------------------------------------

+ kubectl_bin delete namespace pitr-27834 --ignore-not-found
+ kubectl_bin get ns
+ xargs kubectl delete ns
++ mktemp
+ local LAST_OUT=/tmp/tmp.C2ikBfFTKD
++ mktemp
+ local LAST_OUT=/tmp/tmp.syufdpbdh8
++ mktemp
++ mktemp
+ local LAST_ERR=/tmp/tmp.vRdj6ntdEm
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ local LAST_ERR=/tmp/tmp.QEK1JJ1Hpt
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl delete namespace pitr-27834 --ignore-not-found
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl get ns
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.syufdpbdh8
+ cat /tmp/tmp.QEK1JJ1Hpt
+ rm /tmp/tmp.syufdpbdh8 /tmp/tmp.QEK1JJ1Hpt
+ return 0
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.C2ikBfFTKD
+ cat /tmp/tmp.vRdj6ntdEm
+ rm /tmp/tmp.C2ikBfFTKD /tmp/tmp.vRdj6ntdEm
+ return 0
+ kubectl_bin wait --for=delete namespace pitr-27834
++ mktemp
+ local LAST_OUT=/tmp/tmp.QsvLQ9rnYS
++ mktemp
+ local LAST_ERR=/tmp/tmp.QC2nIeA8jQ
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl wait --for=delete namespace pitr-27834
namespace "pitr-2615" deleted
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.QsvLQ9rnYS
+ cat /tmp/tmp.QC2nIeA8jQ
+ rm /tmp/tmp.QsvLQ9rnYS /tmp/tmp.QC2nIeA8jQ
+ return 0
+ desc 'create namespace pitr-27834'
+ set +o xtrace


-----------------------------------------------------------------------------------
create namespace pitr-27834
-----------------------------------------------------------------------------------

+ kubectl_bin create namespace pitr-27834
++ mktemp
+ local LAST_OUT=/tmp/tmp.U23IGaYGdL
++ mktemp
+ local LAST_ERR=/tmp/tmp.063CaC9uZG
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl create namespace pitr-27834
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.U23IGaYGdL
namespace/pitr-27834 created
+ cat /tmp/tmp.063CaC9uZG
+ rm /tmp/tmp.U23IGaYGdL /tmp/tmp.063CaC9uZG
+ return 0
++ kubectl_bin config current-context
+++ mktemp
++ local LAST_OUT=/tmp/tmp.RGkXebQ9Wf
+++ mktemp
++ local LAST_ERR=/tmp/tmp.bn3UvNjkC8
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl config current-context
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.RGkXebQ9Wf
++ cat /tmp/tmp.bn3UvNjkC8
++ rm /tmp/tmp.RGkXebQ9Wf /tmp/tmp.bn3UvNjkC8
++ return 0
+ kubectl_bin config set-context gke_cloud-dev-112233_us-central1-a_jen-psmdb-1352-9793733e-5-cluster2 --namespace=pitr-27834
++ mktemp
+ local LAST_OUT=/tmp/tmp.V2YqogfY3l
++ mktemp
+ local LAST_ERR=/tmp/tmp.EYlb1PFV74
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl config set-context gke_cloud-dev-112233_us-central1-a_jen-psmdb-1352-9793733e-5-cluster2 --namespace=pitr-27834
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.V2YqogfY3l
Context "gke_cloud-dev-112233_us-central1-a_jen-psmdb-1352-9793733e-5-cluster2" modified.
+ cat /tmp/tmp.EYlb1PFV74
+ rm /tmp/tmp.V2YqogfY3l /tmp/tmp.EYlb1PFV74
+ return 0
+ deploy_operator
+ desc 'start PSMDB operator'
+ set +o xtrace


-----------------------------------------------------------------------------------
start PSMDB operator
-----------------------------------------------------------------------------------

+ local cr_file
+ '[' -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1352/e2e-tests/pitr/conf/crd.yaml ']'
+ cr_file=/mnt/jenkins/workspace/cloud-psmdb-operator_PR-1352/deploy/crd.yaml
+ kubectl_bin apply --server-side --force-conflicts -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1352/deploy/crd.yaml
++ mktemp
+ local LAST_OUT=/tmp/tmp.vEcuLr8ZBo
++ mktemp
+ local LAST_ERR=/tmp/tmp.UACMJXFTCx
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl apply --server-side --force-conflicts -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1352/deploy/crd.yaml
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.vEcuLr8ZBo
customresourcedefinition.apiextensions.k8s.io/perconaservermongodbbackups.psmdb.percona.com serverside-applied
customresourcedefinition.apiextensions.k8s.io/perconaservermongodbrestores.psmdb.percona.com serverside-applied
customresourcedefinition.apiextensions.k8s.io/perconaservermongodbs.psmdb.percona.com serverside-applied
+ cat /tmp/tmp.UACMJXFTCx
+ rm /tmp/tmp.vEcuLr8ZBo /tmp/tmp.UACMJXFTCx
+ return 0
+ '[' -n '' ']'
+ apply_rbac rbac
+ local operator_namespace=psmdb-operator
+ local rbac=rbac
+ kubectl_bin apply -f -
+ cat /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1352/deploy/rbac.yaml
+ sed -e 's^namespace: .*^namespace: psmdb-operator^'
++ mktemp
+ local LAST_OUT=/tmp/tmp.dxmMzXtlPs
++ mktemp
+ local LAST_ERR=/tmp/tmp.UjPHp3NTFo
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl apply -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.dxmMzXtlPs
role.rbac.authorization.k8s.io/percona-server-mongodb-operator created
serviceaccount/percona-server-mongodb-operator created
rolebinding.rbac.authorization.k8s.io/service-account-percona-server-mongodb-operator created
+ cat /tmp/tmp.UjPHp3NTFo
+ rm /tmp/tmp.dxmMzXtlPs /tmp/tmp.UjPHp3NTFo
+ return 0
+ yq eval '
			(.spec.template.spec.containers[].image = "perconalab/percona-server-mongodb-operator:PR-1352-9793733e") |
			((.. | select(.[] == "DISABLE_TELEMETRY")) |= .value="true") |
			((.. | select(.[] == "LOG_LEVEL")) |= .value="DEBUG")' /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1352/deploy/operator.yaml
+ kubectl_bin apply -f -
++ mktemp
+ local LAST_OUT=/tmp/tmp.FYyvSWoYeK
++ mktemp
+ local LAST_ERR=/tmp/tmp.UUN6l2hIMU
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl apply -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.FYyvSWoYeK
deployment.apps/percona-server-mongodb-operator created
+ cat /tmp/tmp.UUN6l2hIMU
+ rm /tmp/tmp.FYyvSWoYeK /tmp/tmp.UUN6l2hIMU
+ return 0
+ sleep 2
++ get_operator_pod
++ kubectl_bin get pods --selector=name=percona-server-mongodb-operator -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.ZOygZPVasC
+++ mktemp
++ local LAST_ERR=/tmp/tmp.Gtu4XafQLI
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=percona-server-mongodb-operator -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.ZOygZPVasC
++ cat /tmp/tmp.Gtu4XafQLI
++ rm /tmp/tmp.ZOygZPVasC /tmp/tmp.Gtu4XafQLI
++ return 0
+ wait_pod percona-server-mongodb-operator-6b6bcb669-c78t2
+ local pod=percona-server-mongodb-operator-6b6bcb669-c78t2
+ set +o xtrace
waiting for pod/percona-server-mongodb-operator-6b6bcb669-c78t2 to be ready.OK
+ deploy_minio
+ desc 'install Minio'
+ set +o xtrace


-----------------------------------------------------------------------------------
install Minio
-----------------------------------------------------------------------------------

+ helm uninstall minio-service
Error: uninstall: Release not loaded: minio-service: release: not found
+ :
+ helm repo remove minio
"minio" has been removed from your repositories
+ helm repo add minio https://helm.min.io/
"minio" has been added to your repositories
+ retry 10 60 helm install minio-service --version 8.0.5 --set accessKey=some-access-key --set secretKey=some-secret-key --set service.type=ClusterIP --set configPathmc=/tmp/.minio/ --set persistence.size=2G --set environment.MINIO_REGION=us-east-1 --set environment.MINIO_HTTP_TRACE=/tmp/trace.log --set securityContext.enabled=false minio/minio
+ local max=10
+ local delay=60
+ shift 2
+ local n=1
+ helm install minio-service --version 8.0.5 --set accessKey=some-access-key --set secretKey=some-secret-key --set service.type=ClusterIP --set configPathmc=/tmp/.minio/ --set persistence.size=2G --set environment.MINIO_REGION=us-east-1 --set environment.MINIO_HTTP_TRACE=/tmp/trace.log --set securityContext.enabled=false minio/minio
NAME: minio-service
LAST DEPLOYED: Thu Oct  5 15:21:44 2023
NAMESPACE: pitr-27834
STATUS: deployed
REVISION: 1
TEST SUITE: None
NOTES:
Minio can be accessed via port 9000 on the following DNS name from within your cluster:
minio-service.pitr-27834.svc.cluster.local

To access Minio from localhost, run the below commands:

  1. export POD_NAME=$(kubectl get pods --namespace pitr-27834 -l "release=minio-service" -o jsonpath="{.items[0].metadata.name}")

  2. kubectl port-forward $POD_NAME 9000 --namespace pitr-27834

Read more about port forwarding here: http://kubernetes.io/docs/user-guide/kubectl/kubectl_port-forward/

You can now access Minio server on http://localhost:9000. Follow the below steps to connect to Minio server with mc client:

  1. Download the Minio mc client - https://docs.minio.io/docs/minio-client-quickstart-guide

  2. Get the ACCESS_KEY=$(kubectl get secret minio-service -o jsonpath="{.data.accesskey}" | base64 --decode) and the SECRET_KEY=$(kubectl get secret minio-service -o jsonpath="{.data.secretkey}" | base64 --decode)

  3. mc alias set minio-service-local http://localhost:9000 "$ACCESS_KEY" "$SECRET_KEY" --api s3v4

  4. mc ls minio-service-local

Alternately, you can use your browser or the Minio SDK to access the server - https://docs.minio.io/categories/17
++ kubectl_bin get pods --selector=release=minio-service -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.OoVxByMlNz
+++ mktemp
++ local LAST_ERR=/tmp/tmp.VU7T9wHxqP
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=release=minio-service -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.OoVxByMlNz
++ cat /tmp/tmp.VU7T9wHxqP
++ rm /tmp/tmp.OoVxByMlNz /tmp/tmp.VU7T9wHxqP
++ return 0
+ MINIO_POD=minio-service-5fc4dc898-r6qxw
+ wait_pod minio-service-5fc4dc898-r6qxw
+ local pod=minio-service-5fc4dc898-r6qxw
+ set +o xtrace
waiting for pod/minio-service-5fc4dc898-r6qxw to be ready....OK
+ '[' -n '' ']'
+ kubectl_bin run -i --rm aws-cli --image=perconalab/awscli --restart=Never -- bash -c 'AWS_ACCESS_KEY_ID=some-access-key AWS_SECRET_ACCESS_KEY=some-secret-key AWS_DEFAULT_REGION=us-east-1 \
        /usr/bin/aws --endpoint-url http://minio-service:9000 s3 mb s3://operator-testing'
++ mktemp
+ local LAST_OUT=/tmp/tmp.OAtjfLYxMa
++ mktemp
+ local LAST_ERR=/tmp/tmp.d3AddHkygR
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl run -i --rm aws-cli --image=perconalab/awscli --restart=Never -- bash -c 'AWS_ACCESS_KEY_ID=some-access-key AWS_SECRET_ACCESS_KEY=some-secret-key AWS_DEFAULT_REGION=us-east-1 \
        /usr/bin/aws --endpoint-url http://minio-service:9000 s3 mb s3://operator-testing'
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.OAtjfLYxMa
make_bucket: operator-testing
pod "aws-cli" deleted
+ cat /tmp/tmp.d3AddHkygR
If you don't see a command prompt, try pressing enter.
Error attaching, falling back to logs: Internal error occurred: error attaching to container: container is in CONTAINER_EXITED state
+ rm /tmp/tmp.OAtjfLYxMa /tmp/tmp.d3AddHkygR
+ return 0
+ desc 'create secrets and start client'
+ set +o xtrace


-----------------------------------------------------------------------------------
create secrets and start client
-----------------------------------------------------------------------------------

+ kubectl_bin apply -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1352/e2e-tests/conf/secrets.yml -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1352/e2e-tests/conf/client.yml -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1352/e2e-tests/conf/minio-secret.yml
++ mktemp
+ local LAST_OUT=/tmp/tmp.zIImsEkS9I
++ mktemp
+ local LAST_ERR=/tmp/tmp.z4RB841dc8
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl apply -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1352/e2e-tests/conf/secrets.yml -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1352/e2e-tests/conf/client.yml -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1352/e2e-tests/conf/minio-secret.yml
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.zIImsEkS9I
secret/some-users created
deployment.apps/psmdb-client created
secret/minio-secret created
+ cat /tmp/tmp.z4RB841dc8
+ rm /tmp/tmp.zIImsEkS9I /tmp/tmp.z4RB841dc8
+ return 0
+ cluster=some-name-rs0
+ desc 'create first PSMDB cluster some-name-rs0'
+ set +o xtrace


-----------------------------------------------------------------------------------
create first PSMDB cluster some-name-rs0
-----------------------------------------------------------------------------------

+ apply_cluster /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1352/e2e-tests/pitr/conf/some-name-rs0.yml
+ '[' -z '' ']'
+ kubectl_bin apply -f -
+ cat_config /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1352/e2e-tests/pitr/conf/some-name-rs0.yml
+ yq eval '.spec.upgradeOptions.apply="Never"'
++ mktemp
+ yq eval '(.spec | select(.image == null)).image = "perconalab/percona-server-mongodb-operator:main-mongod5.0"'
+ yq eval '(.spec | select(has("backup"))).backup.image = "perconalab/percona-server-mongodb-operator:main-backup"'
+ local LAST_OUT=/tmp/tmp.knXMxaMDgI
+ cat /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1352/e2e-tests/pitr/conf/some-name-rs0.yml
+ yq eval '(.spec | select(has("pmm"))).pmm.image = "perconalab/pmm-client:dev-latest"'
++ mktemp
+ local LAST_ERR=/tmp/tmp.o4WqQsqwkJ
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl apply -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.knXMxaMDgI
perconaservermongodb.psmdb.percona.com/some-name created
+ cat /tmp/tmp.o4WqQsqwkJ
+ rm /tmp/tmp.knXMxaMDgI /tmp/tmp.o4WqQsqwkJ
+ return 0
+ desc 'check if all 3 Pods started'
+ set +o xtrace


-----------------------------------------------------------------------------------
check if all 3 Pods started
-----------------------------------------------------------------------------------

+ wait_for_running some-name-rs0 3
+ local name=some-name-rs0
+ let last_pod=2
+ local check_cluster_readyness=true
+ set_debug
+ [[ 1 == 1 ]]
+ set -o xtrace
+ local rs_name=rs0
+ local cluster_name=some-name
++ seq 0 2
+ for i in '$(seq 0 $last_pod)'
+ [[ 0 -eq 2 ]]
+ wait_pod some-name-rs0-0
+ local pod=some-name-rs0-0
+ set +o xtrace
waiting for pod/some-name-rs0-0 to be ready.....OK
+ for i in '$(seq 0 $last_pod)'
+ [[ 1 -eq 2 ]]
+ wait_pod some-name-rs0-1
+ local pod=some-name-rs0-1
+ set +o xtrace
waiting for pod/some-name-rs0-1 to be ready.....OK
+ for i in '$(seq 0 $last_pod)'
+ [[ 2 -eq 2 ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].arbiter.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.KLe93nMlMY
+++ mktemp
++ local LAST_ERR=/tmp/tmp.1T3UGTSfCR
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].arbiter.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.KLe93nMlMY
++ cat /tmp/tmp.1T3UGTSfCR
++ rm /tmp/tmp.KLe93nMlMY /tmp/tmp.1T3UGTSfCR
++ return 0
+ [[ '' == \t\r\u\e ]]
+ wait_pod some-name-rs0-2
+ local pod=some-name-rs0-2
+ set +o xtrace
waiting for pod/some-name-rs0-2 to be ready......OK
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].non_voting.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.7bTzNXwz8t
+++ mktemp
++ local LAST_ERR=/tmp/tmp.Jwou1STDVZ
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].non_voting.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.7bTzNXwz8t
++ cat /tmp/tmp.Jwou1STDVZ
++ rm /tmp/tmp.7bTzNXwz8t /tmp/tmp.Jwou1STDVZ
++ return 0
+ [[ '' == \t\r\u\e ]]
+ sleep 10
+ [[ true == \t\r\u\e ]]
+ set +x
Waiting for cluster readyness
+ desc 'check if statefulset created with expected config'
+ set +o xtrace


-----------------------------------------------------------------------------------
check if statefulset created with expected config
-----------------------------------------------------------------------------------

+ compare_kubectl statefulset/some-name-rs0
+ local resource=statefulset/some-name-rs0
+ local postfix=
+ local expected_result=/mnt/jenkins/workspace/cloud-psmdb-operator_PR-1352/e2e-tests/pitr/compare/statefulset_some-name-rs0.yml
+ local new_result=/tmp/tmp.iKog3JtyCU/statefulset_some-name-rs0.yml
+ '[' -n '' -a -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1352/e2e-tests/pitr/compare/statefulset_some-name-rs0-oc.yml ']'
+ kubectl_bin get -o yaml statefulset/some-name-rs0
+ yq eval '
			del(.metadata.ownerReferences[].apiVersion) |
			del(.metadata.managedFields) |
			del(.. | select(has("creationTimestamp")).creationTimestamp) |
			del(.. | select(has("namespace")).namespace) |
			del(.. | select(has("uid")).uid) |
			del(.metadata.resourceVersion) |
			del(.spec.template.spec.containers[].env[] | select(.name == "NAMESPACE")) |
			del(.metadata.selfLink) |
			del(.metadata.annotations."cloud.google.com/neg") |
			del(.. | select(has("image")).image) |
			del(.. | select(has("clusterIP")).clusterIP) |
			del(.. | select(has("clusterIPs")).clusterIPs) |
			del(.. | select(has("dataSource")).dataSource) |
			del(.. | select(has("procMount")).procMount) |
			del(.. | select(has("storageClassName")).storageClassName) |
			del(.. | select(has("finalizers")).finalizers) |
			del(.. | select(has("kubernetes.io/pvc-protection"))."kubernetes.io/pvc-protection") |
			del(.. | select(has("volumeName")).volumeName) |
			del(.. | select(has("volume.beta.kubernetes.io/storage-provisioner"))."volume.beta.kubernetes.io/storage-provisioner") |
			del(.. | select(has("volume.kubernetes.io/storage-provisioner"))."volume.kubernetes.io/storage-provisioner") |
			del(.spec.volumeMode) |
			del(.. | select(has("volume.kubernetes.io/selected-node"))."volume.kubernetes.io/selected-node") |
			del(.. | select(has("percona.com/last-config-hash"))."percona.com/last-config-hash") |
			del(.. | select(has("percona.com/configuration-hash"))."percona.com/configuration-hash") |
			del(.. | select(has("percona.com/ssl-hash"))."percona.com/ssl-hash") |
			del(.. | select(has("percona.com/ssl-internal-hash"))."percona.com/ssl-internal-hash") |
			del(.spec.volumeClaimTemplates[].spec.volumeMode | select(. == "Filesystem")) |
			del(.. | select(has("healthCheckNodePort")).healthCheckNodePort) |
			del(.. | select(has("nodePort")).nodePort) |
			del(.status) |
			(.. | select(tag == "!!str")) |= sub("pitr-27834", "NAME_SPACE") |
			del(.spec.volumeClaimTemplates[].apiVersion) |
			del(.spec.volumeClaimTemplates[].kind) |
			del(.spec.ipFamilies) |
			del(.spec.ipFamilyPolicy) |
			(.. | select(. == "extensions/v1beta1")) = "apps/v1" |
			(.. | select(. == "batch/v1beta1")) = "batch/v1" ' -
++ mktemp
+ local LAST_OUT=/tmp/tmp.Vz5GMwfKMx
++ mktemp
+ local LAST_ERR=/tmp/tmp.kJ9uiVORkB
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl get -o yaml statefulset/some-name-rs0
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.Vz5GMwfKMx
+ cat /tmp/tmp.kJ9uiVORkB
+ rm /tmp/tmp.Vz5GMwfKMx /tmp/tmp.kJ9uiVORkB
+ return 0
+ yq -i eval 'del(.spec.persistentVolumeClaimRetentionPolicy)' /tmp/tmp.iKog3JtyCU/statefulset_some-name-rs0.yml
+ version_gt 1.22
++ bc -l
++ echo '1.24 >= 1.22'
+ '[' 1 -eq 1 ']'
+ return 0
+ yq -i eval 'del(.spec.internalTrafficPolicy)' /tmp/tmp.iKog3JtyCU/statefulset_some-name-rs0.yml
+ yq -i eval 'del(.spec.allocateLoadBalancerNodePorts)' /tmp/tmp.iKog3JtyCU/statefulset_some-name-rs0.yml
+ [[ /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1352/e2e-tests/pitr/compare/statefulset_some-name-rs0.yml == */cronjob* ]]
+ diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1352/e2e-tests/pitr/compare/statefulset_some-name-rs0.yml /tmp/tmp.iKog3JtyCU/statefulset_some-name-rs0.yml
+ write_initial_data
+ desc 'create user myApp'
+ set +o xtrace


-----------------------------------------------------------------------------------
create user myApp
-----------------------------------------------------------------------------------

+ run_mongo 'db.createUser({user:"myApp",pwd:"myPass",roles:[{db:"myApp",role:"readWrite"}]})' userAdmin:userAdmin123456@some-name-rs0.pitr-27834
+ local 'command=db.createUser({user:"myApp",pwd:"myPass",roles:[{db:"myApp",role:"readWrite"}]})'
+ local uri=userAdmin:userAdmin123456@some-name-rs0.pitr-27834
+ local driver=mongodb+srv
+ local suffix=.svc.cluster.local
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.ZYlF3em4Wu
+++ mktemp
++ local LAST_ERR=/tmp/tmp.S5BREscPSF
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.ZYlF3em4Wu
++ cat /tmp/tmp.S5BREscPSF
++ rm /tmp/tmp.ZYlF3em4Wu /tmp/tmp.S5BREscPSF
++ return 0
+ local client_container=psmdb-client-9b7b5fdcc-46nql
+ local mongo_flag=
+ [[ userAdmin:userAdmin123456@some-name-rs0.pitr-27834 == *cfg* ]]
+ replica_set=rs0
+ kubectl_bin exec psmdb-client-9b7b5fdcc-46nql -- bash -c 'printf '\''db.createUser({user:"myApp",pwd:"myPass",roles:[{db:"myApp",role:"readWrite"}]})\n'\'' | mongo mongodb+srv://userAdmin:userAdmin123456@some-name-rs0.pitr-27834.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
++ mktemp
+ local LAST_OUT=/tmp/tmp.rUBBOIATAc
++ mktemp
+ local LAST_ERR=/tmp/tmp.WEQNNsWhTz
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl exec psmdb-client-9b7b5fdcc-46nql -- bash -c 'printf '\''db.createUser({user:"myApp",pwd:"myPass",roles:[{db:"myApp",role:"readWrite"}]})\n'\'' | mongo mongodb+srv://userAdmin:userAdmin123456@some-name-rs0.pitr-27834.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.rUBBOIATAc
Percona Server for MongoDB shell version v4.4.24-23
connecting to: mongodb://some-name-rs0-1.some-name-rs0.pitr-27834.svc.cluster.local:27017,some-name-rs0-0.some-name-rs0.pitr-27834.svc.cluster.local:27017,some-name-rs0-2.some-name-rs0.pitr-27834.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb&replicaSet=rs0&ssl=false
Implicit session: session { "id" : UUID("b6e1ad84-1302-4d14-8e08-1b93bf954fc3") }
Percona Server for MongoDB server version: v5.0.20-17
WARNING: shell and server versions do not match
Successfully added user: {
	"user" : "myApp",
	"roles" : [
		{
			"db" : "myApp",
			"role" : "readWrite"
		}
	]
}
bye
+ cat /tmp/tmp.WEQNNsWhTz
+ rm /tmp/tmp.rUBBOIATAc /tmp/tmp.WEQNNsWhTz
+ return 0
+ sleep 2
+ write_document
+ local cmp_postfix=
+ desc 'write initial data, read from all'
+ set +o xtrace


-----------------------------------------------------------------------------------
write initial data, read from all
-----------------------------------------------------------------------------------

+ run_mongo 'use myApp\n db.test.insert({ x: 100500 })' myApp:myPass@some-name-rs0.pitr-27834
+ local 'command=use myApp\n db.test.insert({ x: 100500 })'
+ local uri=myApp:myPass@some-name-rs0.pitr-27834
+ local driver=mongodb+srv
+ local suffix=.svc.cluster.local
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.MzAQBLLmYS
+++ mktemp
++ local LAST_ERR=/tmp/tmp.QDUjYioU2i
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.MzAQBLLmYS
++ cat /tmp/tmp.QDUjYioU2i
++ rm /tmp/tmp.MzAQBLLmYS /tmp/tmp.QDUjYioU2i
++ return 0
+ local client_container=psmdb-client-9b7b5fdcc-46nql
+ local mongo_flag=
+ [[ myApp:myPass@some-name-rs0.pitr-27834 == *cfg* ]]
+ replica_set=rs0
+ kubectl_bin exec psmdb-client-9b7b5fdcc-46nql -- bash -c 'printf '\''use myApp\n db.test.insert({ x: 100500 })\n'\'' | mongo mongodb+srv://myApp:myPass@some-name-rs0.pitr-27834.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
++ mktemp
+ local LAST_OUT=/tmp/tmp.H3XgKgnKpv
++ mktemp
+ local LAST_ERR=/tmp/tmp.ukzPaT7xqv
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl exec psmdb-client-9b7b5fdcc-46nql -- bash -c 'printf '\''use myApp\n db.test.insert({ x: 100500 })\n'\'' | mongo mongodb+srv://myApp:myPass@some-name-rs0.pitr-27834.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.H3XgKgnKpv
Percona Server for MongoDB shell version v4.4.24-23
connecting to: mongodb://some-name-rs0-1.some-name-rs0.pitr-27834.svc.cluster.local:27017,some-name-rs0-0.some-name-rs0.pitr-27834.svc.cluster.local:27017,some-name-rs0-2.some-name-rs0.pitr-27834.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb&replicaSet=rs0&ssl=false
Implicit session: session { "id" : UUID("fcd1c09f-3266-4dd3-a748-ee2845ac5c6b") }
Percona Server for MongoDB server version: v5.0.20-17
WARNING: shell and server versions do not match
switched to db myApp
WriteResult({ "nInserted" : 1 })
bye
+ cat /tmp/tmp.ukzPaT7xqv
+ rm /tmp/tmp.H3XgKgnKpv /tmp/tmp.ukzPaT7xqv
+ return 0
+ minikube_sleep
+ sleep_time=10
+ [[ '' == 1 ]]
+ compare_mongo_cmd find myApp:myPass@some-name-rs0-0.some-name-rs0.pitr-27834 ''
+ local command=find
+ local uri=myApp:myPass@some-name-rs0-0.some-name-rs0.pitr-27834
+ local postfix=
+ local suffix=
+ local database=myApp
+ local collection=test
+ egrep -v 'I NETWORK|W NETWORK|F NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match'
+ run_mongo 'use myApp\n db.test.find()' myApp:myPass@some-name-rs0-0.some-name-rs0.pitr-27834 mongodb ''
+ local 'command=use myApp\n db.test.find()'
+ local uri=myApp:myPass@some-name-rs0-0.some-name-rs0.pitr-27834
+ local driver=mongodb
+ local suffix=.svc.cluster.local
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+ /usr/bin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.hltaMgmlup
+++ mktemp
++ local LAST_ERR=/tmp/tmp.qYSZGvk2sf
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.hltaMgmlup
++ cat /tmp/tmp.qYSZGvk2sf
++ rm /tmp/tmp.hltaMgmlup /tmp/tmp.qYSZGvk2sf
++ return 0
+ local client_container=psmdb-client-9b7b5fdcc-46nql
+ local mongo_flag=
+ [[ myApp:myPass@some-name-rs0-0.some-name-rs0.pitr-27834 == *cfg* ]]
+ replica_set=rs0
+ kubectl_bin exec psmdb-client-9b7b5fdcc-46nql -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-0.some-name-rs0.pitr-27834.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
++ mktemp
+ local LAST_OUT=/tmp/tmp.kNsegYxSRj
++ mktemp
+ local LAST_ERR=/tmp/tmp.6J9F7PbFwb
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl exec psmdb-client-9b7b5fdcc-46nql -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-0.some-name-rs0.pitr-27834.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.kNsegYxSRj
+ cat /tmp/tmp.6J9F7PbFwb
+ rm /tmp/tmp.kNsegYxSRj /tmp/tmp.6J9F7PbFwb
+ return 0
+ diff /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1352/e2e-tests/pitr/compare/find.json /tmp/tmp.iKog3JtyCU/find
+ compare_mongo_cmd find myApp:myPass@some-name-rs0-1.some-name-rs0.pitr-27834 ''
+ local command=find
+ local uri=myApp:myPass@some-name-rs0-1.some-name-rs0.pitr-27834
+ local postfix=
+ local suffix=
+ local database=myApp
+ local collection=test
+ run_mongo 'use myApp\n db.test.find()' myApp:myPass@some-name-rs0-1.some-name-rs0.pitr-27834 mongodb ''
+ local 'command=use myApp\n db.test.find()'
+ local uri=myApp:myPass@some-name-rs0-1.some-name-rs0.pitr-27834
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ /usr/bin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
+ egrep -v 'I NETWORK|W NETWORK|F NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match'
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.UbO4b68til
+++ mktemp
++ local LAST_ERR=/tmp/tmp.v46uRINxI4
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.UbO4b68til
++ cat /tmp/tmp.v46uRINxI4
++ rm /tmp/tmp.UbO4b68til /tmp/tmp.v46uRINxI4
++ return 0
+ local client_container=psmdb-client-9b7b5fdcc-46nql
+ local mongo_flag=
+ [[ myApp:myPass@some-name-rs0-1.some-name-rs0.pitr-27834 == *cfg* ]]
+ replica_set=rs0
+ kubectl_bin exec psmdb-client-9b7b5fdcc-46nql -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-1.some-name-rs0.pitr-27834.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
++ mktemp
+ local LAST_OUT=/tmp/tmp.9JcwMNYNkn
++ mktemp
+ local LAST_ERR=/tmp/tmp.qVi4gjVt7N
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl exec psmdb-client-9b7b5fdcc-46nql -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-1.some-name-rs0.pitr-27834.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.9JcwMNYNkn
+ cat /tmp/tmp.qVi4gjVt7N
+ rm /tmp/tmp.9JcwMNYNkn /tmp/tmp.qVi4gjVt7N
+ return 0
+ diff /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1352/e2e-tests/pitr/compare/find.json /tmp/tmp.iKog3JtyCU/find
+ compare_mongo_cmd find myApp:myPass@some-name-rs0-2.some-name-rs0.pitr-27834 ''
+ local command=find
+ local uri=myApp:myPass@some-name-rs0-2.some-name-rs0.pitr-27834
+ local postfix=
+ local suffix=
+ local database=myApp
+ local collection=test
+ egrep -v 'I NETWORK|W NETWORK|F NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match'
+ run_mongo 'use myApp\n db.test.find()' myApp:myPass@some-name-rs0-2.some-name-rs0.pitr-27834 mongodb ''
+ local 'command=use myApp\n db.test.find()'
+ local uri=myApp:myPass@some-name-rs0-2.some-name-rs0.pitr-27834
+ local driver=mongodb
+ local suffix=.svc.cluster.local
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.zY6d3oS6WA
+++ mktemp
+ /usr/bin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
++ local LAST_ERR=/tmp/tmp.jSSiFYYRfq
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.zY6d3oS6WA
++ cat /tmp/tmp.jSSiFYYRfq
++ rm /tmp/tmp.zY6d3oS6WA /tmp/tmp.jSSiFYYRfq
++ return 0
+ local client_container=psmdb-client-9b7b5fdcc-46nql
+ local mongo_flag=
+ [[ myApp:myPass@some-name-rs0-2.some-name-rs0.pitr-27834 == *cfg* ]]
+ replica_set=rs0
+ kubectl_bin exec psmdb-client-9b7b5fdcc-46nql -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-2.some-name-rs0.pitr-27834.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
++ mktemp
+ local LAST_OUT=/tmp/tmp.m2Kt4XSZBb
++ mktemp
+ local LAST_ERR=/tmp/tmp.Oi2qQwMZxA
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl exec psmdb-client-9b7b5fdcc-46nql -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-2.some-name-rs0.pitr-27834.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.m2Kt4XSZBb
+ cat /tmp/tmp.Oi2qQwMZxA
+ rm /tmp/tmp.m2Kt4XSZBb /tmp/tmp.Oi2qQwMZxA
+ return 0
+ diff /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1352/e2e-tests/pitr/compare/find.json /tmp/tmp.iKog3JtyCU/find
+ wait_backup_agent some-name-rs0-0
+ local agent_pod=some-name-rs0-0
+ set +o xtrace
some-name-rs0-0
+ wait_backup_agent some-name-rs0-1
+ local agent_pod=some-name-rs0-1
+ set +o xtrace
some-name-rs0-1
+ wait_backup_agent some-name-rs0-2
+ local agent_pod=some-name-rs0-2
+ set +o xtrace
some-name-rs0-2
+ backup_name_minio=backup-minio
+ run_backup backup-minio 0
+ local name=backup-minio
+ local idx=0
+ desc 'run backup backup-minio-0'
+ set +o xtrace


-----------------------------------------------------------------------------------
run backup backup-minio-0
-----------------------------------------------------------------------------------

+ kubectl_bin apply -f -
+ /usr/bin/sed -e 's/name:/name: backup-minio-0/'
+ cat /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1352/e2e-tests/pitr/conf/backup-minio.yml
++ mktemp
+ local LAST_OUT=/tmp/tmp.TFNtiLKVsb
++ mktemp
+ local LAST_ERR=/tmp/tmp.hwc2PtXyF3
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl apply -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.TFNtiLKVsb
perconaservermongodbbackup.psmdb.percona.com/backup-minio-0 created
+ cat /tmp/tmp.hwc2PtXyF3
+ rm /tmp/tmp.TFNtiLKVsb /tmp/tmp.hwc2PtXyF3
+ return 0
+ wait_backup backup-minio-0
+ local backup_name=backup-minio-0
+ set +o xtrace
backup-minio-0........
+ write_document -2nd
+ local cmp_postfix=-2nd
+ desc 'write initial data, read from all'
+ set +o xtrace


-----------------------------------------------------------------------------------
write initial data, read from all
-----------------------------------------------------------------------------------

+ run_mongo 'use myApp\n db.test.insert({ x: 100500 })' myApp:myPass@some-name-rs0.pitr-27834
+ local 'command=use myApp\n db.test.insert({ x: 100500 })'
+ local uri=myApp:myPass@some-name-rs0.pitr-27834
+ local driver=mongodb+srv
+ local suffix=.svc.cluster.local
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.cbEGr3KlgA
+++ mktemp
++ local LAST_ERR=/tmp/tmp.cPlsMSRuko
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.cbEGr3KlgA
++ cat /tmp/tmp.cPlsMSRuko
++ rm /tmp/tmp.cbEGr3KlgA /tmp/tmp.cPlsMSRuko
++ return 0
+ local client_container=psmdb-client-9b7b5fdcc-46nql
+ local mongo_flag=
+ [[ myApp:myPass@some-name-rs0.pitr-27834 == *cfg* ]]
+ replica_set=rs0
+ kubectl_bin exec psmdb-client-9b7b5fdcc-46nql -- bash -c 'printf '\''use myApp\n db.test.insert({ x: 100500 })\n'\'' | mongo mongodb+srv://myApp:myPass@some-name-rs0.pitr-27834.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
++ mktemp
+ local LAST_OUT=/tmp/tmp.dE0sMbdv8C
++ mktemp
+ local LAST_ERR=/tmp/tmp.7YcsgHP2Nl
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl exec psmdb-client-9b7b5fdcc-46nql -- bash -c 'printf '\''use myApp\n db.test.insert({ x: 100500 })\n'\'' | mongo mongodb+srv://myApp:myPass@some-name-rs0.pitr-27834.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.dE0sMbdv8C
Percona Server for MongoDB shell version v4.4.24-23
connecting to: mongodb://some-name-rs0-1.some-name-rs0.pitr-27834.svc.cluster.local:27017,some-name-rs0-0.some-name-rs0.pitr-27834.svc.cluster.local:27017,some-name-rs0-2.some-name-rs0.pitr-27834.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb&replicaSet=rs0&ssl=false
Implicit session: session { "id" : UUID("70faf68c-7c4a-4af6-9077-9cccfd05da67") }
Percona Server for MongoDB server version: v5.0.20-17
WARNING: shell and server versions do not match
switched to db myApp
WriteResult({ "nInserted" : 1 })
bye
+ cat /tmp/tmp.7YcsgHP2Nl
+ rm /tmp/tmp.dE0sMbdv8C /tmp/tmp.7YcsgHP2Nl
+ return 0
+ minikube_sleep
+ sleep_time=10
+ [[ '' == 1 ]]
+ compare_mongo_cmd find myApp:myPass@some-name-rs0-0.some-name-rs0.pitr-27834 -2nd
+ local command=find
+ local uri=myApp:myPass@some-name-rs0-0.some-name-rs0.pitr-27834
+ local postfix=-2nd
+ local suffix=
+ local database=myApp
+ local collection=test
+ /usr/bin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
+ egrep -v 'I NETWORK|W NETWORK|F NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match'
+ run_mongo 'use myApp\n db.test.find()' myApp:myPass@some-name-rs0-0.some-name-rs0.pitr-27834 mongodb ''
+ local 'command=use myApp\n db.test.find()'
+ local uri=myApp:myPass@some-name-rs0-0.some-name-rs0.pitr-27834
+ local driver=mongodb
+ local suffix=.svc.cluster.local
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.fnJTcNObGV
+++ mktemp
++ local LAST_ERR=/tmp/tmp.bXtJAgJeHQ
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.fnJTcNObGV
++ cat /tmp/tmp.bXtJAgJeHQ
++ rm /tmp/tmp.fnJTcNObGV /tmp/tmp.bXtJAgJeHQ
++ return 0
+ local client_container=psmdb-client-9b7b5fdcc-46nql
+ local mongo_flag=
+ [[ myApp:myPass@some-name-rs0-0.some-name-rs0.pitr-27834 == *cfg* ]]
+ replica_set=rs0
+ kubectl_bin exec psmdb-client-9b7b5fdcc-46nql -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-0.some-name-rs0.pitr-27834.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
++ mktemp
+ local LAST_OUT=/tmp/tmp.oNkF1iXmw8
++ mktemp
+ local LAST_ERR=/tmp/tmp.o26wGNOkYn
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl exec psmdb-client-9b7b5fdcc-46nql -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-0.some-name-rs0.pitr-27834.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.oNkF1iXmw8
+ cat /tmp/tmp.o26wGNOkYn
+ rm /tmp/tmp.oNkF1iXmw8 /tmp/tmp.o26wGNOkYn
+ return 0
+ diff /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1352/e2e-tests/pitr/compare/find-2nd.json /tmp/tmp.iKog3JtyCU/find-2nd
+ compare_mongo_cmd find myApp:myPass@some-name-rs0-1.some-name-rs0.pitr-27834 -2nd
+ local command=find
+ local uri=myApp:myPass@some-name-rs0-1.some-name-rs0.pitr-27834
+ local postfix=-2nd
+ local suffix=
+ local database=myApp
+ local collection=test
+ run_mongo 'use myApp\n db.test.find()' myApp:myPass@some-name-rs0-1.some-name-rs0.pitr-27834 mongodb ''
+ local 'command=use myApp\n db.test.find()'
+ local uri=myApp:myPass@some-name-rs0-1.some-name-rs0.pitr-27834
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ egrep -v 'I NETWORK|W NETWORK|F NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match'
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
+ /usr/bin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
++ local LAST_OUT=/tmp/tmp.PeB9YPWNPt
+++ mktemp
++ local LAST_ERR=/tmp/tmp.ObsWktSgwd
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.PeB9YPWNPt
++ cat /tmp/tmp.ObsWktSgwd
++ rm /tmp/tmp.PeB9YPWNPt /tmp/tmp.ObsWktSgwd
++ return 0
+ local client_container=psmdb-client-9b7b5fdcc-46nql
+ local mongo_flag=
+ [[ myApp:myPass@some-name-rs0-1.some-name-rs0.pitr-27834 == *cfg* ]]
+ replica_set=rs0
+ kubectl_bin exec psmdb-client-9b7b5fdcc-46nql -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-1.some-name-rs0.pitr-27834.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
++ mktemp
+ local LAST_OUT=/tmp/tmp.zMaaTRBkaw
++ mktemp
+ local LAST_ERR=/tmp/tmp.QcEcVStDZU
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl exec psmdb-client-9b7b5fdcc-46nql -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-1.some-name-rs0.pitr-27834.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.zMaaTRBkaw
+ cat /tmp/tmp.QcEcVStDZU
+ rm /tmp/tmp.zMaaTRBkaw /tmp/tmp.QcEcVStDZU
+ return 0
+ diff /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1352/e2e-tests/pitr/compare/find-2nd.json /tmp/tmp.iKog3JtyCU/find-2nd
+ compare_mongo_cmd find myApp:myPass@some-name-rs0-2.some-name-rs0.pitr-27834 -2nd
+ local command=find
+ local uri=myApp:myPass@some-name-rs0-2.some-name-rs0.pitr-27834
+ local postfix=-2nd
+ local suffix=
+ local database=myApp
+ local collection=test
+ run_mongo 'use myApp\n db.test.find()' myApp:myPass@some-name-rs0-2.some-name-rs0.pitr-27834 mongodb ''
+ local 'command=use myApp\n db.test.find()'
+ local uri=myApp:myPass@some-name-rs0-2.some-name-rs0.pitr-27834
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ /usr/bin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
+ egrep -v 'I NETWORK|W NETWORK|F NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match'
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.lNR0Sw0eUA
+++ mktemp
++ local LAST_ERR=/tmp/tmp.ETsDeiWTIq
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.lNR0Sw0eUA
++ cat /tmp/tmp.ETsDeiWTIq
++ rm /tmp/tmp.lNR0Sw0eUA /tmp/tmp.ETsDeiWTIq
++ return 0
+ local client_container=psmdb-client-9b7b5fdcc-46nql
+ local mongo_flag=
+ [[ myApp:myPass@some-name-rs0-2.some-name-rs0.pitr-27834 == *cfg* ]]
+ replica_set=rs0
+ kubectl_bin exec psmdb-client-9b7b5fdcc-46nql -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-2.some-name-rs0.pitr-27834.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
++ mktemp
+ local LAST_OUT=/tmp/tmp.VBffUv7Pek
++ mktemp
+ local LAST_ERR=/tmp/tmp.x0Z9VUOT9c
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl exec psmdb-client-9b7b5fdcc-46nql -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-2.some-name-rs0.pitr-27834.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.VBffUv7Pek
+ cat /tmp/tmp.x0Z9VUOT9c
+ rm /tmp/tmp.VBffUv7Pek /tmp/tmp.x0Z9VUOT9c
+ return 0
+ diff /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1352/e2e-tests/pitr/compare/find-2nd.json /tmp/tmp.iKog3JtyCU/find-2nd
+ sleep 2
++ run_mongo 'new Date().toISOString()' myApp:myPass@some-name-rs0.pitr-27834 mongodb '' --quiet
++ local 'command=new Date().toISOString()'
++ local uri=myApp:myPass@some-name-rs0.pitr-27834
++ local driver=mongodb
++ grep -E -v 'I NETWORK|W NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match'
++ local suffix=.svc.cluster.local
++ tr T ' '
++ cut -c1-19
+++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.ixwVA4iYnR
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.spGgitcp44
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in '$(seq 0 2)'
+++ set +e
+++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.ixwVA4iYnR
+++ cat /tmp/tmp.spGgitcp44
+++ rm /tmp/tmp.ixwVA4iYnR /tmp/tmp.spGgitcp44
+++ return 0
++ local client_container=psmdb-client-9b7b5fdcc-46nql
++ local mongo_flag=--quiet
++ [[ myApp:myPass@some-name-rs0.pitr-27834 == *cfg* ]]
++ replica_set=rs0
++ kubectl_bin exec psmdb-client-9b7b5fdcc-46nql -- bash -c 'printf '\''new Date().toISOString()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0.pitr-27834.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 --quiet'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.GxMcYOnGUo
+++ mktemp
++ local LAST_ERR=/tmp/tmp.oSak1D2wEx
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl exec psmdb-client-9b7b5fdcc-46nql -- bash -c 'printf '\''new Date().toISOString()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0.pitr-27834.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 --quiet'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.GxMcYOnGUo
++ cat /tmp/tmp.oSak1D2wEx
++ rm /tmp/tmp.GxMcYOnGUo /tmp/tmp.oSak1D2wEx
++ return 0
+ time_now='2023-10-05 15:25:44'
+ check_recovery backup-minio-0 date '2023-10-05 15:25:44' -2nd '' some-name
+ local backup_name=backup-minio-0
+ local restore_type=date
+ local 'restore_date=2023-10-05 15:25:44'
+ local cmp_postfix=-2nd
+ local backupSource=
+ local cluster_name=some-name
+ desc 'write more data before restore by date'
+ set +o xtrace


-----------------------------------------------------------------------------------
write more data before restore by date
-----------------------------------------------------------------------------------

+ sleep 60
+ run_mongo 'use myApp\n db.test.insert({ x: 100501 })' myApp:myPass@some-name-rs0.pitr-27834
+ local 'command=use myApp\n db.test.insert({ x: 100501 })'
+ local uri=myApp:myPass@some-name-rs0.pitr-27834
+ local driver=mongodb+srv
+ local suffix=.svc.cluster.local
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.7Z7tmH1giB
+++ mktemp
++ local LAST_ERR=/tmp/tmp.zgfgCtMfJY
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.7Z7tmH1giB
++ cat /tmp/tmp.zgfgCtMfJY
++ rm /tmp/tmp.7Z7tmH1giB /tmp/tmp.zgfgCtMfJY
++ return 0
+ local client_container=psmdb-client-9b7b5fdcc-46nql
+ local mongo_flag=
+ [[ myApp:myPass@some-name-rs0.pitr-27834 == *cfg* ]]
+ replica_set=rs0
+ kubectl_bin exec psmdb-client-9b7b5fdcc-46nql -- bash -c 'printf '\''use myApp\n db.test.insert({ x: 100501 })\n'\'' | mongo mongodb+srv://myApp:myPass@some-name-rs0.pitr-27834.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
++ mktemp
+ local LAST_OUT=/tmp/tmp.ZWnasDxiKB
++ mktemp
+ local LAST_ERR=/tmp/tmp.Z5Qf8BBhBA
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl exec psmdb-client-9b7b5fdcc-46nql -- bash -c 'printf '\''use myApp\n db.test.insert({ x: 100501 })\n'\'' | mongo mongodb+srv://myApp:myPass@some-name-rs0.pitr-27834.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.ZWnasDxiKB
Percona Server for MongoDB shell version v4.4.24-23
connecting to: mongodb://some-name-rs0-1.some-name-rs0.pitr-27834.svc.cluster.local:27017,some-name-rs0-0.some-name-rs0.pitr-27834.svc.cluster.local:27017,some-name-rs0-2.some-name-rs0.pitr-27834.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb&replicaSet=rs0&ssl=false
Implicit session: session { "id" : UUID("d7a76077-b8c4-4492-bb2d-f48dadd8771e") }
Percona Server for MongoDB server version: v5.0.20-17
WARNING: shell and server versions do not match
switched to db myApp
WriteResult({ "nInserted" : 1 })
bye
+ cat /tmp/tmp.Z5Qf8BBhBA
+ rm /tmp/tmp.ZWnasDxiKB /tmp/tmp.Z5Qf8BBhBA
+ return 0
+ desc 'waiting for chunks to be uploaded'
+ set +o xtrace


-----------------------------------------------------------------------------------
waiting for chunks to be uploaded
-----------------------------------------------------------------------------------

+ sleep 120
+ desc 'check restore by date'
+ set +o xtrace


-----------------------------------------------------------------------------------
check restore by date
-----------------------------------------------------------------------------------

+ '[' -z '' ']'
+ /usr/bin/sed -e 's/name:/name: restore-backup-minio-0/'
+ cat /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1352/e2e-tests/pitr/conf/restore.yml
+ /usr/bin/sed -e 's/type:/type: date/'
+ /usr/bin/sed -e 's/backupName:/backupName: backup-minio-0/'
+ '[' -z '2023-10-05 15:25:44' ']'
+ /usr/bin/sed -e 's/date:/date: 2023-10-05 15:25:44/'
+ kubectl_bin apply -f -
+ /usr/bin/sed -e /backupSource/,+2d
++ mktemp
+ local LAST_OUT=/tmp/tmp.kV3O0NXzb7
++ mktemp
+ local LAST_ERR=/tmp/tmp.25MgbpUq5h
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl apply -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.kV3O0NXzb7
perconaservermongodbrestore.psmdb.percona.com/restore-backup-minio-0 created
+ cat /tmp/tmp.25MgbpUq5h
+ rm /tmp/tmp.kV3O0NXzb7 /tmp/tmp.25MgbpUq5h
+ return 0
+ wait_restore backup-minio-0 some-name
+ local backup_name=backup-minio-0
+ local cluster_name=some-name
+ local target_state=ready
+ local wait_cluster_consistency=1
+ local wait_time=780
+ set +o xtrace
waiting psmdb-restore/backup-minio-0 to reach ready state.......
+ '[' 1 -eq 1 ']'
+ wait_cluster_consistency some-name
+ local cluster_name=some-name
+ local wait_time=32
+ retry=0
+ sleep 7
+ echo -n 'waiting for cluster readyness'
waiting for cluster readyness++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.H36KFJZraY
+++ mktemp
++ local LAST_ERR=/tmp/tmp.Ff6bKUXJYc
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.H36KFJZraY
++ cat /tmp/tmp.Ff6bKUXJYc
++ rm /tmp/tmp.H36KFJZraY /tmp/tmp.Ff6bKUXJYc
++ return 0
+ [[ ready == \r\e\a\d\y ]]
+ echo

+ set -o xtrace
+ compare_mongo_cmd find myApp:myPass@some-name-rs0-0.some-name-rs0.pitr-27834 -2nd
+ local command=find
+ local uri=myApp:myPass@some-name-rs0-0.some-name-rs0.pitr-27834
+ local postfix=-2nd
+ local suffix=
+ local database=myApp
+ local collection=test
+ /usr/bin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
+ egrep -v 'I NETWORK|W NETWORK|F NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match'
+ run_mongo 'use myApp\n db.test.find()' myApp:myPass@some-name-rs0-0.some-name-rs0.pitr-27834 mongodb ''
+ local 'command=use myApp\n db.test.find()'
+ local uri=myApp:myPass@some-name-rs0-0.some-name-rs0.pitr-27834
+ local driver=mongodb
+ local suffix=.svc.cluster.local
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.ec434eHqQs
+++ mktemp
++ local LAST_ERR=/tmp/tmp.FPU81sEdeN
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.ec434eHqQs
++ cat /tmp/tmp.FPU81sEdeN
++ rm /tmp/tmp.ec434eHqQs /tmp/tmp.FPU81sEdeN
++ return 0
+ local client_container=psmdb-client-9b7b5fdcc-46nql
+ local mongo_flag=
+ [[ myApp:myPass@some-name-rs0-0.some-name-rs0.pitr-27834 == *cfg* ]]
+ replica_set=rs0
+ kubectl_bin exec psmdb-client-9b7b5fdcc-46nql -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-0.some-name-rs0.pitr-27834.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
++ mktemp
+ local LAST_OUT=/tmp/tmp.kAsVpJ3s3w
++ mktemp
+ local LAST_ERR=/tmp/tmp.n4HDv5neKa
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl exec psmdb-client-9b7b5fdcc-46nql -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-0.some-name-rs0.pitr-27834.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.kAsVpJ3s3w
+ cat /tmp/tmp.n4HDv5neKa
+ rm /tmp/tmp.kAsVpJ3s3w /tmp/tmp.n4HDv5neKa
+ return 0
+ diff /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1352/e2e-tests/pitr/compare/find-2nd.json /tmp/tmp.iKog3JtyCU/find-2nd
+ compare_mongo_cmd find myApp:myPass@some-name-rs0-1.some-name-rs0.pitr-27834 -2nd
+ local command=find
+ local uri=myApp:myPass@some-name-rs0-1.some-name-rs0.pitr-27834
+ local postfix=-2nd
+ local suffix=
+ local database=myApp
+ local collection=test
+ run_mongo 'use myApp\n db.test.find()' myApp:myPass@some-name-rs0-1.some-name-rs0.pitr-27834 mongodb ''
+ local 'command=use myApp\n db.test.find()'
+ local uri=myApp:myPass@some-name-rs0-1.some-name-rs0.pitr-27834
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ egrep -v 'I NETWORK|W NETWORK|F NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match'
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.XL5yEYCsYj
+++ mktemp
+ /usr/bin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
++ local LAST_ERR=/tmp/tmp.HL9UmVp95j
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.XL5yEYCsYj
++ cat /tmp/tmp.HL9UmVp95j
++ rm /tmp/tmp.XL5yEYCsYj /tmp/tmp.HL9UmVp95j
++ return 0
+ local client_container=psmdb-client-9b7b5fdcc-46nql
+ local mongo_flag=
+ [[ myApp:myPass@some-name-rs0-1.some-name-rs0.pitr-27834 == *cfg* ]]
+ replica_set=rs0
+ kubectl_bin exec psmdb-client-9b7b5fdcc-46nql -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-1.some-name-rs0.pitr-27834.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
++ mktemp
+ local LAST_OUT=/tmp/tmp.WoWWhFLn8H
++ mktemp
+ local LAST_ERR=/tmp/tmp.aR1htx4CfS
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl exec psmdb-client-9b7b5fdcc-46nql -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-1.some-name-rs0.pitr-27834.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.WoWWhFLn8H
+ cat /tmp/tmp.aR1htx4CfS
+ rm /tmp/tmp.WoWWhFLn8H /tmp/tmp.aR1htx4CfS
+ return 0
+ diff /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1352/e2e-tests/pitr/compare/find-2nd.json /tmp/tmp.iKog3JtyCU/find-2nd
+ compare_mongo_cmd find myApp:myPass@some-name-rs0-2.some-name-rs0.pitr-27834 -2nd
+ local command=find
+ local uri=myApp:myPass@some-name-rs0-2.some-name-rs0.pitr-27834
+ local postfix=-2nd
+ local suffix=
+ local database=myApp
+ local collection=test
+ run_mongo 'use myApp\n db.test.find()' myApp:myPass@some-name-rs0-2.some-name-rs0.pitr-27834 mongodb ''
+ local 'command=use myApp\n db.test.find()'
+ local uri=myApp:myPass@some-name-rs0-2.some-name-rs0.pitr-27834
+ local driver=mongodb
+ local suffix=.svc.cluster.local
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+ egrep -v 'I NETWORK|W NETWORK|F NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match'
+ /usr/bin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.VJGrOdDWy1
+++ mktemp
++ local LAST_ERR=/tmp/tmp.XC8BHbfRmZ
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.VJGrOdDWy1
++ cat /tmp/tmp.XC8BHbfRmZ
++ rm /tmp/tmp.VJGrOdDWy1 /tmp/tmp.XC8BHbfRmZ
++ return 0
+ local client_container=psmdb-client-9b7b5fdcc-46nql
+ local mongo_flag=
+ [[ myApp:myPass@some-name-rs0-2.some-name-rs0.pitr-27834 == *cfg* ]]
+ replica_set=rs0
+ kubectl_bin exec psmdb-client-9b7b5fdcc-46nql -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-2.some-name-rs0.pitr-27834.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
++ mktemp
+ local LAST_OUT=/tmp/tmp.OH5ZiScnpD
++ mktemp
+ local LAST_ERR=/tmp/tmp.CGzT1ozFyn
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl exec psmdb-client-9b7b5fdcc-46nql -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-2.some-name-rs0.pitr-27834.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.OH5ZiScnpD
+ cat /tmp/tmp.CGzT1ozFyn
+ rm /tmp/tmp.OH5ZiScnpD /tmp/tmp.CGzT1ozFyn
+ return 0
+ diff /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1352/e2e-tests/pitr/compare/find-2nd.json /tmp/tmp.iKog3JtyCU/find-2nd
+ run_backup backup-minio 1
+ local name=backup-minio
+ local idx=1
+ desc 'run backup backup-minio-1'
+ set +o xtrace


-----------------------------------------------------------------------------------
run backup backup-minio-1
-----------------------------------------------------------------------------------

+ /usr/bin/sed -e 's/name:/name: backup-minio-1/'
+ cat /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1352/e2e-tests/pitr/conf/backup-minio.yml
+ kubectl_bin apply -f -
++ mktemp
+ local LAST_OUT=/tmp/tmp.JBrStN46mN
++ mktemp
+ local LAST_ERR=/tmp/tmp.cUmNx5VcWU
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl apply -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.JBrStN46mN
perconaservermongodbbackup.psmdb.percona.com/backup-minio-1 created
+ cat /tmp/tmp.cUmNx5VcWU
+ rm /tmp/tmp.JBrStN46mN /tmp/tmp.cUmNx5VcWU
+ return 0
+ wait_backup backup-minio-1
+ local backup_name=backup-minio-1
+ set +o xtrace
backup-minio-1.......
+ check_recovery backup-minio-1 latest '' -3rd '' some-name
+ local backup_name=backup-minio-1
+ local restore_type=latest
+ local restore_date=
+ local cmp_postfix=-3rd
+ local backupSource=
+ local cluster_name=some-name
+ desc 'write more data before restore by latest'
+ set +o xtrace


-----------------------------------------------------------------------------------
write more data before restore by latest
-----------------------------------------------------------------------------------

+ sleep 60
+ run_mongo 'use myApp\n db.test.insert({ x: 100501 })' myApp:myPass@some-name-rs0.pitr-27834
+ local 'command=use myApp\n db.test.insert({ x: 100501 })'
+ local uri=myApp:myPass@some-name-rs0.pitr-27834
+ local driver=mongodb+srv
+ local suffix=.svc.cluster.local
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.8FbfCzLW3d
+++ mktemp
++ local LAST_ERR=/tmp/tmp.ZQDVq9oN2i
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.8FbfCzLW3d
++ cat /tmp/tmp.ZQDVq9oN2i
++ rm /tmp/tmp.8FbfCzLW3d /tmp/tmp.ZQDVq9oN2i
++ return 0
+ local client_container=psmdb-client-9b7b5fdcc-46nql
+ local mongo_flag=
+ [[ myApp:myPass@some-name-rs0.pitr-27834 == *cfg* ]]
+ replica_set=rs0
+ kubectl_bin exec psmdb-client-9b7b5fdcc-46nql -- bash -c 'printf '\''use myApp\n db.test.insert({ x: 100501 })\n'\'' | mongo mongodb+srv://myApp:myPass@some-name-rs0.pitr-27834.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
++ mktemp
+ local LAST_OUT=/tmp/tmp.xLU2AhC6vK
++ mktemp
+ local LAST_ERR=/tmp/tmp.xaGEgsVlBT
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl exec psmdb-client-9b7b5fdcc-46nql -- bash -c 'printf '\''use myApp\n db.test.insert({ x: 100501 })\n'\'' | mongo mongodb+srv://myApp:myPass@some-name-rs0.pitr-27834.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.xLU2AhC6vK
Percona Server for MongoDB shell version v4.4.24-23
connecting to: mongodb://some-name-rs0-1.some-name-rs0.pitr-27834.svc.cluster.local:27017,some-name-rs0-0.some-name-rs0.pitr-27834.svc.cluster.local:27017,some-name-rs0-2.some-name-rs0.pitr-27834.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb&replicaSet=rs0&ssl=false
Implicit session: session { "id" : UUID("eab449bb-7702-4e18-b89e-76b33b5bc8fe") }
Percona Server for MongoDB server version: v5.0.20-17
WARNING: shell and server versions do not match
switched to db myApp
WriteResult({ "nInserted" : 1 })
bye
+ cat /tmp/tmp.xaGEgsVlBT
+ rm /tmp/tmp.xLU2AhC6vK /tmp/tmp.xaGEgsVlBT
+ return 0
+ desc 'waiting for chunks to be uploaded'
+ set +o xtrace


-----------------------------------------------------------------------------------
waiting for chunks to be uploaded
-----------------------------------------------------------------------------------

+ sleep 120
+ desc 'check restore by latest'
+ set +o xtrace


-----------------------------------------------------------------------------------
check restore by latest
-----------------------------------------------------------------------------------

+ '[' -z '' ']'
+ /usr/bin/sed -e 's/name:/name: restore-backup-minio-1/'
+ /usr/bin/sed -e 's/backupName:/backupName: backup-minio-1/'
+ '[' -z '' ']'
+ /usr/bin/sed -e /date:/d
+ cat /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1352/e2e-tests/pitr/conf/restore.yml
+ /usr/bin/sed -e /backupSource/,+2d
+ kubectl_bin apply -f -
++ mktemp
+ local LAST_OUT=/tmp/tmp.i7MgobyQ7N
++ mktemp
+ /usr/bin/sed -e 's/type:/type: latest/'
+ local LAST_ERR=/tmp/tmp.CI5mypbTFC
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl apply -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.i7MgobyQ7N
perconaservermongodbrestore.psmdb.percona.com/restore-backup-minio-1 created
+ cat /tmp/tmp.CI5mypbTFC
+ rm /tmp/tmp.i7MgobyQ7N /tmp/tmp.CI5mypbTFC
+ return 0
+ wait_restore backup-minio-1 some-name
+ local backup_name=backup-minio-1
+ local cluster_name=some-name
+ local target_state=ready
+ local wait_cluster_consistency=1
+ local wait_time=780
+ set +o xtrace
waiting psmdb-restore/backup-minio-1 to reach ready state......
+ '[' 1 -eq 1 ']'
+ wait_cluster_consistency some-name
+ local cluster_name=some-name
+ local wait_time=32
+ retry=0
+ sleep 7
+ echo -n 'waiting for cluster readyness'
waiting for cluster readyness++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.fzB5UV0Qhv
+++ mktemp
++ local LAST_ERR=/tmp/tmp.5qvPgh7jtM
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.fzB5UV0Qhv
++ cat /tmp/tmp.5qvPgh7jtM
++ rm /tmp/tmp.fzB5UV0Qhv /tmp/tmp.5qvPgh7jtM
++ return 0
+ [[ ready == \r\e\a\d\y ]]
+ echo

+ set -o xtrace
+ compare_mongo_cmd find myApp:myPass@some-name-rs0-0.some-name-rs0.pitr-27834 -3rd
+ local command=find
+ local uri=myApp:myPass@some-name-rs0-0.some-name-rs0.pitr-27834
+ local postfix=-3rd
+ local suffix=
+ local database=myApp
+ local collection=test
+ /usr/bin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
+ run_mongo 'use myApp\n db.test.find()' myApp:myPass@some-name-rs0-0.some-name-rs0.pitr-27834 mongodb ''
+ egrep -v 'I NETWORK|W NETWORK|F NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match'
+ local 'command=use myApp\n db.test.find()'
+ local uri=myApp:myPass@some-name-rs0-0.some-name-rs0.pitr-27834
+ local driver=mongodb
+ local suffix=.svc.cluster.local
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.XaMjiqlC71
+++ mktemp
++ local LAST_ERR=/tmp/tmp.s6gwjOfC7w
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.XaMjiqlC71
++ cat /tmp/tmp.s6gwjOfC7w
++ rm /tmp/tmp.XaMjiqlC71 /tmp/tmp.s6gwjOfC7w
++ return 0
+ local client_container=psmdb-client-9b7b5fdcc-46nql
+ local mongo_flag=
+ [[ myApp:myPass@some-name-rs0-0.some-name-rs0.pitr-27834 == *cfg* ]]
+ replica_set=rs0
+ kubectl_bin exec psmdb-client-9b7b5fdcc-46nql -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-0.some-name-rs0.pitr-27834.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
++ mktemp
+ local LAST_OUT=/tmp/tmp.XiNaSZk5wN
++ mktemp
+ local LAST_ERR=/tmp/tmp.Pchz6tAJb3
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl exec psmdb-client-9b7b5fdcc-46nql -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-0.some-name-rs0.pitr-27834.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.XiNaSZk5wN
+ cat /tmp/tmp.Pchz6tAJb3
+ rm /tmp/tmp.XiNaSZk5wN /tmp/tmp.Pchz6tAJb3
+ return 0
+ diff /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1352/e2e-tests/pitr/compare/find-3rd.json /tmp/tmp.iKog3JtyCU/find-3rd
+ compare_mongo_cmd find myApp:myPass@some-name-rs0-1.some-name-rs0.pitr-27834 -3rd
+ local command=find
+ local uri=myApp:myPass@some-name-rs0-1.some-name-rs0.pitr-27834
+ local postfix=-3rd
+ local suffix=
+ local database=myApp
+ local collection=test
+ run_mongo 'use myApp\n db.test.find()' myApp:myPass@some-name-rs0-1.some-name-rs0.pitr-27834 mongodb ''
+ local 'command=use myApp\n db.test.find()'
+ local uri=myApp:myPass@some-name-rs0-1.some-name-rs0.pitr-27834
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ /usr/bin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+ egrep -v 'I NETWORK|W NETWORK|F NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.Mkeszfidmf
+++ mktemp
++ local LAST_ERR=/tmp/tmp.WTDz54SEqe
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.Mkeszfidmf
++ cat /tmp/tmp.WTDz54SEqe
++ rm /tmp/tmp.Mkeszfidmf /tmp/tmp.WTDz54SEqe
++ return 0
+ local client_container=psmdb-client-9b7b5fdcc-46nql
+ local mongo_flag=
+ [[ myApp:myPass@some-name-rs0-1.some-name-rs0.pitr-27834 == *cfg* ]]
+ replica_set=rs0
+ kubectl_bin exec psmdb-client-9b7b5fdcc-46nql -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-1.some-name-rs0.pitr-27834.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
++ mktemp
+ local LAST_OUT=/tmp/tmp.VkvwkMyaXs
++ mktemp
+ local LAST_ERR=/tmp/tmp.XGgZyaISZh
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl exec psmdb-client-9b7b5fdcc-46nql -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-1.some-name-rs0.pitr-27834.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.VkvwkMyaXs
+ cat /tmp/tmp.XGgZyaISZh
+ rm /tmp/tmp.VkvwkMyaXs /tmp/tmp.XGgZyaISZh
+ return 0
+ diff /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1352/e2e-tests/pitr/compare/find-3rd.json /tmp/tmp.iKog3JtyCU/find-3rd
+ compare_mongo_cmd find myApp:myPass@some-name-rs0-2.some-name-rs0.pitr-27834 -3rd
+ local command=find
+ local uri=myApp:myPass@some-name-rs0-2.some-name-rs0.pitr-27834
+ local postfix=-3rd
+ local suffix=
+ local database=myApp
+ local collection=test
+ run_mongo 'use myApp\n db.test.find()' myApp:myPass@some-name-rs0-2.some-name-rs0.pitr-27834 mongodb ''
+ local 'command=use myApp\n db.test.find()'
+ local uri=myApp:myPass@some-name-rs0-2.some-name-rs0.pitr-27834
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ /usr/bin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
+ egrep -v 'I NETWORK|W NETWORK|F NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match'
++ local LAST_OUT=/tmp/tmp.3lGsEXwDrC
+++ mktemp
++ local LAST_ERR=/tmp/tmp.iZs78JC1vX
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.3lGsEXwDrC
++ cat /tmp/tmp.iZs78JC1vX
++ rm /tmp/tmp.3lGsEXwDrC /tmp/tmp.iZs78JC1vX
++ return 0
+ local client_container=psmdb-client-9b7b5fdcc-46nql
+ local mongo_flag=
+ [[ myApp:myPass@some-name-rs0-2.some-name-rs0.pitr-27834 == *cfg* ]]
+ replica_set=rs0
+ kubectl_bin exec psmdb-client-9b7b5fdcc-46nql -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-2.some-name-rs0.pitr-27834.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
++ mktemp
+ local LAST_OUT=/tmp/tmp.rVY29vql86
++ mktemp
+ local LAST_ERR=/tmp/tmp.e30ILbLzMI
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl exec psmdb-client-9b7b5fdcc-46nql -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-2.some-name-rs0.pitr-27834.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.rVY29vql86
+ cat /tmp/tmp.e30ILbLzMI
+ rm /tmp/tmp.rVY29vql86 /tmp/tmp.e30ILbLzMI
+ return 0
+ diff /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1352/e2e-tests/pitr/compare/find-3rd.json /tmp/tmp.iKog3JtyCU/find-3rd
+ desc 'restore pitr using backupSource'
+ set +o xtrace


-----------------------------------------------------------------------------------
restore pitr using backupSource
-----------------------------------------------------------------------------------

+ reset_collection
+ desc 'reset data'
+ set +o xtrace


-----------------------------------------------------------------------------------
reset data
-----------------------------------------------------------------------------------

+ run_mongo 'use myApp\n db.test.remove({})' myApp:myPass@some-name-rs0.pitr-27834
+ local 'command=use myApp\n db.test.remove({})'
+ local uri=myApp:myPass@some-name-rs0.pitr-27834
+ local driver=mongodb+srv
+ local suffix=.svc.cluster.local
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.RR86waGE8o
+++ mktemp
++ local LAST_ERR=/tmp/tmp.Th3r1EXb4I
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.RR86waGE8o
++ cat /tmp/tmp.Th3r1EXb4I
++ rm /tmp/tmp.RR86waGE8o /tmp/tmp.Th3r1EXb4I
++ return 0
+ local client_container=psmdb-client-9b7b5fdcc-46nql
+ local mongo_flag=
+ [[ myApp:myPass@some-name-rs0.pitr-27834 == *cfg* ]]
+ replica_set=rs0
+ kubectl_bin exec psmdb-client-9b7b5fdcc-46nql -- bash -c 'printf '\''use myApp\n db.test.remove({})\n'\'' | mongo mongodb+srv://myApp:myPass@some-name-rs0.pitr-27834.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
++ mktemp
+ local LAST_OUT=/tmp/tmp.ByuTPc03DF
++ mktemp
+ local LAST_ERR=/tmp/tmp.19YzgDPkYz
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl exec psmdb-client-9b7b5fdcc-46nql -- bash -c 'printf '\''use myApp\n db.test.remove({})\n'\'' | mongo mongodb+srv://myApp:myPass@some-name-rs0.pitr-27834.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.ByuTPc03DF
Percona Server for MongoDB shell version v4.4.24-23
connecting to: mongodb://some-name-rs0-1.some-name-rs0.pitr-27834.svc.cluster.local:27017,some-name-rs0-0.some-name-rs0.pitr-27834.svc.cluster.local:27017,some-name-rs0-2.some-name-rs0.pitr-27834.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb&replicaSet=rs0&ssl=false
Implicit session: session { "id" : UUID("1d99013c-ab38-443e-8557-8d46adb3b0dd") }
Percona Server for MongoDB server version: v5.0.20-17
WARNING: shell and server versions do not match
switched to db myApp
WriteResult({ "nRemoved" : 3 })
bye
+ cat /tmp/tmp.19YzgDPkYz
+ rm /tmp/tmp.ByuTPc03DF /tmp/tmp.19YzgDPkYz
+ return 0
+ sleep 2
+ write_document
+ local cmp_postfix=
+ desc 'write initial data, read from all'
+ set +o xtrace


-----------------------------------------------------------------------------------
write initial data, read from all
-----------------------------------------------------------------------------------

+ run_mongo 'use myApp\n db.test.insert({ x: 100500 })' myApp:myPass@some-name-rs0.pitr-27834
+ local 'command=use myApp\n db.test.insert({ x: 100500 })'
+ local uri=myApp:myPass@some-name-rs0.pitr-27834
+ local driver=mongodb+srv
+ local suffix=.svc.cluster.local
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.xIvBvBV64w
+++ mktemp
++ local LAST_ERR=/tmp/tmp.SqY5KExTvr
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.xIvBvBV64w
++ cat /tmp/tmp.SqY5KExTvr
++ rm /tmp/tmp.xIvBvBV64w /tmp/tmp.SqY5KExTvr
++ return 0
+ local client_container=psmdb-client-9b7b5fdcc-46nql
+ local mongo_flag=
+ [[ myApp:myPass@some-name-rs0.pitr-27834 == *cfg* ]]
+ replica_set=rs0
+ kubectl_bin exec psmdb-client-9b7b5fdcc-46nql -- bash -c 'printf '\''use myApp\n db.test.insert({ x: 100500 })\n'\'' | mongo mongodb+srv://myApp:myPass@some-name-rs0.pitr-27834.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
++ mktemp
+ local LAST_OUT=/tmp/tmp.zTChntwcZZ
++ mktemp
+ local LAST_ERR=/tmp/tmp.VMoufURTDP
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl exec psmdb-client-9b7b5fdcc-46nql -- bash -c 'printf '\''use myApp\n db.test.insert({ x: 100500 })\n'\'' | mongo mongodb+srv://myApp:myPass@some-name-rs0.pitr-27834.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.zTChntwcZZ
Percona Server for MongoDB shell version v4.4.24-23
connecting to: mongodb://some-name-rs0-2.some-name-rs0.pitr-27834.svc.cluster.local:27017,some-name-rs0-1.some-name-rs0.pitr-27834.svc.cluster.local:27017,some-name-rs0-0.some-name-rs0.pitr-27834.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb&replicaSet=rs0&ssl=false
Implicit session: session { "id" : UUID("c9d0f9f1-78fb-40e3-a0b6-a72fc04d1936") }
Percona Server for MongoDB server version: v5.0.20-17
WARNING: shell and server versions do not match
switched to db myApp
WriteResult({ "nInserted" : 1 })
bye
+ cat /tmp/tmp.VMoufURTDP
+ rm /tmp/tmp.zTChntwcZZ /tmp/tmp.VMoufURTDP
+ return 0
+ minikube_sleep
+ sleep_time=10
+ [[ '' == 1 ]]
+ compare_mongo_cmd find myApp:myPass@some-name-rs0-0.some-name-rs0.pitr-27834 ''
+ local command=find
+ local uri=myApp:myPass@some-name-rs0-0.some-name-rs0.pitr-27834
+ local postfix=
+ local suffix=
+ local database=myApp
+ local collection=test
+ run_mongo 'use myApp\n db.test.find()' myApp:myPass@some-name-rs0-0.some-name-rs0.pitr-27834 mongodb ''
+ local 'command=use myApp\n db.test.find()'
+ local uri=myApp:myPass@some-name-rs0-0.some-name-rs0.pitr-27834
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ egrep -v 'I NETWORK|W NETWORK|F NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match'
+ /usr/bin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.LPDhABRnUN
+++ mktemp
++ local LAST_ERR=/tmp/tmp.bOJhd83JWd
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.LPDhABRnUN
++ cat /tmp/tmp.bOJhd83JWd
++ rm /tmp/tmp.LPDhABRnUN /tmp/tmp.bOJhd83JWd
++ return 0
+ local client_container=psmdb-client-9b7b5fdcc-46nql
+ local mongo_flag=
+ [[ myApp:myPass@some-name-rs0-0.some-name-rs0.pitr-27834 == *cfg* ]]
+ replica_set=rs0
+ kubectl_bin exec psmdb-client-9b7b5fdcc-46nql -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-0.some-name-rs0.pitr-27834.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
++ mktemp
+ local LAST_OUT=/tmp/tmp.f7ATrcPYsA
++ mktemp
+ local LAST_ERR=/tmp/tmp.OMHy5ZOffN
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl exec psmdb-client-9b7b5fdcc-46nql -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-0.some-name-rs0.pitr-27834.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.f7ATrcPYsA
+ cat /tmp/tmp.OMHy5ZOffN
+ rm /tmp/tmp.f7ATrcPYsA /tmp/tmp.OMHy5ZOffN
+ return 0
+ diff /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1352/e2e-tests/pitr/compare/find.json /tmp/tmp.iKog3JtyCU/find
+ compare_mongo_cmd find myApp:myPass@some-name-rs0-1.some-name-rs0.pitr-27834 ''
+ local command=find
+ local uri=myApp:myPass@some-name-rs0-1.some-name-rs0.pitr-27834
+ local postfix=
+ local suffix=
+ local database=myApp
+ local collection=test
+ run_mongo 'use myApp\n db.test.find()' myApp:myPass@some-name-rs0-1.some-name-rs0.pitr-27834 mongodb ''
+ local 'command=use myApp\n db.test.find()'
+ local uri=myApp:myPass@some-name-rs0-1.some-name-rs0.pitr-27834
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ /usr/bin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+ egrep -v 'I NETWORK|W NETWORK|F NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.FcOZCbNVPE
+++ mktemp
++ local LAST_ERR=/tmp/tmp.pm8SJnpArT
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.FcOZCbNVPE
++ cat /tmp/tmp.pm8SJnpArT
++ rm /tmp/tmp.FcOZCbNVPE /tmp/tmp.pm8SJnpArT
++ return 0
+ local client_container=psmdb-client-9b7b5fdcc-46nql
+ local mongo_flag=
+ [[ myApp:myPass@some-name-rs0-1.some-name-rs0.pitr-27834 == *cfg* ]]
+ replica_set=rs0
+ kubectl_bin exec psmdb-client-9b7b5fdcc-46nql -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-1.some-name-rs0.pitr-27834.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
++ mktemp
+ local LAST_OUT=/tmp/tmp.gYbxbiRu89
++ mktemp
+ local LAST_ERR=/tmp/tmp.MUkjbgXpLe
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl exec psmdb-client-9b7b5fdcc-46nql -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-1.some-name-rs0.pitr-27834.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.gYbxbiRu89
+ cat /tmp/tmp.MUkjbgXpLe
+ rm /tmp/tmp.gYbxbiRu89 /tmp/tmp.MUkjbgXpLe
+ return 0
+ diff /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1352/e2e-tests/pitr/compare/find.json /tmp/tmp.iKog3JtyCU/find
+ compare_mongo_cmd find myApp:myPass@some-name-rs0-2.some-name-rs0.pitr-27834 ''
+ local command=find
+ local uri=myApp:myPass@some-name-rs0-2.some-name-rs0.pitr-27834
+ local postfix=
+ local suffix=
+ local database=myApp
+ local collection=test
+ egrep -v 'I NETWORK|W NETWORK|F NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match'
+ /usr/bin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
+ run_mongo 'use myApp\n db.test.find()' myApp:myPass@some-name-rs0-2.some-name-rs0.pitr-27834 mongodb ''
+ local 'command=use myApp\n db.test.find()'
+ local uri=myApp:myPass@some-name-rs0-2.some-name-rs0.pitr-27834
+ local driver=mongodb
+ local suffix=.svc.cluster.local
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.qUhUVd8ft0
+++ mktemp
++ local LAST_ERR=/tmp/tmp.J8APFUmNjZ
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.qUhUVd8ft0
++ cat /tmp/tmp.J8APFUmNjZ
++ rm /tmp/tmp.qUhUVd8ft0 /tmp/tmp.J8APFUmNjZ
++ return 0
+ local client_container=psmdb-client-9b7b5fdcc-46nql
+ local mongo_flag=
+ [[ myApp:myPass@some-name-rs0-2.some-name-rs0.pitr-27834 == *cfg* ]]
+ replica_set=rs0
+ kubectl_bin exec psmdb-client-9b7b5fdcc-46nql -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-2.some-name-rs0.pitr-27834.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
++ mktemp
+ local LAST_OUT=/tmp/tmp.uxKe8e1K0D
++ mktemp
+ local LAST_ERR=/tmp/tmp.BefcMj7qEc
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl exec psmdb-client-9b7b5fdcc-46nql -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-2.some-name-rs0.pitr-27834.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.uxKe8e1K0D
+ cat /tmp/tmp.BefcMj7qEc
+ rm /tmp/tmp.uxKe8e1K0D /tmp/tmp.BefcMj7qEc
+ return 0
+ diff /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1352/e2e-tests/pitr/compare/find.json /tmp/tmp.iKog3JtyCU/find
+ run_backup backup-minio 2
+ local name=backup-minio
+ local idx=2
+ desc 'run backup backup-minio-2'
+ set +o xtrace


-----------------------------------------------------------------------------------
run backup backup-minio-2
-----------------------------------------------------------------------------------

+ cat /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1352/e2e-tests/pitr/conf/backup-minio.yml
+ /usr/bin/sed -e 's/name:/name: backup-minio-2/'
+ kubectl_bin apply -f -
++ mktemp
+ local LAST_OUT=/tmp/tmp.Nd3EaHwRjH
++ mktemp
+ local LAST_ERR=/tmp/tmp.I4o0hLUvTb
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl apply -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.Nd3EaHwRjH
perconaservermongodbbackup.psmdb.percona.com/backup-minio-2 created
+ cat /tmp/tmp.I4o0hLUvTb
+ rm /tmp/tmp.Nd3EaHwRjH /tmp/tmp.I4o0hLUvTb
+ return 0
+ wait_backup backup-minio-2
+ local backup_name=backup-minio-2
+ set +o xtrace
backup-minio-2..........
++ run_mongo 'new Date().toISOString()' myApp:myPass@some-name-rs0.pitr-27834 mongodb '' --quiet
++ local 'command=new Date().toISOString()'
++ local uri=myApp:myPass@some-name-rs0.pitr-27834
++ local driver=mongodb
++ local suffix=.svc.cluster.local
++ grep -E -v 'I NETWORK|W NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match'
++ cut -c1-19
+++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++++ mktemp
++ tr T ' '
+++ local LAST_OUT=/tmp/tmp.IkzHGwATR3
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.RbHWGJxUmV
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in '$(seq 0 2)'
+++ set +e
+++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.IkzHGwATR3
+++ cat /tmp/tmp.RbHWGJxUmV
+++ rm /tmp/tmp.IkzHGwATR3 /tmp/tmp.RbHWGJxUmV
+++ return 0
++ local client_container=psmdb-client-9b7b5fdcc-46nql
++ local mongo_flag=--quiet
++ [[ myApp:myPass@some-name-rs0.pitr-27834 == *cfg* ]]
++ replica_set=rs0
++ kubectl_bin exec psmdb-client-9b7b5fdcc-46nql -- bash -c 'printf '\''new Date().toISOString()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0.pitr-27834.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 --quiet'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.3yuKerq1qO
+++ mktemp
++ local LAST_ERR=/tmp/tmp.NUoCUgWQcD
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl exec psmdb-client-9b7b5fdcc-46nql -- bash -c 'printf '\''new Date().toISOString()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0.pitr-27834.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 --quiet'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.3yuKerq1qO
++ cat /tmp/tmp.NUoCUgWQcD
++ rm /tmp/tmp.3yuKerq1qO /tmp/tmp.NUoCUgWQcD
++ return 0
+ time_now='2023-10-05 15:35:58'
+ check_recovery backup-minio-2 date '2023-10-05 15:35:58' '' backupSource some-name
+ local backup_name=backup-minio-2
+ local restore_type=date
+ local 'restore_date=2023-10-05 15:35:58'
+ local cmp_postfix=
+ local backupSource=backupSource
+ local cluster_name=some-name
+ desc 'write more data before restore by date'
+ set +o xtrace


-----------------------------------------------------------------------------------
write more data before restore by date
-----------------------------------------------------------------------------------

+ sleep 60
+ run_mongo 'use myApp\n db.test.insert({ x: 100501 })' myApp:myPass@some-name-rs0.pitr-27834
+ local 'command=use myApp\n db.test.insert({ x: 100501 })'
+ local uri=myApp:myPass@some-name-rs0.pitr-27834
+ local driver=mongodb+srv
+ local suffix=.svc.cluster.local
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.rQh3KOqgOo
+++ mktemp
++ local LAST_ERR=/tmp/tmp.DRSxcYfvJz
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.rQh3KOqgOo
++ cat /tmp/tmp.DRSxcYfvJz
++ rm /tmp/tmp.rQh3KOqgOo /tmp/tmp.DRSxcYfvJz
++ return 0
+ local client_container=psmdb-client-9b7b5fdcc-46nql
+ local mongo_flag=
+ [[ myApp:myPass@some-name-rs0.pitr-27834 == *cfg* ]]
+ replica_set=rs0
+ kubectl_bin exec psmdb-client-9b7b5fdcc-46nql -- bash -c 'printf '\''use myApp\n db.test.insert({ x: 100501 })\n'\'' | mongo mongodb+srv://myApp:myPass@some-name-rs0.pitr-27834.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
++ mktemp
+ local LAST_OUT=/tmp/tmp.G8tW8LxnAd
++ mktemp
+ local LAST_ERR=/tmp/tmp.pX2pPN1YT9
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl exec psmdb-client-9b7b5fdcc-46nql -- bash -c 'printf '\''use myApp\n db.test.insert({ x: 100501 })\n'\'' | mongo mongodb+srv://myApp:myPass@some-name-rs0.pitr-27834.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.G8tW8LxnAd
Percona Server for MongoDB shell version v4.4.24-23
connecting to: mongodb://some-name-rs0-0.some-name-rs0.pitr-27834.svc.cluster.local:27017,some-name-rs0-2.some-name-rs0.pitr-27834.svc.cluster.local:27017,some-name-rs0-1.some-name-rs0.pitr-27834.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb&replicaSet=rs0&ssl=false
Implicit session: session { "id" : UUID("0c1e3c76-8ea0-46a8-a462-24011480eb70") }
Percona Server for MongoDB server version: v5.0.20-17
WARNING: shell and server versions do not match
switched to db myApp
WriteResult({ "nInserted" : 1 })
bye
+ cat /tmp/tmp.pX2pPN1YT9
+ rm /tmp/tmp.G8tW8LxnAd /tmp/tmp.pX2pPN1YT9
+ return 0
+ desc 'waiting for chunks to be uploaded'
+ set +o xtrace


-----------------------------------------------------------------------------------
waiting for chunks to be uploaded
-----------------------------------------------------------------------------------

+ sleep 120
+ desc 'check restore by date'
+ set +o xtrace


-----------------------------------------------------------------------------------
check restore by date
-----------------------------------------------------------------------------------

+ '[' -z backupSource ']'
++ get_backup_dest backup-minio-2
++ local backup_name=backup-minio-2
++ kubectl_bin get psmdb-backup backup-minio-2 -o 'jsonpath={.status.destination}'
++ sed -e 's/.json$//'
++ sed 's|s3://||'
++ sed 's|azure://||'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.dFthIF59IL
+++ mktemp
++ local LAST_ERR=/tmp/tmp.9mzhl6QLD9
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb-backup backup-minio-2 -o 'jsonpath={.status.destination}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.dFthIF59IL
++ cat /tmp/tmp.9mzhl6QLD9
++ rm /tmp/tmp.dFthIF59IL /tmp/tmp.9mzhl6QLD9
++ return 0
+ backup_dest=operator-testing/2023-10-05T15:35:37Z
+ cat /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1352/e2e-tests/pitr/conf/restore.yml
+ /usr/bin/sed -e /backupName/d
+ /usr/bin/sed -e 's/type:/type: date/'
+ '[' -z '2023-10-05 15:35:58' ']'
+ /usr/bin/sed -e 's/date:/date: 2023-10-05 15:35:58/'
+ kubectl_bin apply -f -
+ /usr/bin/sed -e 's/name:/name: restore-backup-minio-2/'
++ mktemp
+ /usr/bin/sed -e 's|BACKUP-NAME|operator-testing/2023-10-05T15:35:37Z|'
+ local LAST_OUT=/tmp/tmp.1ruReGkYEP
++ mktemp
+ local LAST_ERR=/tmp/tmp.OJFdzFwtps
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl apply -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.1ruReGkYEP
perconaservermongodbrestore.psmdb.percona.com/restore-backup-minio-2 created
+ cat /tmp/tmp.OJFdzFwtps
+ rm /tmp/tmp.1ruReGkYEP /tmp/tmp.OJFdzFwtps
+ return 0
+ wait_restore backup-minio-2 some-name
+ local backup_name=backup-minio-2
+ local cluster_name=some-name
+ local target_state=ready
+ local wait_cluster_consistency=1
+ local wait_time=780
+ set +o xtrace
waiting psmdb-restore/backup-minio-2 to reach ready state.......
+ '[' 1 -eq 1 ']'
+ wait_cluster_consistency some-name
+ local cluster_name=some-name
+ local wait_time=32
+ retry=0
+ sleep 7
+ echo -n 'waiting for cluster readyness'
waiting for cluster readyness++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.26nNqeaHzi
+++ mktemp
++ local LAST_ERR=/tmp/tmp.M85mQkEziK
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.26nNqeaHzi
++ cat /tmp/tmp.M85mQkEziK
++ rm /tmp/tmp.26nNqeaHzi /tmp/tmp.M85mQkEziK
++ return 0
+ [[ ready == \r\e\a\d\y ]]
+ echo

+ set -o xtrace
+ compare_mongo_cmd find myApp:myPass@some-name-rs0-0.some-name-rs0.pitr-27834 ''
+ local command=find
+ local uri=myApp:myPass@some-name-rs0-0.some-name-rs0.pitr-27834
+ local postfix=
+ local suffix=
+ local database=myApp
+ local collection=test
+ run_mongo 'use myApp\n db.test.find()' myApp:myPass@some-name-rs0-0.some-name-rs0.pitr-27834 mongodb ''
+ local 'command=use myApp\n db.test.find()'
+ local uri=myApp:myPass@some-name-rs0-0.some-name-rs0.pitr-27834
+ local driver=mongodb
+ local suffix=.svc.cluster.local
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
+ /usr/bin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
+ egrep -v 'I NETWORK|W NETWORK|F NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match'
++ local LAST_OUT=/tmp/tmp.L3UvoAS7f3
+++ mktemp
++ local LAST_ERR=/tmp/tmp.NdNjdzKkBD
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.L3UvoAS7f3
++ cat /tmp/tmp.NdNjdzKkBD
++ rm /tmp/tmp.L3UvoAS7f3 /tmp/tmp.NdNjdzKkBD
++ return 0
+ local client_container=psmdb-client-9b7b5fdcc-46nql
+ local mongo_flag=
+ [[ myApp:myPass@some-name-rs0-0.some-name-rs0.pitr-27834 == *cfg* ]]
+ replica_set=rs0
+ kubectl_bin exec psmdb-client-9b7b5fdcc-46nql -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-0.some-name-rs0.pitr-27834.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
++ mktemp
+ local LAST_OUT=/tmp/tmp.plAcwwkD0R
++ mktemp
+ local LAST_ERR=/tmp/tmp.05SrzhRFjV
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl exec psmdb-client-9b7b5fdcc-46nql -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-0.some-name-rs0.pitr-27834.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.plAcwwkD0R
+ cat /tmp/tmp.05SrzhRFjV
+ rm /tmp/tmp.plAcwwkD0R /tmp/tmp.05SrzhRFjV
+ return 0
+ diff /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1352/e2e-tests/pitr/compare/find.json /tmp/tmp.iKog3JtyCU/find
+ compare_mongo_cmd find myApp:myPass@some-name-rs0-1.some-name-rs0.pitr-27834 ''
+ local command=find
+ local uri=myApp:myPass@some-name-rs0-1.some-name-rs0.pitr-27834
+ local postfix=
+ local suffix=
+ local database=myApp
+ local collection=test
+ run_mongo 'use myApp\n db.test.find()' myApp:myPass@some-name-rs0-1.some-name-rs0.pitr-27834 mongodb ''
+ local 'command=use myApp\n db.test.find()'
+ local uri=myApp:myPass@some-name-rs0-1.some-name-rs0.pitr-27834
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ /usr/bin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
+ egrep -v 'I NETWORK|W NETWORK|F NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match'
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.wcIlOvhB5w
+++ mktemp
++ local LAST_ERR=/tmp/tmp.hmsqbPTXEY
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.wcIlOvhB5w
++ cat /tmp/tmp.hmsqbPTXEY
++ rm /tmp/tmp.wcIlOvhB5w /tmp/tmp.hmsqbPTXEY
++ return 0
+ local client_container=psmdb-client-9b7b5fdcc-46nql
+ local mongo_flag=
+ [[ myApp:myPass@some-name-rs0-1.some-name-rs0.pitr-27834 == *cfg* ]]
+ replica_set=rs0
+ kubectl_bin exec psmdb-client-9b7b5fdcc-46nql -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-1.some-name-rs0.pitr-27834.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
++ mktemp
+ local LAST_OUT=/tmp/tmp.3BqlzGYIrY
++ mktemp
+ local LAST_ERR=/tmp/tmp.qPM5kCiBAI
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl exec psmdb-client-9b7b5fdcc-46nql -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-1.some-name-rs0.pitr-27834.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.3BqlzGYIrY
+ cat /tmp/tmp.qPM5kCiBAI
+ rm /tmp/tmp.3BqlzGYIrY /tmp/tmp.qPM5kCiBAI
+ return 0
+ diff /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1352/e2e-tests/pitr/compare/find.json /tmp/tmp.iKog3JtyCU/find
+ compare_mongo_cmd find myApp:myPass@some-name-rs0-2.some-name-rs0.pitr-27834 ''
+ local command=find
+ local uri=myApp:myPass@some-name-rs0-2.some-name-rs0.pitr-27834
+ local postfix=
+ local suffix=
+ local database=myApp
+ local collection=test
+ egrep -v 'I NETWORK|W NETWORK|F NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match'
+ run_mongo 'use myApp\n db.test.find()' myApp:myPass@some-name-rs0-2.some-name-rs0.pitr-27834 mongodb ''
+ local 'command=use myApp\n db.test.find()'
+ local uri=myApp:myPass@some-name-rs0-2.some-name-rs0.pitr-27834
+ local driver=mongodb
+ local suffix=.svc.cluster.local
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
+ /usr/bin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
++ local LAST_OUT=/tmp/tmp.SnzjKRc3xo
+++ mktemp
++ local LAST_ERR=/tmp/tmp.ScaGxld62l
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.SnzjKRc3xo
++ cat /tmp/tmp.ScaGxld62l
++ rm /tmp/tmp.SnzjKRc3xo /tmp/tmp.ScaGxld62l
++ return 0
+ local client_container=psmdb-client-9b7b5fdcc-46nql
+ local mongo_flag=
+ [[ myApp:myPass@some-name-rs0-2.some-name-rs0.pitr-27834 == *cfg* ]]
+ replica_set=rs0
+ kubectl_bin exec psmdb-client-9b7b5fdcc-46nql -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-2.some-name-rs0.pitr-27834.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
++ mktemp
+ local LAST_OUT=/tmp/tmp.aNFe5702IR
++ mktemp
+ local LAST_ERR=/tmp/tmp.GbyJN85jAt
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl exec psmdb-client-9b7b5fdcc-46nql -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-2.some-name-rs0.pitr-27834.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.aNFe5702IR
+ cat /tmp/tmp.GbyJN85jAt
+ rm /tmp/tmp.aNFe5702IR /tmp/tmp.GbyJN85jAt
+ return 0
+ diff /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1352/e2e-tests/pitr/compare/find.json /tmp/tmp.iKog3JtyCU/find
+ run_backup backup-minio 3
+ local name=backup-minio
+ local idx=3
+ desc 'run backup backup-minio-3'
+ set +o xtrace


-----------------------------------------------------------------------------------
run backup backup-minio-3
-----------------------------------------------------------------------------------

+ /usr/bin/sed -e 's/name:/name: backup-minio-3/'
+ cat /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1352/e2e-tests/pitr/conf/backup-minio.yml
+ kubectl_bin apply -f -
++ mktemp
+ local LAST_OUT=/tmp/tmp.xt1vPt9E86
++ mktemp
+ local LAST_ERR=/tmp/tmp.AhBKyj7h5Z
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl apply -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.xt1vPt9E86
perconaservermongodbbackup.psmdb.percona.com/backup-minio-3 created
+ cat /tmp/tmp.AhBKyj7h5Z
+ rm /tmp/tmp.xt1vPt9E86 /tmp/tmp.AhBKyj7h5Z
+ return 0
+ wait_backup backup-minio-3
+ local backup_name=backup-minio-3
+ set +o xtrace
backup-minio-3..........
+ check_recovery backup-minio-3 latest '' -4th backupSource some-name
+ local backup_name=backup-minio-3
+ local restore_type=latest
+ local restore_date=
+ local cmp_postfix=-4th
+ local backupSource=backupSource
+ local cluster_name=some-name
+ desc 'write more data before restore by latest'
+ set +o xtrace


-----------------------------------------------------------------------------------
write more data before restore by latest
-----------------------------------------------------------------------------------

+ sleep 60
+ run_mongo 'use myApp\n db.test.insert({ x: 100501 })' myApp:myPass@some-name-rs0.pitr-27834
+ local 'command=use myApp\n db.test.insert({ x: 100501 })'
+ local uri=myApp:myPass@some-name-rs0.pitr-27834
+ local driver=mongodb+srv
+ local suffix=.svc.cluster.local
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.ldilERshy2
+++ mktemp
++ local LAST_ERR=/tmp/tmp.VRgy06CeOK
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.ldilERshy2
++ cat /tmp/tmp.VRgy06CeOK
++ rm /tmp/tmp.ldilERshy2 /tmp/tmp.VRgy06CeOK
++ return 0
+ local client_container=psmdb-client-9b7b5fdcc-46nql
+ local mongo_flag=
+ [[ myApp:myPass@some-name-rs0.pitr-27834 == *cfg* ]]
+ replica_set=rs0
+ kubectl_bin exec psmdb-client-9b7b5fdcc-46nql -- bash -c 'printf '\''use myApp\n db.test.insert({ x: 100501 })\n'\'' | mongo mongodb+srv://myApp:myPass@some-name-rs0.pitr-27834.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
++ mktemp
+ local LAST_OUT=/tmp/tmp.R5ch3CuFyC
++ mktemp
+ local LAST_ERR=/tmp/tmp.vymXb4nwBD
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl exec psmdb-client-9b7b5fdcc-46nql -- bash -c 'printf '\''use myApp\n db.test.insert({ x: 100501 })\n'\'' | mongo mongodb+srv://myApp:myPass@some-name-rs0.pitr-27834.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.R5ch3CuFyC
Percona Server for MongoDB shell version v4.4.24-23
connecting to: mongodb://some-name-rs0-1.some-name-rs0.pitr-27834.svc.cluster.local:27017,some-name-rs0-0.some-name-rs0.pitr-27834.svc.cluster.local:27017,some-name-rs0-2.some-name-rs0.pitr-27834.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb&replicaSet=rs0&ssl=false
Implicit session: session { "id" : UUID("2aee5c24-8d03-4d37-a6eb-eb9367b54294") }
Percona Server for MongoDB server version: v5.0.20-17
WARNING: shell and server versions do not match
switched to db myApp
WriteResult({ "nInserted" : 1 })
bye
+ cat /tmp/tmp.vymXb4nwBD
+ rm /tmp/tmp.R5ch3CuFyC /tmp/tmp.vymXb4nwBD
+ return 0
+ desc 'waiting for chunks to be uploaded'
+ set +o xtrace


-----------------------------------------------------------------------------------
waiting for chunks to be uploaded
-----------------------------------------------------------------------------------

+ sleep 120
+ desc 'check restore by latest'
+ set +o xtrace


-----------------------------------------------------------------------------------
check restore by latest
-----------------------------------------------------------------------------------

+ '[' -z backupSource ']'
++ get_backup_dest backup-minio-3
++ local backup_name=backup-minio-3
++ kubectl_bin get psmdb-backup backup-minio-3 -o 'jsonpath={.status.destination}'
++ sed -e 's/.json$//'
+++ mktemp
++ sed 's|s3://||'
++ sed 's|azure://||'
++ local LAST_OUT=/tmp/tmp.1Nfs0CikdQ
+++ mktemp
++ local LAST_ERR=/tmp/tmp.50piuwW9ew
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb-backup backup-minio-3 -o 'jsonpath={.status.destination}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.1Nfs0CikdQ
++ cat /tmp/tmp.50piuwW9ew
++ rm /tmp/tmp.1Nfs0CikdQ /tmp/tmp.50piuwW9ew
++ return 0
+ backup_dest=operator-testing/2023-10-05T15:40:29Z
+ /usr/bin/sed -e 's/name:/name: restore-backup-minio-3/'
+ /usr/bin/sed -e 's|BACKUP-NAME|operator-testing/2023-10-05T15:40:29Z|'
+ '[' -z '' ']'
+ /usr/bin/sed -e /date:/d
+ /usr/bin/sed -e 's/type:/type: latest/'
+ kubectl_bin apply -f -
+ /usr/bin/sed -e /backupName/d
+ cat /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1352/e2e-tests/pitr/conf/restore.yml
++ mktemp
+ local LAST_OUT=/tmp/tmp.L7JHTe9ETj
++ mktemp
+ local LAST_ERR=/tmp/tmp.wYfG0iGp26
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl apply -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.L7JHTe9ETj
perconaservermongodbrestore.psmdb.percona.com/restore-backup-minio-3 created
+ cat /tmp/tmp.wYfG0iGp26
+ rm /tmp/tmp.L7JHTe9ETj /tmp/tmp.wYfG0iGp26
+ return 0
+ wait_restore backup-minio-3 some-name
+ local backup_name=backup-minio-3
+ local cluster_name=some-name
+ local target_state=ready
+ local wait_cluster_consistency=1
+ local wait_time=780
+ set +o xtrace
waiting psmdb-restore/backup-minio-3 to reach ready state.......
+ '[' 1 -eq 1 ']'
+ wait_cluster_consistency some-name
+ local cluster_name=some-name
+ local wait_time=32
+ retry=0
+ sleep 7
+ echo -n 'waiting for cluster readyness'
waiting for cluster readyness++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.m9QeLEMFnd
+++ mktemp
++ local LAST_ERR=/tmp/tmp.uSfWJunpdj
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.m9QeLEMFnd
++ cat /tmp/tmp.uSfWJunpdj
++ rm /tmp/tmp.m9QeLEMFnd /tmp/tmp.uSfWJunpdj
++ return 0
+ [[ ready == \r\e\a\d\y ]]
+ echo

+ set -o xtrace
+ compare_mongo_cmd find myApp:myPass@some-name-rs0-0.some-name-rs0.pitr-27834 -4th
+ local command=find
+ local uri=myApp:myPass@some-name-rs0-0.some-name-rs0.pitr-27834
+ local postfix=-4th
+ local suffix=
+ local database=myApp
+ local collection=test
+ run_mongo 'use myApp\n db.test.find()' myApp:myPass@some-name-rs0-0.some-name-rs0.pitr-27834 mongodb ''
+ local 'command=use myApp\n db.test.find()'
+ local uri=myApp:myPass@some-name-rs0-0.some-name-rs0.pitr-27834
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ egrep -v 'I NETWORK|W NETWORK|F NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match'
+ /usr/bin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.l3luRUnVdV
+++ mktemp
++ local LAST_ERR=/tmp/tmp.Y41iwdh08l
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.l3luRUnVdV
++ cat /tmp/tmp.Y41iwdh08l
++ rm /tmp/tmp.l3luRUnVdV /tmp/tmp.Y41iwdh08l
++ return 0
+ local client_container=psmdb-client-9b7b5fdcc-46nql
+ local mongo_flag=
+ [[ myApp:myPass@some-name-rs0-0.some-name-rs0.pitr-27834 == *cfg* ]]
+ replica_set=rs0
+ kubectl_bin exec psmdb-client-9b7b5fdcc-46nql -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-0.some-name-rs0.pitr-27834.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
++ mktemp
+ local LAST_OUT=/tmp/tmp.OMFuJvKvmv
++ mktemp
+ local LAST_ERR=/tmp/tmp.UCl5BpVnei
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl exec psmdb-client-9b7b5fdcc-46nql -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-0.some-name-rs0.pitr-27834.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.OMFuJvKvmv
+ cat /tmp/tmp.UCl5BpVnei
+ rm /tmp/tmp.OMFuJvKvmv /tmp/tmp.UCl5BpVnei
+ return 0
+ diff /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1352/e2e-tests/pitr/compare/find-4th.json /tmp/tmp.iKog3JtyCU/find-4th
+ compare_mongo_cmd find myApp:myPass@some-name-rs0-1.some-name-rs0.pitr-27834 -4th
+ local command=find
+ local uri=myApp:myPass@some-name-rs0-1.some-name-rs0.pitr-27834
+ local postfix=-4th
+ local suffix=
+ local database=myApp
+ local collection=test
+ run_mongo 'use myApp\n db.test.find()' myApp:myPass@some-name-rs0-1.some-name-rs0.pitr-27834 mongodb ''
+ local 'command=use myApp\n db.test.find()'
+ local uri=myApp:myPass@some-name-rs0-1.some-name-rs0.pitr-27834
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ egrep -v 'I NETWORK|W NETWORK|F NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match'
+ /usr/bin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.9NHUNpfrBu
+++ mktemp
++ local LAST_ERR=/tmp/tmp.9HrIYaC48L
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.9NHUNpfrBu
++ cat /tmp/tmp.9HrIYaC48L
++ rm /tmp/tmp.9NHUNpfrBu /tmp/tmp.9HrIYaC48L
++ return 0
+ local client_container=psmdb-client-9b7b5fdcc-46nql
+ local mongo_flag=
+ [[ myApp:myPass@some-name-rs0-1.some-name-rs0.pitr-27834 == *cfg* ]]
+ replica_set=rs0
+ kubectl_bin exec psmdb-client-9b7b5fdcc-46nql -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-1.some-name-rs0.pitr-27834.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
++ mktemp
+ local LAST_OUT=/tmp/tmp.CMajec4nRy
++ mktemp
+ local LAST_ERR=/tmp/tmp.zbatQOKlnI
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl exec psmdb-client-9b7b5fdcc-46nql -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-1.some-name-rs0.pitr-27834.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.CMajec4nRy
+ cat /tmp/tmp.zbatQOKlnI
+ rm /tmp/tmp.CMajec4nRy /tmp/tmp.zbatQOKlnI
+ return 0
+ diff /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1352/e2e-tests/pitr/compare/find-4th.json /tmp/tmp.iKog3JtyCU/find-4th
+ compare_mongo_cmd find myApp:myPass@some-name-rs0-2.some-name-rs0.pitr-27834 -4th
+ local command=find
+ local uri=myApp:myPass@some-name-rs0-2.some-name-rs0.pitr-27834
+ local postfix=-4th
+ local suffix=
+ local database=myApp
+ local collection=test
+ egrep -v 'I NETWORK|W NETWORK|F NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match'
+ run_mongo 'use myApp\n db.test.find()' myApp:myPass@some-name-rs0-2.some-name-rs0.pitr-27834 mongodb ''
+ local 'command=use myApp\n db.test.find()'
+ local uri=myApp:myPass@some-name-rs0-2.some-name-rs0.pitr-27834
+ local driver=mongodb
+ local suffix=.svc.cluster.local
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.yPzbN1noJH
+++ mktemp
++ local LAST_ERR=/tmp/tmp.zjlz0ADcwj
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+ /usr/bin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.yPzbN1noJH
++ cat /tmp/tmp.zjlz0ADcwj
++ rm /tmp/tmp.yPzbN1noJH /tmp/tmp.zjlz0ADcwj
++ return 0
+ local client_container=psmdb-client-9b7b5fdcc-46nql
+ local mongo_flag=
+ [[ myApp:myPass@some-name-rs0-2.some-name-rs0.pitr-27834 == *cfg* ]]
+ replica_set=rs0
+ kubectl_bin exec psmdb-client-9b7b5fdcc-46nql -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-2.some-name-rs0.pitr-27834.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
++ mktemp
+ local LAST_OUT=/tmp/tmp.VJWRWmUUi4
++ mktemp
+ local LAST_ERR=/tmp/tmp.f9j5LeUGJ1
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl exec psmdb-client-9b7b5fdcc-46nql -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-2.some-name-rs0.pitr-27834.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.VJWRWmUUi4
+ cat /tmp/tmp.f9j5LeUGJ1
+ rm /tmp/tmp.VJWRWmUUi4 /tmp/tmp.f9j5LeUGJ1
+ return 0
+ diff /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1352/e2e-tests/pitr/compare/find-4th.json /tmp/tmp.iKog3JtyCU/find-4th
+ kubectl patch psmdb some-name --type=merge --patch '{"spec": {"backup": {"pitr": {"enabled": false}}}}'
perconaservermongodb.psmdb.percona.com/some-name patched
+ sleep 20
+ kubectl_bin delete psmdb-backup --all
++ mktemp
+ local LAST_OUT=/tmp/tmp.UoxWzfspPG
++ mktemp
+ local LAST_ERR=/tmp/tmp.v1EUFDxihA
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl delete psmdb-backup --all
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.UoxWzfspPG
perconaservermongodbbackup.psmdb.percona.com "backup-minio-0" deleted
perconaservermongodbbackup.psmdb.percona.com "backup-minio-1" deleted
perconaservermongodbbackup.psmdb.percona.com "backup-minio-2" deleted
perconaservermongodbbackup.psmdb.percona.com "backup-minio-3" deleted
+ cat /tmp/tmp.v1EUFDxihA
+ rm /tmp/tmp.UoxWzfspPG /tmp/tmp.v1EUFDxihA
+ return 0
+ destroy pitr-27834
+ local namespace=pitr-27834
+ local ignore_logs=true
+ desc 'destroy cluster/operator and all other resources'
+ set +o xtrace


-----------------------------------------------------------------------------------
destroy cluster/operator and all other resources
-----------------------------------------------------------------------------------

+ '[' true == false ']'
+ delete_crd
+ desc 'get and delete old CRDs and RBAC'
+ set +o xtrace


-----------------------------------------------------------------------------------
get and delete old CRDs and RBAC
-----------------------------------------------------------------------------------

+ kubectl_bin delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1352/deploy/crd.yaml --ignore-not-found --wait=false
++ mktemp
+ local LAST_OUT=/tmp/tmp.h09p6USCK8
++ mktemp
+ local LAST_ERR=/tmp/tmp.elkORCEjZb
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1352/deploy/crd.yaml --ignore-not-found --wait=false
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.h09p6USCK8
customresourcedefinition.apiextensions.k8s.io "perconaservermongodbbackups.psmdb.percona.com" deleted
customresourcedefinition.apiextensions.k8s.io "perconaservermongodbrestores.psmdb.percona.com" deleted
customresourcedefinition.apiextensions.k8s.io "perconaservermongodbs.psmdb.percona.com" deleted
+ cat /tmp/tmp.elkORCEjZb
+ rm /tmp/tmp.h09p6USCK8 /tmp/tmp.elkORCEjZb
+ return 0
++ grep -v '\-\-\-'
++ yq eval .metadata.name /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1352/deploy/crd.yaml
+ for crd_name in '$(yq eval '\''.metadata.name'\'' "${src_dir}/deploy/crd.yaml" | grep -v '\''\-\-\-'\'')'
+ kubectl get perconaservermongodbbackups.psmdb.percona.com --all-namespaces -o wide
+ xargs -L 1 sh -xc 'kubectl patch perconaservermongodbbackups.psmdb.percona.com -n $0 $1 --type=merge -p "{\"metadata\":{\"finalizers\":[]}}"'
+ grep -v NAMESPACE
error: the server doesn't have a resource type "perconaservermongodbbackups"
+ kubectl patch perconaservermongodbbackups.psmdb.percona.com -n sh --type=merge -p '{"metadata":{"finalizers":[]}}'
error: the server doesn't have a resource type "perconaservermongodbbackups"
+ :
+ kubectl_bin wait --for=delete crd perconaservermongodbbackups.psmdb.percona.com
++ mktemp
+ local LAST_OUT=/tmp/tmp.nauUbToOGr
++ mktemp
+ local LAST_ERR=/tmp/tmp.K6Xpd53HVj
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl wait --for=delete crd perconaservermongodbbackups.psmdb.percona.com
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.nauUbToOGr
+ cat /tmp/tmp.K6Xpd53HVj
+ rm /tmp/tmp.nauUbToOGr /tmp/tmp.K6Xpd53HVj
+ return 0
+ for crd_name in '$(yq eval '\''.metadata.name'\'' "${src_dir}/deploy/crd.yaml" | grep -v '\''\-\-\-'\'')'
+ kubectl get perconaservermongodbrestores.psmdb.percona.com --all-namespaces -o wide
+ xargs -L 1 sh -xc 'kubectl patch perconaservermongodbrestores.psmdb.percona.com -n $0 $1 --type=merge -p "{\"metadata\":{\"finalizers\":[]}}"'
+ grep -v NAMESPACE
error: the server doesn't have a resource type "perconaservermongodbrestores"
+ kubectl patch perconaservermongodbrestores.psmdb.percona.com -n sh --type=merge -p '{"metadata":{"finalizers":[]}}'
error: the server doesn't have a resource type "perconaservermongodbrestores"
+ :
+ kubectl_bin wait --for=delete crd perconaservermongodbrestores.psmdb.percona.com
++ mktemp
+ local LAST_OUT=/tmp/tmp.Ucqr8ClfcN
++ mktemp
+ local LAST_ERR=/tmp/tmp.v7yYzKzqVr
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl wait --for=delete crd perconaservermongodbrestores.psmdb.percona.com
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.Ucqr8ClfcN
+ cat /tmp/tmp.v7yYzKzqVr
+ rm /tmp/tmp.Ucqr8ClfcN /tmp/tmp.v7yYzKzqVr
+ return 0
+ for crd_name in '$(yq eval '\''.metadata.name'\'' "${src_dir}/deploy/crd.yaml" | grep -v '\''\-\-\-'\'')'
+ kubectl get perconaservermongodbs.psmdb.percona.com --all-namespaces -o wide
+ xargs -L 1 sh -xc 'kubectl patch perconaservermongodbs.psmdb.percona.com -n $0 $1 --type=merge -p "{\"metadata\":{\"finalizers\":[]}}"'
+ grep -v NAMESPACE
error: the server doesn't have a resource type "perconaservermongodbs"
+ kubectl patch perconaservermongodbs.psmdb.percona.com -n sh --type=merge -p '{"metadata":{"finalizers":[]}}'
error: the server doesn't have a resource type "perconaservermongodbs"
+ :
+ kubectl_bin wait --for=delete crd perconaservermongodbs.psmdb.percona.com
++ mktemp
+ local LAST_OUT=/tmp/tmp.rjuC5vswZj
++ mktemp
+ local LAST_ERR=/tmp/tmp.SPhGUPxNax
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl wait --for=delete crd perconaservermongodbs.psmdb.percona.com
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.rjuC5vswZj
+ cat /tmp/tmp.SPhGUPxNax
+ rm /tmp/tmp.rjuC5vswZj /tmp/tmp.SPhGUPxNax
+ return 0
+ local rbac_yaml=rbac.yaml
+ '[' -n '' ']'
+ kubectl_bin delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1352/deploy/rbac.yaml --ignore-not-found
++ mktemp
+ local LAST_OUT=/tmp/tmp.IenGqCJw0Y
++ mktemp
+ local LAST_ERR=/tmp/tmp.BlMpjKohs5
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1352/deploy/rbac.yaml --ignore-not-found
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.IenGqCJw0Y
role.rbac.authorization.k8s.io "percona-server-mongodb-operator" deleted
serviceaccount "percona-server-mongodb-operator" deleted
rolebinding.rbac.authorization.k8s.io "service-account-percona-server-mongodb-operator" deleted
+ cat /tmp/tmp.BlMpjKohs5
+ rm /tmp/tmp.IenGqCJw0Y /tmp/tmp.BlMpjKohs5
+ return 0
+ kubectl_bin delete -f https://github.com/cert-manager/cert-manager/releases/download/v1.12.4/cert-manager.yaml
+ :
+ '[' -n '' ']'
+ '[' -n '' ']'
+ rm -rf /tmp/tmp.iKog3JtyCU
+ kubectl_bin delete --grace-period=0 --force=true namespace pitr-27834
++ mktemp
+ local LAST_OUT=/tmp/tmp.xYNkmDO7df
++ mktemp
+ local LAST_ERR=/tmp/tmp.9X2QWlLzDJ
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl delete --grace-period=0 --force=true namespace pitr-27834
+ desc 'test passed'
+ set +o xtrace


-----------------------------------------------------------------------------------
test passed
-----------------------------------------------------------------------------------