Log: /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1961/e2e-tests/logs/pitr-physical-backup-source.log
WARNING: version difference between client (1.33) and server (1.30) exceeds the supported minor version skew of +/-1
WARNING: version difference between client (1.33) and server (1.30) exceeds the supported minor version skew of +/-1
WARNING: version difference between client (1.33) and server (1.30) exceeds the supported minor version skew of +/-1
+ main
+ create_infra pitr-physical-backup-source-28726
+ local ns=pitr-physical-backup-source-28726
+ [[ 1 == 1 ]]
+ delete_crd
+ desc 'get and delete old CRDs and RBAC'
+ set +o xtrace


-----------------------------------------------------------------------------------
get and delete old CRDs and RBAC
-----------------------------------------------------------------------------------

+ kubectl_bin delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1961/deploy/crd.yaml --ignore-not-found --wait=false
++ mktemp
+ local LAST_OUT=/tmp/tmp.V70qS43GFS
++ mktemp
+ local LAST_ERR=/tmp/tmp.JCGGaUDX2G
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1961/deploy/crd.yaml --ignore-not-found --wait=false
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.V70qS43GFS
customresourcedefinition.apiextensions.k8s.io "perconaservermongodbbackups.psmdb.percona.com" deleted
customresourcedefinition.apiextensions.k8s.io "perconaservermongodbrestores.psmdb.percona.com" deleted
customresourcedefinition.apiextensions.k8s.io "perconaservermongodbs.psmdb.percona.com" deleted
+ cat /tmp/tmp.JCGGaUDX2G
+ rm /tmp/tmp.V70qS43GFS /tmp/tmp.JCGGaUDX2G
+ return 0
++ yq eval .metadata.name /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1961/deploy/crd.yaml
++ grep -v '\-\-\-'
+ for crd_name in '$(yq eval '\''.metadata.name'\'' "${src_dir}/deploy/crd.yaml" | grep -v '\''\-\-\-'\'')'
+ kubectl get perconaservermongodbbackups.psmdb.percona.com --all-namespaces -o wide
+ grep -v NAMESPACE
+ xargs -L 1 sh -xc 'kubectl patch perconaservermongodbbackups.psmdb.percona.com -n $0 $1 --type=merge -p "{\"metadata\":{\"finalizers\":[]}}"'
No resources found
+ kubectl patch perconaservermongodbbackups.psmdb.percona.com -n sh --type=merge -p '{"metadata":{"finalizers":[]}}'
error: resource(s) were provided, but no name was specified
+ :
+ kubectl_bin wait --for=delete crd perconaservermongodbbackups.psmdb.percona.com
++ mktemp
+ local LAST_OUT=/tmp/tmp.vDaNc03xI2
++ mktemp
+ local LAST_ERR=/tmp/tmp.E42Dio8Uzc
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl wait --for=delete crd perconaservermongodbbackups.psmdb.percona.com
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.vDaNc03xI2
customresourcedefinition.apiextensions.k8s.io/perconaservermongodbbackups.psmdb.percona.com condition met
+ cat /tmp/tmp.E42Dio8Uzc
+ rm /tmp/tmp.vDaNc03xI2 /tmp/tmp.E42Dio8Uzc
+ return 0
+ for crd_name in '$(yq eval '\''.metadata.name'\'' "${src_dir}/deploy/crd.yaml" | grep -v '\''\-\-\-'\'')'
+ kubectl get perconaservermongodbrestores.psmdb.percona.com --all-namespaces -o wide
+ grep -v NAMESPACE
+ xargs -L 1 sh -xc 'kubectl patch perconaservermongodbrestores.psmdb.percona.com -n $0 $1 --type=merge -p "{\"metadata\":{\"finalizers\":[]}}"'
error: the server doesn't have a resource type "perconaservermongodbrestores"
+ kubectl patch perconaservermongodbrestores.psmdb.percona.com -n sh --type=merge -p '{"metadata":{"finalizers":[]}}'
error: the server doesn't have a resource type "perconaservermongodbrestores"
+ :
+ kubectl_bin wait --for=delete crd perconaservermongodbrestores.psmdb.percona.com
++ mktemp
+ local LAST_OUT=/tmp/tmp.ktzpYky6Zm
++ mktemp
+ local LAST_ERR=/tmp/tmp.Hh340Jn2Zh
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl wait --for=delete crd perconaservermongodbrestores.psmdb.percona.com
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.ktzpYky6Zm
+ cat /tmp/tmp.Hh340Jn2Zh
+ rm /tmp/tmp.ktzpYky6Zm /tmp/tmp.Hh340Jn2Zh
+ return 0
+ for crd_name in '$(yq eval '\''.metadata.name'\'' "${src_dir}/deploy/crd.yaml" | grep -v '\''\-\-\-'\'')'
+ kubectl get perconaservermongodbs.psmdb.percona.com --all-namespaces -o wide
+ grep -v NAMESPACE
+ xargs -L 1 sh -xc 'kubectl patch perconaservermongodbs.psmdb.percona.com -n $0 $1 --type=merge -p "{\"metadata\":{\"finalizers\":[]}}"'
error: the server doesn't have a resource type "perconaservermongodbs"
+ kubectl patch perconaservermongodbs.psmdb.percona.com -n sh --type=merge -p '{"metadata":{"finalizers":[]}}'
error: the server doesn't have a resource type "perconaservermongodbs"
+ :
+ kubectl_bin wait --for=delete crd perconaservermongodbs.psmdb.percona.com
++ mktemp
+ local LAST_OUT=/tmp/tmp.NAFotk8RRE
++ mktemp
+ local LAST_ERR=/tmp/tmp.yqj3KLNIf9
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl wait --for=delete crd perconaservermongodbs.psmdb.percona.com
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.NAFotk8RRE
+ cat /tmp/tmp.yqj3KLNIf9
+ rm /tmp/tmp.NAFotk8RRE /tmp/tmp.yqj3KLNIf9
+ return 0
+ local rbac_yaml=rbac.yaml
+ '[' -n psmdb-operator ']'
+ rbac_yaml=cw-rbac.yaml
+ kubectl_bin delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1961/deploy/cw-rbac.yaml --ignore-not-found
++ mktemp
+ local LAST_OUT=/tmp/tmp.d3ioj59wTx
++ mktemp
+ local LAST_ERR=/tmp/tmp.VQHgrdyWx4
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1961/deploy/cw-rbac.yaml --ignore-not-found
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.d3ioj59wTx
clusterrole.rbac.authorization.k8s.io "percona-server-mongodb-operator" deleted
clusterrolebinding.rbac.authorization.k8s.io "service-account-percona-server-mongodb-operator" deleted
+ cat /tmp/tmp.VQHgrdyWx4
+ rm /tmp/tmp.d3ioj59wTx /tmp/tmp.VQHgrdyWx4
+ return 0
+ check_crd_for_deletion PR-1961-970eeaa4
+ local git_tag=PR-1961-970eeaa4
++ curl -s https://raw.githubusercontent.com/percona/percona-server-mongodb-operator/PR-1961-970eeaa4/deploy/crd.yaml
++ /usr/bin/sed s/---//g
++ yq eval .metadata.name
++ /usr/bin/sed ':a;N;$!ba;s/\n/ /g'
+ for crd_name in '$(curl -s https://raw.githubusercontent.com/percona/percona-server-mongodb-operator/${git_tag}/deploy/crd.yaml | yq eval '\''.metadata.name'\'' | $sed '\''s/---//g'\'' | $sed '\'':a;N;$!ba;s/\n/ /g'\'')'
++ kubectl_bin get crd/null -o 'jsonpath={.status.conditions[-1].type}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.iqi89BEnt1
+++ mktemp
++ local LAST_ERR=/tmp/tmp.xG6JIaWDW7
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get crd/null -o 'jsonpath={.status.conditions[-1].type}'
++ exit_status=1
++ set -e
++ '[' 1 '!=' 0 -a -n 1 ']'
++ cat /tmp/tmp.iqi89BEnt1
++ cat /tmp/tmp.xG6JIaWDW7
Error from server (NotFound): customresourcedefinitions.apiextensions.k8s.io "null" not found
++ sleep 0
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get crd/null -o 'jsonpath={.status.conditions[-1].type}'
++ exit_status=1
++ set -e
++ '[' 1 '!=' 0 -a -n 1 ']'
++ cat /tmp/tmp.iqi89BEnt1
++ cat /tmp/tmp.xG6JIaWDW7
Error from server (NotFound): customresourcedefinitions.apiextensions.k8s.io "null" not found
++ sleep 4
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get crd/null -o 'jsonpath={.status.conditions[-1].type}'
++ exit_status=1
++ set -e
++ '[' 1 '!=' 0 -a -n 1 ']'
++ cat /tmp/tmp.iqi89BEnt1
++ cat /tmp/tmp.xG6JIaWDW7
Error from server (NotFound): customresourcedefinitions.apiextensions.k8s.io "null" not found
++ sleep 8
++ cat /tmp/tmp.iqi89BEnt1
++ cat /tmp/tmp.xG6JIaWDW7
Error from server (NotFound): customresourcedefinitions.apiextensions.k8s.io "null" not found
++ rm /tmp/tmp.iqi89BEnt1 /tmp/tmp.xG6JIaWDW7
++ return 1
+ [[ '' == \T\e\r\m\i\n\a\t\i\n\g ]]
+ '[' -n psmdb-operator ']'
+ create_namespace psmdb-operator
+ local namespace=psmdb-operator
+ local skip_clean_namespace=
+ [[ 1 == 1 ]]
+ [[ -z '' ]]
+ destroy_chaos_mesh
++ tail -n1
++ helm list --all-namespaces --filter chaos-mesh
++ sed s/NAMESPACE//
++ awk '-F ' '{print $2}'
+ local chaos_mesh_ns=
+ desc 'destroy chaos-mesh'
+ set +o xtrace


-----------------------------------------------------------------------------------
destroy chaos-mesh
-----------------------------------------------------------------------------------

+ '[' -n '' ']'
++ kubectl get MutatingWebhookConfiguration
++ grep chaos-mesh
++ awk '{print $1}'
+ timeout 30 kubectl delete MutatingWebhookConfiguration
error: resource(s) were provided, but no name was specified
+ :
++ kubectl get ValidatingWebhookConfiguration
++ awk '{print $1}'
++ grep chaos-mesh
+ timeout 30 kubectl delete ValidatingWebhookConfiguration
error: resource(s) were provided, but no name was specified
+ :
++ kubectl get ValidatingWebhookConfiguration
++ awk '{print $1}'
++ grep validate-auth
+ timeout 30 kubectl delete ValidatingWebhookConfiguration
error: resource(s) were provided, but no name was specified
+ :
++ kubectl api-resources
++ grep chaos-mesh
++ awk '{print $1}'
++ kubectl get crd
++ grep chaos-mesh.org
++ awk '{print $1}'
+ timeout 30 kubectl delete crd
error: resource(s) were provided, but no name was specified
+ :
++ grep chaos-mesh
++ awk '{print $1}'
++ kubectl get clusterrolebinding
+ timeout 30 kubectl delete clusterrolebinding
error: resource(s) were provided, but no name was specified
+ :
++ grep chaos-mesh
++ awk '{print $1}'
++ kubectl get clusterrole
+ timeout 30 kubectl delete clusterrole
error: resource(s) were provided, but no name was specified
+ :
+ desc 'cleaned up all old namespaces'
+ set +o xtrace


-----------------------------------------------------------------------------------
cleaned up all old namespaces
-----------------------------------------------------------------------------------

+ kubectl_bin get ns
+ '[' -n '' ']'
+ desc 'cleaned up old namespaces psmdb-operator'
+ set +o xtrace


-----------------------------------------------------------------------------------
cleaned up old namespaces psmdb-operator
-----------------------------------------------------------------------------------

+ awk '{print$1}'
+ kubectl_bin delete namespace psmdb-operator --ignore-not-found
+ xargs kubectl delete ns
++ mktemp
+ egrep -v '^kube-|^default|Terminating|psmdb-operator|openshift|^gke-|^gmp-|^NAME'
++ mktemp
+ local LAST_OUT=/tmp/tmp.VbH5dWSUR1
+ local LAST_OUT=/tmp/tmp.pljsooHcjQ
++ mktemp
+ local LAST_ERR=/tmp/tmp.QcZCJHsz5d
+ local exit_status=0
+ local timeout=4
++ seq 0 2
++ mktemp
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl delete namespace psmdb-operator --ignore-not-found
+ local LAST_ERR=/tmp/tmp.9ZoE3ycJs8
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl get ns
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.pljsooHcjQ
+ cat /tmp/tmp.9ZoE3ycJs8
+ rm /tmp/tmp.pljsooHcjQ /tmp/tmp.9ZoE3ycJs8
+ return 0
namespace "pitr-physical-backup-source-4586" deleted
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.VbH5dWSUR1
namespace "psmdb-operator" deleted
+ cat /tmp/tmp.QcZCJHsz5d
+ rm /tmp/tmp.VbH5dWSUR1 /tmp/tmp.QcZCJHsz5d
+ return 0
+ kubectl_bin wait --for=delete namespace psmdb-operator
++ mktemp
+ local LAST_OUT=/tmp/tmp.E17jg3K4xf
++ mktemp
+ local LAST_ERR=/tmp/tmp.DYBo9lW0uC
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl wait --for=delete namespace psmdb-operator
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.E17jg3K4xf
+ cat /tmp/tmp.DYBo9lW0uC
+ rm /tmp/tmp.E17jg3K4xf /tmp/tmp.DYBo9lW0uC
+ return 0
+ desc 'create namespace psmdb-operator'
+ set +o xtrace


-----------------------------------------------------------------------------------
create namespace psmdb-operator
-----------------------------------------------------------------------------------

+ kubectl_bin create namespace psmdb-operator
++ mktemp
+ local LAST_OUT=/tmp/tmp.ikwEUndVHl
++ mktemp
+ local LAST_ERR=/tmp/tmp.5NGfodjsFx
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl create namespace psmdb-operator
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.ikwEUndVHl
namespace/psmdb-operator created
+ cat /tmp/tmp.5NGfodjsFx
+ rm /tmp/tmp.ikwEUndVHl /tmp/tmp.5NGfodjsFx
+ return 0
+ set_kube_ctx psmdb-operator
+ local namespace=psmdb-operator
++ kubectl_bin config current-context
+++ mktemp
++ local LAST_OUT=/tmp/tmp.1EFq8mlALJ
+++ mktemp
++ local LAST_ERR=/tmp/tmp.g91Qk0Y1Kk
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl config current-context
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.1EFq8mlALJ
++ cat /tmp/tmp.g91Qk0Y1Kk
++ rm /tmp/tmp.1EFq8mlALJ /tmp/tmp.g91Qk0Y1Kk
++ return 0
+ kubectl_bin config set-context gke_cloud-dev-112233_us-central1-a_jen-psmdb-1961-970eeaa4-5-cluster6 --namespace=psmdb-operator
++ mktemp
+ local LAST_OUT=/tmp/tmp.ymVbnaxuwr
++ mktemp
+ local LAST_ERR=/tmp/tmp.GaQ1INzxng
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl config set-context gke_cloud-dev-112233_us-central1-a_jen-psmdb-1961-970eeaa4-5-cluster6 --namespace=psmdb-operator
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.ymVbnaxuwr
Context "gke_cloud-dev-112233_us-central1-a_jen-psmdb-1961-970eeaa4-5-cluster6" modified.
+ cat /tmp/tmp.GaQ1INzxng
+ rm /tmp/tmp.ymVbnaxuwr /tmp/tmp.GaQ1INzxng
+ return 0
+ deploy_operator
+ desc 'start PSMDB operator'
+ set +o xtrace


-----------------------------------------------------------------------------------
start PSMDB operator
-----------------------------------------------------------------------------------

+ local cr_file
+ '[' -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1961/e2e-tests/pitr-physical-backup-source/conf/crd.yaml ']'
+ cr_file=/mnt/jenkins/workspace/cloud-psmdb-operator_PR-1961/deploy/crd.yaml
+ kubectl_bin apply --server-side --force-conflicts -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1961/deploy/crd.yaml
++ mktemp
+ local LAST_OUT=/tmp/tmp.P4oU0vI0ps
++ mktemp
+ local LAST_ERR=/tmp/tmp.GPRN8nIgoj
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl apply --server-side --force-conflicts -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1961/deploy/crd.yaml
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.P4oU0vI0ps
customresourcedefinition.apiextensions.k8s.io/perconaservermongodbbackups.psmdb.percona.com serverside-applied
customresourcedefinition.apiextensions.k8s.io/perconaservermongodbrestores.psmdb.percona.com serverside-applied
customresourcedefinition.apiextensions.k8s.io/perconaservermongodbs.psmdb.percona.com serverside-applied
+ cat /tmp/tmp.GPRN8nIgoj
+ rm /tmp/tmp.P4oU0vI0ps /tmp/tmp.GPRN8nIgoj
+ return 0
+ '[' -n psmdb-operator ']'
+ apply_rbac cw-rbac
+ local operator_namespace=psmdb-operator
+ local rbac=cw-rbac
+ cat /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1961/deploy/cw-rbac.yaml
+ sed -e 's^namespace: .*^namespace: psmdb-operator^'
+ kubectl_bin apply -n psmdb-operator -f -
++ mktemp
+ local LAST_OUT=/tmp/tmp.mWezzP4z5W
++ mktemp
+ local LAST_ERR=/tmp/tmp.IzF2OoArIx
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl apply -n psmdb-operator -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.mWezzP4z5W
clusterrole.rbac.authorization.k8s.io/percona-server-mongodb-operator created
serviceaccount/percona-server-mongodb-operator created
clusterrolebinding.rbac.authorization.k8s.io/service-account-percona-server-mongodb-operator created
+ cat /tmp/tmp.IzF2OoArIx
+ rm /tmp/tmp.mWezzP4z5W /tmp/tmp.IzF2OoArIx
+ return 0
+ yq eval '
			(.spec.template.spec.containers[].image = "perconalab/percona-server-mongodb-operator:PR-1961-970eeaa4") |
			((.. | select(.[] == "DISABLE_TELEMETRY")) |= .value="true") |
			((.. | select(.[] == "LOG_LEVEL")) |= .value="DEBUG")' /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1961/deploy/cw-operator.yaml
+ kubectl_bin apply -f -
++ mktemp
+ local LAST_OUT=/tmp/tmp.jANQEU8fxN
++ mktemp
+ local LAST_ERR=/tmp/tmp.VBATZ8PKSz
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl apply -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.jANQEU8fxN
deployment.apps/percona-server-mongodb-operator created
+ cat /tmp/tmp.VBATZ8PKSz
+ rm /tmp/tmp.jANQEU8fxN /tmp/tmp.VBATZ8PKSz
+ return 0
+ sleep 2
++ get_operator_pod
++ kubectl_bin get pods --selector=name=percona-server-mongodb-operator -o 'jsonpath={.items[].metadata.name}' -n psmdb-operator
+++ mktemp
++ local LAST_OUT=/tmp/tmp.iWHXamYL9X
+++ mktemp
++ local LAST_ERR=/tmp/tmp.YBC9n8AwCU
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=percona-server-mongodb-operator -o 'jsonpath={.items[].metadata.name}' -n psmdb-operator
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.iWHXamYL9X
++ cat /tmp/tmp.YBC9n8AwCU
++ rm /tmp/tmp.iWHXamYL9X /tmp/tmp.YBC9n8AwCU
++ return 0
+ wait_pod percona-server-mongodb-operator-7cfd5675c9-5btm4
+ local pod=percona-server-mongodb-operator-7cfd5675c9-5btm4
+ set +o xtrace
waiting for pod/percona-server-mongodb-operator-7cfd5675c9-5btm4 to be ready.OK
+ echo 'Print operator info from log'
Print operator info from log
+ grep 'Manager starting up'
++ get_operator_pod
++ kubectl_bin get pods --selector=name=percona-server-mongodb-operator -o 'jsonpath={.items[].metadata.name}' -n psmdb-operator
+++ mktemp
++ local LAST_OUT=/tmp/tmp.6kca6Pp2TG
+++ mktemp
++ local LAST_ERR=/tmp/tmp.1NNDd5sG19
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=percona-server-mongodb-operator -o 'jsonpath={.items[].metadata.name}' -n psmdb-operator
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.6kca6Pp2TG
++ cat /tmp/tmp.1NNDd5sG19
++ rm /tmp/tmp.6kca6Pp2TG /tmp/tmp.1NNDd5sG19
++ return 0
+ kubectl_bin logs percona-server-mongodb-operator-7cfd5675c9-5btm4
++ mktemp
+ local LAST_OUT=/tmp/tmp.TppaHx7Vmt
++ mktemp
+ local LAST_ERR=/tmp/tmp.EqTGnJewYd
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl logs percona-server-mongodb-operator-7cfd5675c9-5btm4
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.TppaHx7Vmt
+ cat /tmp/tmp.EqTGnJewYd
+ rm /tmp/tmp.TppaHx7Vmt /tmp/tmp.EqTGnJewYd
+ return 0
2025-08-06T21:36:19.997Z	INFO	setup	Manager starting up	{"gitCommit": "970eeaa4b725700bdc63e3d7ec04ab0be1ecce33", "gitBranch": "PR-1961-970eeaa4", "buildTime": "", "goVersion": "go1.24.5", "os": "linux", "arch": "amd64"}
+ create_namespace pitr-physical-backup-source-28726
+ local namespace=pitr-physical-backup-source-28726
+ local skip_clean_namespace=
+ [[ 1 == 1 ]]
+ [[ -z '' ]]
+ destroy_chaos_mesh
++ helm list --all-namespaces --filter chaos-mesh
++ tail -n1
++ awk '-F ' '{print $2}'
++ sed s/NAMESPACE//
+ local chaos_mesh_ns=
+ desc 'destroy chaos-mesh'
+ set +o xtrace


-----------------------------------------------------------------------------------
destroy chaos-mesh
-----------------------------------------------------------------------------------

+ '[' -n '' ']'
++ kubectl get MutatingWebhookConfiguration
++ grep chaos-mesh
++ awk '{print $1}'
+ timeout 30 kubectl delete MutatingWebhookConfiguration
error: resource(s) were provided, but no name was specified
+ :
++ grep chaos-mesh
++ kubectl get ValidatingWebhookConfiguration
++ awk '{print $1}'
+ timeout 30 kubectl delete ValidatingWebhookConfiguration
error: resource(s) were provided, but no name was specified
+ :
++ kubectl get ValidatingWebhookConfiguration
++ grep validate-auth
++ awk '{print $1}'
+ timeout 30 kubectl delete ValidatingWebhookConfiguration
error: resource(s) were provided, but no name was specified
+ :
++ kubectl api-resources
++ awk '{print $1}'
++ grep chaos-mesh
++ kubectl get crd
++ grep chaos-mesh.org
++ awk '{print $1}'
+ timeout 30 kubectl delete crd
error: resource(s) were provided, but no name was specified
+ :
++ awk '{print $1}'
++ grep chaos-mesh
++ kubectl get clusterrolebinding
+ timeout 30 kubectl delete clusterrolebinding
error: resource(s) were provided, but no name was specified
+ :
++ kubectl get clusterrole
++ grep chaos-mesh
++ awk '{print $1}'
+ timeout 30 kubectl delete clusterrole
error: resource(s) were provided, but no name was specified
+ :
+ desc 'cleaned up all old namespaces'
+ set +o xtrace


-----------------------------------------------------------------------------------
cleaned up all old namespaces
-----------------------------------------------------------------------------------

+ kubectl_bin get ns
+ egrep -v '^kube-|^default|Terminating|psmdb-operator|openshift|^gke-|^gmp-|^NAME'
+ awk '{print$1}'
+ '[' -n '' ']'
+ desc 'cleaned up old namespaces pitr-physical-backup-source-28726'
+ set +o xtrace
++ mktemp


-----------------------------------------------------------------------------------
cleaned up old namespaces pitr-physical-backup-source-28726
-----------------------------------------------------------------------------------

+ kubectl_bin delete namespace pitr-physical-backup-source-28726 --ignore-not-found
++ mktemp
+ local LAST_OUT=/tmp/tmp.bw3cMopoYt
++ mktemp
+ local LAST_OUT=/tmp/tmp.2YFlJlyYN7
++ mktemp
+ local LAST_ERR=/tmp/tmp.8RBngCtwMQ
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ local LAST_ERR=/tmp/tmp.uckIsbaW0L
+ local exit_status=0
+ local timeout=4
+ xargs kubectl delete ns
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl get ns
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl delete namespace pitr-physical-backup-source-28726 --ignore-not-found
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.bw3cMopoYt
+ cat /tmp/tmp.8RBngCtwMQ
+ rm /tmp/tmp.bw3cMopoYt /tmp/tmp.8RBngCtwMQ
+ return 0
error: resource(s) were provided, but no name was specified
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.2YFlJlyYN7
+ cat /tmp/tmp.uckIsbaW0L
+ rm /tmp/tmp.2YFlJlyYN7 /tmp/tmp.uckIsbaW0L
+ return 0
+ kubectl_bin wait --for=delete namespace pitr-physical-backup-source-28726
++ mktemp
+ local LAST_OUT=/tmp/tmp.RjtBWtS5mA
++ mktemp
+ local LAST_ERR=/tmp/tmp.NaguWXjc08
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl wait --for=delete namespace pitr-physical-backup-source-28726
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.RjtBWtS5mA
+ cat /tmp/tmp.NaguWXjc08
+ rm /tmp/tmp.RjtBWtS5mA /tmp/tmp.NaguWXjc08
+ return 0
+ desc 'create namespace pitr-physical-backup-source-28726'
+ set +o xtrace


-----------------------------------------------------------------------------------
create namespace pitr-physical-backup-source-28726
-----------------------------------------------------------------------------------

+ kubectl_bin create namespace pitr-physical-backup-source-28726
++ mktemp
+ local LAST_OUT=/tmp/tmp.noBgQl2YFt
++ mktemp
+ local LAST_ERR=/tmp/tmp.F6oXQ8x4FR
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl create namespace pitr-physical-backup-source-28726
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.noBgQl2YFt
namespace/pitr-physical-backup-source-28726 created
+ cat /tmp/tmp.F6oXQ8x4FR
+ rm /tmp/tmp.noBgQl2YFt /tmp/tmp.F6oXQ8x4FR
+ return 0
+ set_kube_ctx pitr-physical-backup-source-28726
+ local namespace=pitr-physical-backup-source-28726
++ kubectl_bin config current-context
+++ mktemp
++ local LAST_OUT=/tmp/tmp.F5nbaduY7O
+++ mktemp
++ local LAST_ERR=/tmp/tmp.fBepSfteGw
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl config current-context
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.F5nbaduY7O
++ cat /tmp/tmp.fBepSfteGw
++ rm /tmp/tmp.F5nbaduY7O /tmp/tmp.fBepSfteGw
++ return 0
+ kubectl_bin config set-context gke_cloud-dev-112233_us-central1-a_jen-psmdb-1961-970eeaa4-5-cluster6 --namespace=pitr-physical-backup-source-28726
++ mktemp
+ local LAST_OUT=/tmp/tmp.8YjiJv5CTl
++ mktemp
+ local LAST_ERR=/tmp/tmp.g7n6lOzSlF
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl config set-context gke_cloud-dev-112233_us-central1-a_jen-psmdb-1961-970eeaa4-5-cluster6 --namespace=pitr-physical-backup-source-28726
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.8YjiJv5CTl
Context "gke_cloud-dev-112233_us-central1-a_jen-psmdb-1961-970eeaa4-5-cluster6" modified.
+ cat /tmp/tmp.g7n6lOzSlF
+ rm /tmp/tmp.8YjiJv5CTl /tmp/tmp.g7n6lOzSlF
+ return 0
+ deploy_minio
+ desc 'install Minio'
+ set +o xtrace


-----------------------------------------------------------------------------------
install Minio
-----------------------------------------------------------------------------------

+ helm uninstall minio-service
Error: uninstall: Release not loaded: minio-service: release: not found
+ :
+ helm repo remove minio
"minio" has been removed from your repositories
+ helm repo add minio https://charts.min.io/
"minio" has been added to your repositories
+ retry 10 60 helm install minio-service --version 5.4.0 --set replicas=1 --set mode=standalone --set resources.requests.memory=256Mi --set rootUser=rootuser --set rootPassword=rootpass123 --set 'users[0].accessKey=some-access-key' --set 'users[0].secretKey=some-secret-key' --set 'users[0].policy=consoleAdmin' --set service.type=ClusterIP --set configPathmc=/tmp/.minio/ --set persistence.size=2G --set securityContext.enabled=false minio/minio
+ local max=10
+ local delay=60
+ shift 2
+ local n=1
+ helm install minio-service --version 5.4.0 --set replicas=1 --set mode=standalone --set resources.requests.memory=256Mi --set rootUser=rootuser --set rootPassword=rootpass123 --set 'users[0].accessKey=some-access-key' --set 'users[0].secretKey=some-secret-key' --set 'users[0].policy=consoleAdmin' --set service.type=ClusterIP --set configPathmc=/tmp/.minio/ --set persistence.size=2G --set securityContext.enabled=false minio/minio
NAME: minio-service
LAST DEPLOYED: Wed Aug  6 21:36:37 2025
NAMESPACE: pitr-physical-backup-source-28726
STATUS: deployed
REVISION: 1
TEST SUITE: None
NOTES:
MinIO can be accessed via port 9000 on the following DNS name from within your cluster:
minio-service.pitr-physical-backup-source-28726.cluster.local

To access MinIO from localhost, run the below commands:

  1. export POD_NAME=$(kubectl get pods --namespace pitr-physical-backup-source-28726 -l "release=minio-service" -o jsonpath="{.items[0].metadata.name}")

  2. kubectl port-forward $POD_NAME 9000 --namespace pitr-physical-backup-source-28726

Read more about port forwarding here: http://kubernetes.io/docs/user-guide/kubectl/kubectl_port-forward/

You can now access MinIO server on http://localhost:9000. Follow the below steps to connect to MinIO server with mc client:

  1. Download the MinIO mc client - https://min.io/docs/minio/linux/reference/minio-mc.html#quickstart

  2. export MC_HOST_minio-service-local=http://$(kubectl get secret --namespace pitr-physical-backup-source-28726 minio-service -o jsonpath="{.data.rootUser}" | base64 --decode):$(kubectl get secret --namespace pitr-physical-backup-source-28726 minio-service -o jsonpath="{.data.rootPassword}" | base64 --decode)@localhost:9000

  3. mc ls minio-service-local
++ kubectl_bin get pods --selector=release=minio-service -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.QozGAbHvBw
+++ mktemp
++ local LAST_ERR=/tmp/tmp.UV1JSF5Oz4
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=release=minio-service -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.QozGAbHvBw
++ cat /tmp/tmp.UV1JSF5Oz4
++ rm /tmp/tmp.QozGAbHvBw /tmp/tmp.UV1JSF5Oz4
++ return 0
+ MINIO_POD=minio-service-86dfccd949-j665g
+ wait_pod minio-service-86dfccd949-j665g
+ local pod=minio-service-86dfccd949-j665g
+ set +o xtrace
waiting for pod/minio-service-86dfccd949-j665g to be ready.OK
+ '[' -n psmdb-operator ']'
+ kubectl_bin create svc -n psmdb-operator externalname minio-service --external-name=minio-service.pitr-physical-backup-source-28726.svc.cluster.local --tcp=9000
++ mktemp
+ local LAST_OUT=/tmp/tmp.slB6h4YP0k
++ mktemp
+ local LAST_ERR=/tmp/tmp.dH2ExWOPLy
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl create svc -n psmdb-operator externalname minio-service --external-name=minio-service.pitr-physical-backup-source-28726.svc.cluster.local --tcp=9000
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.slB6h4YP0k
service/minio-service created
+ cat /tmp/tmp.dH2ExWOPLy
+ rm /tmp/tmp.slB6h4YP0k /tmp/tmp.dH2ExWOPLy
+ return 0
+ create_minio_bucket operator-testing
+ local bucket=operator-testing
+ kubectl_bin run -i --rm aws-cli --image=perconalab/awscli --restart=Never -- bash -c 'AWS_ACCESS_KEY_ID=some-access-key 		AWS_SECRET_ACCESS_KEY=some-secret-key 		AWS_DEFAULT_REGION=us-east-1         /usr/bin/aws --endpoint-url http://minio-service:9000 s3 mb s3://operator-testing'
++ mktemp
+ local LAST_OUT=/tmp/tmp.63RYj811Zy
++ mktemp
+ local LAST_ERR=/tmp/tmp.TuZVgpSF00
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl run -i --rm aws-cli --image=perconalab/awscli --restart=Never -- bash -c 'AWS_ACCESS_KEY_ID=some-access-key 		AWS_SECRET_ACCESS_KEY=some-secret-key 		AWS_DEFAULT_REGION=us-east-1         /usr/bin/aws --endpoint-url http://minio-service:9000 s3 mb s3://operator-testing'
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.63RYj811Zy
make_bucket: operator-testing
pod "aws-cli" deleted
+ cat /tmp/tmp.TuZVgpSF00
+ rm /tmp/tmp.63RYj811Zy /tmp/tmp.TuZVgpSF00
+ return 0
+ desc 'create secrets and start client'
+ set +o xtrace


-----------------------------------------------------------------------------------
create secrets and start client
-----------------------------------------------------------------------------------

+ kubectl_bin apply -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1961/e2e-tests/conf/secrets.yml -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1961/e2e-tests/conf/client.yml -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1961/e2e-tests/conf/minio-secret.yml
++ mktemp
+ local LAST_OUT=/tmp/tmp.BUqBhMAwi2
++ mktemp
+ local LAST_ERR=/tmp/tmp.2G6oStAHu8
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl apply -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1961/e2e-tests/conf/secrets.yml -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1961/e2e-tests/conf/client.yml -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1961/e2e-tests/conf/minio-secret.yml
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.BUqBhMAwi2
secret/some-users created
deployment.apps/psmdb-client created
secret/minio-secret created
+ cat /tmp/tmp.2G6oStAHu8
+ rm /tmp/tmp.BUqBhMAwi2 /tmp/tmp.2G6oStAHu8
+ return 0
+ cluster=some-name
+ desc 'create first PSMDB cluster some-name'
+ set +o xtrace


-----------------------------------------------------------------------------------
create first PSMDB cluster some-name
-----------------------------------------------------------------------------------

+ apply_cluster /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1961/e2e-tests/pitr-physical-backup-source/conf/some-name-rs0.yml
+ '[' -z '' ']'
+ cat_config /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1961/e2e-tests/pitr-physical-backup-source/conf/some-name-rs0.yml
+ kubectl_bin apply -f -
+ cat /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1961/e2e-tests/pitr-physical-backup-source/conf/some-name-rs0.yml
++ mktemp
+ yq eval '(.spec | select(.image == null)).image = "perconalab/percona-server-mongodb-operator:main-mongod7.0"'
+ yq eval '(.spec | select(has("initImage"))).initImage = "perconalab/percona-server-mongodb-operator:PR-1961-970eeaa4"'
+ yq eval '(.spec | select(has("pmm"))).pmm.image = "perconalab/pmm-client:dev-latest"'
+ yq eval '(.spec | select(has("backup"))).backup.image = "perconalab/percona-server-mongodb-operator:main-backup"'
+ yq eval '.spec.upgradeOptions.apply="Never"'
+ local LAST_OUT=/tmp/tmp.mneuksoxn7
++ mktemp
+ local LAST_ERR=/tmp/tmp.IzWLmDRVug
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl apply -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.mneuksoxn7
perconaservermongodb.psmdb.percona.com/some-name created
+ cat /tmp/tmp.IzWLmDRVug
+ rm /tmp/tmp.mneuksoxn7 /tmp/tmp.IzWLmDRVug
+ return 0
+ desc 'check if all 3 Pods started'
+ set +o xtrace


-----------------------------------------------------------------------------------
check if all 3 Pods started
-----------------------------------------------------------------------------------

+ wait_for_running some-name-rs0 3
+ local name=some-name-rs0
+ let last_pod=2
+ local check_cluster_readyness=true
+ set_debug
+ [[ 1 == 1 ]]
+ set -o xtrace
+ local rs_name=rs0
+ local cluster_name=some-name
++ seq 0 2
+ for i in '$(seq 0 $last_pod)'
+ [[ 0 -eq 2 ]]
+ wait_pod some-name-rs0-0
+ local pod=some-name-rs0-0
+ set +o xtrace
waiting for pod/some-name-rs0-0 to be ready........OK
+ for i in '$(seq 0 $last_pod)'
+ [[ 1 -eq 2 ]]
+ wait_pod some-name-rs0-1
+ local pod=some-name-rs0-1
+ set +o xtrace
waiting for pod/some-name-rs0-1 to be ready........OK
+ for i in '$(seq 0 $last_pod)'
+ [[ 2 -eq 2 ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].arbiter.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.KqEXQGTZpT
+++ mktemp
++ local LAST_ERR=/tmp/tmp.9br3YUy6sJ
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].arbiter.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.KqEXQGTZpT
++ cat /tmp/tmp.9br3YUy6sJ
++ rm /tmp/tmp.KqEXQGTZpT /tmp/tmp.9br3YUy6sJ
++ return 0
+ [[ '' == \t\r\u\e ]]
+ wait_pod some-name-rs0-2
+ local pod=some-name-rs0-2
+ set +o xtrace
waiting for pod/some-name-rs0-2 to be ready......OK
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].non_voting.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.mq9Y1TzNEq
+++ mktemp
++ local LAST_ERR=/tmp/tmp.XWdXO7SE1G
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].non_voting.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.mq9Y1TzNEq
++ cat /tmp/tmp.XWdXO7SE1G
++ rm /tmp/tmp.mq9Y1TzNEq /tmp/tmp.XWdXO7SE1G
++ return 0
+ [[ '' == \t\r\u\e ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].hidden.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.6PoTPfqAhJ
+++ mktemp
++ local LAST_ERR=/tmp/tmp.gJqZq1ZeTP
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].hidden.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.6PoTPfqAhJ
++ cat /tmp/tmp.gJqZq1ZeTP
++ rm /tmp/tmp.6PoTPfqAhJ /tmp/tmp.gJqZq1ZeTP
++ return 0
+ [[ '' == \t\r\u\e ]]
+ sleep 10
+ [[ true == \t\r\u\e ]]
+ set +x
Waiting for cluster readyness..................................................
+ wait_for_running some-name-cfg 3 false
+ local name=some-name-cfg
+ let last_pod=2
+ local check_cluster_readyness=false
+ set_debug
+ [[ 1 == 1 ]]
+ set -o xtrace
+ local rs_name=cfg
+ local cluster_name=some-name
++ seq 0 2
+ for i in '$(seq 0 $last_pod)'
+ [[ 0 -eq 2 ]]
+ wait_pod some-name-cfg-0
+ local pod=some-name-cfg-0
+ set +o xtrace
waiting for pod/some-name-cfg-0 to be ready.OK
+ for i in '$(seq 0 $last_pod)'
+ [[ 1 -eq 2 ]]
+ wait_pod some-name-cfg-1
+ local pod=some-name-cfg-1
+ set +o xtrace
waiting for pod/some-name-cfg-1 to be ready.OK
+ for i in '$(seq 0 $last_pod)'
+ [[ 2 -eq 2 ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].arbiter.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.zuo1hd1Glu
+++ mktemp
++ local LAST_ERR=/tmp/tmp.J6pQ7XTzzK
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].arbiter.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.zuo1hd1Glu
++ cat /tmp/tmp.J6pQ7XTzzK
++ rm /tmp/tmp.zuo1hd1Glu /tmp/tmp.J6pQ7XTzzK
++ return 0
+ [[ '' == \t\r\u\e ]]
+ wait_pod some-name-cfg-2
+ local pod=some-name-cfg-2
+ set +o xtrace
waiting for pod/some-name-cfg-2 to be ready.OK
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].non_voting.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.HJrofEGbLd
+++ mktemp
++ local LAST_ERR=/tmp/tmp.ndNenDSpgW
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].non_voting.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.HJrofEGbLd
++ cat /tmp/tmp.ndNenDSpgW
++ rm /tmp/tmp.HJrofEGbLd /tmp/tmp.ndNenDSpgW
++ return 0
+ [[ '' == \t\r\u\e ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].hidden.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.Gh3IrgOvqi
+++ mktemp
++ local LAST_ERR=/tmp/tmp.KWEoqUWqwM
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].hidden.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.Gh3IrgOvqi
++ cat /tmp/tmp.KWEoqUWqwM
++ rm /tmp/tmp.Gh3IrgOvqi /tmp/tmp.KWEoqUWqwM
++ return 0
+ [[ '' == \t\r\u\e ]]
+ sleep 10
+ [[ false == \t\r\u\e ]]
+ sleep 10
+ write_initial_data
+ desc 'create user myApp'
+ set +o xtrace


-----------------------------------------------------------------------------------
create user myApp
-----------------------------------------------------------------------------------

+ run_mongos 'db.createUser({user:"myApp",pwd:"myPass",roles:[{db:"myApp",role:"readWrite"}]})' userAdmin:userAdmin123456@some-name-mongos.pitr-physical-backup-source-28726
+ local 'command=db.createUser({user:"myApp",pwd:"myPass",roles:[{db:"myApp",role:"readWrite"}]})'
+ local uri=userAdmin:userAdmin123456@some-name-mongos.pitr-physical-backup-source-28726
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local port=27017
+ local mongo_bin=mongo
++ echo .svc.cluster.local
++ awk -F: '{print $2}'
+ suffix_port=
+ [[ -z '' ]]
+ suffix=.svc.cluster.local:27017
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.dlLYcPU5SJ
+++ mktemp
++ local LAST_ERR=/tmp/tmp.dBMjr0YVM0
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.dlLYcPU5SJ
++ cat /tmp/tmp.dBMjr0YVM0
++ rm /tmp/tmp.dlLYcPU5SJ /tmp/tmp.dBMjr0YVM0
++ return 0
+ local client_container=psmdb-client-66f577db5f-d4gt7
+ kubectl_bin exec psmdb-client-66f577db5f-d4gt7 -- bash -c 'printf '\''db.createUser({user:"myApp",pwd:"myPass",roles:[{db:"myApp",role:"readWrite"}]})\n'\'' | mongo mongodb://userAdmin:userAdmin123456@some-name-mongos.pitr-physical-backup-source-28726.svc.cluster.local:27017/admin '
++ mktemp
+ local LAST_OUT=/tmp/tmp.BKJPXTEAGu
++ mktemp
+ local LAST_ERR=/tmp/tmp.M3ios58erQ
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl exec psmdb-client-66f577db5f-d4gt7 -- bash -c 'printf '\''db.createUser({user:"myApp",pwd:"myPass",roles:[{db:"myApp",role:"readWrite"}]})\n'\'' | mongo mongodb://userAdmin:userAdmin123456@some-name-mongos.pitr-physical-backup-source-28726.svc.cluster.local:27017/admin '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.BKJPXTEAGu
Percona Server for MongoDB shell version v4.4.29-28
connecting to: mongodb://some-name-mongos.pitr-physical-backup-source-28726.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb
Implicit session: session { "id" : UUID("d93f6101-4bd4-4d3b-af16-38df76bca15e") }
Percona Server for MongoDB server version: v7.0.22-12
WARNING: shell and server versions do not match
Successfully added user: {
	"user" : "myApp",
	"roles" : [
		{
			"db" : "myApp",
			"role" : "readWrite"
		}
	]
}
bye
+ cat /tmp/tmp.M3ios58erQ
+ rm /tmp/tmp.BKJPXTEAGu /tmp/tmp.M3ios58erQ
+ return 0
+ sleep 2
+ write_document
+ local cmp_postfix=
+ local sleep_value=0
+ log 'write initial data, read from all'
+ set +o xtrace
[2025-08-06T21:40:38+0000] write initial data, read from all
+ run_mongos 'use myApp\n db.test.insert({ x: 100500 })' myApp:myPass@some-name-mongos.pitr-physical-backup-source-28726
+ local 'command=use myApp\n db.test.insert({ x: 100500 })'
+ local uri=myApp:myPass@some-name-mongos.pitr-physical-backup-source-28726
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local port=27017
+ local mongo_bin=mongo
++ echo .svc.cluster.local
++ awk -F: '{print $2}'
+ suffix_port=
+ [[ -z '' ]]
+ suffix=.svc.cluster.local:27017
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.WE5ScSRfDx
+++ mktemp
++ local LAST_ERR=/tmp/tmp.Nf3irgMgGD
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.WE5ScSRfDx
++ cat /tmp/tmp.Nf3irgMgGD
++ rm /tmp/tmp.WE5ScSRfDx /tmp/tmp.Nf3irgMgGD
++ return 0
+ local client_container=psmdb-client-66f577db5f-d4gt7
+ kubectl_bin exec psmdb-client-66f577db5f-d4gt7 -- bash -c 'printf '\''use myApp\n db.test.insert({ x: 100500 })\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.pitr-physical-backup-source-28726.svc.cluster.local:27017/admin '
++ mktemp
+ local LAST_OUT=/tmp/tmp.qmWkOFpLyA
++ mktemp
+ local LAST_ERR=/tmp/tmp.eZOHH7kzIq
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl exec psmdb-client-66f577db5f-d4gt7 -- bash -c 'printf '\''use myApp\n db.test.insert({ x: 100500 })\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.pitr-physical-backup-source-28726.svc.cluster.local:27017/admin '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.qmWkOFpLyA
Percona Server for MongoDB shell version v4.4.29-28
connecting to: mongodb://some-name-mongos.pitr-physical-backup-source-28726.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb
Implicit session: session { "id" : UUID("db4523d5-ba62-4b38-bed6-49f22038d5e2") }
Percona Server for MongoDB server version: v7.0.22-12
WARNING: shell and server versions do not match
switched to db myApp
WriteResult({ "nInserted" : 1 })
bye
+ cat /tmp/tmp.eZOHH7kzIq
+ rm /tmp/tmp.qmWkOFpLyA /tmp/tmp.eZOHH7kzIq
+ return 0
+ sleep 0
+ compare_mongos_cmd find myApp:myPass@some-name-mongos.pitr-physical-backup-source-28726
+ local command=find
+ local uri=myApp:myPass@some-name-mongos.pitr-physical-backup-source-28726
+ local postfix=
+ local suffix=
+ local database=myApp
+ local collection=test
+ local port=27017
+ run_mongos 'use myApp\n db.test.find()' myApp:myPass@some-name-mongos.pitr-physical-backup-source-28726 mongodb '' '' 27017
+ local 'command=use myApp\n db.test.find()'
+ local uri=myApp:myPass@some-name-mongos.pitr-physical-backup-source-28726
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local port=27017
+ local mongo_bin=mongo
+ egrep -v 'I NETWORK|W NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:'
+ /usr/bin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
++ echo .svc.cluster.local
++ awk -F: '{print $2}'
+ suffix_port=
+ [[ -z '' ]]
+ suffix=.svc.cluster.local:27017
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.gC8ReLOsXv
+++ mktemp
++ local LAST_ERR=/tmp/tmp.FlaQ4oDT27
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.gC8ReLOsXv
++ cat /tmp/tmp.FlaQ4oDT27
++ rm /tmp/tmp.gC8ReLOsXv /tmp/tmp.FlaQ4oDT27
++ return 0
+ local client_container=psmdb-client-66f577db5f-d4gt7
+ kubectl_bin exec psmdb-client-66f577db5f-d4gt7 -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.pitr-physical-backup-source-28726.svc.cluster.local:27017/admin '
++ mktemp
+ local LAST_OUT=/tmp/tmp.QmzWxaR6I2
++ mktemp
+ local LAST_ERR=/tmp/tmp.AkBEZpVKP3
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl exec psmdb-client-66f577db5f-d4gt7 -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.pitr-physical-backup-source-28726.svc.cluster.local:27017/admin '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.QmzWxaR6I2
+ cat /tmp/tmp.AkBEZpVKP3
+ rm /tmp/tmp.QmzWxaR6I2 /tmp/tmp.AkBEZpVKP3
+ return 0
+ [[ 0 -eq 0 ]]
+ diff /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1961/e2e-tests/pitr-physical-backup-source/compare/find.json /tmp/tmp.Yk9PLuHVxt/find
+ wait_backup_agent some-name-rs0-0
+ local agent_pod=some-name-rs0-0
+ set +o xtrace
waiting for pbm-agent to be ready in some-name-rs0-0...2025-08-06T21:40:15.000+0000 I listening for the commands

+ wait_backup_agent some-name-rs0-1
+ local agent_pod=some-name-rs0-1
+ set +o xtrace
waiting for pbm-agent to be ready in some-name-rs0-1...2025-08-06T21:40:14.000+0000 I listening for the commands

+ wait_backup_agent some-name-rs0-2
+ local agent_pod=some-name-rs0-2
+ set +o xtrace
waiting for pbm-agent to be ready in some-name-rs0-2...2025-08-06T21:40:14.000+0000 I listening for the commands

+ wait_backup_agent some-name-rs1-0
+ local agent_pod=some-name-rs1-0
+ set +o xtrace
waiting for pbm-agent to be ready in some-name-rs1-0...2025-08-06T21:40:15.000+0000 I listening for the commands

+ wait_backup_agent some-name-rs1-1
+ local agent_pod=some-name-rs1-1
+ set +o xtrace
waiting for pbm-agent to be ready in some-name-rs1-1...2025-08-06T21:40:14.000+0000 I listening for the commands

+ wait_backup_agent some-name-rs1-2
+ local agent_pod=some-name-rs1-2
+ set +o xtrace
waiting for pbm-agent to be ready in some-name-rs1-2...2025-08-06T21:40:15.000+0000 I listening for the commands

+ wait_backup_agent some-name-rs2-0
+ local agent_pod=some-name-rs2-0
+ set +o xtrace
waiting for pbm-agent to be ready in some-name-rs2-0...2025-08-06T21:40:17.000+0000 I listening for the commands

+ wait_backup_agent some-name-rs2-1
+ local agent_pod=some-name-rs2-1
+ set +o xtrace
waiting for pbm-agent to be ready in some-name-rs2-1...2025-08-06T21:40:15.000+0000 I listening for the commands

+ wait_backup_agent some-name-rs2-2
+ local agent_pod=some-name-rs2-2
+ set +o xtrace
waiting for pbm-agent to be ready in some-name-rs2-2...2025-08-06T21:40:15.000+0000 I listening for the commands

+ echo 'Sleeping for 360 seconds'
Sleeping for 360 seconds
+ sleep 360
+ backup_name_minio=backup-minio
+ desc 'restore pitr type date using backupSource'
+ set +o xtrace


-----------------------------------------------------------------------------------
restore pitr type date using backupSource
-----------------------------------------------------------------------------------

+ run_backup backup-minio 1 physical
+ local name=backup-minio
+ local idx=1
+ local type=physical
+ desc 'run backup backup-minio-1'
+ set +o xtrace


-----------------------------------------------------------------------------------
run backup backup-minio-1
-----------------------------------------------------------------------------------

+ cat /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1961/e2e-tests/pitr-physical-backup-source/conf/backup-minio.yml
+ /usr/bin/sed -e 's/name:/name: backup-minio-1/'
+ /usr/bin/sed -e 's/type:/type: physical/'
+ kubectl_bin apply -f -
++ mktemp
+ local LAST_OUT=/tmp/tmp.zEb07BaeNs
++ mktemp
+ local LAST_ERR=/tmp/tmp.if0R8fvInS
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl apply -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.zEb07BaeNs
perconaservermongodbbackup.psmdb.percona.com/backup-minio-1 created
+ cat /tmp/tmp.if0R8fvInS
+ rm /tmp/tmp.zEb07BaeNs /tmp/tmp.if0R8fvInS
+ return 0
+ wait_backup backup-minio-1
+ local backup_name=backup-minio-1
+ local target_state=ready
+ set +o xtrace
waiting for backup-minio-1 to reach ready state..........
+ compare_latest_restorable_time some-name-rs0 backup-minio-1
+ local cluster=some-name-rs0
+ local backup_name=backup-minio-1
+ local latest_restorable_time
+ local backup_time
++ get_latest_restorable_time some-name-rs0
++ local cluster=some-name-rs0
++ local first_timestamp
++ local second_timestamp
++ local retry=0
++ [[ '' != '' ]]
+++ kubectl_bin exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ jq '.backups.pitrChunks.pitrChunks | last | .range.end'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.uOBFtk55dx
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.6SW73vBqlP
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in '$(seq 0 2)'
+++ set +e
+++ kubectl exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.uOBFtk55dx
+++ cat /tmp/tmp.6SW73vBqlP
+++ rm /tmp/tmp.uOBFtk55dx /tmp/tmp.6SW73vBqlP
+++ return 0
++ first_timestamp=null
++ sleep 5
++ [[ null != '' ]]
++ [[ null != \n\u\l\l ]]
++ let retry+=1
++ [[ 1 -gt 30 ]]
++ [[ null != '' ]]
++ [[ null != \n\u\l\l ]]
+++ kubectl_bin exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ jq '.backups.pitrChunks.pitrChunks | last | .range.end'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.6pFKLWvpHH
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.uSZ9DzDUAm
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in '$(seq 0 2)'
+++ set +e
+++ kubectl exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.6pFKLWvpHH
+++ cat /tmp/tmp.uSZ9DzDUAm
+++ rm /tmp/tmp.6pFKLWvpHH /tmp/tmp.uSZ9DzDUAm
+++ return 0
++ first_timestamp=null
++ sleep 5
++ [[ null != '' ]]
++ [[ null != \n\u\l\l ]]
++ let retry+=1
++ [[ 2 -gt 30 ]]
++ [[ null != '' ]]
++ [[ null != \n\u\l\l ]]
+++ kubectl_bin exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ jq '.backups.pitrChunks.pitrChunks | last | .range.end'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.PNuoZl3zGT
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.fKpdx0XoJs
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in '$(seq 0 2)'
+++ set +e
+++ kubectl exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.PNuoZl3zGT
+++ cat /tmp/tmp.fKpdx0XoJs
+++ rm /tmp/tmp.PNuoZl3zGT /tmp/tmp.fKpdx0XoJs
+++ return 0
++ first_timestamp=null
++ sleep 5
++ [[ null != '' ]]
++ [[ null != \n\u\l\l ]]
++ let retry+=1
++ [[ 3 -gt 30 ]]
++ [[ null != '' ]]
++ [[ null != \n\u\l\l ]]
+++ kubectl_bin exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ jq '.backups.pitrChunks.pitrChunks | last | .range.end'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.K14lgTj4An
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.oBhFUGjtWY
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in '$(seq 0 2)'
+++ set +e
+++ kubectl exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.K14lgTj4An
+++ cat /tmp/tmp.oBhFUGjtWY
+++ rm /tmp/tmp.K14lgTj4An /tmp/tmp.oBhFUGjtWY
+++ return 0
++ first_timestamp=null
++ sleep 5
++ [[ null != '' ]]
++ [[ null != \n\u\l\l ]]
++ let retry+=1
++ [[ 4 -gt 30 ]]
++ [[ null != '' ]]
++ [[ null != \n\u\l\l ]]
+++ kubectl_bin exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ jq '.backups.pitrChunks.pitrChunks | last | .range.end'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.XzKKF5mzrp
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.ivN1bDCd2l
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in '$(seq 0 2)'
+++ set +e
+++ kubectl exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.XzKKF5mzrp
+++ cat /tmp/tmp.ivN1bDCd2l
+++ rm /tmp/tmp.XzKKF5mzrp /tmp/tmp.ivN1bDCd2l
+++ return 0
++ first_timestamp=null
++ sleep 5
++ [[ null != '' ]]
++ [[ null != \n\u\l\l ]]
++ let retry+=1
++ [[ 5 -gt 30 ]]
++ [[ null != '' ]]
++ [[ null != \n\u\l\l ]]
+++ jq '.backups.pitrChunks.pitrChunks | last | .range.end'
+++ kubectl_bin exec some-name-rs0-0 -c backup-agent -- pbm status -o json
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.cxsgB6jtvw
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.fXQCHX0Hje
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in '$(seq 0 2)'
+++ set +e
+++ kubectl exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.cxsgB6jtvw
+++ cat /tmp/tmp.fXQCHX0Hje
+++ rm /tmp/tmp.cxsgB6jtvw /tmp/tmp.fXQCHX0Hje
+++ return 0
++ first_timestamp=null
++ sleep 5
++ [[ null != '' ]]
++ [[ null != \n\u\l\l ]]
++ let retry+=1
++ [[ 6 -gt 30 ]]
++ [[ null != '' ]]
++ [[ null != \n\u\l\l ]]
+++ kubectl_bin exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ jq '.backups.pitrChunks.pitrChunks | last | .range.end'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.oCHWkuHj1V
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.HFmAtwlabi
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in '$(seq 0 2)'
+++ set +e
+++ kubectl exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.oCHWkuHj1V
+++ cat /tmp/tmp.HFmAtwlabi
+++ rm /tmp/tmp.oCHWkuHj1V /tmp/tmp.HFmAtwlabi
+++ return 0
++ first_timestamp=null
++ sleep 5
++ [[ null != '' ]]
++ [[ null != \n\u\l\l ]]
++ let retry+=1
++ [[ 7 -gt 30 ]]
++ [[ null != '' ]]
++ [[ null != \n\u\l\l ]]
+++ kubectl_bin exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ jq '.backups.pitrChunks.pitrChunks | last | .range.end'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.0srRFirjmD
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.Kc7X7Au2hf
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in '$(seq 0 2)'
+++ set +e
+++ kubectl exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.0srRFirjmD
+++ cat /tmp/tmp.Kc7X7Au2hf
+++ rm /tmp/tmp.0srRFirjmD /tmp/tmp.Kc7X7Au2hf
+++ return 0
++ first_timestamp=null
++ sleep 5
++ [[ null != '' ]]
++ [[ null != \n\u\l\l ]]
++ let retry+=1
++ [[ 8 -gt 30 ]]
++ [[ null != '' ]]
++ [[ null != \n\u\l\l ]]
+++ kubectl_bin exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ jq '.backups.pitrChunks.pitrChunks | last | .range.end'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.115nRGKr6w
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.OboqSL9yoZ
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in '$(seq 0 2)'
+++ set +e
+++ kubectl exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.115nRGKr6w
+++ cat /tmp/tmp.OboqSL9yoZ
+++ rm /tmp/tmp.115nRGKr6w /tmp/tmp.OboqSL9yoZ
+++ return 0
++ first_timestamp=null
++ sleep 5
++ [[ null != '' ]]
++ [[ null != \n\u\l\l ]]
++ let retry+=1
++ [[ 9 -gt 30 ]]
++ [[ null != '' ]]
++ [[ null != \n\u\l\l ]]
+++ kubectl_bin exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ jq '.backups.pitrChunks.pitrChunks | last | .range.end'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.WgqTZ2S64a
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.O3bK5t6oH3
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in '$(seq 0 2)'
+++ set +e
+++ kubectl exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.WgqTZ2S64a
+++ cat /tmp/tmp.O3bK5t6oH3
+++ rm /tmp/tmp.WgqTZ2S64a /tmp/tmp.O3bK5t6oH3
+++ return 0
++ first_timestamp=null
++ sleep 5
++ [[ null != '' ]]
++ [[ null != \n\u\l\l ]]
++ let retry+=1
++ [[ 10 -gt 30 ]]
++ [[ null != '' ]]
++ [[ null != \n\u\l\l ]]
+++ jq '.backups.pitrChunks.pitrChunks | last | .range.end'
+++ kubectl_bin exec some-name-rs0-0 -c backup-agent -- pbm status -o json
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.0bHK3vsbxk
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.6J3V0axe9j
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in '$(seq 0 2)'
+++ set +e
+++ kubectl exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.0bHK3vsbxk
+++ cat /tmp/tmp.6J3V0axe9j
+++ rm /tmp/tmp.0bHK3vsbxk /tmp/tmp.6J3V0axe9j
+++ return 0
++ first_timestamp=null
++ sleep 5
++ [[ null != '' ]]
++ [[ null != \n\u\l\l ]]
++ let retry+=1
++ [[ 11 -gt 30 ]]
++ [[ null != '' ]]
++ [[ null != \n\u\l\l ]]
+++ kubectl_bin exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ jq '.backups.pitrChunks.pitrChunks | last | .range.end'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.6ofPL7rcq7
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.pldeevhM8Y
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in '$(seq 0 2)'
+++ set +e
+++ kubectl exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.6ofPL7rcq7
+++ cat /tmp/tmp.pldeevhM8Y
+++ rm /tmp/tmp.6ofPL7rcq7 /tmp/tmp.pldeevhM8Y
+++ return 0
++ first_timestamp=null
++ sleep 5
++ [[ null != '' ]]
++ [[ null != \n\u\l\l ]]
++ let retry+=1
++ [[ 12 -gt 30 ]]
++ [[ null != '' ]]
++ [[ null != \n\u\l\l ]]
+++ kubectl_bin exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ jq '.backups.pitrChunks.pitrChunks | last | .range.end'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.IWWl9k0AmQ
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.IttL1NNqZ3
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in '$(seq 0 2)'
+++ set +e
+++ kubectl exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.IWWl9k0AmQ
+++ cat /tmp/tmp.IttL1NNqZ3
+++ rm /tmp/tmp.IWWl9k0AmQ /tmp/tmp.IttL1NNqZ3
+++ return 0
++ first_timestamp=null
++ sleep 5
++ [[ null != '' ]]
++ [[ null != \n\u\l\l ]]
++ let retry+=1
++ [[ 13 -gt 30 ]]
++ [[ null != '' ]]
++ [[ null != \n\u\l\l ]]
+++ jq '.backups.pitrChunks.pitrChunks | last | .range.end'
+++ kubectl_bin exec some-name-rs0-0 -c backup-agent -- pbm status -o json
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.GrkENRoHg8
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.O1YD0InuzO
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in '$(seq 0 2)'
+++ set +e
+++ kubectl exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.GrkENRoHg8
+++ cat /tmp/tmp.O1YD0InuzO
+++ rm /tmp/tmp.GrkENRoHg8 /tmp/tmp.O1YD0InuzO
+++ return 0
++ first_timestamp=null
++ sleep 5
++ [[ null != '' ]]
++ [[ null != \n\u\l\l ]]
++ let retry+=1
++ [[ 14 -gt 30 ]]
++ [[ null != '' ]]
++ [[ null != \n\u\l\l ]]
+++ jq '.backups.pitrChunks.pitrChunks | last | .range.end'
+++ kubectl_bin exec some-name-rs0-0 -c backup-agent -- pbm status -o json
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.q9MAEJ1DRH
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.86MOhkp6Z4
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in '$(seq 0 2)'
+++ set +e
+++ kubectl exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.q9MAEJ1DRH
+++ cat /tmp/tmp.86MOhkp6Z4
+++ rm /tmp/tmp.q9MAEJ1DRH /tmp/tmp.86MOhkp6Z4
+++ return 0
++ first_timestamp=null
++ sleep 5
++ [[ null != '' ]]
++ [[ null != \n\u\l\l ]]
++ let retry+=1
++ [[ 15 -gt 30 ]]
++ [[ null != '' ]]
++ [[ null != \n\u\l\l ]]
+++ jq '.backups.pitrChunks.pitrChunks | last | .range.end'
+++ kubectl_bin exec some-name-rs0-0 -c backup-agent -- pbm status -o json
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.8W1Vj0nnw3
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.vRHLwAa0f7
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in '$(seq 0 2)'
+++ set +e
+++ kubectl exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.8W1Vj0nnw3
+++ cat /tmp/tmp.vRHLwAa0f7
+++ rm /tmp/tmp.8W1Vj0nnw3 /tmp/tmp.vRHLwAa0f7
+++ return 0
++ first_timestamp=null
++ sleep 5
++ [[ null != '' ]]
++ [[ null != \n\u\l\l ]]
++ let retry+=1
++ [[ 16 -gt 30 ]]
++ [[ null != '' ]]
++ [[ null != \n\u\l\l ]]
+++ kubectl_bin exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ jq '.backups.pitrChunks.pitrChunks | last | .range.end'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.jx6mv4Jodf
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.FdjDi7p7Qx
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in '$(seq 0 2)'
+++ set +e
+++ kubectl exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.jx6mv4Jodf
+++ cat /tmp/tmp.FdjDi7p7Qx
+++ rm /tmp/tmp.jx6mv4Jodf /tmp/tmp.FdjDi7p7Qx
+++ return 0
++ first_timestamp=null
++ sleep 5
++ [[ null != '' ]]
++ [[ null != \n\u\l\l ]]
++ let retry+=1
++ [[ 17 -gt 30 ]]
++ [[ null != '' ]]
++ [[ null != \n\u\l\l ]]
+++ kubectl_bin exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ jq '.backups.pitrChunks.pitrChunks | last | .range.end'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.frUcotPJVo
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.ROwLoBGm9s
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in '$(seq 0 2)'
+++ set +e
+++ kubectl exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.frUcotPJVo
+++ cat /tmp/tmp.ROwLoBGm9s
+++ rm /tmp/tmp.frUcotPJVo /tmp/tmp.ROwLoBGm9s
+++ return 0
++ first_timestamp=null
++ sleep 5
++ [[ null != '' ]]
++ [[ null != \n\u\l\l ]]
++ let retry+=1
++ [[ 18 -gt 30 ]]
++ [[ null != '' ]]
++ [[ null != \n\u\l\l ]]
+++ kubectl_bin exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ jq '.backups.pitrChunks.pitrChunks | last | .range.end'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.SPrrN0zTpe
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.H1Qsv4CyvQ
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in '$(seq 0 2)'
+++ set +e
+++ kubectl exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.SPrrN0zTpe
+++ cat /tmp/tmp.H1Qsv4CyvQ
+++ rm /tmp/tmp.SPrrN0zTpe /tmp/tmp.H1Qsv4CyvQ
+++ return 0
++ first_timestamp=null
++ sleep 5
++ [[ null != '' ]]
++ [[ null != \n\u\l\l ]]
++ let retry+=1
++ [[ 19 -gt 30 ]]
++ [[ null != '' ]]
++ [[ null != \n\u\l\l ]]
+++ kubectl_bin exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ jq '.backups.pitrChunks.pitrChunks | last | .range.end'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.6iPDwwkuCM
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.eyVvBjeX9z
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in '$(seq 0 2)'
+++ set +e
+++ kubectl exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.6iPDwwkuCM
+++ cat /tmp/tmp.eyVvBjeX9z
+++ rm /tmp/tmp.6iPDwwkuCM /tmp/tmp.eyVvBjeX9z
+++ return 0
++ first_timestamp=null
++ sleep 5
++ [[ null != '' ]]
++ [[ null != \n\u\l\l ]]
++ let retry+=1
++ [[ 20 -gt 30 ]]
++ [[ null != '' ]]
++ [[ null != \n\u\l\l ]]
+++ kubectl_bin exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ jq '.backups.pitrChunks.pitrChunks | last | .range.end'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.slEd67AsLm
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.xbIf4DrilJ
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in '$(seq 0 2)'
+++ set +e
+++ kubectl exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.slEd67AsLm
+++ cat /tmp/tmp.xbIf4DrilJ
+++ rm /tmp/tmp.slEd67AsLm /tmp/tmp.xbIf4DrilJ
+++ return 0
++ first_timestamp=null
++ sleep 5
++ [[ null != '' ]]
++ [[ null != \n\u\l\l ]]
++ let retry+=1
++ [[ 21 -gt 30 ]]
++ [[ null != '' ]]
++ [[ null != \n\u\l\l ]]
+++ kubectl_bin exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ jq '.backups.pitrChunks.pitrChunks | last | .range.end'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.PMswcvh05J
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.K9JouNYjz2
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in '$(seq 0 2)'
+++ set +e
+++ kubectl exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.PMswcvh05J
+++ cat /tmp/tmp.K9JouNYjz2
+++ rm /tmp/tmp.PMswcvh05J /tmp/tmp.K9JouNYjz2
+++ return 0
++ first_timestamp=1754516972
++ sleep 5
++ [[ 1754516972 != '' ]]
++ [[ 1754516972 != \n\u\l\l ]]
+++ jq '.backups.pitrChunks.pitrChunks | last | .range.end'
+++ kubectl_bin exec some-name-rs0-0 -c backup-agent -- pbm status -o json
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.p0laGaMC4Q
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.iGkC4bRtZp
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in '$(seq 0 2)'
+++ set +e
+++ kubectl exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.p0laGaMC4Q
+++ cat /tmp/tmp.iGkC4bRtZp
+++ rm /tmp/tmp.p0laGaMC4Q /tmp/tmp.iGkC4bRtZp
+++ return 0
++ second_timestamp=1754516961
++ let retry+=1
++ [[ 22 -gt 30 ]]
++ [[ 1754516972 != '' ]]
++ [[ 1754516972 != \n\u\l\l ]]
++ [[ 1754516972 == 1754516961 ]]
+++ kubectl_bin exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ jq '.backups.pitrChunks.pitrChunks | last | .range.end'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.IuZP5pCnlR
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.fvzaom1vxl
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in '$(seq 0 2)'
+++ set +e
+++ kubectl exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.IuZP5pCnlR
+++ cat /tmp/tmp.fvzaom1vxl
+++ rm /tmp/tmp.IuZP5pCnlR /tmp/tmp.fvzaom1vxl
+++ return 0
++ first_timestamp=1754516961
++ sleep 5
++ [[ 1754516961 != '' ]]
++ [[ 1754516961 != \n\u\l\l ]]
+++ kubectl_bin exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ jq '.backups.pitrChunks.pitrChunks | last | .range.end'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.TBEoc2o6Te
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.QecyA6kRux
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in '$(seq 0 2)'
+++ set +e
+++ kubectl exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.TBEoc2o6Te
+++ cat /tmp/tmp.QecyA6kRux
+++ rm /tmp/tmp.TBEoc2o6Te /tmp/tmp.QecyA6kRux
+++ return 0
++ second_timestamp=1754516961
++ let retry+=1
++ [[ 23 -gt 30 ]]
++ [[ 1754516961 != '' ]]
++ [[ 1754516961 != \n\u\l\l ]]
++ [[ 1754516961 == 1754516961 ]]
++ /usr/bin/date -u -d @1754516961 +%Y-%m-%dT%H:%M:%SZ
+ latest_restorable_time=2025-08-06T21:49:21Z
++ get_latest_restorable_time_from_backup_object backup-minio-1
++ local backup_name=backup-minio-1
++ local latestRestorableTime
++ local retry=0
++ [[ '' != '' ]]
++ sleep 5
+++ kubectl_bin get psmdb-backup backup-minio-1 -o 'jsonpath={.status.latestRestorableTime}'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.Rx40VxcPBM
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.VsEkvBpbjj
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in '$(seq 0 2)'
+++ set +e
+++ kubectl get psmdb-backup backup-minio-1 -o 'jsonpath={.status.latestRestorableTime}'
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.Rx40VxcPBM
+++ cat /tmp/tmp.VsEkvBpbjj
+++ rm /tmp/tmp.Rx40VxcPBM /tmp/tmp.VsEkvBpbjj
+++ return 0
++ latestRestorableTime=2025-08-06T21:49:21Z
++ let retry+=1
++ [[ 1 -gt 30 ]]
++ [[ 2025-08-06T21:49:21Z != '' ]]
++ [[ 2025-08-06T21:49:21Z != \n\u\l\l ]]
++ echo 2025-08-06T21:49:21Z
+ backup_time=2025-08-06T21:49:21Z
+ [[ 2025-08-06T21:49:21Z != \2\0\2\5\-\0\8\-\0\6\T\2\1\:\4\9\:\2\1\Z ]]
+ reset_collection
+ desc 'reset data'
+ set +o xtrace


-----------------------------------------------------------------------------------
reset data
-----------------------------------------------------------------------------------

+ run_mongos 'use myApp\n db.test.remove({})' myApp:myPass@some-name-mongos.pitr-physical-backup-source-28726
+ local 'command=use myApp\n db.test.remove({})'
+ local uri=myApp:myPass@some-name-mongos.pitr-physical-backup-source-28726
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local port=27017
+ local mongo_bin=mongo
++ echo .svc.cluster.local
++ awk -F: '{print $2}'
+ suffix_port=
+ [[ -z '' ]]
+ suffix=.svc.cluster.local:27017
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.H1MIsZLiNx
+++ mktemp
++ local LAST_ERR=/tmp/tmp.UEi0lX4m6X
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.H1MIsZLiNx
++ cat /tmp/tmp.UEi0lX4m6X
++ rm /tmp/tmp.H1MIsZLiNx /tmp/tmp.UEi0lX4m6X
++ return 0
+ local client_container=psmdb-client-66f577db5f-d4gt7
+ kubectl_bin exec psmdb-client-66f577db5f-d4gt7 -- bash -c 'printf '\''use myApp\n db.test.remove({})\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.pitr-physical-backup-source-28726.svc.cluster.local:27017/admin '
++ mktemp
+ local LAST_OUT=/tmp/tmp.lXAUd3FhkJ
++ mktemp
+ local LAST_ERR=/tmp/tmp.9RHnTFo2Bu
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl exec psmdb-client-66f577db5f-d4gt7 -- bash -c 'printf '\''use myApp\n db.test.remove({})\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.pitr-physical-backup-source-28726.svc.cluster.local:27017/admin '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.lXAUd3FhkJ
Percona Server for MongoDB shell version v4.4.29-28
connecting to: mongodb://some-name-mongos.pitr-physical-backup-source-28726.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb
Implicit session: session { "id" : UUID("abbad160-4d96-46c8-b57c-e3a255f761a3") }
Percona Server for MongoDB server version: v7.0.22-12
WARNING: shell and server versions do not match
switched to db myApp
WriteResult({ "nRemoved" : 1 })
bye
+ cat /tmp/tmp.9RHnTFo2Bu
+ rm /tmp/tmp.lXAUd3FhkJ /tmp/tmp.9RHnTFo2Bu
+ return 0
+ sleep 2
+ write_document '' 120
+ local cmp_postfix=
+ local sleep_value=120
+ log 'write initial data, read from all'
+ set +o xtrace
[2025-08-06T21:50:02+0000] write initial data, read from all
+ run_mongos 'use myApp\n db.test.insert({ x: 100500 })' myApp:myPass@some-name-mongos.pitr-physical-backup-source-28726
+ local 'command=use myApp\n db.test.insert({ x: 100500 })'
+ local uri=myApp:myPass@some-name-mongos.pitr-physical-backup-source-28726
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local port=27017
+ local mongo_bin=mongo
++ awk -F: '{print $2}'
++ echo .svc.cluster.local
+ suffix_port=
+ [[ -z '' ]]
+ suffix=.svc.cluster.local:27017
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.aRlK5gtPFl
+++ mktemp
++ local LAST_ERR=/tmp/tmp.CuvrfFHCDQ
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.aRlK5gtPFl
++ cat /tmp/tmp.CuvrfFHCDQ
++ rm /tmp/tmp.aRlK5gtPFl /tmp/tmp.CuvrfFHCDQ
++ return 0
+ local client_container=psmdb-client-66f577db5f-d4gt7
+ kubectl_bin exec psmdb-client-66f577db5f-d4gt7 -- bash -c 'printf '\''use myApp\n db.test.insert({ x: 100500 })\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.pitr-physical-backup-source-28726.svc.cluster.local:27017/admin '
++ mktemp
+ local LAST_OUT=/tmp/tmp.Gtpbi7aoP9
++ mktemp
+ local LAST_ERR=/tmp/tmp.qOVI21vs6Q
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl exec psmdb-client-66f577db5f-d4gt7 -- bash -c 'printf '\''use myApp\n db.test.insert({ x: 100500 })\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.pitr-physical-backup-source-28726.svc.cluster.local:27017/admin '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.Gtpbi7aoP9
Percona Server for MongoDB shell version v4.4.29-28
connecting to: mongodb://some-name-mongos.pitr-physical-backup-source-28726.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb
Implicit session: session { "id" : UUID("54727ff5-1896-4a9d-8f79-7d2b19e0f48e") }
Percona Server for MongoDB server version: v7.0.22-12
WARNING: shell and server versions do not match
switched to db myApp
WriteResult({ "nInserted" : 1 })
bye
+ cat /tmp/tmp.qOVI21vs6Q
+ rm /tmp/tmp.Gtpbi7aoP9 /tmp/tmp.qOVI21vs6Q
+ return 0
+ sleep 120
+ compare_mongos_cmd find myApp:myPass@some-name-mongos.pitr-physical-backup-source-28726
+ local command=find
+ local uri=myApp:myPass@some-name-mongos.pitr-physical-backup-source-28726
+ local postfix=
+ local suffix=
+ local database=myApp
+ local collection=test
+ local port=27017
+ run_mongos 'use myApp\n db.test.find()' myApp:myPass@some-name-mongos.pitr-physical-backup-source-28726 mongodb '' '' 27017
+ egrep -v 'I NETWORK|W NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:'
+ local 'command=use myApp\n db.test.find()'
+ local uri=myApp:myPass@some-name-mongos.pitr-physical-backup-source-28726
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local port=27017
+ local mongo_bin=mongo
+ /usr/bin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
++ echo .svc.cluster.local
++ awk -F: '{print $2}'
+ suffix_port=
+ [[ -z '' ]]
+ suffix=.svc.cluster.local:27017
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.udWktJR7xa
+++ mktemp
++ local LAST_ERR=/tmp/tmp.Enb4tQAKAd
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.udWktJR7xa
++ cat /tmp/tmp.Enb4tQAKAd
++ rm /tmp/tmp.udWktJR7xa /tmp/tmp.Enb4tQAKAd
++ return 0
+ local client_container=psmdb-client-66f577db5f-d4gt7
+ kubectl_bin exec psmdb-client-66f577db5f-d4gt7 -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.pitr-physical-backup-source-28726.svc.cluster.local:27017/admin '
++ mktemp
+ local LAST_OUT=/tmp/tmp.y1NO8xznob
++ mktemp
+ local LAST_ERR=/tmp/tmp.yJejqXmb65
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl exec psmdb-client-66f577db5f-d4gt7 -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.pitr-physical-backup-source-28726.svc.cluster.local:27017/admin '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.y1NO8xznob
+ cat /tmp/tmp.yJejqXmb65
+ rm /tmp/tmp.y1NO8xznob /tmp/tmp.yJejqXmb65
+ return 0
+ [[ 0 -eq 0 ]]
+ diff /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1961/e2e-tests/pitr-physical-backup-source/compare/find.json /tmp/tmp.Yk9PLuHVxt/find
++ run_mongos 'new Date().getTime() / 1000' myApp:myPass@some-name-mongos.pitr-physical-backup-source-28726 mongodb '' --quiet
++ local 'command=new Date().getTime() / 1000'
++ grep -E -v 'I NETWORK|W NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match'
++ local uri=myApp:myPass@some-name-mongos.pitr-physical-backup-source-28726
++ local driver=mongodb
++ cut -d. -f1
++ local suffix=.svc.cluster.local
++ local mongo_flag=--quiet
++ local port=27017
++ local mongo_bin=mongo
+++ awk -F: '{print $2}'
+++ echo .svc.cluster.local
++ suffix_port=
++ [[ -z '' ]]
++ suffix=.svc.cluster.local:27017
+++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.oVC8ZIHfng
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.9w46Vl2vNj
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in '$(seq 0 2)'
+++ set +e
+++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.oVC8ZIHfng
+++ cat /tmp/tmp.9w46Vl2vNj
+++ rm /tmp/tmp.oVC8ZIHfng /tmp/tmp.9w46Vl2vNj
+++ return 0
++ local client_container=psmdb-client-66f577db5f-d4gt7
++ kubectl_bin exec psmdb-client-66f577db5f-d4gt7 -- bash -c 'printf '\''new Date().getTime() / 1000\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.pitr-physical-backup-source-28726.svc.cluster.local:27017/admin --quiet'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.zAHhKsrV9p
+++ mktemp
++ local LAST_ERR=/tmp/tmp.VlkNqYCnIU
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl exec psmdb-client-66f577db5f-d4gt7 -- bash -c 'printf '\''new Date().getTime() / 1000\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.pitr-physical-backup-source-28726.svc.cluster.local:27017/admin --quiet'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.zAHhKsrV9p
++ cat /tmp/tmp.VlkNqYCnIU
++ rm /tmp/tmp.zAHhKsrV9p /tmp/tmp.VlkNqYCnIU
++ return 0
+ time_now=1754517129
+ check_recovery backup-minio-1 date 1754517129 '' some-name backupSource
+ local backup_name=backup-minio-1
+ local restore_type=date
+ local restore_date=1754517129
+ local cmp_postfix=
+ local cluster_name=some-name
+ local backupSource=backupSource
++ get_latest_oplog_chunk_ts some-name
++ local cluster=some-name
+++ kubectl_bin exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ jq '.backups.pitrChunks.pitrChunks | last | .range.end'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.AfJtna5Stk
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.twFqYlaZOt
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in '$(seq 0 2)'
+++ set +e
+++ kubectl exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.AfJtna5Stk
+++ cat /tmp/tmp.twFqYlaZOt
+++ rm /tmp/tmp.AfJtna5Stk /tmp/tmp.twFqYlaZOt
+++ return 0
++ echo 1754517088
+ local latest_ts=1754517088
+ desc 'write more data before restore by date'
+ set +o xtrace


-----------------------------------------------------------------------------------
write more data before restore by date
-----------------------------------------------------------------------------------

+ run_mongos 'use myApp\n db.test.insert({ x: 100501 })' myApp:myPass@some-name-mongos.pitr-physical-backup-source-28726
+ local 'command=use myApp\n db.test.insert({ x: 100501 })'
+ local uri=myApp:myPass@some-name-mongos.pitr-physical-backup-source-28726
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local port=27017
+ local mongo_bin=mongo
++ echo .svc.cluster.local
++ awk -F: '{print $2}'
+ suffix_port=
+ [[ -z '' ]]
+ suffix=.svc.cluster.local:27017
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.RyHCdZwR3R
+++ mktemp
++ local LAST_ERR=/tmp/tmp.laJfJnVC4T
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.RyHCdZwR3R
++ cat /tmp/tmp.laJfJnVC4T
++ rm /tmp/tmp.RyHCdZwR3R /tmp/tmp.laJfJnVC4T
++ return 0
+ local client_container=psmdb-client-66f577db5f-d4gt7
+ kubectl_bin exec psmdb-client-66f577db5f-d4gt7 -- bash -c 'printf '\''use myApp\n db.test.insert({ x: 100501 })\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.pitr-physical-backup-source-28726.svc.cluster.local:27017/admin '
++ mktemp
+ local LAST_OUT=/tmp/tmp.q56v3Uzapb
++ mktemp
+ local LAST_ERR=/tmp/tmp.XIMDrVr06e
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl exec psmdb-client-66f577db5f-d4gt7 -- bash -c 'printf '\''use myApp\n db.test.insert({ x: 100501 })\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.pitr-physical-backup-source-28726.svc.cluster.local:27017/admin '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.q56v3Uzapb
Percona Server for MongoDB shell version v4.4.29-28
connecting to: mongodb://some-name-mongos.pitr-physical-backup-source-28726.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb
Implicit session: session { "id" : UUID("e1c2c1c9-0fc7-4e97-b5fa-19da7ca5588a") }
Percona Server for MongoDB server version: v7.0.22-12
WARNING: shell and server versions do not match
switched to db myApp
WriteResult({ "nInserted" : 1 })
bye
+ cat /tmp/tmp.XIMDrVr06e
+ rm /tmp/tmp.q56v3Uzapb /tmp/tmp.XIMDrVr06e
+ return 0
+ [[ -n 1754517129 ]]
++ format_date 1754517129
++ local timestamp=1754517129
+++ TZ=UTC
+++ /usr/bin/date -d@1754517129 '+%Y-%m-%d %H:%M:%S'
++ echo 2025-08-06 21:52:09
+ desc 'Restoring to time 2025-08-06 21:52:09'
+ set +o xtrace


-----------------------------------------------------------------------------------
Restoring to time 2025-08-06 21:52:09
-----------------------------------------------------------------------------------

+ retries=0
+ [[ 1754517088 -gt 1754517129 ]]
+ [[ 0 -gt 30 ]]
++ get_latest_oplog_chunk_ts some-name
++ local cluster=some-name
+++ kubectl_bin exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ jq '.backups.pitrChunks.pitrChunks | last | .range.end'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.FO21m1a1BL
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.Umw33HiD59
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in '$(seq 0 2)'
+++ set +e
+++ kubectl exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.FO21m1a1BL
+++ cat /tmp/tmp.Umw33HiD59
+++ rm /tmp/tmp.FO21m1a1BL /tmp/tmp.Umw33HiD59
+++ return 0
++ echo 1754517088
+ latest_ts=1754517088
+ retries=1
++ format_date 1754517088
++ local timestamp=1754517088
+++ TZ=UTC
+++ /usr/bin/date -d@1754517088 '+%Y-%m-%d %H:%M:%S'
++ echo 2025-08-06 21:51:28
++ format_date 1754517129
++ local timestamp=1754517129
+++ TZ=UTC
+++ /usr/bin/date -d@1754517129 '+%Y-%m-%d %H:%M:%S'
++ echo 2025-08-06 21:52:09
+ echo 'Waiting for last oplog chunk (2025-08-06 21:51:28) to be greater than restore target (2025-08-06 21:52:09)'
Waiting for last oplog chunk (2025-08-06 21:51:28) to be greater than restore target (2025-08-06 21:52:09)
+ sleep 10
+ [[ 1754517088 -gt 1754517129 ]]
+ [[ 1 -gt 30 ]]
++ get_latest_oplog_chunk_ts some-name
++ local cluster=some-name
+++ kubectl_bin exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ jq '.backups.pitrChunks.pitrChunks | last | .range.end'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.Gpf8UxLFoD
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.upKeHTRUQz
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in '$(seq 0 2)'
+++ set +e
+++ kubectl exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.Gpf8UxLFoD
+++ cat /tmp/tmp.upKeHTRUQz
+++ rm /tmp/tmp.Gpf8UxLFoD /tmp/tmp.upKeHTRUQz
+++ return 0
++ echo 1754517088
+ latest_ts=1754517088
+ retries=2
++ format_date 1754517088
++ local timestamp=1754517088
+++ TZ=UTC
+++ /usr/bin/date -d@1754517088 '+%Y-%m-%d %H:%M:%S'
++ echo 2025-08-06 21:51:28
++ format_date 1754517129
++ local timestamp=1754517129
+++ TZ=UTC
+++ /usr/bin/date -d@1754517129 '+%Y-%m-%d %H:%M:%S'
++ echo 2025-08-06 21:52:09
+ echo 'Waiting for last oplog chunk (2025-08-06 21:51:28) to be greater than restore target (2025-08-06 21:52:09)'
Waiting for last oplog chunk (2025-08-06 21:51:28) to be greater than restore target (2025-08-06 21:52:09)
+ sleep 10
+ [[ 1754517088 -gt 1754517129 ]]
+ [[ 2 -gt 30 ]]
++ get_latest_oplog_chunk_ts some-name
++ local cluster=some-name
+++ kubectl_bin exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ jq '.backups.pitrChunks.pitrChunks | last | .range.end'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.EM2qX7AGmL
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.TScyAMottC
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in '$(seq 0 2)'
+++ set +e
+++ kubectl exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.EM2qX7AGmL
+++ cat /tmp/tmp.TScyAMottC
+++ rm /tmp/tmp.EM2qX7AGmL /tmp/tmp.TScyAMottC
+++ return 0
++ echo 1754517088
+ latest_ts=1754517088
+ retries=3
++ format_date 1754517088
++ local timestamp=1754517088
+++ TZ=UTC
+++ /usr/bin/date -d@1754517088 '+%Y-%m-%d %H:%M:%S'
++ echo 2025-08-06 21:51:28
++ format_date 1754517129
++ local timestamp=1754517129
+++ TZ=UTC
+++ /usr/bin/date -d@1754517129 '+%Y-%m-%d %H:%M:%S'
++ echo 2025-08-06 21:52:09
+ echo 'Waiting for last oplog chunk (2025-08-06 21:51:28) to be greater than restore target (2025-08-06 21:52:09)'
Waiting for last oplog chunk (2025-08-06 21:51:28) to be greater than restore target (2025-08-06 21:52:09)
+ sleep 10
+ [[ 1754517088 -gt 1754517129 ]]
+ [[ 3 -gt 30 ]]
++ get_latest_oplog_chunk_ts some-name
++ local cluster=some-name
+++ kubectl_bin exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ jq '.backups.pitrChunks.pitrChunks | last | .range.end'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.nttb5P5FqM
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.ii2iRxT0MS
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in '$(seq 0 2)'
+++ set +e
+++ kubectl exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.nttb5P5FqM
+++ cat /tmp/tmp.ii2iRxT0MS
+++ rm /tmp/tmp.nttb5P5FqM /tmp/tmp.ii2iRxT0MS
+++ return 0
++ echo 1754517088
+ latest_ts=1754517088
+ retries=4
++ format_date 1754517088
++ local timestamp=1754517088
+++ TZ=UTC
+++ /usr/bin/date -d@1754517088 '+%Y-%m-%d %H:%M:%S'
++ echo 2025-08-06 21:51:28
++ format_date 1754517129
++ local timestamp=1754517129
+++ TZ=UTC
+++ /usr/bin/date -d@1754517129 '+%Y-%m-%d %H:%M:%S'
++ echo 2025-08-06 21:52:09
+ echo 'Waiting for last oplog chunk (2025-08-06 21:51:28) to be greater than restore target (2025-08-06 21:52:09)'
Waiting for last oplog chunk (2025-08-06 21:51:28) to be greater than restore target (2025-08-06 21:52:09)
+ sleep 10
+ [[ 1754517088 -gt 1754517129 ]]
+ [[ 4 -gt 30 ]]
++ get_latest_oplog_chunk_ts some-name
++ local cluster=some-name
+++ jq '.backups.pitrChunks.pitrChunks | last | .range.end'
+++ kubectl_bin exec some-name-rs0-0 -c backup-agent -- pbm status -o json
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.ItFJQEZKzb
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.Awt9PkQTMS
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in '$(seq 0 2)'
+++ set +e
+++ kubectl exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.ItFJQEZKzb
+++ cat /tmp/tmp.Awt9PkQTMS
+++ rm /tmp/tmp.ItFJQEZKzb /tmp/tmp.Awt9PkQTMS
+++ return 0
++ echo 1754517088
+ latest_ts=1754517088
+ retries=5
++ format_date 1754517088
++ local timestamp=1754517088
+++ TZ=UTC
+++ /usr/bin/date -d@1754517088 '+%Y-%m-%d %H:%M:%S'
++ echo 2025-08-06 21:51:28
++ format_date 1754517129
++ local timestamp=1754517129
+++ TZ=UTC
+++ /usr/bin/date -d@1754517129 '+%Y-%m-%d %H:%M:%S'
++ echo 2025-08-06 21:52:09
+ echo 'Waiting for last oplog chunk (2025-08-06 21:51:28) to be greater than restore target (2025-08-06 21:52:09)'
Waiting for last oplog chunk (2025-08-06 21:51:28) to be greater than restore target (2025-08-06 21:52:09)
+ sleep 10
+ [[ 1754517088 -gt 1754517129 ]]
+ [[ 5 -gt 30 ]]
++ get_latest_oplog_chunk_ts some-name
++ local cluster=some-name
+++ kubectl_bin exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ jq '.backups.pitrChunks.pitrChunks | last | .range.end'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.mcGKXLlgn0
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.ojydtAIXDD
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in '$(seq 0 2)'
+++ set +e
+++ kubectl exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.mcGKXLlgn0
+++ cat /tmp/tmp.ojydtAIXDD
+++ rm /tmp/tmp.mcGKXLlgn0 /tmp/tmp.ojydtAIXDD
+++ return 0
++ echo 1754517088
+ latest_ts=1754517088
+ retries=6
++ format_date 1754517088
++ local timestamp=1754517088
+++ TZ=UTC
+++ /usr/bin/date -d@1754517088 '+%Y-%m-%d %H:%M:%S'
++ echo 2025-08-06 21:51:28
++ format_date 1754517129
++ local timestamp=1754517129
+++ TZ=UTC
+++ /usr/bin/date -d@1754517129 '+%Y-%m-%d %H:%M:%S'
++ echo 2025-08-06 21:52:09
+ echo 'Waiting for last oplog chunk (2025-08-06 21:51:28) to be greater than restore target (2025-08-06 21:52:09)'
Waiting for last oplog chunk (2025-08-06 21:51:28) to be greater than restore target (2025-08-06 21:52:09)
+ sleep 10
+ [[ 1754517088 -gt 1754517129 ]]
+ [[ 6 -gt 30 ]]
++ get_latest_oplog_chunk_ts some-name
++ local cluster=some-name
+++ kubectl_bin exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ jq '.backups.pitrChunks.pitrChunks | last | .range.end'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.IlYf6S1aql
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.m8rC94tNuq
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in '$(seq 0 2)'
+++ set +e
+++ kubectl exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.IlYf6S1aql
+++ cat /tmp/tmp.m8rC94tNuq
+++ rm /tmp/tmp.IlYf6S1aql /tmp/tmp.m8rC94tNuq
+++ return 0
++ echo 1754517088
+ latest_ts=1754517088
+ retries=7
++ format_date 1754517088
++ local timestamp=1754517088
+++ TZ=UTC
+++ /usr/bin/date -d@1754517088 '+%Y-%m-%d %H:%M:%S'
++ echo 2025-08-06 21:51:28
++ format_date 1754517129
++ local timestamp=1754517129
+++ TZ=UTC
+++ /usr/bin/date -d@1754517129 '+%Y-%m-%d %H:%M:%S'
++ echo 2025-08-06 21:52:09
+ echo 'Waiting for last oplog chunk (2025-08-06 21:51:28) to be greater than restore target (2025-08-06 21:52:09)'
Waiting for last oplog chunk (2025-08-06 21:51:28) to be greater than restore target (2025-08-06 21:52:09)
+ sleep 10
+ [[ 1754517088 -gt 1754517129 ]]
+ [[ 7 -gt 30 ]]
++ get_latest_oplog_chunk_ts some-name
++ local cluster=some-name
+++ kubectl_bin exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ jq '.backups.pitrChunks.pitrChunks | last | .range.end'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.kheCsUv4SB
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.lpEoXqg7H8
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in '$(seq 0 2)'
+++ set +e
+++ kubectl exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.kheCsUv4SB
+++ cat /tmp/tmp.lpEoXqg7H8
+++ rm /tmp/tmp.kheCsUv4SB /tmp/tmp.lpEoXqg7H8
+++ return 0
++ echo 1754517088
+ latest_ts=1754517088
+ retries=8
++ format_date 1754517088
++ local timestamp=1754517088
+++ TZ=UTC
+++ /usr/bin/date -d@1754517088 '+%Y-%m-%d %H:%M:%S'
++ echo 2025-08-06 21:51:28
++ format_date 1754517129
++ local timestamp=1754517129
+++ TZ=UTC
+++ /usr/bin/date -d@1754517129 '+%Y-%m-%d %H:%M:%S'
++ echo 2025-08-06 21:52:09
+ echo 'Waiting for last oplog chunk (2025-08-06 21:51:28) to be greater than restore target (2025-08-06 21:52:09)'
Waiting for last oplog chunk (2025-08-06 21:51:28) to be greater than restore target (2025-08-06 21:52:09)
+ sleep 10
+ [[ 1754517088 -gt 1754517129 ]]
+ [[ 8 -gt 30 ]]
++ get_latest_oplog_chunk_ts some-name
++ local cluster=some-name
+++ kubectl_bin exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ jq '.backups.pitrChunks.pitrChunks | last | .range.end'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.rdLsLdszx9
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.xZKpZlky18
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in '$(seq 0 2)'
+++ set +e
+++ kubectl exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.rdLsLdszx9
+++ cat /tmp/tmp.xZKpZlky18
+++ rm /tmp/tmp.rdLsLdszx9 /tmp/tmp.xZKpZlky18
+++ return 0
++ echo 1754517206
+ latest_ts=1754517206
+ retries=9
++ format_date 1754517206
++ local timestamp=1754517206
+++ TZ=UTC
+++ /usr/bin/date -d@1754517206 '+%Y-%m-%d %H:%M:%S'
++ echo 2025-08-06 21:53:26
++ format_date 1754517129
++ local timestamp=1754517129
+++ TZ=UTC
+++ /usr/bin/date -d@1754517129 '+%Y-%m-%d %H:%M:%S'
++ echo 2025-08-06 21:52:09
+ echo 'Waiting for last oplog chunk (2025-08-06 21:53:26) to be greater than restore target (2025-08-06 21:52:09)'
Waiting for last oplog chunk (2025-08-06 21:53:26) to be greater than restore target (2025-08-06 21:52:09)
+ sleep 10
+ [[ 1754517206 -gt 1754517129 ]]
+ '[' -z backupSource ']'
+ desc 'check restore by date backupSource'
+ set +o xtrace


-----------------------------------------------------------------------------------
check restore by date backupSource
-----------------------------------------------------------------------------------

++ get_backup_dest backup-minio-1
++ local backup_name=backup-minio-1
++ kubectl_bin get psmdb-backup backup-minio-1 -o 'jsonpath={.status.destination}'
++ sed 's|s3://||'
++ sed -e 's/.json$//'
++ sed 's|azure://||'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.jcyZNFQnxk
+++ mktemp
++ local LAST_ERR=/tmp/tmp.ZBwzoRy5Ak
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb-backup backup-minio-1 -o 'jsonpath={.status.destination}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.jcyZNFQnxk
++ cat /tmp/tmp.ZBwzoRy5Ak
++ rm /tmp/tmp.jcyZNFQnxk /tmp/tmp.ZBwzoRy5Ak
++ return 0
+ backup_dest=operator-testing/2025-08-06T21:46:53Z
+ cat /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1961/e2e-tests/pitr-physical-backup-source/conf/restore.yml
+ /usr/bin/sed -e 's/name:/name: restore-backup-minio-1/'
+ /usr/bin/sed -e 's/pitrType:/type: date/'
+ /usr/bin/sed -e /backupName/d
+ '[' -z 1754517129 ']'
+ /usr/bin/sed -e 's|DESTINATION|operator-testing/2025-08-06T21:46:53Z|'
+ '[' -n '' ']'
+ yq
+ kubectl_bin apply -f -
++ get_bucket_name backup-minio-1
++ local backup_name=backup-minio-1
++ kubectl_bin get psmdb-backup backup-minio-1 -o 'jsonpath={.status.s3.bucket}'
++ mktemp
+++ mktemp
++ format_date 1754517129
++ local timestamp=1754517129
+++ TZ=UTC
+++ /usr/bin/date -d@1754517129 '+%Y-%m-%d %H:%M:%S'
+ local LAST_OUT=/tmp/tmp.iWaebaocMy
++ mktemp
++ local LAST_OUT=/tmp/tmp.h96WaCo5ZU
+ local LAST_ERR=/tmp/tmp.9zAcF77Lzx
+ local exit_status=0
+ local timeout=4
+++ mktemp
++ seq 0 2
++ echo 2025-08-06 21:52:09
+ /usr/bin/sed -e 's/date:/date: 2025-08-06 21:52:09/'
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl apply -f -
++ local LAST_ERR=/tmp/tmp.bsUOwaeHRd
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb-backup backup-minio-1 -o 'jsonpath={.status.s3.bucket}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.h96WaCo5ZU
++ cat /tmp/tmp.bsUOwaeHRd
++ rm /tmp/tmp.h96WaCo5ZU /tmp/tmp.bsUOwaeHRd
++ return 0
+ /usr/bin/sed -e 's|BUCKET-NAME|operator-testing|'
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.iWaebaocMy
perconaservermongodbrestore.psmdb.percona.com/restore-backup-minio-1 created
+ cat /tmp/tmp.9zAcF77Lzx
+ rm /tmp/tmp.iWaebaocMy /tmp/tmp.9zAcF77Lzx
+ return 0
+ wait_restore backup-minio-1 some-name requested 0 1200
+ local backup_name=backup-minio-1
+ local cluster_name=some-name
+ local target_state=requested
+ local wait_cluster_consistency=0
+ local wait_time=1200
+ local ok_if_ready=0
+ set +o xtrace
Waiting for the psmdb-restore/restore-backup-minio-1 object to be createdOK
Waiting psmdb-restore/restore-backup-minio-1 to reach state "requested" .........OK after 9 minutes
+ [[ 0 -eq 1 ]]
+ echo

+ wait_restore backup-minio-1 some-name ready 0 1600
+ local backup_name=backup-minio-1
+ local cluster_name=some-name
+ local target_state=ready
+ local wait_cluster_consistency=0
+ local wait_time=1600
+ local ok_if_ready=0
+ set +o xtrace
Waiting for the psmdb-restore/restore-backup-minio-1 object to be createdOK
Waiting psmdb-restore/restore-backup-minio-1 to reach state "ready" ..OK after 2 minutes
+ [[ 0 -eq 1 ]]
+ echo

+ set -o xtrace
+ wait_for_running some-name-rs0 3
+ local name=some-name-rs0
+ let last_pod=2
+ local check_cluster_readyness=true
+ set_debug
+ [[ 1 == 1 ]]
+ set -o xtrace
+ local rs_name=rs0
+ local cluster_name=some-name
++ seq 0 2
+ for i in '$(seq 0 $last_pod)'
+ [[ 0 -eq 2 ]]
+ wait_pod some-name-rs0-0
+ local pod=some-name-rs0-0
+ set +o xtrace
waiting for pod/some-name-rs0-0 to be ready.OK
+ for i in '$(seq 0 $last_pod)'
+ [[ 1 -eq 2 ]]
+ wait_pod some-name-rs0-1
+ local pod=some-name-rs0-1
+ set +o xtrace
waiting for pod/some-name-rs0-1 to be ready.OK
+ for i in '$(seq 0 $last_pod)'
+ [[ 2 -eq 2 ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].arbiter.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.LJ53KKXpXL
+++ mktemp
++ local LAST_ERR=/tmp/tmp.G3Gdgo5SZ9
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].arbiter.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.LJ53KKXpXL
++ cat /tmp/tmp.G3Gdgo5SZ9
++ rm /tmp/tmp.LJ53KKXpXL /tmp/tmp.G3Gdgo5SZ9
++ return 0
+ [[ '' == \t\r\u\e ]]
+ wait_pod some-name-rs0-2
+ local pod=some-name-rs0-2
+ set +o xtrace
waiting for pod/some-name-rs0-2 to be ready.OK
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].non_voting.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.WyYHMNdtba
+++ mktemp
++ local LAST_ERR=/tmp/tmp.IM1pvaZPX8
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].non_voting.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.WyYHMNdtba
++ cat /tmp/tmp.IM1pvaZPX8
++ rm /tmp/tmp.WyYHMNdtba /tmp/tmp.IM1pvaZPX8
++ return 0
+ [[ '' == \t\r\u\e ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].hidden.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.DZLFUaJC22
+++ mktemp
++ local LAST_ERR=/tmp/tmp.FwBFeSSpDg
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].hidden.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.DZLFUaJC22
++ cat /tmp/tmp.FwBFeSSpDg
++ rm /tmp/tmp.DZLFUaJC22 /tmp/tmp.FwBFeSSpDg
++ return 0
+ [[ '' == \t\r\u\e ]]
+ sleep 10
+ [[ true == \t\r\u\e ]]
+ set +x
Waiting for cluster readyness............................................................................................
+ wait_for_running some-name-cfg 3
+ local name=some-name-cfg
+ let last_pod=2
+ local check_cluster_readyness=true
+ set_debug
+ [[ 1 == 1 ]]
+ set -o xtrace
+ local rs_name=cfg
+ local cluster_name=some-name
++ seq 0 2
+ for i in '$(seq 0 $last_pod)'
+ [[ 0 -eq 2 ]]
+ wait_pod some-name-cfg-0
+ local pod=some-name-cfg-0
+ set +o xtrace
waiting for pod/some-name-cfg-0 to be ready.OK
+ for i in '$(seq 0 $last_pod)'
+ [[ 1 -eq 2 ]]
+ wait_pod some-name-cfg-1
+ local pod=some-name-cfg-1
+ set +o xtrace
waiting for pod/some-name-cfg-1 to be ready.OK
+ for i in '$(seq 0 $last_pod)'
+ [[ 2 -eq 2 ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].arbiter.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.ihbaMat7NO
+++ mktemp
++ local LAST_ERR=/tmp/tmp.FWwtb6v43p
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].arbiter.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.ihbaMat7NO
++ cat /tmp/tmp.FWwtb6v43p
++ rm /tmp/tmp.ihbaMat7NO /tmp/tmp.FWwtb6v43p
++ return 0
+ [[ '' == \t\r\u\e ]]
+ wait_pod some-name-cfg-2
+ local pod=some-name-cfg-2
+ set +o xtrace
waiting for pod/some-name-cfg-2 to be ready.OK
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].non_voting.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.ZzHE65I0F8
+++ mktemp
++ local LAST_ERR=/tmp/tmp.KkdlH3AXmJ
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].non_voting.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.ZzHE65I0F8
++ cat /tmp/tmp.KkdlH3AXmJ
++ rm /tmp/tmp.ZzHE65I0F8 /tmp/tmp.KkdlH3AXmJ
++ return 0
+ [[ '' == \t\r\u\e ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].hidden.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.1MVpjwJRny
+++ mktemp
++ local LAST_ERR=/tmp/tmp.ScxomK3AJy
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].hidden.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.1MVpjwJRny
++ cat /tmp/tmp.ScxomK3AJy
++ rm /tmp/tmp.1MVpjwJRny /tmp/tmp.ScxomK3AJy
++ return 0
+ [[ '' == \t\r\u\e ]]
+ sleep 10
+ [[ true == \t\r\u\e ]]
+ set +x
Waiting for cluster readyness
+ wait_for_running some-name-mongos 3
+ local name=some-name-mongos
+ let last_pod=2
+ local check_cluster_readyness=true
+ set_debug
+ [[ 1 == 1 ]]
+ set -o xtrace
+ local rs_name=mongos
+ local cluster_name=some-name
++ seq 0 2
+ for i in '$(seq 0 $last_pod)'
+ [[ 0 -eq 2 ]]
+ wait_pod some-name-mongos-0
+ local pod=some-name-mongos-0
+ set +o xtrace
waiting for pod/some-name-mongos-0 to be ready.OK
+ for i in '$(seq 0 $last_pod)'
+ [[ 1 -eq 2 ]]
+ wait_pod some-name-mongos-1
+ local pod=some-name-mongos-1
+ set +o xtrace
waiting for pod/some-name-mongos-1 to be ready.OK
+ for i in '$(seq 0 $last_pod)'
+ [[ 2 -eq 2 ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="mongos")].arbiter.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.KxB9wtfI5t
+++ mktemp
++ local LAST_ERR=/tmp/tmp.5CTVRm7zZk
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="mongos")].arbiter.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.KxB9wtfI5t
++ cat /tmp/tmp.5CTVRm7zZk
++ rm /tmp/tmp.KxB9wtfI5t /tmp/tmp.5CTVRm7zZk
++ return 0
+ [[ '' == \t\r\u\e ]]
+ wait_pod some-name-mongos-2
+ local pod=some-name-mongos-2
+ set +o xtrace
waiting for pod/some-name-mongos-2 to be ready.OK
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="mongos")].non_voting.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.rRcseqSP4T
+++ mktemp
++ local LAST_ERR=/tmp/tmp.8jzC9RzoKF
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="mongos")].non_voting.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.rRcseqSP4T
++ cat /tmp/tmp.8jzC9RzoKF
++ rm /tmp/tmp.rRcseqSP4T /tmp/tmp.8jzC9RzoKF
++ return 0
+ [[ '' == \t\r\u\e ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="mongos")].hidden.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.gjUt0idmpY
+++ mktemp
++ local LAST_ERR=/tmp/tmp.jD91BOPPF1
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="mongos")].hidden.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.gjUt0idmpY
++ cat /tmp/tmp.jD91BOPPF1
++ rm /tmp/tmp.gjUt0idmpY /tmp/tmp.jD91BOPPF1
++ return 0
+ [[ '' == \t\r\u\e ]]
+ sleep 10
+ [[ true == \t\r\u\e ]]
+ set +x
Waiting for cluster readyness
+ sleep 10
+ compare_mongos_cmd find myApp:myPass@some-name-mongos.pitr-physical-backup-source-28726 ''
+ local command=find
+ local uri=myApp:myPass@some-name-mongos.pitr-physical-backup-source-28726
+ local postfix=
+ local suffix=
+ local database=myApp
+ local collection=test
+ local port=27017
+ run_mongos 'use myApp\n db.test.find()' myApp:myPass@some-name-mongos.pitr-physical-backup-source-28726 mongodb '' '' 27017
+ egrep -v 'I NETWORK|W NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:'
+ local 'command=use myApp\n db.test.find()'
+ local uri=myApp:myPass@some-name-mongos.pitr-physical-backup-source-28726
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local port=27017
+ local mongo_bin=mongo
+ /usr/bin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
++ echo .svc.cluster.local
++ awk -F: '{print $2}'
+ suffix_port=
+ [[ -z '' ]]
+ suffix=.svc.cluster.local:27017
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.dYDyE92lPx
+++ mktemp
++ local LAST_ERR=/tmp/tmp.EfVTMXrzwj
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.dYDyE92lPx
++ cat /tmp/tmp.EfVTMXrzwj
++ rm /tmp/tmp.dYDyE92lPx /tmp/tmp.EfVTMXrzwj
++ return 0
+ local client_container=psmdb-client-66f577db5f-d4gt7
+ kubectl_bin exec psmdb-client-66f577db5f-d4gt7 -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.pitr-physical-backup-source-28726.svc.cluster.local:27017/admin '
++ mktemp
+ local LAST_OUT=/tmp/tmp.XA4bxSPx7B
++ mktemp
+ local LAST_ERR=/tmp/tmp.M9v7xTnufb
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl exec psmdb-client-66f577db5f-d4gt7 -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.pitr-physical-backup-source-28726.svc.cluster.local:27017/admin '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.XA4bxSPx7B
+ cat /tmp/tmp.M9v7xTnufb
+ rm /tmp/tmp.XA4bxSPx7B /tmp/tmp.M9v7xTnufb
+ return 0
+ [[ 0 -eq 0 ]]
+ diff /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1961/e2e-tests/pitr-physical-backup-source/compare/find.json /tmp/tmp.Yk9PLuHVxt/find
+ desc 'delete PSMDB cluster some-name'
+ set +o xtrace


-----------------------------------------------------------------------------------
delete PSMDB cluster some-name
-----------------------------------------------------------------------------------

+ kubectl_bin delete psmdb some-name
++ mktemp
+ local LAST_OUT=/tmp/tmp.N055uc1kqX
++ mktemp
+ local LAST_ERR=/tmp/tmp.q1D37XL8hH
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl delete psmdb some-name
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.N055uc1kqX
perconaservermongodb.psmdb.percona.com "some-name" deleted
+ cat /tmp/tmp.q1D37XL8hH
+ rm /tmp/tmp.N055uc1kqX /tmp/tmp.q1D37XL8hH
+ return 0
+ kubectl_bin delete pvc -l app.kubernetes.io/managed-by=percona-server-mongodb-operator
++ mktemp
+ local LAST_OUT=/tmp/tmp.acCWSAZ32q
++ mktemp
+ local LAST_ERR=/tmp/tmp.PBcu6BLRCV
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl delete pvc -l app.kubernetes.io/managed-by=percona-server-mongodb-operator
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.acCWSAZ32q
persistentvolumeclaim "mongod-data-some-name-cfg-0" deleted
persistentvolumeclaim "mongod-data-some-name-cfg-1" deleted
persistentvolumeclaim "mongod-data-some-name-cfg-2" deleted
persistentvolumeclaim "mongod-data-some-name-rs0-0" deleted
persistentvolumeclaim "mongod-data-some-name-rs0-1" deleted
persistentvolumeclaim "mongod-data-some-name-rs0-2" deleted
persistentvolumeclaim "mongod-data-some-name-rs1-0" deleted
persistentvolumeclaim "mongod-data-some-name-rs1-1" deleted
persistentvolumeclaim "mongod-data-some-name-rs1-2" deleted
persistentvolumeclaim "mongod-data-some-name-rs2-0" deleted
persistentvolumeclaim "mongod-data-some-name-rs2-1" deleted
persistentvolumeclaim "mongod-data-some-name-rs2-2" deleted
+ cat /tmp/tmp.PBcu6BLRCV
+ rm /tmp/tmp.acCWSAZ32q /tmp/tmp.PBcu6BLRCV
+ return 0
+ sleep 10
+ desc 'recreate PSMDB cluster some-name'
+ set +o xtrace


-----------------------------------------------------------------------------------
recreate PSMDB cluster some-name
-----------------------------------------------------------------------------------

+ desc 'create second PSMDB cluster some-name'
+ set +o xtrace


-----------------------------------------------------------------------------------
create second PSMDB cluster some-name
-----------------------------------------------------------------------------------

+ apply_cluster /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1961/e2e-tests/pitr-physical-backup-source/conf/some-name-rs0.yml
+ '[' -z '' ']'
+ cat_config /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1961/e2e-tests/pitr-physical-backup-source/conf/some-name-rs0.yml
+ kubectl_bin apply -f -
+ cat /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1961/e2e-tests/pitr-physical-backup-source/conf/some-name-rs0.yml
+ yq eval '(.spec | select(.image == null)).image = "perconalab/percona-server-mongodb-operator:main-mongod7.0"'
++ mktemp
+ yq eval '(.spec | select(has("pmm"))).pmm.image = "perconalab/pmm-client:dev-latest"'
+ yq eval '(.spec | select(has("initImage"))).initImage = "perconalab/percona-server-mongodb-operator:PR-1961-970eeaa4"'
+ yq eval '.spec.upgradeOptions.apply="Never"'
+ yq eval '(.spec | select(has("backup"))).backup.image = "perconalab/percona-server-mongodb-operator:main-backup"'
+ local LAST_OUT=/tmp/tmp.GXkhqNMPoo
++ mktemp
+ local LAST_ERR=/tmp/tmp.OPFTnYYgdh
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl apply -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.GXkhqNMPoo
perconaservermongodb.psmdb.percona.com/some-name created
+ cat /tmp/tmp.OPFTnYYgdh
+ rm /tmp/tmp.GXkhqNMPoo /tmp/tmp.OPFTnYYgdh
+ return 0
+ desc 'check if all 3 Pods started'
+ set +o xtrace


-----------------------------------------------------------------------------------
check if all 3 Pods started
-----------------------------------------------------------------------------------

+ wait_for_running some-name-rs0 3
+ local name=some-name-rs0
+ let last_pod=2
+ local check_cluster_readyness=true
+ set_debug
+ [[ 1 == 1 ]]
+ set -o xtrace
+ local rs_name=rs0
+ local cluster_name=some-name
++ seq 0 2
+ for i in '$(seq 0 $last_pod)'
+ [[ 0 -eq 2 ]]
+ wait_pod some-name-rs0-0
+ local pod=some-name-rs0-0
+ set +o xtrace
waiting for pod/some-name-rs0-0 to be ready..............OK
+ for i in '$(seq 0 $last_pod)'
+ [[ 1 -eq 2 ]]
+ wait_pod some-name-rs0-1
+ local pod=some-name-rs0-1
+ set +o xtrace
waiting for pod/some-name-rs0-1 to be ready.......OK
+ for i in '$(seq 0 $last_pod)'
+ [[ 2 -eq 2 ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].arbiter.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.baSNWZ06rX
+++ mktemp
++ local LAST_ERR=/tmp/tmp.RPQ2hEZFVX
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].arbiter.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.baSNWZ06rX
++ cat /tmp/tmp.RPQ2hEZFVX
++ rm /tmp/tmp.baSNWZ06rX /tmp/tmp.RPQ2hEZFVX
++ return 0
+ [[ '' == \t\r\u\e ]]
+ wait_pod some-name-rs0-2
+ local pod=some-name-rs0-2
+ set +o xtrace
waiting for pod/some-name-rs0-2 to be ready......OK
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].non_voting.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.MbYCt4gsDa
+++ mktemp
++ local LAST_ERR=/tmp/tmp.qlpKnD6yRs
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].non_voting.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.MbYCt4gsDa
++ cat /tmp/tmp.qlpKnD6yRs
++ rm /tmp/tmp.MbYCt4gsDa /tmp/tmp.qlpKnD6yRs
++ return 0
+ [[ '' == \t\r\u\e ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].hidden.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.US9XycCCei
+++ mktemp
++ local LAST_ERR=/tmp/tmp.SaBRIJC6Je
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].hidden.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.US9XycCCei
++ cat /tmp/tmp.SaBRIJC6Je
++ rm /tmp/tmp.US9XycCCei /tmp/tmp.SaBRIJC6Je
++ return 0
+ [[ '' == \t\r\u\e ]]
+ sleep 10
+ [[ true == \t\r\u\e ]]
+ set +x
Waiting for cluster readyness.............................................
+ wait_for_running some-name-cfg 3 false
+ local name=some-name-cfg
+ let last_pod=2
+ local check_cluster_readyness=false
+ set_debug
+ [[ 1 == 1 ]]
+ set -o xtrace
+ local rs_name=cfg
+ local cluster_name=some-name
++ seq 0 2
+ for i in '$(seq 0 $last_pod)'
+ [[ 0 -eq 2 ]]
+ wait_pod some-name-cfg-0
+ local pod=some-name-cfg-0
+ set +o xtrace
waiting for pod/some-name-cfg-0 to be ready.OK
+ for i in '$(seq 0 $last_pod)'
+ [[ 1 -eq 2 ]]
+ wait_pod some-name-cfg-1
+ local pod=some-name-cfg-1
+ set +o xtrace
waiting for pod/some-name-cfg-1 to be ready.OK
+ for i in '$(seq 0 $last_pod)'
+ [[ 2 -eq 2 ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].arbiter.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.pPmph0Nrgm
+++ mktemp
++ local LAST_ERR=/tmp/tmp.HZy6bGFTLd
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].arbiter.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.pPmph0Nrgm
++ cat /tmp/tmp.HZy6bGFTLd
++ rm /tmp/tmp.pPmph0Nrgm /tmp/tmp.HZy6bGFTLd
++ return 0
+ [[ '' == \t\r\u\e ]]
+ wait_pod some-name-cfg-2
+ local pod=some-name-cfg-2
+ set +o xtrace
waiting for pod/some-name-cfg-2 to be ready.OK
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].non_voting.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.RmV0rtUDFk
+++ mktemp
++ local LAST_ERR=/tmp/tmp.o328YKkq8z
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].non_voting.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.RmV0rtUDFk
++ cat /tmp/tmp.o328YKkq8z
++ rm /tmp/tmp.RmV0rtUDFk /tmp/tmp.o328YKkq8z
++ return 0
+ [[ '' == \t\r\u\e ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].hidden.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.SRCUiMDG9Z
+++ mktemp
++ local LAST_ERR=/tmp/tmp.BSqeuO8tHs
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].hidden.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.SRCUiMDG9Z
++ cat /tmp/tmp.BSqeuO8tHs
++ rm /tmp/tmp.SRCUiMDG9Z /tmp/tmp.BSqeuO8tHs
++ return 0
+ [[ '' == \t\r\u\e ]]
+ sleep 10
+ [[ false == \t\r\u\e ]]
+ wait_for_running some-name-rs1 3
+ local name=some-name-rs1
+ let last_pod=2
+ local check_cluster_readyness=true
+ set_debug
+ [[ 1 == 1 ]]
+ set -o xtrace
+ local rs_name=rs1
+ local cluster_name=some-name
++ seq 0 2
+ for i in '$(seq 0 $last_pod)'
+ [[ 0 -eq 2 ]]
+ wait_pod some-name-rs1-0
+ local pod=some-name-rs1-0
+ set +o xtrace
waiting for pod/some-name-rs1-0 to be ready.OK
+ for i in '$(seq 0 $last_pod)'
+ [[ 1 -eq 2 ]]
+ wait_pod some-name-rs1-1
+ local pod=some-name-rs1-1
+ set +o xtrace
waiting for pod/some-name-rs1-1 to be ready.OK
+ for i in '$(seq 0 $last_pod)'
+ [[ 2 -eq 2 ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs1")].arbiter.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.6khQZmGw8F
+++ mktemp
++ local LAST_ERR=/tmp/tmp.qgWbVyOQAB
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs1")].arbiter.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.6khQZmGw8F
++ cat /tmp/tmp.qgWbVyOQAB
++ rm /tmp/tmp.6khQZmGw8F /tmp/tmp.qgWbVyOQAB
++ return 0
+ [[ '' == \t\r\u\e ]]
+ wait_pod some-name-rs1-2
+ local pod=some-name-rs1-2
+ set +o xtrace
waiting for pod/some-name-rs1-2 to be ready.OK
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs1")].non_voting.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.qJCJKIBrU2
+++ mktemp
++ local LAST_ERR=/tmp/tmp.BE5wikRG1k
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs1")].non_voting.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.qJCJKIBrU2
++ cat /tmp/tmp.BE5wikRG1k
++ rm /tmp/tmp.qJCJKIBrU2 /tmp/tmp.BE5wikRG1k
++ return 0
+ [[ '' == \t\r\u\e ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs1")].hidden.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.8TIH8qt0Ra
+++ mktemp
++ local LAST_ERR=/tmp/tmp.XZYT5q9Ofy
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs1")].hidden.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.8TIH8qt0Ra
++ cat /tmp/tmp.XZYT5q9Ofy
++ rm /tmp/tmp.8TIH8qt0Ra /tmp/tmp.XZYT5q9Ofy
++ return 0
+ [[ '' == \t\r\u\e ]]
+ sleep 10
+ [[ true == \t\r\u\e ]]
+ set +x
Waiting for cluster readyness
+ wait_for_running some-name-rs2 3
+ local name=some-name-rs2
+ let last_pod=2
+ local check_cluster_readyness=true
+ set_debug
+ [[ 1 == 1 ]]
+ set -o xtrace
+ local rs_name=rs2
+ local cluster_name=some-name
++ seq 0 2
+ for i in '$(seq 0 $last_pod)'
+ [[ 0 -eq 2 ]]
+ wait_pod some-name-rs2-0
+ local pod=some-name-rs2-0
+ set +o xtrace
waiting for pod/some-name-rs2-0 to be ready.OK
+ for i in '$(seq 0 $last_pod)'
+ [[ 1 -eq 2 ]]
+ wait_pod some-name-rs2-1
+ local pod=some-name-rs2-1
+ set +o xtrace
waiting for pod/some-name-rs2-1 to be ready.OK
+ for i in '$(seq 0 $last_pod)'
+ [[ 2 -eq 2 ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs2")].arbiter.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.aXwIwMFQCU
+++ mktemp
++ local LAST_ERR=/tmp/tmp.cPgrflnzf0
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs2")].arbiter.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.aXwIwMFQCU
++ cat /tmp/tmp.cPgrflnzf0
++ rm /tmp/tmp.aXwIwMFQCU /tmp/tmp.cPgrflnzf0
++ return 0
+ [[ '' == \t\r\u\e ]]
+ wait_pod some-name-rs2-2
+ local pod=some-name-rs2-2
+ set +o xtrace
waiting for pod/some-name-rs2-2 to be ready.OK
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs2")].non_voting.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.vCkYx3ELqa
+++ mktemp
++ local LAST_ERR=/tmp/tmp.c9dVk9wkb3
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs2")].non_voting.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.vCkYx3ELqa
++ cat /tmp/tmp.c9dVk9wkb3
++ rm /tmp/tmp.vCkYx3ELqa /tmp/tmp.c9dVk9wkb3
++ return 0
+ [[ '' == \t\r\u\e ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs2")].hidden.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.Hdz7quU1BZ
+++ mktemp
++ local LAST_ERR=/tmp/tmp.8piE58fmfZ
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs2")].hidden.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.Hdz7quU1BZ
++ cat /tmp/tmp.8piE58fmfZ
++ rm /tmp/tmp.Hdz7quU1BZ /tmp/tmp.8piE58fmfZ
++ return 0
+ [[ '' == \t\r\u\e ]]
+ sleep 10
+ [[ true == \t\r\u\e ]]
+ set +x
Waiting for cluster readyness
+ sleep 10
+ write_initial_data
+ desc 'create user myApp'
+ set +o xtrace


-----------------------------------------------------------------------------------
create user myApp
-----------------------------------------------------------------------------------

+ run_mongos 'db.createUser({user:"myApp",pwd:"myPass",roles:[{db:"myApp",role:"readWrite"}]})' userAdmin:userAdmin123456@some-name-mongos.pitr-physical-backup-source-28726
+ local 'command=db.createUser({user:"myApp",pwd:"myPass",roles:[{db:"myApp",role:"readWrite"}]})'
+ local uri=userAdmin:userAdmin123456@some-name-mongos.pitr-physical-backup-source-28726
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local port=27017
+ local mongo_bin=mongo
++ echo .svc.cluster.local
++ awk -F: '{print $2}'
+ suffix_port=
+ [[ -z '' ]]
+ suffix=.svc.cluster.local:27017
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.5aZYmJ07Uo
+++ mktemp
++ local LAST_ERR=/tmp/tmp.DcQL3GapOi
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.5aZYmJ07Uo
++ cat /tmp/tmp.DcQL3GapOi
++ rm /tmp/tmp.5aZYmJ07Uo /tmp/tmp.DcQL3GapOi
++ return 0
+ local client_container=psmdb-client-66f577db5f-d4gt7
+ kubectl_bin exec psmdb-client-66f577db5f-d4gt7 -- bash -c 'printf '\''db.createUser({user:"myApp",pwd:"myPass",roles:[{db:"myApp",role:"readWrite"}]})\n'\'' | mongo mongodb://userAdmin:userAdmin123456@some-name-mongos.pitr-physical-backup-source-28726.svc.cluster.local:27017/admin '
++ mktemp
+ local LAST_OUT=/tmp/tmp.E1hbFFeRzH
++ mktemp
+ local LAST_ERR=/tmp/tmp.RbmAn3fO2M
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl exec psmdb-client-66f577db5f-d4gt7 -- bash -c 'printf '\''db.createUser({user:"myApp",pwd:"myPass",roles:[{db:"myApp",role:"readWrite"}]})\n'\'' | mongo mongodb://userAdmin:userAdmin123456@some-name-mongos.pitr-physical-backup-source-28726.svc.cluster.local:27017/admin '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.E1hbFFeRzH
Percona Server for MongoDB shell version v4.4.29-28
connecting to: mongodb://some-name-mongos.pitr-physical-backup-source-28726.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb
Implicit session: session { "id" : UUID("1e799a02-b5ab-401a-b081-4e144b751519") }
Percona Server for MongoDB server version: v7.0.22-12
WARNING: shell and server versions do not match
Successfully added user: {
	"user" : "myApp",
	"roles" : [
		{
			"db" : "myApp",
			"role" : "readWrite"
		}
	]
}
bye
+ cat /tmp/tmp.RbmAn3fO2M
+ rm /tmp/tmp.E1hbFFeRzH /tmp/tmp.RbmAn3fO2M
+ return 0
+ sleep 2
+ write_document
+ local cmp_postfix=
+ local sleep_value=0
+ log 'write initial data, read from all'
+ set +o xtrace
[2025-08-06T22:14:31+0000] write initial data, read from all
+ run_mongos 'use myApp\n db.test.insert({ x: 100500 })' myApp:myPass@some-name-mongos.pitr-physical-backup-source-28726
+ local 'command=use myApp\n db.test.insert({ x: 100500 })'
+ local uri=myApp:myPass@some-name-mongos.pitr-physical-backup-source-28726
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local port=27017
+ local mongo_bin=mongo
++ awk -F: '{print $2}'
++ echo .svc.cluster.local
+ suffix_port=
+ [[ -z '' ]]
+ suffix=.svc.cluster.local:27017
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.VmdItt0GG4
+++ mktemp
++ local LAST_ERR=/tmp/tmp.WmMxHp2N6V
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.VmdItt0GG4
++ cat /tmp/tmp.WmMxHp2N6V
++ rm /tmp/tmp.VmdItt0GG4 /tmp/tmp.WmMxHp2N6V
++ return 0
+ local client_container=psmdb-client-66f577db5f-d4gt7
+ kubectl_bin exec psmdb-client-66f577db5f-d4gt7 -- bash -c 'printf '\''use myApp\n db.test.insert({ x: 100500 })\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.pitr-physical-backup-source-28726.svc.cluster.local:27017/admin '
++ mktemp
+ local LAST_OUT=/tmp/tmp.L4srkhVOzZ
++ mktemp
+ local LAST_ERR=/tmp/tmp.UIvps1796s
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl exec psmdb-client-66f577db5f-d4gt7 -- bash -c 'printf '\''use myApp\n db.test.insert({ x: 100500 })\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.pitr-physical-backup-source-28726.svc.cluster.local:27017/admin '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.L4srkhVOzZ
Percona Server for MongoDB shell version v4.4.29-28
connecting to: mongodb://some-name-mongos.pitr-physical-backup-source-28726.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb
Implicit session: session { "id" : UUID("935c4a83-4bb3-4cd9-aa69-39141767e45f") }
Percona Server for MongoDB server version: v7.0.22-12
WARNING: shell and server versions do not match
switched to db myApp
WriteResult({ "nInserted" : 1 })
bye
+ cat /tmp/tmp.UIvps1796s
+ rm /tmp/tmp.L4srkhVOzZ /tmp/tmp.UIvps1796s
+ return 0
+ sleep 0
+ compare_mongos_cmd find myApp:myPass@some-name-mongos.pitr-physical-backup-source-28726
+ local command=find
+ local uri=myApp:myPass@some-name-mongos.pitr-physical-backup-source-28726
+ local postfix=
+ local suffix=
+ local database=myApp
+ local collection=test
+ local port=27017
+ run_mongos 'use myApp\n db.test.find()' myApp:myPass@some-name-mongos.pitr-physical-backup-source-28726 mongodb '' '' 27017
+ local 'command=use myApp\n db.test.find()'
+ local uri=myApp:myPass@some-name-mongos.pitr-physical-backup-source-28726
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local port=27017
+ local mongo_bin=mongo
+ /usr/bin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
+ egrep -v 'I NETWORK|W NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:'
++ echo .svc.cluster.local
++ awk -F: '{print $2}'
+ suffix_port=
+ [[ -z '' ]]
+ suffix=.svc.cluster.local:27017
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.AxFmj2c0Hp
+++ mktemp
++ local LAST_ERR=/tmp/tmp.AI8rrIzrDZ
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.AxFmj2c0Hp
++ cat /tmp/tmp.AI8rrIzrDZ
++ rm /tmp/tmp.AxFmj2c0Hp /tmp/tmp.AI8rrIzrDZ
++ return 0
+ local client_container=psmdb-client-66f577db5f-d4gt7
+ kubectl_bin exec psmdb-client-66f577db5f-d4gt7 -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.pitr-physical-backup-source-28726.svc.cluster.local:27017/admin '
++ mktemp
+ local LAST_OUT=/tmp/tmp.IVDqrvQmk1
++ mktemp
+ local LAST_ERR=/tmp/tmp.nkwFIHy65f
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl exec psmdb-client-66f577db5f-d4gt7 -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.pitr-physical-backup-source-28726.svc.cluster.local:27017/admin '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.IVDqrvQmk1
+ cat /tmp/tmp.nkwFIHy65f
+ rm /tmp/tmp.IVDqrvQmk1 /tmp/tmp.nkwFIHy65f
+ return 0
+ [[ 0 -eq 0 ]]
+ diff /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1961/e2e-tests/pitr-physical-backup-source/compare/find.json /tmp/tmp.Yk9PLuHVxt/find
+ desc 'restore pitr type latest using backupSource'
+ set +o xtrace


-----------------------------------------------------------------------------------
restore pitr type latest using backupSource
-----------------------------------------------------------------------------------

+ write_document -2nd
+ local cmp_postfix=-2nd
+ local sleep_value=0
+ log 'write initial data, read from all'
+ set +o xtrace
[2025-08-06T22:14:34+0000] write initial data, read from all
+ run_mongos 'use myApp\n db.test.insert({ x: 100500 })' myApp:myPass@some-name-mongos.pitr-physical-backup-source-28726
+ local 'command=use myApp\n db.test.insert({ x: 100500 })'
+ local uri=myApp:myPass@some-name-mongos.pitr-physical-backup-source-28726
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local port=27017
+ local mongo_bin=mongo
++ echo .svc.cluster.local
++ awk -F: '{print $2}'
+ suffix_port=
+ [[ -z '' ]]
+ suffix=.svc.cluster.local:27017
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.Iu1KieNNUS
+++ mktemp
++ local LAST_ERR=/tmp/tmp.a88HaCR75J
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.Iu1KieNNUS
++ cat /tmp/tmp.a88HaCR75J
++ rm /tmp/tmp.Iu1KieNNUS /tmp/tmp.a88HaCR75J
++ return 0
+ local client_container=psmdb-client-66f577db5f-d4gt7
+ kubectl_bin exec psmdb-client-66f577db5f-d4gt7 -- bash -c 'printf '\''use myApp\n db.test.insert({ x: 100500 })\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.pitr-physical-backup-source-28726.svc.cluster.local:27017/admin '
++ mktemp
+ local LAST_OUT=/tmp/tmp.GwZxmuLmc1
++ mktemp
+ local LAST_ERR=/tmp/tmp.VANYYXKHoB
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl exec psmdb-client-66f577db5f-d4gt7 -- bash -c 'printf '\''use myApp\n db.test.insert({ x: 100500 })\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.pitr-physical-backup-source-28726.svc.cluster.local:27017/admin '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.GwZxmuLmc1
Percona Server for MongoDB shell version v4.4.29-28
connecting to: mongodb://some-name-mongos.pitr-physical-backup-source-28726.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb
Implicit session: session { "id" : UUID("45a5969e-6b99-4dda-bc3f-e4139244d3b3") }
Percona Server for MongoDB server version: v7.0.22-12
WARNING: shell and server versions do not match
switched to db myApp
WriteResult({ "nInserted" : 1 })
bye
+ cat /tmp/tmp.VANYYXKHoB
+ rm /tmp/tmp.GwZxmuLmc1 /tmp/tmp.VANYYXKHoB
+ return 0
+ sleep 0
+ compare_mongos_cmd find myApp:myPass@some-name-mongos.pitr-physical-backup-source-28726 -2nd
+ local command=find
+ local uri=myApp:myPass@some-name-mongos.pitr-physical-backup-source-28726
+ local postfix=-2nd
+ local suffix=
+ local database=myApp
+ local collection=test
+ local port=27017
+ /usr/bin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
+ run_mongos 'use myApp\n db.test.find()' myApp:myPass@some-name-mongos.pitr-physical-backup-source-28726 mongodb '' '' 27017
+ egrep -v 'I NETWORK|W NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:'
+ local 'command=use myApp\n db.test.find()'
+ local uri=myApp:myPass@some-name-mongos.pitr-physical-backup-source-28726
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local port=27017
+ local mongo_bin=mongo
++ awk -F: '{print $2}'
++ echo .svc.cluster.local
+ suffix_port=
+ [[ -z '' ]]
+ suffix=.svc.cluster.local:27017
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.le3vKROSQN
+++ mktemp
++ local LAST_ERR=/tmp/tmp.8oShnoWPiO
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.le3vKROSQN
++ cat /tmp/tmp.8oShnoWPiO
++ rm /tmp/tmp.le3vKROSQN /tmp/tmp.8oShnoWPiO
++ return 0
+ local client_container=psmdb-client-66f577db5f-d4gt7
+ kubectl_bin exec psmdb-client-66f577db5f-d4gt7 -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.pitr-physical-backup-source-28726.svc.cluster.local:27017/admin '
++ mktemp
+ local LAST_OUT=/tmp/tmp.9qkT05eMoD
++ mktemp
+ local LAST_ERR=/tmp/tmp.CwAUMedFe3
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl exec psmdb-client-66f577db5f-d4gt7 -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.pitr-physical-backup-source-28726.svc.cluster.local:27017/admin '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.9qkT05eMoD
+ cat /tmp/tmp.CwAUMedFe3
+ rm /tmp/tmp.9qkT05eMoD /tmp/tmp.CwAUMedFe3
+ return 0
+ [[ 0 -eq 0 ]]
+ diff /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1961/e2e-tests/pitr-physical-backup-source/compare/find-2nd.json /tmp/tmp.Yk9PLuHVxt/find-2nd
+ run_backup backup-minio 2 physical
+ local name=backup-minio
+ local idx=2
+ local type=physical
+ desc 'run backup backup-minio-2'
+ set +o xtrace


-----------------------------------------------------------------------------------
run backup backup-minio-2
-----------------------------------------------------------------------------------

+ cat /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1961/e2e-tests/pitr-physical-backup-source/conf/backup-minio.yml
+ kubectl_bin apply -f -
+ /usr/bin/sed -e 's/name:/name: backup-minio-2/'
+ /usr/bin/sed -e 's/type:/type: physical/'
++ mktemp
+ local LAST_OUT=/tmp/tmp.CzzTNQ56b1
++ mktemp
+ local LAST_ERR=/tmp/tmp.qTKdWP2siU
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl apply -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.CzzTNQ56b1
perconaservermongodbbackup.psmdb.percona.com/backup-minio-2 created
+ cat /tmp/tmp.qTKdWP2siU
+ rm /tmp/tmp.CzzTNQ56b1 /tmp/tmp.qTKdWP2siU
+ return 0
+ wait_backup backup-minio-2
+ local backup_name=backup-minio-2
+ local target_state=ready
+ set +o xtrace
waiting for backup-minio-2 to reach ready state............
+ compare_latest_restorable_time some-name-rs0 backup-minio-2
+ local cluster=some-name-rs0
+ local backup_name=backup-minio-2
+ local latest_restorable_time
+ local backup_time
++ get_latest_restorable_time some-name-rs0
++ local cluster=some-name-rs0
++ local first_timestamp
++ local second_timestamp
++ local retry=0
++ [[ '' != '' ]]
+++ kubectl_bin exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ jq '.backups.pitrChunks.pitrChunks | last | .range.end'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.mAml3HvUld
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.sbxPdC4ecB
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in '$(seq 0 2)'
+++ set +e
+++ kubectl exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.mAml3HvUld
+++ cat /tmp/tmp.sbxPdC4ecB
+++ rm /tmp/tmp.mAml3HvUld /tmp/tmp.sbxPdC4ecB
+++ return 0
++ first_timestamp=1754517212
++ sleep 5
++ [[ 1754517212 != '' ]]
++ [[ 1754517212 != \n\u\l\l ]]
+++ kubectl_bin exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ jq '.backups.pitrChunks.pitrChunks | last | .range.end'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.Ri4mfWQHfP
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.QG2kSKE5uB
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in '$(seq 0 2)'
+++ set +e
+++ kubectl exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.Ri4mfWQHfP
+++ cat /tmp/tmp.QG2kSKE5uB
+++ rm /tmp/tmp.Ri4mfWQHfP /tmp/tmp.QG2kSKE5uB
+++ return 0
++ second_timestamp=1754517212
++ let retry+=1
++ [[ 1 -gt 30 ]]
++ [[ 1754517212 != '' ]]
++ [[ 1754517212 != \n\u\l\l ]]
++ [[ 1754517212 == 1754517212 ]]
++ /usr/bin/date -u -d @1754517212 +%Y-%m-%dT%H:%M:%SZ
+ latest_restorable_time=2025-08-06T21:53:32Z
++ get_latest_restorable_time_from_backup_object backup-minio-2
++ local backup_name=backup-minio-2
++ local latestRestorableTime
++ local retry=0
++ [[ '' != '' ]]
++ sleep 5
+++ kubectl_bin get psmdb-backup backup-minio-2 -o 'jsonpath={.status.latestRestorableTime}'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.eTz65Zifdj
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.Qr68ExDY6K
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in '$(seq 0 2)'
+++ set +e
+++ kubectl get psmdb-backup backup-minio-2 -o 'jsonpath={.status.latestRestorableTime}'
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.eTz65Zifdj
+++ cat /tmp/tmp.Qr68ExDY6K
+++ rm /tmp/tmp.eTz65Zifdj /tmp/tmp.Qr68ExDY6K
+++ return 0
++ latestRestorableTime=2025-08-06T21:53:32Z
++ let retry+=1
++ [[ 1 -gt 30 ]]
++ [[ 2025-08-06T21:53:32Z != '' ]]
++ [[ 2025-08-06T21:53:32Z != \n\u\l\l ]]
++ echo 2025-08-06T21:53:32Z
+ backup_time=2025-08-06T21:53:32Z
+ [[ 2025-08-06T21:53:32Z != \2\0\2\5\-\0\8\-\0\6\T\2\1\:\5\3\:\3\2\Z ]]
+ check_recovery backup-minio-2 latest '' -3rd some-name backupSource
+ local backup_name=backup-minio-2
+ local restore_type=latest
+ local restore_date=
+ local cmp_postfix=-3rd
+ local cluster_name=some-name
+ local backupSource=backupSource
++ get_latest_oplog_chunk_ts some-name
++ local cluster=some-name
+++ kubectl_bin exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ jq '.backups.pitrChunks.pitrChunks | last | .range.end'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.Kv29haWKvs
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.e5ULVAstyq
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in '$(seq 0 2)'
+++ set +e
+++ kubectl exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.Kv29haWKvs
+++ cat /tmp/tmp.e5ULVAstyq
+++ rm /tmp/tmp.Kv29haWKvs /tmp/tmp.e5ULVAstyq
+++ return 0
++ echo 1754517212
+ local latest_ts=1754517212
+ desc 'write more data before restore by latest'
+ set +o xtrace


-----------------------------------------------------------------------------------
write more data before restore by latest
-----------------------------------------------------------------------------------

+ run_mongos 'use myApp\n db.test.insert({ x: 100501 })' myApp:myPass@some-name-mongos.pitr-physical-backup-source-28726
+ local 'command=use myApp\n db.test.insert({ x: 100501 })'
+ local uri=myApp:myPass@some-name-mongos.pitr-physical-backup-source-28726
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local port=27017
+ local mongo_bin=mongo
++ echo .svc.cluster.local
++ awk -F: '{print $2}'
+ suffix_port=
+ [[ -z '' ]]
+ suffix=.svc.cluster.local:27017
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.6CjExsVJNZ
+++ mktemp
++ local LAST_ERR=/tmp/tmp.USGqq6NZcP
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.6CjExsVJNZ
++ cat /tmp/tmp.USGqq6NZcP
++ rm /tmp/tmp.6CjExsVJNZ /tmp/tmp.USGqq6NZcP
++ return 0
+ local client_container=psmdb-client-66f577db5f-d4gt7
+ kubectl_bin exec psmdb-client-66f577db5f-d4gt7 -- bash -c 'printf '\''use myApp\n db.test.insert({ x: 100501 })\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.pitr-physical-backup-source-28726.svc.cluster.local:27017/admin '
++ mktemp
+ local LAST_OUT=/tmp/tmp.7NSuqoXhRR
++ mktemp
+ local LAST_ERR=/tmp/tmp.xX0HuaWfKT
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl exec psmdb-client-66f577db5f-d4gt7 -- bash -c 'printf '\''use myApp\n db.test.insert({ x: 100501 })\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.pitr-physical-backup-source-28726.svc.cluster.local:27017/admin '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.7NSuqoXhRR
Percona Server for MongoDB shell version v4.4.29-28
connecting to: mongodb://some-name-mongos.pitr-physical-backup-source-28726.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb
Implicit session: session { "id" : UUID("e1b580ec-7630-4e8b-8192-b075455497fd") }
Percona Server for MongoDB server version: v7.0.22-12
WARNING: shell and server versions do not match
switched to db myApp
WriteResult({ "nInserted" : 1 })
bye
+ cat /tmp/tmp.xX0HuaWfKT
+ rm /tmp/tmp.7NSuqoXhRR /tmp/tmp.xX0HuaWfKT
+ return 0
+ [[ -n '' ]]
+ desc 'Restoring to latest'
+ set +o xtrace


-----------------------------------------------------------------------------------
Restoring to latest
-----------------------------------------------------------------------------------

++ get_latest_oplog_chunk_ts some-name
++ local cluster=some-name
+++ kubectl_bin exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ jq '.backups.pitrChunks.pitrChunks | last | .range.end'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.0D19Z4tQrs
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.L5xB6Whxf2
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in '$(seq 0 2)'
+++ set +e
+++ kubectl exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.0D19Z4tQrs
+++ cat /tmp/tmp.L5xB6Whxf2
+++ rm /tmp/tmp.0D19Z4tQrs /tmp/tmp.L5xB6Whxf2
+++ return 0
++ echo 1754517212
+ local current_ts=1754517212
+ retries=0
+ [[ 1754517212 -gt 1754517212 ]]
+ [[ 0 -gt 30 ]]
++ get_latest_oplog_chunk_ts some-name
++ local cluster=some-name
+++ jq '.backups.pitrChunks.pitrChunks | last | .range.end'
+++ kubectl_bin exec some-name-rs0-0 -c backup-agent -- pbm status -o json
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.Tq8shxVzsv
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.Lgk6NLiuX9
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in '$(seq 0 2)'
+++ set +e
+++ kubectl exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.Tq8shxVzsv
+++ cat /tmp/tmp.Lgk6NLiuX9
+++ rm /tmp/tmp.Tq8shxVzsv /tmp/tmp.Lgk6NLiuX9
+++ return 0
++ echo 1754517212
+ latest_ts=1754517212
+ retries=1
++ format_date 1754517212
++ local timestamp=1754517212
+++ TZ=UTC
+++ /usr/bin/date -d@1754517212 '+%Y-%m-%d %H:%M:%S'
++ echo 2025-08-06 21:53:32
++ format_date 1754517212
++ local timestamp=1754517212
+++ TZ=UTC
+++ /usr/bin/date -d@1754517212 '+%Y-%m-%d %H:%M:%S'
++ echo 2025-08-06 21:53:32
+ echo 'Waiting for last oplog chunk (2025-08-06 21:53:32) to be 120 seconds older than starting chunk (2025-08-06 21:53:32)'
Waiting for last oplog chunk (2025-08-06 21:53:32) to be 120 seconds older than starting chunk (2025-08-06 21:53:32)
+ sleep 10
+ [[ 1754517212 -gt 1754517212 ]]
+ [[ 1 -gt 30 ]]
++ get_latest_oplog_chunk_ts some-name
++ local cluster=some-name
+++ kubectl_bin exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ jq '.backups.pitrChunks.pitrChunks | last | .range.end'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.Nltqadvb6E
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.DK9JR9JSG2
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in '$(seq 0 2)'
+++ set +e
+++ kubectl exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.Nltqadvb6E
+++ cat /tmp/tmp.DK9JR9JSG2
+++ rm /tmp/tmp.Nltqadvb6E /tmp/tmp.DK9JR9JSG2
+++ return 0
++ echo 1754517212
+ latest_ts=1754517212
+ retries=2
++ format_date 1754517212
++ local timestamp=1754517212
+++ TZ=UTC
+++ /usr/bin/date -d@1754517212 '+%Y-%m-%d %H:%M:%S'
++ echo 2025-08-06 21:53:32
++ format_date 1754517212
++ local timestamp=1754517212
+++ TZ=UTC
+++ /usr/bin/date -d@1754517212 '+%Y-%m-%d %H:%M:%S'
++ echo 2025-08-06 21:53:32
+ echo 'Waiting for last oplog chunk (2025-08-06 21:53:32) to be 120 seconds older than starting chunk (2025-08-06 21:53:32)'
Waiting for last oplog chunk (2025-08-06 21:53:32) to be 120 seconds older than starting chunk (2025-08-06 21:53:32)
+ sleep 10
+ [[ 1754517212 -gt 1754517212 ]]
+ [[ 2 -gt 30 ]]
++ get_latest_oplog_chunk_ts some-name
++ local cluster=some-name
+++ kubectl_bin exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ jq '.backups.pitrChunks.pitrChunks | last | .range.end'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.RMPZoYBkBo
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.LMLPghp7KE
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in '$(seq 0 2)'
+++ set +e
+++ kubectl exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.RMPZoYBkBo
+++ cat /tmp/tmp.LMLPghp7KE
+++ rm /tmp/tmp.RMPZoYBkBo /tmp/tmp.LMLPghp7KE
+++ return 0
++ echo 1754517212
+ latest_ts=1754517212
+ retries=3
++ format_date 1754517212
++ local timestamp=1754517212
+++ TZ=UTC
+++ /usr/bin/date -d@1754517212 '+%Y-%m-%d %H:%M:%S'
++ echo 2025-08-06 21:53:32
++ format_date 1754517212
++ local timestamp=1754517212
+++ TZ=UTC
+++ /usr/bin/date -d@1754517212 '+%Y-%m-%d %H:%M:%S'
++ echo 2025-08-06 21:53:32
+ echo 'Waiting for last oplog chunk (2025-08-06 21:53:32) to be 120 seconds older than starting chunk (2025-08-06 21:53:32)'
Waiting for last oplog chunk (2025-08-06 21:53:32) to be 120 seconds older than starting chunk (2025-08-06 21:53:32)
+ sleep 10
+ [[ 1754517212 -gt 1754517212 ]]
+ [[ 3 -gt 30 ]]
++ get_latest_oplog_chunk_ts some-name
++ local cluster=some-name
+++ kubectl_bin exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ jq '.backups.pitrChunks.pitrChunks | last | .range.end'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.H3V2yAWaB3
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.CoYIhtZOyi
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in '$(seq 0 2)'
+++ set +e
+++ kubectl exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.H3V2yAWaB3
+++ cat /tmp/tmp.CoYIhtZOyi
+++ rm /tmp/tmp.H3V2yAWaB3 /tmp/tmp.CoYIhtZOyi
+++ return 0
++ echo 1754517212
+ latest_ts=1754517212
+ retries=4
++ format_date 1754517212
++ local timestamp=1754517212
+++ TZ=UTC
+++ /usr/bin/date -d@1754517212 '+%Y-%m-%d %H:%M:%S'
++ echo 2025-08-06 21:53:32
++ format_date 1754517212
++ local timestamp=1754517212
+++ TZ=UTC
+++ /usr/bin/date -d@1754517212 '+%Y-%m-%d %H:%M:%S'
++ echo 2025-08-06 21:53:32
+ echo 'Waiting for last oplog chunk (2025-08-06 21:53:32) to be 120 seconds older than starting chunk (2025-08-06 21:53:32)'
Waiting for last oplog chunk (2025-08-06 21:53:32) to be 120 seconds older than starting chunk (2025-08-06 21:53:32)
+ sleep 10
+ [[ 1754517212 -gt 1754517212 ]]
+ [[ 4 -gt 30 ]]
++ get_latest_oplog_chunk_ts some-name
++ local cluster=some-name
+++ kubectl_bin exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ jq '.backups.pitrChunks.pitrChunks | last | .range.end'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.t6SVywgmHi
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.AwwjzgFYQ3
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in '$(seq 0 2)'
+++ set +e
+++ kubectl exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.t6SVywgmHi
+++ cat /tmp/tmp.AwwjzgFYQ3
+++ rm /tmp/tmp.t6SVywgmHi /tmp/tmp.AwwjzgFYQ3
+++ return 0
++ echo 1754517212
+ latest_ts=1754517212
+ retries=5
++ format_date 1754517212
++ local timestamp=1754517212
+++ TZ=UTC
+++ /usr/bin/date -d@1754517212 '+%Y-%m-%d %H:%M:%S'
++ echo 2025-08-06 21:53:32
++ format_date 1754517212
++ local timestamp=1754517212
+++ TZ=UTC
+++ /usr/bin/date -d@1754517212 '+%Y-%m-%d %H:%M:%S'
++ echo 2025-08-06 21:53:32
+ echo 'Waiting for last oplog chunk (2025-08-06 21:53:32) to be 120 seconds older than starting chunk (2025-08-06 21:53:32)'
Waiting for last oplog chunk (2025-08-06 21:53:32) to be 120 seconds older than starting chunk (2025-08-06 21:53:32)
+ sleep 10
+ [[ 1754517212 -gt 1754517212 ]]
+ [[ 5 -gt 30 ]]
++ get_latest_oplog_chunk_ts some-name
++ local cluster=some-name
+++ kubectl_bin exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ jq '.backups.pitrChunks.pitrChunks | last | .range.end'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.9APLFQOQ7Q
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.ZrzEpi7N5L
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in '$(seq 0 2)'
+++ set +e
+++ kubectl exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.9APLFQOQ7Q
+++ cat /tmp/tmp.ZrzEpi7N5L
+++ rm /tmp/tmp.9APLFQOQ7Q /tmp/tmp.ZrzEpi7N5L
+++ return 0
++ echo 1754517212
+ latest_ts=1754517212
+ retries=6
++ format_date 1754517212
++ local timestamp=1754517212
+++ TZ=UTC
+++ /usr/bin/date -d@1754517212 '+%Y-%m-%d %H:%M:%S'
++ echo 2025-08-06 21:53:32
++ format_date 1754517212
++ local timestamp=1754517212
+++ TZ=UTC
+++ /usr/bin/date -d@1754517212 '+%Y-%m-%d %H:%M:%S'
++ echo 2025-08-06 21:53:32
+ echo 'Waiting for last oplog chunk (2025-08-06 21:53:32) to be 120 seconds older than starting chunk (2025-08-06 21:53:32)'
Waiting for last oplog chunk (2025-08-06 21:53:32) to be 120 seconds older than starting chunk (2025-08-06 21:53:32)
+ sleep 10
+ [[ 1754517212 -gt 1754517212 ]]
+ [[ 6 -gt 30 ]]
++ get_latest_oplog_chunk_ts some-name
++ local cluster=some-name
+++ kubectl_bin exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ jq '.backups.pitrChunks.pitrChunks | last | .range.end'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.PgZlhRd9SE
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.ADzwQeSZ9c
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in '$(seq 0 2)'
+++ set +e
+++ kubectl exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.PgZlhRd9SE
+++ cat /tmp/tmp.ADzwQeSZ9c
+++ rm /tmp/tmp.PgZlhRd9SE /tmp/tmp.ADzwQeSZ9c
+++ return 0
++ echo 1754517212
+ latest_ts=1754517212
+ retries=7
++ format_date 1754517212
++ local timestamp=1754517212
+++ TZ=UTC
+++ /usr/bin/date -d@1754517212 '+%Y-%m-%d %H:%M:%S'
++ echo 2025-08-06 21:53:32
++ format_date 1754517212
++ local timestamp=1754517212
+++ TZ=UTC
+++ /usr/bin/date -d@1754517212 '+%Y-%m-%d %H:%M:%S'
++ echo 2025-08-06 21:53:32
+ echo 'Waiting for last oplog chunk (2025-08-06 21:53:32) to be 120 seconds older than starting chunk (2025-08-06 21:53:32)'
Waiting for last oplog chunk (2025-08-06 21:53:32) to be 120 seconds older than starting chunk (2025-08-06 21:53:32)
+ sleep 10
+ [[ 1754517212 -gt 1754517212 ]]
+ [[ 7 -gt 30 ]]
++ get_latest_oplog_chunk_ts some-name
++ local cluster=some-name
+++ kubectl_bin exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ jq '.backups.pitrChunks.pitrChunks | last | .range.end'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.9WEp9MPsbP
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.yKwgfWbIFL
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in '$(seq 0 2)'
+++ set +e
+++ kubectl exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.9WEp9MPsbP
+++ cat /tmp/tmp.yKwgfWbIFL
+++ rm /tmp/tmp.9WEp9MPsbP /tmp/tmp.yKwgfWbIFL
+++ return 0
++ echo 1754517212
+ latest_ts=1754517212
+ retries=8
++ format_date 1754517212
++ local timestamp=1754517212
+++ TZ=UTC
+++ /usr/bin/date -d@1754517212 '+%Y-%m-%d %H:%M:%S'
++ echo 2025-08-06 21:53:32
++ format_date 1754517212
++ local timestamp=1754517212
+++ TZ=UTC
+++ /usr/bin/date -d@1754517212 '+%Y-%m-%d %H:%M:%S'
++ echo 2025-08-06 21:53:32
+ echo 'Waiting for last oplog chunk (2025-08-06 21:53:32) to be 120 seconds older than starting chunk (2025-08-06 21:53:32)'
Waiting for last oplog chunk (2025-08-06 21:53:32) to be 120 seconds older than starting chunk (2025-08-06 21:53:32)
+ sleep 10
+ [[ 1754517212 -gt 1754517212 ]]
+ [[ 8 -gt 30 ]]
++ get_latest_oplog_chunk_ts some-name
++ local cluster=some-name
+++ kubectl_bin exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ jq '.backups.pitrChunks.pitrChunks | last | .range.end'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.fbofQQ9v98
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.bw2kD30D3g
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in '$(seq 0 2)'
+++ set +e
+++ kubectl exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.fbofQQ9v98
+++ cat /tmp/tmp.bw2kD30D3g
+++ rm /tmp/tmp.fbofQQ9v98 /tmp/tmp.bw2kD30D3g
+++ return 0
++ echo 1754517212
+ latest_ts=1754517212
+ retries=9
++ format_date 1754517212
++ local timestamp=1754517212
+++ TZ=UTC
+++ /usr/bin/date -d@1754517212 '+%Y-%m-%d %H:%M:%S'
++ echo 2025-08-06 21:53:32
++ format_date 1754517212
++ local timestamp=1754517212
+++ TZ=UTC
+++ /usr/bin/date -d@1754517212 '+%Y-%m-%d %H:%M:%S'
++ echo 2025-08-06 21:53:32
+ echo 'Waiting for last oplog chunk (2025-08-06 21:53:32) to be 120 seconds older than starting chunk (2025-08-06 21:53:32)'
Waiting for last oplog chunk (2025-08-06 21:53:32) to be 120 seconds older than starting chunk (2025-08-06 21:53:32)
+ sleep 10
+ [[ 1754517212 -gt 1754517212 ]]
+ [[ 9 -gt 30 ]]
++ get_latest_oplog_chunk_ts some-name
++ local cluster=some-name
+++ kubectl_bin exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ jq '.backups.pitrChunks.pitrChunks | last | .range.end'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.jafIaHEFbS
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.a5ZPQijCjy
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in '$(seq 0 2)'
+++ set +e
+++ kubectl exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.jafIaHEFbS
+++ cat /tmp/tmp.a5ZPQijCjy
+++ rm /tmp/tmp.jafIaHEFbS /tmp/tmp.a5ZPQijCjy
+++ return 0
++ echo 1754517212
+ latest_ts=1754517212
+ retries=10
++ format_date 1754517212
++ local timestamp=1754517212
+++ TZ=UTC
+++ /usr/bin/date -d@1754517212 '+%Y-%m-%d %H:%M:%S'
++ echo 2025-08-06 21:53:32
++ format_date 1754517212
++ local timestamp=1754517212
+++ TZ=UTC
+++ /usr/bin/date -d@1754517212 '+%Y-%m-%d %H:%M:%S'
++ echo 2025-08-06 21:53:32
+ echo 'Waiting for last oplog chunk (2025-08-06 21:53:32) to be 120 seconds older than starting chunk (2025-08-06 21:53:32)'
Waiting for last oplog chunk (2025-08-06 21:53:32) to be 120 seconds older than starting chunk (2025-08-06 21:53:32)
+ sleep 10
+ [[ 1754517212 -gt 1754517212 ]]
+ [[ 10 -gt 30 ]]
++ get_latest_oplog_chunk_ts some-name
++ local cluster=some-name
+++ kubectl_bin exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ jq '.backups.pitrChunks.pitrChunks | last | .range.end'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.OCCw9ckVJM
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.tHlvPEfB2I
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in '$(seq 0 2)'
+++ set +e
+++ kubectl exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.OCCw9ckVJM
+++ cat /tmp/tmp.tHlvPEfB2I
+++ rm /tmp/tmp.OCCw9ckVJM /tmp/tmp.tHlvPEfB2I
+++ return 0
++ echo 1754518613
+ latest_ts=1754518613
+ retries=11
++ format_date 1754518613
++ local timestamp=1754518613
+++ TZ=UTC
+++ /usr/bin/date -d@1754518613 '+%Y-%m-%d %H:%M:%S'
++ echo 2025-08-06 22:16:53
++ format_date 1754517212
++ local timestamp=1754517212
+++ TZ=UTC
+++ /usr/bin/date -d@1754517212 '+%Y-%m-%d %H:%M:%S'
++ echo 2025-08-06 21:53:32
+ echo 'Waiting for last oplog chunk (2025-08-06 22:16:53) to be 120 seconds older than starting chunk (2025-08-06 21:53:32)'
Waiting for last oplog chunk (2025-08-06 22:16:53) to be 120 seconds older than starting chunk (2025-08-06 21:53:32)
+ sleep 10