Log: /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2328/e2e-tests/logs/pitr-physical-backup-source.log
Warning: version difference between client (1.36) and server (1.33) exceeds the supported minor version skew of +/-1
Warning: version difference between client (1.36) and server (1.33) exceeds the supported minor version skew of +/-1
Warning: version difference between client (1.36) and server (1.33) exceeds the supported minor version skew of +/-1
+ main
+ create_infra pitr-physical-backup-source-27258
+ local ns=pitr-physical-backup-source-27258
+ [[ 1 == 1 ]]
+ delete_crd
+ desc 'get and delete old CRDs and RBAC'
+ set +o xtrace


-----------------------------------------------------------------------------------
get and delete old CRDs and RBAC
-----------------------------------------------------------------------------------

+ kubectl_bin delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2328/deploy/crd.yaml --ignore-not-found --wait=false
++ mktemp
+ local LAST_OUT=/tmp/tmp.8R9KB9ZeDM
++ mktemp
+ local LAST_ERR=/tmp/tmp.sYD6Q8qpbQ
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2328/deploy/crd.yaml --ignore-not-found --wait=false
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.8R9KB9ZeDM
customresourcedefinition.apiextensions.k8s.io "perconaservermongodbbackups.psmdb.percona.com" deleted
customresourcedefinition.apiextensions.k8s.io "perconaservermongodbrestores.psmdb.percona.com" deleted
customresourcedefinition.apiextensions.k8s.io "perconaservermongodbs.psmdb.percona.com" deleted
+ cat /tmp/tmp.sYD6Q8qpbQ
+ rm /tmp/tmp.8R9KB9ZeDM /tmp/tmp.sYD6Q8qpbQ
+ return 0
++ yq eval .metadata.name /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2328/deploy/crd.yaml
++ grep -v '\-\-\-'
grep: warning: stray \ before -
grep: warning: stray \ before -
+ for crd_name in $(yq eval '.metadata.name' "${src_dir}/deploy/crd.yaml" | grep -v '\-\-\-')
+ xargs -L 1 sh -xc 'kubectl patch perconaservermongodbbackups.psmdb.percona.com -n $0 $1 --type=merge -p "{\"metadata\":{\"finalizers\":[]}}"'
+ kubectl get perconaservermongodbbackups.psmdb.percona.com --all-namespaces -o wide
+ grep -v NAMESPACE
No resources found
+ kubectl patch perconaservermongodbbackups.psmdb.percona.com -n sh --type=merge -p '{"metadata":{"finalizers":[]}}'
error: resource(s) were provided, but no name was specified
+ :
+ kubectl_bin wait --for=delete crd perconaservermongodbbackups.psmdb.percona.com
++ mktemp
+ local LAST_OUT=/tmp/tmp.yxsVWg0efD
++ mktemp
+ local LAST_ERR=/tmp/tmp.0UNvgM2fv3
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl wait --for=delete crd perconaservermongodbbackups.psmdb.percona.com
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.yxsVWg0efD
+ cat /tmp/tmp.0UNvgM2fv3
+ rm /tmp/tmp.yxsVWg0efD /tmp/tmp.0UNvgM2fv3
+ return 0
+ for crd_name in $(yq eval '.metadata.name' "${src_dir}/deploy/crd.yaml" | grep -v '\-\-\-')
+ kubectl get perconaservermongodbrestores.psmdb.percona.com --all-namespaces -o wide
+ xargs -L 1 sh -xc 'kubectl patch perconaservermongodbrestores.psmdb.percona.com -n $0 $1 --type=merge -p "{\"metadata\":{\"finalizers\":[]}}"'
+ grep -v NAMESPACE
error: the server doesn't have a resource type "perconaservermongodbrestores"
+ kubectl patch perconaservermongodbrestores.psmdb.percona.com -n sh --type=merge -p '{"metadata":{"finalizers":[]}}'
error: the server doesn't have a resource type "perconaservermongodbrestores"
+ :
+ kubectl_bin wait --for=delete crd perconaservermongodbrestores.psmdb.percona.com
++ mktemp
+ local LAST_OUT=/tmp/tmp.Blm4vv2TIv
++ mktemp
+ local LAST_ERR=/tmp/tmp.CWm88p9Epk
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl wait --for=delete crd perconaservermongodbrestores.psmdb.percona.com
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.Blm4vv2TIv
+ cat /tmp/tmp.CWm88p9Epk
+ rm /tmp/tmp.Blm4vv2TIv /tmp/tmp.CWm88p9Epk
+ return 0
+ for crd_name in $(yq eval '.metadata.name' "${src_dir}/deploy/crd.yaml" | grep -v '\-\-\-')
+ grep -v NAMESPACE
+ kubectl get perconaservermongodbs.psmdb.percona.com --all-namespaces -o wide
+ xargs -L 1 sh -xc 'kubectl patch perconaservermongodbs.psmdb.percona.com -n $0 $1 --type=merge -p "{\"metadata\":{\"finalizers\":[]}}"'
error: the server doesn't have a resource type "perconaservermongodbs"
+ kubectl patch perconaservermongodbs.psmdb.percona.com -n sh --type=merge -p '{"metadata":{"finalizers":[]}}'
error: the server doesn't have a resource type "perconaservermongodbs"
+ :
+ kubectl_bin wait --for=delete crd perconaservermongodbs.psmdb.percona.com
++ mktemp
+ local LAST_OUT=/tmp/tmp.XcOBlA5QS7
++ mktemp
+ local LAST_ERR=/tmp/tmp.CTF2iRisCL
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl wait --for=delete crd perconaservermongodbs.psmdb.percona.com
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.XcOBlA5QS7
+ cat /tmp/tmp.CTF2iRisCL
+ rm /tmp/tmp.XcOBlA5QS7 /tmp/tmp.CTF2iRisCL
+ return 0
+ local rbac_yaml=rbac.yaml
+ '[' -n psmdb-operator ']'
+ rbac_yaml=cw-rbac.yaml
+ kubectl_bin delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2328/deploy/cw-rbac.yaml --ignore-not-found
++ mktemp
+ local LAST_OUT=/tmp/tmp.2nvRfkTjdH
++ mktemp
+ local LAST_ERR=/tmp/tmp.Bkm2IiadT2
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2328/deploy/cw-rbac.yaml --ignore-not-found
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.2nvRfkTjdH
clusterrole.rbac.authorization.k8s.io "percona-server-mongodb-operator" deleted
clusterrolebinding.rbac.authorization.k8s.io "service-account-percona-server-mongodb-operator" deleted
+ cat /tmp/tmp.Bkm2IiadT2
+ rm /tmp/tmp.2nvRfkTjdH /tmp/tmp.Bkm2IiadT2
+ return 0
+ check_crd_for_deletion PR-2328-37c432905
+ local git_tag=PR-2328-37c432905
++ curl -s https://raw.githubusercontent.com/percona/percona-server-mongodb-operator/PR-2328-37c432905/deploy/crd.yaml
++ yq eval .metadata.name
++ /usr/sbin/sed s/---//g
++ /usr/sbin/sed ':a;N;$!ba;s/\n/ /g'
+ for crd_name in $(curl -s https://raw.githubusercontent.com/percona/percona-server-mongodb-operator/${git_tag}/deploy/crd.yaml | yq eval '.metadata.name' | $sed 's/---//g' | $sed ':a;N;$!ba;s/\n/ /g')
++ kubectl_bin get crd/null -o 'jsonpath={.status.conditions[-1].type}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.7mtVNwBvN6
+++ mktemp
++ local LAST_ERR=/tmp/tmp.HgZAI1W5UT
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get crd/null -o 'jsonpath={.status.conditions[-1].type}'
++ exit_status=1
++ set -e
++ '[' 1 '!=' 0 -a -n 1 ']'
++ cat /tmp/tmp.7mtVNwBvN6
++ cat /tmp/tmp.HgZAI1W5UT
Error from server (NotFound): customresourcedefinitions.apiextensions.k8s.io "null" not found
++ sleep 0
++ for i in $(seq 0 2)
++ set +e
++ kubectl get crd/null -o 'jsonpath={.status.conditions[-1].type}'
++ exit_status=1
++ set -e
++ '[' 1 '!=' 0 -a -n 1 ']'
++ cat /tmp/tmp.7mtVNwBvN6
++ cat /tmp/tmp.HgZAI1W5UT
Error from server (NotFound): customresourcedefinitions.apiextensions.k8s.io "null" not found
++ sleep 4
++ for i in $(seq 0 2)
++ set +e
++ kubectl get crd/null -o 'jsonpath={.status.conditions[-1].type}'
++ exit_status=1
++ set -e
++ '[' 1 '!=' 0 -a -n 1 ']'
++ cat /tmp/tmp.7mtVNwBvN6
++ cat /tmp/tmp.HgZAI1W5UT
Error from server (NotFound): customresourcedefinitions.apiextensions.k8s.io "null" not found
++ sleep 8
++ cat /tmp/tmp.7mtVNwBvN6
++ cat /tmp/tmp.HgZAI1W5UT
Error from server (NotFound): customresourcedefinitions.apiextensions.k8s.io "null" not found
++ rm /tmp/tmp.7mtVNwBvN6 /tmp/tmp.HgZAI1W5UT
++ return 1
+ [[ '' == Terminating ]]
+ '[' -n psmdb-operator ']'
+ create_namespace psmdb-operator
+ local namespace=psmdb-operator
+ local skip_clean_namespace=
+ [[ 1 == 1 ]]
+ [[ -z '' ]]
+ destroy_chaos_mesh
++ awk '-F ' '{print $2}'
++ sed s/NAMESPACE//
++ helm list --all-namespaces --filter chaos-mesh
++ tail -n1
+ local chaos_mesh_ns=
+ desc 'destroy chaos-mesh'
+ set +o xtrace


-----------------------------------------------------------------------------------
destroy chaos-mesh
-----------------------------------------------------------------------------------

+ '[' -n '' ']'
++ grep chaos-mesh
++ awk '{print $1}'
++ kubectl get MutatingWebhookConfiguration
+ timeout 30 kubectl delete MutatingWebhookConfiguration
error: resource(s) were provided, but no name was specified
+ :
++ kubectl get ValidatingWebhookConfiguration
++ grep chaos-mesh
++ awk '{print $1}'
+ timeout 30 kubectl delete ValidatingWebhookConfiguration
error: resource(s) were provided, but no name was specified
+ :
++ grep validate-auth
++ awk '{print $1}'
++ kubectl get ValidatingWebhookConfiguration
+ timeout 30 kubectl delete ValidatingWebhookConfiguration
error: resource(s) were provided, but no name was specified
+ :
++ kubectl api-resources
++ grep chaos-mesh
++ awk '{print $1}'
++ kubectl get crd
++ grep chaos-mesh.org
++ awk '{print $1}'
+ timeout 30 kubectl delete crd
error: resource(s) were provided, but no name was specified
+ :
++ kubectl get clusterrolebinding
++ grep chaos-mesh
++ awk '{print $1}'
+ timeout 30 kubectl delete clusterrolebinding
error: resource(s) were provided, but no name was specified
+ :
++ kubectl get clusterrole
++ grep chaos-mesh
++ awk '{print $1}'
+ timeout 30 kubectl delete clusterrole
error: resource(s) were provided, but no name was specified
+ :
+ desc 'cleaned up all old namespaces'
+ set +o xtrace


-----------------------------------------------------------------------------------
cleaned up all old namespaces
-----------------------------------------------------------------------------------

+ kubectl_bin get ns
+ grep -E -v '^kube-|^default|Terminating|psmdb-operator|openshift|^gke-|^gmp-|^NAME'
+ awk '{print$1}'
++ mktemp
+ '[' -n '' ']'
+ xargs kubectl delete ns
+ desc 'cleaned up old namespaces psmdb-operator'
+ set +o xtrace


-----------------------------------------------------------------------------------
cleaned up old namespaces psmdb-operator
-----------------------------------------------------------------------------------

+ kubectl_bin delete namespace psmdb-operator --ignore-not-found
++ mktemp
+ local LAST_OUT=/tmp/tmp.B8ivv87dTX
++ mktemp
+ local LAST_OUT=/tmp/tmp.czBCyQA5mN
++ mktemp
+ local LAST_ERR=/tmp/tmp.3GdJBdjp7Y
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ local LAST_ERR=/tmp/tmp.FRym6kyOBM
+ local exit_status=0
+ local timeout=4
+ for i in $(seq 0 2)
+ set +e
+ kubectl get ns
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl delete namespace psmdb-operator --ignore-not-found
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.B8ivv87dTX
+ cat /tmp/tmp.3GdJBdjp7Y
+ rm /tmp/tmp.B8ivv87dTX /tmp/tmp.3GdJBdjp7Y
+ return 0
namespace "pitr-physical-backup-source-19784" deleted
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.czBCyQA5mN
namespace "psmdb-operator" deleted
+ cat /tmp/tmp.FRym6kyOBM
+ rm /tmp/tmp.czBCyQA5mN /tmp/tmp.FRym6kyOBM
+ return 0
+ kubectl_bin wait --for=delete namespace psmdb-operator
++ mktemp
+ local LAST_OUT=/tmp/tmp.wpNz0pjASD
++ mktemp
+ local LAST_ERR=/tmp/tmp.wpwWr44mV9
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl wait --for=delete namespace psmdb-operator
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.wpNz0pjASD
+ cat /tmp/tmp.wpwWr44mV9
+ rm /tmp/tmp.wpNz0pjASD /tmp/tmp.wpwWr44mV9
+ return 0
+ desc 'create namespace psmdb-operator'
+ set +o xtrace


-----------------------------------------------------------------------------------
create namespace psmdb-operator
-----------------------------------------------------------------------------------

+ kubectl_bin create namespace psmdb-operator
++ mktemp
+ local LAST_OUT=/tmp/tmp.lv9TGS0x0T
++ mktemp
+ local LAST_ERR=/tmp/tmp.Zd3Tf9HbRS
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl create namespace psmdb-operator
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.lv9TGS0x0T
namespace/psmdb-operator created
+ cat /tmp/tmp.Zd3Tf9HbRS
+ rm /tmp/tmp.lv9TGS0x0T /tmp/tmp.Zd3Tf9HbRS
+ return 0
+ set_kube_ctx psmdb-operator
+ local namespace=psmdb-operator
++ kubectl_bin config current-context
+++ mktemp
++ local LAST_OUT=/tmp/tmp.AHmnavoYJm
+++ mktemp
++ local LAST_ERR=/tmp/tmp.tQXIdhWGgA
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl config current-context
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.AHmnavoYJm
++ cat /tmp/tmp.tQXIdhWGgA
++ rm /tmp/tmp.AHmnavoYJm /tmp/tmp.tQXIdhWGgA
++ return 0
+ kubectl_bin config set-context gke_cloud-dev-112233_us-central1-a_jen-psmdb-2328-37c432905-3-cluster5 --namespace=psmdb-operator
++ mktemp
+ local LAST_OUT=/tmp/tmp.HJ7lEFDwr4
++ mktemp
+ local LAST_ERR=/tmp/tmp.vnfKdH88ez
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl config set-context gke_cloud-dev-112233_us-central1-a_jen-psmdb-2328-37c432905-3-cluster5 --namespace=psmdb-operator
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.HJ7lEFDwr4
Context "gke_cloud-dev-112233_us-central1-a_jen-psmdb-2328-37c432905-3-cluster5" modified.
+ cat /tmp/tmp.vnfKdH88ez
+ rm /tmp/tmp.HJ7lEFDwr4 /tmp/tmp.vnfKdH88ez
+ return 0
+ deploy_operator
+ desc 'start PSMDB operator: docker.io/perconalab/percona-server-mongodb-operator:PR-2328-37c432905'
+ set +o xtrace


-----------------------------------------------------------------------------------
start PSMDB operator: docker.io/perconalab/percona-server-mongodb-operator:PR-2328-37c432905
-----------------------------------------------------------------------------------

+ local cr_file
+ '[' -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2328/e2e-tests/pitr-physical-backup-source/conf/crd.yaml ']'
+ cr_file=/mnt/jenkins/workspace/cloud-psmdb-operator_PR-2328/deploy/crd.yaml
+ kubectl_bin apply --server-side --force-conflicts -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2328/deploy/crd.yaml
++ mktemp
+ local LAST_OUT=/tmp/tmp.OD7SUV67zq
++ mktemp
+ local LAST_ERR=/tmp/tmp.fGRF3RxH9j
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl apply --server-side --force-conflicts -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2328/deploy/crd.yaml
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.OD7SUV67zq
customresourcedefinition.apiextensions.k8s.io/perconaservermongodbbackups.psmdb.percona.com serverside-applied
customresourcedefinition.apiextensions.k8s.io/perconaservermongodbrestores.psmdb.percona.com serverside-applied
customresourcedefinition.apiextensions.k8s.io/perconaservermongodbs.psmdb.percona.com serverside-applied
+ cat /tmp/tmp.fGRF3RxH9j
+ rm /tmp/tmp.OD7SUV67zq /tmp/tmp.fGRF3RxH9j
+ return 0
+ '[' -n psmdb-operator ']'
+ apply_rbac cw-rbac
+ local operator_namespace=psmdb-operator
+ local rbac=cw-rbac
+ cat /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2328/deploy/cw-rbac.yaml
+ sed -e 's^namespace: .*^namespace: psmdb-operator^'
+ kubectl_bin apply -n psmdb-operator -f -
++ mktemp
+ local LAST_OUT=/tmp/tmp.1NgWecAxDR
++ mktemp
+ local LAST_ERR=/tmp/tmp.nTGdQ5zTXv
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl apply -n psmdb-operator -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.1NgWecAxDR
clusterrole.rbac.authorization.k8s.io/percona-server-mongodb-operator created
serviceaccount/percona-server-mongodb-operator created
clusterrolebinding.rbac.authorization.k8s.io/service-account-percona-server-mongodb-operator created
+ cat /tmp/tmp.nTGdQ5zTXv
+ rm /tmp/tmp.1NgWecAxDR /tmp/tmp.nTGdQ5zTXv
+ return 0
+ yq eval $'\n\t\t\t(.spec.template.spec.containers[].image = "docker.io/perconalab/percona-server-mongodb-operator:PR-2328-37c432905") |\n\t\t\t((.. | select(.[] == "DISABLE_TELEMETRY")) |= .value="true") |\n\t\t\t((.. | select(.[] == "LOG_LEVEL")) |= .value="DEBUG")' /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2328/deploy/cw-operator.yaml
+ kubectl_bin apply -n psmdb-operator -f -
++ mktemp
+ local LAST_OUT=/tmp/tmp.K2b3pttS04
++ mktemp
+ local LAST_ERR=/tmp/tmp.W1rYbr0dX0
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl apply -n psmdb-operator -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.K2b3pttS04
deployment.apps/percona-server-mongodb-operator created
+ cat /tmp/tmp.W1rYbr0dX0
+ rm /tmp/tmp.K2b3pttS04 /tmp/tmp.W1rYbr0dX0
+ return 0
+ sleep 20
++ get_operator_pod
++ kubectl_bin get pods --selector=name=percona-server-mongodb-operator -o 'jsonpath={.items[].metadata.name}' -n psmdb-operator
+++ mktemp
++ local LAST_OUT=/tmp/tmp.Yz4pNrpRoa
+++ mktemp
++ local LAST_ERR=/tmp/tmp.p1FDTeTglp
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=percona-server-mongodb-operator -o 'jsonpath={.items[].metadata.name}' -n psmdb-operator
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.Yz4pNrpRoa
++ cat /tmp/tmp.p1FDTeTglp
++ rm /tmp/tmp.Yz4pNrpRoa /tmp/tmp.p1FDTeTglp
++ return 0
+ wait_operator_pod percona-server-mongodb-operator-7dbf5fc47b-gt2jw
+ local pod=percona-server-mongodb-operator-7dbf5fc47b-gt2jw
+ set +o xtrace
waiting for pod/percona-server-mongodb-operator-7dbf5fc47b-gt2jw to be ready.OK
+ echo 'Print operator info from log'
Print operator info from log
+ grep 'Manager starting up'
++ get_operator_pod
++ kubectl_bin get pods --selector=name=percona-server-mongodb-operator -o 'jsonpath={.items[].metadata.name}' -n psmdb-operator
+++ mktemp
++ local LAST_OUT=/tmp/tmp.hX691Ivq6V
+++ mktemp
++ local LAST_ERR=/tmp/tmp.liTee4Zdbq
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=percona-server-mongodb-operator -o 'jsonpath={.items[].metadata.name}' -n psmdb-operator
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.hX691Ivq6V
++ cat /tmp/tmp.liTee4Zdbq
++ rm /tmp/tmp.hX691Ivq6V /tmp/tmp.liTee4Zdbq
++ return 0
+ kubectl_bin logs -n psmdb-operator percona-server-mongodb-operator-7dbf5fc47b-gt2jw
++ mktemp
+ local LAST_OUT=/tmp/tmp.lJyD03ryLu
++ mktemp
+ local LAST_ERR=/tmp/tmp.uOfoNxSR3P
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl logs -n psmdb-operator percona-server-mongodb-operator-7dbf5fc47b-gt2jw
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.lJyD03ryLu
+ cat /tmp/tmp.uOfoNxSR3P
+ rm /tmp/tmp.lJyD03ryLu /tmp/tmp.uOfoNxSR3P
+ return 0
2026-05-21T08:59:36.270Z	INFO	setup	Manager starting up	{"gitCommit": "37c432905eccde48178453d4542cff5f669efa6b", "gitBranch": "PR-2328-37c432905", "buildTime": "", "goVersion": "go1.26.3", "os": "linux", "arch": "amd64"}
+ create_namespace pitr-physical-backup-source-27258
+ local namespace=pitr-physical-backup-source-27258
+ local skip_clean_namespace=
+ [[ 1 == 1 ]]
+ [[ -z '' ]]
+ destroy_chaos_mesh
++ helm list --all-namespaces --filter chaos-mesh
++ awk '-F ' '{print $2}'
++ sed s/NAMESPACE//
++ tail -n1
+ local chaos_mesh_ns=
+ desc 'destroy chaos-mesh'
+ set +o xtrace


-----------------------------------------------------------------------------------
destroy chaos-mesh
-----------------------------------------------------------------------------------

+ '[' -n '' ']'
++ kubectl get MutatingWebhookConfiguration
++ awk '{print $1}'
++ grep chaos-mesh
+ timeout 30 kubectl delete MutatingWebhookConfiguration
error: resource(s) were provided, but no name was specified
+ :
++ kubectl get ValidatingWebhookConfiguration
++ awk '{print $1}'
++ grep chaos-mesh
+ timeout 30 kubectl delete ValidatingWebhookConfiguration
error: resource(s) were provided, but no name was specified
+ :
++ kubectl get ValidatingWebhookConfiguration
++ awk '{print $1}'
++ grep validate-auth
+ timeout 30 kubectl delete ValidatingWebhookConfiguration
error: resource(s) were provided, but no name was specified
+ :
++ kubectl api-resources
++ grep chaos-mesh
++ awk '{print $1}'
++ awk '{print $1}'
++ kubectl get crd
++ grep chaos-mesh.org
+ timeout 30 kubectl delete crd
error: resource(s) were provided, but no name was specified
+ :
++ grep chaos-mesh
++ awk '{print $1}'
++ kubectl get clusterrolebinding
+ timeout 30 kubectl delete clusterrolebinding
error: resource(s) were provided, but no name was specified
+ :
++ kubectl get clusterrole
++ grep chaos-mesh
++ awk '{print $1}'
+ timeout 30 kubectl delete clusterrole
error: resource(s) were provided, but no name was specified
+ :
+ desc 'cleaned up all old namespaces'
+ set +o xtrace


-----------------------------------------------------------------------------------
cleaned up all old namespaces
-----------------------------------------------------------------------------------

+ '[' -n '' ']'
+ kubectl_bin get ns
+ desc 'cleaned up old namespaces pitr-physical-backup-source-27258'
+ grep -E -v '^kube-|^default|Terminating|psmdb-operator|openshift|^gke-|^gmp-|^NAME'
+ xargs kubectl delete ns
+ set +o xtrace


-----------------------------------------------------------------------------------
cleaned up old namespaces pitr-physical-backup-source-27258
-----------------------------------------------------------------------------------

+ kubectl_bin delete namespace pitr-physical-backup-source-27258 --ignore-not-found
+ awk '{print$1}'
++ mktemp
++ mktemp
+ local LAST_OUT=/tmp/tmp.DQbvlwBPDZ
++ mktemp
+ local LAST_OUT=/tmp/tmp.7xmYB8POzb
++ mktemp
+ local LAST_ERR=/tmp/tmp.I8geoJwNN4
+ local exit_status=0
+ local timeout=4
+ local LAST_ERR=/tmp/tmp.sVeMzuOeYJ
+ local exit_status=0
+ local timeout=4
++ seq 0 2
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl delete namespace pitr-physical-backup-source-27258 --ignore-not-found
+ for i in $(seq 0 2)
+ set +e
+ kubectl get ns
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.DQbvlwBPDZ
+ cat /tmp/tmp.I8geoJwNN4
+ rm /tmp/tmp.DQbvlwBPDZ /tmp/tmp.I8geoJwNN4
+ return 0
error: resource(s) were provided, but no name was specified
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.7xmYB8POzb
+ cat /tmp/tmp.sVeMzuOeYJ
+ rm /tmp/tmp.7xmYB8POzb /tmp/tmp.sVeMzuOeYJ
+ return 0
+ kubectl_bin wait --for=delete namespace pitr-physical-backup-source-27258
++ mktemp
+ local LAST_OUT=/tmp/tmp.3Y45VfqtIa
++ mktemp
+ local LAST_ERR=/tmp/tmp.fqgtaqTZQZ
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl wait --for=delete namespace pitr-physical-backup-source-27258
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.3Y45VfqtIa
+ cat /tmp/tmp.fqgtaqTZQZ
+ rm /tmp/tmp.3Y45VfqtIa /tmp/tmp.fqgtaqTZQZ
+ return 0
+ desc 'create namespace pitr-physical-backup-source-27258'
+ set +o xtrace


-----------------------------------------------------------------------------------
create namespace pitr-physical-backup-source-27258
-----------------------------------------------------------------------------------

+ kubectl_bin create namespace pitr-physical-backup-source-27258
++ mktemp
+ local LAST_OUT=/tmp/tmp.7G5I43dIJD
++ mktemp
+ local LAST_ERR=/tmp/tmp.h1JfdlJMPy
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl create namespace pitr-physical-backup-source-27258
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.7G5I43dIJD
namespace/pitr-physical-backup-source-27258 created
+ cat /tmp/tmp.h1JfdlJMPy
+ rm /tmp/tmp.7G5I43dIJD /tmp/tmp.h1JfdlJMPy
+ return 0
+ set_kube_ctx pitr-physical-backup-source-27258
+ local namespace=pitr-physical-backup-source-27258
++ kubectl_bin config current-context
+++ mktemp
++ local LAST_OUT=/tmp/tmp.lya7sEHioB
+++ mktemp
++ local LAST_ERR=/tmp/tmp.PT6sXkXfcQ
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl config current-context
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.lya7sEHioB
++ cat /tmp/tmp.PT6sXkXfcQ
++ rm /tmp/tmp.lya7sEHioB /tmp/tmp.PT6sXkXfcQ
++ return 0
+ kubectl_bin config set-context gke_cloud-dev-112233_us-central1-a_jen-psmdb-2328-37c432905-3-cluster5 --namespace=pitr-physical-backup-source-27258
++ mktemp
+ local LAST_OUT=/tmp/tmp.LTtogl5yjK
++ mktemp
+ local LAST_ERR=/tmp/tmp.BpGy5Q9XUr
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl config set-context gke_cloud-dev-112233_us-central1-a_jen-psmdb-2328-37c432905-3-cluster5 --namespace=pitr-physical-backup-source-27258
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.LTtogl5yjK
Context "gke_cloud-dev-112233_us-central1-a_jen-psmdb-2328-37c432905-3-cluster5" modified.
+ cat /tmp/tmp.BpGy5Q9XUr
+ rm /tmp/tmp.LTtogl5yjK /tmp/tmp.BpGy5Q9XUr
+ return 0
+ deploy_minio
+ local cert_secret=
+ local service_name=minio-service
+ desc 'install MinIO: minio-service'
+ set +o xtrace


-----------------------------------------------------------------------------------
install MinIO: minio-service
-----------------------------------------------------------------------------------

+ helm uninstall minio-service
+ :
+ helm repo remove minio
"minio" has been removed from your repositories
+ helm repo add minio https://charts.min.io/
"minio" has been added to your repositories
+ local endpoint=http://minio-service:9000
+ minio_args=('--version' '5.4.0' '--set' 'replicas=1' '--set' 'mode=standalone' '--set' 'resources.requests.memory=256Mi' '--set' 'rootUser=rootuser' '--set' 'rootPassword=rootpass123' '--set' 'users[0].accessKey=some-access-key' '--set' 'users[0].secretKey=some-secret-key' '--set' 'users[0].policy=consoleAdmin' '--set' 'service.type=ClusterIP' '--set' 'configPathmc=/tmp/' '--set' 'securityContext.enabled=false' '--set' 'persistence.size=2G' '--set' 'fullnameOverride=minio-service' '--set' 'serviceAccount.create=true' '--set' 'serviceAccount.name=minio-service-sa')
+ local minio_args
+ [[ -n '' ]]
+ retry 10 60 helm install minio-service --version 5.4.0 --set replicas=1 --set mode=standalone --set resources.requests.memory=256Mi --set rootUser=rootuser --set rootPassword=rootpass123 --set 'users[0].accessKey=some-access-key' --set 'users[0].secretKey=some-secret-key' --set 'users[0].policy=consoleAdmin' --set service.type=ClusterIP --set configPathmc=/tmp/ --set securityContext.enabled=false --set persistence.size=2G --set fullnameOverride=minio-service --set serviceAccount.create=true --set serviceAccount.name=minio-service-sa minio/minio
+ local max=10
+ local delay=60
+ shift 2
+ local n=1
+ helm install minio-service --version 5.4.0 --set replicas=1 --set mode=standalone --set resources.requests.memory=256Mi --set rootUser=rootuser --set rootPassword=rootpass123 --set 'users[0].accessKey=some-access-key' --set 'users[0].secretKey=some-secret-key' --set 'users[0].policy=consoleAdmin' --set service.type=ClusterIP --set configPathmc=/tmp/ --set securityContext.enabled=false --set persistence.size=2G --set fullnameOverride=minio-service --set serviceAccount.create=true --set serviceAccount.name=minio-service-sa minio/minio
NAME: minio-service
LAST DEPLOYED: Thu May 21 09:00:17 2026
NAMESPACE: pitr-physical-backup-source-27258
STATUS: deployed
REVISION: 1
TEST SUITE: None
NOTES:
MinIO can be accessed via port 9000 on the following DNS name from within your cluster:
minio-service.pitr-physical-backup-source-27258.cluster.local

To access MinIO from localhost, run the below commands:

  1. export POD_NAME=$(kubectl get pods --namespace pitr-physical-backup-source-27258 -l "release=minio-service" -o jsonpath="{.items[0].metadata.name}")

  2. kubectl port-forward $POD_NAME 9000 --namespace pitr-physical-backup-source-27258

Read more about port forwarding here: http://kubernetes.io/docs/user-guide/kubectl/kubectl_port-forward/

You can now access MinIO server on http://localhost:9000. Follow the below steps to connect to MinIO server with mc client:

  1. Download the MinIO mc client - https://min.io/docs/minio/linux/reference/minio-mc.html#quickstart

  2. export MC_HOST_minio-service-local=http://$(kubectl get secret --namespace pitr-physical-backup-source-27258 minio-service -o jsonpath="{.data.rootUser}" | base64 --decode):$(kubectl get secret --namespace pitr-physical-backup-source-27258 minio-service -o jsonpath="{.data.rootPassword}" | base64 --decode)@localhost:9000

  3. mc ls minio-service-local
++ kubectl_bin get pods --selector=release=minio-service -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.wRbnEnrVuM
+++ mktemp
++ local LAST_ERR=/tmp/tmp.MVbhbfL2GU
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=release=minio-service -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.wRbnEnrVuM
++ cat /tmp/tmp.MVbhbfL2GU
++ rm /tmp/tmp.wRbnEnrVuM /tmp/tmp.MVbhbfL2GU
++ return 0
+ local MINIO_POD=minio-service-6d5f646cdc-5bgjs
+ wait_pod minio-service-6d5f646cdc-5bgjs
+ local pod=minio-service-6d5f646cdc-5bgjs
+ set +o xtrace
waiting for pod/minio-service-6d5f646cdc-5bgjs to be ready.OK
+ '[' -n psmdb-operator ']'
+ kubectl_bin create svc -n psmdb-operator externalname minio-service --external-name=minio-service.pitr-physical-backup-source-27258.svc.cluster.local --tcp=9000
service/minio-service created
+ create_minio_bucket operator-testing http://minio-service:9000
+ local bucket=operator-testing
+ local endpoint=http://minio-service:9000
+ kubectl_bin run -i --rm aws-cli --image=docker.io/perconalab/awscli --restart=Never -- bash -c $'AWS_ACCESS_KEY_ID=some-access-key \t\tAWS_SECRET_ACCESS_KEY=some-secret-key \t\tAWS_DEFAULT_REGION=us-east-1         /usr/bin/aws --no-verify-ssl --endpoint-url http://minio-service:9000 s3 mb s3://operator-testing'
++ mktemp
+ local LAST_OUT=/tmp/tmp.PWxM7h2BBU
++ mktemp
+ local LAST_ERR=/tmp/tmp.uRFBiOCiZf
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl run -i --rm aws-cli --image=docker.io/perconalab/awscli --restart=Never -- bash -c $'AWS_ACCESS_KEY_ID=some-access-key \t\tAWS_SECRET_ACCESS_KEY=some-secret-key \t\tAWS_DEFAULT_REGION=us-east-1         /usr/bin/aws --no-verify-ssl --endpoint-url http://minio-service:9000 s3 mb s3://operator-testing'
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.PWxM7h2BBU
make_bucket: operator-testing
pod "aws-cli" deleted from pitr-physical-backup-source-27258 namespace
+ cat /tmp/tmp.uRFBiOCiZf
All commands and output from this session will be recorded in container logs, including credentials and sensitive information passed through the command prompt.
If you don't see a command prompt, try pressing enter.
warning: couldn't attach to pod/aws-cli, falling back to streaming logs: Internal error occurred: unable to upgrade connection: container aws-cli not found in pod aws-cli_pitr-physical-backup-source-27258
+ rm /tmp/tmp.PWxM7h2BBU /tmp/tmp.uRFBiOCiZf
+ return 0
+ desc 'create secrets and start client'
+ set +o xtrace


-----------------------------------------------------------------------------------
create secrets and start client
-----------------------------------------------------------------------------------

+ kubectl_bin apply -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2328/e2e-tests/conf/secrets.yml -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2328/e2e-tests/conf/client.yml -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2328/e2e-tests/conf/minio-secret.yml
++ mktemp
+ local LAST_OUT=/tmp/tmp.IzTUrmxVLI
++ mktemp
+ local LAST_ERR=/tmp/tmp.hEfDFszjZW
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl apply -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2328/e2e-tests/conf/secrets.yml -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2328/e2e-tests/conf/client.yml -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2328/e2e-tests/conf/minio-secret.yml
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.IzTUrmxVLI
secret/some-users created
deployment.apps/psmdb-client created
secret/minio-secret created
+ cat /tmp/tmp.hEfDFszjZW
+ rm /tmp/tmp.IzTUrmxVLI /tmp/tmp.hEfDFszjZW
+ return 0
+ cluster=some-name
+ desc 'create first PSMDB cluster some-name'
+ set +o xtrace


-----------------------------------------------------------------------------------
create first PSMDB cluster some-name
-----------------------------------------------------------------------------------

+ apply_cluster /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2328/e2e-tests/pitr-physical-backup-source/conf/some-name-rs0.yml
+ '[' -z '' ']'
+ cat_config /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2328/e2e-tests/pitr-physical-backup-source/conf/some-name-rs0.yml
+ kubectl_bin apply -f -
+ cat /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2328/e2e-tests/pitr-physical-backup-source/conf/some-name-rs0.yml
+ yq eval '(.spec | select(has("pmm"))).pmm.image = "docker.io/percona/pmm-client:2.44.1-1"'
++ mktemp
+ yq eval '(.spec | select(.image == null)).image = "docker.io/perconalab/percona-server-mongodb-operator:main-mongod8.0"'
+ /usr/sbin/sed -e s/NAME_SPACE/pitr-physical-backup-source-27258/g
+ yq eval '.spec.upgradeOptions.apply="Never"'
+ yq eval '(.spec | select(has("backup"))).backup.image = "docker.io/perconalab/percona-server-mongodb-operator:main-backup"'
+ yq eval '(.spec | select(has("initImage"))).initImage = "docker.io/perconalab/percona-server-mongodb-operator:PR-2328-37c432905"'
+ local LAST_OUT=/tmp/tmp.2ZAbSUJhDW
++ mktemp
+ local LAST_ERR=/tmp/tmp.sX6QUSR7VD
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl apply -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.2ZAbSUJhDW
perconaservermongodb.psmdb.percona.com/some-name created
+ cat /tmp/tmp.sX6QUSR7VD
+ rm /tmp/tmp.2ZAbSUJhDW /tmp/tmp.sX6QUSR7VD
+ return 0
+ desc 'check if all 3 Pods started'
+ set +o xtrace


-----------------------------------------------------------------------------------
check if all 3 Pods started
-----------------------------------------------------------------------------------

+ wait_for_running some-name-rs0 3
+ local name=some-name-rs0
+ let last_pod=2
+ local check_cluster_readyness=true
+ set_debug
+ [[ 1 == 1 ]]
+ set -o xtrace
+ local rs_name=rs0
+ local cluster_name=some-name
++ seq 0 2
+ for i in $(seq 0 $last_pod)
+ [[ 0 -eq 2 ]]
+ wait_pod some-name-rs0-0
+ local pod=some-name-rs0-0
+ set +o xtrace
waiting for pod/some-name-rs0-0 to be ready......OK
+ for i in $(seq 0 $last_pod)
+ [[ 1 -eq 2 ]]
+ wait_pod some-name-rs0-1
+ local pod=some-name-rs0-1
+ set +o xtrace
waiting for pod/some-name-rs0-1 to be ready.......OK
+ for i in $(seq 0 $last_pod)
+ [[ 2 -eq 2 ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].arbiter.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.D8gapdUlRa
+++ mktemp
++ local LAST_ERR=/tmp/tmp.3FJrq03Jp0
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].arbiter.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.D8gapdUlRa
++ cat /tmp/tmp.3FJrq03Jp0
++ rm /tmp/tmp.D8gapdUlRa /tmp/tmp.3FJrq03Jp0
++ return 0
+ [[ '' == true ]]
+ wait_pod some-name-rs0-2
+ local pod=some-name-rs0-2
+ set +o xtrace
waiting for pod/some-name-rs0-2 to be ready......OK
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].nonvoting.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.Y9RFssLl4A
+++ mktemp
++ local LAST_ERR=/tmp/tmp.FhixZOvaZM
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].nonvoting.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.Y9RFssLl4A
++ cat /tmp/tmp.FhixZOvaZM
++ rm /tmp/tmp.Y9RFssLl4A /tmp/tmp.FhixZOvaZM
++ return 0
+ [[ '' == true ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].hidden.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.3RjKk2ncsL
+++ mktemp
++ local LAST_ERR=/tmp/tmp.m2xASZ9MxL
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].hidden.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.3RjKk2ncsL
++ cat /tmp/tmp.m2xASZ9MxL
++ rm /tmp/tmp.3RjKk2ncsL /tmp/tmp.m2xASZ9MxL
++ return 0
+ [[ '' == true ]]
+ sleep 10
+ [[ true == true ]]
+ set +x
Waiting for cluster readyness..........................................................
+ wait_for_running some-name-cfg 3 false
+ local name=some-name-cfg
+ let last_pod=2
+ local check_cluster_readyness=false
+ set_debug
+ [[ 1 == 1 ]]
+ set -o xtrace
+ local rs_name=cfg
+ local cluster_name=some-name
++ seq 0 2
+ for i in $(seq 0 $last_pod)
+ [[ 0 -eq 2 ]]
+ wait_pod some-name-cfg-0
+ local pod=some-name-cfg-0
+ set +o xtrace
waiting for pod/some-name-cfg-0 to be ready.OK
+ for i in $(seq 0 $last_pod)
+ [[ 1 -eq 2 ]]
+ wait_pod some-name-cfg-1
+ local pod=some-name-cfg-1
+ set +o xtrace
waiting for pod/some-name-cfg-1 to be ready.OK
+ for i in $(seq 0 $last_pod)
+ [[ 2 -eq 2 ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].arbiter.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.nqM9xktPS9
+++ mktemp
++ local LAST_ERR=/tmp/tmp.l5VYprPB97
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].arbiter.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.nqM9xktPS9
++ cat /tmp/tmp.l5VYprPB97
++ rm /tmp/tmp.nqM9xktPS9 /tmp/tmp.l5VYprPB97
++ return 0
+ [[ '' == true ]]
+ wait_pod some-name-cfg-2
+ local pod=some-name-cfg-2
+ set +o xtrace
waiting for pod/some-name-cfg-2 to be ready.OK
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].nonvoting.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.x2RDVDpbKa
+++ mktemp
++ local LAST_ERR=/tmp/tmp.oZbJBXb4dN
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].nonvoting.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.x2RDVDpbKa
++ cat /tmp/tmp.oZbJBXb4dN
++ rm /tmp/tmp.x2RDVDpbKa /tmp/tmp.oZbJBXb4dN
++ return 0
+ [[ '' == true ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].hidden.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.FCKm5LbHdt
+++ mktemp
++ local LAST_ERR=/tmp/tmp.xnAQHxyiu9
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].hidden.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.FCKm5LbHdt
++ cat /tmp/tmp.xnAQHxyiu9
++ rm /tmp/tmp.FCKm5LbHdt /tmp/tmp.xnAQHxyiu9
++ return 0
+ [[ '' == true ]]
+ sleep 10
+ [[ false == true ]]
+ sleep 10
+ write_initial_data
+ desc 'create user myApp'
+ set +o xtrace


-----------------------------------------------------------------------------------
create user myApp
-----------------------------------------------------------------------------------

+ run_mongos 'db.createUser({user:"myApp",pwd:"myPass",roles:[{db:"myApp",role:"readWrite"}]})' userAdmin:userAdmin123456@some-name-mongos.pitr-physical-backup-source-27258
+ local 'command=db.createUser({user:"myApp",pwd:"myPass",roles:[{db:"myApp",role:"readWrite"}]})'
+ local uri=userAdmin:userAdmin123456@some-name-mongos.pitr-physical-backup-source-27258
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local port=27017
+ local mongo_bin=mongo
++ echo .svc.cluster.local
++ awk -F: '{print $2}'
+ suffix_port=
+ [[ -z '' ]]
+ suffix=.svc.cluster.local:27017
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.d421sX8B8S
+++ mktemp
++ local LAST_ERR=/tmp/tmp.eeFTXkFmtr
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.d421sX8B8S
++ cat /tmp/tmp.eeFTXkFmtr
++ rm /tmp/tmp.d421sX8B8S /tmp/tmp.eeFTXkFmtr
++ return 0
+ local client_container=psmdb-client-bb8b97679-c5ptc
+ kubectl_bin exec psmdb-client-bb8b97679-c5ptc -- bash -c 'printf '\''db.createUser({user:"myApp",pwd:"myPass",roles:[{db:"myApp",role:"readWrite"}]})\n'\'' | mongo mongodb://userAdmin:userAdmin123456@some-name-mongos.pitr-physical-backup-source-27258.svc.cluster.local:27017/admin '
++ mktemp
+ local LAST_OUT=/tmp/tmp.0grbEUrhIe
++ mktemp
+ local LAST_ERR=/tmp/tmp.fgnKcEzZn5
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-bb8b97679-c5ptc -- bash -c 'printf '\''db.createUser({user:"myApp",pwd:"myPass",roles:[{db:"myApp",role:"readWrite"}]})\n'\'' | mongo mongodb://userAdmin:userAdmin123456@some-name-mongos.pitr-physical-backup-source-27258.svc.cluster.local:27017/admin '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.0grbEUrhIe
Percona Server for MongoDB shell version v4.4.29-28
connecting to: mongodb://some-name-mongos.pitr-physical-backup-source-27258.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb
Implicit session: session { "id" : UUID("80d0d840-57f6-462c-94d5-f981d6141625") }
Percona Server for MongoDB server version: v8.0.21-9
WARNING: shell and server versions do not match
Successfully added user: {
	"user" : "myApp",
	"roles" : [
		{
			"db" : "myApp",
			"role" : "readWrite"
		}
	]
}
bye
+ cat /tmp/tmp.fgnKcEzZn5
+ rm /tmp/tmp.0grbEUrhIe /tmp/tmp.fgnKcEzZn5
+ return 0
+ sleep 2
+ write_document
+ local cmp_postfix=
+ local sleep_value=0
+ log 'write initial data, read from all'
+ set +o xtrace
[2026-05-21T09:04:52+0000] write initial data, read from all
+ run_mongos 'use myApp\n db.test.insert({ x: 100500 })' myApp:myPass@some-name-mongos.pitr-physical-backup-source-27258
+ local 'command=use myApp\n db.test.insert({ x: 100500 })'
+ local uri=myApp:myPass@some-name-mongos.pitr-physical-backup-source-27258
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local port=27017
+ local mongo_bin=mongo
++ echo .svc.cluster.local
++ awk -F: '{print $2}'
+ suffix_port=
+ [[ -z '' ]]
+ suffix=.svc.cluster.local:27017
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.HbiJo1Opso
+++ mktemp
++ local LAST_ERR=/tmp/tmp.1RblHhLDcp
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.HbiJo1Opso
++ cat /tmp/tmp.1RblHhLDcp
++ rm /tmp/tmp.HbiJo1Opso /tmp/tmp.1RblHhLDcp
++ return 0
+ local client_container=psmdb-client-bb8b97679-c5ptc
+ kubectl_bin exec psmdb-client-bb8b97679-c5ptc -- bash -c 'printf '\''use myApp\n db.test.insert({ x: 100500 })\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.pitr-physical-backup-source-27258.svc.cluster.local:27017/admin '
++ mktemp
+ local LAST_OUT=/tmp/tmp.YaBTpysx7v
++ mktemp
+ local LAST_ERR=/tmp/tmp.LuveRbAXKA
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-bb8b97679-c5ptc -- bash -c 'printf '\''use myApp\n db.test.insert({ x: 100500 })\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.pitr-physical-backup-source-27258.svc.cluster.local:27017/admin '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.YaBTpysx7v
Percona Server for MongoDB shell version v4.4.29-28
connecting to: mongodb://some-name-mongos.pitr-physical-backup-source-27258.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb
Implicit session: session { "id" : UUID("ac8d4b46-ea65-4099-bc84-54f39196ce02") }
Percona Server for MongoDB server version: v8.0.21-9
WARNING: shell and server versions do not match
switched to db myApp
WriteResult({ "nInserted" : 1 })
bye
+ cat /tmp/tmp.LuveRbAXKA
+ rm /tmp/tmp.YaBTpysx7v /tmp/tmp.LuveRbAXKA
+ return 0
+ sleep 0
+ compare_mongos_cmd find myApp:myPass@some-name-mongos.pitr-physical-backup-source-27258
+ local command=find
+ local uri=myApp:myPass@some-name-mongos.pitr-physical-backup-source-27258
+ local postfix=
+ local suffix=
+ local database=myApp
+ local collection=test
+ local port=27017
+ local tls=false
+ [[ false == true ]]
+ mongos_command=run_mongos
+ log 'running db.test.command() in myApp'
+ set +o xtrace
[2026-05-21T09:04:55+0000] running db.test.command() in myApp
+ run_mongos 'use myApp\n db.test.find()' myApp:myPass@some-name-mongos.pitr-physical-backup-source-27258 mongodb '' '' 27017
+ local 'command=use myApp\n db.test.find()'
+ local uri=myApp:myPass@some-name-mongos.pitr-physical-backup-source-27258
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local port=27017
+ grep -E -v 'I NETWORK|W NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|Started a new thread for the timer service'
+ local mongo_bin=mongo
+ /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
++ echo .svc.cluster.local
++ awk -F: '{print $2}'
+ suffix_port=
+ [[ -z '' ]]
+ suffix=.svc.cluster.local:27017
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.a8Vf5EDjat
+++ mktemp
++ local LAST_ERR=/tmp/tmp.XO8a6IQTvm
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.a8Vf5EDjat
++ cat /tmp/tmp.XO8a6IQTvm
++ rm /tmp/tmp.a8Vf5EDjat /tmp/tmp.XO8a6IQTvm
++ return 0
+ local client_container=psmdb-client-bb8b97679-c5ptc
+ kubectl_bin exec psmdb-client-bb8b97679-c5ptc -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.pitr-physical-backup-source-27258.svc.cluster.local:27017/admin '
++ mktemp
+ local LAST_OUT=/tmp/tmp.BtpiuwPCvt
++ mktemp
+ local LAST_ERR=/tmp/tmp.j8QCpgoeGP
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-bb8b97679-c5ptc -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.pitr-physical-backup-source-27258.svc.cluster.local:27017/admin '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.BtpiuwPCvt
+ cat /tmp/tmp.j8QCpgoeGP
+ rm /tmp/tmp.BtpiuwPCvt /tmp/tmp.j8QCpgoeGP
+ return 0
+ [[ 0 -eq 0 ]]
+ diff /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2328/e2e-tests/pitr-physical-backup-source/compare/find.json /tmp/tmp.DTCahJpmH8/find
+ wait_backup_agent some-name-rs0-0
+ local agent_pod=some-name-rs0-0
+ set +o xtrace
waiting for pbm-agent to be ready in some-name-rs0-0...2026-05-21T09:03:59.000+0000 I listening for the commands

+ wait_backup_agent some-name-rs0-1
+ local agent_pod=some-name-rs0-1
+ set +o xtrace
waiting for pbm-agent to be ready in some-name-rs0-1...2026-05-21T09:04:02.000+0000 I listening for the commands

+ wait_backup_agent some-name-rs0-2
+ local agent_pod=some-name-rs0-2
+ set +o xtrace
waiting for pbm-agent to be ready in some-name-rs0-2...2026-05-21T09:04:14.000+0000 I listening for the commands

+ wait_backup_agent some-name-rs1-0
+ local agent_pod=some-name-rs1-0
+ set +o xtrace
waiting for pbm-agent to be ready in some-name-rs1-0...2026-05-21T09:04:01.000+0000 I listening for the commands

+ wait_backup_agent some-name-rs1-1
+ local agent_pod=some-name-rs1-1
+ set +o xtrace
waiting for pbm-agent to be ready in some-name-rs1-1...2026-05-21T09:04:04.000+0000 I listening for the commands

+ wait_backup_agent some-name-rs1-2
+ local agent_pod=some-name-rs1-2
+ set +o xtrace
waiting for pbm-agent to be ready in some-name-rs1-2...2026-05-21T09:04:15.000+0000 I listening for the commands

+ wait_backup_agent some-name-rs2-0
+ local agent_pod=some-name-rs2-0
+ set +o xtrace
waiting for pbm-agent to be ready in some-name-rs2-0...2026-05-21T09:04:01.000+0000 I listening for the commands

+ wait_backup_agent some-name-rs2-1
+ local agent_pod=some-name-rs2-1
+ set +o xtrace
waiting for pbm-agent to be ready in some-name-rs2-1...2026-05-21T09:04:06.000+0000 I listening for the commands

+ wait_backup_agent some-name-rs2-2
+ local agent_pod=some-name-rs2-2
+ set +o xtrace
waiting for pbm-agent to be ready in some-name-rs2-2...2026-05-21T09:04:16.000+0000 I listening for the commands

+ echo 'Sleeping for 360 seconds'
Sleeping for 360 seconds
+ sleep 360
+ backup_name_minio=backup-minio
+ desc 'restore pitr type date using backupSource'
+ set +o xtrace


-----------------------------------------------------------------------------------
restore pitr type date using backupSource
-----------------------------------------------------------------------------------

+ run_backup backup-minio 1 physical
+ local name=backup-minio
+ local idx=1
+ local type=physical
+ desc 'run backup backup-minio-1'
+ set +o xtrace


-----------------------------------------------------------------------------------
run backup backup-minio-1
-----------------------------------------------------------------------------------

+ cat /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2328/e2e-tests/pitr-physical-backup-source/conf/backup-minio.yml
+ /usr/sbin/sed -e 's/name:/name: backup-minio-1/'
+ /usr/sbin/sed -e 's/type:/type: physical/'
+ kubectl_bin apply -f -
++ mktemp
+ local LAST_OUT=/tmp/tmp.aWuON81YtG
++ mktemp
+ local LAST_ERR=/tmp/tmp.UlaJfwBj0N
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl apply -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.aWuON81YtG
perconaservermongodbbackup.psmdb.percona.com/backup-minio-1 created
+ cat /tmp/tmp.UlaJfwBj0N
+ rm /tmp/tmp.aWuON81YtG /tmp/tmp.UlaJfwBj0N
+ return 0
+ wait_backup backup-minio-1
+ local backup_name=backup-minio-1
+ local target_state=ready
+ set +o xtrace
waiting for backup-minio-1 to reach ready state........OK
+ compare_latest_restorable_time some-name-rs0 backup-minio-1
+ local cluster=some-name-rs0
+ local backup_name=backup-minio-1
+ local latest_restorable_time
+ local backup_time
++ get_latest_restorable_time some-name-rs0
++ local cluster=some-name-rs0
++ local first_timestamp
++ local second_timestamp
++ local retry=0
++ [[ '' != '' ]]
+++ kubectl_bin exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ jq '.backups.pitrChunks.pitrChunks | last | .range.end'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.lxQpnQXhrd
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.vwssX4QjLm
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in $(seq 0 2)
+++ set +e
+++ kubectl exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.lxQpnQXhrd
+++ cat /tmp/tmp.vwssX4QjLm
+++ rm /tmp/tmp.lxQpnQXhrd /tmp/tmp.vwssX4QjLm
+++ return 0
++ first_timestamp=null
++ sleep 5
++ [[ null != '' ]]
++ [[ null != null ]]
++ let retry+=1
++ [[ 1 -gt 30 ]]
++ [[ null != '' ]]
++ [[ null != null ]]
+++ kubectl_bin exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ jq '.backups.pitrChunks.pitrChunks | last | .range.end'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.JkO4X2aij9
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.NsY466uaUv
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in $(seq 0 2)
+++ set +e
+++ kubectl exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.JkO4X2aij9
+++ cat /tmp/tmp.NsY466uaUv
+++ rm /tmp/tmp.JkO4X2aij9 /tmp/tmp.NsY466uaUv
+++ return 0
++ first_timestamp=null
++ sleep 5
++ [[ null != '' ]]
++ [[ null != null ]]
++ let retry+=1
++ [[ 2 -gt 30 ]]
++ [[ null != '' ]]
++ [[ null != null ]]
+++ kubectl_bin exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ jq '.backups.pitrChunks.pitrChunks | last | .range.end'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.wNTS1rCEGb
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.Yy2sTuegLv
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in $(seq 0 2)
+++ set +e
+++ kubectl exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.wNTS1rCEGb
+++ cat /tmp/tmp.Yy2sTuegLv
+++ rm /tmp/tmp.wNTS1rCEGb /tmp/tmp.Yy2sTuegLv
+++ return 0
++ first_timestamp=null
++ sleep 5
++ [[ null != '' ]]
++ [[ null != null ]]
++ let retry+=1
++ [[ 3 -gt 30 ]]
++ [[ null != '' ]]
++ [[ null != null ]]
+++ kubectl_bin exec some-name-rs0-0 -c backup-agent -- pbm status -o json
++++ mktemp
+++ jq '.backups.pitrChunks.pitrChunks | last | .range.end'
+++ local LAST_OUT=/tmp/tmp.wbFiGps0g3
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.iddVBh6XyV
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in $(seq 0 2)
+++ set +e
+++ kubectl exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.wbFiGps0g3
+++ cat /tmp/tmp.iddVBh6XyV
+++ rm /tmp/tmp.wbFiGps0g3 /tmp/tmp.iddVBh6XyV
+++ return 0
++ first_timestamp=null
++ sleep 5
++ [[ null != '' ]]
++ [[ null != null ]]
++ let retry+=1
++ [[ 4 -gt 30 ]]
++ [[ null != '' ]]
++ [[ null != null ]]
+++ kubectl_bin exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ jq '.backups.pitrChunks.pitrChunks | last | .range.end'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.nUDHav2sKg
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.RcRU6bLhKh
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in $(seq 0 2)
+++ set +e
+++ kubectl exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.nUDHav2sKg
+++ cat /tmp/tmp.RcRU6bLhKh
+++ rm /tmp/tmp.nUDHav2sKg /tmp/tmp.RcRU6bLhKh
+++ return 0
++ first_timestamp=null
++ sleep 5
++ [[ null != '' ]]
++ [[ null != null ]]
++ let retry+=1
++ [[ 5 -gt 30 ]]
++ [[ null != '' ]]
++ [[ null != null ]]
+++ kubectl_bin exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ jq '.backups.pitrChunks.pitrChunks | last | .range.end'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.Fe8iDko00V
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.bB7YqTFDC0
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in $(seq 0 2)
+++ set +e
+++ kubectl exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.Fe8iDko00V
+++ cat /tmp/tmp.bB7YqTFDC0
+++ rm /tmp/tmp.Fe8iDko00V /tmp/tmp.bB7YqTFDC0
+++ return 0
++ first_timestamp=null
++ sleep 5
++ [[ null != '' ]]
++ [[ null != null ]]
++ let retry+=1
++ [[ 6 -gt 30 ]]
++ [[ null != '' ]]
++ [[ null != null ]]
+++ kubectl_bin exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ jq '.backups.pitrChunks.pitrChunks | last | .range.end'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.aFRifxF71u
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.yITumDpWha
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in $(seq 0 2)
+++ set +e
+++ kubectl exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.aFRifxF71u
+++ cat /tmp/tmp.yITumDpWha
+++ rm /tmp/tmp.aFRifxF71u /tmp/tmp.yITumDpWha
+++ return 0
++ first_timestamp=null
++ sleep 5
++ [[ null != '' ]]
++ [[ null != null ]]
++ let retry+=1
++ [[ 7 -gt 30 ]]
++ [[ null != '' ]]
++ [[ null != null ]]
+++ kubectl_bin exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ jq '.backups.pitrChunks.pitrChunks | last | .range.end'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.VKx3EZ2fbS
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.2rJEbCUCHC
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in $(seq 0 2)
+++ set +e
+++ kubectl exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.VKx3EZ2fbS
+++ cat /tmp/tmp.2rJEbCUCHC
+++ rm /tmp/tmp.VKx3EZ2fbS /tmp/tmp.2rJEbCUCHC
+++ return 0
++ first_timestamp=null
++ sleep 5
++ [[ null != '' ]]
++ [[ null != null ]]
++ let retry+=1
++ [[ 8 -gt 30 ]]
++ [[ null != '' ]]
++ [[ null != null ]]
+++ kubectl_bin exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ jq '.backups.pitrChunks.pitrChunks | last | .range.end'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.zkvRzVQK1W
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.DJfrWt1Zvb
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in $(seq 0 2)
+++ set +e
+++ kubectl exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.zkvRzVQK1W
+++ cat /tmp/tmp.DJfrWt1Zvb
+++ rm /tmp/tmp.zkvRzVQK1W /tmp/tmp.DJfrWt1Zvb
+++ return 0
++ first_timestamp=null
++ sleep 5
++ [[ null != '' ]]
++ [[ null != null ]]
++ let retry+=1
++ [[ 9 -gt 30 ]]
++ [[ null != '' ]]
++ [[ null != null ]]
+++ kubectl_bin exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ jq '.backups.pitrChunks.pitrChunks | last | .range.end'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.FIlhD75Z7U
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.xOgzykLHlB
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in $(seq 0 2)
+++ set +e
+++ kubectl exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.FIlhD75Z7U
+++ cat /tmp/tmp.xOgzykLHlB
+++ rm /tmp/tmp.FIlhD75Z7U /tmp/tmp.xOgzykLHlB
+++ return 0
++ first_timestamp=null
++ sleep 5
++ [[ null != '' ]]
++ [[ null != null ]]
++ let retry+=1
++ [[ 10 -gt 30 ]]
++ [[ null != '' ]]
++ [[ null != null ]]
+++ kubectl_bin exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ jq '.backups.pitrChunks.pitrChunks | last | .range.end'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.oBLCFa1UqP
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.LviA1gYB3j
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in $(seq 0 2)
+++ set +e
+++ kubectl exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.oBLCFa1UqP
+++ cat /tmp/tmp.LviA1gYB3j
+++ rm /tmp/tmp.oBLCFa1UqP /tmp/tmp.LviA1gYB3j
+++ return 0
++ first_timestamp=null
++ sleep 5
++ [[ null != '' ]]
++ [[ null != null ]]
++ let retry+=1
++ [[ 11 -gt 30 ]]
++ [[ null != '' ]]
++ [[ null != null ]]
+++ kubectl_bin exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ jq '.backups.pitrChunks.pitrChunks | last | .range.end'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.Mv6VIPr7NK
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.1n09QySLLI
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in $(seq 0 2)
+++ set +e
+++ kubectl exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.Mv6VIPr7NK
+++ cat /tmp/tmp.1n09QySLLI
+++ rm /tmp/tmp.Mv6VIPr7NK /tmp/tmp.1n09QySLLI
+++ return 0
++ first_timestamp=null
++ sleep 5
++ [[ null != '' ]]
++ [[ null != null ]]
++ let retry+=1
++ [[ 12 -gt 30 ]]
++ [[ null != '' ]]
++ [[ null != null ]]
+++ kubectl_bin exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ jq '.backups.pitrChunks.pitrChunks | last | .range.end'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.Mq4yvTpbXo
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.kBhb6JXCSa
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in $(seq 0 2)
+++ set +e
+++ kubectl exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.Mq4yvTpbXo
+++ cat /tmp/tmp.kBhb6JXCSa
+++ rm /tmp/tmp.Mq4yvTpbXo /tmp/tmp.kBhb6JXCSa
+++ return 0
++ first_timestamp=null
++ sleep 5
++ [[ null != '' ]]
++ [[ null != null ]]
++ let retry+=1
++ [[ 13 -gt 30 ]]
++ [[ null != '' ]]
++ [[ null != null ]]
+++ kubectl_bin exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ jq '.backups.pitrChunks.pitrChunks | last | .range.end'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.bsJjOjkjcK
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.Y6gNe1r3N7
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in $(seq 0 2)
+++ set +e
+++ kubectl exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.bsJjOjkjcK
+++ cat /tmp/tmp.Y6gNe1r3N7
+++ rm /tmp/tmp.bsJjOjkjcK /tmp/tmp.Y6gNe1r3N7
+++ return 0
++ first_timestamp=null
++ sleep 5
++ [[ null != '' ]]
++ [[ null != null ]]
++ let retry+=1
++ [[ 14 -gt 30 ]]
++ [[ null != '' ]]
++ [[ null != null ]]
+++ kubectl_bin exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ jq '.backups.pitrChunks.pitrChunks | last | .range.end'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.ZUMM0h1b7d
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.ilNbNvQdQ4
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in $(seq 0 2)
+++ set +e
+++ kubectl exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.ZUMM0h1b7d
+++ cat /tmp/tmp.ilNbNvQdQ4
+++ rm /tmp/tmp.ZUMM0h1b7d /tmp/tmp.ilNbNvQdQ4
+++ return 0
++ first_timestamp=null
++ sleep 5
++ [[ null != '' ]]
++ [[ null != null ]]
++ let retry+=1
++ [[ 15 -gt 30 ]]
++ [[ null != '' ]]
++ [[ null != null ]]
+++ kubectl_bin exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ jq '.backups.pitrChunks.pitrChunks | last | .range.end'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.VCBJBcGYPM
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.2yAO5KQXZ3
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in $(seq 0 2)
+++ set +e
+++ kubectl exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.VCBJBcGYPM
+++ cat /tmp/tmp.2yAO5KQXZ3
+++ rm /tmp/tmp.VCBJBcGYPM /tmp/tmp.2yAO5KQXZ3
+++ return 0
++ first_timestamp=null
++ sleep 5
++ [[ null != '' ]]
++ [[ null != null ]]
++ let retry+=1
++ [[ 16 -gt 30 ]]
++ [[ null != '' ]]
++ [[ null != null ]]
+++ kubectl_bin exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ jq '.backups.pitrChunks.pitrChunks | last | .range.end'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.j6XfM8DqGn
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.yilZCynSlP
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in $(seq 0 2)
+++ set +e
+++ kubectl exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.j6XfM8DqGn
+++ cat /tmp/tmp.yilZCynSlP
+++ rm /tmp/tmp.j6XfM8DqGn /tmp/tmp.yilZCynSlP
+++ return 0
++ first_timestamp=null
++ sleep 5
++ [[ null != '' ]]
++ [[ null != null ]]
++ let retry+=1
++ [[ 17 -gt 30 ]]
++ [[ null != '' ]]
++ [[ null != null ]]
+++ kubectl_bin exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ jq '.backups.pitrChunks.pitrChunks | last | .range.end'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.laaKrd3MNm
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.vyRVgqCK7q
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in $(seq 0 2)
+++ set +e
+++ kubectl exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.laaKrd3MNm
+++ cat /tmp/tmp.vyRVgqCK7q
+++ rm /tmp/tmp.laaKrd3MNm /tmp/tmp.vyRVgqCK7q
+++ return 0
++ first_timestamp=null
++ sleep 5
++ [[ null != '' ]]
++ [[ null != null ]]
++ let retry+=1
++ [[ 18 -gt 30 ]]
++ [[ null != '' ]]
++ [[ null != null ]]
+++ kubectl_bin exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ jq '.backups.pitrChunks.pitrChunks | last | .range.end'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.MYe8Vo7L5s
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.KbDNPpEQDP
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in $(seq 0 2)
+++ set +e
+++ kubectl exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.MYe8Vo7L5s
+++ cat /tmp/tmp.KbDNPpEQDP
+++ rm /tmp/tmp.MYe8Vo7L5s /tmp/tmp.KbDNPpEQDP
+++ return 0
++ first_timestamp=null
++ sleep 5
++ [[ null != '' ]]
++ [[ null != null ]]
++ let retry+=1
++ [[ 19 -gt 30 ]]
++ [[ null != '' ]]
++ [[ null != null ]]
+++ kubectl_bin exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ jq '.backups.pitrChunks.pitrChunks | last | .range.end'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.9pgBNAUJ6N
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.5LQ6xgd7da
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in $(seq 0 2)
+++ set +e
+++ kubectl exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.9pgBNAUJ6N
+++ cat /tmp/tmp.5LQ6xgd7da
+++ rm /tmp/tmp.9pgBNAUJ6N /tmp/tmp.5LQ6xgd7da
+++ return 0
++ first_timestamp=null
++ sleep 5
++ [[ null != '' ]]
++ [[ null != null ]]
++ let retry+=1
++ [[ 20 -gt 30 ]]
++ [[ null != '' ]]
++ [[ null != null ]]
+++ kubectl_bin exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ jq '.backups.pitrChunks.pitrChunks | last | .range.end'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.VY1HIlcQ5J
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.9482U6ljHX
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in $(seq 0 2)
+++ set +e
+++ kubectl exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.VY1HIlcQ5J
+++ cat /tmp/tmp.9482U6ljHX
+++ rm /tmp/tmp.VY1HIlcQ5J /tmp/tmp.9482U6ljHX
+++ return 0
++ first_timestamp=1779354814
++ sleep 5
++ [[ 1779354814 != '' ]]
++ [[ 1779354814 != null ]]
+++ kubectl_bin exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ jq '.backups.pitrChunks.pitrChunks | last | .range.end'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.scw6wni2qF
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.vDI3NAX6YJ
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in $(seq 0 2)
+++ set +e
+++ kubectl exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.scw6wni2qF
+++ cat /tmp/tmp.vDI3NAX6YJ
+++ rm /tmp/tmp.scw6wni2qF /tmp/tmp.vDI3NAX6YJ
+++ return 0
++ second_timestamp=1779354814
++ let retry+=1
++ [[ 21 -gt 30 ]]
++ [[ 1779354814 != '' ]]
++ [[ 1779354814 != null ]]
++ [[ 1779354814 == 1779354814 ]]
++ /usr/sbin/date -u -d @1779354814 +%Y-%m-%dT%H:%M:%SZ
+ latest_restorable_time=2026-05-21T09:13:34Z
++ get_latest_restorable_time_from_backup_object backup-minio-1
++ local backup_name=backup-minio-1
++ local latestRestorableTime
++ local retry=0
++ [[ '' != '' ]]
++ sleep 5
+++ kubectl_bin get psmdb-backup backup-minio-1 -o 'jsonpath={.status.latestRestorableTime}'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.LU5outx51M
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.XeCUMRN1L8
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in $(seq 0 2)
+++ set +e
+++ kubectl get psmdb-backup backup-minio-1 -o 'jsonpath={.status.latestRestorableTime}'
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.LU5outx51M
+++ cat /tmp/tmp.XeCUMRN1L8
+++ rm /tmp/tmp.LU5outx51M /tmp/tmp.XeCUMRN1L8
+++ return 0
++ latestRestorableTime=2026-05-21T09:13:34Z
++ let retry+=1
++ [[ 1 -gt 30 ]]
++ [[ 2026-05-21T09:13:34Z != '' ]]
++ [[ 2026-05-21T09:13:34Z != null ]]
++ echo 2026-05-21T09:13:34Z
+ backup_time=2026-05-21T09:13:34Z
+ [[ 2026-05-21T09:13:34Z != 2026\-05\-21T09\:13\:34Z ]]
+ reset_collection
+ desc 'reset data'
+ set +o xtrace


-----------------------------------------------------------------------------------
reset data
-----------------------------------------------------------------------------------

+ run_mongos 'use myApp\n db.test.remove({})' myApp:myPass@some-name-mongos.pitr-physical-backup-source-27258
+ local 'command=use myApp\n db.test.remove({})'
+ local uri=myApp:myPass@some-name-mongos.pitr-physical-backup-source-27258
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local port=27017
+ local mongo_bin=mongo
++ echo .svc.cluster.local
++ awk -F: '{print $2}'
+ suffix_port=
+ [[ -z '' ]]
+ suffix=.svc.cluster.local:27017
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.a1ugJHGxO4
+++ mktemp
++ local LAST_ERR=/tmp/tmp.HxeFw1pE63
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.a1ugJHGxO4
++ cat /tmp/tmp.HxeFw1pE63
++ rm /tmp/tmp.a1ugJHGxO4 /tmp/tmp.HxeFw1pE63
++ return 0
+ local client_container=psmdb-client-bb8b97679-c5ptc
+ kubectl_bin exec psmdb-client-bb8b97679-c5ptc -- bash -c 'printf '\''use myApp\n db.test.remove({})\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.pitr-physical-backup-source-27258.svc.cluster.local:27017/admin '
++ mktemp
+ local LAST_OUT=/tmp/tmp.zh7DmBzsca
++ mktemp
+ local LAST_ERR=/tmp/tmp.DZVQaOe5vc
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-bb8b97679-c5ptc -- bash -c 'printf '\''use myApp\n db.test.remove({})\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.pitr-physical-backup-source-27258.svc.cluster.local:27017/admin '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.zh7DmBzsca
Percona Server for MongoDB shell version v4.4.29-28
connecting to: mongodb://some-name-mongos.pitr-physical-backup-source-27258.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb
Implicit session: session { "id" : UUID("bd6ddc02-f6ea-4588-b4a1-849cfc9fdfcb") }
Percona Server for MongoDB server version: v8.0.21-9
WARNING: shell and server versions do not match
switched to db myApp
WriteResult({ "nRemoved" : 1 })
bye
+ cat /tmp/tmp.DZVQaOe5vc
+ rm /tmp/tmp.zh7DmBzsca /tmp/tmp.DZVQaOe5vc
+ return 0
+ sleep 2
+ write_document '' 120
+ local cmp_postfix=
+ local sleep_value=120
+ log 'write initial data, read from all'
+ set +o xtrace
[2026-05-21T09:14:01+0000] write initial data, read from all
+ run_mongos 'use myApp\n db.test.insert({ x: 100500 })' myApp:myPass@some-name-mongos.pitr-physical-backup-source-27258
+ local 'command=use myApp\n db.test.insert({ x: 100500 })'
+ local uri=myApp:myPass@some-name-mongos.pitr-physical-backup-source-27258
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local port=27017
+ local mongo_bin=mongo
++ echo .svc.cluster.local
++ awk -F: '{print $2}'
+ suffix_port=
+ [[ -z '' ]]
+ suffix=.svc.cluster.local:27017
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.YKKDMjNy68
+++ mktemp
++ local LAST_ERR=/tmp/tmp.oUDrO8M3sh
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.YKKDMjNy68
++ cat /tmp/tmp.oUDrO8M3sh
++ rm /tmp/tmp.YKKDMjNy68 /tmp/tmp.oUDrO8M3sh
++ return 0
+ local client_container=psmdb-client-bb8b97679-c5ptc
+ kubectl_bin exec psmdb-client-bb8b97679-c5ptc -- bash -c 'printf '\''use myApp\n db.test.insert({ x: 100500 })\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.pitr-physical-backup-source-27258.svc.cluster.local:27017/admin '
++ mktemp
+ local LAST_OUT=/tmp/tmp.Niqyof4hMQ
++ mktemp
+ local LAST_ERR=/tmp/tmp.ExYAFTP5cL
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-bb8b97679-c5ptc -- bash -c 'printf '\''use myApp\n db.test.insert({ x: 100500 })\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.pitr-physical-backup-source-27258.svc.cluster.local:27017/admin '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.Niqyof4hMQ
Percona Server for MongoDB shell version v4.4.29-28
connecting to: mongodb://some-name-mongos.pitr-physical-backup-source-27258.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb
Implicit session: session { "id" : UUID("4301dae5-9a79-4f50-9c2f-0e399feb6716") }
Percona Server for MongoDB server version: v8.0.21-9
WARNING: shell and server versions do not match
switched to db myApp
WriteResult({ "nInserted" : 1 })
bye
+ cat /tmp/tmp.ExYAFTP5cL
+ rm /tmp/tmp.Niqyof4hMQ /tmp/tmp.ExYAFTP5cL
+ return 0
+ sleep 120
+ compare_mongos_cmd find myApp:myPass@some-name-mongos.pitr-physical-backup-source-27258
+ local command=find
+ local uri=myApp:myPass@some-name-mongos.pitr-physical-backup-source-27258
+ local postfix=
+ local suffix=
+ local database=myApp
+ local collection=test
+ local port=27017
+ local tls=false
+ [[ false == true ]]
+ mongos_command=run_mongos
+ log 'running db.test.command() in myApp'
+ set +o xtrace
[2026-05-21T09:16:03+0000] running db.test.command() in myApp
+ run_mongos 'use myApp\n db.test.find()' myApp:myPass@some-name-mongos.pitr-physical-backup-source-27258 mongodb '' '' 27017
+ local 'command=use myApp\n db.test.find()'
+ local uri=myApp:myPass@some-name-mongos.pitr-physical-backup-source-27258
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local port=27017
+ local mongo_bin=mongo
+ grep -E -v 'I NETWORK|W NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|Started a new thread for the timer service'
+ /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
++ echo .svc.cluster.local
++ awk -F: '{print $2}'
+ suffix_port=
+ [[ -z '' ]]
+ suffix=.svc.cluster.local:27017
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.XMjPxrbgYt
+++ mktemp
++ local LAST_ERR=/tmp/tmp.UJNMpz7KxM
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.XMjPxrbgYt
++ cat /tmp/tmp.UJNMpz7KxM
++ rm /tmp/tmp.XMjPxrbgYt /tmp/tmp.UJNMpz7KxM
++ return 0
+ local client_container=psmdb-client-bb8b97679-c5ptc
+ kubectl_bin exec psmdb-client-bb8b97679-c5ptc -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.pitr-physical-backup-source-27258.svc.cluster.local:27017/admin '
++ mktemp
+ local LAST_OUT=/tmp/tmp.8kBazcR5FA
++ mktemp
+ local LAST_ERR=/tmp/tmp.MTvrTKHdk4
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-bb8b97679-c5ptc -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.pitr-physical-backup-source-27258.svc.cluster.local:27017/admin '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.8kBazcR5FA
+ cat /tmp/tmp.MTvrTKHdk4
+ rm /tmp/tmp.8kBazcR5FA /tmp/tmp.MTvrTKHdk4
+ return 0
+ [[ 0 -eq 0 ]]
+ diff /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2328/e2e-tests/pitr-physical-backup-source/compare/find.json /tmp/tmp.DTCahJpmH8/find
++ run_mongos 'new Date().getTime() / 1000' myApp:myPass@some-name-mongos.pitr-physical-backup-source-27258 mongodb '' --quiet
++ local 'command=new Date().getTime() / 1000'
++ local uri=myApp:myPass@some-name-mongos.pitr-physical-backup-source-27258
++ local driver=mongodb
++ local suffix=.svc.cluster.local
++ local mongo_flag=--quiet
++ local port=27017
++ local mongo_bin=mongo
++ cut -d. -f1
++ grep -E -v 'I NETWORK|W NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match'
+++ echo .svc.cluster.local
+++ awk -F: '{print $2}'
++ suffix_port=
++ [[ -z '' ]]
++ suffix=.svc.cluster.local:27017
+++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.ggQ2vwxKeh
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.88BYM6Tg09
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in $(seq 0 2)
+++ set +e
+++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.ggQ2vwxKeh
+++ cat /tmp/tmp.88BYM6Tg09
+++ rm /tmp/tmp.ggQ2vwxKeh /tmp/tmp.88BYM6Tg09
+++ return 0
++ local client_container=psmdb-client-bb8b97679-c5ptc
++ kubectl_bin exec psmdb-client-bb8b97679-c5ptc -- bash -c 'printf '\''new Date().getTime() / 1000\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.pitr-physical-backup-source-27258.svc.cluster.local:27017/admin --quiet'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.N5HuDmeupw
+++ mktemp
++ local LAST_ERR=/tmp/tmp.Vk66oBZswa
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl exec psmdb-client-bb8b97679-c5ptc -- bash -c 'printf '\''new Date().getTime() / 1000\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.pitr-physical-backup-source-27258.svc.cluster.local:27017/admin --quiet'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.N5HuDmeupw
++ cat /tmp/tmp.Vk66oBZswa
++ rm /tmp/tmp.N5HuDmeupw /tmp/tmp.Vk66oBZswa
++ return 0
+ time_now=1779354969
+ check_recovery backup-minio-1 date 1779354969 '' some-name backupSource
+ local backup_name=backup-minio-1
+ local restore_type=date
+ local restore_date=1779354969
+ local cmp_postfix=
+ local cluster_name=some-name
+ local backupSource=backupSource
++ get_latest_oplog_chunk_ts some-name
++ local cluster=some-name
+++ kubectl_bin exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ jq '.backups.pitrChunks.pitrChunks | last | .range.end'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.1pphgqZEY1
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.7JCJs6pdyG
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in $(seq 0 2)
+++ set +e
+++ kubectl exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.1pphgqZEY1
+++ cat /tmp/tmp.7JCJs6pdyG
+++ rm /tmp/tmp.1pphgqZEY1 /tmp/tmp.7JCJs6pdyG
+++ return 0
++ echo 1779354934
+ local latest_ts=1779354934
+ desc 'write more data before restore by date'
+ set +o xtrace


-----------------------------------------------------------------------------------
write more data before restore by date
-----------------------------------------------------------------------------------

+ run_mongos 'use myApp\n db.test.insert({ x: 100501 })' myApp:myPass@some-name-mongos.pitr-physical-backup-source-27258
+ local 'command=use myApp\n db.test.insert({ x: 100501 })'
+ local uri=myApp:myPass@some-name-mongos.pitr-physical-backup-source-27258
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local port=27017
+ local mongo_bin=mongo
++ echo .svc.cluster.local
++ awk -F: '{print $2}'
+ suffix_port=
+ [[ -z '' ]]
+ suffix=.svc.cluster.local:27017
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.5fA7zFbyGH
+++ mktemp
++ local LAST_ERR=/tmp/tmp.FsRESv5qeA
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.5fA7zFbyGH
++ cat /tmp/tmp.FsRESv5qeA
++ rm /tmp/tmp.5fA7zFbyGH /tmp/tmp.FsRESv5qeA
++ return 0
+ local client_container=psmdb-client-bb8b97679-c5ptc
+ kubectl_bin exec psmdb-client-bb8b97679-c5ptc -- bash -c 'printf '\''use myApp\n db.test.insert({ x: 100501 })\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.pitr-physical-backup-source-27258.svc.cluster.local:27017/admin '
++ mktemp
+ local LAST_OUT=/tmp/tmp.zsmTPxfLeJ
++ mktemp
+ local LAST_ERR=/tmp/tmp.eFjAqRxBV9
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-bb8b97679-c5ptc -- bash -c 'printf '\''use myApp\n db.test.insert({ x: 100501 })\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.pitr-physical-backup-source-27258.svc.cluster.local:27017/admin '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.zsmTPxfLeJ
Percona Server for MongoDB shell version v4.4.29-28
connecting to: mongodb://some-name-mongos.pitr-physical-backup-source-27258.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb
Implicit session: session { "id" : UUID("33b24034-cd8c-40f1-ae09-a29c2cd8bc77") }
Percona Server for MongoDB server version: v8.0.21-9
WARNING: shell and server versions do not match
switched to db myApp
WriteResult({ "nInserted" : 1 })
bye
+ cat /tmp/tmp.eFjAqRxBV9
+ rm /tmp/tmp.zsmTPxfLeJ /tmp/tmp.eFjAqRxBV9
+ return 0
+ [[ -n 1779354969 ]]
++ format_date 1779354969
++ local timestamp=1779354969
+++ TZ=UTC
+++ /usr/sbin/date -d@1779354969 '+%Y-%m-%d %H:%M:%S'
++ echo 2026-05-21 09:16:09
+ desc 'Restoring to time 2026-05-21 09:16:09'
+ set +o xtrace


-----------------------------------------------------------------------------------
Restoring to time 2026-05-21 09:16:09
-----------------------------------------------------------------------------------

+ retries=0
+ [[ 1779354934 -gt 1779354969 ]]
+ [[ 0 -gt 30 ]]
++ get_latest_oplog_chunk_ts some-name
++ local cluster=some-name
+++ kubectl_bin exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ jq '.backups.pitrChunks.pitrChunks | last | .range.end'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.MDvjQD48ge
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.w6V3fDF2pW
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in $(seq 0 2)
+++ set +e
+++ kubectl exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.MDvjQD48ge
+++ cat /tmp/tmp.w6V3fDF2pW
+++ rm /tmp/tmp.MDvjQD48ge /tmp/tmp.w6V3fDF2pW
+++ return 0
++ echo 1779354934
+ latest_ts=1779354934
+ retries=1
++ format_date 1779354934
++ local timestamp=1779354934
+++ TZ=UTC
+++ /usr/sbin/date -d@1779354934 '+%Y-%m-%d %H:%M:%S'
++ echo 2026-05-21 09:15:34
++ format_date 1779354969
++ local timestamp=1779354969
+++ TZ=UTC
+++ /usr/sbin/date -d@1779354969 '+%Y-%m-%d %H:%M:%S'
++ echo 2026-05-21 09:16:09
+ echo 'Waiting for last oplog chunk (2026-05-21 09:15:34) to be greater than restore target (2026-05-21 09:16:09)'
Waiting for last oplog chunk (2026-05-21 09:15:34) to be greater than restore target (2026-05-21 09:16:09)
+ sleep 10
+ [[ 1779354934 -gt 1779354969 ]]
+ [[ 1 -gt 30 ]]
++ get_latest_oplog_chunk_ts some-name
++ local cluster=some-name
+++ kubectl_bin exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ jq '.backups.pitrChunks.pitrChunks | last | .range.end'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.BOcZV6DGQR
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.eyHSmlgYim
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in $(seq 0 2)
+++ set +e
+++ kubectl exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.BOcZV6DGQR
+++ cat /tmp/tmp.eyHSmlgYim
+++ rm /tmp/tmp.BOcZV6DGQR /tmp/tmp.eyHSmlgYim
+++ return 0
++ echo 1779354934
+ latest_ts=1779354934
+ retries=2
++ format_date 1779354934
++ local timestamp=1779354934
+++ TZ=UTC
+++ /usr/sbin/date -d@1779354934 '+%Y-%m-%d %H:%M:%S'
++ echo 2026-05-21 09:15:34
++ format_date 1779354969
++ local timestamp=1779354969
+++ TZ=UTC
+++ /usr/sbin/date -d@1779354969 '+%Y-%m-%d %H:%M:%S'
++ echo 2026-05-21 09:16:09
+ echo 'Waiting for last oplog chunk (2026-05-21 09:15:34) to be greater than restore target (2026-05-21 09:16:09)'
Waiting for last oplog chunk (2026-05-21 09:15:34) to be greater than restore target (2026-05-21 09:16:09)
+ sleep 10
+ [[ 1779354934 -gt 1779354969 ]]
+ [[ 2 -gt 30 ]]
++ get_latest_oplog_chunk_ts some-name
++ local cluster=some-name
+++ kubectl_bin exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ jq '.backups.pitrChunks.pitrChunks | last | .range.end'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.0Uj4VktAf4
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.8Vx4Jii5Pa
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in $(seq 0 2)
+++ set +e
+++ kubectl exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.0Uj4VktAf4
+++ cat /tmp/tmp.8Vx4Jii5Pa
+++ rm /tmp/tmp.0Uj4VktAf4 /tmp/tmp.8Vx4Jii5Pa
+++ return 0
++ echo 1779354934
+ latest_ts=1779354934
+ retries=3
++ format_date 1779354934
++ local timestamp=1779354934
+++ TZ=UTC
+++ /usr/sbin/date -d@1779354934 '+%Y-%m-%d %H:%M:%S'
++ echo 2026-05-21 09:15:34
++ format_date 1779354969
++ local timestamp=1779354969
+++ TZ=UTC
+++ /usr/sbin/date -d@1779354969 '+%Y-%m-%d %H:%M:%S'
++ echo 2026-05-21 09:16:09
+ echo 'Waiting for last oplog chunk (2026-05-21 09:15:34) to be greater than restore target (2026-05-21 09:16:09)'
Waiting for last oplog chunk (2026-05-21 09:15:34) to be greater than restore target (2026-05-21 09:16:09)
+ sleep 10
+ [[ 1779354934 -gt 1779354969 ]]
+ [[ 3 -gt 30 ]]
++ get_latest_oplog_chunk_ts some-name
++ local cluster=some-name
+++ kubectl_bin exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ jq '.backups.pitrChunks.pitrChunks | last | .range.end'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.jZrlf0og1S
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.JoynsLatub
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in $(seq 0 2)
+++ set +e
+++ kubectl exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.jZrlf0og1S
+++ cat /tmp/tmp.JoynsLatub
+++ rm /tmp/tmp.jZrlf0og1S /tmp/tmp.JoynsLatub
+++ return 0
++ echo 1779354934
+ latest_ts=1779354934
+ retries=4
++ format_date 1779354934
++ local timestamp=1779354934
+++ TZ=UTC
+++ /usr/sbin/date -d@1779354934 '+%Y-%m-%d %H:%M:%S'
++ echo 2026-05-21 09:15:34
++ format_date 1779354969
++ local timestamp=1779354969
+++ TZ=UTC
+++ /usr/sbin/date -d@1779354969 '+%Y-%m-%d %H:%M:%S'
++ echo 2026-05-21 09:16:09
+ echo 'Waiting for last oplog chunk (2026-05-21 09:15:34) to be greater than restore target (2026-05-21 09:16:09)'
Waiting for last oplog chunk (2026-05-21 09:15:34) to be greater than restore target (2026-05-21 09:16:09)
+ sleep 10
+ [[ 1779354934 -gt 1779354969 ]]
+ [[ 4 -gt 30 ]]
++ get_latest_oplog_chunk_ts some-name
++ local cluster=some-name
+++ kubectl_bin exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ jq '.backups.pitrChunks.pitrChunks | last | .range.end'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.bvvASJCTeY
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.U2trlE2UTU
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in $(seq 0 2)
+++ set +e
+++ kubectl exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.bvvASJCTeY
+++ cat /tmp/tmp.U2trlE2UTU
+++ rm /tmp/tmp.bvvASJCTeY /tmp/tmp.U2trlE2UTU
+++ return 0
++ echo 1779354934
+ latest_ts=1779354934
+ retries=5
++ format_date 1779354934
++ local timestamp=1779354934
+++ TZ=UTC
+++ /usr/sbin/date -d@1779354934 '+%Y-%m-%d %H:%M:%S'
++ echo 2026-05-21 09:15:34
++ format_date 1779354969
++ local timestamp=1779354969
+++ TZ=UTC
+++ /usr/sbin/date -d@1779354969 '+%Y-%m-%d %H:%M:%S'
++ echo 2026-05-21 09:16:09
+ echo 'Waiting for last oplog chunk (2026-05-21 09:15:34) to be greater than restore target (2026-05-21 09:16:09)'
Waiting for last oplog chunk (2026-05-21 09:15:34) to be greater than restore target (2026-05-21 09:16:09)
+ sleep 10
+ [[ 1779354934 -gt 1779354969 ]]
+ [[ 5 -gt 30 ]]
++ get_latest_oplog_chunk_ts some-name
++ local cluster=some-name
+++ kubectl_bin exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ jq '.backups.pitrChunks.pitrChunks | last | .range.end'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.meweycqcOC
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.9DSlgPgBL5
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in $(seq 0 2)
+++ set +e
+++ kubectl exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.meweycqcOC
+++ cat /tmp/tmp.9DSlgPgBL5
+++ rm /tmp/tmp.meweycqcOC /tmp/tmp.9DSlgPgBL5
+++ return 0
++ echo 1779354934
+ latest_ts=1779354934
+ retries=6
++ format_date 1779354934
++ local timestamp=1779354934
+++ TZ=UTC
+++ /usr/sbin/date -d@1779354934 '+%Y-%m-%d %H:%M:%S'
++ echo 2026-05-21 09:15:34
++ format_date 1779354969
++ local timestamp=1779354969
+++ TZ=UTC
+++ /usr/sbin/date -d@1779354969 '+%Y-%m-%d %H:%M:%S'
++ echo 2026-05-21 09:16:09
+ echo 'Waiting for last oplog chunk (2026-05-21 09:15:34) to be greater than restore target (2026-05-21 09:16:09)'
Waiting for last oplog chunk (2026-05-21 09:15:34) to be greater than restore target (2026-05-21 09:16:09)
+ sleep 10
+ [[ 1779354934 -gt 1779354969 ]]
+ [[ 6 -gt 30 ]]
++ get_latest_oplog_chunk_ts some-name
++ local cluster=some-name
+++ kubectl_bin exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ jq '.backups.pitrChunks.pitrChunks | last | .range.end'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.EqJpznAhwr
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.ILoA476F1b
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in $(seq 0 2)
+++ set +e
+++ kubectl exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.EqJpznAhwr
+++ cat /tmp/tmp.ILoA476F1b
+++ rm /tmp/tmp.EqJpznAhwr /tmp/tmp.ILoA476F1b
+++ return 0
++ echo 1779354934
+ latest_ts=1779354934
+ retries=7
++ format_date 1779354934
++ local timestamp=1779354934
+++ TZ=UTC
+++ /usr/sbin/date -d@1779354934 '+%Y-%m-%d %H:%M:%S'
++ echo 2026-05-21 09:15:34
++ format_date 1779354969
++ local timestamp=1779354969
+++ TZ=UTC
+++ /usr/sbin/date -d@1779354969 '+%Y-%m-%d %H:%M:%S'
++ echo 2026-05-21 09:16:09
+ echo 'Waiting for last oplog chunk (2026-05-21 09:15:34) to be greater than restore target (2026-05-21 09:16:09)'
Waiting for last oplog chunk (2026-05-21 09:15:34) to be greater than restore target (2026-05-21 09:16:09)
+ sleep 10
+ [[ 1779354934 -gt 1779354969 ]]
+ [[ 7 -gt 30 ]]
++ get_latest_oplog_chunk_ts some-name
++ local cluster=some-name
+++ kubectl_bin exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ jq '.backups.pitrChunks.pitrChunks | last | .range.end'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.DEbOE2wmq1
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.bjkBZZomSQ
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in $(seq 0 2)
+++ set +e
+++ kubectl exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.DEbOE2wmq1
+++ cat /tmp/tmp.bjkBZZomSQ
+++ rm /tmp/tmp.DEbOE2wmq1 /tmp/tmp.bjkBZZomSQ
+++ return 0
++ echo 1779354934
+ latest_ts=1779354934
+ retries=8
++ format_date 1779354934
++ local timestamp=1779354934
+++ TZ=UTC
+++ /usr/sbin/date -d@1779354934 '+%Y-%m-%d %H:%M:%S'
++ echo 2026-05-21 09:15:34
++ format_date 1779354969
++ local timestamp=1779354969
+++ TZ=UTC
+++ /usr/sbin/date -d@1779354969 '+%Y-%m-%d %H:%M:%S'
++ echo 2026-05-21 09:16:09
+ echo 'Waiting for last oplog chunk (2026-05-21 09:15:34) to be greater than restore target (2026-05-21 09:16:09)'
Waiting for last oplog chunk (2026-05-21 09:15:34) to be greater than restore target (2026-05-21 09:16:09)
+ sleep 10
+ [[ 1779354934 -gt 1779354969 ]]
+ [[ 8 -gt 30 ]]
++ get_latest_oplog_chunk_ts some-name
++ local cluster=some-name
+++ kubectl_bin exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ jq '.backups.pitrChunks.pitrChunks | last | .range.end'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.JK5AJKn1da
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.xBfoqe5rWS
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in $(seq 0 2)
+++ set +e
+++ kubectl exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.JK5AJKn1da
+++ cat /tmp/tmp.xBfoqe5rWS
+++ rm /tmp/tmp.JK5AJKn1da /tmp/tmp.xBfoqe5rWS
+++ return 0
++ echo 1779355054
+ latest_ts=1779355054
+ retries=9
++ format_date 1779355054
++ local timestamp=1779355054
+++ TZ=UTC
+++ /usr/sbin/date -d@1779355054 '+%Y-%m-%d %H:%M:%S'
++ echo 2026-05-21 09:17:34
++ format_date 1779354969
++ local timestamp=1779354969
+++ TZ=UTC
+++ /usr/sbin/date -d@1779354969 '+%Y-%m-%d %H:%M:%S'
++ echo 2026-05-21 09:16:09
+ echo 'Waiting for last oplog chunk (2026-05-21 09:17:34) to be greater than restore target (2026-05-21 09:16:09)'
Waiting for last oplog chunk (2026-05-21 09:17:34) to be greater than restore target (2026-05-21 09:16:09)
+ sleep 10
+ [[ 1779355054 -gt 1779354969 ]]
+ '[' -z backupSource ']'
+ desc 'check restore by date backupSource'
+ set +o xtrace


-----------------------------------------------------------------------------------
check restore by date backupSource
-----------------------------------------------------------------------------------

++ get_backup_dest backup-minio-1
++ local backup_name=backup-minio-1
++ kubectl_bin get psmdb-backup backup-minio-1 -o 'jsonpath={.status.destination}'
++ sed -e 's/.json$//'
++ sed 's|s3://||'
+++ mktemp
++ sed 's|azure://||'
++ local LAST_OUT=/tmp/tmp.He7xjkUkKw
++ sed 's|gs://||'
+++ mktemp
++ local LAST_ERR=/tmp/tmp.yUXYkR62c8
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb-backup backup-minio-1 -o 'jsonpath={.status.destination}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.He7xjkUkKw
++ cat /tmp/tmp.yUXYkR62c8
++ rm /tmp/tmp.He7xjkUkKw /tmp/tmp.yUXYkR62c8
++ return 0
+ backup_dest=operator-testing/2026-05-21T09:11:10Z
+ cat /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2328/e2e-tests/pitr-physical-backup-source/conf/restore.yml
+ /usr/sbin/sed -e 's/name:/name: restore-backup-minio-1/'
+ /usr/sbin/sed -e /backupName/d
+ /usr/sbin/sed -e 's/pitrType:/type: date/'
+ '[' -z 1779354969 ']'
+ /usr/sbin/sed -e 's|DESTINATION|operator-testing/2026-05-21T09:11:10Z|'
+ '[' -n '' ']'
+ yq
+ kubectl_bin apply -f -
++ format_date 1779354969
++ local timestamp=1779354969
++ mktemp
++ get_bucket_name backup-minio-1
++ local backup_name=backup-minio-1
++ kubectl_bin get psmdb-backup backup-minio-1 -o 'jsonpath={.status.s3.bucket}'
+++ TZ=UTC
+++ /usr/sbin/date -d@1779354969 '+%Y-%m-%d %H:%M:%S'
+++ mktemp
+ local LAST_OUT=/tmp/tmp.E9gCqzbDIu
++ mktemp
++ echo 2026-05-21 09:16:09
++ local LAST_OUT=/tmp/tmp.56WNNbQJMQ
+ /usr/sbin/sed -e 's/date:/date: 2026-05-21 09:16:09/'
+++ mktemp
+ local LAST_ERR=/tmp/tmp.YmPhLEndr0
+ local exit_status=0
+ local timeout=4
++ seq 0 2
++ local LAST_ERR=/tmp/tmp.CKeYt3EFr2
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl apply -f -
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb-backup backup-minio-1 -o 'jsonpath={.status.s3.bucket}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.56WNNbQJMQ
++ cat /tmp/tmp.CKeYt3EFr2
++ rm /tmp/tmp.56WNNbQJMQ /tmp/tmp.CKeYt3EFr2
++ return 0
+ /usr/sbin/sed -e 's|BUCKET-NAME|operator-testing|'
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.E9gCqzbDIu
perconaservermongodbrestore.psmdb.percona.com/restore-backup-minio-1 created
+ cat /tmp/tmp.YmPhLEndr0
+ rm /tmp/tmp.E9gCqzbDIu /tmp/tmp.YmPhLEndr0
+ return 0
+ wait_restore backup-minio-1 some-name requested 0 1200
+ local backup_name=backup-minio-1
+ local cluster_name=some-name
+ local target_state=requested
+ local wait_cluster_consistency=0
+ local wait_time=1200
+ local ok_if_ready=0
+ set +o xtrace
Waiting for the psmdb-restore/restore-backup-minio-1 object to be created.OK
Waiting psmdb-restore/restore-backup-minio-1 to reach state "requested" ........OK after 7 minutes
+ [[ 0 -eq 1 ]]
+ echo

+ wait_restore backup-minio-1 some-name ready 0 1600
+ local backup_name=backup-minio-1
+ local cluster_name=some-name
+ local target_state=ready
+ local wait_cluster_consistency=0
+ local wait_time=1600
+ local ok_if_ready=0
+ set +o xtrace
Waiting for the psmdb-restore/restore-backup-minio-1 object to be created.OK
Waiting psmdb-restore/restore-backup-minio-1 to reach state "ready" ...OK after 2 minutes
+ [[ 0 -eq 1 ]]
+ echo

+ set -o xtrace
+ wait_for_running some-name-rs0 3
+ local name=some-name-rs0
+ let last_pod=2
+ local check_cluster_readyness=true
+ set_debug
+ [[ 1 == 1 ]]
+ set -o xtrace
+ local rs_name=rs0
+ local cluster_name=some-name
++ seq 0 2
+ for i in $(seq 0 $last_pod)
+ [[ 0 -eq 2 ]]
+ wait_pod some-name-rs0-0
+ local pod=some-name-rs0-0
+ set +o xtrace
waiting for pod/some-name-rs0-0 to be ready...........OK
+ for i in $(seq 0 $last_pod)
+ [[ 1 -eq 2 ]]
+ wait_pod some-name-rs0-1
+ local pod=some-name-rs0-1
+ set +o xtrace
waiting for pod/some-name-rs0-1 to be ready.....OK
+ for i in $(seq 0 $last_pod)
+ [[ 2 -eq 2 ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].arbiter.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.dNdbSdFjNs
+++ mktemp
++ local LAST_ERR=/tmp/tmp.XJ7lAWhWLs
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].arbiter.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.dNdbSdFjNs
++ cat /tmp/tmp.XJ7lAWhWLs
++ rm /tmp/tmp.dNdbSdFjNs /tmp/tmp.XJ7lAWhWLs
++ return 0
+ [[ '' == true ]]
+ wait_pod some-name-rs0-2
+ local pod=some-name-rs0-2
+ set +o xtrace
waiting for pod/some-name-rs0-2 to be ready...OK
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].nonvoting.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.pPbgflWFIC
+++ mktemp
++ local LAST_ERR=/tmp/tmp.B0SL3ut94X
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].nonvoting.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.pPbgflWFIC
++ cat /tmp/tmp.B0SL3ut94X
++ rm /tmp/tmp.pPbgflWFIC /tmp/tmp.B0SL3ut94X
++ return 0
+ [[ '' == true ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].hidden.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.GsNPLjhsKh
+++ mktemp
++ local LAST_ERR=/tmp/tmp.AHRdmqvGoR
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].hidden.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.GsNPLjhsKh
++ cat /tmp/tmp.AHRdmqvGoR
++ rm /tmp/tmp.GsNPLjhsKh /tmp/tmp.AHRdmqvGoR
++ return 0
+ [[ '' == true ]]
+ sleep 10
+ [[ true == true ]]
+ set +x
Waiting for cluster readyness..........
+ wait_for_running some-name-cfg 3
+ local name=some-name-cfg
+ let last_pod=2
+ local check_cluster_readyness=true
+ set_debug
+ [[ 1 == 1 ]]
+ set -o xtrace
+ local rs_name=cfg
+ local cluster_name=some-name
++ seq 0 2
+ for i in $(seq 0 $last_pod)
+ [[ 0 -eq 2 ]]
+ wait_pod some-name-cfg-0
+ local pod=some-name-cfg-0
+ set +o xtrace
waiting for pod/some-name-cfg-0 to be ready.OK
+ for i in $(seq 0 $last_pod)
+ [[ 1 -eq 2 ]]
+ wait_pod some-name-cfg-1
+ local pod=some-name-cfg-1
+ set +o xtrace
waiting for pod/some-name-cfg-1 to be ready.OK
+ for i in $(seq 0 $last_pod)
+ [[ 2 -eq 2 ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].arbiter.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.qE4I095UHK
+++ mktemp
++ local LAST_ERR=/tmp/tmp.arZMnvYdit
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].arbiter.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.qE4I095UHK
++ cat /tmp/tmp.arZMnvYdit
++ rm /tmp/tmp.qE4I095UHK /tmp/tmp.arZMnvYdit
++ return 0
+ [[ '' == true ]]
+ wait_pod some-name-cfg-2
+ local pod=some-name-cfg-2
+ set +o xtrace
waiting for pod/some-name-cfg-2 to be ready.OK
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].nonvoting.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.BprwLsHyiW
+++ mktemp
++ local LAST_ERR=/tmp/tmp.GFGzD2IZTt
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].nonvoting.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.BprwLsHyiW
++ cat /tmp/tmp.GFGzD2IZTt
++ rm /tmp/tmp.BprwLsHyiW /tmp/tmp.GFGzD2IZTt
++ return 0
+ [[ '' == true ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].hidden.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.3D99nUpSO6
+++ mktemp
++ local LAST_ERR=/tmp/tmp.BOXJQ5Ygkm
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].hidden.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.3D99nUpSO6
++ cat /tmp/tmp.BOXJQ5Ygkm
++ rm /tmp/tmp.3D99nUpSO6 /tmp/tmp.BOXJQ5Ygkm
++ return 0
+ [[ '' == true ]]
+ sleep 10
+ [[ true == true ]]
+ set +x
Waiting for cluster readyness
+ wait_for_running some-name-mongos 3
+ local name=some-name-mongos
+ let last_pod=2
+ local check_cluster_readyness=true
+ set_debug
+ [[ 1 == 1 ]]
+ set -o xtrace
+ local rs_name=mongos
+ local cluster_name=some-name
++ seq 0 2
+ for i in $(seq 0 $last_pod)
+ [[ 0 -eq 2 ]]
+ wait_pod some-name-mongos-0
+ local pod=some-name-mongos-0
+ set +o xtrace
waiting for pod/some-name-mongos-0 to be ready.OK
+ for i in $(seq 0 $last_pod)
+ [[ 1 -eq 2 ]]
+ wait_pod some-name-mongos-1
+ local pod=some-name-mongos-1
+ set +o xtrace
waiting for pod/some-name-mongos-1 to be ready.OK
+ for i in $(seq 0 $last_pod)
+ [[ 2 -eq 2 ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="mongos")].arbiter.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.5NCKGiBH0f
+++ mktemp
++ local LAST_ERR=/tmp/tmp.RdF0YbxNia
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="mongos")].arbiter.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.5NCKGiBH0f
++ cat /tmp/tmp.RdF0YbxNia
++ rm /tmp/tmp.5NCKGiBH0f /tmp/tmp.RdF0YbxNia
++ return 0
+ [[ '' == true ]]
+ wait_pod some-name-mongos-2
+ local pod=some-name-mongos-2
+ set +o xtrace
waiting for pod/some-name-mongos-2 to be ready.OK
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="mongos")].nonvoting.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.p9RB2jMweS
+++ mktemp
++ local LAST_ERR=/tmp/tmp.DUdaXf3JVI
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="mongos")].nonvoting.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.p9RB2jMweS
++ cat /tmp/tmp.DUdaXf3JVI
++ rm /tmp/tmp.p9RB2jMweS /tmp/tmp.DUdaXf3JVI
++ return 0
+ [[ '' == true ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="mongos")].hidden.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.nVyJhBhDNu
+++ mktemp
++ local LAST_ERR=/tmp/tmp.0O4ZQdgoKF
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="mongos")].hidden.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.nVyJhBhDNu
++ cat /tmp/tmp.0O4ZQdgoKF
++ rm /tmp/tmp.nVyJhBhDNu /tmp/tmp.0O4ZQdgoKF
++ return 0
+ [[ '' == true ]]
+ sleep 10
+ [[ true == true ]]
+ set +x
Waiting for cluster readyness
+ sleep 10
+ compare_mongos_cmd find myApp:myPass@some-name-mongos.pitr-physical-backup-source-27258 ''
+ local command=find
+ local uri=myApp:myPass@some-name-mongos.pitr-physical-backup-source-27258
+ local postfix=
+ local suffix=
+ local database=myApp
+ local collection=test
+ local port=27017
+ local tls=false
+ [[ false == true ]]
+ mongos_command=run_mongos
+ log 'running db.test.command() in myApp'
+ set +o xtrace
[2026-05-21T09:31:41+0000] running db.test.command() in myApp
+ run_mongos 'use myApp\n db.test.find()' myApp:myPass@some-name-mongos.pitr-physical-backup-source-27258 mongodb '' '' 27017
+ local 'command=use myApp\n db.test.find()'
+ local uri=myApp:myPass@some-name-mongos.pitr-physical-backup-source-27258
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local port=27017
+ local mongo_bin=mongo
+ grep -E -v 'I NETWORK|W NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|Started a new thread for the timer service'
+ /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
++ echo .svc.cluster.local
++ awk -F: '{print $2}'
+ suffix_port=
+ [[ -z '' ]]
+ suffix=.svc.cluster.local:27017
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.NYTbHyfgN1
+++ mktemp
++ local LAST_ERR=/tmp/tmp.JoktPRbAxO
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.NYTbHyfgN1
++ cat /tmp/tmp.JoktPRbAxO
++ rm /tmp/tmp.NYTbHyfgN1 /tmp/tmp.JoktPRbAxO
++ return 0
+ local client_container=psmdb-client-bb8b97679-c5ptc
+ kubectl_bin exec psmdb-client-bb8b97679-c5ptc -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.pitr-physical-backup-source-27258.svc.cluster.local:27017/admin '
++ mktemp
+ local LAST_OUT=/tmp/tmp.4hee3iKtVd
++ mktemp
+ local LAST_ERR=/tmp/tmp.u5RADcKscQ
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-bb8b97679-c5ptc -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.pitr-physical-backup-source-27258.svc.cluster.local:27017/admin '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.4hee3iKtVd
+ cat /tmp/tmp.u5RADcKscQ
+ rm /tmp/tmp.4hee3iKtVd /tmp/tmp.u5RADcKscQ
+ return 0
+ [[ 0 -eq 0 ]]
+ diff /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2328/e2e-tests/pitr-physical-backup-source/compare/find.json /tmp/tmp.DTCahJpmH8/find
+ desc 'delete PSMDB cluster some-name'
+ set +o xtrace


-----------------------------------------------------------------------------------
delete PSMDB cluster some-name
-----------------------------------------------------------------------------------

+ kubectl_bin delete psmdb some-name
++ mktemp
+ local LAST_OUT=/tmp/tmp.vAYAol7xLL
++ mktemp
+ local LAST_ERR=/tmp/tmp.ZCcOa9ipei
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl delete psmdb some-name
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.vAYAol7xLL
perconaservermongodb.psmdb.percona.com "some-name" deleted from pitr-physical-backup-source-27258 namespace
+ cat /tmp/tmp.ZCcOa9ipei
+ rm /tmp/tmp.vAYAol7xLL /tmp/tmp.ZCcOa9ipei
+ return 0
+ kubectl_bin delete pvc -l app.kubernetes.io/managed-by=percona-server-mongodb-operator
++ mktemp
+ local LAST_OUT=/tmp/tmp.K5jbq8kE6q
++ mktemp
+ local LAST_ERR=/tmp/tmp.EWKcJ1EGvw
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl delete pvc -l app.kubernetes.io/managed-by=percona-server-mongodb-operator
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.K5jbq8kE6q
persistentvolumeclaim "mongod-data-some-name-cfg-0" deleted from pitr-physical-backup-source-27258 namespace
persistentvolumeclaim "mongod-data-some-name-cfg-1" deleted from pitr-physical-backup-source-27258 namespace
persistentvolumeclaim "mongod-data-some-name-cfg-2" deleted from pitr-physical-backup-source-27258 namespace
persistentvolumeclaim "mongod-data-some-name-rs0-0" deleted from pitr-physical-backup-source-27258 namespace
persistentvolumeclaim "mongod-data-some-name-rs0-1" deleted from pitr-physical-backup-source-27258 namespace
persistentvolumeclaim "mongod-data-some-name-rs0-2" deleted from pitr-physical-backup-source-27258 namespace
persistentvolumeclaim "mongod-data-some-name-rs1-0" deleted from pitr-physical-backup-source-27258 namespace
persistentvolumeclaim "mongod-data-some-name-rs1-1" deleted from pitr-physical-backup-source-27258 namespace
persistentvolumeclaim "mongod-data-some-name-rs1-2" deleted from pitr-physical-backup-source-27258 namespace
persistentvolumeclaim "mongod-data-some-name-rs2-0" deleted from pitr-physical-backup-source-27258 namespace
persistentvolumeclaim "mongod-data-some-name-rs2-1" deleted from pitr-physical-backup-source-27258 namespace
persistentvolumeclaim "mongod-data-some-name-rs2-2" deleted from pitr-physical-backup-source-27258 namespace
+ cat /tmp/tmp.EWKcJ1EGvw
+ rm /tmp/tmp.K5jbq8kE6q /tmp/tmp.EWKcJ1EGvw
+ return 0
+ sleep 10
+ desc 'recreate PSMDB cluster some-name'
+ set +o xtrace


-----------------------------------------------------------------------------------
recreate PSMDB cluster some-name
-----------------------------------------------------------------------------------

+ desc 'create second PSMDB cluster some-name'
+ set +o xtrace


-----------------------------------------------------------------------------------
create second PSMDB cluster some-name
-----------------------------------------------------------------------------------

+ apply_cluster /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2328/e2e-tests/pitr-physical-backup-source/conf/some-name-rs0.yml
+ '[' -z '' ']'
+ cat_config /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2328/e2e-tests/pitr-physical-backup-source/conf/some-name-rs0.yml
+ kubectl_bin apply -f -
+ cat /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2328/e2e-tests/pitr-physical-backup-source/conf/some-name-rs0.yml
++ mktemp
+ yq eval '(.spec | select(.image == null)).image = "docker.io/perconalab/percona-server-mongodb-operator:main-mongod8.0"'
+ yq eval '(.spec | select(has("pmm"))).pmm.image = "docker.io/percona/pmm-client:2.44.1-1"'
+ yq eval '(.spec | select(has("initImage"))).initImage = "docker.io/perconalab/percona-server-mongodb-operator:PR-2328-37c432905"'
+ /usr/sbin/sed -e s/NAME_SPACE/pitr-physical-backup-source-27258/g
+ local LAST_OUT=/tmp/tmp.ylSa55H3PG
+ yq eval '(.spec | select(has("backup"))).backup.image = "docker.io/perconalab/percona-server-mongodb-operator:main-backup"'
+ yq eval '.spec.upgradeOptions.apply="Never"'
++ mktemp
+ local LAST_ERR=/tmp/tmp.lGP2aJVsN7
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl apply -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.ylSa55H3PG
perconaservermongodb.psmdb.percona.com/some-name created
+ cat /tmp/tmp.lGP2aJVsN7
+ rm /tmp/tmp.ylSa55H3PG /tmp/tmp.lGP2aJVsN7
+ return 0
+ desc 'check if all 3 Pods started'
+ set +o xtrace


-----------------------------------------------------------------------------------
check if all 3 Pods started
-----------------------------------------------------------------------------------

+ wait_for_running some-name-rs0 3
+ local name=some-name-rs0
+ let last_pod=2
+ local check_cluster_readyness=true
+ set_debug
+ [[ 1 == 1 ]]
+ set -o xtrace
+ local rs_name=rs0
+ local cluster_name=some-name
++ seq 0 2
+ for i in $(seq 0 $last_pod)
+ [[ 0 -eq 2 ]]
+ wait_pod some-name-rs0-0
+ local pod=some-name-rs0-0
+ set +o xtrace
waiting for pod/some-name-rs0-0 to be ready........OK
+ for i in $(seq 0 $last_pod)
+ [[ 1 -eq 2 ]]
+ wait_pod some-name-rs0-1
+ local pod=some-name-rs0-1
+ set +o xtrace
waiting for pod/some-name-rs0-1 to be ready......OK
+ for i in $(seq 0 $last_pod)
+ [[ 2 -eq 2 ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].arbiter.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.EdGfqmMG0k
+++ mktemp
++ local LAST_ERR=/tmp/tmp.bAeZhhoGRy
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].arbiter.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.EdGfqmMG0k
++ cat /tmp/tmp.bAeZhhoGRy
++ rm /tmp/tmp.EdGfqmMG0k /tmp/tmp.bAeZhhoGRy
++ return 0
+ [[ '' == true ]]
+ wait_pod some-name-rs0-2
+ local pod=some-name-rs0-2
+ set +o xtrace
waiting for pod/some-name-rs0-2 to be ready......OK
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].nonvoting.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.DPjHGZIZ5L
+++ mktemp
++ local LAST_ERR=/tmp/tmp.B3A76FhkNU
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].nonvoting.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.DPjHGZIZ5L
++ cat /tmp/tmp.B3A76FhkNU
++ rm /tmp/tmp.DPjHGZIZ5L /tmp/tmp.B3A76FhkNU
++ return 0
+ [[ '' == true ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].hidden.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.lDfv3flOTh
+++ mktemp
++ local LAST_ERR=/tmp/tmp.8x30z7Ps8L
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].hidden.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.lDfv3flOTh
++ cat /tmp/tmp.8x30z7Ps8L
++ rm /tmp/tmp.lDfv3flOTh /tmp/tmp.8x30z7Ps8L
++ return 0
+ [[ '' == true ]]
+ sleep 10
+ [[ true == true ]]
+ set +x
Waiting for cluster readyness...........................................................
+ wait_for_running some-name-cfg 3 false
+ local name=some-name-cfg
+ let last_pod=2
+ local check_cluster_readyness=false
+ set_debug
+ [[ 1 == 1 ]]
+ set -o xtrace
+ local rs_name=cfg
+ local cluster_name=some-name
++ seq 0 2
+ for i in $(seq 0 $last_pod)
+ [[ 0 -eq 2 ]]
+ wait_pod some-name-cfg-0
+ local pod=some-name-cfg-0
+ set +o xtrace
waiting for pod/some-name-cfg-0 to be ready.OK
+ for i in $(seq 0 $last_pod)
+ [[ 1 -eq 2 ]]
+ wait_pod some-name-cfg-1
+ local pod=some-name-cfg-1
+ set +o xtrace
waiting for pod/some-name-cfg-1 to be ready.OK
+ for i in $(seq 0 $last_pod)
+ [[ 2 -eq 2 ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].arbiter.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.i0Y9Ks5soC
+++ mktemp
++ local LAST_ERR=/tmp/tmp.ElzQSBvSF8
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].arbiter.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.i0Y9Ks5soC
++ cat /tmp/tmp.ElzQSBvSF8
++ rm /tmp/tmp.i0Y9Ks5soC /tmp/tmp.ElzQSBvSF8
++ return 0
+ [[ '' == true ]]
+ wait_pod some-name-cfg-2
+ local pod=some-name-cfg-2
+ set +o xtrace
waiting for pod/some-name-cfg-2 to be ready.OK
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].nonvoting.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.25f3Uyfw3p
+++ mktemp
++ local LAST_ERR=/tmp/tmp.OGGugPn8ap
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].nonvoting.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.25f3Uyfw3p
++ cat /tmp/tmp.OGGugPn8ap
++ rm /tmp/tmp.25f3Uyfw3p /tmp/tmp.OGGugPn8ap
++ return 0
+ [[ '' == true ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].hidden.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.691ZjVLpsM
+++ mktemp
++ local LAST_ERR=/tmp/tmp.DS3VdChBNX
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].hidden.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.691ZjVLpsM
++ cat /tmp/tmp.DS3VdChBNX
++ rm /tmp/tmp.691ZjVLpsM /tmp/tmp.DS3VdChBNX
++ return 0
+ [[ '' == true ]]
+ sleep 10
+ [[ false == true ]]
+ wait_for_running some-name-rs1 3
+ local name=some-name-rs1
+ let last_pod=2
+ local check_cluster_readyness=true
+ set_debug
+ [[ 1 == 1 ]]
+ set -o xtrace
+ local rs_name=rs1
+ local cluster_name=some-name
++ seq 0 2
+ for i in $(seq 0 $last_pod)
+ [[ 0 -eq 2 ]]
+ wait_pod some-name-rs1-0
+ local pod=some-name-rs1-0
+ set +o xtrace
waiting for pod/some-name-rs1-0 to be ready.OK
+ for i in $(seq 0 $last_pod)
+ [[ 1 -eq 2 ]]
+ wait_pod some-name-rs1-1
+ local pod=some-name-rs1-1
+ set +o xtrace
waiting for pod/some-name-rs1-1 to be ready.OK
+ for i in $(seq 0 $last_pod)
+ [[ 2 -eq 2 ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs1")].arbiter.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.5LKwi0d6CJ
+++ mktemp
++ local LAST_ERR=/tmp/tmp.wtwmLbOLYM
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs1")].arbiter.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.5LKwi0d6CJ
++ cat /tmp/tmp.wtwmLbOLYM
++ rm /tmp/tmp.5LKwi0d6CJ /tmp/tmp.wtwmLbOLYM
++ return 0
+ [[ '' == true ]]
+ wait_pod some-name-rs1-2
+ local pod=some-name-rs1-2
+ set +o xtrace
waiting for pod/some-name-rs1-2 to be ready.OK
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs1")].nonvoting.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.edMH62L7eU
+++ mktemp
++ local LAST_ERR=/tmp/tmp.senwFklNJ5
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs1")].nonvoting.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.edMH62L7eU
++ cat /tmp/tmp.senwFklNJ5
++ rm /tmp/tmp.edMH62L7eU /tmp/tmp.senwFklNJ5
++ return 0
+ [[ '' == true ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs1")].hidden.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.KOEQ2iHtW0
+++ mktemp
++ local LAST_ERR=/tmp/tmp.OcZP8RcQdB
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs1")].hidden.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.KOEQ2iHtW0
++ cat /tmp/tmp.OcZP8RcQdB
++ rm /tmp/tmp.KOEQ2iHtW0 /tmp/tmp.OcZP8RcQdB
++ return 0
+ [[ '' == true ]]
+ sleep 10
+ [[ true == true ]]
+ set +x
Waiting for cluster readyness
+ wait_for_running some-name-rs2 3
+ local name=some-name-rs2
+ let last_pod=2
+ local check_cluster_readyness=true
+ set_debug
+ [[ 1 == 1 ]]
+ set -o xtrace
+ local rs_name=rs2
+ local cluster_name=some-name
++ seq 0 2
+ for i in $(seq 0 $last_pod)
+ [[ 0 -eq 2 ]]
+ wait_pod some-name-rs2-0
+ local pod=some-name-rs2-0
+ set +o xtrace
waiting for pod/some-name-rs2-0 to be ready.OK
+ for i in $(seq 0 $last_pod)
+ [[ 1 -eq 2 ]]
+ wait_pod some-name-rs2-1
+ local pod=some-name-rs2-1
+ set +o xtrace
waiting for pod/some-name-rs2-1 to be ready.OK
+ for i in $(seq 0 $last_pod)
+ [[ 2 -eq 2 ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs2")].arbiter.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.ym916RH7oJ
+++ mktemp
++ local LAST_ERR=/tmp/tmp.pGlFIPiqqG
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs2")].arbiter.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.ym916RH7oJ
++ cat /tmp/tmp.pGlFIPiqqG
++ rm /tmp/tmp.ym916RH7oJ /tmp/tmp.pGlFIPiqqG
++ return 0
+ [[ '' == true ]]
+ wait_pod some-name-rs2-2
+ local pod=some-name-rs2-2
+ set +o xtrace
waiting for pod/some-name-rs2-2 to be ready.OK
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs2")].nonvoting.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.FPHRwkFNxR
+++ mktemp
++ local LAST_ERR=/tmp/tmp.9MTMzDFNvb
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs2")].nonvoting.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.FPHRwkFNxR
++ cat /tmp/tmp.9MTMzDFNvb
++ rm /tmp/tmp.FPHRwkFNxR /tmp/tmp.9MTMzDFNvb
++ return 0
+ [[ '' == true ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs2")].hidden.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.mZbPvxotaD
+++ mktemp
++ local LAST_ERR=/tmp/tmp.Wo0AWcJL24
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs2")].hidden.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.mZbPvxotaD
++ cat /tmp/tmp.Wo0AWcJL24
++ rm /tmp/tmp.mZbPvxotaD /tmp/tmp.Wo0AWcJL24
++ return 0
+ [[ '' == true ]]
+ sleep 10
+ [[ true == true ]]
+ set +x
Waiting for cluster readyness
+ sleep 10
+ write_initial_data
+ desc 'create user myApp'
+ set +o xtrace


-----------------------------------------------------------------------------------
create user myApp
-----------------------------------------------------------------------------------

+ run_mongos 'db.createUser({user:"myApp",pwd:"myPass",roles:[{db:"myApp",role:"readWrite"}]})' userAdmin:userAdmin123456@some-name-mongos.pitr-physical-backup-source-27258
+ local 'command=db.createUser({user:"myApp",pwd:"myPass",roles:[{db:"myApp",role:"readWrite"}]})'
+ local uri=userAdmin:userAdmin123456@some-name-mongos.pitr-physical-backup-source-27258
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local port=27017
+ local mongo_bin=mongo
++ echo .svc.cluster.local
++ awk -F: '{print $2}'
+ suffix_port=
+ [[ -z '' ]]
+ suffix=.svc.cluster.local:27017
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.nQf3eHEJOc
+++ mktemp
++ local LAST_ERR=/tmp/tmp.sTfFJk9EmQ
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.nQf3eHEJOc
++ cat /tmp/tmp.sTfFJk9EmQ
++ rm /tmp/tmp.nQf3eHEJOc /tmp/tmp.sTfFJk9EmQ
++ return 0
+ local client_container=psmdb-client-bb8b97679-c5ptc
+ kubectl_bin exec psmdb-client-bb8b97679-c5ptc -- bash -c 'printf '\''db.createUser({user:"myApp",pwd:"myPass",roles:[{db:"myApp",role:"readWrite"}]})\n'\'' | mongo mongodb://userAdmin:userAdmin123456@some-name-mongos.pitr-physical-backup-source-27258.svc.cluster.local:27017/admin '
++ mktemp
+ local LAST_OUT=/tmp/tmp.xmnXihPUNI
++ mktemp
+ local LAST_ERR=/tmp/tmp.wcdSuiH38W
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-bb8b97679-c5ptc -- bash -c 'printf '\''db.createUser({user:"myApp",pwd:"myPass",roles:[{db:"myApp",role:"readWrite"}]})\n'\'' | mongo mongodb://userAdmin:userAdmin123456@some-name-mongos.pitr-physical-backup-source-27258.svc.cluster.local:27017/admin '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.xmnXihPUNI
Percona Server for MongoDB shell version v4.4.29-28
connecting to: mongodb://some-name-mongos.pitr-physical-backup-source-27258.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb
Implicit session: session { "id" : UUID("969c75c6-4cb2-4cb9-a065-9310e6a122d3") }
Percona Server for MongoDB server version: v8.0.21-9
WARNING: shell and server versions do not match
Successfully added user: {
	"user" : "myApp",
	"roles" : [
		{
			"db" : "myApp",
			"role" : "readWrite"
		}
	]
}
bye
+ cat /tmp/tmp.wcdSuiH38W
+ rm /tmp/tmp.xmnXihPUNI /tmp/tmp.wcdSuiH38W
+ return 0
+ sleep 2
+ write_document
+ local cmp_postfix=
+ local sleep_value=0
+ log 'write initial data, read from all'
+ set +o xtrace
[2026-05-21T09:36:45+0000] write initial data, read from all
+ run_mongos 'use myApp\n db.test.insert({ x: 100500 })' myApp:myPass@some-name-mongos.pitr-physical-backup-source-27258
+ local 'command=use myApp\n db.test.insert({ x: 100500 })'
+ local uri=myApp:myPass@some-name-mongos.pitr-physical-backup-source-27258
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local port=27017
+ local mongo_bin=mongo
++ echo .svc.cluster.local
++ awk -F: '{print $2}'
+ suffix_port=
+ [[ -z '' ]]
+ suffix=.svc.cluster.local:27017
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.Scr3UpglOs
+++ mktemp
++ local LAST_ERR=/tmp/tmp.JEwzkymOzs
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.Scr3UpglOs
++ cat /tmp/tmp.JEwzkymOzs
++ rm /tmp/tmp.Scr3UpglOs /tmp/tmp.JEwzkymOzs
++ return 0
+ local client_container=psmdb-client-bb8b97679-c5ptc
+ kubectl_bin exec psmdb-client-bb8b97679-c5ptc -- bash -c 'printf '\''use myApp\n db.test.insert({ x: 100500 })\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.pitr-physical-backup-source-27258.svc.cluster.local:27017/admin '
++ mktemp
+ local LAST_OUT=/tmp/tmp.UZkclqDHLq
++ mktemp
+ local LAST_ERR=/tmp/tmp.JJpKLobGlW
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-bb8b97679-c5ptc -- bash -c 'printf '\''use myApp\n db.test.insert({ x: 100500 })\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.pitr-physical-backup-source-27258.svc.cluster.local:27017/admin '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.UZkclqDHLq
Percona Server for MongoDB shell version v4.4.29-28
connecting to: mongodb://some-name-mongos.pitr-physical-backup-source-27258.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb
Implicit session: session { "id" : UUID("56485260-6faf-4e10-909a-659a4274fc20") }
Percona Server for MongoDB server version: v8.0.21-9
WARNING: shell and server versions do not match
switched to db myApp
WriteResult({ "nInserted" : 1 })
bye
+ cat /tmp/tmp.JJpKLobGlW
+ rm /tmp/tmp.UZkclqDHLq /tmp/tmp.JJpKLobGlW
+ return 0
+ sleep 0
+ compare_mongos_cmd find myApp:myPass@some-name-mongos.pitr-physical-backup-source-27258
+ local command=find
+ local uri=myApp:myPass@some-name-mongos.pitr-physical-backup-source-27258
+ local postfix=
+ local suffix=
+ local database=myApp
+ local collection=test
+ local port=27017
+ local tls=false
+ [[ false == true ]]
+ mongos_command=run_mongos
+ log 'running db.test.command() in myApp'
+ set +o xtrace
[2026-05-21T09:36:48+0000] running db.test.command() in myApp
+ run_mongos 'use myApp\n db.test.find()' myApp:myPass@some-name-mongos.pitr-physical-backup-source-27258 mongodb '' '' 27017
+ local 'command=use myApp\n db.test.find()'
+ local uri=myApp:myPass@some-name-mongos.pitr-physical-backup-source-27258
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local port=27017
+ local mongo_bin=mongo
+ grep -E -v 'I NETWORK|W NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|Started a new thread for the timer service'
+ /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
++ echo .svc.cluster.local
++ awk -F: '{print $2}'
+ suffix_port=
+ [[ -z '' ]]
+ suffix=.svc.cluster.local:27017
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.X97qrVv2UR
+++ mktemp
++ local LAST_ERR=/tmp/tmp.cZnpH8Y2XK
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.X97qrVv2UR
++ cat /tmp/tmp.cZnpH8Y2XK
++ rm /tmp/tmp.X97qrVv2UR /tmp/tmp.cZnpH8Y2XK
++ return 0
+ local client_container=psmdb-client-bb8b97679-c5ptc
+ kubectl_bin exec psmdb-client-bb8b97679-c5ptc -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.pitr-physical-backup-source-27258.svc.cluster.local:27017/admin '
++ mktemp
+ local LAST_OUT=/tmp/tmp.DIFT73063v
++ mktemp
+ local LAST_ERR=/tmp/tmp.IpQBptPDXK
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-bb8b97679-c5ptc -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.pitr-physical-backup-source-27258.svc.cluster.local:27017/admin '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.DIFT73063v
+ cat /tmp/tmp.IpQBptPDXK
+ rm /tmp/tmp.DIFT73063v /tmp/tmp.IpQBptPDXK
+ return 0
+ [[ 0 -eq 0 ]]
+ diff /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2328/e2e-tests/pitr-physical-backup-source/compare/find.json /tmp/tmp.DTCahJpmH8/find
+ desc 'restore pitr type latest using backupSource'
+ set +o xtrace


-----------------------------------------------------------------------------------
restore pitr type latest using backupSource
-----------------------------------------------------------------------------------

+ write_document -2nd
+ local cmp_postfix=-2nd
+ local sleep_value=0
+ log 'write initial data, read from all'
+ set +o xtrace
[2026-05-21T09:36:50+0000] write initial data, read from all
+ run_mongos 'use myApp\n db.test.insert({ x: 100500 })' myApp:myPass@some-name-mongos.pitr-physical-backup-source-27258
+ local 'command=use myApp\n db.test.insert({ x: 100500 })'
+ local uri=myApp:myPass@some-name-mongos.pitr-physical-backup-source-27258
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local port=27017
+ local mongo_bin=mongo
++ echo .svc.cluster.local
++ awk -F: '{print $2}'
+ suffix_port=
+ [[ -z '' ]]
+ suffix=.svc.cluster.local:27017
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.t6dtHOHJ7M
+++ mktemp
++ local LAST_ERR=/tmp/tmp.fk5P0DG9Fz
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.t6dtHOHJ7M
++ cat /tmp/tmp.fk5P0DG9Fz
++ rm /tmp/tmp.t6dtHOHJ7M /tmp/tmp.fk5P0DG9Fz
++ return 0
+ local client_container=psmdb-client-bb8b97679-c5ptc
+ kubectl_bin exec psmdb-client-bb8b97679-c5ptc -- bash -c 'printf '\''use myApp\n db.test.insert({ x: 100500 })\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.pitr-physical-backup-source-27258.svc.cluster.local:27017/admin '
++ mktemp
+ local LAST_OUT=/tmp/tmp.n10WPMQ5gI
++ mktemp
+ local LAST_ERR=/tmp/tmp.AothmvOBXN
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-bb8b97679-c5ptc -- bash -c 'printf '\''use myApp\n db.test.insert({ x: 100500 })\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.pitr-physical-backup-source-27258.svc.cluster.local:27017/admin '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.n10WPMQ5gI
Percona Server for MongoDB shell version v4.4.29-28
connecting to: mongodb://some-name-mongos.pitr-physical-backup-source-27258.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb
Implicit session: session { "id" : UUID("6451182e-73fd-4e98-9a42-3e62c783561f") }
Percona Server for MongoDB server version: v8.0.21-9
WARNING: shell and server versions do not match
switched to db myApp
WriteResult({ "nInserted" : 1 })
bye
+ cat /tmp/tmp.AothmvOBXN
+ rm /tmp/tmp.n10WPMQ5gI /tmp/tmp.AothmvOBXN
+ return 0
+ sleep 0
+ compare_mongos_cmd find myApp:myPass@some-name-mongos.pitr-physical-backup-source-27258 -2nd
+ local command=find
+ local uri=myApp:myPass@some-name-mongos.pitr-physical-backup-source-27258
+ local postfix=-2nd
+ local suffix=
+ local database=myApp
+ local collection=test
+ local port=27017
+ local tls=false
+ [[ false == true ]]
+ mongos_command=run_mongos
+ log 'running db.test.command() in myApp'
+ set +o xtrace
[2026-05-21T09:36:52+0000] running db.test.command() in myApp
+ run_mongos 'use myApp\n db.test.find()' myApp:myPass@some-name-mongos.pitr-physical-backup-source-27258 mongodb '' '' 27017
+ local 'command=use myApp\n db.test.find()'
+ local uri=myApp:myPass@some-name-mongos.pitr-physical-backup-source-27258
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local port=27017
+ local mongo_bin=mongo
+ grep -E -v 'I NETWORK|W NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|Started a new thread for the timer service'
+ /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
++ echo .svc.cluster.local
++ awk -F: '{print $2}'
+ suffix_port=
+ [[ -z '' ]]
+ suffix=.svc.cluster.local:27017
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.2j3Ub4R0Zz
+++ mktemp
++ local LAST_ERR=/tmp/tmp.dD1xyvhU3I
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.2j3Ub4R0Zz
++ cat /tmp/tmp.dD1xyvhU3I
++ rm /tmp/tmp.2j3Ub4R0Zz /tmp/tmp.dD1xyvhU3I
++ return 0
+ local client_container=psmdb-client-bb8b97679-c5ptc
+ kubectl_bin exec psmdb-client-bb8b97679-c5ptc -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.pitr-physical-backup-source-27258.svc.cluster.local:27017/admin '
++ mktemp
+ local LAST_OUT=/tmp/tmp.1G7zEgkHJz
++ mktemp
+ local LAST_ERR=/tmp/tmp.LxzmUOv9B1
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-bb8b97679-c5ptc -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.pitr-physical-backup-source-27258.svc.cluster.local:27017/admin '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.1G7zEgkHJz
+ cat /tmp/tmp.LxzmUOv9B1
+ rm /tmp/tmp.1G7zEgkHJz /tmp/tmp.LxzmUOv9B1
+ return 0
+ [[ 0 -eq 0 ]]
+ diff /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2328/e2e-tests/pitr-physical-backup-source/compare/find-2nd.json /tmp/tmp.DTCahJpmH8/find-2nd
+ run_backup backup-minio 2 physical
+ local name=backup-minio
+ local idx=2
+ local type=physical
+ desc 'run backup backup-minio-2'
+ set +o xtrace


-----------------------------------------------------------------------------------
run backup backup-minio-2
-----------------------------------------------------------------------------------

+ cat /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2328/e2e-tests/pitr-physical-backup-source/conf/backup-minio.yml
+ /usr/sbin/sed -e 's/name:/name: backup-minio-2/'
+ /usr/sbin/sed -e 's/type:/type: physical/'
+ kubectl_bin apply -f -
++ mktemp
+ local LAST_OUT=/tmp/tmp.ARKZRScLw5
++ mktemp
+ local LAST_ERR=/tmp/tmp.PGwELcZ6Im
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl apply -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.ARKZRScLw5
perconaservermongodbbackup.psmdb.percona.com/backup-minio-2 created
+ cat /tmp/tmp.PGwELcZ6Im
+ rm /tmp/tmp.ARKZRScLw5 /tmp/tmp.PGwELcZ6Im
+ return 0
+ wait_backup backup-minio-2
+ local backup_name=backup-minio-2
+ local target_state=ready
+ set +o xtrace
waiting for backup-minio-2 to reach ready state.............OK
+ compare_latest_restorable_time some-name-rs0 backup-minio-2
+ local cluster=some-name-rs0
+ local backup_name=backup-minio-2
+ local latest_restorable_time
+ local backup_time
++ get_latest_restorable_time some-name-rs0
++ local cluster=some-name-rs0
++ local first_timestamp
++ local second_timestamp
++ local retry=0
++ [[ '' != '' ]]
+++ kubectl_bin exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ jq '.backups.pitrChunks.pitrChunks | last | .range.end'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.apbwBUy2Uq
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.7PHHqG2zIP
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in $(seq 0 2)
+++ set +e
+++ kubectl exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.apbwBUy2Uq
+++ cat /tmp/tmp.7PHHqG2zIP
+++ rm /tmp/tmp.apbwBUy2Uq /tmp/tmp.7PHHqG2zIP
+++ return 0
++ first_timestamp=1779355104
++ sleep 5
++ [[ 1779355104 != '' ]]
++ [[ 1779355104 != null ]]
+++ kubectl_bin exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ jq '.backups.pitrChunks.pitrChunks | last | .range.end'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.9RP5LxNr2G
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.GDBCoNJ7oe
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in $(seq 0 2)
+++ set +e
+++ kubectl exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.9RP5LxNr2G
+++ cat /tmp/tmp.GDBCoNJ7oe
+++ rm /tmp/tmp.9RP5LxNr2G /tmp/tmp.GDBCoNJ7oe
+++ return 0
++ second_timestamp=1779355104
++ let retry+=1
++ [[ 1 -gt 30 ]]
++ [[ 1779355104 != '' ]]
++ [[ 1779355104 != null ]]
++ [[ 1779355104 == 1779355104 ]]
++ /usr/sbin/date -u -d @1779355104 +%Y-%m-%dT%H:%M:%SZ
+ latest_restorable_time=2026-05-21T09:18:24Z
++ get_latest_restorable_time_from_backup_object backup-minio-2
++ local backup_name=backup-minio-2
++ local latestRestorableTime
++ local retry=0
++ [[ '' != '' ]]
++ sleep 5
+++ kubectl_bin get psmdb-backup backup-minio-2 -o 'jsonpath={.status.latestRestorableTime}'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.r0hiKp3FXc
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.3mlATe2eh2
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in $(seq 0 2)
+++ set +e
+++ kubectl get psmdb-backup backup-minio-2 -o 'jsonpath={.status.latestRestorableTime}'
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.r0hiKp3FXc
+++ cat /tmp/tmp.3mlATe2eh2
+++ rm /tmp/tmp.r0hiKp3FXc /tmp/tmp.3mlATe2eh2
+++ return 0
++ latestRestorableTime=2026-05-21T09:18:24Z
++ let retry+=1
++ [[ 1 -gt 30 ]]
++ [[ 2026-05-21T09:18:24Z != '' ]]
++ [[ 2026-05-21T09:18:24Z != null ]]
++ echo 2026-05-21T09:18:24Z
+ backup_time=2026-05-21T09:18:24Z
+ [[ 2026-05-21T09:18:24Z != 2026\-05\-21T09\:18\:24Z ]]
+ check_recovery backup-minio-2 latest '' -3rd some-name backupSource
+ local backup_name=backup-minio-2
+ local restore_type=latest
+ local restore_date=
+ local cmp_postfix=-3rd
+ local cluster_name=some-name
+ local backupSource=backupSource
++ get_latest_oplog_chunk_ts some-name
++ local cluster=some-name
+++ kubectl_bin exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ jq '.backups.pitrChunks.pitrChunks | last | .range.end'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.sJdt9PkXCa
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.bln6sgaYvg
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in $(seq 0 2)
+++ set +e
+++ kubectl exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.sJdt9PkXCa
+++ cat /tmp/tmp.bln6sgaYvg
+++ rm /tmp/tmp.sJdt9PkXCa /tmp/tmp.bln6sgaYvg
+++ return 0
++ echo 1779355104
+ local latest_ts=1779355104
+ desc 'write more data before restore by latest'
+ set +o xtrace


-----------------------------------------------------------------------------------
write more data before restore by latest
-----------------------------------------------------------------------------------

+ run_mongos 'use myApp\n db.test.insert({ x: 100501 })' myApp:myPass@some-name-mongos.pitr-physical-backup-source-27258
+ local 'command=use myApp\n db.test.insert({ x: 100501 })'
+ local uri=myApp:myPass@some-name-mongos.pitr-physical-backup-source-27258
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local port=27017
+ local mongo_bin=mongo
++ echo .svc.cluster.local
++ awk -F: '{print $2}'
+ suffix_port=
+ [[ -z '' ]]
+ suffix=.svc.cluster.local:27017
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.SMIOXZ6Wqw
+++ mktemp
++ local LAST_ERR=/tmp/tmp.UMQrVQEk4S
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.SMIOXZ6Wqw
++ cat /tmp/tmp.UMQrVQEk4S
++ rm /tmp/tmp.SMIOXZ6Wqw /tmp/tmp.UMQrVQEk4S
++ return 0
+ local client_container=psmdb-client-bb8b97679-c5ptc
+ kubectl_bin exec psmdb-client-bb8b97679-c5ptc -- bash -c 'printf '\''use myApp\n db.test.insert({ x: 100501 })\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.pitr-physical-backup-source-27258.svc.cluster.local:27017/admin '
++ mktemp
+ local LAST_OUT=/tmp/tmp.U2piVq97PV
++ mktemp
+ local LAST_ERR=/tmp/tmp.W7aAkPwOQM
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-bb8b97679-c5ptc -- bash -c 'printf '\''use myApp\n db.test.insert({ x: 100501 })\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.pitr-physical-backup-source-27258.svc.cluster.local:27017/admin '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.U2piVq97PV
Percona Server for MongoDB shell version v4.4.29-28
connecting to: mongodb://some-name-mongos.pitr-physical-backup-source-27258.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb
Implicit session: session { "id" : UUID("151ffc1f-a5e2-4642-929c-61fcec776a81") }
Percona Server for MongoDB server version: v8.0.21-9
WARNING: shell and server versions do not match
switched to db myApp
WriteResult({ "nInserted" : 1 })
bye
+ cat /tmp/tmp.W7aAkPwOQM
+ rm /tmp/tmp.U2piVq97PV /tmp/tmp.W7aAkPwOQM
+ return 0
+ [[ -n '' ]]
+ desc 'Restoring to latest'
+ set +o xtrace


-----------------------------------------------------------------------------------
Restoring to latest
-----------------------------------------------------------------------------------

++ get_latest_oplog_chunk_ts some-name
++ local cluster=some-name
+++ kubectl_bin exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ jq '.backups.pitrChunks.pitrChunks | last | .range.end'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.jjf8ksAMgN
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.m5VeDgM9IQ
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in $(seq 0 2)
+++ set +e
+++ kubectl exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.jjf8ksAMgN
+++ cat /tmp/tmp.m5VeDgM9IQ
+++ rm /tmp/tmp.jjf8ksAMgN /tmp/tmp.m5VeDgM9IQ
+++ return 0
++ echo 1779355104
+ local current_ts=1779355104
+ retries=0
+ [[ 1779355104 -gt 1779355104 ]]
+ [[ 0 -gt 30 ]]
++ get_latest_oplog_chunk_ts some-name
++ local cluster=some-name
+++ kubectl_bin exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ jq '.backups.pitrChunks.pitrChunks | last | .range.end'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.GxGH9M8bHw
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.iRLLkc0ajI
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in $(seq 0 2)
+++ set +e
+++ kubectl exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.GxGH9M8bHw
+++ cat /tmp/tmp.iRLLkc0ajI
+++ rm /tmp/tmp.GxGH9M8bHw /tmp/tmp.iRLLkc0ajI
+++ return 0
++ echo 1779355104
+ latest_ts=1779355104
+ retries=1
++ format_date 1779355104
++ local timestamp=1779355104
+++ TZ=UTC
+++ /usr/sbin/date -d@1779355104 '+%Y-%m-%d %H:%M:%S'
++ echo 2026-05-21 09:18:24
++ format_date 1779355104
++ local timestamp=1779355104
+++ TZ=UTC
+++ /usr/sbin/date -d@1779355104 '+%Y-%m-%d %H:%M:%S'
++ echo 2026-05-21 09:18:24
+ echo 'Waiting for last oplog chunk (2026-05-21 09:18:24) to be 120 seconds older than starting chunk (2026-05-21 09:18:24)'
Waiting for last oplog chunk (2026-05-21 09:18:24) to be 120 seconds older than starting chunk (2026-05-21 09:18:24)
+ sleep 10
+ [[ 1779355104 -gt 1779355104 ]]
+ [[ 1 -gt 30 ]]
++ get_latest_oplog_chunk_ts some-name
++ local cluster=some-name
+++ kubectl_bin exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ jq '.backups.pitrChunks.pitrChunks | last | .range.end'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.6uc7yey1T0
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.xnv8t5kFww
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in $(seq 0 2)
+++ set +e
+++ kubectl exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.6uc7yey1T0
+++ cat /tmp/tmp.xnv8t5kFww
+++ rm /tmp/tmp.6uc7yey1T0 /tmp/tmp.xnv8t5kFww
+++ return 0
++ echo 1779355104
+ latest_ts=1779355104
+ retries=2
++ format_date 1779355104
++ local timestamp=1779355104
+++ TZ=UTC
+++ /usr/sbin/date -d@1779355104 '+%Y-%m-%d %H:%M:%S'
++ echo 2026-05-21 09:18:24
++ format_date 1779355104
++ local timestamp=1779355104
+++ TZ=UTC
+++ /usr/sbin/date -d@1779355104 '+%Y-%m-%d %H:%M:%S'
++ echo 2026-05-21 09:18:24
+ echo 'Waiting for last oplog chunk (2026-05-21 09:18:24) to be 120 seconds older than starting chunk (2026-05-21 09:18:24)'
Waiting for last oplog chunk (2026-05-21 09:18:24) to be 120 seconds older than starting chunk (2026-05-21 09:18:24)
+ sleep 10
+ [[ 1779355104 -gt 1779355104 ]]
+ [[ 2 -gt 30 ]]
++ get_latest_oplog_chunk_ts some-name
++ local cluster=some-name
+++ kubectl_bin exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ jq '.backups.pitrChunks.pitrChunks | last | .range.end'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.bTDPvLBSrF
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.KP0vxKkLCX
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in $(seq 0 2)
+++ set +e
+++ kubectl exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.bTDPvLBSrF
+++ cat /tmp/tmp.KP0vxKkLCX
+++ rm /tmp/tmp.bTDPvLBSrF /tmp/tmp.KP0vxKkLCX
+++ return 0
++ echo 1779355104
+ latest_ts=1779355104
+ retries=3
++ format_date 1779355104
++ local timestamp=1779355104
+++ TZ=UTC
+++ /usr/sbin/date -d@1779355104 '+%Y-%m-%d %H:%M:%S'
++ echo 2026-05-21 09:18:24
++ format_date 1779355104
++ local timestamp=1779355104
+++ TZ=UTC
+++ /usr/sbin/date -d@1779355104 '+%Y-%m-%d %H:%M:%S'
++ echo 2026-05-21 09:18:24
+ echo 'Waiting for last oplog chunk (2026-05-21 09:18:24) to be 120 seconds older than starting chunk (2026-05-21 09:18:24)'
Waiting for last oplog chunk (2026-05-21 09:18:24) to be 120 seconds older than starting chunk (2026-05-21 09:18:24)
+ sleep 10
+ [[ 1779355104 -gt 1779355104 ]]
+ [[ 3 -gt 30 ]]
++ get_latest_oplog_chunk_ts some-name
++ local cluster=some-name
+++ kubectl_bin exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ jq '.backups.pitrChunks.pitrChunks | last | .range.end'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.6T1lLAAOai
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.uBKoHq61Hf
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in $(seq 0 2)
+++ set +e
+++ kubectl exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.6T1lLAAOai
+++ cat /tmp/tmp.uBKoHq61Hf
+++ rm /tmp/tmp.6T1lLAAOai /tmp/tmp.uBKoHq61Hf
+++ return 0
++ echo 1779355104
+ latest_ts=1779355104
+ retries=4
++ format_date 1779355104
++ local timestamp=1779355104
+++ TZ=UTC
+++ /usr/sbin/date -d@1779355104 '+%Y-%m-%d %H:%M:%S'
++ echo 2026-05-21 09:18:24
++ format_date 1779355104
++ local timestamp=1779355104
+++ TZ=UTC
+++ /usr/sbin/date -d@1779355104 '+%Y-%m-%d %H:%M:%S'
++ echo 2026-05-21 09:18:24
+ echo 'Waiting for last oplog chunk (2026-05-21 09:18:24) to be 120 seconds older than starting chunk (2026-05-21 09:18:24)'
Waiting for last oplog chunk (2026-05-21 09:18:24) to be 120 seconds older than starting chunk (2026-05-21 09:18:24)
+ sleep 10
+ [[ 1779355104 -gt 1779355104 ]]
+ [[ 4 -gt 30 ]]
++ get_latest_oplog_chunk_ts some-name
++ local cluster=some-name
+++ kubectl_bin exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ jq '.backups.pitrChunks.pitrChunks | last | .range.end'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.avbqAVLOOZ
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.xLJz4hZ1cg
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in $(seq 0 2)
+++ set +e
+++ kubectl exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.avbqAVLOOZ
+++ cat /tmp/tmp.xLJz4hZ1cg
+++ rm /tmp/tmp.avbqAVLOOZ /tmp/tmp.xLJz4hZ1cg
+++ return 0
++ echo 1779355104
+ latest_ts=1779355104
+ retries=5
++ format_date 1779355104
++ local timestamp=1779355104
+++ TZ=UTC
+++ /usr/sbin/date -d@1779355104 '+%Y-%m-%d %H:%M:%S'
++ echo 2026-05-21 09:18:24
++ format_date 1779355104
++ local timestamp=1779355104
+++ TZ=UTC
+++ /usr/sbin/date -d@1779355104 '+%Y-%m-%d %H:%M:%S'
++ echo 2026-05-21 09:18:24
+ echo 'Waiting for last oplog chunk (2026-05-21 09:18:24) to be 120 seconds older than starting chunk (2026-05-21 09:18:24)'
Waiting for last oplog chunk (2026-05-21 09:18:24) to be 120 seconds older than starting chunk (2026-05-21 09:18:24)
+ sleep 10
+ [[ 1779355104 -gt 1779355104 ]]
+ [[ 5 -gt 30 ]]
++ get_latest_oplog_chunk_ts some-name
++ local cluster=some-name
+++ kubectl_bin exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ jq '.backups.pitrChunks.pitrChunks | last | .range.end'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.5VB6hRL1TQ
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.6LoISPdlpV
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in $(seq 0 2)
+++ set +e
+++ kubectl exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.5VB6hRL1TQ
+++ cat /tmp/tmp.6LoISPdlpV
+++ rm /tmp/tmp.5VB6hRL1TQ /tmp/tmp.6LoISPdlpV
+++ return 0
++ echo 1779355104
+ latest_ts=1779355104
+ retries=6
++ format_date 1779355104
++ local timestamp=1779355104
+++ TZ=UTC
+++ /usr/sbin/date -d@1779355104 '+%Y-%m-%d %H:%M:%S'
++ echo 2026-05-21 09:18:24
++ format_date 1779355104
++ local timestamp=1779355104
+++ TZ=UTC
+++ /usr/sbin/date -d@1779355104 '+%Y-%m-%d %H:%M:%S'
++ echo 2026-05-21 09:18:24
+ echo 'Waiting for last oplog chunk (2026-05-21 09:18:24) to be 120 seconds older than starting chunk (2026-05-21 09:18:24)'
Waiting for last oplog chunk (2026-05-21 09:18:24) to be 120 seconds older than starting chunk (2026-05-21 09:18:24)
+ sleep 10
+ [[ 1779355104 -gt 1779355104 ]]
+ [[ 6 -gt 30 ]]
++ get_latest_oplog_chunk_ts some-name
++ local cluster=some-name
+++ kubectl_bin exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ jq '.backups.pitrChunks.pitrChunks | last | .range.end'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.NukR6SdWbo
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.cEKmCuTKeU
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in $(seq 0 2)
+++ set +e
+++ kubectl exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.NukR6SdWbo
+++ cat /tmp/tmp.cEKmCuTKeU
+++ rm /tmp/tmp.NukR6SdWbo /tmp/tmp.cEKmCuTKeU
+++ return 0
++ echo 1779355104
+ latest_ts=1779355104
+ retries=7
++ format_date 1779355104
++ local timestamp=1779355104
+++ TZ=UTC
+++ /usr/sbin/date -d@1779355104 '+%Y-%m-%d %H:%M:%S'
++ echo 2026-05-21 09:18:24
++ format_date 1779355104
++ local timestamp=1779355104
+++ TZ=UTC
+++ /usr/sbin/date -d@1779355104 '+%Y-%m-%d %H:%M:%S'
++ echo 2026-05-21 09:18:24
+ echo 'Waiting for last oplog chunk (2026-05-21 09:18:24) to be 120 seconds older than starting chunk (2026-05-21 09:18:24)'
Waiting for last oplog chunk (2026-05-21 09:18:24) to be 120 seconds older than starting chunk (2026-05-21 09:18:24)
+ sleep 10
+ [[ 1779355104 -gt 1779355104 ]]
+ [[ 7 -gt 30 ]]
++ get_latest_oplog_chunk_ts some-name
++ local cluster=some-name
+++ kubectl_bin exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ jq '.backups.pitrChunks.pitrChunks | last | .range.end'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.tqSigJ97lt
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.383XjqpZUx
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in $(seq 0 2)
+++ set +e
+++ kubectl exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.tqSigJ97lt
+++ cat /tmp/tmp.383XjqpZUx
+++ rm /tmp/tmp.tqSigJ97lt /tmp/tmp.383XjqpZUx
+++ return 0
++ echo 1779355104
+ latest_ts=1779355104
+ retries=8
++ format_date 1779355104
++ local timestamp=1779355104
+++ TZ=UTC
+++ /usr/sbin/date -d@1779355104 '+%Y-%m-%d %H:%M:%S'
++ echo 2026-05-21 09:18:24
++ format_date 1779355104
++ local timestamp=1779355104
+++ TZ=UTC
+++ /usr/sbin/date -d@1779355104 '+%Y-%m-%d %H:%M:%S'
++ echo 2026-05-21 09:18:24
+ echo 'Waiting for last oplog chunk (2026-05-21 09:18:24) to be 120 seconds older than starting chunk (2026-05-21 09:18:24)'
Waiting for last oplog chunk (2026-05-21 09:18:24) to be 120 seconds older than starting chunk (2026-05-21 09:18:24)
+ sleep 10
+ [[ 1779355104 -gt 1779355104 ]]
+ [[ 8 -gt 30 ]]
++ get_latest_oplog_chunk_ts some-name
++ local cluster=some-name
+++ kubectl_bin exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ jq '.backups.pitrChunks.pitrChunks | last | .range.end'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.a6DwK6cikp
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.HFffzusQd3
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in $(seq 0 2)
+++ set +e
+++ kubectl exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.a6DwK6cikp
+++ cat /tmp/tmp.HFffzusQd3
+++ rm /tmp/tmp.a6DwK6cikp /tmp/tmp.HFffzusQd3
+++ return 0
++ echo 1779355104
+ latest_ts=1779355104
+ retries=9
++ format_date 1779355104
++ local timestamp=1779355104
+++ TZ=UTC
+++ /usr/sbin/date -d@1779355104 '+%Y-%m-%d %H:%M:%S'
++ echo 2026-05-21 09:18:24
++ format_date 1779355104
++ local timestamp=1779355104
+++ TZ=UTC
+++ /usr/sbin/date -d@1779355104 '+%Y-%m-%d %H:%M:%S'
++ echo 2026-05-21 09:18:24
+ echo 'Waiting for last oplog chunk (2026-05-21 09:18:24) to be 120 seconds older than starting chunk (2026-05-21 09:18:24)'
Waiting for last oplog chunk (2026-05-21 09:18:24) to be 120 seconds older than starting chunk (2026-05-21 09:18:24)
+ sleep 10
+ [[ 1779355104 -gt 1779355104 ]]
+ [[ 9 -gt 30 ]]
++ get_latest_oplog_chunk_ts some-name
++ local cluster=some-name
+++ kubectl_bin exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ jq '.backups.pitrChunks.pitrChunks | last | .range.end'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.DcJecGH96z
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.YgHjEOdLxh
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in $(seq 0 2)
+++ set +e
+++ kubectl exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.DcJecGH96z
+++ cat /tmp/tmp.YgHjEOdLxh
+++ rm /tmp/tmp.DcJecGH96z /tmp/tmp.YgHjEOdLxh
+++ return 0
++ echo 1779355104
+ latest_ts=1779355104
+ retries=10
++ format_date 1779355104
++ local timestamp=1779355104
+++ TZ=UTC
+++ /usr/sbin/date -d@1779355104 '+%Y-%m-%d %H:%M:%S'
++ echo 2026-05-21 09:18:24
++ format_date 1779355104
++ local timestamp=1779355104
+++ TZ=UTC
+++ /usr/sbin/date -d@1779355104 '+%Y-%m-%d %H:%M:%S'
++ echo 2026-05-21 09:18:24
+ echo 'Waiting for last oplog chunk (2026-05-21 09:18:24) to be 120 seconds older than starting chunk (2026-05-21 09:18:24)'
Waiting for last oplog chunk (2026-05-21 09:18:24) to be 120 seconds older than starting chunk (2026-05-21 09:18:24)
+ sleep 10
+ [[ 1779355104 -gt 1779355104 ]]
+ [[ 10 -gt 30 ]]
++ get_latest_oplog_chunk_ts some-name
++ local cluster=some-name
+++ kubectl_bin exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ jq '.backups.pitrChunks.pitrChunks | last | .range.end'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.b3fb8o8WPt
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.1xRjKRJY5x
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in $(seq 0 2)
+++ set +e
+++ kubectl exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.b3fb8o8WPt
+++ cat /tmp/tmp.1xRjKRJY5x
+++ rm /tmp/tmp.b3fb8o8WPt /tmp/tmp.1xRjKRJY5x
+++ return 0
++ echo 1779355104
+ latest_ts=1779355104
+ retries=11
++ format_date 1779355104
++ local timestamp=1779355104
+++ TZ=UTC
+++ /usr/sbin/date -d@1779355104 '+%Y-%m-%d %H:%M:%S'
++ echo 2026-05-21 09:18:24
++ format_date 1779355104
++ local timestamp=1779355104
+++ TZ=UTC
+++ /usr/sbin/date -d@1779355104 '+%Y-%m-%d %H:%M:%S'
++ echo 2026-05-21 09:18:24
+ echo 'Waiting for last oplog chunk (2026-05-21 09:18:24) to be 120 seconds older than starting chunk (2026-05-21 09:18:24)'
Waiting for last oplog chunk (2026-05-21 09:18:24) to be 120 seconds older than starting chunk (2026-05-21 09:18:24)
+ sleep 10
+ [[ 1779355104 -gt 1779355104 ]]
+ [[ 11 -gt 30 ]]
++ get_latest_oplog_chunk_ts some-name
++ local cluster=some-name
+++ kubectl_bin exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ jq '.backups.pitrChunks.pitrChunks | last | .range.end'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.FVV2Lcb0uB
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.fRp2wHs3eB
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in $(seq 0 2)
+++ set +e
+++ kubectl exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.FVV2Lcb0uB
+++ cat /tmp/tmp.fRp2wHs3eB
+++ rm /tmp/tmp.FVV2Lcb0uB /tmp/tmp.fRp2wHs3eB
+++ return 0
++ echo 1779356371
+ latest_ts=1779356371
+ retries=12
++ format_date 1779356371
++ local timestamp=1779356371
+++ TZ=UTC
+++ /usr/sbin/date -d@1779356371 '+%Y-%m-%d %H:%M:%S'
++ echo 2026-05-21 09:39:31
++ format_date 1779355104
++ local timestamp=1779355104
+++ TZ=UTC
+++ /usr/sbin/date -d@1779355104 '+%Y-%m-%d %H:%M:%S'
++ echo 2026-05-21 09:18:24
+ echo 'Waiting for last oplog chunk (2026-05-21 09:39:31) to be 120 seconds older than starting chunk (2026-05-21 09:18:24)'
Waiting for last oplog chunk (2026-05-21 09:39:31) to be 120 seconds older than starting chunk (2026-05-21 09:18:24)
+ sleep 10
+ [[ 1779356371 -gt 1779355104 ]]
+ '[' -z backupSource ']'
+ desc 'check restore by latest backupSource'
+ set +o xtrace


-----------------------------------------------------------------------------------
check restore by latest backupSource
-----------------------------------------------------------------------------------

++ get_backup_dest backup-minio-2
++ local backup_name=backup-minio-2
++ kubectl_bin get psmdb-backup backup-minio-2 -o 'jsonpath={.status.destination}'
++ sed -e 's/.json$//'
++ sed 's|s3://||'
++ sed 's|azure://||'
++ sed 's|gs://||'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.dGGyExuTG1
+++ mktemp
++ local LAST_ERR=/tmp/tmp.IhoQCfkz3P
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb-backup backup-minio-2 -o 'jsonpath={.status.destination}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.dGGyExuTG1
++ cat /tmp/tmp.IhoQCfkz3P
++ rm /tmp/tmp.dGGyExuTG1 /tmp/tmp.IhoQCfkz3P
++ return 0
+ backup_dest=operator-testing/2026-05-21T09:36:57Z
+ cat /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2328/e2e-tests/pitr-physical-backup-source/conf/restore.yml
+ /usr/sbin/sed -e 's/name:/name: restore-backup-minio-2/'
+ /usr/sbin/sed -e /backupName/d
+ '[' -z '' ']'
+ /usr/sbin/sed -e /date:/d
+ '[' -n '' ']'
+ yq
+ /usr/sbin/sed -e 's/pitrType:/type: latest/'
+ kubectl_bin apply -f -
++ mktemp
+ /usr/sbin/sed -e 's|DESTINATION|operator-testing/2026-05-21T09:36:57Z|'
+ local LAST_OUT=/tmp/tmp.jzb19KGioq
++ mktemp
++ get_bucket_name backup-minio-2
++ local backup_name=backup-minio-2
++ kubectl_bin get psmdb-backup backup-minio-2 -o 'jsonpath={.status.s3.bucket}'
+++ mktemp
+ local LAST_ERR=/tmp/tmp.stoEOGZq0d
+ local exit_status=0
+ local timeout=4
++ seq 0 2
++ local LAST_OUT=/tmp/tmp.YkQ2HgssMC
+++ mktemp
+ for i in $(seq 0 2)
+ set +e
+ kubectl apply -f -
++ local LAST_ERR=/tmp/tmp.xhe0PsE3TM
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb-backup backup-minio-2 -o 'jsonpath={.status.s3.bucket}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.YkQ2HgssMC
++ cat /tmp/tmp.xhe0PsE3TM
++ rm /tmp/tmp.YkQ2HgssMC /tmp/tmp.xhe0PsE3TM
++ return 0
+ /usr/sbin/sed -e 's|BUCKET-NAME|operator-testing|'
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.jzb19KGioq
perconaservermongodbrestore.psmdb.percona.com/restore-backup-minio-2 created
+ cat /tmp/tmp.stoEOGZq0d
+ rm /tmp/tmp.jzb19KGioq /tmp/tmp.stoEOGZq0d
+ return 0
+ wait_restore backup-minio-2 some-name requested 0 1200
+ local backup_name=backup-minio-2
+ local cluster_name=some-name
+ local target_state=requested
+ local wait_cluster_consistency=0
+ local wait_time=1200
+ local ok_if_ready=0
+ set +o xtrace
Waiting for the psmdb-restore/restore-backup-minio-2 object to be created.OK
Waiting psmdb-restore/restore-backup-minio-2 to reach state "requested" ........OK after 7 minutes
+ [[ 0 -eq 1 ]]
+ echo

+ wait_restore backup-minio-2 some-name ready 0 1600
+ local backup_name=backup-minio-2
+ local cluster_name=some-name
+ local target_state=ready
+ local wait_cluster_consistency=0
+ local wait_time=1600
+ local ok_if_ready=0
+ set +o xtrace
Waiting for the psmdb-restore/restore-backup-minio-2 object to be created.OK
Waiting psmdb-restore/restore-backup-minio-2 to reach state "ready" ...OK after 2 minutes
+ [[ 0 -eq 1 ]]
+ echo

+ set -o xtrace
+ wait_for_running some-name-rs0 3
+ local name=some-name-rs0
+ let last_pod=2
+ local check_cluster_readyness=true
+ set_debug
+ [[ 1 == 1 ]]
+ set -o xtrace
+ local rs_name=rs0
+ local cluster_name=some-name
++ seq 0 2
+ for i in $(seq 0 $last_pod)
+ [[ 0 -eq 2 ]]
+ wait_pod some-name-rs0-0
+ local pod=some-name-rs0-0
+ set +o xtrace
waiting for pod/some-name-rs0-0 to be ready.OK
+ for i in $(seq 0 $last_pod)
+ [[ 1 -eq 2 ]]
+ wait_pod some-name-rs0-1
+ local pod=some-name-rs0-1
+ set +o xtrace
waiting for pod/some-name-rs0-1 to be ready.OK
+ for i in $(seq 0 $last_pod)
+ [[ 2 -eq 2 ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].arbiter.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.4xhtRo1gHS
+++ mktemp
++ local LAST_ERR=/tmp/tmp.D7yXBfBYKu
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].arbiter.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.4xhtRo1gHS
++ cat /tmp/tmp.D7yXBfBYKu
++ rm /tmp/tmp.4xhtRo1gHS /tmp/tmp.D7yXBfBYKu
++ return 0
+ [[ '' == true ]]
+ wait_pod some-name-rs0-2
+ local pod=some-name-rs0-2
+ set +o xtrace
waiting for pod/some-name-rs0-2 to be ready.OK
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].nonvoting.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.xF3mLVJ1BF
+++ mktemp
++ local LAST_ERR=/tmp/tmp.UKvYQrAm1p
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].nonvoting.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.xF3mLVJ1BF
++ cat /tmp/tmp.UKvYQrAm1p
++ rm /tmp/tmp.xF3mLVJ1BF /tmp/tmp.UKvYQrAm1p
++ return 0
+ [[ '' == true ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].hidden.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.BNr45JQC5B
+++ mktemp
++ local LAST_ERR=/tmp/tmp.oDASSJm5GY
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].hidden.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.BNr45JQC5B
++ cat /tmp/tmp.oDASSJm5GY
++ rm /tmp/tmp.BNr45JQC5B /tmp/tmp.oDASSJm5GY
++ return 0
+ [[ '' == true ]]
+ sleep 10
+ [[ true == true ]]
+ set +x
Waiting for cluster readyness.....................................................................
+ wait_for_running some-name-cfg 3
+ local name=some-name-cfg
+ let last_pod=2
+ local check_cluster_readyness=true
+ set_debug
+ [[ 1 == 1 ]]
+ set -o xtrace
+ local rs_name=cfg
+ local cluster_name=some-name
++ seq 0 2
+ for i in $(seq 0 $last_pod)
+ [[ 0 -eq 2 ]]
+ wait_pod some-name-cfg-0
+ local pod=some-name-cfg-0
+ set +o xtrace
waiting for pod/some-name-cfg-0 to be ready.OK
+ for i in $(seq 0 $last_pod)
+ [[ 1 -eq 2 ]]
+ wait_pod some-name-cfg-1
+ local pod=some-name-cfg-1
+ set +o xtrace
waiting for pod/some-name-cfg-1 to be ready.OK
+ for i in $(seq 0 $last_pod)
+ [[ 2 -eq 2 ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].arbiter.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.n8DtSXrG33
+++ mktemp
++ local LAST_ERR=/tmp/tmp.UGprA4e1PL
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].arbiter.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.n8DtSXrG33
++ cat /tmp/tmp.UGprA4e1PL
++ rm /tmp/tmp.n8DtSXrG33 /tmp/tmp.UGprA4e1PL
++ return 0
+ [[ '' == true ]]
+ wait_pod some-name-cfg-2
+ local pod=some-name-cfg-2
+ set +o xtrace
waiting for pod/some-name-cfg-2 to be ready.OK
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].nonvoting.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.pOlJTOofi3
+++ mktemp
++ local LAST_ERR=/tmp/tmp.jmMwvbNPtV
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].nonvoting.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.pOlJTOofi3
++ cat /tmp/tmp.jmMwvbNPtV
++ rm /tmp/tmp.pOlJTOofi3 /tmp/tmp.jmMwvbNPtV
++ return 0
+ [[ '' == true ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].hidden.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.NkHdlKv727
+++ mktemp
++ local LAST_ERR=/tmp/tmp.JKiLfVk0nY
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].hidden.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.NkHdlKv727
++ cat /tmp/tmp.JKiLfVk0nY
++ rm /tmp/tmp.NkHdlKv727 /tmp/tmp.JKiLfVk0nY
++ return 0
+ [[ '' == true ]]
+ sleep 10
+ [[ true == true ]]
+ set +x
Waiting for cluster readyness
+ wait_for_running some-name-mongos 3
+ local name=some-name-mongos
+ let last_pod=2
+ local check_cluster_readyness=true
+ set_debug
+ [[ 1 == 1 ]]
+ set -o xtrace
+ local rs_name=mongos
+ local cluster_name=some-name
++ seq 0 2
+ for i in $(seq 0 $last_pod)
+ [[ 0 -eq 2 ]]
+ wait_pod some-name-mongos-0
+ local pod=some-name-mongos-0
+ set +o xtrace
waiting for pod/some-name-mongos-0 to be ready.OK
+ for i in $(seq 0 $last_pod)
+ [[ 1 -eq 2 ]]
+ wait_pod some-name-mongos-1
+ local pod=some-name-mongos-1
+ set +o xtrace
waiting for pod/some-name-mongos-1 to be ready.OK
+ for i in $(seq 0 $last_pod)
+ [[ 2 -eq 2 ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="mongos")].arbiter.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.KzH1aWJv4x
+++ mktemp
++ local LAST_ERR=/tmp/tmp.2kScxY5FZp
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="mongos")].arbiter.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.KzH1aWJv4x
++ cat /tmp/tmp.2kScxY5FZp
++ rm /tmp/tmp.KzH1aWJv4x /tmp/tmp.2kScxY5FZp
++ return 0
+ [[ '' == true ]]
+ wait_pod some-name-mongos-2
+ local pod=some-name-mongos-2
+ set +o xtrace
waiting for pod/some-name-mongos-2 to be ready.OK
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="mongos")].nonvoting.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.f7k8n3rwkl
+++ mktemp
++ local LAST_ERR=/tmp/tmp.LyWJzMsA2X
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="mongos")].nonvoting.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.f7k8n3rwkl
++ cat /tmp/tmp.LyWJzMsA2X
++ rm /tmp/tmp.f7k8n3rwkl /tmp/tmp.LyWJzMsA2X
++ return 0
+ [[ '' == true ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="mongos")].hidden.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.mwhrTFQIID
+++ mktemp
++ local LAST_ERR=/tmp/tmp.fmEPGV0UyH
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="mongos")].hidden.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.mwhrTFQIID
++ cat /tmp/tmp.fmEPGV0UyH
++ rm /tmp/tmp.mwhrTFQIID /tmp/tmp.fmEPGV0UyH
++ return 0
+ [[ '' == true ]]
+ sleep 10
+ [[ true == true ]]
+ set +x
Waiting for cluster readyness
+ sleep 10
+ compare_mongos_cmd find myApp:myPass@some-name-mongos.pitr-physical-backup-source-27258 -3rd
+ local command=find
+ local uri=myApp:myPass@some-name-mongos.pitr-physical-backup-source-27258
+ local postfix=-3rd
+ local suffix=
+ local database=myApp
+ local collection=test
+ local port=27017
+ local tls=false
+ [[ false == true ]]
+ mongos_command=run_mongos
+ log 'running db.test.command() in myApp'
+ set +o xtrace
[2026-05-21T09:53:44+0000] running db.test.command() in myApp
+ run_mongos 'use myApp\n db.test.find()' myApp:myPass@some-name-mongos.pitr-physical-backup-source-27258 mongodb '' '' 27017
+ local 'command=use myApp\n db.test.find()'
+ local uri=myApp:myPass@some-name-mongos.pitr-physical-backup-source-27258
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local port=27017
+ local mongo_bin=mongo
+ grep -E -v 'I NETWORK|W NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|Started a new thread for the timer service'
+ /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
++ echo .svc.cluster.local
++ awk -F: '{print $2}'
+ suffix_port=
+ [[ -z '' ]]
+ suffix=.svc.cluster.local:27017
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.UGXDjDo1Sb
+++ mktemp
++ local LAST_ERR=/tmp/tmp.LasdFwYNoR
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.UGXDjDo1Sb
++ cat /tmp/tmp.LasdFwYNoR
++ rm /tmp/tmp.UGXDjDo1Sb /tmp/tmp.LasdFwYNoR
++ return 0
+ local client_container=psmdb-client-bb8b97679-c5ptc
+ kubectl_bin exec psmdb-client-bb8b97679-c5ptc -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.pitr-physical-backup-source-27258.svc.cluster.local:27017/admin '
++ mktemp
+ local LAST_OUT=/tmp/tmp.hyRKsTBlCU
++ mktemp
+ local LAST_ERR=/tmp/tmp.oaxHgv9wYH
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-bb8b97679-c5ptc -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.pitr-physical-backup-source-27258.svc.cluster.local:27017/admin '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.hyRKsTBlCU
+ cat /tmp/tmp.oaxHgv9wYH
+ rm /tmp/tmp.hyRKsTBlCU /tmp/tmp.oaxHgv9wYH
+ return 0
+ [[ 0 -eq 0 ]]
+ diff /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2328/e2e-tests/pitr-physical-backup-source/compare/find-3rd.json /tmp/tmp.DTCahJpmH8/find-3rd
+ desc 'disable pitr'
+ set +o xtrace


-----------------------------------------------------------------------------------
disable pitr
-----------------------------------------------------------------------------------

+ kubectl patch psmdb some-name --type=merge --patch '{"spec": {"backup": {"pitr": {"enabled": false}}}}'
perconaservermongodb.psmdb.percona.com/some-name patched
+ sleep 20
+ desc 'delete all backups'
+ set +o xtrace


-----------------------------------------------------------------------------------
delete all backups
-----------------------------------------------------------------------------------

+ kubectl_bin delete psmdb-backup --all
++ mktemp
+ local LAST_OUT=/tmp/tmp.bLBR02Xo7P
++ mktemp
+ local LAST_ERR=/tmp/tmp.Z3PdlyonJI
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl delete psmdb-backup --all
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.bLBR02Xo7P
perconaservermongodbbackup.psmdb.percona.com "backup-minio-1" deleted from pitr-physical-backup-source-27258 namespace
perconaservermongodbbackup.psmdb.percona.com "backup-minio-2" deleted from pitr-physical-backup-source-27258 namespace
+ cat /tmp/tmp.Z3PdlyonJI
+ rm /tmp/tmp.bLBR02Xo7P /tmp/tmp.Z3PdlyonJI
+ return 0
+ desc 'destroy cluster'
+ set +o xtrace


-----------------------------------------------------------------------------------
destroy cluster
-----------------------------------------------------------------------------------

+ destroy pitr-physical-backup-source-27258
+ local namespace=pitr-physical-backup-source-27258
+ local ignore_logs=true
+ [[ 0 == 1 ]]
+ desc 'destroy cluster/operator and all other resources'
+ set +o xtrace


-----------------------------------------------------------------------------------
destroy cluster/operator and all other resources
-----------------------------------------------------------------------------------

+ '[' true == false ']'
+ delete_backups
+ desc 'Delete psmdb-backup'
+ set +o xtrace


-----------------------------------------------------------------------------------
Delete psmdb-backup
-----------------------------------------------------------------------------------

++ kubectl_bin get psmdb-backup --no-headers
++ wc -l
+++ mktemp
++ local LAST_OUT=/tmp/tmp.y9LTpoW3zv
+++ mktemp
++ local LAST_ERR=/tmp/tmp.QJpIzsHLyM
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb-backup --no-headers
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.y9LTpoW3zv
++ cat /tmp/tmp.QJpIzsHLyM
No resources found in pitr-physical-backup-source-27258 namespace.
++ rm /tmp/tmp.y9LTpoW3zv /tmp/tmp.QJpIzsHLyM
++ return 0
+ '[' 0 '!=' 0 ']'
+ delete_crd
+ desc 'get and delete old CRDs and RBAC'
+ set +o xtrace


-----------------------------------------------------------------------------------
get and delete old CRDs and RBAC
-----------------------------------------------------------------------------------

+ kubectl_bin delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2328/deploy/crd.yaml --ignore-not-found --wait=false
++ mktemp
+ local LAST_OUT=/tmp/tmp.LY2HcfIay9
++ mktemp
+ local LAST_ERR=/tmp/tmp.Xn7GTcBfsP
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2328/deploy/crd.yaml --ignore-not-found --wait=false
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.LY2HcfIay9
customresourcedefinition.apiextensions.k8s.io "perconaservermongodbbackups.psmdb.percona.com" deleted
customresourcedefinition.apiextensions.k8s.io "perconaservermongodbrestores.psmdb.percona.com" deleted
customresourcedefinition.apiextensions.k8s.io "perconaservermongodbs.psmdb.percona.com" deleted
+ cat /tmp/tmp.Xn7GTcBfsP
+ rm /tmp/tmp.LY2HcfIay9 /tmp/tmp.Xn7GTcBfsP
+ return 0
++ yq eval .metadata.name /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2328/deploy/crd.yaml
++ grep -v '\-\-\-'
grep: warning: stray \ before -
grep: warning: stray \ before -
+ for crd_name in $(yq eval '.metadata.name' "${src_dir}/deploy/crd.yaml" | grep -v '\-\-\-')
+ kubectl get perconaservermongodbbackups.psmdb.percona.com --all-namespaces -o wide
+ grep -v NAMESPACE
+ xargs -L 1 sh -xc 'kubectl patch perconaservermongodbbackups.psmdb.percona.com -n $0 $1 --type=merge -p "{\"metadata\":{\"finalizers\":[]}}"'
error: the server doesn't have a resource type "perconaservermongodbbackups"
+ kubectl patch perconaservermongodbbackups.psmdb.percona.com -n sh --type=merge -p '{"metadata":{"finalizers":[]}}'
error: the server doesn't have a resource type "perconaservermongodbbackups"
+ :
+ kubectl_bin wait --for=delete crd perconaservermongodbbackups.psmdb.percona.com
++ mktemp
+ local LAST_OUT=/tmp/tmp.SJrAT4XbjF
++ mktemp
+ local LAST_ERR=/tmp/tmp.Nt1iIEwSwI
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl wait --for=delete crd perconaservermongodbbackups.psmdb.percona.com
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.SJrAT4XbjF
+ cat /tmp/tmp.Nt1iIEwSwI
+ rm /tmp/tmp.SJrAT4XbjF /tmp/tmp.Nt1iIEwSwI
+ return 0
+ for crd_name in $(yq eval '.metadata.name' "${src_dir}/deploy/crd.yaml" | grep -v '\-\-\-')
+ kubectl get perconaservermongodbrestores.psmdb.percona.com --all-namespaces -o wide
+ grep -v NAMESPACE
+ xargs -L 1 sh -xc 'kubectl patch perconaservermongodbrestores.psmdb.percona.com -n $0 $1 --type=merge -p "{\"metadata\":{\"finalizers\":[]}}"'
error: the server doesn't have a resource type "perconaservermongodbrestores"
+ kubectl patch perconaservermongodbrestores.psmdb.percona.com -n sh --type=merge -p '{"metadata":{"finalizers":[]}}'
error: the server doesn't have a resource type "perconaservermongodbrestores"
+ :
+ kubectl_bin wait --for=delete crd perconaservermongodbrestores.psmdb.percona.com
++ mktemp
+ local LAST_OUT=/tmp/tmp.YJ4tk2CzL1
++ mktemp
+ local LAST_ERR=/tmp/tmp.BZ066btQ3i
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl wait --for=delete crd perconaservermongodbrestores.psmdb.percona.com
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.YJ4tk2CzL1
+ cat /tmp/tmp.BZ066btQ3i
+ rm /tmp/tmp.YJ4tk2CzL1 /tmp/tmp.BZ066btQ3i
+ return 0
+ for crd_name in $(yq eval '.metadata.name' "${src_dir}/deploy/crd.yaml" | grep -v '\-\-\-')
+ kubectl get perconaservermongodbs.psmdb.percona.com --all-namespaces -o wide
+ grep -v NAMESPACE
+ xargs -L 1 sh -xc 'kubectl patch perconaservermongodbs.psmdb.percona.com -n $0 $1 --type=merge -p "{\"metadata\":{\"finalizers\":[]}}"'
error: the server doesn't have a resource type "perconaservermongodbs"
+ kubectl patch perconaservermongodbs.psmdb.percona.com -n sh --type=merge -p '{"metadata":{"finalizers":[]}}'
error: the server doesn't have a resource type "perconaservermongodbs"
+ :
+ kubectl_bin wait --for=delete crd perconaservermongodbs.psmdb.percona.com
++ mktemp
+ local LAST_OUT=/tmp/tmp.KFXMai5BFB
++ mktemp
+ local LAST_ERR=/tmp/tmp.h60f20Bvu5
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl wait --for=delete crd perconaservermongodbs.psmdb.percona.com
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.KFXMai5BFB
+ cat /tmp/tmp.h60f20Bvu5
+ rm /tmp/tmp.KFXMai5BFB /tmp/tmp.h60f20Bvu5
+ return 0
+ local rbac_yaml=rbac.yaml
+ '[' -n psmdb-operator ']'
+ rbac_yaml=cw-rbac.yaml
+ kubectl_bin delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2328/deploy/cw-rbac.yaml --ignore-not-found
++ mktemp
+ local LAST_OUT=/tmp/tmp.Humv1EASpN
++ mktemp
+ local LAST_ERR=/tmp/tmp.ZKUjjD1lyO
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2328/deploy/cw-rbac.yaml --ignore-not-found
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.Humv1EASpN
clusterrole.rbac.authorization.k8s.io "percona-server-mongodb-operator" deleted
clusterrolebinding.rbac.authorization.k8s.io "service-account-percona-server-mongodb-operator" deleted
+ cat /tmp/tmp.ZKUjjD1lyO
+ rm /tmp/tmp.Humv1EASpN /tmp/tmp.ZKUjjD1lyO
+ return 0
+ destroy_cert_manager
+ kubectl_bin delete -f https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml
++ mktemp
+ local LAST_OUT=/tmp/tmp.ftpw5pSxfB
++ mktemp
+ local LAST_ERR=/tmp/tmp.GWYh3Qov11
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl delete -f https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml
+ exit_status=1
+ set -e
+ '[' 1 '!=' 0 -a -n 1 ']'
+ cat /tmp/tmp.ftpw5pSxfB
+ cat /tmp/tmp.GWYh3Qov11
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": namespaces "cert-manager" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "challenges.acme.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "orders.acme.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "certificaterequests.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "certificates.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "clusterissuers.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "issuers.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": serviceaccounts "cert-manager-cainjector" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": serviceaccounts "cert-manager" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": serviceaccounts "cert-manager-webhook" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-cainjector" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-issuers" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-clusterissuers" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-certificates" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-orders" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-challenges" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-ingress-shim" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-cluster-view" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-view" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-edit" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-approve:cert-manager-io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-certificatesigningrequests" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-webhook:subjectaccessreviews" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-cainjector" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-issuers" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-clusterissuers" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-certificates" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-orders" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-challenges" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-ingress-shim" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-approve:cert-manager-io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-certificatesigningrequests" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-webhook:subjectaccessreviews" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager-cainjector:leaderelection" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager:leaderelection" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager-tokenrequest" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager-webhook:dynamic-serving" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager-cainjector:leaderelection" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager:leaderelection" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager-tokenrequest" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager-webhook:dynamic-serving" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": services "cert-manager-cainjector" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": services "cert-manager" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": services "cert-manager-webhook" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": deployments.apps "cert-manager-cainjector" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": deployments.apps "cert-manager" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": deployments.apps "cert-manager-webhook" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": mutatingwebhookconfigurations.admissionregistration.k8s.io "cert-manager-webhook" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": validatingwebhookconfigurations.admissionregistration.k8s.io "cert-manager-webhook" not found
+ sleep 0
+ for i in $(seq 0 2)
+ set +e
+ kubectl delete -f https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml
+ exit_status=1
+ set -e
+ '[' 1 '!=' 0 -a -n 1 ']'
+ cat /tmp/tmp.ftpw5pSxfB
+ cat /tmp/tmp.GWYh3Qov11
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": namespaces "cert-manager" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "challenges.acme.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "orders.acme.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "certificaterequests.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "certificates.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "clusterissuers.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "issuers.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": serviceaccounts "cert-manager-cainjector" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": serviceaccounts "cert-manager" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": serviceaccounts "cert-manager-webhook" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-cainjector" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-issuers" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-clusterissuers" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-certificates" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-orders" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-challenges" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-ingress-shim" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-cluster-view" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-view" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-edit" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-approve:cert-manager-io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-certificatesigningrequests" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-webhook:subjectaccessreviews" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-cainjector" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-issuers" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-clusterissuers" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-certificates" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-orders" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-challenges" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-ingress-shim" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-approve:cert-manager-io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-certificatesigningrequests" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-webhook:subjectaccessreviews" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager-cainjector:leaderelection" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager:leaderelection" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager-tokenrequest" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager-webhook:dynamic-serving" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager-cainjector:leaderelection" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager:leaderelection" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager-tokenrequest" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager-webhook:dynamic-serving" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": services "cert-manager-cainjector" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": services "cert-manager" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": services "cert-manager-webhook" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": deployments.apps "cert-manager-cainjector" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": deployments.apps "cert-manager" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": deployments.apps "cert-manager-webhook" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": mutatingwebhookconfigurations.admissionregistration.k8s.io "cert-manager-webhook" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": validatingwebhookconfigurations.admissionregistration.k8s.io "cert-manager-webhook" not found
+ sleep 4
+ for i in $(seq 0 2)
+ set +e
+ kubectl delete -f https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml
+ exit_status=1
+ set -e
+ '[' 1 '!=' 0 -a -n 1 ']'
+ cat /tmp/tmp.ftpw5pSxfB
+ cat /tmp/tmp.GWYh3Qov11
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": namespaces "cert-manager" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "challenges.acme.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "orders.acme.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "certificaterequests.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "certificates.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "clusterissuers.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "issuers.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": serviceaccounts "cert-manager-cainjector" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": serviceaccounts "cert-manager" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": serviceaccounts "cert-manager-webhook" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-cainjector" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-issuers" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-clusterissuers" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-certificates" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-orders" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-challenges" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-ingress-shim" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-cluster-view" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-view" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-edit" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-approve:cert-manager-io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-certificatesigningrequests" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-webhook:subjectaccessreviews" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-cainjector" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-issuers" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-clusterissuers" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-certificates" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-orders" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-challenges" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-ingress-shim" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-approve:cert-manager-io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-certificatesigningrequests" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-webhook:subjectaccessreviews" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager-cainjector:leaderelection" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager:leaderelection" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager-tokenrequest" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager-webhook:dynamic-serving" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager-cainjector:leaderelection" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager:leaderelection" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager-tokenrequest" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager-webhook:dynamic-serving" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": services "cert-manager-cainjector" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": services "cert-manager" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": services "cert-manager-webhook" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": deployments.apps "cert-manager-cainjector" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": deployments.apps "cert-manager" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": deployments.apps "cert-manager-webhook" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": mutatingwebhookconfigurations.admissionregistration.k8s.io "cert-manager-webhook" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": validatingwebhookconfigurations.admissionregistration.k8s.io "cert-manager-webhook" not found
+ sleep 8
+ cat /tmp/tmp.ftpw5pSxfB
+ cat /tmp/tmp.GWYh3Qov11
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": namespaces "cert-manager" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "challenges.acme.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "orders.acme.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "certificaterequests.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "certificates.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "clusterissuers.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "issuers.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": serviceaccounts "cert-manager-cainjector" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": serviceaccounts "cert-manager" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": serviceaccounts "cert-manager-webhook" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-cainjector" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-issuers" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-clusterissuers" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-certificates" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-orders" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-challenges" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-ingress-shim" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-cluster-view" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-view" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-edit" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-approve:cert-manager-io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-certificatesigningrequests" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-webhook:subjectaccessreviews" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-cainjector" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-issuers" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-clusterissuers" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-certificates" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-orders" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-challenges" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-ingress-shim" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-approve:cert-manager-io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-certificatesigningrequests" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-webhook:subjectaccessreviews" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager-cainjector:leaderelection" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager:leaderelection" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager-tokenrequest" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager-webhook:dynamic-serving" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager-cainjector:leaderelection" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager:leaderelection" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager-tokenrequest" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager-webhook:dynamic-serving" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": services "cert-manager-cainjector" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": services "cert-manager" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": services "cert-manager-webhook" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": deployments.apps "cert-manager-cainjector" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": deployments.apps "cert-manager" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": deployments.apps "cert-manager-webhook" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": mutatingwebhookconfigurations.admissionregistration.k8s.io "cert-manager-webhook" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": validatingwebhookconfigurations.admissionregistration.k8s.io "cert-manager-webhook" not found
+ rm /tmp/tmp.ftpw5pSxfB /tmp/tmp.GWYh3Qov11
+ return 1
+ true
+ '[' -n '' ']'
+ '[' -n psmdb-operator ']'
+ kubectl_bin delete --grace-period=0 --force=true namespace pitr-physical-backup-source-27258
+ rm -rf /tmp/tmp.DTCahJpmH8
+ kubectl_bin delete --grace-period=0 --force=true namespace psmdb-operator
++ mktemp
++ mktemp
+ desc 'test passed'
+ set +o xtrace


-----------------------------------------------------------------------------------
test passed
-----------------------------------------------------------------------------------

+ local LAST_OUT=/tmp/tmp.AuYxInRDhm
+ local LAST_OUT=/tmp/tmp.BK8kRaHC0M
++ mktemp
++ mktemp
+ local LAST_ERR=/tmp/tmp.X7OLVoXy8c
+ local exit_status=0
+ local timeout=4
+ local LAST_ERR=/tmp/tmp.BjGR6bTyi4
+ local exit_status=0
+ local timeout=4
++ seq 0 2
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl delete --grace-period=0 --force=true namespace pitr-physical-backup-source-27258
+ for i in $(seq 0 2)
+ set +e
+ kubectl delete --grace-period=0 --force=true namespace psmdb-operator