Log: /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2406/e2e-tests/logs/pitr-physical-backup-source.log
Warning: version difference between client (1.36) and server (1.33) exceeds the supported minor version skew of +/-1
Warning: version difference between client (1.36) and server (1.33) exceeds the supported minor version skew of +/-1
Warning: version difference between client (1.36) and server (1.33) exceeds the supported minor version skew of +/-1
+ main
+ create_infra pitr-physical-backup-source-28616
+ local ns=pitr-physical-backup-source-28616
+ [[ 1 == 1 ]]
+ delete_crd
+ desc 'get and delete old CRDs and RBAC'
+ set +o xtrace


-----------------------------------------------------------------------------------
get and delete old CRDs and RBAC
-----------------------------------------------------------------------------------

+ kubectl_bin delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2406/deploy/crd.yaml --ignore-not-found --wait=false
++ mktemp
+ local LAST_OUT=/tmp/tmp.tXWD8atQiP
++ mktemp
+ local LAST_ERR=/tmp/tmp.OuolzA3py3
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2406/deploy/crd.yaml --ignore-not-found --wait=false
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.tXWD8atQiP
customresourcedefinition.apiextensions.k8s.io "perconaservermongodbbackups.psmdb.percona.com" deleted
customresourcedefinition.apiextensions.k8s.io "perconaservermongodbclustersyncs.psmdb.percona.com" deleted
customresourcedefinition.apiextensions.k8s.io "perconaservermongodbrestores.psmdb.percona.com" deleted
customresourcedefinition.apiextensions.k8s.io "perconaservermongodbs.psmdb.percona.com" deleted
+ cat /tmp/tmp.OuolzA3py3
+ rm /tmp/tmp.tXWD8atQiP /tmp/tmp.OuolzA3py3
+ return 0
++ yq eval .metadata.name /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2406/deploy/crd.yaml
++ grep -v '\-\-\-'
grep: warning: stray \ before -
grep: warning: stray \ before -
+ for crd_name in $(yq eval '.metadata.name' "${src_dir}/deploy/crd.yaml" | grep -v '\-\-\-')
+ kubectl get perconaservermongodbbackups.psmdb.percona.com --all-namespaces -o wide
+ grep -v NAMESPACE
+ xargs -L 1 sh -xc 'kubectl patch perconaservermongodbbackups.psmdb.percona.com -n $0 $1 --type=merge -p "{\"metadata\":{\"finalizers\":[]}}"'
No resources found
+ kubectl patch perconaservermongodbbackups.psmdb.percona.com -n sh --type=merge -p '{"metadata":{"finalizers":[]}}'
error: resource(s) were provided, but no name was specified
+ :
+ kubectl_bin wait --for=delete crd perconaservermongodbbackups.psmdb.percona.com
++ mktemp
+ local LAST_OUT=/tmp/tmp.L20kZ8oEH8
++ mktemp
+ local LAST_ERR=/tmp/tmp.EOD1lk7r0p
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl wait --for=delete crd perconaservermongodbbackups.psmdb.percona.com
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.L20kZ8oEH8
+ cat /tmp/tmp.EOD1lk7r0p
+ rm /tmp/tmp.L20kZ8oEH8 /tmp/tmp.EOD1lk7r0p
+ return 0
+ for crd_name in $(yq eval '.metadata.name' "${src_dir}/deploy/crd.yaml" | grep -v '\-\-\-')
+ kubectl get perconaservermongodbclustersyncs.psmdb.percona.com --all-namespaces -o wide
+ grep -v NAMESPACE
+ xargs -L 1 sh -xc 'kubectl patch perconaservermongodbclustersyncs.psmdb.percona.com -n $0 $1 --type=merge -p "{\"metadata\":{\"finalizers\":[]}}"'
error: the server doesn't have a resource type "perconaservermongodbclustersyncs"
+ kubectl patch perconaservermongodbclustersyncs.psmdb.percona.com -n sh --type=merge -p '{"metadata":{"finalizers":[]}}'
error: the server doesn't have a resource type "perconaservermongodbclustersyncs"
+ :
+ kubectl_bin wait --for=delete crd perconaservermongodbclustersyncs.psmdb.percona.com
++ mktemp
+ local LAST_OUT=/tmp/tmp.azG2onTxTV
++ mktemp
+ local LAST_ERR=/tmp/tmp.0vPOf4q0TH
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl wait --for=delete crd perconaservermongodbclustersyncs.psmdb.percona.com
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.azG2onTxTV
+ cat /tmp/tmp.0vPOf4q0TH
+ rm /tmp/tmp.azG2onTxTV /tmp/tmp.0vPOf4q0TH
+ return 0
+ for crd_name in $(yq eval '.metadata.name' "${src_dir}/deploy/crd.yaml" | grep -v '\-\-\-')
+ kubectl get perconaservermongodbrestores.psmdb.percona.com --all-namespaces -o wide
+ grep -v NAMESPACE
+ xargs -L 1 sh -xc 'kubectl patch perconaservermongodbrestores.psmdb.percona.com -n $0 $1 --type=merge -p "{\"metadata\":{\"finalizers\":[]}}"'
error: the server doesn't have a resource type "perconaservermongodbrestores"
+ kubectl patch perconaservermongodbrestores.psmdb.percona.com -n sh --type=merge -p '{"metadata":{"finalizers":[]}}'
error: the server doesn't have a resource type "perconaservermongodbrestores"
+ :
+ kubectl_bin wait --for=delete crd perconaservermongodbrestores.psmdb.percona.com
++ mktemp
+ local LAST_OUT=/tmp/tmp.lU3KkKr5g2
++ mktemp
+ local LAST_ERR=/tmp/tmp.E8A0zSznkV
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl wait --for=delete crd perconaservermongodbrestores.psmdb.percona.com
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.lU3KkKr5g2
+ cat /tmp/tmp.E8A0zSznkV
+ rm /tmp/tmp.lU3KkKr5g2 /tmp/tmp.E8A0zSznkV
+ return 0
+ for crd_name in $(yq eval '.metadata.name' "${src_dir}/deploy/crd.yaml" | grep -v '\-\-\-')
+ kubectl get perconaservermongodbs.psmdb.percona.com --all-namespaces -o wide
+ grep -v NAMESPACE
+ xargs -L 1 sh -xc 'kubectl patch perconaservermongodbs.psmdb.percona.com -n $0 $1 --type=merge -p "{\"metadata\":{\"finalizers\":[]}}"'
error: the server doesn't have a resource type "perconaservermongodbs"
+ kubectl patch perconaservermongodbs.psmdb.percona.com -n sh --type=merge -p '{"metadata":{"finalizers":[]}}'
error: the server doesn't have a resource type "perconaservermongodbs"
+ :
+ kubectl_bin wait --for=delete crd perconaservermongodbs.psmdb.percona.com
++ mktemp
+ local LAST_OUT=/tmp/tmp.bfNI3YHgR7
++ mktemp
+ local LAST_ERR=/tmp/tmp.TojncLaEOK
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl wait --for=delete crd perconaservermongodbs.psmdb.percona.com
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.bfNI3YHgR7
+ cat /tmp/tmp.TojncLaEOK
+ rm /tmp/tmp.bfNI3YHgR7 /tmp/tmp.TojncLaEOK
+ return 0
+ local rbac_yaml=rbac.yaml
+ '[' -n psmdb-operator ']'
+ rbac_yaml=cw-rbac.yaml
+ kubectl_bin delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2406/deploy/cw-rbac.yaml --ignore-not-found
++ mktemp
+ local LAST_OUT=/tmp/tmp.lOqiu7CAzv
++ mktemp
+ local LAST_ERR=/tmp/tmp.PwMzIioIzM
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2406/deploy/cw-rbac.yaml --ignore-not-found
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.lOqiu7CAzv
clusterrole.rbac.authorization.k8s.io "percona-server-mongodb-operator" deleted
clusterrolebinding.rbac.authorization.k8s.io "service-account-percona-server-mongodb-operator" deleted
+ cat /tmp/tmp.PwMzIioIzM
+ rm /tmp/tmp.lOqiu7CAzv /tmp/tmp.PwMzIioIzM
+ return 0
+ check_crd_for_deletion PR-2406-1c5b15f05
+ local git_tag=PR-2406-1c5b15f05
++ curl -s https://raw.githubusercontent.com/percona/percona-server-mongodb-operator/PR-2406-1c5b15f05/deploy/crd.yaml
++ yq eval .metadata.name
++ /usr/sbin/sed ':a;N;$!ba;s/\n/ /g'
++ /usr/sbin/sed s/---//g
+ for crd_name in $(curl -s https://raw.githubusercontent.com/percona/percona-server-mongodb-operator/${git_tag}/deploy/crd.yaml | yq eval '.metadata.name' | $sed 's/---//g' | $sed ':a;N;$!ba;s/\n/ /g')
++ kubectl_bin get crd/null -o 'jsonpath={.status.conditions[-1].type}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.1GzA3hNsJx
+++ mktemp
++ local LAST_ERR=/tmp/tmp.rTZVklbWY3
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get crd/null -o 'jsonpath={.status.conditions[-1].type}'
++ exit_status=1
++ set -e
++ '[' 1 '!=' 0 -a -n 1 ']'
++ cat /tmp/tmp.1GzA3hNsJx
++ cat /tmp/tmp.rTZVklbWY3
Error from server (NotFound): customresourcedefinitions.apiextensions.k8s.io "null" not found
++ sleep 0
++ for i in $(seq 0 2)
++ set +e
++ kubectl get crd/null -o 'jsonpath={.status.conditions[-1].type}'
++ exit_status=1
++ set -e
++ '[' 1 '!=' 0 -a -n 1 ']'
++ cat /tmp/tmp.1GzA3hNsJx
++ cat /tmp/tmp.rTZVklbWY3
Error from server (NotFound): customresourcedefinitions.apiextensions.k8s.io "null" not found
++ sleep 4
++ for i in $(seq 0 2)
++ set +e
++ kubectl get crd/null -o 'jsonpath={.status.conditions[-1].type}'
++ exit_status=1
++ set -e
++ '[' 1 '!=' 0 -a -n 1 ']'
++ cat /tmp/tmp.1GzA3hNsJx
++ cat /tmp/tmp.rTZVklbWY3
Error from server (NotFound): customresourcedefinitions.apiextensions.k8s.io "null" not found
++ sleep 8
++ cat /tmp/tmp.1GzA3hNsJx
++ cat /tmp/tmp.rTZVklbWY3
Error from server (NotFound): customresourcedefinitions.apiextensions.k8s.io "null" not found
++ rm /tmp/tmp.1GzA3hNsJx /tmp/tmp.rTZVklbWY3
++ return 1
+ [[ '' == Terminating ]]
+ '[' -n psmdb-operator ']'
+ create_namespace psmdb-operator
+ local namespace=psmdb-operator
+ local skip_clean_namespace=
+ [[ 1 == 1 ]]
+ [[ -z '' ]]
+ destroy_chaos_mesh
++ helm list --all-namespaces --filter chaos-mesh
++ tail -n1
++ sed s/NAMESPACE//
++ awk '-F ' '{print $2}'
+ local chaos_mesh_ns=
+ desc 'destroy chaos-mesh'
+ set +o xtrace


-----------------------------------------------------------------------------------
destroy chaos-mesh
-----------------------------------------------------------------------------------

+ '[' -n '' ']'
++ kubectl get MutatingWebhookConfiguration
++ grep chaos-mesh
++ awk '{print $1}'
+ timeout 30 kubectl delete MutatingWebhookConfiguration
error: resource(s) were provided, but no name was specified
+ :
++ kubectl get ValidatingWebhookConfiguration
++ grep chaos-mesh
++ awk '{print $1}'
+ timeout 30 kubectl delete ValidatingWebhookConfiguration
error: resource(s) were provided, but no name was specified
+ :
++ kubectl get ValidatingWebhookConfiguration
++ grep validate-auth
++ awk '{print $1}'
+ timeout 30 kubectl delete ValidatingWebhookConfiguration
error: resource(s) were provided, but no name was specified
+ :
++ kubectl api-resources
++ grep chaos-mesh
++ awk '{print $1}'
++ kubectl get crd
++ grep chaos-mesh.org
++ awk '{print $1}'
+ timeout 30 kubectl delete crd
error: resource(s) were provided, but no name was specified
+ :
++ kubectl get clusterrolebinding
++ grep chaos-mesh
++ awk '{print $1}'
+ timeout 30 kubectl delete clusterrolebinding
error: resource(s) were provided, but no name was specified
+ :
++ kubectl get clusterrole
++ grep chaos-mesh
++ awk '{print $1}'
+ timeout 30 kubectl delete clusterrole
error: resource(s) were provided, but no name was specified
+ :
+ desc 'cleaned up all old namespaces'
+ set +o xtrace


-----------------------------------------------------------------------------------
cleaned up all old namespaces
-----------------------------------------------------------------------------------

+ kubectl_bin get ns
+ grep -E -v '^kube-|^default|Terminating|psmdb-operator|openshift|^gke-|^gmp-|^NAME'
+ awk '{print$1}'
+ '[' -n '' ']'
+ desc 'cleaned up old namespaces psmdb-operator'
+ xargs kubectl delete ns
+ set +o xtrace


-----------------------------------------------------------------------------------
cleaned up old namespaces psmdb-operator
-----------------------------------------------------------------------------------

+ kubectl_bin delete namespace psmdb-operator --ignore-not-found
++ mktemp
++ mktemp
+ local LAST_OUT=/tmp/tmp.0Qay1FXeig
++ mktemp
+ local LAST_OUT=/tmp/tmp.wKr2o7UX4b
++ mktemp
+ local LAST_ERR=/tmp/tmp.q9WXFu11Ab
+ local exit_status=0
+ local timeout=4
+ local LAST_ERR=/tmp/tmp.tEpOy8piPJ
++ seq 0 2
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl get ns
+ for i in $(seq 0 2)
+ set +e
+ kubectl delete namespace psmdb-operator --ignore-not-found
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.0Qay1FXeig
+ cat /tmp/tmp.q9WXFu11Ab
+ rm /tmp/tmp.0Qay1FXeig /tmp/tmp.q9WXFu11Ab
+ return 0
namespace "pitr-physical-backup-source-16329" deleted
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.wKr2o7UX4b
namespace "psmdb-operator" deleted
+ cat /tmp/tmp.tEpOy8piPJ
+ rm /tmp/tmp.wKr2o7UX4b /tmp/tmp.tEpOy8piPJ
+ return 0
+ kubectl_bin wait --for=delete namespace psmdb-operator
++ mktemp
+ local LAST_OUT=/tmp/tmp.9MztGggexo
++ mktemp
+ local LAST_ERR=/tmp/tmp.a13bXvRIMw
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl wait --for=delete namespace psmdb-operator
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.9MztGggexo
+ cat /tmp/tmp.a13bXvRIMw
+ rm /tmp/tmp.9MztGggexo /tmp/tmp.a13bXvRIMw
+ return 0
+ desc 'create namespace psmdb-operator'
+ set +o xtrace


-----------------------------------------------------------------------------------
create namespace psmdb-operator
-----------------------------------------------------------------------------------

+ kubectl_bin create namespace psmdb-operator
++ mktemp
+ local LAST_OUT=/tmp/tmp.Tz7xgHP5i4
++ mktemp
+ local LAST_ERR=/tmp/tmp.VJ1U4DG01A
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl create namespace psmdb-operator
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.Tz7xgHP5i4
namespace/psmdb-operator created
+ cat /tmp/tmp.VJ1U4DG01A
+ rm /tmp/tmp.Tz7xgHP5i4 /tmp/tmp.VJ1U4DG01A
+ return 0
+ set_kube_ctx psmdb-operator
+ local namespace=psmdb-operator
++ kubectl_bin config current-context
+++ mktemp
++ local LAST_OUT=/tmp/tmp.kIHPq3IZHg
+++ mktemp
++ local LAST_ERR=/tmp/tmp.rUXRMJZZwT
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl config current-context
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.kIHPq3IZHg
++ cat /tmp/tmp.rUXRMJZZwT
++ rm /tmp/tmp.kIHPq3IZHg /tmp/tmp.rUXRMJZZwT
++ return 0
+ kubectl_bin config set-context gke_cloud-dev-112233_us-central1-a_jen-psmdb-2406-1c5b15f05-8-cluster3 --namespace=psmdb-operator
++ mktemp
+ local LAST_OUT=/tmp/tmp.ecHKoY4chl
++ mktemp
+ local LAST_ERR=/tmp/tmp.a8zSAwwgwj
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl config set-context gke_cloud-dev-112233_us-central1-a_jen-psmdb-2406-1c5b15f05-8-cluster3 --namespace=psmdb-operator
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.ecHKoY4chl
Context "gke_cloud-dev-112233_us-central1-a_jen-psmdb-2406-1c5b15f05-8-cluster3" modified.
+ cat /tmp/tmp.a8zSAwwgwj
+ rm /tmp/tmp.ecHKoY4chl /tmp/tmp.a8zSAwwgwj
+ return 0
+ deploy_operator
+ desc 'start PSMDB operator: docker.io/perconalab/percona-server-mongodb-operator:PR-2406-1c5b15f05'
+ set +o xtrace


-----------------------------------------------------------------------------------
start PSMDB operator: docker.io/perconalab/percona-server-mongodb-operator:PR-2406-1c5b15f05
-----------------------------------------------------------------------------------

+ local cr_file
+ '[' -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2406/e2e-tests/pitr-physical-backup-source/conf/crd.yaml ']'
+ cr_file=/mnt/jenkins/workspace/cloud-psmdb-operator_PR-2406/deploy/crd.yaml
+ kubectl_bin apply --server-side --force-conflicts -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2406/deploy/crd.yaml
++ mktemp
+ local LAST_OUT=/tmp/tmp.xdPM24Sgq3
++ mktemp
+ local LAST_ERR=/tmp/tmp.fgW730zAGp
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl apply --server-side --force-conflicts -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2406/deploy/crd.yaml
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.xdPM24Sgq3
customresourcedefinition.apiextensions.k8s.io/perconaservermongodbbackups.psmdb.percona.com serverside-applied
customresourcedefinition.apiextensions.k8s.io/perconaservermongodbclustersyncs.psmdb.percona.com serverside-applied
customresourcedefinition.apiextensions.k8s.io/perconaservermongodbrestores.psmdb.percona.com serverside-applied
customresourcedefinition.apiextensions.k8s.io/perconaservermongodbs.psmdb.percona.com serverside-applied
+ cat /tmp/tmp.fgW730zAGp
+ rm /tmp/tmp.xdPM24Sgq3 /tmp/tmp.fgW730zAGp
+ return 0
+ '[' -n psmdb-operator ']'
+ apply_rbac cw-rbac
+ local operator_namespace=psmdb-operator
+ local rbac=cw-rbac
+ cat /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2406/deploy/cw-rbac.yaml
+ sed -e 's^namespace: .*^namespace: psmdb-operator^'
+ kubectl_bin apply -n psmdb-operator -f -
++ mktemp
+ local LAST_OUT=/tmp/tmp.alt442rIjt
++ mktemp
+ local LAST_ERR=/tmp/tmp.VaXdo7SzOM
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl apply -n psmdb-operator -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.alt442rIjt
clusterrole.rbac.authorization.k8s.io/percona-server-mongodb-operator created
serviceaccount/percona-server-mongodb-operator created
clusterrolebinding.rbac.authorization.k8s.io/service-account-percona-server-mongodb-operator created
+ cat /tmp/tmp.VaXdo7SzOM
+ rm /tmp/tmp.alt442rIjt /tmp/tmp.VaXdo7SzOM
+ return 0
+ yq eval $'\n\t\t\t(.spec.template.spec.containers[].image = "docker.io/perconalab/percona-server-mongodb-operator:PR-2406-1c5b15f05") |\n\t\t\t((.. | select(.[] == "DISABLE_TELEMETRY")) |= .value="true") |\n\t\t\t((.. | select(.[] == "LOG_LEVEL")) |= .value="DEBUG")' /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2406/deploy/cw-operator.yaml
+ kubectl_bin apply -n psmdb-operator -f -
++ mktemp
+ local LAST_OUT=/tmp/tmp.NcXebzVJU3
++ mktemp
+ local LAST_ERR=/tmp/tmp.DbadDkfkS5
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl apply -n psmdb-operator -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.NcXebzVJU3
deployment.apps/percona-server-mongodb-operator created
+ cat /tmp/tmp.DbadDkfkS5
+ rm /tmp/tmp.NcXebzVJU3 /tmp/tmp.DbadDkfkS5
+ return 0
+ sleep 20
++ get_operator_pod
++ kubectl_bin get pods --selector=name=percona-server-mongodb-operator -o 'jsonpath={.items[].metadata.name}' -n psmdb-operator
+++ mktemp
++ local LAST_OUT=/tmp/tmp.uiS1HoFyYt
+++ mktemp
++ local LAST_ERR=/tmp/tmp.ezdQbDQtaf
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=percona-server-mongodb-operator -o 'jsonpath={.items[].metadata.name}' -n psmdb-operator
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.uiS1HoFyYt
++ cat /tmp/tmp.ezdQbDQtaf
++ rm /tmp/tmp.uiS1HoFyYt /tmp/tmp.ezdQbDQtaf
++ return 0
+ wait_operator_pod percona-server-mongodb-operator-f74db4f97-6jtnn
+ local pod=percona-server-mongodb-operator-f74db4f97-6jtnn
+ set +o xtrace
waiting for pod/percona-server-mongodb-operator-f74db4f97-6jtnn to be ready.OK
+ echo 'Print operator info from log'
Print operator info from log
+ grep 'Manager starting up'
++ get_operator_pod
++ kubectl_bin get pods --selector=name=percona-server-mongodb-operator -o 'jsonpath={.items[].metadata.name}' -n psmdb-operator
+++ mktemp
++ local LAST_OUT=/tmp/tmp.BjtaWQdKpv
+++ mktemp
++ local LAST_ERR=/tmp/tmp.oyuhvzEKZc
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=percona-server-mongodb-operator -o 'jsonpath={.items[].metadata.name}' -n psmdb-operator
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.BjtaWQdKpv
++ cat /tmp/tmp.oyuhvzEKZc
++ rm /tmp/tmp.BjtaWQdKpv /tmp/tmp.oyuhvzEKZc
++ return 0
+ kubectl_bin logs -n psmdb-operator percona-server-mongodb-operator-f74db4f97-6jtnn
++ mktemp
+ local LAST_OUT=/tmp/tmp.OyrOLoYtG3
++ mktemp
+ local LAST_ERR=/tmp/tmp.mn3HFfQWtS
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl logs -n psmdb-operator percona-server-mongodb-operator-f74db4f97-6jtnn
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.OyrOLoYtG3
+ cat /tmp/tmp.mn3HFfQWtS
+ rm /tmp/tmp.OyrOLoYtG3 /tmp/tmp.mn3HFfQWtS
+ return 0
2026-06-18T20:51:00.920Z	INFO	setup	Manager starting up	{"gitCommit": "1c5b15f0523892e2a9831596c60cb774f11b7aff", "gitBranch": "PR-2406-1c5b15f05", "buildTime": "", "goVersion": "go1.26.4", "os": "linux", "arch": "amd64"}
+ create_namespace pitr-physical-backup-source-28616
+ local namespace=pitr-physical-backup-source-28616
+ local skip_clean_namespace=
+ [[ 1 == 1 ]]
+ [[ -z '' ]]
+ destroy_chaos_mesh
++ helm list --all-namespaces --filter chaos-mesh
++ tail -n1
++ awk '-F ' '{print $2}'
++ sed s/NAMESPACE//
+ local chaos_mesh_ns=
+ desc 'destroy chaos-mesh'
+ set +o xtrace


-----------------------------------------------------------------------------------
destroy chaos-mesh
-----------------------------------------------------------------------------------

+ '[' -n '' ']'
++ kubectl get MutatingWebhookConfiguration
++ grep chaos-mesh
++ awk '{print $1}'
+ timeout 30 kubectl delete MutatingWebhookConfiguration
error: resource(s) were provided, but no name was specified
+ :
++ kubectl get ValidatingWebhookConfiguration
++ grep chaos-mesh
++ awk '{print $1}'
+ timeout 30 kubectl delete ValidatingWebhookConfiguration
error: resource(s) were provided, but no name was specified
+ :
++ kubectl get ValidatingWebhookConfiguration
++ grep validate-auth
++ awk '{print $1}'
+ timeout 30 kubectl delete ValidatingWebhookConfiguration
error: resource(s) were provided, but no name was specified
+ :
++ kubectl api-resources
++ grep chaos-mesh
++ awk '{print $1}'
++ kubectl get crd
++ grep chaos-mesh.org
++ awk '{print $1}'
+ timeout 30 kubectl delete crd
error: resource(s) were provided, but no name was specified
+ :
++ kubectl get clusterrolebinding
++ grep chaos-mesh
++ awk '{print $1}'
+ timeout 30 kubectl delete clusterrolebinding
error: resource(s) were provided, but no name was specified
+ :
++ kubectl get clusterrole
++ grep chaos-mesh
++ awk '{print $1}'
+ timeout 30 kubectl delete clusterrole
error: resource(s) were provided, but no name was specified
+ :
+ desc 'cleaned up all old namespaces'
+ set +o xtrace


-----------------------------------------------------------------------------------
cleaned up all old namespaces
-----------------------------------------------------------------------------------

+ kubectl_bin get ns
+ grep -E -v '^kube-|^default|Terminating|psmdb-operator|openshift|^gke-|^gmp-|^NAME'
+ '[' -n '' ']'
+ awk '{print$1}'
+ desc 'cleaned up old namespaces pitr-physical-backup-source-28616'
+ set +o xtrace
+ xargs kubectl delete ns


-----------------------------------------------------------------------------------
cleaned up old namespaces pitr-physical-backup-source-28616
-----------------------------------------------------------------------------------

+ kubectl_bin delete namespace pitr-physical-backup-source-28616 --ignore-not-found
++ mktemp
++ mktemp
+ local LAST_OUT=/tmp/tmp.rUyMhK0zrk
+ local LAST_OUT=/tmp/tmp.0BjonUrRlu
++ mktemp
++ mktemp
+ local LAST_ERR=/tmp/tmp.LLWw28kSiI
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ local LAST_ERR=/tmp/tmp.3wo0reBRX7
+ local exit_status=0
+ local timeout=4
+ for i in $(seq 0 2)
+ set +e
+ kubectl get ns
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl delete namespace pitr-physical-backup-source-28616 --ignore-not-found
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.rUyMhK0zrk
+ cat /tmp/tmp.LLWw28kSiI
+ rm /tmp/tmp.rUyMhK0zrk /tmp/tmp.LLWw28kSiI
+ return 0
error: resource(s) were provided, but no name was specified
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.0BjonUrRlu
+ cat /tmp/tmp.3wo0reBRX7
+ rm /tmp/tmp.0BjonUrRlu /tmp/tmp.3wo0reBRX7
+ return 0
+ kubectl_bin wait --for=delete namespace pitr-physical-backup-source-28616
++ mktemp
+ local LAST_OUT=/tmp/tmp.TgVkz69HZe
++ mktemp
+ local LAST_ERR=/tmp/tmp.XUF7G9nHvn
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl wait --for=delete namespace pitr-physical-backup-source-28616
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.TgVkz69HZe
+ cat /tmp/tmp.XUF7G9nHvn
+ rm /tmp/tmp.TgVkz69HZe /tmp/tmp.XUF7G9nHvn
+ return 0
+ desc 'create namespace pitr-physical-backup-source-28616'
+ set +o xtrace


-----------------------------------------------------------------------------------
create namespace pitr-physical-backup-source-28616
-----------------------------------------------------------------------------------

+ kubectl_bin create namespace pitr-physical-backup-source-28616
++ mktemp
+ local LAST_OUT=/tmp/tmp.bkFU7lHHQd
++ mktemp
+ local LAST_ERR=/tmp/tmp.YGRjQbwG6V
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl create namespace pitr-physical-backup-source-28616
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.bkFU7lHHQd
namespace/pitr-physical-backup-source-28616 created
+ cat /tmp/tmp.YGRjQbwG6V
+ rm /tmp/tmp.bkFU7lHHQd /tmp/tmp.YGRjQbwG6V
+ return 0
+ set_kube_ctx pitr-physical-backup-source-28616
+ local namespace=pitr-physical-backup-source-28616
++ kubectl_bin config current-context
+++ mktemp
++ local LAST_OUT=/tmp/tmp.fuIFPpbHaI
+++ mktemp
++ local LAST_ERR=/tmp/tmp.ZrAMQKSaR8
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl config current-context
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.fuIFPpbHaI
++ cat /tmp/tmp.ZrAMQKSaR8
++ rm /tmp/tmp.fuIFPpbHaI /tmp/tmp.ZrAMQKSaR8
++ return 0
+ kubectl_bin config set-context gke_cloud-dev-112233_us-central1-a_jen-psmdb-2406-1c5b15f05-8-cluster3 --namespace=pitr-physical-backup-source-28616
++ mktemp
+ local LAST_OUT=/tmp/tmp.iA4qDUlJgn
++ mktemp
+ local LAST_ERR=/tmp/tmp.X4Wwi1nvnM
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl config set-context gke_cloud-dev-112233_us-central1-a_jen-psmdb-2406-1c5b15f05-8-cluster3 --namespace=pitr-physical-backup-source-28616
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.iA4qDUlJgn
Context "gke_cloud-dev-112233_us-central1-a_jen-psmdb-2406-1c5b15f05-8-cluster3" modified.
+ cat /tmp/tmp.X4Wwi1nvnM
+ rm /tmp/tmp.iA4qDUlJgn /tmp/tmp.X4Wwi1nvnM
+ return 0
+ deploy_minio
+ local cert_secret=
+ local service_name=minio-service
+ desc 'install MinIO: minio-service'
+ set +o xtrace


-----------------------------------------------------------------------------------
install MinIO: minio-service
-----------------------------------------------------------------------------------

+ helm uninstall minio-service
+ :
+ helm repo remove minio
"minio" has been removed from your repositories
+ helm repo add minio https://charts.min.io/
"minio" has been added to your repositories
+ local endpoint=http://minio-service:9000
+ minio_args=('--version' '5.4.0' '--set' 'replicas=1' '--set' 'mode=standalone' '--set' 'resources.requests.memory=256Mi' '--set' 'rootUser=rootuser' '--set' 'rootPassword=rootpass123' '--set' 'users[0].accessKey=some-access-key' '--set' 'users[0].secretKey=some-secret-key' '--set' 'users[0].policy=consoleAdmin' '--set' 'service.type=ClusterIP' '--set' 'configPathmc=/tmp/' '--set' 'securityContext.enabled=false' '--set' 'persistence.size=2G' '--set' 'fullnameOverride=minio-service' '--set' 'serviceAccount.create=true' '--set' 'serviceAccount.name=minio-service-sa')
+ local minio_args
+ [[ -n '' ]]
+ retry 10 60 helm install minio-service --version 5.4.0 --set replicas=1 --set mode=standalone --set resources.requests.memory=256Mi --set rootUser=rootuser --set rootPassword=rootpass123 --set 'users[0].accessKey=some-access-key' --set 'users[0].secretKey=some-secret-key' --set 'users[0].policy=consoleAdmin' --set service.type=ClusterIP --set configPathmc=/tmp/ --set securityContext.enabled=false --set persistence.size=2G --set fullnameOverride=minio-service --set serviceAccount.create=true --set serviceAccount.name=minio-service-sa minio/minio
+ local max=10
+ local delay=60
+ shift 2
+ local n=1
+ helm install minio-service --version 5.4.0 --set replicas=1 --set mode=standalone --set resources.requests.memory=256Mi --set rootUser=rootuser --set rootPassword=rootpass123 --set 'users[0].accessKey=some-access-key' --set 'users[0].secretKey=some-secret-key' --set 'users[0].policy=consoleAdmin' --set service.type=ClusterIP --set configPathmc=/tmp/ --set securityContext.enabled=false --set persistence.size=2G --set fullnameOverride=minio-service --set serviceAccount.create=true --set serviceAccount.name=minio-service-sa minio/minio
NAME: minio-service
LAST DEPLOYED: Thu Jun 18 20:51:38 2026
NAMESPACE: pitr-physical-backup-source-28616
STATUS: deployed
REVISION: 1
TEST SUITE: None
NOTES:
MinIO can be accessed via port 9000 on the following DNS name from within your cluster:
minio-service.pitr-physical-backup-source-28616.cluster.local

To access MinIO from localhost, run the below commands:

  1. export POD_NAME=$(kubectl get pods --namespace pitr-physical-backup-source-28616 -l "release=minio-service" -o jsonpath="{.items[0].metadata.name}")

  2. kubectl port-forward $POD_NAME 9000 --namespace pitr-physical-backup-source-28616

Read more about port forwarding here: http://kubernetes.io/docs/user-guide/kubectl/kubectl_port-forward/

You can now access MinIO server on http://localhost:9000. Follow the below steps to connect to MinIO server with mc client:

  1. Download the MinIO mc client - https://min.io/docs/minio/linux/reference/minio-mc.html#quickstart

  2. export MC_HOST_minio-service-local=http://$(kubectl get secret --namespace pitr-physical-backup-source-28616 minio-service -o jsonpath="{.data.rootUser}" | base64 --decode):$(kubectl get secret --namespace pitr-physical-backup-source-28616 minio-service -o jsonpath="{.data.rootPassword}" | base64 --decode)@localhost:9000

  3. mc ls minio-service-local
++ kubectl_bin get pods --selector=release=minio-service -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.Bpi1gaCVYb
+++ mktemp
++ local LAST_ERR=/tmp/tmp.zTWABI0HKp
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=release=minio-service -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.Bpi1gaCVYb
++ cat /tmp/tmp.zTWABI0HKp
++ rm /tmp/tmp.Bpi1gaCVYb /tmp/tmp.zTWABI0HKp
++ return 0
+ local MINIO_POD=minio-service-6d5f646cdc-9s5bv
+ wait_pod minio-service-6d5f646cdc-9s5bv
+ local pod=minio-service-6d5f646cdc-9s5bv
+ set +o xtrace
waiting for pod/minio-service-6d5f646cdc-9s5bv to be ready.OK
+ '[' -n psmdb-operator ']'
+ kubectl_bin create svc -n psmdb-operator externalname minio-service --external-name=minio-service.pitr-physical-backup-source-28616.svc.cluster.local --tcp=9000
service/minio-service created
+ create_minio_bucket operator-testing http://minio-service:9000
+ local bucket=operator-testing
+ local endpoint=http://minio-service:9000
+ kubectl_bin run -i --rm aws-cli --image=docker.io/perconalab/awscli --restart=Never -- bash -c $'AWS_ACCESS_KEY_ID=some-access-key \t\tAWS_SECRET_ACCESS_KEY=some-secret-key \t\tAWS_DEFAULT_REGION=us-east-1         /usr/bin/aws --no-verify-ssl --endpoint-url http://minio-service:9000 s3 mb s3://operator-testing'
++ mktemp
+ local LAST_OUT=/tmp/tmp.UYoqAFkdDi
++ mktemp
+ local LAST_ERR=/tmp/tmp.1ObA64g3nF
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl run -i --rm aws-cli --image=docker.io/perconalab/awscli --restart=Never -- bash -c $'AWS_ACCESS_KEY_ID=some-access-key \t\tAWS_SECRET_ACCESS_KEY=some-secret-key \t\tAWS_DEFAULT_REGION=us-east-1         /usr/bin/aws --no-verify-ssl --endpoint-url http://minio-service:9000 s3 mb s3://operator-testing'
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.UYoqAFkdDi
make_bucket: operator-testing
make_bucket: operator-testing
pod "aws-cli" deleted from pitr-physical-backup-source-28616 namespace
+ cat /tmp/tmp.1ObA64g3nF
All commands and output from this session will be recorded in container logs, including credentials and sensitive information passed through the command prompt.
If you don't see a command prompt, try pressing enter.
warning: couldn't attach to pod/aws-cli, falling back to streaming logs: Internal error occurred: unable to upgrade connection: container aws-cli not found in pod aws-cli_pitr-physical-backup-source-28616
+ rm /tmp/tmp.UYoqAFkdDi /tmp/tmp.1ObA64g3nF
+ return 0
+ desc 'create secrets and start client'
+ set +o xtrace


-----------------------------------------------------------------------------------
create secrets and start client
-----------------------------------------------------------------------------------

+ kubectl_bin apply -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2406/e2e-tests/conf/secrets.yml -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2406/e2e-tests/conf/client.yml -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2406/e2e-tests/conf/minio-secret.yml
++ mktemp
+ local LAST_OUT=/tmp/tmp.3CnnJO6FvP
++ mktemp
+ local LAST_ERR=/tmp/tmp.FA5ayhQ2Rz
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl apply -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2406/e2e-tests/conf/secrets.yml -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2406/e2e-tests/conf/client.yml -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2406/e2e-tests/conf/minio-secret.yml
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.3CnnJO6FvP
secret/some-users created
deployment.apps/psmdb-client created
secret/minio-secret created
+ cat /tmp/tmp.FA5ayhQ2Rz
+ rm /tmp/tmp.3CnnJO6FvP /tmp/tmp.FA5ayhQ2Rz
+ return 0
+ cluster=some-name
+ desc 'create first PSMDB cluster some-name'
+ set +o xtrace


-----------------------------------------------------------------------------------
create first PSMDB cluster some-name
-----------------------------------------------------------------------------------

+ apply_cluster /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2406/e2e-tests/pitr-physical-backup-source/conf/some-name-rs0.yml
+ '[' -z '' ']'
+ cat_config /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2406/e2e-tests/pitr-physical-backup-source/conf/some-name-rs0.yml
+ kubectl_bin apply -f -
+ cat /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2406/e2e-tests/pitr-physical-backup-source/conf/some-name-rs0.yml
+ yq eval '(.spec | select(.image == null)).image = "docker.io/perconalab/percona-server-mongodb-operator:main-mongod8.0"'
++ mktemp
+ yq eval '(.spec | select(has("pmm"))).pmm.image = "docker.io/percona/pmm-client:2.44.1-1"'
+ yq eval '(.spec | select(has("initImage"))).initImage = "docker.io/perconalab/percona-server-mongodb-operator:PR-2406-1c5b15f05"'
+ yq eval '(.spec | select(has("backup"))).backup.image = "docker.io/perconalab/percona-server-mongodb-operator:main-backup"'
+ local LAST_OUT=/tmp/tmp.nCbPFWDT2y
++ mktemp
+ yq eval '(.spec | select(has("search"))).search.image = "mongodb/mongodb-community-search:latest"'
+ /usr/sbin/sed -e s/NAME_SPACE/pitr-physical-backup-source-28616/g
+ yq eval '.spec.upgradeOptions.apply="Never"'
+ local LAST_ERR=/tmp/tmp.OZ6tPQo1Op
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl apply -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.nCbPFWDT2y
perconaservermongodb.psmdb.percona.com/some-name created
+ cat /tmp/tmp.OZ6tPQo1Op
+ rm /tmp/tmp.nCbPFWDT2y /tmp/tmp.OZ6tPQo1Op
+ return 0
+ desc 'check if all 3 Pods started'
+ set +o xtrace


-----------------------------------------------------------------------------------
check if all 3 Pods started
-----------------------------------------------------------------------------------

+ wait_for_running some-name-rs0 3
+ local name=some-name-rs0
+ let last_pod=2
+ local check_cluster_readyness=true
+ set_debug
+ [[ 1 == 1 ]]
+ set -o xtrace
+ local rs_name=rs0
+ local cluster_name=some-name
++ seq 0 2
+ for i in $(seq 0 $last_pod)
+ [[ 0 -eq 2 ]]
+ wait_pod some-name-rs0-0
+ local pod=some-name-rs0-0
+ set +o xtrace
waiting for pod/some-name-rs0-0 to be ready......OK
+ for i in $(seq 0 $last_pod)
+ [[ 1 -eq 2 ]]
+ wait_pod some-name-rs0-1
+ local pod=some-name-rs0-1
+ set +o xtrace
waiting for pod/some-name-rs0-1 to be ready......OK
+ for i in $(seq 0 $last_pod)
+ [[ 2 -eq 2 ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].arbiter.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.SiJptKRJTA
+++ mktemp
++ local LAST_ERR=/tmp/tmp.Yaqqu0HFZW
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].arbiter.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.SiJptKRJTA
++ cat /tmp/tmp.Yaqqu0HFZW
++ rm /tmp/tmp.SiJptKRJTA /tmp/tmp.Yaqqu0HFZW
++ return 0
+ [[ '' == true ]]
+ wait_pod some-name-rs0-2
+ local pod=some-name-rs0-2
+ set +o xtrace
waiting for pod/some-name-rs0-2 to be ready.......OK
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].nonvoting.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.QrrTEEExTh
+++ mktemp
++ local LAST_ERR=/tmp/tmp.altN0BuekV
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].nonvoting.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.QrrTEEExTh
++ cat /tmp/tmp.altN0BuekV
++ rm /tmp/tmp.QrrTEEExTh /tmp/tmp.altN0BuekV
++ return 0
+ [[ '' == true ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].hidden.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.864Ia8VBSg
+++ mktemp
++ local LAST_ERR=/tmp/tmp.EYKeFaCb4X
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].hidden.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.864Ia8VBSg
++ cat /tmp/tmp.EYKeFaCb4X
++ rm /tmp/tmp.864Ia8VBSg /tmp/tmp.EYKeFaCb4X
++ return 0
+ [[ '' == true ]]
+ sleep 10
+ [[ true == true ]]
+ set +x
Waiting for cluster readyness..........................................
+ wait_for_running some-name-cfg 3 false
+ local name=some-name-cfg
+ let last_pod=2
+ local check_cluster_readyness=false
+ set_debug
+ [[ 1 == 1 ]]
+ set -o xtrace
+ local rs_name=cfg
+ local cluster_name=some-name
++ seq 0 2
+ for i in $(seq 0 $last_pod)
+ [[ 0 -eq 2 ]]
+ wait_pod some-name-cfg-0
+ local pod=some-name-cfg-0
+ set +o xtrace
waiting for pod/some-name-cfg-0 to be ready.OK
+ for i in $(seq 0 $last_pod)
+ [[ 1 -eq 2 ]]
+ wait_pod some-name-cfg-1
+ local pod=some-name-cfg-1
+ set +o xtrace
waiting for pod/some-name-cfg-1 to be ready.OK
+ for i in $(seq 0 $last_pod)
+ [[ 2 -eq 2 ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].arbiter.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.UAIg42QyQj
+++ mktemp
++ local LAST_ERR=/tmp/tmp.RXzyOPCX21
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].arbiter.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.UAIg42QyQj
++ cat /tmp/tmp.RXzyOPCX21
++ rm /tmp/tmp.UAIg42QyQj /tmp/tmp.RXzyOPCX21
++ return 0
+ [[ '' == true ]]
+ wait_pod some-name-cfg-2
+ local pod=some-name-cfg-2
+ set +o xtrace
waiting for pod/some-name-cfg-2 to be ready.OK
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].nonvoting.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.P9wwJZl24q
+++ mktemp
++ local LAST_ERR=/tmp/tmp.4PnvEtnq7k
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].nonvoting.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.P9wwJZl24q
++ cat /tmp/tmp.4PnvEtnq7k
++ rm /tmp/tmp.P9wwJZl24q /tmp/tmp.4PnvEtnq7k
++ return 0
+ [[ '' == true ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].hidden.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.RHFNXndPxI
+++ mktemp
++ local LAST_ERR=/tmp/tmp.autYHUXX6X
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].hidden.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.RHFNXndPxI
++ cat /tmp/tmp.autYHUXX6X
++ rm /tmp/tmp.RHFNXndPxI /tmp/tmp.autYHUXX6X
++ return 0
+ [[ '' == true ]]
+ sleep 10
+ [[ false == true ]]
+ sleep 10
+ write_initial_data
+ desc 'create user myApp'
+ set +o xtrace


-----------------------------------------------------------------------------------
create user myApp
-----------------------------------------------------------------------------------

+ run_mongos 'db.createUser({user:"myApp",pwd:"myPass",roles:[{db:"myApp",role:"readWrite"}]})' userAdmin:userAdmin123456@some-name-mongos.pitr-physical-backup-source-28616
+ local 'command=db.createUser({user:"myApp",pwd:"myPass",roles:[{db:"myApp",role:"readWrite"}]})'
+ local uri=userAdmin:userAdmin123456@some-name-mongos.pitr-physical-backup-source-28616
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local port=27017
+ local mongo_bin=mongo
++ echo .svc.cluster.local
++ awk -F: '{print $2}'
+ suffix_port=
+ [[ -z '' ]]
+ suffix=.svc.cluster.local:27017
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.5hyBVLehNV
+++ mktemp
++ local LAST_ERR=/tmp/tmp.YmR2nlyes2
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.5hyBVLehNV
++ cat /tmp/tmp.YmR2nlyes2
++ rm /tmp/tmp.5hyBVLehNV /tmp/tmp.YmR2nlyes2
++ return 0
+ local client_container=psmdb-client-bb8b97679-8kmm8
+ kubectl_bin exec psmdb-client-bb8b97679-8kmm8 -- bash -c 'printf '\''db.createUser({user:"myApp",pwd:"myPass",roles:[{db:"myApp",role:"readWrite"}]})\n'\'' | mongo mongodb://userAdmin:userAdmin123456@some-name-mongos.pitr-physical-backup-source-28616.svc.cluster.local:27017/admin '
++ mktemp
+ local LAST_OUT=/tmp/tmp.lftEkPUdgn
++ mktemp
+ local LAST_ERR=/tmp/tmp.XWceOl2PvL
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-bb8b97679-8kmm8 -- bash -c 'printf '\''db.createUser({user:"myApp",pwd:"myPass",roles:[{db:"myApp",role:"readWrite"}]})\n'\'' | mongo mongodb://userAdmin:userAdmin123456@some-name-mongos.pitr-physical-backup-source-28616.svc.cluster.local:27017/admin '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.lftEkPUdgn
Percona Server for MongoDB shell version v4.4.29-28
connecting to: mongodb://some-name-mongos.pitr-physical-backup-source-28616.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb
Implicit session: session { "id" : UUID("c519807d-da3e-49a4-8cbb-788a25e8b89d") }
Percona Server for MongoDB server version: v8.0.23-10
WARNING: shell and server versions do not match
Successfully added user: {
	"user" : "myApp",
	"roles" : [
		{
			"db" : "myApp",
			"role" : "readWrite"
		}
	]
}
bye
+ cat /tmp/tmp.XWceOl2PvL
+ rm /tmp/tmp.lftEkPUdgn /tmp/tmp.XWceOl2PvL
+ return 0
+ sleep 2
+ write_document
+ local cmp_postfix=
+ local sleep_value=0
+ log 'write initial data, read from all'
+ set +o xtrace
[2026-06-18T20:55:41+0000] write initial data, read from all
+ run_mongos 'use myApp\n db.test.insert({ x: 100500 })' myApp:myPass@some-name-mongos.pitr-physical-backup-source-28616
+ local 'command=use myApp\n db.test.insert({ x: 100500 })'
+ local uri=myApp:myPass@some-name-mongos.pitr-physical-backup-source-28616
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local port=27017
+ local mongo_bin=mongo
++ echo .svc.cluster.local
++ awk -F: '{print $2}'
+ suffix_port=
+ [[ -z '' ]]
+ suffix=.svc.cluster.local:27017
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.hM1EEsSIfN
+++ mktemp
++ local LAST_ERR=/tmp/tmp.do0jr6lnFg
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.hM1EEsSIfN
++ cat /tmp/tmp.do0jr6lnFg
++ rm /tmp/tmp.hM1EEsSIfN /tmp/tmp.do0jr6lnFg
++ return 0
+ local client_container=psmdb-client-bb8b97679-8kmm8
+ kubectl_bin exec psmdb-client-bb8b97679-8kmm8 -- bash -c 'printf '\''use myApp\n db.test.insert({ x: 100500 })\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.pitr-physical-backup-source-28616.svc.cluster.local:27017/admin '
++ mktemp
+ local LAST_OUT=/tmp/tmp.kx3UOy0t5j
++ mktemp
+ local LAST_ERR=/tmp/tmp.0xJLYMmd8W
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-bb8b97679-8kmm8 -- bash -c 'printf '\''use myApp\n db.test.insert({ x: 100500 })\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.pitr-physical-backup-source-28616.svc.cluster.local:27017/admin '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.kx3UOy0t5j
Percona Server for MongoDB shell version v4.4.29-28
connecting to: mongodb://some-name-mongos.pitr-physical-backup-source-28616.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb
Implicit session: session { "id" : UUID("b0ed49de-9788-4783-817f-69e04dae74de") }
Percona Server for MongoDB server version: v8.0.23-10
WARNING: shell and server versions do not match
switched to db myApp
WriteResult({ "nInserted" : 1 })
bye
+ cat /tmp/tmp.0xJLYMmd8W
+ rm /tmp/tmp.kx3UOy0t5j /tmp/tmp.0xJLYMmd8W
+ return 0
+ sleep 0
+ compare_mongos_cmd find myApp:myPass@some-name-mongos.pitr-physical-backup-source-28616
+ local command=find
+ local uri=myApp:myPass@some-name-mongos.pitr-physical-backup-source-28616
+ local postfix=
+ local suffix=
+ local database=myApp
+ local collection=test
+ local port=27017
+ local tls=false
+ [[ false == true ]]
+ mongos_command=run_mongos
+ log 'running db.test.command() in myApp'
+ set +o xtrace
[2026-06-18T20:55:44+0000] running db.test.command() in myApp
+ run_mongos 'use myApp\n db.test.find()' myApp:myPass@some-name-mongos.pitr-physical-backup-source-28616 mongodb '' '' 27017
+ local 'command=use myApp\n db.test.find()'
+ local uri=myApp:myPass@some-name-mongos.pitr-physical-backup-source-28616
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local port=27017
+ local mongo_bin=mongo
+ grep -E -v 'I NETWORK|W NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|Started a new thread for the timer service'
+ /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
++ echo .svc.cluster.local
++ awk -F: '{print $2}'
+ suffix_port=
+ [[ -z '' ]]
+ suffix=.svc.cluster.local:27017
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.P8Gp9Jv6Bi
+++ mktemp
++ local LAST_ERR=/tmp/tmp.iluMsDAm2q
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.P8Gp9Jv6Bi
++ cat /tmp/tmp.iluMsDAm2q
++ rm /tmp/tmp.P8Gp9Jv6Bi /tmp/tmp.iluMsDAm2q
++ return 0
+ local client_container=psmdb-client-bb8b97679-8kmm8
+ kubectl_bin exec psmdb-client-bb8b97679-8kmm8 -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.pitr-physical-backup-source-28616.svc.cluster.local:27017/admin '
++ mktemp
+ local LAST_OUT=/tmp/tmp.WuQOKqH4si
++ mktemp
+ local LAST_ERR=/tmp/tmp.6OuHiIkMVP
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-bb8b97679-8kmm8 -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.pitr-physical-backup-source-28616.svc.cluster.local:27017/admin '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.WuQOKqH4si
+ cat /tmp/tmp.6OuHiIkMVP
+ rm /tmp/tmp.WuQOKqH4si /tmp/tmp.6OuHiIkMVP
+ return 0
+ [[ 0 -eq 0 ]]
+ diff /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2406/e2e-tests/pitr-physical-backup-source/compare/find.json /tmp/tmp.4E31yTnNi1/find
+ wait_backup_agent some-name-rs0-0
+ local agent_pod=some-name-rs0-0
+ set +o xtrace
waiting for pbm-agent to be ready in some-name-rs0-0...2026-06-18T20:55:03.000+0000 I listening for the commands

+ wait_backup_agent some-name-rs0-1
+ local agent_pod=some-name-rs0-1
+ set +o xtrace
waiting for pbm-agent to be ready in some-name-rs0-1...2026-06-18T20:55:08.000+0000 I listening for the commands

+ wait_backup_agent some-name-rs0-2
+ local agent_pod=some-name-rs0-2
+ set +o xtrace
waiting for pbm-agent to be ready in some-name-rs0-2...2026-06-18T20:55:12.000+0000 I listening for the commands

+ wait_backup_agent some-name-rs1-0
+ local agent_pod=some-name-rs1-0
+ set +o xtrace
waiting for pbm-agent to be ready in some-name-rs1-0...2026-06-18T20:55:05.000+0000 I listening for the commands

+ wait_backup_agent some-name-rs1-1
+ local agent_pod=some-name-rs1-1
+ set +o xtrace
waiting for pbm-agent to be ready in some-name-rs1-1...2026-06-18T20:55:09.000+0000 I listening for the commands

+ wait_backup_agent some-name-rs1-2
+ local agent_pod=some-name-rs1-2
+ set +o xtrace
waiting for pbm-agent to be ready in some-name-rs1-2...2026-06-18T20:55:12.000+0000 I listening for the commands

+ wait_backup_agent some-name-rs2-0
+ local agent_pod=some-name-rs2-0
+ set +o xtrace
waiting for pbm-agent to be ready in some-name-rs2-0...2026-06-18T20:55:08.000+0000 I listening for the commands

+ wait_backup_agent some-name-rs2-1
+ local agent_pod=some-name-rs2-1
+ set +o xtrace
waiting for pbm-agent to be ready in some-name-rs2-1...2026-06-18T20:55:10.000+0000 I listening for the commands

+ wait_backup_agent some-name-rs2-2
+ local agent_pod=some-name-rs2-2
+ set +o xtrace
waiting for pbm-agent to be ready in some-name-rs2-2...2026-06-18T20:55:13.000+0000 I listening for the commands

+ echo 'Sleeping for 360 seconds'
Sleeping for 360 seconds
+ sleep 360
+ backup_name_minio=backup-minio
+ desc 'restore pitr type date using backupSource'
+ set +o xtrace


-----------------------------------------------------------------------------------
restore pitr type date using backupSource
-----------------------------------------------------------------------------------

+ run_backup backup-minio 1 physical
+ local name=backup-minio
+ local idx=1
+ local type=physical
+ desc 'run backup backup-minio-1'
+ set +o xtrace


-----------------------------------------------------------------------------------
run backup backup-minio-1
-----------------------------------------------------------------------------------

+ cat /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2406/e2e-tests/pitr-physical-backup-source/conf/backup-minio.yml
+ /usr/sbin/sed -e 's/type:/type: physical/'
+ /usr/sbin/sed -e 's/name:/name: backup-minio-1/'
+ kubectl_bin apply -f -
++ mktemp
+ local LAST_OUT=/tmp/tmp.tCPN9gVafX
++ mktemp
+ local LAST_ERR=/tmp/tmp.msjHVvDllf
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl apply -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.tCPN9gVafX
perconaservermongodbbackup.psmdb.percona.com/backup-minio-1 created
+ cat /tmp/tmp.msjHVvDllf
+ rm /tmp/tmp.tCPN9gVafX /tmp/tmp.msjHVvDllf
+ return 0
+ wait_backup backup-minio-1
+ local backup_name=backup-minio-1
+ local target_state=ready
+ set +o xtrace
waiting for backup-minio-1 to reach ready state..........OK
+ compare_latest_restorable_time some-name-rs0 backup-minio-1
+ local cluster=some-name-rs0
+ local backup_name=backup-minio-1
+ local latest_restorable_time
+ local backup_time
++ get_latest_restorable_time some-name-rs0
++ local cluster=some-name-rs0
++ local first_timestamp
++ local second_timestamp
++ local retry=0
++ [[ '' != '' ]]
+++ kubectl_bin exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ jq '.backups.pitrChunks.pitrChunks | last | .range.end'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.3J7W8tLrcd
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.qay7lGvESk
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in $(seq 0 2)
+++ set +e
+++ kubectl exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.3J7W8tLrcd
+++ cat /tmp/tmp.qay7lGvESk
+++ rm /tmp/tmp.3J7W8tLrcd /tmp/tmp.qay7lGvESk
+++ return 0
++ first_timestamp=null
++ sleep 5
++ [[ null != '' ]]
++ [[ null != null ]]
++ let retry+=1
++ [[ 1 -gt 30 ]]
++ [[ null != '' ]]
++ [[ null != null ]]
+++ kubectl_bin exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ jq '.backups.pitrChunks.pitrChunks | last | .range.end'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.1RkTkxDghQ
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.yj0C4bnEOY
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in $(seq 0 2)
+++ set +e
+++ kubectl exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.1RkTkxDghQ
+++ cat /tmp/tmp.yj0C4bnEOY
+++ rm /tmp/tmp.1RkTkxDghQ /tmp/tmp.yj0C4bnEOY
+++ return 0
++ first_timestamp=null
++ sleep 5
++ [[ null != '' ]]
++ [[ null != null ]]
++ let retry+=1
++ [[ 2 -gt 30 ]]
++ [[ null != '' ]]
++ [[ null != null ]]
+++ kubectl_bin exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ jq '.backups.pitrChunks.pitrChunks | last | .range.end'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.QmvMvKpyeB
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.yRVdDq6IIv
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in $(seq 0 2)
+++ set +e
+++ kubectl exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.QmvMvKpyeB
+++ cat /tmp/tmp.yRVdDq6IIv
+++ rm /tmp/tmp.QmvMvKpyeB /tmp/tmp.yRVdDq6IIv
+++ return 0
++ first_timestamp=null
++ sleep 5
++ [[ null != '' ]]
++ [[ null != null ]]
++ let retry+=1
++ [[ 3 -gt 30 ]]
++ [[ null != '' ]]
++ [[ null != null ]]
+++ kubectl_bin exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ jq '.backups.pitrChunks.pitrChunks | last | .range.end'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.mZ3EeSDIfl
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.U1rr0t2FS1
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in $(seq 0 2)
+++ set +e
+++ kubectl exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.mZ3EeSDIfl
+++ cat /tmp/tmp.U1rr0t2FS1
+++ rm /tmp/tmp.mZ3EeSDIfl /tmp/tmp.U1rr0t2FS1
+++ return 0
++ first_timestamp=null
++ sleep 5
++ [[ null != '' ]]
++ [[ null != null ]]
++ let retry+=1
++ [[ 4 -gt 30 ]]
++ [[ null != '' ]]
++ [[ null != null ]]
+++ kubectl_bin exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ jq '.backups.pitrChunks.pitrChunks | last | .range.end'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.CPO2bm9wvZ
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.wWTAacAc6z
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in $(seq 0 2)
+++ set +e
+++ kubectl exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.CPO2bm9wvZ
+++ cat /tmp/tmp.wWTAacAc6z
+++ rm /tmp/tmp.CPO2bm9wvZ /tmp/tmp.wWTAacAc6z
+++ return 0
++ first_timestamp=1781816554
++ sleep 5
++ [[ 1781816554 != '' ]]
++ [[ 1781816554 != null ]]
+++ kubectl_bin exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ jq '.backups.pitrChunks.pitrChunks | last | .range.end'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.YZmZ56eVR4
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.8DuKoBBrZo
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in $(seq 0 2)
+++ set +e
+++ kubectl exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.YZmZ56eVR4
+++ cat /tmp/tmp.8DuKoBBrZo
+++ rm /tmp/tmp.YZmZ56eVR4 /tmp/tmp.8DuKoBBrZo
+++ return 0
++ second_timestamp=1781816554
++ let retry+=1
++ [[ 5 -gt 30 ]]
++ [[ 1781816554 != '' ]]
++ [[ 1781816554 != null ]]
++ [[ 1781816554 == 1781816554 ]]
++ /usr/sbin/date -u -d @1781816554 +%Y-%m-%dT%H:%M:%SZ
+ latest_restorable_time=2026-06-18T21:02:34Z
++ get_latest_restorable_time_from_backup_object backup-minio-1
++ local backup_name=backup-minio-1
++ local latestRestorableTime
++ local retry=0
++ [[ '' != '' ]]
++ sleep 5
+++ kubectl_bin get psmdb-backup backup-minio-1 -o 'jsonpath={.status.latestRestorableTime}'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.KJGpi8yKjr
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.XSuvbv9AfF
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in $(seq 0 2)
+++ set +e
+++ kubectl get psmdb-backup backup-minio-1 -o 'jsonpath={.status.latestRestorableTime}'
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.KJGpi8yKjr
+++ cat /tmp/tmp.XSuvbv9AfF
+++ rm /tmp/tmp.KJGpi8yKjr /tmp/tmp.XSuvbv9AfF
+++ return 0
++ latestRestorableTime=2026-06-18T21:02:34Z
++ let retry+=1
++ [[ 1 -gt 30 ]]
++ [[ 2026-06-18T21:02:34Z != '' ]]
++ [[ 2026-06-18T21:02:34Z != null ]]
++ echo 2026-06-18T21:02:34Z
+ backup_time=2026-06-18T21:02:34Z
+ [[ 2026-06-18T21:02:34Z != 2026\-06\-18T21\:02\:34Z ]]
+ reset_collection
+ desc 'reset data'
+ set +o xtrace


-----------------------------------------------------------------------------------
reset data
-----------------------------------------------------------------------------------

+ run_mongos 'use myApp\n db.test.remove({})' myApp:myPass@some-name-mongos.pitr-physical-backup-source-28616
+ local 'command=use myApp\n db.test.remove({})'
+ local uri=myApp:myPass@some-name-mongos.pitr-physical-backup-source-28616
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local port=27017
+ local mongo_bin=mongo
++ echo .svc.cluster.local
++ awk -F: '{print $2}'
+ suffix_port=
+ [[ -z '' ]]
+ suffix=.svc.cluster.local:27017
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.vWooCOoEga
+++ mktemp
++ local LAST_ERR=/tmp/tmp.NXVW9nJgTN
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.vWooCOoEga
++ cat /tmp/tmp.NXVW9nJgTN
++ rm /tmp/tmp.vWooCOoEga /tmp/tmp.NXVW9nJgTN
++ return 0
+ local client_container=psmdb-client-bb8b97679-8kmm8
+ kubectl_bin exec psmdb-client-bb8b97679-8kmm8 -- bash -c 'printf '\''use myApp\n db.test.remove({})\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.pitr-physical-backup-source-28616.svc.cluster.local:27017/admin '
++ mktemp
+ local LAST_OUT=/tmp/tmp.jsFz5Irtdd
++ mktemp
+ local LAST_ERR=/tmp/tmp.iXEwicmMor
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-bb8b97679-8kmm8 -- bash -c 'printf '\''use myApp\n db.test.remove({})\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.pitr-physical-backup-source-28616.svc.cluster.local:27017/admin '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.jsFz5Irtdd
Percona Server for MongoDB shell version v4.4.29-28
connecting to: mongodb://some-name-mongos.pitr-physical-backup-source-28616.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb
Implicit session: session { "id" : UUID("f42926d7-a8b3-434a-9783-7e9ee356921b") }
Percona Server for MongoDB server version: v8.0.23-10
WARNING: shell and server versions do not match
switched to db myApp
WriteResult({ "nRemoved" : 1 })
bye
+ cat /tmp/tmp.iXEwicmMor
+ rm /tmp/tmp.jsFz5Irtdd /tmp/tmp.iXEwicmMor
+ return 0
+ sleep 2
+ write_document '' 120
+ local cmp_postfix=
+ local sleep_value=120
+ log 'write initial data, read from all'
+ set +o xtrace
[2026-06-18T21:03:02+0000] write initial data, read from all
+ run_mongos 'use myApp\n db.test.insert({ x: 100500 })' myApp:myPass@some-name-mongos.pitr-physical-backup-source-28616
+ local 'command=use myApp\n db.test.insert({ x: 100500 })'
+ local uri=myApp:myPass@some-name-mongos.pitr-physical-backup-source-28616
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local port=27017
+ local mongo_bin=mongo
++ echo .svc.cluster.local
++ awk -F: '{print $2}'
+ suffix_port=
+ [[ -z '' ]]
+ suffix=.svc.cluster.local:27017
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.zdtnBfidtt
+++ mktemp
++ local LAST_ERR=/tmp/tmp.naL0fZfZs5
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.zdtnBfidtt
++ cat /tmp/tmp.naL0fZfZs5
++ rm /tmp/tmp.zdtnBfidtt /tmp/tmp.naL0fZfZs5
++ return 0
+ local client_container=psmdb-client-bb8b97679-8kmm8
+ kubectl_bin exec psmdb-client-bb8b97679-8kmm8 -- bash -c 'printf '\''use myApp\n db.test.insert({ x: 100500 })\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.pitr-physical-backup-source-28616.svc.cluster.local:27017/admin '
++ mktemp
+ local LAST_OUT=/tmp/tmp.04zVtSIbJb
++ mktemp
+ local LAST_ERR=/tmp/tmp.Uvtp3FNIVN
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-bb8b97679-8kmm8 -- bash -c 'printf '\''use myApp\n db.test.insert({ x: 100500 })\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.pitr-physical-backup-source-28616.svc.cluster.local:27017/admin '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.04zVtSIbJb
Percona Server for MongoDB shell version v4.4.29-28
connecting to: mongodb://some-name-mongos.pitr-physical-backup-source-28616.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb
Implicit session: session { "id" : UUID("f015ee27-2ecd-4216-aa0c-b946a18fb37d") }
Percona Server for MongoDB server version: v8.0.23-10
WARNING: shell and server versions do not match
switched to db myApp
WriteResult({ "nInserted" : 1 })
bye
+ cat /tmp/tmp.Uvtp3FNIVN
+ rm /tmp/tmp.04zVtSIbJb /tmp/tmp.Uvtp3FNIVN
+ return 0
+ sleep 120
+ compare_mongos_cmd find myApp:myPass@some-name-mongos.pitr-physical-backup-source-28616
+ local command=find
+ local uri=myApp:myPass@some-name-mongos.pitr-physical-backup-source-28616
+ local postfix=
+ local suffix=
+ local database=myApp
+ local collection=test
+ local port=27017
+ local tls=false
+ [[ false == true ]]
+ mongos_command=run_mongos
+ log 'running db.test.command() in myApp'
+ set +o xtrace
[2026-06-18T21:05:04+0000] running db.test.command() in myApp
+ run_mongos 'use myApp\n db.test.find()' myApp:myPass@some-name-mongos.pitr-physical-backup-source-28616 mongodb '' '' 27017
+ local 'command=use myApp\n db.test.find()'
+ local uri=myApp:myPass@some-name-mongos.pitr-physical-backup-source-28616
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ grep -E -v 'I NETWORK|W NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|Started a new thread for the timer service'
+ local port=27017
+ local mongo_bin=mongo
+ /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
++ echo .svc.cluster.local
++ awk -F: '{print $2}'
+ suffix_port=
+ [[ -z '' ]]
+ suffix=.svc.cluster.local:27017
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.8xnDinLKRQ
+++ mktemp
++ local LAST_ERR=/tmp/tmp.YgDqmGbbxa
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.8xnDinLKRQ
++ cat /tmp/tmp.YgDqmGbbxa
++ rm /tmp/tmp.8xnDinLKRQ /tmp/tmp.YgDqmGbbxa
++ return 0
+ local client_container=psmdb-client-bb8b97679-8kmm8
+ kubectl_bin exec psmdb-client-bb8b97679-8kmm8 -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.pitr-physical-backup-source-28616.svc.cluster.local:27017/admin '
++ mktemp
+ local LAST_OUT=/tmp/tmp.Ox7mtSZdqh
++ mktemp
+ local LAST_ERR=/tmp/tmp.snZgAomDE4
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-bb8b97679-8kmm8 -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.pitr-physical-backup-source-28616.svc.cluster.local:27017/admin '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.Ox7mtSZdqh
+ cat /tmp/tmp.snZgAomDE4
+ rm /tmp/tmp.Ox7mtSZdqh /tmp/tmp.snZgAomDE4
+ return 0
+ [[ 0 -eq 0 ]]
+ diff /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2406/e2e-tests/pitr-physical-backup-source/compare/find.json /tmp/tmp.4E31yTnNi1/find
++ run_mongos 'new Date().getTime() / 1000' myApp:myPass@some-name-mongos.pitr-physical-backup-source-28616 mongodb '' --quiet
++ local 'command=new Date().getTime() / 1000'
++ local uri=myApp:myPass@some-name-mongos.pitr-physical-backup-source-28616
++ local driver=mongodb
++ local suffix=.svc.cluster.local
++ local mongo_flag=--quiet
++ local port=27017
++ local mongo_bin=mongo
++ grep -E -v 'I NETWORK|W NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match'
++ cut -d. -f1
+++ echo .svc.cluster.local
+++ awk -F: '{print $2}'
++ suffix_port=
++ [[ -z '' ]]
++ suffix=.svc.cluster.local:27017
+++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.9flqZg6Hwb
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.thQfgSE9ry
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in $(seq 0 2)
+++ set +e
+++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.9flqZg6Hwb
+++ cat /tmp/tmp.thQfgSE9ry
+++ rm /tmp/tmp.9flqZg6Hwb /tmp/tmp.thQfgSE9ry
+++ return 0
++ local client_container=psmdb-client-bb8b97679-8kmm8
++ kubectl_bin exec psmdb-client-bb8b97679-8kmm8 -- bash -c 'printf '\''new Date().getTime() / 1000\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.pitr-physical-backup-source-28616.svc.cluster.local:27017/admin --quiet'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.G6cBOtGLI2
+++ mktemp
++ local LAST_ERR=/tmp/tmp.z4hSSTEeEe
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl exec psmdb-client-bb8b97679-8kmm8 -- bash -c 'printf '\''new Date().getTime() / 1000\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.pitr-physical-backup-source-28616.svc.cluster.local:27017/admin --quiet'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.G6cBOtGLI2
++ cat /tmp/tmp.z4hSSTEeEe
++ rm /tmp/tmp.G6cBOtGLI2 /tmp/tmp.z4hSSTEeEe
++ return 0
+ time_now=1781816709
+ check_recovery backup-minio-1 date 1781816709 '' some-name backupSource
+ local backup_name=backup-minio-1
+ local restore_type=date
+ local restore_date=1781816709
+ local cmp_postfix=
+ local cluster_name=some-name
+ local backupSource=backupSource
++ get_latest_oplog_chunk_ts some-name
++ local cluster=some-name
+++ kubectl_bin exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ jq '.backups.pitrChunks.pitrChunks | last | .range.end'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.TOFbluvFmG
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.p4hhloGZsZ
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in $(seq 0 2)
+++ set +e
+++ kubectl exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.TOFbluvFmG
+++ cat /tmp/tmp.p4hhloGZsZ
+++ rm /tmp/tmp.TOFbluvFmG /tmp/tmp.p4hhloGZsZ
+++ return 0
++ echo 1781816681
+ local latest_ts=1781816681
+ desc 'write more data before restore by date'
+ set +o xtrace


-----------------------------------------------------------------------------------
write more data before restore by date
-----------------------------------------------------------------------------------

+ run_mongos 'use myApp\n db.test.insert({ x: 100501 })' myApp:myPass@some-name-mongos.pitr-physical-backup-source-28616
+ local 'command=use myApp\n db.test.insert({ x: 100501 })'
+ local uri=myApp:myPass@some-name-mongos.pitr-physical-backup-source-28616
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local port=27017
+ local mongo_bin=mongo
++ echo .svc.cluster.local
++ awk -F: '{print $2}'
+ suffix_port=
+ [[ -z '' ]]
+ suffix=.svc.cluster.local:27017
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.ZFUANVWNvG
+++ mktemp
++ local LAST_ERR=/tmp/tmp.Koh5sraSKz
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.ZFUANVWNvG
++ cat /tmp/tmp.Koh5sraSKz
++ rm /tmp/tmp.ZFUANVWNvG /tmp/tmp.Koh5sraSKz
++ return 0
+ local client_container=psmdb-client-bb8b97679-8kmm8
+ kubectl_bin exec psmdb-client-bb8b97679-8kmm8 -- bash -c 'printf '\''use myApp\n db.test.insert({ x: 100501 })\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.pitr-physical-backup-source-28616.svc.cluster.local:27017/admin '
++ mktemp
+ local LAST_OUT=/tmp/tmp.1WvSbBSE0z
++ mktemp
+ local LAST_ERR=/tmp/tmp.bo6MZoBFNB
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-bb8b97679-8kmm8 -- bash -c 'printf '\''use myApp\n db.test.insert({ x: 100501 })\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.pitr-physical-backup-source-28616.svc.cluster.local:27017/admin '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.1WvSbBSE0z
Percona Server for MongoDB shell version v4.4.29-28
connecting to: mongodb://some-name-mongos.pitr-physical-backup-source-28616.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb
Implicit session: session { "id" : UUID("f6e6ef4a-4b62-4adc-8738-64aa0b8faa5e") }
Percona Server for MongoDB server version: v8.0.23-10
WARNING: shell and server versions do not match
switched to db myApp
WriteResult({ "nInserted" : 1 })
bye
+ cat /tmp/tmp.bo6MZoBFNB
+ rm /tmp/tmp.1WvSbBSE0z /tmp/tmp.bo6MZoBFNB
+ return 0
+ [[ -n 1781816709 ]]
++ format_date 1781816709
++ local timestamp=1781816709
+++ TZ=UTC
+++ /usr/sbin/date -d@1781816709 '+%Y-%m-%d %H:%M:%S'
++ echo 2026-06-18 21:05:09
+ desc 'Restoring to time 2026-06-18 21:05:09'
+ set +o xtrace


-----------------------------------------------------------------------------------
Restoring to time 2026-06-18 21:05:09
-----------------------------------------------------------------------------------

+ retries=0
+ [[ 1781816681 -gt 1781816709 ]]
+ [[ 0 -gt 30 ]]
++ get_latest_oplog_chunk_ts some-name
++ local cluster=some-name
+++ kubectl_bin exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ jq '.backups.pitrChunks.pitrChunks | last | .range.end'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.lK7TxQVy88
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.ViUQNnbaA0
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in $(seq 0 2)
+++ set +e
+++ kubectl exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.lK7TxQVy88
+++ cat /tmp/tmp.ViUQNnbaA0
+++ rm /tmp/tmp.lK7TxQVy88 /tmp/tmp.ViUQNnbaA0
+++ return 0
++ echo 1781816681
+ latest_ts=1781816681
+ retries=1
++ format_date 1781816681
++ local timestamp=1781816681
+++ TZ=UTC
+++ /usr/sbin/date -d@1781816681 '+%Y-%m-%d %H:%M:%S'
++ echo 2026-06-18 21:04:41
++ format_date 1781816709
++ local timestamp=1781816709
+++ TZ=UTC
+++ /usr/sbin/date -d@1781816709 '+%Y-%m-%d %H:%M:%S'
++ echo 2026-06-18 21:05:09
+ echo 'Waiting for last oplog chunk (2026-06-18 21:04:41) to be greater than restore target (2026-06-18 21:05:09)'
Waiting for last oplog chunk (2026-06-18 21:04:41) to be greater than restore target (2026-06-18 21:05:09)
+ sleep 10
+ [[ 1781816681 -gt 1781816709 ]]
+ [[ 1 -gt 30 ]]
++ get_latest_oplog_chunk_ts some-name
++ local cluster=some-name
+++ kubectl_bin exec some-name-rs0-0 -c backup-agent -- pbm status -o json
++++ mktemp
+++ jq '.backups.pitrChunks.pitrChunks | last | .range.end'
+++ local LAST_OUT=/tmp/tmp.GXv5A9eOXx
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.mpdhy1pEwP
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in $(seq 0 2)
+++ set +e
+++ kubectl exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.GXv5A9eOXx
+++ cat /tmp/tmp.mpdhy1pEwP
+++ rm /tmp/tmp.GXv5A9eOXx /tmp/tmp.mpdhy1pEwP
+++ return 0
++ echo 1781816681
+ latest_ts=1781816681
+ retries=2
++ format_date 1781816681
++ local timestamp=1781816681
+++ TZ=UTC
+++ /usr/sbin/date -d@1781816681 '+%Y-%m-%d %H:%M:%S'
++ echo 2026-06-18 21:04:41
++ format_date 1781816709
++ local timestamp=1781816709
+++ TZ=UTC
+++ /usr/sbin/date -d@1781816709 '+%Y-%m-%d %H:%M:%S'
++ echo 2026-06-18 21:05:09
+ echo 'Waiting for last oplog chunk (2026-06-18 21:04:41) to be greater than restore target (2026-06-18 21:05:09)'
Waiting for last oplog chunk (2026-06-18 21:04:41) to be greater than restore target (2026-06-18 21:05:09)
+ sleep 10
+ [[ 1781816681 -gt 1781816709 ]]
+ [[ 2 -gt 30 ]]
++ get_latest_oplog_chunk_ts some-name
++ local cluster=some-name
+++ kubectl_bin exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ jq '.backups.pitrChunks.pitrChunks | last | .range.end'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.Crsp4yf3Wb
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.qgnOL0GBnH
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in $(seq 0 2)
+++ set +e
+++ kubectl exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.Crsp4yf3Wb
+++ cat /tmp/tmp.qgnOL0GBnH
+++ rm /tmp/tmp.Crsp4yf3Wb /tmp/tmp.qgnOL0GBnH
+++ return 0
++ echo 1781816681
+ latest_ts=1781816681
+ retries=3
++ format_date 1781816681
++ local timestamp=1781816681
+++ TZ=UTC
+++ /usr/sbin/date -d@1781816681 '+%Y-%m-%d %H:%M:%S'
++ echo 2026-06-18 21:04:41
++ format_date 1781816709
++ local timestamp=1781816709
+++ TZ=UTC
+++ /usr/sbin/date -d@1781816709 '+%Y-%m-%d %H:%M:%S'
++ echo 2026-06-18 21:05:09
+ echo 'Waiting for last oplog chunk (2026-06-18 21:04:41) to be greater than restore target (2026-06-18 21:05:09)'
Waiting for last oplog chunk (2026-06-18 21:04:41) to be greater than restore target (2026-06-18 21:05:09)
+ sleep 10
+ [[ 1781816681 -gt 1781816709 ]]
+ [[ 3 -gt 30 ]]
++ get_latest_oplog_chunk_ts some-name
++ local cluster=some-name
+++ kubectl_bin exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ jq '.backups.pitrChunks.pitrChunks | last | .range.end'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.eTosNVYVvD
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.6KgbQ2TBi8
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in $(seq 0 2)
+++ set +e
+++ kubectl exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.eTosNVYVvD
+++ cat /tmp/tmp.6KgbQ2TBi8
+++ rm /tmp/tmp.eTosNVYVvD /tmp/tmp.6KgbQ2TBi8
+++ return 0
++ echo 1781816681
+ latest_ts=1781816681
+ retries=4
++ format_date 1781816681
++ local timestamp=1781816681
+++ TZ=UTC
+++ /usr/sbin/date -d@1781816681 '+%Y-%m-%d %H:%M:%S'
++ echo 2026-06-18 21:04:41
++ format_date 1781816709
++ local timestamp=1781816709
+++ TZ=UTC
+++ /usr/sbin/date -d@1781816709 '+%Y-%m-%d %H:%M:%S'
++ echo 2026-06-18 21:05:09
+ echo 'Waiting for last oplog chunk (2026-06-18 21:04:41) to be greater than restore target (2026-06-18 21:05:09)'
Waiting for last oplog chunk (2026-06-18 21:04:41) to be greater than restore target (2026-06-18 21:05:09)
+ sleep 10
+ [[ 1781816681 -gt 1781816709 ]]
+ [[ 4 -gt 30 ]]
++ get_latest_oplog_chunk_ts some-name
++ local cluster=some-name
+++ kubectl_bin exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ jq '.backups.pitrChunks.pitrChunks | last | .range.end'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.dH662rykHA
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.DL23fCrBXp
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in $(seq 0 2)
+++ set +e
+++ kubectl exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.dH662rykHA
+++ cat /tmp/tmp.DL23fCrBXp
+++ rm /tmp/tmp.dH662rykHA /tmp/tmp.DL23fCrBXp
+++ return 0
++ echo 1781816681
+ latest_ts=1781816681
+ retries=5
++ format_date 1781816681
++ local timestamp=1781816681
+++ TZ=UTC
+++ /usr/sbin/date -d@1781816681 '+%Y-%m-%d %H:%M:%S'
++ echo 2026-06-18 21:04:41
++ format_date 1781816709
++ local timestamp=1781816709
+++ TZ=UTC
+++ /usr/sbin/date -d@1781816709 '+%Y-%m-%d %H:%M:%S'
++ echo 2026-06-18 21:05:09
+ echo 'Waiting for last oplog chunk (2026-06-18 21:04:41) to be greater than restore target (2026-06-18 21:05:09)'
Waiting for last oplog chunk (2026-06-18 21:04:41) to be greater than restore target (2026-06-18 21:05:09)
+ sleep 10
+ [[ 1781816681 -gt 1781816709 ]]
+ [[ 5 -gt 30 ]]
++ get_latest_oplog_chunk_ts some-name
++ local cluster=some-name
+++ kubectl_bin exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ jq '.backups.pitrChunks.pitrChunks | last | .range.end'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.RYr3zlfWbj
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.1ccia7etvs
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in $(seq 0 2)
+++ set +e
+++ kubectl exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.RYr3zlfWbj
+++ cat /tmp/tmp.1ccia7etvs
+++ rm /tmp/tmp.RYr3zlfWbj /tmp/tmp.1ccia7etvs
+++ return 0
++ echo 1781816681
+ latest_ts=1781816681
+ retries=6
++ format_date 1781816681
++ local timestamp=1781816681
+++ TZ=UTC
+++ /usr/sbin/date -d@1781816681 '+%Y-%m-%d %H:%M:%S'
++ echo 2026-06-18 21:04:41
++ format_date 1781816709
++ local timestamp=1781816709
+++ TZ=UTC
+++ /usr/sbin/date -d@1781816709 '+%Y-%m-%d %H:%M:%S'
++ echo 2026-06-18 21:05:09
+ echo 'Waiting for last oplog chunk (2026-06-18 21:04:41) to be greater than restore target (2026-06-18 21:05:09)'
Waiting for last oplog chunk (2026-06-18 21:04:41) to be greater than restore target (2026-06-18 21:05:09)
+ sleep 10
+ [[ 1781816681 -gt 1781816709 ]]
+ [[ 6 -gt 30 ]]
++ get_latest_oplog_chunk_ts some-name
++ local cluster=some-name
+++ kubectl_bin exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ jq '.backups.pitrChunks.pitrChunks | last | .range.end'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.nyloYkoC4p
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.B7X5qlY0pV
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in $(seq 0 2)
+++ set +e
+++ kubectl exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.nyloYkoC4p
+++ cat /tmp/tmp.B7X5qlY0pV
+++ rm /tmp/tmp.nyloYkoC4p /tmp/tmp.B7X5qlY0pV
+++ return 0
++ echo 1781816681
+ latest_ts=1781816681
+ retries=7
++ format_date 1781816681
++ local timestamp=1781816681
+++ TZ=UTC
+++ /usr/sbin/date -d@1781816681 '+%Y-%m-%d %H:%M:%S'
++ echo 2026-06-18 21:04:41
++ format_date 1781816709
++ local timestamp=1781816709
+++ TZ=UTC
+++ /usr/sbin/date -d@1781816709 '+%Y-%m-%d %H:%M:%S'
++ echo 2026-06-18 21:05:09
+ echo 'Waiting for last oplog chunk (2026-06-18 21:04:41) to be greater than restore target (2026-06-18 21:05:09)'
Waiting for last oplog chunk (2026-06-18 21:04:41) to be greater than restore target (2026-06-18 21:05:09)
+ sleep 10
+ [[ 1781816681 -gt 1781816709 ]]
+ [[ 7 -gt 30 ]]
++ get_latest_oplog_chunk_ts some-name
++ local cluster=some-name
+++ kubectl_bin exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ jq '.backups.pitrChunks.pitrChunks | last | .range.end'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.cEz3WV6RWX
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.EI2ZfJUAv7
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in $(seq 0 2)
+++ set +e
+++ kubectl exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.cEz3WV6RWX
+++ cat /tmp/tmp.EI2ZfJUAv7
+++ rm /tmp/tmp.cEz3WV6RWX /tmp/tmp.EI2ZfJUAv7
+++ return 0
++ echo 1781816681
+ latest_ts=1781816681
+ retries=8
++ format_date 1781816681
++ local timestamp=1781816681
+++ TZ=UTC
+++ /usr/sbin/date -d@1781816681 '+%Y-%m-%d %H:%M:%S'
++ echo 2026-06-18 21:04:41
++ format_date 1781816709
++ local timestamp=1781816709
+++ TZ=UTC
+++ /usr/sbin/date -d@1781816709 '+%Y-%m-%d %H:%M:%S'
++ echo 2026-06-18 21:05:09
+ echo 'Waiting for last oplog chunk (2026-06-18 21:04:41) to be greater than restore target (2026-06-18 21:05:09)'
Waiting for last oplog chunk (2026-06-18 21:04:41) to be greater than restore target (2026-06-18 21:05:09)
+ sleep 10
+ [[ 1781816681 -gt 1781816709 ]]
+ [[ 8 -gt 30 ]]
++ get_latest_oplog_chunk_ts some-name
++ local cluster=some-name
+++ kubectl_bin exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ jq '.backups.pitrChunks.pitrChunks | last | .range.end'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.WUGUGYosdb
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.Vr8VNODo1h
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in $(seq 0 2)
+++ set +e
+++ kubectl exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.WUGUGYosdb
+++ cat /tmp/tmp.Vr8VNODo1h
+++ rm /tmp/tmp.WUGUGYosdb /tmp/tmp.Vr8VNODo1h
+++ return 0
++ echo 1781816684
+ latest_ts=1781816684
+ retries=9
++ format_date 1781816684
++ local timestamp=1781816684
+++ TZ=UTC
+++ /usr/sbin/date -d@1781816684 '+%Y-%m-%d %H:%M:%S'
++ echo 2026-06-18 21:04:44
++ format_date 1781816709
++ local timestamp=1781816709
+++ TZ=UTC
+++ /usr/sbin/date -d@1781816709 '+%Y-%m-%d %H:%M:%S'
++ echo 2026-06-18 21:05:09
+ echo 'Waiting for last oplog chunk (2026-06-18 21:04:44) to be greater than restore target (2026-06-18 21:05:09)'
Waiting for last oplog chunk (2026-06-18 21:04:44) to be greater than restore target (2026-06-18 21:05:09)
+ sleep 10
+ [[ 1781816684 -gt 1781816709 ]]
+ [[ 9 -gt 30 ]]
++ get_latest_oplog_chunk_ts some-name
++ local cluster=some-name
+++ kubectl_bin exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ jq '.backups.pitrChunks.pitrChunks | last | .range.end'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.V1sWprsGgA
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.LvEsIhrPUD
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in $(seq 0 2)
+++ set +e
+++ kubectl exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.V1sWprsGgA
+++ cat /tmp/tmp.LvEsIhrPUD
+++ rm /tmp/tmp.V1sWprsGgA /tmp/tmp.LvEsIhrPUD
+++ return 0
++ echo 1781816691
+ latest_ts=1781816691
+ retries=10
++ format_date 1781816691
++ local timestamp=1781816691
+++ TZ=UTC
+++ /usr/sbin/date -d@1781816691 '+%Y-%m-%d %H:%M:%S'
++ echo 2026-06-18 21:04:51
++ format_date 1781816709
++ local timestamp=1781816709
+++ TZ=UTC
+++ /usr/sbin/date -d@1781816709 '+%Y-%m-%d %H:%M:%S'
++ echo 2026-06-18 21:05:09
+ echo 'Waiting for last oplog chunk (2026-06-18 21:04:51) to be greater than restore target (2026-06-18 21:05:09)'
Waiting for last oplog chunk (2026-06-18 21:04:51) to be greater than restore target (2026-06-18 21:05:09)
+ sleep 10
+ [[ 1781816691 -gt 1781816709 ]]
+ [[ 10 -gt 30 ]]
++ get_latest_oplog_chunk_ts some-name
++ local cluster=some-name
+++ kubectl_bin exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ jq '.backups.pitrChunks.pitrChunks | last | .range.end'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.cVoRtzP174
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.q6fNpxF03l
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in $(seq 0 2)
+++ set +e
+++ kubectl exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.cVoRtzP174
+++ cat /tmp/tmp.q6fNpxF03l
+++ rm /tmp/tmp.cVoRtzP174 /tmp/tmp.q6fNpxF03l
+++ return 0
++ echo 1781816801
+ latest_ts=1781816801
+ retries=11
++ format_date 1781816801
++ local timestamp=1781816801
+++ TZ=UTC
+++ /usr/sbin/date -d@1781816801 '+%Y-%m-%d %H:%M:%S'
++ echo 2026-06-18 21:06:41
++ format_date 1781816709
++ local timestamp=1781816709
+++ TZ=UTC
+++ /usr/sbin/date -d@1781816709 '+%Y-%m-%d %H:%M:%S'
++ echo 2026-06-18 21:05:09
+ echo 'Waiting for last oplog chunk (2026-06-18 21:06:41) to be greater than restore target (2026-06-18 21:05:09)'
Waiting for last oplog chunk (2026-06-18 21:06:41) to be greater than restore target (2026-06-18 21:05:09)
+ sleep 10
+ [[ 1781816801 -gt 1781816709 ]]
+ '[' -z backupSource ']'
+ desc 'check restore by date backupSource'
+ set +o xtrace


-----------------------------------------------------------------------------------
check restore by date backupSource
-----------------------------------------------------------------------------------

++ get_backup_dest backup-minio-1
++ local backup_name=backup-minio-1
++ kubectl_bin get psmdb-backup backup-minio-1 -o 'jsonpath={.status.destination}'
++ sed -e 's/.json$//'
++ sed 's|s3://||'
++ sed 's|azure://||'
+++ mktemp
++ sed 's|gs://||'
++ local LAST_OUT=/tmp/tmp.cq1lEvz5Ii
+++ mktemp
++ local LAST_ERR=/tmp/tmp.xp7oO8egtJ
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb-backup backup-minio-1 -o 'jsonpath={.status.destination}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.cq1lEvz5Ii
++ cat /tmp/tmp.xp7oO8egtJ
++ rm /tmp/tmp.cq1lEvz5Ii /tmp/tmp.xp7oO8egtJ
++ return 0
+ backup_dest=operator-testing/2026-06-18T21:01:58Z
+ cat /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2406/e2e-tests/pitr-physical-backup-source/conf/restore.yml
+ /usr/sbin/sed -e 's/name:/name: restore-backup-minio-1/'
+ /usr/sbin/sed -e /backupName/d
+ /usr/sbin/sed -e 's/pitrType:/type: date/'
+ '[' -z 1781816709 ']'
+ /usr/sbin/sed -e 's|DESTINATION|operator-testing/2026-06-18T21:01:58Z|'
++ format_date 1781816709
++ local timestamp=1781816709
+ '[' -n '' ']'
+ yq
+ kubectl_bin apply -f -
+++ TZ=UTC
+++ /usr/sbin/date -d@1781816709 '+%Y-%m-%d %H:%M:%S'
++ get_bucket_name backup-minio-1
++ local backup_name=backup-minio-1
++ kubectl_bin get psmdb-backup backup-minio-1 -o 'jsonpath={.status.s3.bucket}'
++ mktemp
+++ mktemp
++ echo 2026-06-18 21:05:09
+ /usr/sbin/sed -e 's/date:/date: 2026-06-18 21:05:09/'
++ local LAST_OUT=/tmp/tmp.7vm7T82CM8
+ local LAST_OUT=/tmp/tmp.i6ZF9nmUCt
++ mktemp
+++ mktemp
+ local LAST_ERR=/tmp/tmp.7S7r0KSBwb
+ local exit_status=0
+ local timeout=4
++ local LAST_ERR=/tmp/tmp.obzqQUbhDN
++ local exit_status=0
++ local timeout=4
++ seq 0 2
+++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl apply -f -
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb-backup backup-minio-1 -o 'jsonpath={.status.s3.bucket}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.7vm7T82CM8
++ cat /tmp/tmp.obzqQUbhDN
++ rm /tmp/tmp.7vm7T82CM8 /tmp/tmp.obzqQUbhDN
++ return 0
+ /usr/sbin/sed -e 's|BUCKET-NAME|operator-testing|'
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.i6ZF9nmUCt
perconaservermongodbrestore.psmdb.percona.com/restore-backup-minio-1 created
+ cat /tmp/tmp.7S7r0KSBwb
+ rm /tmp/tmp.i6ZF9nmUCt /tmp/tmp.7S7r0KSBwb
+ return 0
+ wait_restore backup-minio-1 some-name requested 0 1200
+ local backup_name=backup-minio-1
+ local cluster_name=some-name
+ local target_state=requested
+ local wait_cluster_consistency=0
+ local wait_time=1200
+ local ok_if_ready=0
+ set +o xtrace
Waiting for the psmdb-restore/restore-backup-minio-1 object to be created.OK
Waiting psmdb-restore/restore-backup-minio-1 to reach state "requested" ........OK after 7 minutes
+ [[ 0 -eq 1 ]]
+ echo

+ wait_restore backup-minio-1 some-name ready 0 1600
+ local backup_name=backup-minio-1
+ local cluster_name=some-name
+ local target_state=ready
+ local wait_cluster_consistency=0
+ local wait_time=1600
+ local ok_if_ready=0
+ set +o xtrace
Waiting for the psmdb-restore/restore-backup-minio-1 object to be created.OK
Waiting psmdb-restore/restore-backup-minio-1 to reach state "ready" ...OK after 2 minutes
+ [[ 0 -eq 1 ]]
+ echo

+ set -o xtrace
+ wait_for_running some-name-rs0 3
+ local name=some-name-rs0
+ let last_pod=2
+ local check_cluster_readyness=true
+ set_debug
+ [[ 1 == 1 ]]
+ set -o xtrace
+ local rs_name=rs0
+ local cluster_name=some-name
++ seq 0 2
+ for i in $(seq 0 $last_pod)
+ [[ 0 -eq 2 ]]
+ wait_pod some-name-rs0-0
+ local pod=some-name-rs0-0
+ set +o xtrace
waiting for pod/some-name-rs0-0 to be ready.OK
+ for i in $(seq 0 $last_pod)
+ [[ 1 -eq 2 ]]
+ wait_pod some-name-rs0-1
+ local pod=some-name-rs0-1
+ set +o xtrace
waiting for pod/some-name-rs0-1 to be ready.OK
+ for i in $(seq 0 $last_pod)
+ [[ 2 -eq 2 ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].arbiter.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.gKxotpYiPV
+++ mktemp
++ local LAST_ERR=/tmp/tmp.I1ny2oP7Ph
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].arbiter.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.gKxotpYiPV
++ cat /tmp/tmp.I1ny2oP7Ph
++ rm /tmp/tmp.gKxotpYiPV /tmp/tmp.I1ny2oP7Ph
++ return 0
+ [[ '' == true ]]
+ wait_pod some-name-rs0-2
+ local pod=some-name-rs0-2
+ set +o xtrace
waiting for pod/some-name-rs0-2 to be ready.OK
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].nonvoting.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.KaIAPgKhD1
+++ mktemp
++ local LAST_ERR=/tmp/tmp.GawSg1DWjT
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].nonvoting.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.KaIAPgKhD1
++ cat /tmp/tmp.GawSg1DWjT
++ rm /tmp/tmp.KaIAPgKhD1 /tmp/tmp.GawSg1DWjT
++ return 0
+ [[ '' == true ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].hidden.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.tlo2KNrQCs
+++ mktemp
++ local LAST_ERR=/tmp/tmp.XCxAsy3PTH
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].hidden.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.tlo2KNrQCs
++ cat /tmp/tmp.XCxAsy3PTH
++ rm /tmp/tmp.tlo2KNrQCs /tmp/tmp.XCxAsy3PTH
++ return 0
+ [[ '' == true ]]
+ sleep 10
+ [[ true == true ]]
+ set +x
Waiting for cluster readyness.....................................................
+ wait_for_running some-name-cfg 3
+ local name=some-name-cfg
+ let last_pod=2
+ local check_cluster_readyness=true
+ set_debug
+ [[ 1 == 1 ]]
+ set -o xtrace
+ local rs_name=cfg
+ local cluster_name=some-name
++ seq 0 2
+ for i in $(seq 0 $last_pod)
+ [[ 0 -eq 2 ]]
+ wait_pod some-name-cfg-0
+ local pod=some-name-cfg-0
+ set +o xtrace
waiting for pod/some-name-cfg-0 to be ready.OK
+ for i in $(seq 0 $last_pod)
+ [[ 1 -eq 2 ]]
+ wait_pod some-name-cfg-1
+ local pod=some-name-cfg-1
+ set +o xtrace
waiting for pod/some-name-cfg-1 to be ready.OK
+ for i in $(seq 0 $last_pod)
+ [[ 2 -eq 2 ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].arbiter.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.z0fOjelOga
+++ mktemp
++ local LAST_ERR=/tmp/tmp.90XaolCDRt
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].arbiter.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.z0fOjelOga
++ cat /tmp/tmp.90XaolCDRt
++ rm /tmp/tmp.z0fOjelOga /tmp/tmp.90XaolCDRt
++ return 0
+ [[ '' == true ]]
+ wait_pod some-name-cfg-2
+ local pod=some-name-cfg-2
+ set +o xtrace
waiting for pod/some-name-cfg-2 to be ready.OK
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].nonvoting.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.ul1U0FF5H6
+++ mktemp
++ local LAST_ERR=/tmp/tmp.xsLbExXjYR
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].nonvoting.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.ul1U0FF5H6
++ cat /tmp/tmp.xsLbExXjYR
++ rm /tmp/tmp.ul1U0FF5H6 /tmp/tmp.xsLbExXjYR
++ return 0
+ [[ '' == true ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].hidden.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.UsaCkkIBTU
+++ mktemp
++ local LAST_ERR=/tmp/tmp.GBbwCz8Gjo
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].hidden.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.UsaCkkIBTU
++ cat /tmp/tmp.GBbwCz8Gjo
++ rm /tmp/tmp.UsaCkkIBTU /tmp/tmp.GBbwCz8Gjo
++ return 0
+ [[ '' == true ]]
+ sleep 10
+ [[ true == true ]]
+ set +x
Waiting for cluster readyness
+ wait_for_running some-name-mongos 3
+ local name=some-name-mongos
+ let last_pod=2
+ local check_cluster_readyness=true
+ set_debug
+ [[ 1 == 1 ]]
+ set -o xtrace
+ local rs_name=mongos
+ local cluster_name=some-name
++ seq 0 2
+ for i in $(seq 0 $last_pod)
+ [[ 0 -eq 2 ]]
+ wait_pod some-name-mongos-0
+ local pod=some-name-mongos-0
+ set +o xtrace
waiting for pod/some-name-mongos-0 to be ready.OK
+ for i in $(seq 0 $last_pod)
+ [[ 1 -eq 2 ]]
+ wait_pod some-name-mongos-1
+ local pod=some-name-mongos-1
+ set +o xtrace
waiting for pod/some-name-mongos-1 to be ready.OK
+ for i in $(seq 0 $last_pod)
+ [[ 2 -eq 2 ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="mongos")].arbiter.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.YYNQd17Wg7
+++ mktemp
++ local LAST_ERR=/tmp/tmp.8gU60GjgNn
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="mongos")].arbiter.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.YYNQd17Wg7
++ cat /tmp/tmp.8gU60GjgNn
++ rm /tmp/tmp.YYNQd17Wg7 /tmp/tmp.8gU60GjgNn
++ return 0
+ [[ '' == true ]]
+ wait_pod some-name-mongos-2
+ local pod=some-name-mongos-2
+ set +o xtrace
waiting for pod/some-name-mongos-2 to be ready.OK
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="mongos")].nonvoting.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.eG1unrMPJn
+++ mktemp
++ local LAST_ERR=/tmp/tmp.bYL1TuVPcR
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="mongos")].nonvoting.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.eG1unrMPJn
++ cat /tmp/tmp.bYL1TuVPcR
++ rm /tmp/tmp.eG1unrMPJn /tmp/tmp.bYL1TuVPcR
++ return 0
+ [[ '' == true ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="mongos")].hidden.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.vZRn1W7vc4
+++ mktemp
++ local LAST_ERR=/tmp/tmp.PpjuYIknpW
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="mongos")].hidden.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.vZRn1W7vc4
++ cat /tmp/tmp.PpjuYIknpW
++ rm /tmp/tmp.vZRn1W7vc4 /tmp/tmp.PpjuYIknpW
++ return 0
+ [[ '' == true ]]
+ sleep 10
+ [[ true == true ]]
+ set +x
Waiting for cluster readyness
+ sleep 10
+ compare_mongos_cmd find myApp:myPass@some-name-mongos.pitr-physical-backup-source-28616 ''
+ local command=find
+ local uri=myApp:myPass@some-name-mongos.pitr-physical-backup-source-28616
+ local postfix=
+ local suffix=
+ local database=myApp
+ local collection=test
+ local port=27017
+ local tls=false
+ [[ false == true ]]
+ mongos_command=run_mongos
+ log 'running db.test.command() in myApp'
+ set +o xtrace
[2026-06-18T21:20:02+0000] running db.test.command() in myApp
+ run_mongos 'use myApp\n db.test.find()' myApp:myPass@some-name-mongos.pitr-physical-backup-source-28616 mongodb '' '' 27017
+ local 'command=use myApp\n db.test.find()'
+ local uri=myApp:myPass@some-name-mongos.pitr-physical-backup-source-28616
+ grep -E -v 'I NETWORK|W NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|Started a new thread for the timer service'
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local port=27017
+ local mongo_bin=mongo
+ /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
++ echo .svc.cluster.local
++ awk -F: '{print $2}'
+ suffix_port=
+ [[ -z '' ]]
+ suffix=.svc.cluster.local:27017
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.3TJyYM70fn
+++ mktemp
++ local LAST_ERR=/tmp/tmp.2EZjALM9LV
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.3TJyYM70fn
++ cat /tmp/tmp.2EZjALM9LV
++ rm /tmp/tmp.3TJyYM70fn /tmp/tmp.2EZjALM9LV
++ return 0
+ local client_container=psmdb-client-bb8b97679-8kmm8
+ kubectl_bin exec psmdb-client-bb8b97679-8kmm8 -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.pitr-physical-backup-source-28616.svc.cluster.local:27017/admin '
++ mktemp
+ local LAST_OUT=/tmp/tmp.J6uOWx5ArI
++ mktemp
+ local LAST_ERR=/tmp/tmp.P3OYkIidVJ
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-bb8b97679-8kmm8 -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.pitr-physical-backup-source-28616.svc.cluster.local:27017/admin '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.J6uOWx5ArI
+ cat /tmp/tmp.P3OYkIidVJ
+ rm /tmp/tmp.J6uOWx5ArI /tmp/tmp.P3OYkIidVJ
+ return 0
+ [[ 0 -eq 0 ]]
+ diff /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2406/e2e-tests/pitr-physical-backup-source/compare/find.json /tmp/tmp.4E31yTnNi1/find
+ desc 'delete PSMDB cluster some-name'
+ set +o xtrace


-----------------------------------------------------------------------------------
delete PSMDB cluster some-name
-----------------------------------------------------------------------------------

+ kubectl_bin delete psmdb some-name
++ mktemp
+ local LAST_OUT=/tmp/tmp.i4nPYWGnfJ
++ mktemp
+ local LAST_ERR=/tmp/tmp.8cAoOpxKqL
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl delete psmdb some-name
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.i4nPYWGnfJ
perconaservermongodb.psmdb.percona.com "some-name" deleted from pitr-physical-backup-source-28616 namespace
+ cat /tmp/tmp.8cAoOpxKqL
+ rm /tmp/tmp.i4nPYWGnfJ /tmp/tmp.8cAoOpxKqL
+ return 0
+ kubectl_bin delete pvc -l app.kubernetes.io/managed-by=percona-server-mongodb-operator
++ mktemp
+ local LAST_OUT=/tmp/tmp.sGvdnVa6Mf
++ mktemp
+ local LAST_ERR=/tmp/tmp.rRLcqlgtcs
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl delete pvc -l app.kubernetes.io/managed-by=percona-server-mongodb-operator
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.sGvdnVa6Mf
persistentvolumeclaim "mongod-data-some-name-cfg-0" deleted from pitr-physical-backup-source-28616 namespace
persistentvolumeclaim "mongod-data-some-name-cfg-1" deleted from pitr-physical-backup-source-28616 namespace
persistentvolumeclaim "mongod-data-some-name-cfg-2" deleted from pitr-physical-backup-source-28616 namespace
persistentvolumeclaim "mongod-data-some-name-rs0-0" deleted from pitr-physical-backup-source-28616 namespace
persistentvolumeclaim "mongod-data-some-name-rs0-1" deleted from pitr-physical-backup-source-28616 namespace
persistentvolumeclaim "mongod-data-some-name-rs0-2" deleted from pitr-physical-backup-source-28616 namespace
persistentvolumeclaim "mongod-data-some-name-rs1-0" deleted from pitr-physical-backup-source-28616 namespace
persistentvolumeclaim "mongod-data-some-name-rs1-1" deleted from pitr-physical-backup-source-28616 namespace
persistentvolumeclaim "mongod-data-some-name-rs1-2" deleted from pitr-physical-backup-source-28616 namespace
persistentvolumeclaim "mongod-data-some-name-rs2-0" deleted from pitr-physical-backup-source-28616 namespace
persistentvolumeclaim "mongod-data-some-name-rs2-1" deleted from pitr-physical-backup-source-28616 namespace
persistentvolumeclaim "mongod-data-some-name-rs2-2" deleted from pitr-physical-backup-source-28616 namespace
+ cat /tmp/tmp.rRLcqlgtcs
+ rm /tmp/tmp.sGvdnVa6Mf /tmp/tmp.rRLcqlgtcs
+ return 0
+ sleep 10
+ desc 'recreate PSMDB cluster some-name'
+ set +o xtrace


-----------------------------------------------------------------------------------
recreate PSMDB cluster some-name
-----------------------------------------------------------------------------------

+ desc 'create second PSMDB cluster some-name'
+ set +o xtrace


-----------------------------------------------------------------------------------
create second PSMDB cluster some-name
-----------------------------------------------------------------------------------

+ apply_cluster /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2406/e2e-tests/pitr-physical-backup-source/conf/some-name-rs0.yml
+ '[' -z '' ']'
+ cat_config /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2406/e2e-tests/pitr-physical-backup-source/conf/some-name-rs0.yml
+ kubectl_bin apply -f -
+ cat /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2406/e2e-tests/pitr-physical-backup-source/conf/some-name-rs0.yml
++ mktemp
+ yq eval '(.spec | select(.image == null)).image = "docker.io/perconalab/percona-server-mongodb-operator:main-mongod8.0"'
+ yq eval '(.spec | select(has("pmm"))).pmm.image = "docker.io/percona/pmm-client:2.44.1-1"'
+ yq eval '(.spec | select(has("initImage"))).initImage = "docker.io/perconalab/percona-server-mongodb-operator:PR-2406-1c5b15f05"'
+ yq eval '(.spec | select(has("backup"))).backup.image = "docker.io/perconalab/percona-server-mongodb-operator:main-backup"'
+ yq eval '(.spec | select(has("search"))).search.image = "mongodb/mongodb-community-search:latest"'
+ local LAST_OUT=/tmp/tmp.apfnuMfFlw
+ yq eval '.spec.upgradeOptions.apply="Never"'
+ /usr/sbin/sed -e s/NAME_SPACE/pitr-physical-backup-source-28616/g
++ mktemp
+ local LAST_ERR=/tmp/tmp.YRrYQz9TGC
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl apply -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.apfnuMfFlw
perconaservermongodb.psmdb.percona.com/some-name created
+ cat /tmp/tmp.YRrYQz9TGC
+ rm /tmp/tmp.apfnuMfFlw /tmp/tmp.YRrYQz9TGC
+ return 0
+ desc 'check if all 3 Pods started'
+ set +o xtrace


-----------------------------------------------------------------------------------
check if all 3 Pods started
-----------------------------------------------------------------------------------

+ wait_for_running some-name-rs0 3
+ local name=some-name-rs0
+ let last_pod=2
+ local check_cluster_readyness=true
+ set_debug
+ [[ 1 == 1 ]]
+ set -o xtrace
+ local rs_name=rs0
+ local cluster_name=some-name
++ seq 0 2
+ for i in $(seq 0 $last_pod)
+ [[ 0 -eq 2 ]]
+ wait_pod some-name-rs0-0
+ local pod=some-name-rs0-0
+ set +o xtrace
waiting for pod/some-name-rs0-0 to be ready.........OK
+ for i in $(seq 0 $last_pod)
+ [[ 1 -eq 2 ]]
+ wait_pod some-name-rs0-1
+ local pod=some-name-rs0-1
+ set +o xtrace
waiting for pod/some-name-rs0-1 to be ready.........OK
+ for i in $(seq 0 $last_pod)
+ [[ 2 -eq 2 ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].arbiter.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.urzJDjPmfi
+++ mktemp
++ local LAST_ERR=/tmp/tmp.4JiPqBsdsq
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].arbiter.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.urzJDjPmfi
++ cat /tmp/tmp.4JiPqBsdsq
++ rm /tmp/tmp.urzJDjPmfi /tmp/tmp.4JiPqBsdsq
++ return 0
+ [[ '' == true ]]
+ wait_pod some-name-rs0-2
+ local pod=some-name-rs0-2
+ set +o xtrace
waiting for pod/some-name-rs0-2 to be ready.......OK
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].nonvoting.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.tB5WvSdV0L
+++ mktemp
++ local LAST_ERR=/tmp/tmp.zcC2AiA9OC
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].nonvoting.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.tB5WvSdV0L
++ cat /tmp/tmp.zcC2AiA9OC
++ rm /tmp/tmp.tB5WvSdV0L /tmp/tmp.zcC2AiA9OC
++ return 0
+ [[ '' == true ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].hidden.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.lxtLUX7ASa
+++ mktemp
++ local LAST_ERR=/tmp/tmp.g851r7nBvJ
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].hidden.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.lxtLUX7ASa
++ cat /tmp/tmp.g851r7nBvJ
++ rm /tmp/tmp.lxtLUX7ASa /tmp/tmp.g851r7nBvJ
++ return 0
+ [[ '' == true ]]
+ sleep 10
+ [[ true == true ]]
+ set +x
Waiting for cluster readyness...........................................
+ wait_for_running some-name-cfg 3 false
+ local name=some-name-cfg
+ let last_pod=2
+ local check_cluster_readyness=false
+ set_debug
+ [[ 1 == 1 ]]
+ set -o xtrace
+ local rs_name=cfg
+ local cluster_name=some-name
++ seq 0 2
+ for i in $(seq 0 $last_pod)
+ [[ 0 -eq 2 ]]
+ wait_pod some-name-cfg-0
+ local pod=some-name-cfg-0
+ set +o xtrace
waiting for pod/some-name-cfg-0 to be ready.OK
+ for i in $(seq 0 $last_pod)
+ [[ 1 -eq 2 ]]
+ wait_pod some-name-cfg-1
+ local pod=some-name-cfg-1
+ set +o xtrace
waiting for pod/some-name-cfg-1 to be ready.OK
+ for i in $(seq 0 $last_pod)
+ [[ 2 -eq 2 ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].arbiter.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.rLDxleFYR7
+++ mktemp
++ local LAST_ERR=/tmp/tmp.ayflGEdme6
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].arbiter.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.rLDxleFYR7
++ cat /tmp/tmp.ayflGEdme6
++ rm /tmp/tmp.rLDxleFYR7 /tmp/tmp.ayflGEdme6
++ return 0
+ [[ '' == true ]]
+ wait_pod some-name-cfg-2
+ local pod=some-name-cfg-2
+ set +o xtrace
waiting for pod/some-name-cfg-2 to be ready.OK
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].nonvoting.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.7Y6lP53Mn1
+++ mktemp
++ local LAST_ERR=/tmp/tmp.RI9D1D7Pdh
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].nonvoting.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.7Y6lP53Mn1
++ cat /tmp/tmp.RI9D1D7Pdh
++ rm /tmp/tmp.7Y6lP53Mn1 /tmp/tmp.RI9D1D7Pdh
++ return 0
+ [[ '' == true ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].hidden.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.Jl6SW6nkga
+++ mktemp
++ local LAST_ERR=/tmp/tmp.BXybgM5H2M
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].hidden.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.Jl6SW6nkga
++ cat /tmp/tmp.BXybgM5H2M
++ rm /tmp/tmp.Jl6SW6nkga /tmp/tmp.BXybgM5H2M
++ return 0
+ [[ '' == true ]]
+ sleep 10
+ [[ false == true ]]
+ wait_for_running some-name-rs1 3
+ local name=some-name-rs1
+ let last_pod=2
+ local check_cluster_readyness=true
+ set_debug
+ [[ 1 == 1 ]]
+ set -o xtrace
+ local rs_name=rs1
+ local cluster_name=some-name
++ seq 0 2
+ for i in $(seq 0 $last_pod)
+ [[ 0 -eq 2 ]]
+ wait_pod some-name-rs1-0
+ local pod=some-name-rs1-0
+ set +o xtrace
waiting for pod/some-name-rs1-0 to be ready.OK
+ for i in $(seq 0 $last_pod)
+ [[ 1 -eq 2 ]]
+ wait_pod some-name-rs1-1
+ local pod=some-name-rs1-1
+ set +o xtrace
waiting for pod/some-name-rs1-1 to be ready.OK
+ for i in $(seq 0 $last_pod)
+ [[ 2 -eq 2 ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs1")].arbiter.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.K4OteF4XDN
+++ mktemp
++ local LAST_ERR=/tmp/tmp.sVv9jJGmMn
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs1")].arbiter.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.K4OteF4XDN
++ cat /tmp/tmp.sVv9jJGmMn
++ rm /tmp/tmp.K4OteF4XDN /tmp/tmp.sVv9jJGmMn
++ return 0
+ [[ '' == true ]]
+ wait_pod some-name-rs1-2
+ local pod=some-name-rs1-2
+ set +o xtrace
waiting for pod/some-name-rs1-2 to be ready.OK
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs1")].nonvoting.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.EUX4S5yFeT
+++ mktemp
++ local LAST_ERR=/tmp/tmp.o5nTIoxpZe
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs1")].nonvoting.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.EUX4S5yFeT
++ cat /tmp/tmp.o5nTIoxpZe
++ rm /tmp/tmp.EUX4S5yFeT /tmp/tmp.o5nTIoxpZe
++ return 0
+ [[ '' == true ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs1")].hidden.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.8ugd3ehsuv
+++ mktemp
++ local LAST_ERR=/tmp/tmp.tAwVRBpe3Z
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs1")].hidden.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.8ugd3ehsuv
++ cat /tmp/tmp.tAwVRBpe3Z
++ rm /tmp/tmp.8ugd3ehsuv /tmp/tmp.tAwVRBpe3Z
++ return 0
+ [[ '' == true ]]
+ sleep 10
+ [[ true == true ]]
+ set +x
Waiting for cluster readyness
+ wait_for_running some-name-rs2 3
+ local name=some-name-rs2
+ let last_pod=2
+ local check_cluster_readyness=true
+ set_debug
+ [[ 1 == 1 ]]
+ set -o xtrace
+ local rs_name=rs2
+ local cluster_name=some-name
++ seq 0 2
+ for i in $(seq 0 $last_pod)
+ [[ 0 -eq 2 ]]
+ wait_pod some-name-rs2-0
+ local pod=some-name-rs2-0
+ set +o xtrace
waiting for pod/some-name-rs2-0 to be ready.OK
+ for i in $(seq 0 $last_pod)
+ [[ 1 -eq 2 ]]
+ wait_pod some-name-rs2-1
+ local pod=some-name-rs2-1
+ set +o xtrace
waiting for pod/some-name-rs2-1 to be ready.OK
+ for i in $(seq 0 $last_pod)
+ [[ 2 -eq 2 ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs2")].arbiter.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.B1sNhXtXot
+++ mktemp
++ local LAST_ERR=/tmp/tmp.q3M85VrYCH
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs2")].arbiter.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.B1sNhXtXot
++ cat /tmp/tmp.q3M85VrYCH
++ rm /tmp/tmp.B1sNhXtXot /tmp/tmp.q3M85VrYCH
++ return 0
+ [[ '' == true ]]
+ wait_pod some-name-rs2-2
+ local pod=some-name-rs2-2
+ set +o xtrace
waiting for pod/some-name-rs2-2 to be ready.OK
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs2")].nonvoting.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.hWGmD6TP7p
+++ mktemp
++ local LAST_ERR=/tmp/tmp.i3gbfzEoZg
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs2")].nonvoting.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.hWGmD6TP7p
++ cat /tmp/tmp.i3gbfzEoZg
++ rm /tmp/tmp.hWGmD6TP7p /tmp/tmp.i3gbfzEoZg
++ return 0
+ [[ '' == true ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs2")].hidden.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.vpGZefE3r4
+++ mktemp
++ local LAST_ERR=/tmp/tmp.OjELVJRnun
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs2")].hidden.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.vpGZefE3r4
++ cat /tmp/tmp.OjELVJRnun
++ rm /tmp/tmp.vpGZefE3r4 /tmp/tmp.OjELVJRnun
++ return 0
+ [[ '' == true ]]
+ sleep 10
+ [[ true == true ]]
+ set +x
Waiting for cluster readyness
+ sleep 10
+ write_initial_data
+ desc 'create user myApp'
+ set +o xtrace


-----------------------------------------------------------------------------------
create user myApp
-----------------------------------------------------------------------------------

+ run_mongos 'db.createUser({user:"myApp",pwd:"myPass",roles:[{db:"myApp",role:"readWrite"}]})' userAdmin:userAdmin123456@some-name-mongos.pitr-physical-backup-source-28616
+ local 'command=db.createUser({user:"myApp",pwd:"myPass",roles:[{db:"myApp",role:"readWrite"}]})'
+ local uri=userAdmin:userAdmin123456@some-name-mongos.pitr-physical-backup-source-28616
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local port=27017
+ local mongo_bin=mongo
++ echo .svc.cluster.local
++ awk -F: '{print $2}'
+ suffix_port=
+ [[ -z '' ]]
+ suffix=.svc.cluster.local:27017
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.UOVaE0m4UH
+++ mktemp
++ local LAST_ERR=/tmp/tmp.G8rrANVWhg
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.UOVaE0m4UH
++ cat /tmp/tmp.G8rrANVWhg
++ rm /tmp/tmp.UOVaE0m4UH /tmp/tmp.G8rrANVWhg
++ return 0
+ local client_container=psmdb-client-bb8b97679-8kmm8
+ kubectl_bin exec psmdb-client-bb8b97679-8kmm8 -- bash -c 'printf '\''db.createUser({user:"myApp",pwd:"myPass",roles:[{db:"myApp",role:"readWrite"}]})\n'\'' | mongo mongodb://userAdmin:userAdmin123456@some-name-mongos.pitr-physical-backup-source-28616.svc.cluster.local:27017/admin '
++ mktemp
+ local LAST_OUT=/tmp/tmp.ndjgQz9RR1
++ mktemp
+ local LAST_ERR=/tmp/tmp.tVJZvMrXxo
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-bb8b97679-8kmm8 -- bash -c 'printf '\''db.createUser({user:"myApp",pwd:"myPass",roles:[{db:"myApp",role:"readWrite"}]})\n'\'' | mongo mongodb://userAdmin:userAdmin123456@some-name-mongos.pitr-physical-backup-source-28616.svc.cluster.local:27017/admin '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.ndjgQz9RR1
Percona Server for MongoDB shell version v4.4.29-28
connecting to: mongodb://some-name-mongos.pitr-physical-backup-source-28616.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb
Implicit session: session { "id" : UUID("6a87dc83-8420-4eae-8d30-c73858cb85c5") }
Percona Server for MongoDB server version: v8.0.23-10
WARNING: shell and server versions do not match
Successfully added user: {
	"user" : "myApp",
	"roles" : [
		{
			"db" : "myApp",
			"role" : "readWrite"
		}
	]
}
bye
+ cat /tmp/tmp.tVJZvMrXxo
+ rm /tmp/tmp.ndjgQz9RR1 /tmp/tmp.tVJZvMrXxo
+ return 0
+ sleep 2
+ write_document
+ local cmp_postfix=
+ local sleep_value=0
+ log 'write initial data, read from all'
+ set +o xtrace
[2026-06-18T21:24:36+0000] write initial data, read from all
+ run_mongos 'use myApp\n db.test.insert({ x: 100500 })' myApp:myPass@some-name-mongos.pitr-physical-backup-source-28616
+ local 'command=use myApp\n db.test.insert({ x: 100500 })'
+ local uri=myApp:myPass@some-name-mongos.pitr-physical-backup-source-28616
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local port=27017
+ local mongo_bin=mongo
++ echo .svc.cluster.local
++ awk -F: '{print $2}'
+ suffix_port=
+ [[ -z '' ]]
+ suffix=.svc.cluster.local:27017
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.xB7zrolFQ0
+++ mktemp
++ local LAST_ERR=/tmp/tmp.zvk9304WUk
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.xB7zrolFQ0
++ cat /tmp/tmp.zvk9304WUk
++ rm /tmp/tmp.xB7zrolFQ0 /tmp/tmp.zvk9304WUk
++ return 0
+ local client_container=psmdb-client-bb8b97679-8kmm8
+ kubectl_bin exec psmdb-client-bb8b97679-8kmm8 -- bash -c 'printf '\''use myApp\n db.test.insert({ x: 100500 })\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.pitr-physical-backup-source-28616.svc.cluster.local:27017/admin '
++ mktemp
+ local LAST_OUT=/tmp/tmp.SIiWhYHi3J
++ mktemp
+ local LAST_ERR=/tmp/tmp.d3AJU0gwov
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-bb8b97679-8kmm8 -- bash -c 'printf '\''use myApp\n db.test.insert({ x: 100500 })\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.pitr-physical-backup-source-28616.svc.cluster.local:27017/admin '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.SIiWhYHi3J
Percona Server for MongoDB shell version v4.4.29-28
connecting to: mongodb://some-name-mongos.pitr-physical-backup-source-28616.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb
Implicit session: session { "id" : UUID("2c5fa34b-2d87-4414-8db1-f087752af240") }
Percona Server for MongoDB server version: v8.0.23-10
WARNING: shell and server versions do not match
switched to db myApp
WriteResult({ "nInserted" : 1 })
bye
+ cat /tmp/tmp.d3AJU0gwov
+ rm /tmp/tmp.SIiWhYHi3J /tmp/tmp.d3AJU0gwov
+ return 0
+ sleep 0
+ compare_mongos_cmd find myApp:myPass@some-name-mongos.pitr-physical-backup-source-28616
+ local command=find
+ local uri=myApp:myPass@some-name-mongos.pitr-physical-backup-source-28616
+ local postfix=
+ local suffix=
+ local database=myApp
+ local collection=test
+ local port=27017
+ local tls=false
+ [[ false == true ]]
+ mongos_command=run_mongos
+ log 'running db.test.command() in myApp'
+ set +o xtrace
[2026-06-18T21:24:37+0000] running db.test.command() in myApp
+ run_mongos 'use myApp\n db.test.find()' myApp:myPass@some-name-mongos.pitr-physical-backup-source-28616 mongodb '' '' 27017
+ grep -E -v 'I NETWORK|W NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|Started a new thread for the timer service'
+ local 'command=use myApp\n db.test.find()'
+ local uri=myApp:myPass@some-name-mongos.pitr-physical-backup-source-28616
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local port=27017
+ local mongo_bin=mongo
+ /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
++ echo .svc.cluster.local
++ awk -F: '{print $2}'
+ suffix_port=
+ [[ -z '' ]]
+ suffix=.svc.cluster.local:27017
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.2S3b09tlJ5
+++ mktemp
++ local LAST_ERR=/tmp/tmp.tzYDKrrSW7
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.2S3b09tlJ5
++ cat /tmp/tmp.tzYDKrrSW7
++ rm /tmp/tmp.2S3b09tlJ5 /tmp/tmp.tzYDKrrSW7
++ return 0
+ local client_container=psmdb-client-bb8b97679-8kmm8
+ kubectl_bin exec psmdb-client-bb8b97679-8kmm8 -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.pitr-physical-backup-source-28616.svc.cluster.local:27017/admin '
++ mktemp
+ local LAST_OUT=/tmp/tmp.6s4I6ykVue
++ mktemp
+ local LAST_ERR=/tmp/tmp.BVgLbyt43I
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-bb8b97679-8kmm8 -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.pitr-physical-backup-source-28616.svc.cluster.local:27017/admin '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.6s4I6ykVue
+ cat /tmp/tmp.BVgLbyt43I
+ rm /tmp/tmp.6s4I6ykVue /tmp/tmp.BVgLbyt43I
+ return 0
+ [[ 0 -eq 0 ]]
+ diff /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2406/e2e-tests/pitr-physical-backup-source/compare/find.json /tmp/tmp.4E31yTnNi1/find
+ desc 'restore pitr type latest using backupSource'
+ set +o xtrace


-----------------------------------------------------------------------------------
restore pitr type latest using backupSource
-----------------------------------------------------------------------------------

+ write_document -2nd
+ local cmp_postfix=-2nd
+ local sleep_value=0
+ log 'write initial data, read from all'
+ set +o xtrace
[2026-06-18T21:24:39+0000] write initial data, read from all
+ run_mongos 'use myApp\n db.test.insert({ x: 100500 })' myApp:myPass@some-name-mongos.pitr-physical-backup-source-28616
+ local 'command=use myApp\n db.test.insert({ x: 100500 })'
+ local uri=myApp:myPass@some-name-mongos.pitr-physical-backup-source-28616
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local port=27017
+ local mongo_bin=mongo
++ echo .svc.cluster.local
++ awk -F: '{print $2}'
+ suffix_port=
+ [[ -z '' ]]
+ suffix=.svc.cluster.local:27017
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.3BIB2bRHDA
+++ mktemp
++ local LAST_ERR=/tmp/tmp.trz0VQI4lh
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.3BIB2bRHDA
++ cat /tmp/tmp.trz0VQI4lh
++ rm /tmp/tmp.3BIB2bRHDA /tmp/tmp.trz0VQI4lh
++ return 0
+ local client_container=psmdb-client-bb8b97679-8kmm8
+ kubectl_bin exec psmdb-client-bb8b97679-8kmm8 -- bash -c 'printf '\''use myApp\n db.test.insert({ x: 100500 })\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.pitr-physical-backup-source-28616.svc.cluster.local:27017/admin '
++ mktemp
+ local LAST_OUT=/tmp/tmp.RZqd3cYFjy
++ mktemp
+ local LAST_ERR=/tmp/tmp.pcmUoiJXJW
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-bb8b97679-8kmm8 -- bash -c 'printf '\''use myApp\n db.test.insert({ x: 100500 })\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.pitr-physical-backup-source-28616.svc.cluster.local:27017/admin '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.RZqd3cYFjy
Percona Server for MongoDB shell version v4.4.29-28
connecting to: mongodb://some-name-mongos.pitr-physical-backup-source-28616.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb
Implicit session: session { "id" : UUID("f0dd9428-21ce-4669-8c74-65eb6d75cd28") }
Percona Server for MongoDB server version: v8.0.23-10
WARNING: shell and server versions do not match
switched to db myApp
WriteResult({ "nInserted" : 1 })
bye
+ cat /tmp/tmp.pcmUoiJXJW
+ rm /tmp/tmp.RZqd3cYFjy /tmp/tmp.pcmUoiJXJW
+ return 0
+ sleep 0
+ compare_mongos_cmd find myApp:myPass@some-name-mongos.pitr-physical-backup-source-28616 -2nd
+ local command=find
+ local uri=myApp:myPass@some-name-mongos.pitr-physical-backup-source-28616
+ local postfix=-2nd
+ local suffix=
+ local database=myApp
+ local collection=test
+ local port=27017
+ local tls=false
+ [[ false == true ]]
+ mongos_command=run_mongos
+ log 'running db.test.command() in myApp'
+ set +o xtrace
[2026-06-18T21:24:41+0000] running db.test.command() in myApp
+ run_mongos 'use myApp\n db.test.find()' myApp:myPass@some-name-mongos.pitr-physical-backup-source-28616 mongodb '' '' 27017
+ local 'command=use myApp\n db.test.find()'
+ local uri=myApp:myPass@some-name-mongos.pitr-physical-backup-source-28616
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local port=27017
+ grep -E -v 'I NETWORK|W NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|Started a new thread for the timer service'
+ local mongo_bin=mongo
+ /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
++ echo .svc.cluster.local
++ awk -F: '{print $2}'
+ suffix_port=
+ [[ -z '' ]]
+ suffix=.svc.cluster.local:27017
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.STCOjLK9x9
+++ mktemp
++ local LAST_ERR=/tmp/tmp.dAkjLhZz91
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.STCOjLK9x9
++ cat /tmp/tmp.dAkjLhZz91
++ rm /tmp/tmp.STCOjLK9x9 /tmp/tmp.dAkjLhZz91
++ return 0
+ local client_container=psmdb-client-bb8b97679-8kmm8
+ kubectl_bin exec psmdb-client-bb8b97679-8kmm8 -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.pitr-physical-backup-source-28616.svc.cluster.local:27017/admin '
++ mktemp
+ local LAST_OUT=/tmp/tmp.kwtREp8edc
++ mktemp
+ local LAST_ERR=/tmp/tmp.2PlG8Iep8a
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-bb8b97679-8kmm8 -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.pitr-physical-backup-source-28616.svc.cluster.local:27017/admin '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.kwtREp8edc
+ cat /tmp/tmp.2PlG8Iep8a
+ rm /tmp/tmp.kwtREp8edc /tmp/tmp.2PlG8Iep8a
+ return 0
+ [[ 0 -eq 0 ]]
+ diff /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2406/e2e-tests/pitr-physical-backup-source/compare/find-2nd.json /tmp/tmp.4E31yTnNi1/find-2nd
+ run_backup backup-minio 2 physical
+ local name=backup-minio
+ local idx=2
+ local type=physical
+ desc 'run backup backup-minio-2'
+ set +o xtrace


-----------------------------------------------------------------------------------
run backup backup-minio-2
-----------------------------------------------------------------------------------

+ cat /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2406/e2e-tests/pitr-physical-backup-source/conf/backup-minio.yml
+ /usr/sbin/sed -e 's/name:/name: backup-minio-2/'
+ /usr/sbin/sed -e 's/type:/type: physical/'
+ kubectl_bin apply -f -
++ mktemp
+ local LAST_OUT=/tmp/tmp.rYmJlrBnoC
++ mktemp
+ local LAST_ERR=/tmp/tmp.2JhQcEXqLf
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl apply -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.rYmJlrBnoC
perconaservermongodbbackup.psmdb.percona.com/backup-minio-2 created
+ cat /tmp/tmp.2JhQcEXqLf
+ rm /tmp/tmp.rYmJlrBnoC /tmp/tmp.2JhQcEXqLf
+ return 0
+ wait_backup backup-minio-2
+ local backup_name=backup-minio-2
+ local target_state=ready
+ set +o xtrace
waiting for backup-minio-2 to reach ready state............OK
+ compare_latest_restorable_time some-name-rs0 backup-minio-2
+ local cluster=some-name-rs0
+ local backup_name=backup-minio-2
+ local latest_restorable_time
+ local backup_time
++ get_latest_restorable_time some-name-rs0
++ local cluster=some-name-rs0
++ local first_timestamp
++ local second_timestamp
++ local retry=0
++ [[ '' != '' ]]
+++ kubectl_bin exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ jq '.backups.pitrChunks.pitrChunks | last | .range.end'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.zPO3jmT3HS
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.ydN5wS6vzZ
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in $(seq 0 2)
+++ set +e
+++ kubectl exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.zPO3jmT3HS
+++ cat /tmp/tmp.ydN5wS6vzZ
+++ rm /tmp/tmp.zPO3jmT3HS /tmp/tmp.ydN5wS6vzZ
+++ return 0
++ first_timestamp=1781816856
++ sleep 5
++ [[ 1781816856 != '' ]]
++ [[ 1781816856 != null ]]
+++ kubectl_bin exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ jq '.backups.pitrChunks.pitrChunks | last | .range.end'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.h3IHQURVFO
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.NjowsIuoBm
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in $(seq 0 2)
+++ set +e
+++ kubectl exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.h3IHQURVFO
+++ cat /tmp/tmp.NjowsIuoBm
+++ rm /tmp/tmp.h3IHQURVFO /tmp/tmp.NjowsIuoBm
+++ return 0
++ second_timestamp=1781816856
++ let retry+=1
++ [[ 1 -gt 30 ]]
++ [[ 1781816856 != '' ]]
++ [[ 1781816856 != null ]]
++ [[ 1781816856 == 1781816856 ]]
++ /usr/sbin/date -u -d @1781816856 +%Y-%m-%dT%H:%M:%SZ
+ latest_restorable_time=2026-06-18T21:07:36Z
++ get_latest_restorable_time_from_backup_object backup-minio-2
++ local backup_name=backup-minio-2
++ local latestRestorableTime
++ local retry=0
++ [[ '' != '' ]]
++ sleep 5
+++ kubectl_bin get psmdb-backup backup-minio-2 -o 'jsonpath={.status.latestRestorableTime}'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.k5nONpbfEK
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.A9GJddFGbz
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in $(seq 0 2)
+++ set +e
+++ kubectl get psmdb-backup backup-minio-2 -o 'jsonpath={.status.latestRestorableTime}'
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.k5nONpbfEK
+++ cat /tmp/tmp.A9GJddFGbz
+++ rm /tmp/tmp.k5nONpbfEK /tmp/tmp.A9GJddFGbz
+++ return 0
++ latestRestorableTime=2026-06-18T21:07:36Z
++ let retry+=1
++ [[ 1 -gt 30 ]]
++ [[ 2026-06-18T21:07:36Z != '' ]]
++ [[ 2026-06-18T21:07:36Z != null ]]
++ echo 2026-06-18T21:07:36Z
+ backup_time=2026-06-18T21:07:36Z
+ [[ 2026-06-18T21:07:36Z != 2026\-06\-18T21\:07\:36Z ]]
+ check_recovery backup-minio-2 latest '' -3rd some-name backupSource
+ local backup_name=backup-minio-2
+ local restore_type=latest
+ local restore_date=
+ local cmp_postfix=-3rd
+ local cluster_name=some-name
+ local backupSource=backupSource
++ get_latest_oplog_chunk_ts some-name
++ local cluster=some-name
+++ kubectl_bin exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ jq '.backups.pitrChunks.pitrChunks | last | .range.end'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.11vhEmLJyd
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.C0Wa7JqMr5
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in $(seq 0 2)
+++ set +e
+++ kubectl exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.11vhEmLJyd
+++ cat /tmp/tmp.C0Wa7JqMr5
+++ rm /tmp/tmp.11vhEmLJyd /tmp/tmp.C0Wa7JqMr5
+++ return 0
++ echo 1781816856
+ local latest_ts=1781816856
+ desc 'write more data before restore by latest'
+ set +o xtrace


-----------------------------------------------------------------------------------
write more data before restore by latest
-----------------------------------------------------------------------------------

+ run_mongos 'use myApp\n db.test.insert({ x: 100501 })' myApp:myPass@some-name-mongos.pitr-physical-backup-source-28616
+ local 'command=use myApp\n db.test.insert({ x: 100501 })'
+ local uri=myApp:myPass@some-name-mongos.pitr-physical-backup-source-28616
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local port=27017
+ local mongo_bin=mongo
++ echo .svc.cluster.local
++ awk -F: '{print $2}'
+ suffix_port=
+ [[ -z '' ]]
+ suffix=.svc.cluster.local:27017
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.KQnYAas6Bu
+++ mktemp
++ local LAST_ERR=/tmp/tmp.nn7A3dYbhY
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.KQnYAas6Bu
++ cat /tmp/tmp.nn7A3dYbhY
++ rm /tmp/tmp.KQnYAas6Bu /tmp/tmp.nn7A3dYbhY
++ return 0
+ local client_container=psmdb-client-bb8b97679-8kmm8
+ kubectl_bin exec psmdb-client-bb8b97679-8kmm8 -- bash -c 'printf '\''use myApp\n db.test.insert({ x: 100501 })\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.pitr-physical-backup-source-28616.svc.cluster.local:27017/admin '
++ mktemp
+ local LAST_OUT=/tmp/tmp.vcCu9oM3lQ
++ mktemp
+ local LAST_ERR=/tmp/tmp.6UtSRJ3OqE
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-bb8b97679-8kmm8 -- bash -c 'printf '\''use myApp\n db.test.insert({ x: 100501 })\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.pitr-physical-backup-source-28616.svc.cluster.local:27017/admin '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.vcCu9oM3lQ
Percona Server for MongoDB shell version v4.4.29-28
connecting to: mongodb://some-name-mongos.pitr-physical-backup-source-28616.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb
Implicit session: session { "id" : UUID("e5e5378f-6ae4-4d34-b114-cabecaa30430") }
Percona Server for MongoDB server version: v8.0.23-10
WARNING: shell and server versions do not match
switched to db myApp
WriteResult({ "nInserted" : 1 })
bye
+ cat /tmp/tmp.6UtSRJ3OqE
+ rm /tmp/tmp.vcCu9oM3lQ /tmp/tmp.6UtSRJ3OqE
+ return 0
+ [[ -n '' ]]
+ desc 'Restoring to latest'
+ set +o xtrace


-----------------------------------------------------------------------------------
Restoring to latest
-----------------------------------------------------------------------------------

++ get_latest_oplog_chunk_ts some-name
++ local cluster=some-name
+++ kubectl_bin exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ jq '.backups.pitrChunks.pitrChunks | last | .range.end'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.WBiJUj9MZO
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.CPrq2gD2fm
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in $(seq 0 2)
+++ set +e
+++ kubectl exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.WBiJUj9MZO
+++ cat /tmp/tmp.CPrq2gD2fm
+++ rm /tmp/tmp.WBiJUj9MZO /tmp/tmp.CPrq2gD2fm
+++ return 0
++ echo 1781816856
+ local current_ts=1781816856
+ retries=0
+ [[ 1781816856 -gt 1781816856 ]]
+ [[ 0 -gt 30 ]]
++ get_latest_oplog_chunk_ts some-name
++ local cluster=some-name
+++ kubectl_bin exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ jq '.backups.pitrChunks.pitrChunks | last | .range.end'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.vycAphdCSz
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.9tseeolChC
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in $(seq 0 2)
+++ set +e
+++ kubectl exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.vycAphdCSz
+++ cat /tmp/tmp.9tseeolChC
+++ rm /tmp/tmp.vycAphdCSz /tmp/tmp.9tseeolChC
+++ return 0
++ echo 1781816856
+ latest_ts=1781816856
+ retries=1
++ format_date 1781816856
++ local timestamp=1781816856
+++ TZ=UTC
+++ /usr/sbin/date -d@1781816856 '+%Y-%m-%d %H:%M:%S'
++ echo 2026-06-18 21:07:36
++ format_date 1781816856
++ local timestamp=1781816856
+++ TZ=UTC
+++ /usr/sbin/date -d@1781816856 '+%Y-%m-%d %H:%M:%S'
++ echo 2026-06-18 21:07:36
+ echo 'Waiting for last oplog chunk (2026-06-18 21:07:36) to be 120 seconds older than starting chunk (2026-06-18 21:07:36)'
Waiting for last oplog chunk (2026-06-18 21:07:36) to be 120 seconds older than starting chunk (2026-06-18 21:07:36)
+ sleep 10
+ [[ 1781816856 -gt 1781816856 ]]
+ [[ 1 -gt 30 ]]
++ get_latest_oplog_chunk_ts some-name
++ local cluster=some-name
+++ kubectl_bin exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ jq '.backups.pitrChunks.pitrChunks | last | .range.end'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.Yu8a2bTKwD
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.1IpvQHLIOX
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in $(seq 0 2)
+++ set +e
+++ kubectl exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.Yu8a2bTKwD
+++ cat /tmp/tmp.1IpvQHLIOX
+++ rm /tmp/tmp.Yu8a2bTKwD /tmp/tmp.1IpvQHLIOX
+++ return 0
++ echo 1781816856
+ latest_ts=1781816856
+ retries=2
++ format_date 1781816856
++ local timestamp=1781816856
+++ TZ=UTC
+++ /usr/sbin/date -d@1781816856 '+%Y-%m-%d %H:%M:%S'
++ echo 2026-06-18 21:07:36
++ format_date 1781816856
++ local timestamp=1781816856
+++ TZ=UTC
+++ /usr/sbin/date -d@1781816856 '+%Y-%m-%d %H:%M:%S'
++ echo 2026-06-18 21:07:36
+ echo 'Waiting for last oplog chunk (2026-06-18 21:07:36) to be 120 seconds older than starting chunk (2026-06-18 21:07:36)'
Waiting for last oplog chunk (2026-06-18 21:07:36) to be 120 seconds older than starting chunk (2026-06-18 21:07:36)
+ sleep 10
+ [[ 1781816856 -gt 1781816856 ]]
+ [[ 2 -gt 30 ]]
++ get_latest_oplog_chunk_ts some-name
++ local cluster=some-name
+++ kubectl_bin exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ jq '.backups.pitrChunks.pitrChunks | last | .range.end'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.xScuKzcJGN
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.frtPkavDV7
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in $(seq 0 2)
+++ set +e
+++ kubectl exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.xScuKzcJGN
+++ cat /tmp/tmp.frtPkavDV7
+++ rm /tmp/tmp.xScuKzcJGN /tmp/tmp.frtPkavDV7
+++ return 0
++ echo 1781816856
+ latest_ts=1781816856
+ retries=3
++ format_date 1781816856
++ local timestamp=1781816856
+++ TZ=UTC
+++ /usr/sbin/date -d@1781816856 '+%Y-%m-%d %H:%M:%S'
++ echo 2026-06-18 21:07:36
++ format_date 1781816856
++ local timestamp=1781816856
+++ TZ=UTC
+++ /usr/sbin/date -d@1781816856 '+%Y-%m-%d %H:%M:%S'
++ echo 2026-06-18 21:07:36
+ echo 'Waiting for last oplog chunk (2026-06-18 21:07:36) to be 120 seconds older than starting chunk (2026-06-18 21:07:36)'
Waiting for last oplog chunk (2026-06-18 21:07:36) to be 120 seconds older than starting chunk (2026-06-18 21:07:36)
+ sleep 10
+ [[ 1781816856 -gt 1781816856 ]]
+ [[ 3 -gt 30 ]]
++ get_latest_oplog_chunk_ts some-name
++ local cluster=some-name
+++ kubectl_bin exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ jq '.backups.pitrChunks.pitrChunks | last | .range.end'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.wcVQk1E4Zr
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.z19X9vtSUS
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in $(seq 0 2)
+++ set +e
+++ kubectl exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.wcVQk1E4Zr
+++ cat /tmp/tmp.z19X9vtSUS
+++ rm /tmp/tmp.wcVQk1E4Zr /tmp/tmp.z19X9vtSUS
+++ return 0
++ echo 1781816856
+ latest_ts=1781816856
+ retries=4
++ format_date 1781816856
++ local timestamp=1781816856
+++ TZ=UTC
+++ /usr/sbin/date -d@1781816856 '+%Y-%m-%d %H:%M:%S'
++ echo 2026-06-18 21:07:36
++ format_date 1781816856
++ local timestamp=1781816856
+++ TZ=UTC
+++ /usr/sbin/date -d@1781816856 '+%Y-%m-%d %H:%M:%S'
++ echo 2026-06-18 21:07:36
+ echo 'Waiting for last oplog chunk (2026-06-18 21:07:36) to be 120 seconds older than starting chunk (2026-06-18 21:07:36)'
Waiting for last oplog chunk (2026-06-18 21:07:36) to be 120 seconds older than starting chunk (2026-06-18 21:07:36)
+ sleep 10
+ [[ 1781816856 -gt 1781816856 ]]
+ [[ 4 -gt 30 ]]
++ get_latest_oplog_chunk_ts some-name
++ local cluster=some-name
+++ kubectl_bin exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ jq '.backups.pitrChunks.pitrChunks | last | .range.end'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.NaYn6caRFi
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.EgQoNsN14O
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in $(seq 0 2)
+++ set +e
+++ kubectl exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.NaYn6caRFi
+++ cat /tmp/tmp.EgQoNsN14O
+++ rm /tmp/tmp.NaYn6caRFi /tmp/tmp.EgQoNsN14O
+++ return 0
++ echo 1781816856
+ latest_ts=1781816856
+ retries=5
++ format_date 1781816856
++ local timestamp=1781816856
+++ TZ=UTC
+++ /usr/sbin/date -d@1781816856 '+%Y-%m-%d %H:%M:%S'
++ echo 2026-06-18 21:07:36
++ format_date 1781816856
++ local timestamp=1781816856
+++ TZ=UTC
+++ /usr/sbin/date -d@1781816856 '+%Y-%m-%d %H:%M:%S'
++ echo 2026-06-18 21:07:36
+ echo 'Waiting for last oplog chunk (2026-06-18 21:07:36) to be 120 seconds older than starting chunk (2026-06-18 21:07:36)'
Waiting for last oplog chunk (2026-06-18 21:07:36) to be 120 seconds older than starting chunk (2026-06-18 21:07:36)
+ sleep 10
+ [[ 1781816856 -gt 1781816856 ]]
+ [[ 5 -gt 30 ]]
++ get_latest_oplog_chunk_ts some-name
++ local cluster=some-name
+++ kubectl_bin exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ jq '.backups.pitrChunks.pitrChunks | last | .range.end'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.Iwhs5JsDCk
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.C0M7lgxCyN
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in $(seq 0 2)
+++ set +e
+++ kubectl exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.Iwhs5JsDCk
+++ cat /tmp/tmp.C0M7lgxCyN
+++ rm /tmp/tmp.Iwhs5JsDCk /tmp/tmp.C0M7lgxCyN
+++ return 0
++ echo 1781816856
+ latest_ts=1781816856
+ retries=6
++ format_date 1781816856
++ local timestamp=1781816856
+++ TZ=UTC
+++ /usr/sbin/date -d@1781816856 '+%Y-%m-%d %H:%M:%S'
++ echo 2026-06-18 21:07:36
++ format_date 1781816856
++ local timestamp=1781816856
+++ TZ=UTC
+++ /usr/sbin/date -d@1781816856 '+%Y-%m-%d %H:%M:%S'
++ echo 2026-06-18 21:07:36
+ echo 'Waiting for last oplog chunk (2026-06-18 21:07:36) to be 120 seconds older than starting chunk (2026-06-18 21:07:36)'
Waiting for last oplog chunk (2026-06-18 21:07:36) to be 120 seconds older than starting chunk (2026-06-18 21:07:36)
+ sleep 10
+ [[ 1781816856 -gt 1781816856 ]]
+ [[ 6 -gt 30 ]]
++ get_latest_oplog_chunk_ts some-name
++ local cluster=some-name
+++ kubectl_bin exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ jq '.backups.pitrChunks.pitrChunks | last | .range.end'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.seJpz34zNK
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.G74htP7IKv
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in $(seq 0 2)
+++ set +e
+++ kubectl exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.seJpz34zNK
+++ cat /tmp/tmp.G74htP7IKv
+++ rm /tmp/tmp.seJpz34zNK /tmp/tmp.G74htP7IKv
+++ return 0
++ echo 1781816856
+ latest_ts=1781816856
+ retries=7
++ format_date 1781816856
++ local timestamp=1781816856
+++ TZ=UTC
+++ /usr/sbin/date -d@1781816856 '+%Y-%m-%d %H:%M:%S'
++ echo 2026-06-18 21:07:36
++ format_date 1781816856
++ local timestamp=1781816856
+++ TZ=UTC
+++ /usr/sbin/date -d@1781816856 '+%Y-%m-%d %H:%M:%S'
++ echo 2026-06-18 21:07:36
+ echo 'Waiting for last oplog chunk (2026-06-18 21:07:36) to be 120 seconds older than starting chunk (2026-06-18 21:07:36)'
Waiting for last oplog chunk (2026-06-18 21:07:36) to be 120 seconds older than starting chunk (2026-06-18 21:07:36)
+ sleep 10
+ [[ 1781816856 -gt 1781816856 ]]
+ [[ 7 -gt 30 ]]
++ get_latest_oplog_chunk_ts some-name
++ local cluster=some-name
+++ kubectl_bin exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ jq '.backups.pitrChunks.pitrChunks | last | .range.end'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.dRnWx6B1N3
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.Z5QmnDdIUK
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in $(seq 0 2)
+++ set +e
+++ kubectl exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.dRnWx6B1N3
+++ cat /tmp/tmp.Z5QmnDdIUK
+++ rm /tmp/tmp.dRnWx6B1N3 /tmp/tmp.Z5QmnDdIUK
+++ return 0
++ echo 1781816856
+ latest_ts=1781816856
+ retries=8
++ format_date 1781816856
++ local timestamp=1781816856
+++ TZ=UTC
+++ /usr/sbin/date -d@1781816856 '+%Y-%m-%d %H:%M:%S'
++ echo 2026-06-18 21:07:36
++ format_date 1781816856
++ local timestamp=1781816856
+++ TZ=UTC
+++ /usr/sbin/date -d@1781816856 '+%Y-%m-%d %H:%M:%S'
++ echo 2026-06-18 21:07:36
+ echo 'Waiting for last oplog chunk (2026-06-18 21:07:36) to be 120 seconds older than starting chunk (2026-06-18 21:07:36)'
Waiting for last oplog chunk (2026-06-18 21:07:36) to be 120 seconds older than starting chunk (2026-06-18 21:07:36)
+ sleep 10
+ [[ 1781816856 -gt 1781816856 ]]
+ [[ 8 -gt 30 ]]
++ get_latest_oplog_chunk_ts some-name
++ local cluster=some-name
+++ jq '.backups.pitrChunks.pitrChunks | last | .range.end'
+++ kubectl_bin exec some-name-rs0-0 -c backup-agent -- pbm status -o json
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.qInxG1AzHF
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.YtBLUmEsLC
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in $(seq 0 2)
+++ set +e
+++ kubectl exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.qInxG1AzHF
+++ cat /tmp/tmp.YtBLUmEsLC
+++ rm /tmp/tmp.qInxG1AzHF /tmp/tmp.YtBLUmEsLC
+++ return 0
++ echo 1781816856
+ latest_ts=1781816856
+ retries=9
++ format_date 1781816856
++ local timestamp=1781816856
+++ TZ=UTC
+++ /usr/sbin/date -d@1781816856 '+%Y-%m-%d %H:%M:%S'
++ echo 2026-06-18 21:07:36
++ format_date 1781816856
++ local timestamp=1781816856
+++ TZ=UTC
+++ /usr/sbin/date -d@1781816856 '+%Y-%m-%d %H:%M:%S'
++ echo 2026-06-18 21:07:36
+ echo 'Waiting for last oplog chunk (2026-06-18 21:07:36) to be 120 seconds older than starting chunk (2026-06-18 21:07:36)'
Waiting for last oplog chunk (2026-06-18 21:07:36) to be 120 seconds older than starting chunk (2026-06-18 21:07:36)
+ sleep 10
+ [[ 1781816856 -gt 1781816856 ]]
+ [[ 9 -gt 30 ]]
++ get_latest_oplog_chunk_ts some-name
++ local cluster=some-name
+++ kubectl_bin exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ jq '.backups.pitrChunks.pitrChunks | last | .range.end'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.LuXnHCoMW8
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.W4EP5zs97Q
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in $(seq 0 2)
+++ set +e
+++ kubectl exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.LuXnHCoMW8
+++ cat /tmp/tmp.W4EP5zs97Q
+++ rm /tmp/tmp.LuXnHCoMW8 /tmp/tmp.W4EP5zs97Q
+++ return 0
++ echo 1781816856
+ latest_ts=1781816856
+ retries=10
++ format_date 1781816856
++ local timestamp=1781816856
+++ TZ=UTC
+++ /usr/sbin/date -d@1781816856 '+%Y-%m-%d %H:%M:%S'
++ echo 2026-06-18 21:07:36
++ format_date 1781816856
++ local timestamp=1781816856
+++ TZ=UTC
+++ /usr/sbin/date -d@1781816856 '+%Y-%m-%d %H:%M:%S'
++ echo 2026-06-18 21:07:36
+ echo 'Waiting for last oplog chunk (2026-06-18 21:07:36) to be 120 seconds older than starting chunk (2026-06-18 21:07:36)'
Waiting for last oplog chunk (2026-06-18 21:07:36) to be 120 seconds older than starting chunk (2026-06-18 21:07:36)
+ sleep 10
+ [[ 1781816856 -gt 1781816856 ]]
+ [[ 10 -gt 30 ]]
++ get_latest_oplog_chunk_ts some-name
++ local cluster=some-name
+++ kubectl_bin exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ jq '.backups.pitrChunks.pitrChunks | last | .range.end'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.ZKsNWcWBjv
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.Kh1S0N1hhV
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in $(seq 0 2)
+++ set +e
+++ kubectl exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.ZKsNWcWBjv
+++ cat /tmp/tmp.Kh1S0N1hhV
+++ rm /tmp/tmp.ZKsNWcWBjv /tmp/tmp.Kh1S0N1hhV
+++ return 0
++ echo 1781816856
+ latest_ts=1781816856
+ retries=11
++ format_date 1781816856
++ local timestamp=1781816856
+++ TZ=UTC
+++ /usr/sbin/date -d@1781816856 '+%Y-%m-%d %H:%M:%S'
++ echo 2026-06-18 21:07:36
++ format_date 1781816856
++ local timestamp=1781816856
+++ TZ=UTC
+++ /usr/sbin/date -d@1781816856 '+%Y-%m-%d %H:%M:%S'
++ echo 2026-06-18 21:07:36
+ echo 'Waiting for last oplog chunk (2026-06-18 21:07:36) to be 120 seconds older than starting chunk (2026-06-18 21:07:36)'
Waiting for last oplog chunk (2026-06-18 21:07:36) to be 120 seconds older than starting chunk (2026-06-18 21:07:36)
+ sleep 10
+ [[ 1781816856 -gt 1781816856 ]]
+ [[ 11 -gt 30 ]]
++ get_latest_oplog_chunk_ts some-name
++ local cluster=some-name
+++ kubectl_bin exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ jq '.backups.pitrChunks.pitrChunks | last | .range.end'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.EuGbR5ZrPZ
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.emQnMP0ENr
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in $(seq 0 2)
+++ set +e
+++ kubectl exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.EuGbR5ZrPZ
+++ cat /tmp/tmp.emQnMP0ENr
+++ rm /tmp/tmp.EuGbR5ZrPZ /tmp/tmp.emQnMP0ENr
+++ return 0
++ echo 1781816856
+ latest_ts=1781816856
+ retries=12
++ format_date 1781816856
++ local timestamp=1781816856
+++ TZ=UTC
+++ /usr/sbin/date -d@1781816856 '+%Y-%m-%d %H:%M:%S'
++ echo 2026-06-18 21:07:36
++ format_date 1781816856
++ local timestamp=1781816856
+++ TZ=UTC
+++ /usr/sbin/date -d@1781816856 '+%Y-%m-%d %H:%M:%S'
++ echo 2026-06-18 21:07:36
+ echo 'Waiting for last oplog chunk (2026-06-18 21:07:36) to be 120 seconds older than starting chunk (2026-06-18 21:07:36)'
Waiting for last oplog chunk (2026-06-18 21:07:36) to be 120 seconds older than starting chunk (2026-06-18 21:07:36)
+ sleep 10
+ [[ 1781816856 -gt 1781816856 ]]
+ [[ 12 -gt 30 ]]
++ get_latest_oplog_chunk_ts some-name
++ local cluster=some-name
+++ kubectl_bin exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ jq '.backups.pitrChunks.pitrChunks | last | .range.end'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.LILxEISQGw
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.1FOwRTTCCt
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in $(seq 0 2)
+++ set +e
+++ kubectl exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.LILxEISQGw
+++ cat /tmp/tmp.1FOwRTTCCt
+++ rm /tmp/tmp.LILxEISQGw /tmp/tmp.1FOwRTTCCt
+++ return 0
++ echo 1781818044
+ latest_ts=1781818044
+ retries=13
++ format_date 1781818044
++ local timestamp=1781818044
+++ TZ=UTC
+++ /usr/sbin/date -d@1781818044 '+%Y-%m-%d %H:%M:%S'
++ echo 2026-06-18 21:27:24
++ format_date 1781816856
++ local timestamp=1781816856
+++ TZ=UTC
+++ /usr/sbin/date -d@1781816856 '+%Y-%m-%d %H:%M:%S'
++ echo 2026-06-18 21:07:36
+ echo 'Waiting for last oplog chunk (2026-06-18 21:27:24) to be 120 seconds older than starting chunk (2026-06-18 21:07:36)'
Waiting for last oplog chunk (2026-06-18 21:27:24) to be 120 seconds older than starting chunk (2026-06-18 21:07:36)
+ sleep 10
+ [[ 1781818044 -gt 1781816856 ]]
+ '[' -z backupSource ']'
+ desc 'check restore by latest backupSource'
+ set +o xtrace


-----------------------------------------------------------------------------------
check restore by latest backupSource
-----------------------------------------------------------------------------------

++ get_backup_dest backup-minio-2
++ local backup_name=backup-minio-2
++ kubectl_bin get psmdb-backup backup-minio-2 -o 'jsonpath={.status.destination}'
++ sed -e 's/.json$//'
++ sed 's|s3://||'
++ sed 's|azure://||'
+++ mktemp
++ sed 's|gs://||'
++ local LAST_OUT=/tmp/tmp.MtpUjVjKSP
+++ mktemp
++ local LAST_ERR=/tmp/tmp.JXusBtUqcD
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb-backup backup-minio-2 -o 'jsonpath={.status.destination}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.MtpUjVjKSP
++ cat /tmp/tmp.JXusBtUqcD
++ rm /tmp/tmp.MtpUjVjKSP /tmp/tmp.JXusBtUqcD
++ return 0
+ backup_dest=operator-testing/2026-06-18T21:24:45Z
+ cat /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2406/e2e-tests/pitr-physical-backup-source/conf/restore.yml
+ /usr/sbin/sed -e 's/name:/name: restore-backup-minio-2/'
+ /usr/sbin/sed -e /backupName/d
+ /usr/sbin/sed -e 's/pitrType:/type: latest/'
+ '[' -z '' ']'
+ /usr/sbin/sed -e /date:/d
+ /usr/sbin/sed -e 's|DESTINATION|operator-testing/2026-06-18T21:24:45Z|'
+ '[' -n '' ']'
+ yq
+ kubectl_bin apply -f -
++ get_bucket_name backup-minio-2
++ local backup_name=backup-minio-2
++ kubectl_bin get psmdb-backup backup-minio-2 -o 'jsonpath={.status.s3.bucket}'
++ mktemp
+++ mktemp
+ local LAST_OUT=/tmp/tmp.RKiPPGuQje
++ local LAST_OUT=/tmp/tmp.knDw7ryPxB
+++ mktemp
++ mktemp
++ local LAST_ERR=/tmp/tmp.B2NaLTMjYL
++ local exit_status=0
++ local timeout=4
+ local LAST_ERR=/tmp/tmp.2zNgzvhZyj
+ local exit_status=0
+ local timeout=4
+++ seq 0 2
++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb-backup backup-minio-2 -o 'jsonpath={.status.s3.bucket}'
+ for i in $(seq 0 2)
+ set +e
+ kubectl apply -f -
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.knDw7ryPxB
++ cat /tmp/tmp.B2NaLTMjYL
++ rm /tmp/tmp.knDw7ryPxB /tmp/tmp.B2NaLTMjYL
++ return 0
+ /usr/sbin/sed -e 's|BUCKET-NAME|operator-testing|'
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.RKiPPGuQje
perconaservermongodbrestore.psmdb.percona.com/restore-backup-minio-2 created
+ cat /tmp/tmp.2zNgzvhZyj
+ rm /tmp/tmp.RKiPPGuQje /tmp/tmp.2zNgzvhZyj
+ return 0
+ wait_restore backup-minio-2 some-name requested 0 1200
+ local backup_name=backup-minio-2
+ local cluster_name=some-name
+ local target_state=requested
+ local wait_cluster_consistency=0
+ local wait_time=1200
+ local ok_if_ready=0
+ set +o xtrace
Waiting for the psmdb-restore/restore-backup-minio-2 object to be created.OK
Waiting psmdb-restore/restore-backup-minio-2 to reach state "requested" ........OK after 7 minutes
+ [[ 0 -eq 1 ]]
+ echo

+ wait_restore backup-minio-2 some-name ready 0 1600
+ local backup_name=backup-minio-2
+ local cluster_name=some-name
+ local target_state=ready
+ local wait_cluster_consistency=0
+ local wait_time=1600
+ local ok_if_ready=0
+ set +o xtrace
Waiting for the psmdb-restore/restore-backup-minio-2 object to be created.OK
Waiting psmdb-restore/restore-backup-minio-2 to reach state "ready" ...OK after 2 minutes
+ [[ 0 -eq 1 ]]
+ echo

+ set -o xtrace
+ wait_for_running some-name-rs0 3
+ local name=some-name-rs0
+ let last_pod=2
+ local check_cluster_readyness=true
+ set_debug
+ [[ 1 == 1 ]]
+ set -o xtrace
+ local rs_name=rs0
+ local cluster_name=some-name
++ seq 0 2
+ for i in $(seq 0 $last_pod)
+ [[ 0 -eq 2 ]]
+ wait_pod some-name-rs0-0
+ local pod=some-name-rs0-0
+ set +o xtrace
waiting for pod/some-name-rs0-0 to be ready.OK
+ for i in $(seq 0 $last_pod)
+ [[ 1 -eq 2 ]]
+ wait_pod some-name-rs0-1
+ local pod=some-name-rs0-1
+ set +o xtrace
waiting for pod/some-name-rs0-1 to be ready.OK
+ for i in $(seq 0 $last_pod)
+ [[ 2 -eq 2 ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].arbiter.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.wOVky4AhZj
+++ mktemp
++ local LAST_ERR=/tmp/tmp.UbaDWd91ip
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].arbiter.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.wOVky4AhZj
++ cat /tmp/tmp.UbaDWd91ip
++ rm /tmp/tmp.wOVky4AhZj /tmp/tmp.UbaDWd91ip
++ return 0
+ [[ '' == true ]]
+ wait_pod some-name-rs0-2
+ local pod=some-name-rs0-2
+ set +o xtrace
waiting for pod/some-name-rs0-2 to be ready.OK
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].nonvoting.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.93GHgbOTLw
+++ mktemp
++ local LAST_ERR=/tmp/tmp.Vj91EaQZeE
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].nonvoting.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.93GHgbOTLw
++ cat /tmp/tmp.Vj91EaQZeE
++ rm /tmp/tmp.93GHgbOTLw /tmp/tmp.Vj91EaQZeE
++ return 0
+ [[ '' == true ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].hidden.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.1vMV5VlqSl
+++ mktemp
++ local LAST_ERR=/tmp/tmp.s9iBK71tdS
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].hidden.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.1vMV5VlqSl
++ cat /tmp/tmp.s9iBK71tdS
++ rm /tmp/tmp.1vMV5VlqSl /tmp/tmp.s9iBK71tdS
++ return 0
+ [[ '' == true ]]
+ sleep 10
+ [[ true == true ]]
+ set +x
Waiting for cluster readyness.........................................................................
+ wait_for_running some-name-cfg 3
+ local name=some-name-cfg
+ let last_pod=2
+ local check_cluster_readyness=true
+ set_debug
+ [[ 1 == 1 ]]
+ set -o xtrace
+ local rs_name=cfg
+ local cluster_name=some-name
++ seq 0 2
+ for i in $(seq 0 $last_pod)
+ [[ 0 -eq 2 ]]
+ wait_pod some-name-cfg-0
+ local pod=some-name-cfg-0
+ set +o xtrace
waiting for pod/some-name-cfg-0 to be ready.OK
+ for i in $(seq 0 $last_pod)
+ [[ 1 -eq 2 ]]
+ wait_pod some-name-cfg-1
+ local pod=some-name-cfg-1
+ set +o xtrace
waiting for pod/some-name-cfg-1 to be ready.OK
+ for i in $(seq 0 $last_pod)
+ [[ 2 -eq 2 ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].arbiter.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.JyJrNZv5Gg
+++ mktemp
++ local LAST_ERR=/tmp/tmp.YQIxHnN06S
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].arbiter.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.JyJrNZv5Gg
++ cat /tmp/tmp.YQIxHnN06S
++ rm /tmp/tmp.JyJrNZv5Gg /tmp/tmp.YQIxHnN06S
++ return 0
+ [[ '' == true ]]
+ wait_pod some-name-cfg-2
+ local pod=some-name-cfg-2
+ set +o xtrace
waiting for pod/some-name-cfg-2 to be ready.OK
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].nonvoting.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.sQAxPRtGcy
+++ mktemp
++ local LAST_ERR=/tmp/tmp.RpNDr53m6C
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].nonvoting.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.sQAxPRtGcy
++ cat /tmp/tmp.RpNDr53m6C
++ rm /tmp/tmp.sQAxPRtGcy /tmp/tmp.RpNDr53m6C
++ return 0
+ [[ '' == true ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].hidden.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.a8pBN9W4LP
+++ mktemp
++ local LAST_ERR=/tmp/tmp.DMSkN5xTb7
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].hidden.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.a8pBN9W4LP
++ cat /tmp/tmp.DMSkN5xTb7
++ rm /tmp/tmp.a8pBN9W4LP /tmp/tmp.DMSkN5xTb7
++ return 0
+ [[ '' == true ]]
+ sleep 10
+ [[ true == true ]]
+ set +x
Waiting for cluster readyness
+ wait_for_running some-name-mongos 3
+ local name=some-name-mongos
+ let last_pod=2
+ local check_cluster_readyness=true
+ set_debug
+ [[ 1 == 1 ]]
+ set -o xtrace
+ local rs_name=mongos
+ local cluster_name=some-name
++ seq 0 2
+ for i in $(seq 0 $last_pod)
+ [[ 0 -eq 2 ]]
+ wait_pod some-name-mongos-0
+ local pod=some-name-mongos-0
+ set +o xtrace
waiting for pod/some-name-mongos-0 to be ready.OK
+ for i in $(seq 0 $last_pod)
+ [[ 1 -eq 2 ]]
+ wait_pod some-name-mongos-1
+ local pod=some-name-mongos-1
+ set +o xtrace
waiting for pod/some-name-mongos-1 to be ready.OK
+ for i in $(seq 0 $last_pod)
+ [[ 2 -eq 2 ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="mongos")].arbiter.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.3Nbs35qvAG
+++ mktemp
++ local LAST_ERR=/tmp/tmp.bXFj1ljgJl
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="mongos")].arbiter.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.3Nbs35qvAG
++ cat /tmp/tmp.bXFj1ljgJl
++ rm /tmp/tmp.3Nbs35qvAG /tmp/tmp.bXFj1ljgJl
++ return 0
+ [[ '' == true ]]
+ wait_pod some-name-mongos-2
+ local pod=some-name-mongos-2
+ set +o xtrace
waiting for pod/some-name-mongos-2 to be ready.OK
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="mongos")].nonvoting.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.w7FXGyOu1Y
+++ mktemp
++ local LAST_ERR=/tmp/tmp.z47iX12G2m
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="mongos")].nonvoting.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.w7FXGyOu1Y
++ cat /tmp/tmp.z47iX12G2m
++ rm /tmp/tmp.w7FXGyOu1Y /tmp/tmp.z47iX12G2m
++ return 0
+ [[ '' == true ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="mongos")].hidden.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.Dn4WciUR7o
+++ mktemp
++ local LAST_ERR=/tmp/tmp.s6AT0kGnt8
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="mongos")].hidden.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.Dn4WciUR7o
++ cat /tmp/tmp.s6AT0kGnt8
++ rm /tmp/tmp.Dn4WciUR7o /tmp/tmp.s6AT0kGnt8
++ return 0
+ [[ '' == true ]]
+ sleep 10
+ [[ true == true ]]
+ set +x
Waiting for cluster readyness
+ sleep 10
+ compare_mongos_cmd find myApp:myPass@some-name-mongos.pitr-physical-backup-source-28616 -3rd
+ local command=find
+ local uri=myApp:myPass@some-name-mongos.pitr-physical-backup-source-28616
+ local postfix=-3rd
+ local suffix=
+ local database=myApp
+ local collection=test
+ local port=27017
+ local tls=false
+ [[ false == true ]]
+ mongos_command=run_mongos
+ log 'running db.test.command() in myApp'
+ set +o xtrace
[2026-06-18T21:40:56+0000] running db.test.command() in myApp
+ run_mongos 'use myApp\n db.test.find()' myApp:myPass@some-name-mongos.pitr-physical-backup-source-28616 mongodb '' '' 27017
+ local 'command=use myApp\n db.test.find()'
+ local uri=myApp:myPass@some-name-mongos.pitr-physical-backup-source-28616
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local port=27017
+ local mongo_bin=mongo
+ grep -E -v 'I NETWORK|W NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|Started a new thread for the timer service'
+ /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
++ echo .svc.cluster.local
++ awk -F: '{print $2}'
+ suffix_port=
+ [[ -z '' ]]
+ suffix=.svc.cluster.local:27017
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.q9gx8oyfim
+++ mktemp
++ local LAST_ERR=/tmp/tmp.uktJfBiJqZ
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.q9gx8oyfim
++ cat /tmp/tmp.uktJfBiJqZ
++ rm /tmp/tmp.q9gx8oyfim /tmp/tmp.uktJfBiJqZ
++ return 0
+ local client_container=psmdb-client-bb8b97679-8kmm8
+ kubectl_bin exec psmdb-client-bb8b97679-8kmm8 -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.pitr-physical-backup-source-28616.svc.cluster.local:27017/admin '
++ mktemp
+ local LAST_OUT=/tmp/tmp.ZTLh5BnfvF
++ mktemp
+ local LAST_ERR=/tmp/tmp.nyP3rtER1m
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-bb8b97679-8kmm8 -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.pitr-physical-backup-source-28616.svc.cluster.local:27017/admin '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.ZTLh5BnfvF
+ cat /tmp/tmp.nyP3rtER1m
+ rm /tmp/tmp.ZTLh5BnfvF /tmp/tmp.nyP3rtER1m
+ return 0
+ [[ 0 -eq 0 ]]
+ diff /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2406/e2e-tests/pitr-physical-backup-source/compare/find-3rd.json /tmp/tmp.4E31yTnNi1/find-3rd
+ desc 'disable pitr'
+ set +o xtrace


-----------------------------------------------------------------------------------
disable pitr
-----------------------------------------------------------------------------------

+ kubectl patch psmdb some-name --type=merge --patch '{"spec": {"backup": {"pitr": {"enabled": false}}}}'
perconaservermongodb.psmdb.percona.com/some-name patched
+ sleep 20
+ desc 'delete all backups'
+ set +o xtrace


-----------------------------------------------------------------------------------
delete all backups
-----------------------------------------------------------------------------------

+ kubectl_bin delete psmdb-backup --all
++ mktemp
+ local LAST_OUT=/tmp/tmp.b25ZuU6Clq
++ mktemp
+ local LAST_ERR=/tmp/tmp.vorV3A7EHL
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl delete psmdb-backup --all
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.b25ZuU6Clq
perconaservermongodbbackup.psmdb.percona.com "backup-minio-1" deleted from pitr-physical-backup-source-28616 namespace
perconaservermongodbbackup.psmdb.percona.com "backup-minio-2" deleted from pitr-physical-backup-source-28616 namespace
+ cat /tmp/tmp.vorV3A7EHL
+ rm /tmp/tmp.b25ZuU6Clq /tmp/tmp.vorV3A7EHL
+ return 0
+ desc 'destroy cluster'
+ set +o xtrace


-----------------------------------------------------------------------------------
destroy cluster
-----------------------------------------------------------------------------------

+ destroy pitr-physical-backup-source-28616
+ local namespace=pitr-physical-backup-source-28616
+ local ignore_logs=true
+ [[ 0 == 1 ]]
+ desc 'destroy cluster/operator and all other resources'
+ set +o xtrace


-----------------------------------------------------------------------------------
destroy cluster/operator and all other resources
-----------------------------------------------------------------------------------

+ '[' true == false ']'
+ delete_backups
+ desc 'Delete psmdb-backup'
+ set +o xtrace


-----------------------------------------------------------------------------------
Delete psmdb-backup
-----------------------------------------------------------------------------------

++ kubectl_bin get psmdb-backup --no-headers
++ wc -l
+++ mktemp
++ local LAST_OUT=/tmp/tmp.7vltvJQlQX
+++ mktemp
++ local LAST_ERR=/tmp/tmp.hfqdRFnU5g
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb-backup --no-headers
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.7vltvJQlQX
++ cat /tmp/tmp.hfqdRFnU5g
No resources found in pitr-physical-backup-source-28616 namespace.
++ rm /tmp/tmp.7vltvJQlQX /tmp/tmp.hfqdRFnU5g
++ return 0
+ '[' 0 '!=' 0 ']'
+ delete_crd
+ desc 'get and delete old CRDs and RBAC'
+ set +o xtrace


-----------------------------------------------------------------------------------
get and delete old CRDs and RBAC
-----------------------------------------------------------------------------------

+ kubectl_bin delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2406/deploy/crd.yaml --ignore-not-found --wait=false
++ mktemp
+ local LAST_OUT=/tmp/tmp.mMWko4m23r
++ mktemp
+ local LAST_ERR=/tmp/tmp.lAKSG1j87T
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2406/deploy/crd.yaml --ignore-not-found --wait=false
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.mMWko4m23r
customresourcedefinition.apiextensions.k8s.io "perconaservermongodbbackups.psmdb.percona.com" deleted
customresourcedefinition.apiextensions.k8s.io "perconaservermongodbclustersyncs.psmdb.percona.com" deleted
customresourcedefinition.apiextensions.k8s.io "perconaservermongodbrestores.psmdb.percona.com" deleted
customresourcedefinition.apiextensions.k8s.io "perconaservermongodbs.psmdb.percona.com" deleted
+ cat /tmp/tmp.lAKSG1j87T
+ rm /tmp/tmp.mMWko4m23r /tmp/tmp.lAKSG1j87T
+ return 0
++ yq eval .metadata.name /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2406/deploy/crd.yaml
++ grep -v '\-\-\-'
grep: warning: stray \ before -
grep: warning: stray \ before -
+ for crd_name in $(yq eval '.metadata.name' "${src_dir}/deploy/crd.yaml" | grep -v '\-\-\-')
+ kubectl get perconaservermongodbbackups.psmdb.percona.com --all-namespaces -o wide
+ grep -v NAMESPACE
+ xargs -L 1 sh -xc 'kubectl patch perconaservermongodbbackups.psmdb.percona.com -n $0 $1 --type=merge -p "{\"metadata\":{\"finalizers\":[]}}"'
error: the server doesn't have a resource type "perconaservermongodbbackups"
+ kubectl patch perconaservermongodbbackups.psmdb.percona.com -n sh --type=merge -p '{"metadata":{"finalizers":[]}}'
error: the server doesn't have a resource type "perconaservermongodbbackups"
+ :
+ kubectl_bin wait --for=delete crd perconaservermongodbbackups.psmdb.percona.com
++ mktemp
+ local LAST_OUT=/tmp/tmp.uJ3DAxr1tP
++ mktemp
+ local LAST_ERR=/tmp/tmp.bAeykzM0W2
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl wait --for=delete crd perconaservermongodbbackups.psmdb.percona.com
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.uJ3DAxr1tP
+ cat /tmp/tmp.bAeykzM0W2
+ rm /tmp/tmp.uJ3DAxr1tP /tmp/tmp.bAeykzM0W2
+ return 0
+ for crd_name in $(yq eval '.metadata.name' "${src_dir}/deploy/crd.yaml" | grep -v '\-\-\-')
+ kubectl get perconaservermongodbclustersyncs.psmdb.percona.com --all-namespaces -o wide
+ grep -v NAMESPACE
+ xargs -L 1 sh -xc 'kubectl patch perconaservermongodbclustersyncs.psmdb.percona.com -n $0 $1 --type=merge -p "{\"metadata\":{\"finalizers\":[]}}"'
error: the server doesn't have a resource type "perconaservermongodbclustersyncs"
+ kubectl patch perconaservermongodbclustersyncs.psmdb.percona.com -n sh --type=merge -p '{"metadata":{"finalizers":[]}}'
error: the server doesn't have a resource type "perconaservermongodbclustersyncs"
+ :
+ kubectl_bin wait --for=delete crd perconaservermongodbclustersyncs.psmdb.percona.com
++ mktemp
+ local LAST_OUT=/tmp/tmp.PCVyUoUhH0
++ mktemp
+ local LAST_ERR=/tmp/tmp.i5z5wsn4CP
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl wait --for=delete crd perconaservermongodbclustersyncs.psmdb.percona.com
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.PCVyUoUhH0
+ cat /tmp/tmp.i5z5wsn4CP
+ rm /tmp/tmp.PCVyUoUhH0 /tmp/tmp.i5z5wsn4CP
+ return 0
+ for crd_name in $(yq eval '.metadata.name' "${src_dir}/deploy/crd.yaml" | grep -v '\-\-\-')
+ kubectl get perconaservermongodbrestores.psmdb.percona.com --all-namespaces -o wide
+ grep -v NAMESPACE
+ xargs -L 1 sh -xc 'kubectl patch perconaservermongodbrestores.psmdb.percona.com -n $0 $1 --type=merge -p "{\"metadata\":{\"finalizers\":[]}}"'
error: the server doesn't have a resource type "perconaservermongodbrestores"
+ kubectl patch perconaservermongodbrestores.psmdb.percona.com -n sh --type=merge -p '{"metadata":{"finalizers":[]}}'
error: the server doesn't have a resource type "perconaservermongodbrestores"
+ :
+ kubectl_bin wait --for=delete crd perconaservermongodbrestores.psmdb.percona.com
++ mktemp
+ local LAST_OUT=/tmp/tmp.0SdHxUKhOR
++ mktemp
+ local LAST_ERR=/tmp/tmp.tggU2P2eBf
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl wait --for=delete crd perconaservermongodbrestores.psmdb.percona.com
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.0SdHxUKhOR
+ cat /tmp/tmp.tggU2P2eBf
+ rm /tmp/tmp.0SdHxUKhOR /tmp/tmp.tggU2P2eBf
+ return 0
+ for crd_name in $(yq eval '.metadata.name' "${src_dir}/deploy/crd.yaml" | grep -v '\-\-\-')
+ kubectl get perconaservermongodbs.psmdb.percona.com --all-namespaces -o wide
+ grep -v NAMESPACE
+ xargs -L 1 sh -xc 'kubectl patch perconaservermongodbs.psmdb.percona.com -n $0 $1 --type=merge -p "{\"metadata\":{\"finalizers\":[]}}"'
error: the server doesn't have a resource type "perconaservermongodbs"
+ kubectl patch perconaservermongodbs.psmdb.percona.com -n sh --type=merge -p '{"metadata":{"finalizers":[]}}'
error: the server doesn't have a resource type "perconaservermongodbs"
+ :
+ kubectl_bin wait --for=delete crd perconaservermongodbs.psmdb.percona.com
++ mktemp
+ local LAST_OUT=/tmp/tmp.2Bke2EoYKH
++ mktemp
+ local LAST_ERR=/tmp/tmp.Y7upBGXnqT
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl wait --for=delete crd perconaservermongodbs.psmdb.percona.com
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.2Bke2EoYKH
+ cat /tmp/tmp.Y7upBGXnqT
+ rm /tmp/tmp.2Bke2EoYKH /tmp/tmp.Y7upBGXnqT
+ return 0
+ local rbac_yaml=rbac.yaml
+ '[' -n psmdb-operator ']'
+ rbac_yaml=cw-rbac.yaml
+ kubectl_bin delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2406/deploy/cw-rbac.yaml --ignore-not-found
++ mktemp
+ local LAST_OUT=/tmp/tmp.CcH2yRyDIQ
++ mktemp
+ local LAST_ERR=/tmp/tmp.2oS4GpgWds
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2406/deploy/cw-rbac.yaml --ignore-not-found
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.CcH2yRyDIQ
clusterrole.rbac.authorization.k8s.io "percona-server-mongodb-operator" deleted
clusterrolebinding.rbac.authorization.k8s.io "service-account-percona-server-mongodb-operator" deleted
+ cat /tmp/tmp.2oS4GpgWds
+ rm /tmp/tmp.CcH2yRyDIQ /tmp/tmp.2oS4GpgWds
+ return 0
+ destroy_cert_manager
+ kubectl_bin delete -f https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml
++ mktemp
+ local LAST_OUT=/tmp/tmp.1wuFmmpRam
++ mktemp
+ local LAST_ERR=/tmp/tmp.biU5Nt5S5v
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl delete -f https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml
+ exit_status=1
+ set -e
+ '[' 1 '!=' 0 -a -n 1 ']'
+ cat /tmp/tmp.1wuFmmpRam
+ cat /tmp/tmp.biU5Nt5S5v
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": namespaces "cert-manager" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "challenges.acme.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "orders.acme.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "certificaterequests.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "certificates.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "clusterissuers.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "issuers.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": serviceaccounts "cert-manager-cainjector" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": serviceaccounts "cert-manager" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": serviceaccounts "cert-manager-webhook" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-cainjector" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-issuers" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-clusterissuers" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-certificates" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-orders" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-challenges" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-ingress-shim" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-cluster-view" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-view" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-edit" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-approve:cert-manager-io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-certificatesigningrequests" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-webhook:subjectaccessreviews" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-cainjector" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-issuers" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-clusterissuers" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-certificates" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-orders" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-challenges" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-ingress-shim" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-approve:cert-manager-io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-certificatesigningrequests" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-webhook:subjectaccessreviews" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager-cainjector:leaderelection" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager:leaderelection" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager-tokenrequest" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager-webhook:dynamic-serving" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager-cainjector:leaderelection" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager:leaderelection" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager-tokenrequest" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager-webhook:dynamic-serving" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": services "cert-manager-cainjector" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": services "cert-manager" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": services "cert-manager-webhook" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": deployments.apps "cert-manager-cainjector" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": deployments.apps "cert-manager" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": deployments.apps "cert-manager-webhook" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": mutatingwebhookconfigurations.admissionregistration.k8s.io "cert-manager-webhook" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": validatingwebhookconfigurations.admissionregistration.k8s.io "cert-manager-webhook" not found
+ sleep 0
+ for i in $(seq 0 2)
+ set +e
+ kubectl delete -f https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml
+ exit_status=1
+ set -e
+ '[' 1 '!=' 0 -a -n 1 ']'
+ cat /tmp/tmp.1wuFmmpRam
+ cat /tmp/tmp.biU5Nt5S5v
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": namespaces "cert-manager" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "challenges.acme.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "orders.acme.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "certificaterequests.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "certificates.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "clusterissuers.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "issuers.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": serviceaccounts "cert-manager-cainjector" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": serviceaccounts "cert-manager" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": serviceaccounts "cert-manager-webhook" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-cainjector" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-issuers" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-clusterissuers" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-certificates" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-orders" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-challenges" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-ingress-shim" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-cluster-view" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-view" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-edit" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-approve:cert-manager-io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-certificatesigningrequests" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-webhook:subjectaccessreviews" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-cainjector" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-issuers" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-clusterissuers" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-certificates" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-orders" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-challenges" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-ingress-shim" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-approve:cert-manager-io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-certificatesigningrequests" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-webhook:subjectaccessreviews" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager-cainjector:leaderelection" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager:leaderelection" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager-tokenrequest" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager-webhook:dynamic-serving" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager-cainjector:leaderelection" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager:leaderelection" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager-tokenrequest" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager-webhook:dynamic-serving" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": services "cert-manager-cainjector" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": services "cert-manager" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": services "cert-manager-webhook" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": deployments.apps "cert-manager-cainjector" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": deployments.apps "cert-manager" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": deployments.apps "cert-manager-webhook" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": mutatingwebhookconfigurations.admissionregistration.k8s.io "cert-manager-webhook" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": validatingwebhookconfigurations.admissionregistration.k8s.io "cert-manager-webhook" not found
+ sleep 4
+ for i in $(seq 0 2)
+ set +e
+ kubectl delete -f https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml
+ exit_status=1
+ set -e
+ '[' 1 '!=' 0 -a -n 1 ']'
+ cat /tmp/tmp.1wuFmmpRam
+ cat /tmp/tmp.biU5Nt5S5v
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": namespaces "cert-manager" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "challenges.acme.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "orders.acme.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "certificaterequests.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "certificates.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "clusterissuers.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "issuers.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": serviceaccounts "cert-manager-cainjector" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": serviceaccounts "cert-manager" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": serviceaccounts "cert-manager-webhook" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-cainjector" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-issuers" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-clusterissuers" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-certificates" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-orders" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-challenges" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-ingress-shim" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-cluster-view" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-view" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-edit" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-approve:cert-manager-io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-certificatesigningrequests" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-webhook:subjectaccessreviews" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-cainjector" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-issuers" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-clusterissuers" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-certificates" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-orders" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-challenges" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-ingress-shim" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-approve:cert-manager-io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-certificatesigningrequests" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-webhook:subjectaccessreviews" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager-cainjector:leaderelection" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager:leaderelection" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager-tokenrequest" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager-webhook:dynamic-serving" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager-cainjector:leaderelection" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager:leaderelection" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager-tokenrequest" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager-webhook:dynamic-serving" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": services "cert-manager-cainjector" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": services "cert-manager" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": services "cert-manager-webhook" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": deployments.apps "cert-manager-cainjector" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": deployments.apps "cert-manager" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": deployments.apps "cert-manager-webhook" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": mutatingwebhookconfigurations.admissionregistration.k8s.io "cert-manager-webhook" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": validatingwebhookconfigurations.admissionregistration.k8s.io "cert-manager-webhook" not found
+ sleep 8
+ cat /tmp/tmp.1wuFmmpRam
+ cat /tmp/tmp.biU5Nt5S5v
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": namespaces "cert-manager" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "challenges.acme.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "orders.acme.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "certificaterequests.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "certificates.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "clusterissuers.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "issuers.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": serviceaccounts "cert-manager-cainjector" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": serviceaccounts "cert-manager" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": serviceaccounts "cert-manager-webhook" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-cainjector" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-issuers" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-clusterissuers" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-certificates" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-orders" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-challenges" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-ingress-shim" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-cluster-view" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-view" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-edit" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-approve:cert-manager-io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-certificatesigningrequests" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-webhook:subjectaccessreviews" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-cainjector" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-issuers" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-clusterissuers" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-certificates" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-orders" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-challenges" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-ingress-shim" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-approve:cert-manager-io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-certificatesigningrequests" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-webhook:subjectaccessreviews" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager-cainjector:leaderelection" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager:leaderelection" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager-tokenrequest" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager-webhook:dynamic-serving" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager-cainjector:leaderelection" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager:leaderelection" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager-tokenrequest" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager-webhook:dynamic-serving" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": services "cert-manager-cainjector" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": services "cert-manager" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": services "cert-manager-webhook" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": deployments.apps "cert-manager-cainjector" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": deployments.apps "cert-manager" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": deployments.apps "cert-manager-webhook" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": mutatingwebhookconfigurations.admissionregistration.k8s.io "cert-manager-webhook" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": validatingwebhookconfigurations.admissionregistration.k8s.io "cert-manager-webhook" not found
+ rm /tmp/tmp.1wuFmmpRam /tmp/tmp.biU5Nt5S5v
+ return 1
+ true
+ '[' -n '' ']'
+ '[' -n psmdb-operator ']'
+ kubectl_bin delete --grace-period=0 --force=true namespace pitr-physical-backup-source-28616
+ rm -rf /tmp/tmp.4E31yTnNi1
+ kubectl_bin delete --grace-period=0 --force=true namespace psmdb-operator
++ mktemp
++ mktemp
+ desc 'test passed'
+ set +o xtrace


-----------------------------------------------------------------------------------
+ local LAST_OUT=/tmp/tmp.d8yPAckuxm
test passed
-----------------------------------------------------------------------------------

+ local LAST_OUT=/tmp/tmp.ZSYyIVRs5x
++ mktemp
++ mktemp
+ local LAST_ERR=/tmp/tmp.U8cJmbxqGS
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ local LAST_ERR=/tmp/tmp.3TnqPtujEc
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl delete --grace-period=0 --force=true namespace pitr-physical-backup-source-28616
+ for i in $(seq 0 2)
+ set +e
+ kubectl delete --grace-period=0 --force=true namespace psmdb-operator