Log: /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2151/e2e-tests/logs/pitr-physical-backup-source.log
grep: warning: stray \ before -
Warning: version difference between client (1.35) and server (1.31) exceeds the supported minor version skew of +/-1
Warning: version difference between client (1.35) and server (1.31) exceeds the supported minor version skew of +/-1
Warning: version difference between client (1.35) and server (1.31) exceeds the supported minor version skew of +/-1
+ main
+ create_infra pitr-physical-backup-source-28579
+ local ns=pitr-physical-backup-source-28579
+ [[ 1 == 1 ]]
+ delete_crd
+ desc 'get and delete old CRDs and RBAC'
+ set +o xtrace


-----------------------------------------------------------------------------------
get and delete old CRDs and RBAC
-----------------------------------------------------------------------------------

+ kubectl_bin delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2151/deploy/crd.yaml --ignore-not-found --wait=false
++ mktemp
+ local LAST_OUT=/tmp/tmp.dS6rbxnuc7
++ mktemp
+ local LAST_ERR=/tmp/tmp.xB6I3ADDzE
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2151/deploy/crd.yaml --ignore-not-found --wait=false
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.dS6rbxnuc7
customresourcedefinition.apiextensions.k8s.io "perconaservermongodbbackups.psmdb.percona.com" deleted
customresourcedefinition.apiextensions.k8s.io "perconaservermongodbrestores.psmdb.percona.com" deleted
customresourcedefinition.apiextensions.k8s.io "perconaservermongodbs.psmdb.percona.com" deleted
+ cat /tmp/tmp.xB6I3ADDzE
+ rm /tmp/tmp.dS6rbxnuc7 /tmp/tmp.xB6I3ADDzE
+ return 0
++ yq eval .metadata.name /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2151/deploy/crd.yaml
++ grep -v '\-\-\-'
grep: warning: stray \ before -
grep: warning: stray \ before -
+ for crd_name in $(yq eval '.metadata.name' "${src_dir}/deploy/crd.yaml" | grep -v '\-\-\-')
+ kubectl get perconaservermongodbbackups.psmdb.percona.com --all-namespaces -o wide
+ grep -v NAMESPACE
+ xargs -L 1 sh -xc 'kubectl patch perconaservermongodbbackups.psmdb.percona.com -n $0 $1 --type=merge -p "{\"metadata\":{\"finalizers\":[]}}"'
error: the server doesn't have a resource type "perconaservermongodbbackups"
+ kubectl patch perconaservermongodbbackups.psmdb.percona.com -n sh --type=merge -p '{"metadata":{"finalizers":[]}}'
error: the server doesn't have a resource type "perconaservermongodbbackups"
+ :
+ kubectl_bin wait --for=delete crd perconaservermongodbbackups.psmdb.percona.com
++ mktemp
+ local LAST_OUT=/tmp/tmp.jd2ryqPCHy
++ mktemp
+ local LAST_ERR=/tmp/tmp.JBQQ85K6ng
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl wait --for=delete crd perconaservermongodbbackups.psmdb.percona.com
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.jd2ryqPCHy
+ cat /tmp/tmp.JBQQ85K6ng
+ rm /tmp/tmp.jd2ryqPCHy /tmp/tmp.JBQQ85K6ng
+ return 0
+ for crd_name in $(yq eval '.metadata.name' "${src_dir}/deploy/crd.yaml" | grep -v '\-\-\-')
+ kubectl get perconaservermongodbrestores.psmdb.percona.com --all-namespaces -o wide
+ grep -v NAMESPACE
+ xargs -L 1 sh -xc 'kubectl patch perconaservermongodbrestores.psmdb.percona.com -n $0 $1 --type=merge -p "{\"metadata\":{\"finalizers\":[]}}"'
error: the server doesn't have a resource type "perconaservermongodbrestores"
+ kubectl patch perconaservermongodbrestores.psmdb.percona.com -n sh --type=merge -p '{"metadata":{"finalizers":[]}}'
error: the server doesn't have a resource type "perconaservermongodbrestores"
+ :
+ kubectl_bin wait --for=delete crd perconaservermongodbrestores.psmdb.percona.com
++ mktemp
+ local LAST_OUT=/tmp/tmp.Cow9EyiCVL
++ mktemp
+ local LAST_ERR=/tmp/tmp.iatnHPihhj
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl wait --for=delete crd perconaservermongodbrestores.psmdb.percona.com
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.Cow9EyiCVL
+ cat /tmp/tmp.iatnHPihhj
+ rm /tmp/tmp.Cow9EyiCVL /tmp/tmp.iatnHPihhj
+ return 0
+ for crd_name in $(yq eval '.metadata.name' "${src_dir}/deploy/crd.yaml" | grep -v '\-\-\-')
+ kubectl get perconaservermongodbs.psmdb.percona.com --all-namespaces -o wide
+ grep -v NAMESPACE
+ xargs -L 1 sh -xc 'kubectl patch perconaservermongodbs.psmdb.percona.com -n $0 $1 --type=merge -p "{\"metadata\":{\"finalizers\":[]}}"'
error: the server doesn't have a resource type "perconaservermongodbs"
+ kubectl patch perconaservermongodbs.psmdb.percona.com -n sh --type=merge -p '{"metadata":{"finalizers":[]}}'
error: the server doesn't have a resource type "perconaservermongodbs"
+ :
+ kubectl_bin wait --for=delete crd perconaservermongodbs.psmdb.percona.com
++ mktemp
+ local LAST_OUT=/tmp/tmp.fQNFfdCoVU
++ mktemp
+ local LAST_ERR=/tmp/tmp.obovUEdRgn
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl wait --for=delete crd perconaservermongodbs.psmdb.percona.com
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.fQNFfdCoVU
+ cat /tmp/tmp.obovUEdRgn
+ rm /tmp/tmp.fQNFfdCoVU /tmp/tmp.obovUEdRgn
+ return 0
+ local rbac_yaml=rbac.yaml
+ '[' -n psmdb-operator ']'
+ rbac_yaml=cw-rbac.yaml
+ kubectl_bin delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2151/deploy/cw-rbac.yaml --ignore-not-found
++ mktemp
+ local LAST_OUT=/tmp/tmp.PxHOq2Q6ua
++ mktemp
+ local LAST_ERR=/tmp/tmp.o6AdRtr6cg
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2151/deploy/cw-rbac.yaml --ignore-not-found
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.PxHOq2Q6ua
clusterrole.rbac.authorization.k8s.io "percona-server-mongodb-operator" deleted
clusterrolebinding.rbac.authorization.k8s.io "service-account-percona-server-mongodb-operator" deleted
+ cat /tmp/tmp.o6AdRtr6cg
+ rm /tmp/tmp.PxHOq2Q6ua /tmp/tmp.o6AdRtr6cg
+ return 0
+ check_crd_for_deletion PR-2151-37812f21
+ local git_tag=PR-2151-37812f21
++ curl -s https://raw.githubusercontent.com/percona/percona-server-mongodb-operator/PR-2151-37812f21/deploy/crd.yaml
++ yq eval .metadata.name
++ /usr/sbin/sed s/---//g
++ /usr/sbin/sed ':a;N;$!ba;s/\n/ /g'
+ for crd_name in $(curl -s https://raw.githubusercontent.com/percona/percona-server-mongodb-operator/${git_tag}/deploy/crd.yaml | yq eval '.metadata.name' | $sed 's/---//g' | $sed ':a;N;$!ba;s/\n/ /g')
++ kubectl_bin get crd/null -o 'jsonpath={.status.conditions[-1].type}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.MnAcdrhHG0
+++ mktemp
++ local LAST_ERR=/tmp/tmp.GG7sK60Q5M
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get crd/null -o 'jsonpath={.status.conditions[-1].type}'
++ exit_status=1
++ set -e
++ '[' 1 '!=' 0 -a -n 1 ']'
++ cat /tmp/tmp.MnAcdrhHG0
++ cat /tmp/tmp.GG7sK60Q5M
Error from server (NotFound): customresourcedefinitions.apiextensions.k8s.io "null" not found
++ sleep 0
++ for i in $(seq 0 2)
++ set +e
++ kubectl get crd/null -o 'jsonpath={.status.conditions[-1].type}'
++ exit_status=1
++ set -e
++ '[' 1 '!=' 0 -a -n 1 ']'
++ cat /tmp/tmp.MnAcdrhHG0
++ cat /tmp/tmp.GG7sK60Q5M
Error from server (NotFound): customresourcedefinitions.apiextensions.k8s.io "null" not found
++ sleep 4
++ for i in $(seq 0 2)
++ set +e
++ kubectl get crd/null -o 'jsonpath={.status.conditions[-1].type}'
++ exit_status=1
++ set -e
++ '[' 1 '!=' 0 -a -n 1 ']'
++ cat /tmp/tmp.MnAcdrhHG0
++ cat /tmp/tmp.GG7sK60Q5M
Error from server (NotFound): customresourcedefinitions.apiextensions.k8s.io "null" not found
++ sleep 8
++ cat /tmp/tmp.MnAcdrhHG0
++ cat /tmp/tmp.GG7sK60Q5M
Error from server (NotFound): customresourcedefinitions.apiextensions.k8s.io "null" not found
++ rm /tmp/tmp.MnAcdrhHG0 /tmp/tmp.GG7sK60Q5M
++ return 1
+ [[ '' == \T\e\r\m\i\n\a\t\i\n\g ]]
+ '[' -n psmdb-operator ']'
+ create_namespace psmdb-operator
+ local namespace=psmdb-operator
+ local skip_clean_namespace=
+ [[ 1 == 1 ]]
+ [[ -z '' ]]
+ destroy_chaos_mesh
++ helm list --all-namespaces --filter chaos-mesh
++ tail -n1
++ awk '-F ' '{print $2}'
++ sed s/NAMESPACE//
+ local chaos_mesh_ns=
+ desc 'destroy chaos-mesh'
+ set +o xtrace


-----------------------------------------------------------------------------------
destroy chaos-mesh
-----------------------------------------------------------------------------------

+ '[' -n '' ']'
++ kubectl get MutatingWebhookConfiguration
++ grep chaos-mesh
++ awk '{print $1}'
+ timeout 30 kubectl delete MutatingWebhookConfiguration
error: resource(s) were provided, but no name was specified
+ :
++ kubectl get ValidatingWebhookConfiguration
++ grep chaos-mesh
++ awk '{print $1}'
+ timeout 30 kubectl delete ValidatingWebhookConfiguration
error: resource(s) were provided, but no name was specified
+ :
++ kubectl get ValidatingWebhookConfiguration
++ grep validate-auth
++ awk '{print $1}'
+ timeout 30 kubectl delete ValidatingWebhookConfiguration
error: resource(s) were provided, but no name was specified
+ :
++ kubectl api-resources
++ grep chaos-mesh
++ awk '{print $1}'
++ kubectl get crd
++ grep chaos-mesh.org
++ awk '{print $1}'
+ timeout 30 kubectl delete crd
error: resource(s) were provided, but no name was specified
+ :
++ kubectl get clusterrolebinding
++ grep chaos-mesh
++ awk '{print $1}'
+ timeout 30 kubectl delete clusterrolebinding
error: resource(s) were provided, but no name was specified
+ :
++ kubectl get clusterrole
++ grep chaos-mesh
++ awk '{print $1}'
+ timeout 30 kubectl delete clusterrole
error: resource(s) were provided, but no name was specified
+ :
+ desc 'cleaned up all old namespaces'
+ set +o xtrace


-----------------------------------------------------------------------------------
cleaned up all old namespaces
-----------------------------------------------------------------------------------

+ kubectl_bin get ns
+ egrep -v '^kube-|^default|Terminating|psmdb-operator|openshift|^gke-|^gmp-|^NAME'
+ awk '{print$1}'
++ mktemp
+ '[' -n '' ']'
+ desc 'cleaned up old namespaces psmdb-operator'
+ xargs kubectl delete ns
+ set +o xtrace


-----------------------------------------------------------------------------------
cleaned up old namespaces psmdb-operator
-----------------------------------------------------------------------------------

+ kubectl_bin delete namespace psmdb-operator --ignore-not-found
++ mktemp
egrep: warning: egrep is obsolescent; using grep -E
+ local LAST_OUT=/tmp/tmp.OQ2d4mhSrW
+ local LAST_OUT=/tmp/tmp.ZRgJ5LCsI4
++ mktemp
++ mktemp
+ local LAST_ERR=/tmp/tmp.JSyHA77fed
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ local LAST_ERR=/tmp/tmp.5G3ZctVtNw
+ local exit_status=0
+ local timeout=4
+ for i in $(seq 0 2)
++ seq 0 2
+ set +e
+ kubectl delete namespace psmdb-operator --ignore-not-found
+ for i in $(seq 0 2)
+ set +e
+ kubectl get ns
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.ZRgJ5LCsI4
+ cat /tmp/tmp.5G3ZctVtNw
+ rm /tmp/tmp.ZRgJ5LCsI4 /tmp/tmp.5G3ZctVtNw
+ return 0
namespace "pitr-physical-backup-source-17266" deleted
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.OQ2d4mhSrW
namespace "psmdb-operator" deleted
+ cat /tmp/tmp.JSyHA77fed
+ rm /tmp/tmp.OQ2d4mhSrW /tmp/tmp.JSyHA77fed
+ return 0
+ kubectl_bin wait --for=delete namespace psmdb-operator
++ mktemp
+ local LAST_OUT=/tmp/tmp.DB1UQWWc4q
++ mktemp
+ local LAST_ERR=/tmp/tmp.cMwY8SOcd6
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl wait --for=delete namespace psmdb-operator
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.DB1UQWWc4q
+ cat /tmp/tmp.cMwY8SOcd6
+ rm /tmp/tmp.DB1UQWWc4q /tmp/tmp.cMwY8SOcd6
+ return 0
+ desc 'create namespace psmdb-operator'
+ set +o xtrace


-----------------------------------------------------------------------------------
create namespace psmdb-operator
-----------------------------------------------------------------------------------

+ kubectl_bin create namespace psmdb-operator
++ mktemp
+ local LAST_OUT=/tmp/tmp.GKZbVh7a8k
++ mktemp
+ local LAST_ERR=/tmp/tmp.nou1P0brKw
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl create namespace psmdb-operator
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.GKZbVh7a8k
namespace/psmdb-operator created
+ cat /tmp/tmp.nou1P0brKw
+ rm /tmp/tmp.GKZbVh7a8k /tmp/tmp.nou1P0brKw
+ return 0
+ set_kube_ctx psmdb-operator
+ local namespace=psmdb-operator
++ kubectl_bin config current-context
+++ mktemp
++ local LAST_OUT=/tmp/tmp.BhPT4SMG65
+++ mktemp
++ local LAST_ERR=/tmp/tmp.i49O9HzOWk
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl config current-context
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.BhPT4SMG65
++ cat /tmp/tmp.i49O9HzOWk
++ rm /tmp/tmp.BhPT4SMG65 /tmp/tmp.i49O9HzOWk
++ return 0
+ kubectl_bin config set-context gke_cloud-dev-112233_us-central1-a_jen-psmdb-2151-37812f21-16-cluster5 --namespace=psmdb-operator
++ mktemp
+ local LAST_OUT=/tmp/tmp.cva6S04jYb
++ mktemp
+ local LAST_ERR=/tmp/tmp.AQxyOQyDbQ
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl config set-context gke_cloud-dev-112233_us-central1-a_jen-psmdb-2151-37812f21-16-cluster5 --namespace=psmdb-operator
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.cva6S04jYb
Context "gke_cloud-dev-112233_us-central1-a_jen-psmdb-2151-37812f21-16-cluster5" modified.
+ cat /tmp/tmp.AQxyOQyDbQ
+ rm /tmp/tmp.cva6S04jYb /tmp/tmp.AQxyOQyDbQ
+ return 0
+ deploy_operator
+ desc 'start PSMDB operator: perconalab/percona-server-mongodb-operator:PR-2151-37812f21'
+ set +o xtrace


-----------------------------------------------------------------------------------
start PSMDB operator: perconalab/percona-server-mongodb-operator:PR-2151-37812f21
-----------------------------------------------------------------------------------

+ local cr_file
+ '[' -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2151/e2e-tests/pitr-physical-backup-source/conf/crd.yaml ']'
+ cr_file=/mnt/jenkins/workspace/cloud-psmdb-operator_PR-2151/deploy/crd.yaml
+ kubectl_bin apply --server-side --force-conflicts -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2151/deploy/crd.yaml
++ mktemp
+ local LAST_OUT=/tmp/tmp.J84Ka6HF7z
++ mktemp
+ local LAST_ERR=/tmp/tmp.W4oW8HQ0Qy
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl apply --server-side --force-conflicts -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2151/deploy/crd.yaml
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.J84Ka6HF7z
customresourcedefinition.apiextensions.k8s.io/perconaservermongodbbackups.psmdb.percona.com serverside-applied
customresourcedefinition.apiextensions.k8s.io/perconaservermongodbrestores.psmdb.percona.com serverside-applied
customresourcedefinition.apiextensions.k8s.io/perconaservermongodbs.psmdb.percona.com serverside-applied
+ cat /tmp/tmp.W4oW8HQ0Qy
+ rm /tmp/tmp.J84Ka6HF7z /tmp/tmp.W4oW8HQ0Qy
+ return 0
+ '[' -n psmdb-operator ']'
+ apply_rbac cw-rbac
+ local operator_namespace=psmdb-operator
+ local rbac=cw-rbac
+ cat /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2151/deploy/cw-rbac.yaml
+ sed -e 's^namespace: .*^namespace: psmdb-operator^'
+ kubectl_bin apply -n psmdb-operator -f -
++ mktemp
+ local LAST_OUT=/tmp/tmp.BTvTsNGoHV
++ mktemp
+ local LAST_ERR=/tmp/tmp.6CCuJrPCkA
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl apply -n psmdb-operator -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.BTvTsNGoHV
clusterrole.rbac.authorization.k8s.io/percona-server-mongodb-operator created
serviceaccount/percona-server-mongodb-operator created
clusterrolebinding.rbac.authorization.k8s.io/service-account-percona-server-mongodb-operator created
+ cat /tmp/tmp.6CCuJrPCkA
+ rm /tmp/tmp.BTvTsNGoHV /tmp/tmp.6CCuJrPCkA
+ return 0
+ yq eval '
			(.spec.template.spec.containers[].image = "perconalab/percona-server-mongodb-operator:PR-2151-37812f21") |
			((.. | select(.[] == "DISABLE_TELEMETRY")) |= .value="true") |
			((.. | select(.[] == "LOG_LEVEL")) |= .value="DEBUG")' /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2151/deploy/cw-operator.yaml
+ kubectl_bin apply -n psmdb-operator -f -
++ mktemp
+ local LAST_OUT=/tmp/tmp.g5V9Lghwt2
++ mktemp
+ local LAST_ERR=/tmp/tmp.H9l2jhrbQL
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl apply -n psmdb-operator -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.g5V9Lghwt2
deployment.apps/percona-server-mongodb-operator created
+ cat /tmp/tmp.H9l2jhrbQL
+ rm /tmp/tmp.g5V9Lghwt2 /tmp/tmp.H9l2jhrbQL
+ return 0
+ sleep 20
++ get_operator_pod
++ kubectl_bin get pods --selector=name=percona-server-mongodb-operator -o 'jsonpath={.items[].metadata.name}' -n psmdb-operator
+++ mktemp
++ local LAST_OUT=/tmp/tmp.x3FdBgZJ0B
+++ mktemp
++ local LAST_ERR=/tmp/tmp.vjSvVAP9dc
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=percona-server-mongodb-operator -o 'jsonpath={.items[].metadata.name}' -n psmdb-operator
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.x3FdBgZJ0B
++ cat /tmp/tmp.vjSvVAP9dc
++ rm /tmp/tmp.x3FdBgZJ0B /tmp/tmp.vjSvVAP9dc
++ return 0
+ wait_operator_pod percona-server-mongodb-operator-db986cd48-b2t7n
+ local pod=percona-server-mongodb-operator-db986cd48-b2t7n
+ set +o xtrace
waiting for pod/percona-server-mongodb-operator-db986cd48-b2t7n to be ready.OK
+ echo 'Print operator info from log'
Print operator info from log
+ grep 'Manager starting up'
++ get_operator_pod
++ kubectl_bin get pods --selector=name=percona-server-mongodb-operator -o 'jsonpath={.items[].metadata.name}' -n psmdb-operator
+++ mktemp
++ local LAST_OUT=/tmp/tmp.bZ9P33SYwA
+++ mktemp
++ local LAST_ERR=/tmp/tmp.gHJNMMfBc9
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=percona-server-mongodb-operator -o 'jsonpath={.items[].metadata.name}' -n psmdb-operator
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.bZ9P33SYwA
++ cat /tmp/tmp.gHJNMMfBc9
++ rm /tmp/tmp.bZ9P33SYwA /tmp/tmp.gHJNMMfBc9
++ return 0
+ kubectl_bin logs -n psmdb-operator percona-server-mongodb-operator-db986cd48-b2t7n
++ mktemp
+ local LAST_OUT=/tmp/tmp.7L1KWsaiBV
++ mktemp
+ local LAST_ERR=/tmp/tmp.NMCoZKSZew
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl logs -n psmdb-operator percona-server-mongodb-operator-db986cd48-b2t7n
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.7L1KWsaiBV
+ cat /tmp/tmp.NMCoZKSZew
+ rm /tmp/tmp.7L1KWsaiBV /tmp/tmp.NMCoZKSZew
+ return 0
2025-12-24T12:48:15.513Z	INFO	setup	Manager starting up	{"gitCommit": "37812f215eede4eb5b99e48719b6f479c56c9c3d", "gitBranch": "PR-2151-37812f21", "buildTime": "", "goVersion": "go1.25.5", "os": "linux", "arch": "amd64"}
+ create_namespace pitr-physical-backup-source-28579
+ local namespace=pitr-physical-backup-source-28579
+ local skip_clean_namespace=
+ [[ 1 == 1 ]]
+ [[ -z '' ]]
+ destroy_chaos_mesh
++ helm list --all-namespaces --filter chaos-mesh
++ tail -n1
++ awk '-F ' '{print $2}'
++ sed s/NAMESPACE//
+ local chaos_mesh_ns=
+ desc 'destroy chaos-mesh'
+ set +o xtrace


-----------------------------------------------------------------------------------
destroy chaos-mesh
-----------------------------------------------------------------------------------

+ '[' -n '' ']'
++ kubectl get MutatingWebhookConfiguration
++ grep chaos-mesh
++ awk '{print $1}'
+ timeout 30 kubectl delete MutatingWebhookConfiguration
error: resource(s) were provided, but no name was specified
+ :
++ kubectl get ValidatingWebhookConfiguration
++ grep chaos-mesh
++ awk '{print $1}'
+ timeout 30 kubectl delete ValidatingWebhookConfiguration
error: resource(s) were provided, but no name was specified
+ :
++ kubectl get ValidatingWebhookConfiguration
++ grep validate-auth
++ awk '{print $1}'
+ timeout 30 kubectl delete ValidatingWebhookConfiguration
error: resource(s) were provided, but no name was specified
+ :
++ kubectl api-resources
++ grep chaos-mesh
++ awk '{print $1}'
++ kubectl get crd
++ grep chaos-mesh.org
++ awk '{print $1}'
+ timeout 30 kubectl delete crd
error: resource(s) were provided, but no name was specified
+ :
++ kubectl get clusterrolebinding
++ grep chaos-mesh
++ awk '{print $1}'
+ timeout 30 kubectl delete clusterrolebinding
error: resource(s) were provided, but no name was specified
+ :
++ kubectl get clusterrole
++ grep chaos-mesh
++ awk '{print $1}'
+ timeout 30 kubectl delete clusterrole
error: resource(s) were provided, but no name was specified
+ :
+ desc 'cleaned up all old namespaces'
+ set +o xtrace


-----------------------------------------------------------------------------------
cleaned up all old namespaces
-----------------------------------------------------------------------------------

+ kubectl_bin get ns
+ egrep -v '^kube-|^default|Terminating|psmdb-operator|openshift|^gke-|^gmp-|^NAME'
+ awk '{print$1}'
++ mktemp
+ '[' -n '' ']'
+ desc 'cleaned up old namespaces pitr-physical-backup-source-28579'
+ set +o xtrace


-----------------------------------------------------------------------------------
cleaned up old namespaces pitr-physical-backup-source-28579
-----------------------------------------------------------------------------------

+ kubectl_bin delete namespace pitr-physical-backup-source-28579 --ignore-not-found
+ xargs kubectl delete ns
++ mktemp
+ local LAST_OUT=/tmp/tmp.QgI2bvJ5V3
egrep: warning: egrep is obsolescent; using grep -E
++ mktemp
+ local LAST_OUT=/tmp/tmp.mHJ7AJhfBf
++ mktemp
+ local LAST_ERR=/tmp/tmp.3RFF2gtL6H
+ local exit_status=0
+ local timeout=4
+ local LAST_ERR=/tmp/tmp.nK40NojXRA
+ local exit_status=0
+ local timeout=4
++ seq 0 2
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl get ns
+ for i in $(seq 0 2)
+ set +e
+ kubectl delete namespace pitr-physical-backup-source-28579 --ignore-not-found
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.QgI2bvJ5V3
+ cat /tmp/tmp.3RFF2gtL6H
+ rm /tmp/tmp.QgI2bvJ5V3 /tmp/tmp.3RFF2gtL6H
+ return 0
error: resource(s) were provided, but no name was specified
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.mHJ7AJhfBf
+ cat /tmp/tmp.nK40NojXRA
+ rm /tmp/tmp.mHJ7AJhfBf /tmp/tmp.nK40NojXRA
+ return 0
+ kubectl_bin wait --for=delete namespace pitr-physical-backup-source-28579
++ mktemp
+ local LAST_OUT=/tmp/tmp.sa3RU9ePR7
++ mktemp
+ local LAST_ERR=/tmp/tmp.7bQ4rIcwdk
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl wait --for=delete namespace pitr-physical-backup-source-28579
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.sa3RU9ePR7
+ cat /tmp/tmp.7bQ4rIcwdk
+ rm /tmp/tmp.sa3RU9ePR7 /tmp/tmp.7bQ4rIcwdk
+ return 0
+ desc 'create namespace pitr-physical-backup-source-28579'
+ set +o xtrace


-----------------------------------------------------------------------------------
create namespace pitr-physical-backup-source-28579
-----------------------------------------------------------------------------------

+ kubectl_bin create namespace pitr-physical-backup-source-28579
++ mktemp
+ local LAST_OUT=/tmp/tmp.Dqoi1sFEBY
++ mktemp
+ local LAST_ERR=/tmp/tmp.i8xNts0vH8
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl create namespace pitr-physical-backup-source-28579
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.Dqoi1sFEBY
namespace/pitr-physical-backup-source-28579 created
+ cat /tmp/tmp.i8xNts0vH8
+ rm /tmp/tmp.Dqoi1sFEBY /tmp/tmp.i8xNts0vH8
+ return 0
+ set_kube_ctx pitr-physical-backup-source-28579
+ local namespace=pitr-physical-backup-source-28579
++ kubectl_bin config current-context
+++ mktemp
++ local LAST_OUT=/tmp/tmp.9595cBf6kk
+++ mktemp
++ local LAST_ERR=/tmp/tmp.TxISUCFTau
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl config current-context
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.9595cBf6kk
++ cat /tmp/tmp.TxISUCFTau
++ rm /tmp/tmp.9595cBf6kk /tmp/tmp.TxISUCFTau
++ return 0
+ kubectl_bin config set-context gke_cloud-dev-112233_us-central1-a_jen-psmdb-2151-37812f21-16-cluster5 --namespace=pitr-physical-backup-source-28579
++ mktemp
+ local LAST_OUT=/tmp/tmp.Dd3fXGBI3y
++ mktemp
+ local LAST_ERR=/tmp/tmp.5PHR7xG7it
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl config set-context gke_cloud-dev-112233_us-central1-a_jen-psmdb-2151-37812f21-16-cluster5 --namespace=pitr-physical-backup-source-28579
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.Dd3fXGBI3y
Context "gke_cloud-dev-112233_us-central1-a_jen-psmdb-2151-37812f21-16-cluster5" modified.
+ cat /tmp/tmp.5PHR7xG7it
+ rm /tmp/tmp.Dd3fXGBI3y /tmp/tmp.5PHR7xG7it
+ return 0
+ deploy_minio
+ desc 'install Minio'
+ set +o xtrace


-----------------------------------------------------------------------------------
install Minio
-----------------------------------------------------------------------------------

+ helm uninstall minio-service
Error: uninstall: Release not loaded: minio-service: release: not found
+ :
+ helm repo remove minio
"minio" has been removed from your repositories
+ helm repo add minio https://charts.min.io/
"minio" has been added to your repositories
+ retry 10 60 helm install minio-service --version 5.4.0 --set replicas=1 --set mode=standalone --set resources.requests.memory=256Mi --set rootUser=rootuser --set rootPassword=rootpass123 --set 'users[0].accessKey=some-access-key' --set 'users[0].secretKey=some-secret-key' --set 'users[0].policy=consoleAdmin' --set service.type=ClusterIP --set configPathmc=/tmp/.minio/ --set persistence.size=2G --set securityContext.enabled=false minio/minio
+ local max=10
+ local delay=60
+ shift 2
+ local n=1
+ helm install minio-service --version 5.4.0 --set replicas=1 --set mode=standalone --set resources.requests.memory=256Mi --set rootUser=rootuser --set rootPassword=rootpass123 --set 'users[0].accessKey=some-access-key' --set 'users[0].secretKey=some-secret-key' --set 'users[0].policy=consoleAdmin' --set service.type=ClusterIP --set configPathmc=/tmp/.minio/ --set persistence.size=2G --set securityContext.enabled=false minio/minio
NAME: minio-service
LAST DEPLOYED: Wed Dec 24 12:48:55 2025
NAMESPACE: pitr-physical-backup-source-28579
STATUS: deployed
REVISION: 1
TEST SUITE: None
NOTES:
MinIO can be accessed via port 9000 on the following DNS name from within your cluster:
minio-service.pitr-physical-backup-source-28579.cluster.local

To access MinIO from localhost, run the below commands:

  1. export POD_NAME=$(kubectl get pods --namespace pitr-physical-backup-source-28579 -l "release=minio-service" -o jsonpath="{.items[0].metadata.name}")

  2. kubectl port-forward $POD_NAME 9000 --namespace pitr-physical-backup-source-28579

Read more about port forwarding here: http://kubernetes.io/docs/user-guide/kubectl/kubectl_port-forward/

You can now access MinIO server on http://localhost:9000. Follow the below steps to connect to MinIO server with mc client:

  1. Download the MinIO mc client - https://min.io/docs/minio/linux/reference/minio-mc.html#quickstart

  2. export MC_HOST_minio-service-local=http://$(kubectl get secret --namespace pitr-physical-backup-source-28579 minio-service -o jsonpath="{.data.rootUser}" | base64 --decode):$(kubectl get secret --namespace pitr-physical-backup-source-28579 minio-service -o jsonpath="{.data.rootPassword}" | base64 --decode)@localhost:9000

  3. mc ls minio-service-local
++ kubectl_bin get pods --selector=release=minio-service -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.lVR8zzKD7L
+++ mktemp
++ local LAST_ERR=/tmp/tmp.YncMlmFFx7
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=release=minio-service -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.lVR8zzKD7L
++ cat /tmp/tmp.YncMlmFFx7
++ rm /tmp/tmp.lVR8zzKD7L /tmp/tmp.YncMlmFFx7
++ return 0
+ MINIO_POD=minio-service-d9589b474-bshsb
+ wait_pod minio-service-d9589b474-bshsb
+ local pod=minio-service-d9589b474-bshsb
+ set +o xtrace
waiting for pod/minio-service-d9589b474-bshsb to be ready.OK
+ '[' -n psmdb-operator ']'
+ kubectl_bin create svc -n psmdb-operator externalname minio-service --external-name=minio-service.pitr-physical-backup-source-28579.svc.cluster.local --tcp=9000
++ mktemp
+ local LAST_OUT=/tmp/tmp.AwzBeTxD71
++ mktemp
+ local LAST_ERR=/tmp/tmp.MreVAotrjb
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl create svc -n psmdb-operator externalname minio-service --external-name=minio-service.pitr-physical-backup-source-28579.svc.cluster.local --tcp=9000
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.AwzBeTxD71
service/minio-service created
+ cat /tmp/tmp.MreVAotrjb
+ rm /tmp/tmp.AwzBeTxD71 /tmp/tmp.MreVAotrjb
+ return 0
+ create_minio_bucket operator-testing
+ local bucket=operator-testing
+ kubectl_bin run -i --rm aws-cli --image=perconalab/awscli --restart=Never -- bash -c 'AWS_ACCESS_KEY_ID=some-access-key 		AWS_SECRET_ACCESS_KEY=some-secret-key 		AWS_DEFAULT_REGION=us-east-1         /usr/bin/aws --endpoint-url http://minio-service:9000 s3 mb s3://operator-testing'
++ mktemp
+ local LAST_OUT=/tmp/tmp.Ov7IiAC9uy
++ mktemp
+ local LAST_ERR=/tmp/tmp.HZNdgXW17e
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl run -i --rm aws-cli --image=perconalab/awscli --restart=Never -- bash -c 'AWS_ACCESS_KEY_ID=some-access-key 		AWS_SECRET_ACCESS_KEY=some-secret-key 		AWS_DEFAULT_REGION=us-east-1         /usr/bin/aws --endpoint-url http://minio-service:9000 s3 mb s3://operator-testing'
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.Ov7IiAC9uy
make_bucket: operator-testing
pod "aws-cli" deleted from pitr-physical-backup-source-28579 namespace
+ cat /tmp/tmp.HZNdgXW17e
All commands and output from this session will be recorded in container logs, including credentials and sensitive information passed through the command prompt.
If you don't see a command prompt, try pressing enter.
warning: couldn't attach to pod/aws-cli, falling back to streaming logs: Internal error occurred: unable to upgrade connection: container aws-cli not found in pod aws-cli_pitr-physical-backup-source-28579
+ rm /tmp/tmp.Ov7IiAC9uy /tmp/tmp.HZNdgXW17e
+ return 0
+ desc 'create secrets and start client'
+ set +o xtrace


-----------------------------------------------------------------------------------
create secrets and start client
-----------------------------------------------------------------------------------

+ kubectl_bin apply -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2151/e2e-tests/conf/secrets.yml -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2151/e2e-tests/conf/client.yml -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2151/e2e-tests/conf/minio-secret.yml
++ mktemp
+ local LAST_OUT=/tmp/tmp.E79O4oow0B
++ mktemp
+ local LAST_ERR=/tmp/tmp.ASlj5Apzj3
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl apply -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2151/e2e-tests/conf/secrets.yml -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2151/e2e-tests/conf/client.yml -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2151/e2e-tests/conf/minio-secret.yml
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.E79O4oow0B
secret/some-users created
deployment.apps/psmdb-client created
secret/minio-secret created
+ cat /tmp/tmp.ASlj5Apzj3
+ rm /tmp/tmp.E79O4oow0B /tmp/tmp.ASlj5Apzj3
+ return 0
+ cluster=some-name
+ desc 'create first PSMDB cluster some-name'
+ set +o xtrace


-----------------------------------------------------------------------------------
create first PSMDB cluster some-name
-----------------------------------------------------------------------------------

+ apply_cluster /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2151/e2e-tests/pitr-physical-backup-source/conf/some-name-rs0.yml
+ '[' -z '' ']'
+ cat_config /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2151/e2e-tests/pitr-physical-backup-source/conf/some-name-rs0.yml
+ kubectl_bin apply -f -
+ cat /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2151/e2e-tests/pitr-physical-backup-source/conf/some-name-rs0.yml
+ yq eval '(.spec | select(.image == null)).image = "perconalab/percona-server-mongodb-operator:main-mongod8.0"'
++ mktemp
+ yq eval '(.spec | select(has("pmm"))).pmm.image = "percona/pmm-client:2.44.1-1"'
+ yq eval '(.spec | select(has("initImage"))).initImage = "perconalab/percona-server-mongodb-operator:PR-2151-37812f21"'
+ yq eval '(.spec | select(has("backup"))).backup.image = "perconalab/percona-server-mongodb-operator:main-backup"'
+ /usr/sbin/sed -e s/NAME_SPACE/pitr-physical-backup-source-28579/g
+ local LAST_OUT=/tmp/tmp.0ZcW8UIPWE
++ mktemp
+ yq eval '.spec.upgradeOptions.apply="Never"'
+ local LAST_ERR=/tmp/tmp.Y7ur8ab8I9
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl apply -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.0ZcW8UIPWE
perconaservermongodb.psmdb.percona.com/some-name created
+ cat /tmp/tmp.Y7ur8ab8I9
+ rm /tmp/tmp.0ZcW8UIPWE /tmp/tmp.Y7ur8ab8I9
+ return 0
+ desc 'check if all 3 Pods started'
+ set +o xtrace


-----------------------------------------------------------------------------------
check if all 3 Pods started
-----------------------------------------------------------------------------------

+ wait_for_running some-name-rs0 3
+ local name=some-name-rs0
+ let last_pod=2
+ local check_cluster_readyness=true
+ set_debug
+ [[ 1 == 1 ]]
+ set -o xtrace
+ local rs_name=rs0
+ local cluster_name=some-name
++ seq 0 2
+ for i in $(seq 0 $last_pod)
+ [[ 0 -eq 2 ]]
+ wait_pod some-name-rs0-0
+ local pod=some-name-rs0-0
+ set +o xtrace
waiting for pod/some-name-rs0-0 to be ready.......OK
+ for i in $(seq 0 $last_pod)
+ [[ 1 -eq 2 ]]
+ wait_pod some-name-rs0-1
+ local pod=some-name-rs0-1
+ set +o xtrace
waiting for pod/some-name-rs0-1 to be ready.....OK
+ for i in $(seq 0 $last_pod)
+ [[ 2 -eq 2 ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].arbiter.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.UOkYmwzu9u
+++ mktemp
++ local LAST_ERR=/tmp/tmp.fBaR8MPcUZ
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].arbiter.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.UOkYmwzu9u
++ cat /tmp/tmp.fBaR8MPcUZ
++ rm /tmp/tmp.UOkYmwzu9u /tmp/tmp.fBaR8MPcUZ
++ return 0
+ [[ '' == \t\r\u\e ]]
+ wait_pod some-name-rs0-2
+ local pod=some-name-rs0-2
+ set +o xtrace
waiting for pod/some-name-rs0-2 to be ready.....OK
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].nonvoting.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.K6S4fhHc3q
+++ mktemp
++ local LAST_ERR=/tmp/tmp.OYzFY7Etm5
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].nonvoting.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.K6S4fhHc3q
++ cat /tmp/tmp.OYzFY7Etm5
++ rm /tmp/tmp.K6S4fhHc3q /tmp/tmp.OYzFY7Etm5
++ return 0
+ [[ '' == \t\r\u\e ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].hidden.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.YlN1PunKor
+++ mktemp
++ local LAST_ERR=/tmp/tmp.CLuJkibPqa
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].hidden.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.YlN1PunKor
++ cat /tmp/tmp.CLuJkibPqa
++ rm /tmp/tmp.YlN1PunKor /tmp/tmp.CLuJkibPqa
++ return 0
+ [[ '' == \t\r\u\e ]]
+ sleep 10
+ [[ true == \t\r\u\e ]]
+ set +x
Waiting for cluster readyness...................................................
+ wait_for_running some-name-cfg 3 false
+ local name=some-name-cfg
+ let last_pod=2
+ local check_cluster_readyness=false
+ set_debug
+ [[ 1 == 1 ]]
+ set -o xtrace
+ local rs_name=cfg
+ local cluster_name=some-name
++ seq 0 2
+ for i in $(seq 0 $last_pod)
+ [[ 0 -eq 2 ]]
+ wait_pod some-name-cfg-0
+ local pod=some-name-cfg-0
+ set +o xtrace
waiting for pod/some-name-cfg-0 to be ready.OK
+ for i in $(seq 0 $last_pod)
+ [[ 1 -eq 2 ]]
+ wait_pod some-name-cfg-1
+ local pod=some-name-cfg-1
+ set +o xtrace
waiting for pod/some-name-cfg-1 to be ready.OK
+ for i in $(seq 0 $last_pod)
+ [[ 2 -eq 2 ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].arbiter.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.yWHTe6NbFo
+++ mktemp
++ local LAST_ERR=/tmp/tmp.tERzLRBu1O
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].arbiter.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.yWHTe6NbFo
++ cat /tmp/tmp.tERzLRBu1O
++ rm /tmp/tmp.yWHTe6NbFo /tmp/tmp.tERzLRBu1O
++ return 0
+ [[ '' == \t\r\u\e ]]
+ wait_pod some-name-cfg-2
+ local pod=some-name-cfg-2
+ set +o xtrace
waiting for pod/some-name-cfg-2 to be ready.OK
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].nonvoting.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.fmGBxR6R39
+++ mktemp
++ local LAST_ERR=/tmp/tmp.fc2tmNUno5
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].nonvoting.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.fmGBxR6R39
++ cat /tmp/tmp.fc2tmNUno5
++ rm /tmp/tmp.fmGBxR6R39 /tmp/tmp.fc2tmNUno5
++ return 0
+ [[ '' == \t\r\u\e ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].hidden.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.NVtXopVeVQ
+++ mktemp
++ local LAST_ERR=/tmp/tmp.3tdQOYzKyv
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].hidden.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.NVtXopVeVQ
++ cat /tmp/tmp.3tdQOYzKyv
++ rm /tmp/tmp.NVtXopVeVQ /tmp/tmp.3tdQOYzKyv
++ return 0
+ [[ '' == \t\r\u\e ]]
+ sleep 10
+ [[ false == \t\r\u\e ]]
+ sleep 10
+ write_initial_data
+ desc 'create user myApp'
+ set +o xtrace


-----------------------------------------------------------------------------------
create user myApp
-----------------------------------------------------------------------------------

+ run_mongos 'db.createUser({user:"myApp",pwd:"myPass",roles:[{db:"myApp",role:"readWrite"}]})' userAdmin:userAdmin123456@some-name-mongos.pitr-physical-backup-source-28579
+ local 'command=db.createUser({user:"myApp",pwd:"myPass",roles:[{db:"myApp",role:"readWrite"}]})'
+ local uri=userAdmin:userAdmin123456@some-name-mongos.pitr-physical-backup-source-28579
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local port=27017
+ local mongo_bin=mongo
++ echo .svc.cluster.local
++ awk -F: '{print $2}'
+ suffix_port=
+ [[ -z '' ]]
+ suffix=.svc.cluster.local:27017
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.h8O3yHykTX
+++ mktemp
++ local LAST_ERR=/tmp/tmp.RvNZLq1zvv
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.h8O3yHykTX
++ cat /tmp/tmp.RvNZLq1zvv
++ rm /tmp/tmp.h8O3yHykTX /tmp/tmp.RvNZLq1zvv
++ return 0
+ local client_container=psmdb-client-696897d69b-m5f2h
+ kubectl_bin exec psmdb-client-696897d69b-m5f2h -- bash -c 'printf '\''db.createUser({user:"myApp",pwd:"myPass",roles:[{db:"myApp",role:"readWrite"}]})\n'\'' | mongo mongodb://userAdmin:userAdmin123456@some-name-mongos.pitr-physical-backup-source-28579.svc.cluster.local:27017/admin '
++ mktemp
+ local LAST_OUT=/tmp/tmp.t9F2vUGNvS
++ mktemp
+ local LAST_ERR=/tmp/tmp.XITe7eWWoS
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-696897d69b-m5f2h -- bash -c 'printf '\''db.createUser({user:"myApp",pwd:"myPass",roles:[{db:"myApp",role:"readWrite"}]})\n'\'' | mongo mongodb://userAdmin:userAdmin123456@some-name-mongos.pitr-physical-backup-source-28579.svc.cluster.local:27017/admin '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.t9F2vUGNvS
Percona Server for MongoDB shell version v4.4.29-28
connecting to: mongodb://some-name-mongos.pitr-physical-backup-source-28579.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb
Implicit session: session { "id" : UUID("f65a6c3f-2feb-4639-b007-564c337015a7") }
Percona Server for MongoDB server version: v8.0.16-5
WARNING: shell and server versions do not match
Successfully added user: {
	"user" : "myApp",
	"roles" : [
		{
			"db" : "myApp",
			"role" : "readWrite"
		}
	]
}
bye
+ cat /tmp/tmp.XITe7eWWoS
+ rm /tmp/tmp.t9F2vUGNvS /tmp/tmp.XITe7eWWoS
+ return 0
+ sleep 2
+ write_document
+ local cmp_postfix=
+ local sleep_value=0
+ log 'write initial data, read from all'
+ set +o xtrace
[2025-12-24T12:53:19+0000] write initial data, read from all
+ run_mongos 'use myApp\n db.test.insert({ x: 100500 })' myApp:myPass@some-name-mongos.pitr-physical-backup-source-28579
+ local 'command=use myApp\n db.test.insert({ x: 100500 })'
+ local uri=myApp:myPass@some-name-mongos.pitr-physical-backup-source-28579
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local port=27017
+ local mongo_bin=mongo
++ echo .svc.cluster.local
++ awk -F: '{print $2}'
+ suffix_port=
+ [[ -z '' ]]
+ suffix=.svc.cluster.local:27017
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.YkTxBSIQFj
+++ mktemp
++ local LAST_ERR=/tmp/tmp.nhcXFjzdR8
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.YkTxBSIQFj
++ cat /tmp/tmp.nhcXFjzdR8
++ rm /tmp/tmp.YkTxBSIQFj /tmp/tmp.nhcXFjzdR8
++ return 0
+ local client_container=psmdb-client-696897d69b-m5f2h
+ kubectl_bin exec psmdb-client-696897d69b-m5f2h -- bash -c 'printf '\''use myApp\n db.test.insert({ x: 100500 })\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.pitr-physical-backup-source-28579.svc.cluster.local:27017/admin '
++ mktemp
+ local LAST_OUT=/tmp/tmp.g3pWRezDmJ
++ mktemp
+ local LAST_ERR=/tmp/tmp.cwtPSpar9z
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-696897d69b-m5f2h -- bash -c 'printf '\''use myApp\n db.test.insert({ x: 100500 })\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.pitr-physical-backup-source-28579.svc.cluster.local:27017/admin '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.g3pWRezDmJ
Percona Server for MongoDB shell version v4.4.29-28
connecting to: mongodb://some-name-mongos.pitr-physical-backup-source-28579.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb
Implicit session: session { "id" : UUID("5bd5ca4b-55e5-4911-9c64-a6bc8a5a69b4") }
Percona Server for MongoDB server version: v8.0.16-5
WARNING: shell and server versions do not match
switched to db myApp
WriteResult({ "nInserted" : 1 })
bye
+ cat /tmp/tmp.cwtPSpar9z
+ rm /tmp/tmp.g3pWRezDmJ /tmp/tmp.cwtPSpar9z
+ return 0
+ sleep 0
+ compare_mongos_cmd find myApp:myPass@some-name-mongos.pitr-physical-backup-source-28579
+ local command=find
+ local uri=myApp:myPass@some-name-mongos.pitr-physical-backup-source-28579
+ local postfix=
+ local suffix=
+ local database=myApp
+ local collection=test
+ local port=27017
+ local tls=false
+ [[ false == \t\r\u\e ]]
+ mongos_command=run_mongos
+ log 'running db.test.command() in myApp'
+ set +o xtrace
[2025-12-24T12:53:22+0000] running db.test.command() in myApp
+ run_mongos 'use myApp\n db.test.find()' myApp:myPass@some-name-mongos.pitr-physical-backup-source-28579 mongodb '' '' 27017
+ local 'command=use myApp\n db.test.find()'
+ local uri=myApp:myPass@some-name-mongos.pitr-physical-backup-source-28579
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local port=27017
+ local mongo_bin=mongo
+ egrep -v 'I NETWORK|W NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|Started a new thread for the timer service'
+ /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
++ echo .svc.cluster.local
++ awk -F: '{print $2}'
egrep: warning: egrep is obsolescent; using grep -E
+ suffix_port=
+ [[ -z '' ]]
+ suffix=.svc.cluster.local:27017
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.j3Uc0Fzbwr
+++ mktemp
++ local LAST_ERR=/tmp/tmp.tIrAVLE2Uf
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.j3Uc0Fzbwr
++ cat /tmp/tmp.tIrAVLE2Uf
++ rm /tmp/tmp.j3Uc0Fzbwr /tmp/tmp.tIrAVLE2Uf
++ return 0
+ local client_container=psmdb-client-696897d69b-m5f2h
+ kubectl_bin exec psmdb-client-696897d69b-m5f2h -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.pitr-physical-backup-source-28579.svc.cluster.local:27017/admin '
++ mktemp
+ local LAST_OUT=/tmp/tmp.67xAqxDAtD
++ mktemp
+ local LAST_ERR=/tmp/tmp.jNiHAkmTbq
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-696897d69b-m5f2h -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.pitr-physical-backup-source-28579.svc.cluster.local:27017/admin '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.67xAqxDAtD
+ cat /tmp/tmp.jNiHAkmTbq
+ rm /tmp/tmp.67xAqxDAtD /tmp/tmp.jNiHAkmTbq
+ return 0
+ [[ 0 -eq 0 ]]
+ diff /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2151/e2e-tests/pitr-physical-backup-source/compare/find.json /tmp/tmp.e7bs9MurNC/find
+ wait_backup_agent some-name-rs0-0
+ local agent_pod=some-name-rs0-0
+ set +o xtrace
waiting for pbm-agent to be ready in some-name-rs0-0...2025-12-24T12:52:33.000+0000 I listening for the commands

+ wait_backup_agent some-name-rs0-1
+ local agent_pod=some-name-rs0-1
+ set +o xtrace
waiting for pbm-agent to be ready in some-name-rs0-1...2025-12-24T12:52:34.000+0000 I listening for the commands

+ wait_backup_agent some-name-rs0-2
+ local agent_pod=some-name-rs0-2
+ set +o xtrace
waiting for pbm-agent to be ready in some-name-rs0-2...2025-12-24T12:52:42.000+0000 I listening for the commands

+ wait_backup_agent some-name-rs1-0
+ local agent_pod=some-name-rs1-0
+ set +o xtrace
waiting for pbm-agent to be ready in some-name-rs1-0...2025-12-24T12:52:33.000+0000 I listening for the commands

+ wait_backup_agent some-name-rs1-1
+ local agent_pod=some-name-rs1-1
+ set +o xtrace
waiting for pbm-agent to be ready in some-name-rs1-1...2025-12-24T12:52:35.000+0000 I listening for the commands

+ wait_backup_agent some-name-rs1-2
+ local agent_pod=some-name-rs1-2
+ set +o xtrace
waiting for pbm-agent to be ready in some-name-rs1-2...2025-12-24T12:52:43.000+0000 I listening for the commands

+ wait_backup_agent some-name-rs2-0
+ local agent_pod=some-name-rs2-0
+ set +o xtrace
waiting for pbm-agent to be ready in some-name-rs2-0...2025-12-24T12:52:32.000+0000 I listening for the commands

+ wait_backup_agent some-name-rs2-1
+ local agent_pod=some-name-rs2-1
+ set +o xtrace
waiting for pbm-agent to be ready in some-name-rs2-1...2025-12-24T12:52:36.000+0000 I listening for the commands

+ wait_backup_agent some-name-rs2-2
+ local agent_pod=some-name-rs2-2
+ set +o xtrace
waiting for pbm-agent to be ready in some-name-rs2-2...2025-12-24T12:52:44.000+0000 I listening for the commands

+ echo 'Sleeping for 360 seconds'
Sleeping for 360 seconds
+ sleep 360
+ backup_name_minio=backup-minio
+ desc 'restore pitr type date using backupSource'
+ set +o xtrace


-----------------------------------------------------------------------------------
restore pitr type date using backupSource
-----------------------------------------------------------------------------------

+ run_backup backup-minio 1 physical
+ local name=backup-minio
+ local idx=1
+ local type=physical
+ desc 'run backup backup-minio-1'
+ set +o xtrace


-----------------------------------------------------------------------------------
run backup backup-minio-1
-----------------------------------------------------------------------------------

+ cat /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2151/e2e-tests/pitr-physical-backup-source/conf/backup-minio.yml
+ /usr/sbin/sed -e 's/name:/name: backup-minio-1/'
+ /usr/sbin/sed -e 's/type:/type: physical/'
+ kubectl_bin apply -f -
++ mktemp
+ local LAST_OUT=/tmp/tmp.majJ63gau5
++ mktemp
+ local LAST_ERR=/tmp/tmp.MNqt6pJG6p
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl apply -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.majJ63gau5
perconaservermongodbbackup.psmdb.percona.com/backup-minio-1 created
+ cat /tmp/tmp.MNqt6pJG6p
+ rm /tmp/tmp.majJ63gau5 /tmp/tmp.MNqt6pJG6p
+ return 0
+ wait_backup backup-minio-1
+ local backup_name=backup-minio-1
+ local target_state=ready
+ set +o xtrace
waiting for backup-minio-1 to reach ready state.........OK
+ compare_latest_restorable_time some-name-rs0 backup-minio-1
+ local cluster=some-name-rs0
+ local backup_name=backup-minio-1
+ local latest_restorable_time
+ local backup_time
++ get_latest_restorable_time some-name-rs0
++ local cluster=some-name-rs0
++ local first_timestamp
++ local second_timestamp
++ local retry=0
++ [[ '' != '' ]]
+++ kubectl_bin exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ jq '.backups.pitrChunks.pitrChunks | last | .range.end'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.ird2e1TRte
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.Aua1TMnRon
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in $(seq 0 2)
+++ set +e
+++ kubectl exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.ird2e1TRte
+++ cat /tmp/tmp.Aua1TMnRon
+++ rm /tmp/tmp.ird2e1TRte /tmp/tmp.Aua1TMnRon
+++ return 0
++ first_timestamp=null
++ sleep 5
++ [[ null != '' ]]
++ [[ null != \n\u\l\l ]]
++ let retry+=1
++ [[ 1 -gt 30 ]]
++ [[ null != '' ]]
++ [[ null != \n\u\l\l ]]
+++ kubectl_bin exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ jq '.backups.pitrChunks.pitrChunks | last | .range.end'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.0koCb3iPuo
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.hsMK5KwHsL
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in $(seq 0 2)
+++ set +e
+++ kubectl exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.0koCb3iPuo
+++ cat /tmp/tmp.hsMK5KwHsL
+++ rm /tmp/tmp.0koCb3iPuo /tmp/tmp.hsMK5KwHsL
+++ return 0
++ first_timestamp=null
++ sleep 5
++ [[ null != '' ]]
++ [[ null != \n\u\l\l ]]
++ let retry+=1
++ [[ 2 -gt 30 ]]
++ [[ null != '' ]]
++ [[ null != \n\u\l\l ]]
+++ kubectl_bin exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ jq '.backups.pitrChunks.pitrChunks | last | .range.end'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.zBRtv4yKu5
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.nkGIMg9FVW
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in $(seq 0 2)
+++ set +e
+++ kubectl exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.zBRtv4yKu5
+++ cat /tmp/tmp.nkGIMg9FVW
+++ rm /tmp/tmp.zBRtv4yKu5 /tmp/tmp.nkGIMg9FVW
+++ return 0
++ first_timestamp=null
++ sleep 5
++ [[ null != '' ]]
++ [[ null != \n\u\l\l ]]
++ let retry+=1
++ [[ 3 -gt 30 ]]
++ [[ null != '' ]]
++ [[ null != \n\u\l\l ]]
+++ kubectl_bin exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ jq '.backups.pitrChunks.pitrChunks | last | .range.end'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.YzSZjaj1cB
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.pc3ksoqsqp
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in $(seq 0 2)
+++ set +e
+++ kubectl exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.YzSZjaj1cB
+++ cat /tmp/tmp.pc3ksoqsqp
+++ rm /tmp/tmp.YzSZjaj1cB /tmp/tmp.pc3ksoqsqp
+++ return 0
++ first_timestamp=null
++ sleep 5
++ [[ null != '' ]]
++ [[ null != \n\u\l\l ]]
++ let retry+=1
++ [[ 4 -gt 30 ]]
++ [[ null != '' ]]
++ [[ null != \n\u\l\l ]]
+++ jq '.backups.pitrChunks.pitrChunks | last | .range.end'
+++ kubectl_bin exec some-name-rs0-0 -c backup-agent -- pbm status -o json
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.Tw2C9EFOi6
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.jFlz3tCrtj
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in $(seq 0 2)
+++ set +e
+++ kubectl exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.Tw2C9EFOi6
+++ cat /tmp/tmp.jFlz3tCrtj
+++ rm /tmp/tmp.Tw2C9EFOi6 /tmp/tmp.jFlz3tCrtj
+++ return 0
++ first_timestamp=null
++ sleep 5
++ [[ null != '' ]]
++ [[ null != \n\u\l\l ]]
++ let retry+=1
++ [[ 5 -gt 30 ]]
++ [[ null != '' ]]
++ [[ null != \n\u\l\l ]]
+++ kubectl_bin exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ jq '.backups.pitrChunks.pitrChunks | last | .range.end'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.KDUPfsDpsc
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.avtIxFvqRS
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in $(seq 0 2)
+++ set +e
+++ kubectl exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.KDUPfsDpsc
+++ cat /tmp/tmp.avtIxFvqRS
+++ rm /tmp/tmp.KDUPfsDpsc /tmp/tmp.avtIxFvqRS
+++ return 0
++ first_timestamp=1766581217
++ sleep 5
++ [[ 1766581217 != '' ]]
++ [[ 1766581217 != \n\u\l\l ]]
+++ kubectl_bin exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ jq '.backups.pitrChunks.pitrChunks | last | .range.end'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.coijSE04l3
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.CF8kJS5S67
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in $(seq 0 2)
+++ set +e
+++ kubectl exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.coijSE04l3
+++ cat /tmp/tmp.CF8kJS5S67
+++ rm /tmp/tmp.coijSE04l3 /tmp/tmp.CF8kJS5S67
+++ return 0
++ second_timestamp=1766581217
++ let retry+=1
++ [[ 6 -gt 30 ]]
++ [[ 1766581217 != '' ]]
++ [[ 1766581217 != \n\u\l\l ]]
++ [[ 1766581217 == 1766581217 ]]
++ /usr/sbin/date -u -d @1766581217 +%Y-%m-%dT%H:%M:%SZ
+ latest_restorable_time=2025-12-24T13:00:17Z
++ get_latest_restorable_time_from_backup_object backup-minio-1
++ local backup_name=backup-minio-1
++ local latestRestorableTime
++ local retry=0
++ [[ '' != '' ]]
++ sleep 5
+++ kubectl_bin get psmdb-backup backup-minio-1 -o 'jsonpath={.status.latestRestorableTime}'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.DRm00O34qK
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.2yxmXAWLjx
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in $(seq 0 2)
+++ set +e
+++ kubectl get psmdb-backup backup-minio-1 -o 'jsonpath={.status.latestRestorableTime}'
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.DRm00O34qK
+++ cat /tmp/tmp.2yxmXAWLjx
+++ rm /tmp/tmp.DRm00O34qK /tmp/tmp.2yxmXAWLjx
+++ return 0
++ latestRestorableTime=2025-12-24T13:00:17Z
++ let retry+=1
++ [[ 1 -gt 30 ]]
++ [[ 2025-12-24T13:00:17Z != '' ]]
++ [[ 2025-12-24T13:00:17Z != \n\u\l\l ]]
++ echo 2025-12-24T13:00:17Z
+ backup_time=2025-12-24T13:00:17Z
+ [[ 2025-12-24T13:00:17Z != \2\0\2\5\-\1\2\-\2\4\T\1\3\:\0\0\:\1\7\Z ]]
+ reset_collection
+ desc 'reset data'
+ set +o xtrace


-----------------------------------------------------------------------------------
reset data
-----------------------------------------------------------------------------------

+ run_mongos 'use myApp\n db.test.remove({})' myApp:myPass@some-name-mongos.pitr-physical-backup-source-28579
+ local 'command=use myApp\n db.test.remove({})'
+ local uri=myApp:myPass@some-name-mongos.pitr-physical-backup-source-28579
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local port=27017
+ local mongo_bin=mongo
++ echo .svc.cluster.local
++ awk -F: '{print $2}'
+ suffix_port=
+ [[ -z '' ]]
+ suffix=.svc.cluster.local:27017
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.f0renPVCwc
+++ mktemp
++ local LAST_ERR=/tmp/tmp.zluyIPqjRM
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.f0renPVCwc
++ cat /tmp/tmp.zluyIPqjRM
++ rm /tmp/tmp.f0renPVCwc /tmp/tmp.zluyIPqjRM
++ return 0
+ local client_container=psmdb-client-696897d69b-m5f2h
+ kubectl_bin exec psmdb-client-696897d69b-m5f2h -- bash -c 'printf '\''use myApp\n db.test.remove({})\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.pitr-physical-backup-source-28579.svc.cluster.local:27017/admin '
++ mktemp
+ local LAST_OUT=/tmp/tmp.BkoKumkiX3
++ mktemp
+ local LAST_ERR=/tmp/tmp.oL4ATEd4Sy
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-696897d69b-m5f2h -- bash -c 'printf '\''use myApp\n db.test.remove({})\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.pitr-physical-backup-source-28579.svc.cluster.local:27017/admin '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.BkoKumkiX3
Percona Server for MongoDB shell version v4.4.29-28
connecting to: mongodb://some-name-mongos.pitr-physical-backup-source-28579.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb
Implicit session: session { "id" : UUID("41571096-37d6-4797-9ddb-8c5aadde607f") }
Percona Server for MongoDB server version: v8.0.16-5
WARNING: shell and server versions do not match
switched to db myApp
WriteResult({ "nRemoved" : 1 })
bye
+ cat /tmp/tmp.oL4ATEd4Sy
+ rm /tmp/tmp.BkoKumkiX3 /tmp/tmp.oL4ATEd4Sy
+ return 0
+ sleep 2
+ write_document '' 120
+ local cmp_postfix=
+ local sleep_value=120
+ log 'write initial data, read from all'
+ set +o xtrace
[2025-12-24T13:00:50+0000] write initial data, read from all
+ run_mongos 'use myApp\n db.test.insert({ x: 100500 })' myApp:myPass@some-name-mongos.pitr-physical-backup-source-28579
+ local 'command=use myApp\n db.test.insert({ x: 100500 })'
+ local uri=myApp:myPass@some-name-mongos.pitr-physical-backup-source-28579
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local port=27017
+ local mongo_bin=mongo
++ echo .svc.cluster.local
++ awk -F: '{print $2}'
+ suffix_port=
+ [[ -z '' ]]
+ suffix=.svc.cluster.local:27017
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.uiCQUSp01N
+++ mktemp
++ local LAST_ERR=/tmp/tmp.XrgsvXWYc4
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.uiCQUSp01N
++ cat /tmp/tmp.XrgsvXWYc4
++ rm /tmp/tmp.uiCQUSp01N /tmp/tmp.XrgsvXWYc4
++ return 0
+ local client_container=psmdb-client-696897d69b-m5f2h
+ kubectl_bin exec psmdb-client-696897d69b-m5f2h -- bash -c 'printf '\''use myApp\n db.test.insert({ x: 100500 })\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.pitr-physical-backup-source-28579.svc.cluster.local:27017/admin '
++ mktemp
+ local LAST_OUT=/tmp/tmp.jPnlyFWuhb
++ mktemp
+ local LAST_ERR=/tmp/tmp.iE7heblOQq
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-696897d69b-m5f2h -- bash -c 'printf '\''use myApp\n db.test.insert({ x: 100500 })\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.pitr-physical-backup-source-28579.svc.cluster.local:27017/admin '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.jPnlyFWuhb
Percona Server for MongoDB shell version v4.4.29-28
connecting to: mongodb://some-name-mongos.pitr-physical-backup-source-28579.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb
Implicit session: session { "id" : UUID("7462b5c1-d217-4a06-8fa0-60df3113c194") }
Percona Server for MongoDB server version: v8.0.16-5
WARNING: shell and server versions do not match
switched to db myApp
WriteResult({ "nInserted" : 1 })
bye
+ cat /tmp/tmp.iE7heblOQq
+ rm /tmp/tmp.jPnlyFWuhb /tmp/tmp.iE7heblOQq
+ return 0
+ sleep 120
+ compare_mongos_cmd find myApp:myPass@some-name-mongos.pitr-physical-backup-source-28579
+ local command=find
+ local uri=myApp:myPass@some-name-mongos.pitr-physical-backup-source-28579
+ local postfix=
+ local suffix=
+ local database=myApp
+ local collection=test
+ local port=27017
+ local tls=false
+ [[ false == \t\r\u\e ]]
+ mongos_command=run_mongos
+ log 'running db.test.command() in myApp'
+ set +o xtrace
[2025-12-24T13:02:53+0000] running db.test.command() in myApp
+ run_mongos 'use myApp\n db.test.find()' myApp:myPass@some-name-mongos.pitr-physical-backup-source-28579 mongodb '' '' 27017
+ local 'command=use myApp\n db.test.find()'
+ local uri=myApp:myPass@some-name-mongos.pitr-physical-backup-source-28579
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ egrep -v 'I NETWORK|W NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|Started a new thread for the timer service'
+ local mongo_flag=
+ local port=27017
+ local mongo_bin=mongo
+ /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
++ echo .svc.cluster.local
++ awk -F: '{print $2}'
egrep: warning: egrep is obsolescent; using grep -E
+ suffix_port=
+ [[ -z '' ]]
+ suffix=.svc.cluster.local:27017
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.p4PlIRMFUR
+++ mktemp
++ local LAST_ERR=/tmp/tmp.TCqlAYxMXB
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.p4PlIRMFUR
++ cat /tmp/tmp.TCqlAYxMXB
++ rm /tmp/tmp.p4PlIRMFUR /tmp/tmp.TCqlAYxMXB
++ return 0
+ local client_container=psmdb-client-696897d69b-m5f2h
+ kubectl_bin exec psmdb-client-696897d69b-m5f2h -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.pitr-physical-backup-source-28579.svc.cluster.local:27017/admin '
++ mktemp
+ local LAST_OUT=/tmp/tmp.aPbEsB9Ec7
++ mktemp
+ local LAST_ERR=/tmp/tmp.bIztCpOxK3
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-696897d69b-m5f2h -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.pitr-physical-backup-source-28579.svc.cluster.local:27017/admin '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.aPbEsB9Ec7
+ cat /tmp/tmp.bIztCpOxK3
+ rm /tmp/tmp.aPbEsB9Ec7 /tmp/tmp.bIztCpOxK3
+ return 0
+ [[ 0 -eq 0 ]]
+ diff /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2151/e2e-tests/pitr-physical-backup-source/compare/find.json /tmp/tmp.e7bs9MurNC/find
++ run_mongos 'new Date().getTime() / 1000' myApp:myPass@some-name-mongos.pitr-physical-backup-source-28579 mongodb '' --quiet
++ local 'command=new Date().getTime() / 1000'
++ local uri=myApp:myPass@some-name-mongos.pitr-physical-backup-source-28579
++ local driver=mongodb
++ local suffix=.svc.cluster.local
++ local mongo_flag=--quiet
++ local port=27017
++ local mongo_bin=mongo
++ grep -E -v 'I NETWORK|W NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match'
++ cut -d. -f1
+++ echo .svc.cluster.local
+++ awk -F: '{print $2}'
++ suffix_port=
++ [[ -z '' ]]
++ suffix=.svc.cluster.local:27017
+++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.Qv9EiyCMZK
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.VPbO0Colil
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in $(seq 0 2)
+++ set +e
+++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.Qv9EiyCMZK
+++ cat /tmp/tmp.VPbO0Colil
+++ rm /tmp/tmp.Qv9EiyCMZK /tmp/tmp.VPbO0Colil
+++ return 0
++ local client_container=psmdb-client-696897d69b-m5f2h
++ kubectl_bin exec psmdb-client-696897d69b-m5f2h -- bash -c 'printf '\''new Date().getTime() / 1000\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.pitr-physical-backup-source-28579.svc.cluster.local:27017/admin --quiet'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.h5PSTtjxgU
+++ mktemp
++ local LAST_ERR=/tmp/tmp.hIjQIaXkbi
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl exec psmdb-client-696897d69b-m5f2h -- bash -c 'printf '\''new Date().getTime() / 1000\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.pitr-physical-backup-source-28579.svc.cluster.local:27017/admin --quiet'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.h5PSTtjxgU
++ cat /tmp/tmp.hIjQIaXkbi
++ rm /tmp/tmp.h5PSTtjxgU /tmp/tmp.hIjQIaXkbi
++ return 0
+ time_now=1766581378
+ check_recovery backup-minio-1 date 1766581378 '' some-name backupSource
+ local backup_name=backup-minio-1
+ local restore_type=date
+ local restore_date=1766581378
+ local cmp_postfix=
+ local cluster_name=some-name
+ local backupSource=backupSource
++ get_latest_oplog_chunk_ts some-name
++ local cluster=some-name
+++ kubectl_bin exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ jq '.backups.pitrChunks.pitrChunks | last | .range.end'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.qr6g9HpbUP
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.8lP2DHRJyz
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in $(seq 0 2)
+++ set +e
+++ kubectl exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.qr6g9HpbUP
+++ cat /tmp/tmp.8lP2DHRJyz
+++ rm /tmp/tmp.qr6g9HpbUP /tmp/tmp.8lP2DHRJyz
+++ return 0
++ echo 1766581346
+ local latest_ts=1766581346
+ desc 'write more data before restore by date'
+ set +o xtrace


-----------------------------------------------------------------------------------
write more data before restore by date
-----------------------------------------------------------------------------------

+ run_mongos 'use myApp\n db.test.insert({ x: 100501 })' myApp:myPass@some-name-mongos.pitr-physical-backup-source-28579
+ local 'command=use myApp\n db.test.insert({ x: 100501 })'
+ local uri=myApp:myPass@some-name-mongos.pitr-physical-backup-source-28579
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local port=27017
+ local mongo_bin=mongo
++ echo .svc.cluster.local
++ awk -F: '{print $2}'
+ suffix_port=
+ [[ -z '' ]]
+ suffix=.svc.cluster.local:27017
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.b1Ae8sPTWi
+++ mktemp
++ local LAST_ERR=/tmp/tmp.A4Z8oTjjMG
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.b1Ae8sPTWi
++ cat /tmp/tmp.A4Z8oTjjMG
++ rm /tmp/tmp.b1Ae8sPTWi /tmp/tmp.A4Z8oTjjMG
++ return 0
+ local client_container=psmdb-client-696897d69b-m5f2h
+ kubectl_bin exec psmdb-client-696897d69b-m5f2h -- bash -c 'printf '\''use myApp\n db.test.insert({ x: 100501 })\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.pitr-physical-backup-source-28579.svc.cluster.local:27017/admin '
++ mktemp
+ local LAST_OUT=/tmp/tmp.988d21Yx7W
++ mktemp
+ local LAST_ERR=/tmp/tmp.mY7roUPlY9
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-696897d69b-m5f2h -- bash -c 'printf '\''use myApp\n db.test.insert({ x: 100501 })\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.pitr-physical-backup-source-28579.svc.cluster.local:27017/admin '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.988d21Yx7W
Percona Server for MongoDB shell version v4.4.29-28
connecting to: mongodb://some-name-mongos.pitr-physical-backup-source-28579.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb
Implicit session: session { "id" : UUID("d2278ad6-ec0f-4025-b839-855c95604dd9") }
Percona Server for MongoDB server version: v8.0.16-5
WARNING: shell and server versions do not match
switched to db myApp
WriteResult({ "nInserted" : 1 })
bye
+ cat /tmp/tmp.mY7roUPlY9
+ rm /tmp/tmp.988d21Yx7W /tmp/tmp.mY7roUPlY9
+ return 0
+ [[ -n 1766581378 ]]
++ format_date 1766581378
++ local timestamp=1766581378
+++ TZ=UTC
+++ /usr/sbin/date -d@1766581378 '+%Y-%m-%d %H:%M:%S'
++ echo 2025-12-24 13:02:58
+ desc 'Restoring to time 2025-12-24 13:02:58'
+ set +o xtrace


-----------------------------------------------------------------------------------
Restoring to time 2025-12-24 13:02:58
-----------------------------------------------------------------------------------

+ retries=0
+ [[ 1766581346 -gt 1766581378 ]]
+ [[ 0 -gt 30 ]]
++ get_latest_oplog_chunk_ts some-name
++ local cluster=some-name
+++ kubectl_bin exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ jq '.backups.pitrChunks.pitrChunks | last | .range.end'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.nhz3eJs1aq
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.KomloRix9L
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in $(seq 0 2)
+++ set +e
+++ kubectl exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.nhz3eJs1aq
+++ cat /tmp/tmp.KomloRix9L
+++ rm /tmp/tmp.nhz3eJs1aq /tmp/tmp.KomloRix9L
+++ return 0
++ echo 1766581346
+ latest_ts=1766581346
+ retries=1
++ format_date 1766581346
++ local timestamp=1766581346
+++ TZ=UTC
+++ /usr/sbin/date -d@1766581346 '+%Y-%m-%d %H:%M:%S'
++ echo 2025-12-24 13:02:26
++ format_date 1766581378
++ local timestamp=1766581378
+++ TZ=UTC
+++ /usr/sbin/date -d@1766581378 '+%Y-%m-%d %H:%M:%S'
++ echo 2025-12-24 13:02:58
+ echo 'Waiting for last oplog chunk (2025-12-24 13:02:26) to be greater than restore target (2025-12-24 13:02:58)'
Waiting for last oplog chunk (2025-12-24 13:02:26) to be greater than restore target (2025-12-24 13:02:58)
+ sleep 10
+ [[ 1766581346 -gt 1766581378 ]]
+ [[ 1 -gt 30 ]]
++ get_latest_oplog_chunk_ts some-name
++ local cluster=some-name
+++ kubectl_bin exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ jq '.backups.pitrChunks.pitrChunks | last | .range.end'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.iGMxV0mhxL
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.Vx06cVC65X
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in $(seq 0 2)
+++ set +e
+++ kubectl exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.iGMxV0mhxL
+++ cat /tmp/tmp.Vx06cVC65X
+++ rm /tmp/tmp.iGMxV0mhxL /tmp/tmp.Vx06cVC65X
+++ return 0
++ echo 1766581346
+ latest_ts=1766581346
+ retries=2
++ format_date 1766581346
++ local timestamp=1766581346
+++ TZ=UTC
+++ /usr/sbin/date -d@1766581346 '+%Y-%m-%d %H:%M:%S'
++ echo 2025-12-24 13:02:26
++ format_date 1766581378
++ local timestamp=1766581378
+++ TZ=UTC
+++ /usr/sbin/date -d@1766581378 '+%Y-%m-%d %H:%M:%S'
++ echo 2025-12-24 13:02:58
+ echo 'Waiting for last oplog chunk (2025-12-24 13:02:26) to be greater than restore target (2025-12-24 13:02:58)'
Waiting for last oplog chunk (2025-12-24 13:02:26) to be greater than restore target (2025-12-24 13:02:58)
+ sleep 10
+ [[ 1766581346 -gt 1766581378 ]]
+ [[ 2 -gt 30 ]]
++ get_latest_oplog_chunk_ts some-name
++ local cluster=some-name
+++ kubectl_bin exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ jq '.backups.pitrChunks.pitrChunks | last | .range.end'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.MZABsI3T65
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.47SgmxhcPA
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in $(seq 0 2)
+++ set +e
+++ kubectl exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.MZABsI3T65
+++ cat /tmp/tmp.47SgmxhcPA
+++ rm /tmp/tmp.MZABsI3T65 /tmp/tmp.47SgmxhcPA
+++ return 0
++ echo 1766581346
+ latest_ts=1766581346
+ retries=3
++ format_date 1766581346
++ local timestamp=1766581346
+++ TZ=UTC
+++ /usr/sbin/date -d@1766581346 '+%Y-%m-%d %H:%M:%S'
++ echo 2025-12-24 13:02:26
++ format_date 1766581378
++ local timestamp=1766581378
+++ TZ=UTC
+++ /usr/sbin/date -d@1766581378 '+%Y-%m-%d %H:%M:%S'
++ echo 2025-12-24 13:02:58
+ echo 'Waiting for last oplog chunk (2025-12-24 13:02:26) to be greater than restore target (2025-12-24 13:02:58)'
Waiting for last oplog chunk (2025-12-24 13:02:26) to be greater than restore target (2025-12-24 13:02:58)
+ sleep 10
+ [[ 1766581346 -gt 1766581378 ]]
+ [[ 3 -gt 30 ]]
++ get_latest_oplog_chunk_ts some-name
++ local cluster=some-name
+++ kubectl_bin exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ jq '.backups.pitrChunks.pitrChunks | last | .range.end'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.w5jtQIEyDR
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.5xsQ7jE4BI
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in $(seq 0 2)
+++ set +e
+++ kubectl exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.w5jtQIEyDR
+++ cat /tmp/tmp.5xsQ7jE4BI
+++ rm /tmp/tmp.w5jtQIEyDR /tmp/tmp.5xsQ7jE4BI
+++ return 0
++ echo 1766581346
+ latest_ts=1766581346
+ retries=4
++ format_date 1766581346
++ local timestamp=1766581346
+++ TZ=UTC
+++ /usr/sbin/date -d@1766581346 '+%Y-%m-%d %H:%M:%S'
++ echo 2025-12-24 13:02:26
++ format_date 1766581378
++ local timestamp=1766581378
+++ TZ=UTC
+++ /usr/sbin/date -d@1766581378 '+%Y-%m-%d %H:%M:%S'
++ echo 2025-12-24 13:02:58
+ echo 'Waiting for last oplog chunk (2025-12-24 13:02:26) to be greater than restore target (2025-12-24 13:02:58)'
Waiting for last oplog chunk (2025-12-24 13:02:26) to be greater than restore target (2025-12-24 13:02:58)
+ sleep 10
+ [[ 1766581346 -gt 1766581378 ]]
+ [[ 4 -gt 30 ]]
++ get_latest_oplog_chunk_ts some-name
++ local cluster=some-name
+++ kubectl_bin exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ jq '.backups.pitrChunks.pitrChunks | last | .range.end'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.hNTPkt5laD
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.9m6l8PZjZ0
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in $(seq 0 2)
+++ set +e
+++ kubectl exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.hNTPkt5laD
+++ cat /tmp/tmp.9m6l8PZjZ0
+++ rm /tmp/tmp.hNTPkt5laD /tmp/tmp.9m6l8PZjZ0
+++ return 0
++ echo 1766581346
+ latest_ts=1766581346
+ retries=5
++ format_date 1766581346
++ local timestamp=1766581346
+++ TZ=UTC
+++ /usr/sbin/date -d@1766581346 '+%Y-%m-%d %H:%M:%S'
++ echo 2025-12-24 13:02:26
++ format_date 1766581378
++ local timestamp=1766581378
+++ TZ=UTC
+++ /usr/sbin/date -d@1766581378 '+%Y-%m-%d %H:%M:%S'
++ echo 2025-12-24 13:02:58
+ echo 'Waiting for last oplog chunk (2025-12-24 13:02:26) to be greater than restore target (2025-12-24 13:02:58)'
Waiting for last oplog chunk (2025-12-24 13:02:26) to be greater than restore target (2025-12-24 13:02:58)
+ sleep 10
+ [[ 1766581346 -gt 1766581378 ]]
+ [[ 5 -gt 30 ]]
++ get_latest_oplog_chunk_ts some-name
++ local cluster=some-name
+++ kubectl_bin exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ jq '.backups.pitrChunks.pitrChunks | last | .range.end'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.jU2W1jNJdY
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.5jrAm4tbPt
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in $(seq 0 2)
+++ set +e
+++ kubectl exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.jU2W1jNJdY
+++ cat /tmp/tmp.5jrAm4tbPt
+++ rm /tmp/tmp.jU2W1jNJdY /tmp/tmp.5jrAm4tbPt
+++ return 0
++ echo 1766581346
+ latest_ts=1766581346
+ retries=6
++ format_date 1766581346
++ local timestamp=1766581346
+++ TZ=UTC
+++ /usr/sbin/date -d@1766581346 '+%Y-%m-%d %H:%M:%S'
++ echo 2025-12-24 13:02:26
++ format_date 1766581378
++ local timestamp=1766581378
+++ TZ=UTC
+++ /usr/sbin/date -d@1766581378 '+%Y-%m-%d %H:%M:%S'
++ echo 2025-12-24 13:02:58
+ echo 'Waiting for last oplog chunk (2025-12-24 13:02:26) to be greater than restore target (2025-12-24 13:02:58)'
Waiting for last oplog chunk (2025-12-24 13:02:26) to be greater than restore target (2025-12-24 13:02:58)
+ sleep 10
+ [[ 1766581346 -gt 1766581378 ]]
+ [[ 6 -gt 30 ]]
++ get_latest_oplog_chunk_ts some-name
++ local cluster=some-name
+++ kubectl_bin exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ jq '.backups.pitrChunks.pitrChunks | last | .range.end'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.HCy4Ot3xpa
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.Z2MqkrYCXU
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in $(seq 0 2)
+++ set +e
+++ kubectl exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.HCy4Ot3xpa
+++ cat /tmp/tmp.Z2MqkrYCXU
+++ rm /tmp/tmp.HCy4Ot3xpa /tmp/tmp.Z2MqkrYCXU
+++ return 0
++ echo 1766581346
+ latest_ts=1766581346
+ retries=7
++ format_date 1766581346
++ local timestamp=1766581346
+++ TZ=UTC
+++ /usr/sbin/date -d@1766581346 '+%Y-%m-%d %H:%M:%S'
++ echo 2025-12-24 13:02:26
++ format_date 1766581378
++ local timestamp=1766581378
+++ TZ=UTC
+++ /usr/sbin/date -d@1766581378 '+%Y-%m-%d %H:%M:%S'
++ echo 2025-12-24 13:02:58
+ echo 'Waiting for last oplog chunk (2025-12-24 13:02:26) to be greater than restore target (2025-12-24 13:02:58)'
Waiting for last oplog chunk (2025-12-24 13:02:26) to be greater than restore target (2025-12-24 13:02:58)
+ sleep 10
+ [[ 1766581346 -gt 1766581378 ]]
+ [[ 7 -gt 30 ]]
++ get_latest_oplog_chunk_ts some-name
++ local cluster=some-name
+++ kubectl_bin exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ jq '.backups.pitrChunks.pitrChunks | last | .range.end'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.X12GcgeN7c
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.CypbjmETj6
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in $(seq 0 2)
+++ set +e
+++ kubectl exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.X12GcgeN7c
+++ cat /tmp/tmp.CypbjmETj6
+++ rm /tmp/tmp.X12GcgeN7c /tmp/tmp.CypbjmETj6
+++ return 0
++ echo 1766581347
+ latest_ts=1766581347
+ retries=8
++ format_date 1766581347
++ local timestamp=1766581347
+++ TZ=UTC
+++ /usr/sbin/date -d@1766581347 '+%Y-%m-%d %H:%M:%S'
++ echo 2025-12-24 13:02:27
++ format_date 1766581378
++ local timestamp=1766581378
+++ TZ=UTC
+++ /usr/sbin/date -d@1766581378 '+%Y-%m-%d %H:%M:%S'
++ echo 2025-12-24 13:02:58
+ echo 'Waiting for last oplog chunk (2025-12-24 13:02:27) to be greater than restore target (2025-12-24 13:02:58)'
Waiting for last oplog chunk (2025-12-24 13:02:27) to be greater than restore target (2025-12-24 13:02:58)
+ sleep 10
+ [[ 1766581347 -gt 1766581378 ]]
+ [[ 8 -gt 30 ]]
++ get_latest_oplog_chunk_ts some-name
++ local cluster=some-name
+++ kubectl_bin exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ jq '.backups.pitrChunks.pitrChunks | last | .range.end'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.fWgh1nxzWK
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.Dz0Go8s35Q
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in $(seq 0 2)
+++ set +e
+++ kubectl exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.fWgh1nxzWK
+++ cat /tmp/tmp.Dz0Go8s35Q
+++ rm /tmp/tmp.fWgh1nxzWK /tmp/tmp.Dz0Go8s35Q
+++ return 0
++ echo 1766581357
+ latest_ts=1766581357
+ retries=9
++ format_date 1766581357
++ local timestamp=1766581357
+++ TZ=UTC
+++ /usr/sbin/date -d@1766581357 '+%Y-%m-%d %H:%M:%S'
++ echo 2025-12-24 13:02:37
++ format_date 1766581378
++ local timestamp=1766581378
+++ TZ=UTC
+++ /usr/sbin/date -d@1766581378 '+%Y-%m-%d %H:%M:%S'
++ echo 2025-12-24 13:02:58
+ echo 'Waiting for last oplog chunk (2025-12-24 13:02:37) to be greater than restore target (2025-12-24 13:02:58)'
Waiting for last oplog chunk (2025-12-24 13:02:37) to be greater than restore target (2025-12-24 13:02:58)
+ sleep 10
+ [[ 1766581357 -gt 1766581378 ]]
+ [[ 9 -gt 30 ]]
++ get_latest_oplog_chunk_ts some-name
++ local cluster=some-name
+++ kubectl_bin exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ jq '.backups.pitrChunks.pitrChunks | last | .range.end'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.oeyKPSsi6Y
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.WvrjNYfJMz
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in $(seq 0 2)
+++ set +e
+++ kubectl exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.oeyKPSsi6Y
+++ cat /tmp/tmp.WvrjNYfJMz
+++ rm /tmp/tmp.oeyKPSsi6Y /tmp/tmp.WvrjNYfJMz
+++ return 0
++ echo 1766581466
+ latest_ts=1766581466
+ retries=10
++ format_date 1766581466
++ local timestamp=1766581466
+++ TZ=UTC
+++ /usr/sbin/date -d@1766581466 '+%Y-%m-%d %H:%M:%S'
++ echo 2025-12-24 13:04:26
++ format_date 1766581378
++ local timestamp=1766581378
+++ TZ=UTC
+++ /usr/sbin/date -d@1766581378 '+%Y-%m-%d %H:%M:%S'
++ echo 2025-12-24 13:02:58
+ echo 'Waiting for last oplog chunk (2025-12-24 13:04:26) to be greater than restore target (2025-12-24 13:02:58)'
Waiting for last oplog chunk (2025-12-24 13:04:26) to be greater than restore target (2025-12-24 13:02:58)
+ sleep 10
+ [[ 1766581466 -gt 1766581378 ]]
+ '[' -z backupSource ']'
+ desc 'check restore by date backupSource'
+ set +o xtrace


-----------------------------------------------------------------------------------
check restore by date backupSource
-----------------------------------------------------------------------------------

++ get_backup_dest backup-minio-1
++ local backup_name=backup-minio-1
++ kubectl_bin get psmdb-backup backup-minio-1 -o 'jsonpath={.status.destination}'
++ sed -e 's/.json$//'
++ sed 's|s3://||'
++ sed 's|azure://||'
+++ mktemp
++ sed 's|gs://||'
++ local LAST_OUT=/tmp/tmp.cvE6C67Hqt
+++ mktemp
++ local LAST_ERR=/tmp/tmp.0iN0RDfrkQ
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb-backup backup-minio-1 -o 'jsonpath={.status.destination}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.cvE6C67Hqt
++ cat /tmp/tmp.0iN0RDfrkQ
++ rm /tmp/tmp.cvE6C67Hqt /tmp/tmp.0iN0RDfrkQ
++ return 0
+ backup_dest=operator-testing/2025-12-24T12:59:38Z
+ cat /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2151/e2e-tests/pitr-physical-backup-source/conf/restore.yml
+ /usr/sbin/sed -e 's/name:/name: restore-backup-minio-1/'
+ /usr/sbin/sed -e /backupName/d
+ /usr/sbin/sed -e 's/pitrType:/type: date/'
+ '[' -z 1766581378 ']'
+ /usr/sbin/sed -e 's|DESTINATION|operator-testing/2025-12-24T12:59:38Z|'
++ format_date 1766581378
++ local timestamp=1766581378
+ kubectl_bin apply -f -
+ '[' -n '' ']'
+ yq
+++ TZ=UTC
+++ /usr/sbin/date -d@1766581378 '+%Y-%m-%d %H:%M:%S'
++ mktemp
++ get_bucket_name backup-minio-1
++ local backup_name=backup-minio-1
++ kubectl_bin get psmdb-backup backup-minio-1 -o 'jsonpath={.status.s3.bucket}'
+++ mktemp
+ local LAST_OUT=/tmp/tmp.AUV7TqeatO
++ echo 2025-12-24 13:02:58
++ mktemp
+ /usr/sbin/sed -e 's/date:/date: 2025-12-24 13:02:58/'
++ local LAST_OUT=/tmp/tmp.lqTKwmvOMb
+++ mktemp
+ local LAST_ERR=/tmp/tmp.wgm8aWRyQy
+ local exit_status=0
+ local timeout=4
++ seq 0 2
++ local LAST_ERR=/tmp/tmp.S4CIbmQdgv
++ local exit_status=0
++ local timeout=4
+ for i in $(seq 0 2)
+ set +e
+ kubectl apply -f -
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb-backup backup-minio-1 -o 'jsonpath={.status.s3.bucket}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.lqTKwmvOMb
++ cat /tmp/tmp.S4CIbmQdgv
++ rm /tmp/tmp.lqTKwmvOMb /tmp/tmp.S4CIbmQdgv
++ return 0
+ /usr/sbin/sed -e 's|BUCKET-NAME|operator-testing|'
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.AUV7TqeatO
perconaservermongodbrestore.psmdb.percona.com/restore-backup-minio-1 created
+ cat /tmp/tmp.wgm8aWRyQy
+ rm /tmp/tmp.AUV7TqeatO /tmp/tmp.wgm8aWRyQy
+ return 0
+ wait_restore backup-minio-1 some-name requested 0 1200
+ local backup_name=backup-minio-1
+ local cluster_name=some-name
+ local target_state=requested
+ local wait_cluster_consistency=0
+ local wait_time=1200
+ local ok_if_ready=0
+ set +o xtrace
Waiting for the psmdb-restore/restore-backup-minio-1 object to be created.OK
Waiting psmdb-restore/restore-backup-minio-1 to reach state "requested" .........OK after 8 minutes
+ [[ 0 -eq 1 ]]
+ echo

+ wait_restore backup-minio-1 some-name ready 0 1600
+ local backup_name=backup-minio-1
+ local cluster_name=some-name
+ local target_state=ready
+ local wait_cluster_consistency=0
+ local wait_time=1600
+ local ok_if_ready=0
+ set +o xtrace
Waiting for the psmdb-restore/restore-backup-minio-1 object to be created.OK
Waiting psmdb-restore/restore-backup-minio-1 to reach state "ready" ...OK after 2 minutes
+ [[ 0 -eq 1 ]]
+ echo

+ set -o xtrace
+ wait_for_running some-name-rs0 3
+ local name=some-name-rs0
+ let last_pod=2
+ local check_cluster_readyness=true
+ set_debug
+ [[ 1 == 1 ]]
+ set -o xtrace
+ local rs_name=rs0
+ local cluster_name=some-name
++ seq 0 2
+ for i in $(seq 0 $last_pod)
+ [[ 0 -eq 2 ]]
+ wait_pod some-name-rs0-0
+ local pod=some-name-rs0-0
+ set +o xtrace
waiting for pod/some-name-rs0-0 to be ready.OK
+ for i in $(seq 0 $last_pod)
+ [[ 1 -eq 2 ]]
+ wait_pod some-name-rs0-1
+ local pod=some-name-rs0-1
+ set +o xtrace
waiting for pod/some-name-rs0-1 to be ready.OK
+ for i in $(seq 0 $last_pod)
+ [[ 2 -eq 2 ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].arbiter.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.wFrx8yzd5L
+++ mktemp
++ local LAST_ERR=/tmp/tmp.R8XHor9LDK
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].arbiter.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.wFrx8yzd5L
++ cat /tmp/tmp.R8XHor9LDK
++ rm /tmp/tmp.wFrx8yzd5L /tmp/tmp.R8XHor9LDK
++ return 0
+ [[ '' == \t\r\u\e ]]
+ wait_pod some-name-rs0-2
+ local pod=some-name-rs0-2
+ set +o xtrace
waiting for pod/some-name-rs0-2 to be ready.OK
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].nonvoting.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.MTTPWlBB3j
+++ mktemp
++ local LAST_ERR=/tmp/tmp.319rXHTIDx
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].nonvoting.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.MTTPWlBB3j
++ cat /tmp/tmp.319rXHTIDx
++ rm /tmp/tmp.MTTPWlBB3j /tmp/tmp.319rXHTIDx
++ return 0
+ [[ '' == \t\r\u\e ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].hidden.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.PZv7T0Ct9P
+++ mktemp
++ local LAST_ERR=/tmp/tmp.rFBCV79Urn
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].hidden.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.PZv7T0Ct9P
++ cat /tmp/tmp.rFBCV79Urn
++ rm /tmp/tmp.PZv7T0Ct9P /tmp/tmp.rFBCV79Urn
++ return 0
+ [[ '' == \t\r\u\e ]]
+ sleep 10
+ [[ true == \t\r\u\e ]]
+ set +x
Waiting for cluster readyness....................................................
+ wait_for_running some-name-cfg 3
+ local name=some-name-cfg
+ let last_pod=2
+ local check_cluster_readyness=true
+ set_debug
+ [[ 1 == 1 ]]
+ set -o xtrace
+ local rs_name=cfg
+ local cluster_name=some-name
++ seq 0 2
+ for i in $(seq 0 $last_pod)
+ [[ 0 -eq 2 ]]
+ wait_pod some-name-cfg-0
+ local pod=some-name-cfg-0
+ set +o xtrace
waiting for pod/some-name-cfg-0 to be ready.OK
+ for i in $(seq 0 $last_pod)
+ [[ 1 -eq 2 ]]
+ wait_pod some-name-cfg-1
+ local pod=some-name-cfg-1
+ set +o xtrace
waiting for pod/some-name-cfg-1 to be ready.OK
+ for i in $(seq 0 $last_pod)
+ [[ 2 -eq 2 ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].arbiter.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.b89s4K08HM
+++ mktemp
++ local LAST_ERR=/tmp/tmp.foMF6tvNHr
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].arbiter.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.b89s4K08HM
++ cat /tmp/tmp.foMF6tvNHr
++ rm /tmp/tmp.b89s4K08HM /tmp/tmp.foMF6tvNHr
++ return 0
+ [[ '' == \t\r\u\e ]]
+ wait_pod some-name-cfg-2
+ local pod=some-name-cfg-2
+ set +o xtrace
waiting for pod/some-name-cfg-2 to be ready.OK
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].nonvoting.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.CurMfjwCmI
+++ mktemp
++ local LAST_ERR=/tmp/tmp.uwAou36rTo
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].nonvoting.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.CurMfjwCmI
++ cat /tmp/tmp.uwAou36rTo
++ rm /tmp/tmp.CurMfjwCmI /tmp/tmp.uwAou36rTo
++ return 0
+ [[ '' == \t\r\u\e ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].hidden.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.frmEUQqkX5
+++ mktemp
++ local LAST_ERR=/tmp/tmp.qf8z4JEuzG
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].hidden.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.frmEUQqkX5
++ cat /tmp/tmp.qf8z4JEuzG
++ rm /tmp/tmp.frmEUQqkX5 /tmp/tmp.qf8z4JEuzG
++ return 0
+ [[ '' == \t\r\u\e ]]
+ sleep 10
+ [[ true == \t\r\u\e ]]
+ set +x
Waiting for cluster readyness
+ wait_for_running some-name-mongos 3
+ local name=some-name-mongos
+ let last_pod=2
+ local check_cluster_readyness=true
+ set_debug
+ [[ 1 == 1 ]]
+ set -o xtrace
+ local rs_name=mongos
+ local cluster_name=some-name
++ seq 0 2
+ for i in $(seq 0 $last_pod)
+ [[ 0 -eq 2 ]]
+ wait_pod some-name-mongos-0
+ local pod=some-name-mongos-0
+ set +o xtrace
waiting for pod/some-name-mongos-0 to be ready.OK
+ for i in $(seq 0 $last_pod)
+ [[ 1 -eq 2 ]]
+ wait_pod some-name-mongos-1
+ local pod=some-name-mongos-1
+ set +o xtrace
waiting for pod/some-name-mongos-1 to be ready.OK
+ for i in $(seq 0 $last_pod)
+ [[ 2 -eq 2 ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="mongos")].arbiter.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.TFCIvbSbHx
+++ mktemp
++ local LAST_ERR=/tmp/tmp.3q7DqzhsnX
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="mongos")].arbiter.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.TFCIvbSbHx
++ cat /tmp/tmp.3q7DqzhsnX
++ rm /tmp/tmp.TFCIvbSbHx /tmp/tmp.3q7DqzhsnX
++ return 0
+ [[ '' == \t\r\u\e ]]
+ wait_pod some-name-mongos-2
+ local pod=some-name-mongos-2
+ set +o xtrace
waiting for pod/some-name-mongos-2 to be ready.OK
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="mongos")].nonvoting.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.r6YcJiH7p0
+++ mktemp
++ local LAST_ERR=/tmp/tmp.jM4c3lriWs
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="mongos")].nonvoting.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.r6YcJiH7p0
++ cat /tmp/tmp.jM4c3lriWs
++ rm /tmp/tmp.r6YcJiH7p0 /tmp/tmp.jM4c3lriWs
++ return 0
+ [[ '' == \t\r\u\e ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="mongos")].hidden.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.Aq5txk0mWs
+++ mktemp
++ local LAST_ERR=/tmp/tmp.gE4iB1lAno
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="mongos")].hidden.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.Aq5txk0mWs
++ cat /tmp/tmp.gE4iB1lAno
++ rm /tmp/tmp.Aq5txk0mWs /tmp/tmp.gE4iB1lAno
++ return 0
+ [[ '' == \t\r\u\e ]]
+ sleep 10
+ [[ true == \t\r\u\e ]]
+ set +x
Waiting for cluster readyness
+ sleep 10
+ compare_mongos_cmd find myApp:myPass@some-name-mongos.pitr-physical-backup-source-28579 ''
+ local command=find
+ local uri=myApp:myPass@some-name-mongos.pitr-physical-backup-source-28579
+ local postfix=
+ local suffix=
+ local database=myApp
+ local collection=test
+ local port=27017
+ local tls=false
+ [[ false == \t\r\u\e ]]
+ mongos_command=run_mongos
+ log 'running db.test.command() in myApp'
+ set +o xtrace
[2025-12-24T13:18:41+0000] running db.test.command() in myApp
+ run_mongos 'use myApp\n db.test.find()' myApp:myPass@some-name-mongos.pitr-physical-backup-source-28579 mongodb '' '' 27017
+ local 'command=use myApp\n db.test.find()'
+ local uri=myApp:myPass@some-name-mongos.pitr-physical-backup-source-28579
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local port=27017
+ local mongo_bin=mongo
+ egrep -v 'I NETWORK|W NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|Started a new thread for the timer service'
+ /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
++ echo .svc.cluster.local
++ awk -F: '{print $2}'
egrep: warning: egrep is obsolescent; using grep -E
+ suffix_port=
+ [[ -z '' ]]
+ suffix=.svc.cluster.local:27017
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.baewDtJyxO
+++ mktemp
++ local LAST_ERR=/tmp/tmp.CQA7T1C7Gv
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.baewDtJyxO
++ cat /tmp/tmp.CQA7T1C7Gv
++ rm /tmp/tmp.baewDtJyxO /tmp/tmp.CQA7T1C7Gv
++ return 0
+ local client_container=psmdb-client-696897d69b-m5f2h
+ kubectl_bin exec psmdb-client-696897d69b-m5f2h -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.pitr-physical-backup-source-28579.svc.cluster.local:27017/admin '
++ mktemp
+ local LAST_OUT=/tmp/tmp.NLcU1ChzIa
++ mktemp
+ local LAST_ERR=/tmp/tmp.ZP7ZQB70vW
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-696897d69b-m5f2h -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.pitr-physical-backup-source-28579.svc.cluster.local:27017/admin '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.NLcU1ChzIa
+ cat /tmp/tmp.ZP7ZQB70vW
+ rm /tmp/tmp.NLcU1ChzIa /tmp/tmp.ZP7ZQB70vW
+ return 0
+ [[ 0 -eq 0 ]]
+ diff /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2151/e2e-tests/pitr-physical-backup-source/compare/find.json /tmp/tmp.e7bs9MurNC/find
+ desc 'delete PSMDB cluster some-name'
+ set +o xtrace


-----------------------------------------------------------------------------------
delete PSMDB cluster some-name
-----------------------------------------------------------------------------------

+ kubectl_bin delete psmdb some-name
++ mktemp
+ local LAST_OUT=/tmp/tmp.7j2LhDExoi
++ mktemp
+ local LAST_ERR=/tmp/tmp.VuD2zg0Tio
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl delete psmdb some-name
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.7j2LhDExoi
perconaservermongodb.psmdb.percona.com "some-name" deleted from pitr-physical-backup-source-28579 namespace
+ cat /tmp/tmp.VuD2zg0Tio
+ rm /tmp/tmp.7j2LhDExoi /tmp/tmp.VuD2zg0Tio
+ return 0
+ kubectl_bin delete pvc -l app.kubernetes.io/managed-by=percona-server-mongodb-operator
++ mktemp
+ local LAST_OUT=/tmp/tmp.IspmcAKs7x
++ mktemp
+ local LAST_ERR=/tmp/tmp.8ddSae9TiQ
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl delete pvc -l app.kubernetes.io/managed-by=percona-server-mongodb-operator
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.IspmcAKs7x
persistentvolumeclaim "mongod-data-some-name-cfg-0" deleted from pitr-physical-backup-source-28579 namespace
persistentvolumeclaim "mongod-data-some-name-cfg-1" deleted from pitr-physical-backup-source-28579 namespace
persistentvolumeclaim "mongod-data-some-name-cfg-2" deleted from pitr-physical-backup-source-28579 namespace
persistentvolumeclaim "mongod-data-some-name-rs0-0" deleted from pitr-physical-backup-source-28579 namespace
persistentvolumeclaim "mongod-data-some-name-rs0-1" deleted from pitr-physical-backup-source-28579 namespace
persistentvolumeclaim "mongod-data-some-name-rs0-2" deleted from pitr-physical-backup-source-28579 namespace
persistentvolumeclaim "mongod-data-some-name-rs1-0" deleted from pitr-physical-backup-source-28579 namespace
persistentvolumeclaim "mongod-data-some-name-rs1-1" deleted from pitr-physical-backup-source-28579 namespace
persistentvolumeclaim "mongod-data-some-name-rs1-2" deleted from pitr-physical-backup-source-28579 namespace
persistentvolumeclaim "mongod-data-some-name-rs2-0" deleted from pitr-physical-backup-source-28579 namespace
persistentvolumeclaim "mongod-data-some-name-rs2-1" deleted from pitr-physical-backup-source-28579 namespace
persistentvolumeclaim "mongod-data-some-name-rs2-2" deleted from pitr-physical-backup-source-28579 namespace
+ cat /tmp/tmp.8ddSae9TiQ
+ rm /tmp/tmp.IspmcAKs7x /tmp/tmp.8ddSae9TiQ
+ return 0
+ sleep 10
+ desc 'recreate PSMDB cluster some-name'
+ set +o xtrace


-----------------------------------------------------------------------------------
recreate PSMDB cluster some-name
-----------------------------------------------------------------------------------

+ desc 'create second PSMDB cluster some-name'
+ set +o xtrace


-----------------------------------------------------------------------------------
create second PSMDB cluster some-name
-----------------------------------------------------------------------------------

+ apply_cluster /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2151/e2e-tests/pitr-physical-backup-source/conf/some-name-rs0.yml
+ '[' -z '' ']'
+ cat_config /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2151/e2e-tests/pitr-physical-backup-source/conf/some-name-rs0.yml
+ kubectl_bin apply -f -
+ cat /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2151/e2e-tests/pitr-physical-backup-source/conf/some-name-rs0.yml
+ yq eval '(.spec | select(.image == null)).image = "perconalab/percona-server-mongodb-operator:main-mongod8.0"'
+ yq eval '(.spec | select(has("pmm"))).pmm.image = "percona/pmm-client:2.44.1-1"'
++ mktemp
+ yq eval '(.spec | select(has("initImage"))).initImage = "perconalab/percona-server-mongodb-operator:PR-2151-37812f21"'
+ yq eval '(.spec | select(has("backup"))).backup.image = "perconalab/percona-server-mongodb-operator:main-backup"'
+ /usr/sbin/sed -e s/NAME_SPACE/pitr-physical-backup-source-28579/g
+ yq eval '.spec.upgradeOptions.apply="Never"'
+ local LAST_OUT=/tmp/tmp.1smMLzfKv9
++ mktemp
+ local LAST_ERR=/tmp/tmp.csZz0KXTM3
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl apply -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.1smMLzfKv9
perconaservermongodb.psmdb.percona.com/some-name created
+ cat /tmp/tmp.csZz0KXTM3
+ rm /tmp/tmp.1smMLzfKv9 /tmp/tmp.csZz0KXTM3
+ return 0
+ desc 'check if all 3 Pods started'
+ set +o xtrace


-----------------------------------------------------------------------------------
check if all 3 Pods started
-----------------------------------------------------------------------------------

+ wait_for_running some-name-rs0 3
+ local name=some-name-rs0
+ let last_pod=2
+ local check_cluster_readyness=true
+ set_debug
+ [[ 1 == 1 ]]
+ set -o xtrace
+ local rs_name=rs0
+ local cluster_name=some-name
++ seq 0 2
+ for i in $(seq 0 $last_pod)
+ [[ 0 -eq 2 ]]
+ wait_pod some-name-rs0-0
+ local pod=some-name-rs0-0
+ set +o xtrace
waiting for pod/some-name-rs0-0 to be ready.....OK
+ for i in $(seq 0 $last_pod)
+ [[ 1 -eq 2 ]]
+ wait_pod some-name-rs0-1
+ local pod=some-name-rs0-1
+ set +o xtrace
waiting for pod/some-name-rs0-1 to be ready.....OK
+ for i in $(seq 0 $last_pod)
+ [[ 2 -eq 2 ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].arbiter.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.qLp1e3duvM
+++ mktemp
++ local LAST_ERR=/tmp/tmp.A5yXWqUr40
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].arbiter.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.qLp1e3duvM
++ cat /tmp/tmp.A5yXWqUr40
++ rm /tmp/tmp.qLp1e3duvM /tmp/tmp.A5yXWqUr40
++ return 0
+ [[ '' == \t\r\u\e ]]
+ wait_pod some-name-rs0-2
+ local pod=some-name-rs0-2
+ set +o xtrace
waiting for pod/some-name-rs0-2 to be ready.....OK
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].nonvoting.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.E28ugtYxa7
+++ mktemp
++ local LAST_ERR=/tmp/tmp.HsMfJyIsLv
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].nonvoting.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.E28ugtYxa7
++ cat /tmp/tmp.HsMfJyIsLv
++ rm /tmp/tmp.E28ugtYxa7 /tmp/tmp.HsMfJyIsLv
++ return 0
+ [[ '' == \t\r\u\e ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].hidden.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.FgVK0jL3U3
+++ mktemp
++ local LAST_ERR=/tmp/tmp.qPIsOfNTbC
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].hidden.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.FgVK0jL3U3
++ cat /tmp/tmp.qPIsOfNTbC
++ rm /tmp/tmp.FgVK0jL3U3 /tmp/tmp.qPIsOfNTbC
++ return 0
+ [[ '' == \t\r\u\e ]]
+ sleep 10
+ [[ true == \t\r\u\e ]]
+ set +x
Waiting for cluster readyness.................................................................
+ wait_for_running some-name-cfg 3 false
+ local name=some-name-cfg
+ let last_pod=2
+ local check_cluster_readyness=false
+ set_debug
+ [[ 1 == 1 ]]
+ set -o xtrace
+ local rs_name=cfg
+ local cluster_name=some-name
++ seq 0 2
+ for i in $(seq 0 $last_pod)
+ [[ 0 -eq 2 ]]
+ wait_pod some-name-cfg-0
+ local pod=some-name-cfg-0
+ set +o xtrace
waiting for pod/some-name-cfg-0 to be ready.OK
+ for i in $(seq 0 $last_pod)
+ [[ 1 -eq 2 ]]
+ wait_pod some-name-cfg-1
+ local pod=some-name-cfg-1
+ set +o xtrace
waiting for pod/some-name-cfg-1 to be ready.OK
+ for i in $(seq 0 $last_pod)
+ [[ 2 -eq 2 ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].arbiter.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.QTym2fZZ26
+++ mktemp
++ local LAST_ERR=/tmp/tmp.LlLElml5p9
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].arbiter.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.QTym2fZZ26
++ cat /tmp/tmp.LlLElml5p9
++ rm /tmp/tmp.QTym2fZZ26 /tmp/tmp.LlLElml5p9
++ return 0
+ [[ '' == \t\r\u\e ]]
+ wait_pod some-name-cfg-2
+ local pod=some-name-cfg-2
+ set +o xtrace
waiting for pod/some-name-cfg-2 to be ready.OK
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].nonvoting.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.jDjFhzHu0d
+++ mktemp
++ local LAST_ERR=/tmp/tmp.W2pOZ1Jpqh
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].nonvoting.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.jDjFhzHu0d
++ cat /tmp/tmp.W2pOZ1Jpqh
++ rm /tmp/tmp.jDjFhzHu0d /tmp/tmp.W2pOZ1Jpqh
++ return 0
+ [[ '' == \t\r\u\e ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].hidden.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.7lXRIJeBgm
+++ mktemp
++ local LAST_ERR=/tmp/tmp.A9OMLLHB8t
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].hidden.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.7lXRIJeBgm
++ cat /tmp/tmp.A9OMLLHB8t
++ rm /tmp/tmp.7lXRIJeBgm /tmp/tmp.A9OMLLHB8t
++ return 0
+ [[ '' == \t\r\u\e ]]
+ sleep 10
+ [[ false == \t\r\u\e ]]
+ wait_for_running some-name-rs1 3
+ local name=some-name-rs1
+ let last_pod=2
+ local check_cluster_readyness=true
+ set_debug
+ [[ 1 == 1 ]]
+ set -o xtrace
+ local rs_name=rs1
+ local cluster_name=some-name
++ seq 0 2
+ for i in $(seq 0 $last_pod)
+ [[ 0 -eq 2 ]]
+ wait_pod some-name-rs1-0
+ local pod=some-name-rs1-0
+ set +o xtrace
waiting for pod/some-name-rs1-0 to be ready.OK
+ for i in $(seq 0 $last_pod)
+ [[ 1 -eq 2 ]]
+ wait_pod some-name-rs1-1
+ local pod=some-name-rs1-1
+ set +o xtrace
waiting for pod/some-name-rs1-1 to be ready.OK
+ for i in $(seq 0 $last_pod)
+ [[ 2 -eq 2 ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs1")].arbiter.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.5HXugeOW5Y
+++ mktemp
++ local LAST_ERR=/tmp/tmp.dKfoUgmJgn
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs1")].arbiter.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.5HXugeOW5Y
++ cat /tmp/tmp.dKfoUgmJgn
++ rm /tmp/tmp.5HXugeOW5Y /tmp/tmp.dKfoUgmJgn
++ return 0
+ [[ '' == \t\r\u\e ]]
+ wait_pod some-name-rs1-2
+ local pod=some-name-rs1-2
+ set +o xtrace
waiting for pod/some-name-rs1-2 to be ready.OK
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs1")].nonvoting.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.ZSDkI81H2Z
+++ mktemp
++ local LAST_ERR=/tmp/tmp.tzEQMorMBX
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs1")].nonvoting.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.ZSDkI81H2Z
++ cat /tmp/tmp.tzEQMorMBX
++ rm /tmp/tmp.ZSDkI81H2Z /tmp/tmp.tzEQMorMBX
++ return 0
+ [[ '' == \t\r\u\e ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs1")].hidden.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.ZKmIKBnW0r
+++ mktemp
++ local LAST_ERR=/tmp/tmp.uY3s073TOb
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs1")].hidden.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.ZKmIKBnW0r
++ cat /tmp/tmp.uY3s073TOb
++ rm /tmp/tmp.ZKmIKBnW0r /tmp/tmp.uY3s073TOb
++ return 0
+ [[ '' == \t\r\u\e ]]
+ sleep 10
+ [[ true == \t\r\u\e ]]
+ set +x
Waiting for cluster readyness
+ wait_for_running some-name-rs2 3
+ local name=some-name-rs2
+ let last_pod=2
+ local check_cluster_readyness=true
+ set_debug
+ [[ 1 == 1 ]]
+ set -o xtrace
+ local rs_name=rs2
+ local cluster_name=some-name
++ seq 0 2
+ for i in $(seq 0 $last_pod)
+ [[ 0 -eq 2 ]]
+ wait_pod some-name-rs2-0
+ local pod=some-name-rs2-0
+ set +o xtrace
waiting for pod/some-name-rs2-0 to be ready.OK
+ for i in $(seq 0 $last_pod)
+ [[ 1 -eq 2 ]]
+ wait_pod some-name-rs2-1
+ local pod=some-name-rs2-1
+ set +o xtrace
waiting for pod/some-name-rs2-1 to be ready.OK
+ for i in $(seq 0 $last_pod)
+ [[ 2 -eq 2 ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs2")].arbiter.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.BKZiBavosb
+++ mktemp
++ local LAST_ERR=/tmp/tmp.WrrIYobHBq
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs2")].arbiter.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.BKZiBavosb
++ cat /tmp/tmp.WrrIYobHBq
++ rm /tmp/tmp.BKZiBavosb /tmp/tmp.WrrIYobHBq
++ return 0
+ [[ '' == \t\r\u\e ]]
+ wait_pod some-name-rs2-2
+ local pod=some-name-rs2-2
+ set +o xtrace
waiting for pod/some-name-rs2-2 to be ready.OK
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs2")].nonvoting.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.RrR85DrbCD
+++ mktemp
++ local LAST_ERR=/tmp/tmp.Lh7sCrUqJu
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs2")].nonvoting.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.RrR85DrbCD
++ cat /tmp/tmp.Lh7sCrUqJu
++ rm /tmp/tmp.RrR85DrbCD /tmp/tmp.Lh7sCrUqJu
++ return 0
+ [[ '' == \t\r\u\e ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs2")].hidden.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.01rG4weYqW
+++ mktemp
++ local LAST_ERR=/tmp/tmp.ZzVfaUMW0d
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs2")].hidden.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.01rG4weYqW
++ cat /tmp/tmp.ZzVfaUMW0d
++ rm /tmp/tmp.01rG4weYqW /tmp/tmp.ZzVfaUMW0d
++ return 0
+ [[ '' == \t\r\u\e ]]
+ sleep 10
+ [[ true == \t\r\u\e ]]
+ set +x
Waiting for cluster readyness
+ sleep 10
+ write_initial_data
+ desc 'create user myApp'
+ set +o xtrace


-----------------------------------------------------------------------------------
create user myApp
-----------------------------------------------------------------------------------

+ run_mongos 'db.createUser({user:"myApp",pwd:"myPass",roles:[{db:"myApp",role:"readWrite"}]})' userAdmin:userAdmin123456@some-name-mongos.pitr-physical-backup-source-28579
+ local 'command=db.createUser({user:"myApp",pwd:"myPass",roles:[{db:"myApp",role:"readWrite"}]})'
+ local uri=userAdmin:userAdmin123456@some-name-mongos.pitr-physical-backup-source-28579
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local port=27017
+ local mongo_bin=mongo
++ echo .svc.cluster.local
++ awk -F: '{print $2}'
+ suffix_port=
+ [[ -z '' ]]
+ suffix=.svc.cluster.local:27017
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.qn1vVvE1Fc
+++ mktemp
++ local LAST_ERR=/tmp/tmp.WrdsW0J2je
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.qn1vVvE1Fc
++ cat /tmp/tmp.WrdsW0J2je
++ rm /tmp/tmp.qn1vVvE1Fc /tmp/tmp.WrdsW0J2je
++ return 0
+ local client_container=psmdb-client-696897d69b-m5f2h
+ kubectl_bin exec psmdb-client-696897d69b-m5f2h -- bash -c 'printf '\''db.createUser({user:"myApp",pwd:"myPass",roles:[{db:"myApp",role:"readWrite"}]})\n'\'' | mongo mongodb://userAdmin:userAdmin123456@some-name-mongos.pitr-physical-backup-source-28579.svc.cluster.local:27017/admin '
++ mktemp
+ local LAST_OUT=/tmp/tmp.FWRpMOMKlZ
++ mktemp
+ local LAST_ERR=/tmp/tmp.SlgUNPDOXy
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-696897d69b-m5f2h -- bash -c 'printf '\''db.createUser({user:"myApp",pwd:"myPass",roles:[{db:"myApp",role:"readWrite"}]})\n'\'' | mongo mongodb://userAdmin:userAdmin123456@some-name-mongos.pitr-physical-backup-source-28579.svc.cluster.local:27017/admin '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.FWRpMOMKlZ
Percona Server for MongoDB shell version v4.4.29-28
connecting to: mongodb://some-name-mongos.pitr-physical-backup-source-28579.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb
Implicit session: session { "id" : UUID("22cafa70-6972-4738-adfd-f715bec6ba9f") }
Percona Server for MongoDB server version: v8.0.16-5
WARNING: shell and server versions do not match
Successfully added user: {
	"user" : "myApp",
	"roles" : [
		{
			"db" : "myApp",
			"role" : "readWrite"
		}
	]
}
bye
+ cat /tmp/tmp.SlgUNPDOXy
+ rm /tmp/tmp.FWRpMOMKlZ /tmp/tmp.SlgUNPDOXy
+ return 0
+ sleep 2
+ write_document
+ local cmp_postfix=
+ local sleep_value=0
+ log 'write initial data, read from all'
+ set +o xtrace
[2025-12-24T13:23:39+0000] write initial data, read from all
+ run_mongos 'use myApp\n db.test.insert({ x: 100500 })' myApp:myPass@some-name-mongos.pitr-physical-backup-source-28579
+ local 'command=use myApp\n db.test.insert({ x: 100500 })'
+ local uri=myApp:myPass@some-name-mongos.pitr-physical-backup-source-28579
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local port=27017
+ local mongo_bin=mongo
++ echo .svc.cluster.local
++ awk -F: '{print $2}'
+ suffix_port=
+ [[ -z '' ]]
+ suffix=.svc.cluster.local:27017
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.aSbhjgYfVz
+++ mktemp
++ local LAST_ERR=/tmp/tmp.A9LLFEx9xE
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.aSbhjgYfVz
++ cat /tmp/tmp.A9LLFEx9xE
++ rm /tmp/tmp.aSbhjgYfVz /tmp/tmp.A9LLFEx9xE
++ return 0
+ local client_container=psmdb-client-696897d69b-m5f2h
+ kubectl_bin exec psmdb-client-696897d69b-m5f2h -- bash -c 'printf '\''use myApp\n db.test.insert({ x: 100500 })\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.pitr-physical-backup-source-28579.svc.cluster.local:27017/admin '
++ mktemp
+ local LAST_OUT=/tmp/tmp.Af96WYgpnL
++ mktemp
+ local LAST_ERR=/tmp/tmp.NnaBow7vcm
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-696897d69b-m5f2h -- bash -c 'printf '\''use myApp\n db.test.insert({ x: 100500 })\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.pitr-physical-backup-source-28579.svc.cluster.local:27017/admin '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.Af96WYgpnL
Percona Server for MongoDB shell version v4.4.29-28
connecting to: mongodb://some-name-mongos.pitr-physical-backup-source-28579.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb
Implicit session: session { "id" : UUID("7d4d81b3-b840-4740-ae95-0e9dd531498b") }
Percona Server for MongoDB server version: v8.0.16-5
WARNING: shell and server versions do not match
switched to db myApp
WriteResult({ "nInserted" : 1 })
bye
+ cat /tmp/tmp.NnaBow7vcm
+ rm /tmp/tmp.Af96WYgpnL /tmp/tmp.NnaBow7vcm
+ return 0
+ sleep 0
+ compare_mongos_cmd find myApp:myPass@some-name-mongos.pitr-physical-backup-source-28579
+ local command=find
+ local uri=myApp:myPass@some-name-mongos.pitr-physical-backup-source-28579
+ local postfix=
+ local suffix=
+ local database=myApp
+ local collection=test
+ local port=27017
+ local tls=false
+ [[ false == \t\r\u\e ]]
+ mongos_command=run_mongos
+ log 'running db.test.command() in myApp'
+ set +o xtrace
[2025-12-24T13:23:41+0000] running db.test.command() in myApp
+ egrep -v 'I NETWORK|W NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|Started a new thread for the timer service'
+ run_mongos 'use myApp\n db.test.find()' myApp:myPass@some-name-mongos.pitr-physical-backup-source-28579 mongodb '' '' 27017
+ /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
+ local 'command=use myApp\n db.test.find()'
+ local uri=myApp:myPass@some-name-mongos.pitr-physical-backup-source-28579
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local port=27017
+ local mongo_bin=mongo
++ echo .svc.cluster.local
++ awk -F: '{print $2}'
egrep: warning: egrep is obsolescent; using grep -E
+ suffix_port=
+ [[ -z '' ]]
+ suffix=.svc.cluster.local:27017
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.kUrcc1lb3X
+++ mktemp
++ local LAST_ERR=/tmp/tmp.yX9oDYlvQP
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.kUrcc1lb3X
++ cat /tmp/tmp.yX9oDYlvQP
++ rm /tmp/tmp.kUrcc1lb3X /tmp/tmp.yX9oDYlvQP
++ return 0
+ local client_container=psmdb-client-696897d69b-m5f2h
+ kubectl_bin exec psmdb-client-696897d69b-m5f2h -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.pitr-physical-backup-source-28579.svc.cluster.local:27017/admin '
++ mktemp
+ local LAST_OUT=/tmp/tmp.WomospiQth
++ mktemp
+ local LAST_ERR=/tmp/tmp.K5ZDEycohd
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-696897d69b-m5f2h -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.pitr-physical-backup-source-28579.svc.cluster.local:27017/admin '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.WomospiQth
+ cat /tmp/tmp.K5ZDEycohd
+ rm /tmp/tmp.WomospiQth /tmp/tmp.K5ZDEycohd
+ return 0
+ [[ 0 -eq 0 ]]
+ diff /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2151/e2e-tests/pitr-physical-backup-source/compare/find.json /tmp/tmp.e7bs9MurNC/find
+ desc 'restore pitr type latest using backupSource'
+ set +o xtrace


-----------------------------------------------------------------------------------
restore pitr type latest using backupSource
-----------------------------------------------------------------------------------

+ write_document -2nd
+ local cmp_postfix=-2nd
+ local sleep_value=0
+ log 'write initial data, read from all'
+ set +o xtrace
[2025-12-24T13:23:43+0000] write initial data, read from all
+ run_mongos 'use myApp\n db.test.insert({ x: 100500 })' myApp:myPass@some-name-mongos.pitr-physical-backup-source-28579
+ local 'command=use myApp\n db.test.insert({ x: 100500 })'
+ local uri=myApp:myPass@some-name-mongos.pitr-physical-backup-source-28579
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local port=27017
+ local mongo_bin=mongo
++ echo .svc.cluster.local
++ awk -F: '{print $2}'
+ suffix_port=
+ [[ -z '' ]]
+ suffix=.svc.cluster.local:27017
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.yClVojYnzq
+++ mktemp
++ local LAST_ERR=/tmp/tmp.LNyWS5B9JI
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.yClVojYnzq
++ cat /tmp/tmp.LNyWS5B9JI
++ rm /tmp/tmp.yClVojYnzq /tmp/tmp.LNyWS5B9JI
++ return 0
+ local client_container=psmdb-client-696897d69b-m5f2h
+ kubectl_bin exec psmdb-client-696897d69b-m5f2h -- bash -c 'printf '\''use myApp\n db.test.insert({ x: 100500 })\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.pitr-physical-backup-source-28579.svc.cluster.local:27017/admin '
++ mktemp
+ local LAST_OUT=/tmp/tmp.ew50FRSg17
++ mktemp
+ local LAST_ERR=/tmp/tmp.RySPXsz2aJ
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-696897d69b-m5f2h -- bash -c 'printf '\''use myApp\n db.test.insert({ x: 100500 })\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.pitr-physical-backup-source-28579.svc.cluster.local:27017/admin '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.ew50FRSg17
Percona Server for MongoDB shell version v4.4.29-28
connecting to: mongodb://some-name-mongos.pitr-physical-backup-source-28579.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb
Implicit session: session { "id" : UUID("ac399630-5785-4a26-b7cc-55b0aaf58fa1") }
Percona Server for MongoDB server version: v8.0.16-5
WARNING: shell and server versions do not match
switched to db myApp
WriteResult({ "nInserted" : 1 })
bye
+ cat /tmp/tmp.RySPXsz2aJ
+ rm /tmp/tmp.ew50FRSg17 /tmp/tmp.RySPXsz2aJ
+ return 0
+ sleep 0
+ compare_mongos_cmd find myApp:myPass@some-name-mongos.pitr-physical-backup-source-28579 -2nd
+ local command=find
+ local uri=myApp:myPass@some-name-mongos.pitr-physical-backup-source-28579
+ local postfix=-2nd
+ local suffix=
+ local database=myApp
+ local collection=test
+ local port=27017
+ local tls=false
+ [[ false == \t\r\u\e ]]
+ mongos_command=run_mongos
+ log 'running db.test.command() in myApp'
+ set +o xtrace
[2025-12-24T13:23:46+0000] running db.test.command() in myApp
+ run_mongos 'use myApp\n db.test.find()' myApp:myPass@some-name-mongos.pitr-physical-backup-source-28579 mongodb '' '' 27017
+ local 'command=use myApp\n db.test.find()'
+ local uri=myApp:myPass@some-name-mongos.pitr-physical-backup-source-28579
+ /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local port=27017
+ local mongo_bin=mongo
+ egrep -v 'I NETWORK|W NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|Started a new thread for the timer service'
++ echo .svc.cluster.local
++ awk -F: '{print $2}'
egrep: warning: egrep is obsolescent; using grep -E
+ suffix_port=
+ [[ -z '' ]]
+ suffix=.svc.cluster.local:27017
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.tfXuOsJwmB
+++ mktemp
++ local LAST_ERR=/tmp/tmp.LHYF9cYApY
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.tfXuOsJwmB
++ cat /tmp/tmp.LHYF9cYApY
++ rm /tmp/tmp.tfXuOsJwmB /tmp/tmp.LHYF9cYApY
++ return 0
+ local client_container=psmdb-client-696897d69b-m5f2h
+ kubectl_bin exec psmdb-client-696897d69b-m5f2h -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.pitr-physical-backup-source-28579.svc.cluster.local:27017/admin '
++ mktemp
+ local LAST_OUT=/tmp/tmp.43G2YLbt98
++ mktemp
+ local LAST_ERR=/tmp/tmp.b1xRXkw5ub
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-696897d69b-m5f2h -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.pitr-physical-backup-source-28579.svc.cluster.local:27017/admin '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.43G2YLbt98
+ cat /tmp/tmp.b1xRXkw5ub
+ rm /tmp/tmp.43G2YLbt98 /tmp/tmp.b1xRXkw5ub
+ return 0
+ [[ 0 -eq 0 ]]
+ diff /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2151/e2e-tests/pitr-physical-backup-source/compare/find-2nd.json /tmp/tmp.e7bs9MurNC/find-2nd
+ run_backup backup-minio 2 physical
+ local name=backup-minio
+ local idx=2
+ local type=physical
+ desc 'run backup backup-minio-2'
+ set +o xtrace


-----------------------------------------------------------------------------------
run backup backup-minio-2
-----------------------------------------------------------------------------------

+ cat /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2151/e2e-tests/pitr-physical-backup-source/conf/backup-minio.yml
+ /usr/sbin/sed -e 's/name:/name: backup-minio-2/'
+ /usr/sbin/sed -e 's/type:/type: physical/'
+ kubectl_bin apply -f -
++ mktemp
+ local LAST_OUT=/tmp/tmp.BSTCxzHHIS
++ mktemp
+ local LAST_ERR=/tmp/tmp.qPO8CmZRaA
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl apply -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.BSTCxzHHIS
perconaservermongodbbackup.psmdb.percona.com/backup-minio-2 created
+ cat /tmp/tmp.qPO8CmZRaA
+ rm /tmp/tmp.BSTCxzHHIS /tmp/tmp.qPO8CmZRaA
+ return 0
+ wait_backup backup-minio-2
+ local backup_name=backup-minio-2
+ local target_state=ready
+ set +o xtrace
waiting for backup-minio-2 to reach ready state.........OK
+ compare_latest_restorable_time some-name-rs0 backup-minio-2
+ local cluster=some-name-rs0
+ local backup_name=backup-minio-2
+ local latest_restorable_time
+ local backup_time
++ get_latest_restorable_time some-name-rs0
++ local cluster=some-name-rs0
++ local first_timestamp
++ local second_timestamp
++ local retry=0
++ [[ '' != '' ]]
+++ kubectl_bin exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ jq '.backups.pitrChunks.pitrChunks | last | .range.end'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.7D0W95pKEv
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.0cfw2qUvFy
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in $(seq 0 2)
+++ set +e
+++ kubectl exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.7D0W95pKEv
+++ cat /tmp/tmp.0cfw2qUvFy
+++ rm /tmp/tmp.7D0W95pKEv /tmp/tmp.0cfw2qUvFy
+++ return 0
++ first_timestamp=1766581517
++ sleep 5
++ [[ 1766581517 != '' ]]
++ [[ 1766581517 != \n\u\l\l ]]
+++ kubectl_bin exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ jq '.backups.pitrChunks.pitrChunks | last | .range.end'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.Ru2Kv4Mq8r
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.5YsmcLgJn0
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in $(seq 0 2)
+++ set +e
+++ kubectl exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.Ru2Kv4Mq8r
+++ cat /tmp/tmp.5YsmcLgJn0
+++ rm /tmp/tmp.Ru2Kv4Mq8r /tmp/tmp.5YsmcLgJn0
+++ return 0
++ second_timestamp=1766581517
++ let retry+=1
++ [[ 1 -gt 30 ]]
++ [[ 1766581517 != '' ]]
++ [[ 1766581517 != \n\u\l\l ]]
++ [[ 1766581517 == 1766581517 ]]
++ /usr/sbin/date -u -d @1766581517 +%Y-%m-%dT%H:%M:%SZ
+ latest_restorable_time=2025-12-24T13:05:17Z
++ get_latest_restorable_time_from_backup_object backup-minio-2
++ local backup_name=backup-minio-2
++ local latestRestorableTime
++ local retry=0
++ [[ '' != '' ]]
++ sleep 5
+++ kubectl_bin get psmdb-backup backup-minio-2 -o 'jsonpath={.status.latestRestorableTime}'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.7QOjcw4xjf
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.iRYOzLvd0H
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in $(seq 0 2)
+++ set +e
+++ kubectl get psmdb-backup backup-minio-2 -o 'jsonpath={.status.latestRestorableTime}'
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.7QOjcw4xjf
+++ cat /tmp/tmp.iRYOzLvd0H
+++ rm /tmp/tmp.7QOjcw4xjf /tmp/tmp.iRYOzLvd0H
+++ return 0
++ latestRestorableTime=2025-12-24T13:05:17Z
++ let retry+=1
++ [[ 1 -gt 30 ]]
++ [[ 2025-12-24T13:05:17Z != '' ]]
++ [[ 2025-12-24T13:05:17Z != \n\u\l\l ]]
++ echo 2025-12-24T13:05:17Z
+ backup_time=2025-12-24T13:05:17Z
+ [[ 2025-12-24T13:05:17Z != \2\0\2\5\-\1\2\-\2\4\T\1\3\:\0\5\:\1\7\Z ]]
+ check_recovery backup-minio-2 latest '' -3rd some-name backupSource
+ local backup_name=backup-minio-2
+ local restore_type=latest
+ local restore_date=
+ local cmp_postfix=-3rd
+ local cluster_name=some-name
+ local backupSource=backupSource
++ get_latest_oplog_chunk_ts some-name
++ local cluster=some-name
+++ kubectl_bin exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ jq '.backups.pitrChunks.pitrChunks | last | .range.end'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.S05dEqoSDt
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.69fbz5eoZN
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in $(seq 0 2)
+++ set +e
+++ kubectl exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.S05dEqoSDt
+++ cat /tmp/tmp.69fbz5eoZN
+++ rm /tmp/tmp.S05dEqoSDt /tmp/tmp.69fbz5eoZN
+++ return 0
++ echo 1766581517
+ local latest_ts=1766581517
+ desc 'write more data before restore by latest'
+ set +o xtrace


-----------------------------------------------------------------------------------
write more data before restore by latest
-----------------------------------------------------------------------------------

+ run_mongos 'use myApp\n db.test.insert({ x: 100501 })' myApp:myPass@some-name-mongos.pitr-physical-backup-source-28579
+ local 'command=use myApp\n db.test.insert({ x: 100501 })'
+ local uri=myApp:myPass@some-name-mongos.pitr-physical-backup-source-28579
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local port=27017
+ local mongo_bin=mongo
++ echo .svc.cluster.local
++ awk -F: '{print $2}'
+ suffix_port=
+ [[ -z '' ]]
+ suffix=.svc.cluster.local:27017
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.d8zxtHTXLe
+++ mktemp
++ local LAST_ERR=/tmp/tmp.uIWwwGjFAk
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.d8zxtHTXLe
++ cat /tmp/tmp.uIWwwGjFAk
++ rm /tmp/tmp.d8zxtHTXLe /tmp/tmp.uIWwwGjFAk
++ return 0
+ local client_container=psmdb-client-696897d69b-m5f2h
+ kubectl_bin exec psmdb-client-696897d69b-m5f2h -- bash -c 'printf '\''use myApp\n db.test.insert({ x: 100501 })\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.pitr-physical-backup-source-28579.svc.cluster.local:27017/admin '
++ mktemp
+ local LAST_OUT=/tmp/tmp.fIBWPK9hA8
++ mktemp
+ local LAST_ERR=/tmp/tmp.1WTx9RyJHY
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-696897d69b-m5f2h -- bash -c 'printf '\''use myApp\n db.test.insert({ x: 100501 })\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.pitr-physical-backup-source-28579.svc.cluster.local:27017/admin '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.fIBWPK9hA8
Percona Server for MongoDB shell version v4.4.29-28
connecting to: mongodb://some-name-mongos.pitr-physical-backup-source-28579.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb
Implicit session: session { "id" : UUID("3a9dd20d-3521-429e-b55f-052fab9c1e03") }
Percona Server for MongoDB server version: v8.0.16-5
WARNING: shell and server versions do not match
switched to db myApp
WriteResult({ "nInserted" : 1 })
bye
+ cat /tmp/tmp.1WTx9RyJHY
+ rm /tmp/tmp.fIBWPK9hA8 /tmp/tmp.1WTx9RyJHY
+ return 0
+ [[ -n '' ]]
+ desc 'Restoring to latest'
+ set +o xtrace


-----------------------------------------------------------------------------------
Restoring to latest
-----------------------------------------------------------------------------------

++ get_latest_oplog_chunk_ts some-name
++ local cluster=some-name
+++ kubectl_bin exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ jq '.backups.pitrChunks.pitrChunks | last | .range.end'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.iPxotM69Si
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.J6u2r1PjzS
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in $(seq 0 2)
+++ set +e
+++ kubectl exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.iPxotM69Si
+++ cat /tmp/tmp.J6u2r1PjzS
+++ rm /tmp/tmp.iPxotM69Si /tmp/tmp.J6u2r1PjzS
+++ return 0
++ echo 1766581517
+ local current_ts=1766581517
+ retries=0
+ [[ 1766581517 -gt 1766581517 ]]
+ [[ 0 -gt 30 ]]
++ get_latest_oplog_chunk_ts some-name
++ local cluster=some-name
+++ kubectl_bin exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ jq '.backups.pitrChunks.pitrChunks | last | .range.end'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.87K3QCniki
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.Pxgr43gvAw
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in $(seq 0 2)
+++ set +e
+++ kubectl exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.87K3QCniki
+++ cat /tmp/tmp.Pxgr43gvAw
+++ rm /tmp/tmp.87K3QCniki /tmp/tmp.Pxgr43gvAw
+++ return 0
++ echo 1766581517
+ latest_ts=1766581517
+ retries=1
++ format_date 1766581517
++ local timestamp=1766581517
+++ TZ=UTC
+++ /usr/sbin/date -d@1766581517 '+%Y-%m-%d %H:%M:%S'
++ echo 2025-12-24 13:05:17
++ format_date 1766581517
++ local timestamp=1766581517
+++ TZ=UTC
+++ /usr/sbin/date -d@1766581517 '+%Y-%m-%d %H:%M:%S'
++ echo 2025-12-24 13:05:17
+ echo 'Waiting for last oplog chunk (2025-12-24 13:05:17) to be 120 seconds older than starting chunk (2025-12-24 13:05:17)'
Waiting for last oplog chunk (2025-12-24 13:05:17) to be 120 seconds older than starting chunk (2025-12-24 13:05:17)
+ sleep 10
+ [[ 1766581517 -gt 1766581517 ]]
+ [[ 1 -gt 30 ]]
++ get_latest_oplog_chunk_ts some-name
++ local cluster=some-name
+++ kubectl_bin exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ jq '.backups.pitrChunks.pitrChunks | last | .range.end'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.mNC7VmKKY1
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.Q5imzbBsyp
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in $(seq 0 2)
+++ set +e
+++ kubectl exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.mNC7VmKKY1
+++ cat /tmp/tmp.Q5imzbBsyp
+++ rm /tmp/tmp.mNC7VmKKY1 /tmp/tmp.Q5imzbBsyp
+++ return 0
++ echo 1766581517
+ latest_ts=1766581517
+ retries=2
++ format_date 1766581517
++ local timestamp=1766581517
+++ TZ=UTC
+++ /usr/sbin/date -d@1766581517 '+%Y-%m-%d %H:%M:%S'
++ echo 2025-12-24 13:05:17
++ format_date 1766581517
++ local timestamp=1766581517
+++ TZ=UTC
+++ /usr/sbin/date -d@1766581517 '+%Y-%m-%d %H:%M:%S'
++ echo 2025-12-24 13:05:17
+ echo 'Waiting for last oplog chunk (2025-12-24 13:05:17) to be 120 seconds older than starting chunk (2025-12-24 13:05:17)'
Waiting for last oplog chunk (2025-12-24 13:05:17) to be 120 seconds older than starting chunk (2025-12-24 13:05:17)
+ sleep 10
+ [[ 1766581517 -gt 1766581517 ]]
+ [[ 2 -gt 30 ]]
++ get_latest_oplog_chunk_ts some-name
++ local cluster=some-name
+++ kubectl_bin exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ jq '.backups.pitrChunks.pitrChunks | last | .range.end'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.PW5RR85ZKI
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.Qac8pgXxfF
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in $(seq 0 2)
+++ set +e
+++ kubectl exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.PW5RR85ZKI
+++ cat /tmp/tmp.Qac8pgXxfF
+++ rm /tmp/tmp.PW5RR85ZKI /tmp/tmp.Qac8pgXxfF
+++ return 0
++ echo 1766581517
+ latest_ts=1766581517
+ retries=3
++ format_date 1766581517
++ local timestamp=1766581517
+++ TZ=UTC
+++ /usr/sbin/date -d@1766581517 '+%Y-%m-%d %H:%M:%S'
++ echo 2025-12-24 13:05:17
++ format_date 1766581517
++ local timestamp=1766581517
+++ TZ=UTC
+++ /usr/sbin/date -d@1766581517 '+%Y-%m-%d %H:%M:%S'
++ echo 2025-12-24 13:05:17
+ echo 'Waiting for last oplog chunk (2025-12-24 13:05:17) to be 120 seconds older than starting chunk (2025-12-24 13:05:17)'
Waiting for last oplog chunk (2025-12-24 13:05:17) to be 120 seconds older than starting chunk (2025-12-24 13:05:17)
+ sleep 10
+ [[ 1766581517 -gt 1766581517 ]]
+ [[ 3 -gt 30 ]]
++ get_latest_oplog_chunk_ts some-name
++ local cluster=some-name
+++ kubectl_bin exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ jq '.backups.pitrChunks.pitrChunks | last | .range.end'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.FZHaIjMYm0
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.q57kRg2juM
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in $(seq 0 2)
+++ set +e
+++ kubectl exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.FZHaIjMYm0
+++ cat /tmp/tmp.q57kRg2juM
+++ rm /tmp/tmp.FZHaIjMYm0 /tmp/tmp.q57kRg2juM
+++ return 0
++ echo 1766581517
+ latest_ts=1766581517
+ retries=4
++ format_date 1766581517
++ local timestamp=1766581517
+++ TZ=UTC
+++ /usr/sbin/date -d@1766581517 '+%Y-%m-%d %H:%M:%S'
++ echo 2025-12-24 13:05:17
++ format_date 1766581517
++ local timestamp=1766581517
+++ TZ=UTC
+++ /usr/sbin/date -d@1766581517 '+%Y-%m-%d %H:%M:%S'
++ echo 2025-12-24 13:05:17
+ echo 'Waiting for last oplog chunk (2025-12-24 13:05:17) to be 120 seconds older than starting chunk (2025-12-24 13:05:17)'
Waiting for last oplog chunk (2025-12-24 13:05:17) to be 120 seconds older than starting chunk (2025-12-24 13:05:17)
+ sleep 10
+ [[ 1766581517 -gt 1766581517 ]]
+ [[ 4 -gt 30 ]]
++ get_latest_oplog_chunk_ts some-name
++ local cluster=some-name
+++ kubectl_bin exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ jq '.backups.pitrChunks.pitrChunks | last | .range.end'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.EFQ3f0X1Xr
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.4FcwB9yWC7
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in $(seq 0 2)
+++ set +e
+++ kubectl exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.EFQ3f0X1Xr
+++ cat /tmp/tmp.4FcwB9yWC7
+++ rm /tmp/tmp.EFQ3f0X1Xr /tmp/tmp.4FcwB9yWC7
+++ return 0
++ echo 1766581517
+ latest_ts=1766581517
+ retries=5
++ format_date 1766581517
++ local timestamp=1766581517
+++ TZ=UTC
+++ /usr/sbin/date -d@1766581517 '+%Y-%m-%d %H:%M:%S'
++ echo 2025-12-24 13:05:17
++ format_date 1766581517
++ local timestamp=1766581517
+++ TZ=UTC
+++ /usr/sbin/date -d@1766581517 '+%Y-%m-%d %H:%M:%S'
++ echo 2025-12-24 13:05:17
+ echo 'Waiting for last oplog chunk (2025-12-24 13:05:17) to be 120 seconds older than starting chunk (2025-12-24 13:05:17)'
Waiting for last oplog chunk (2025-12-24 13:05:17) to be 120 seconds older than starting chunk (2025-12-24 13:05:17)
+ sleep 10
+ [[ 1766581517 -gt 1766581517 ]]
+ [[ 5 -gt 30 ]]
++ get_latest_oplog_chunk_ts some-name
++ local cluster=some-name
+++ kubectl_bin exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ jq '.backups.pitrChunks.pitrChunks | last | .range.end'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.XIG0V9Xebm
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.dBN9t7vaBJ
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in $(seq 0 2)
+++ set +e
+++ kubectl exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.XIG0V9Xebm
+++ cat /tmp/tmp.dBN9t7vaBJ
+++ rm /tmp/tmp.XIG0V9Xebm /tmp/tmp.dBN9t7vaBJ
+++ return 0
++ echo 1766581517
+ latest_ts=1766581517
+ retries=6
++ format_date 1766581517
++ local timestamp=1766581517
+++ TZ=UTC
+++ /usr/sbin/date -d@1766581517 '+%Y-%m-%d %H:%M:%S'
++ echo 2025-12-24 13:05:17
++ format_date 1766581517
++ local timestamp=1766581517
+++ TZ=UTC
+++ /usr/sbin/date -d@1766581517 '+%Y-%m-%d %H:%M:%S'
++ echo 2025-12-24 13:05:17
+ echo 'Waiting for last oplog chunk (2025-12-24 13:05:17) to be 120 seconds older than starting chunk (2025-12-24 13:05:17)'
Waiting for last oplog chunk (2025-12-24 13:05:17) to be 120 seconds older than starting chunk (2025-12-24 13:05:17)
+ sleep 10
+ [[ 1766581517 -gt 1766581517 ]]
+ [[ 6 -gt 30 ]]
++ get_latest_oplog_chunk_ts some-name
++ local cluster=some-name
+++ kubectl_bin exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ jq '.backups.pitrChunks.pitrChunks | last | .range.end'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.BE9y09MLGe
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.eD4XC49JjO
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in $(seq 0 2)
+++ set +e
+++ kubectl exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.BE9y09MLGe
+++ cat /tmp/tmp.eD4XC49JjO
+++ rm /tmp/tmp.BE9y09MLGe /tmp/tmp.eD4XC49JjO
+++ return 0
++ echo 1766581517
+ latest_ts=1766581517
+ retries=7
++ format_date 1766581517
++ local timestamp=1766581517
+++ TZ=UTC
+++ /usr/sbin/date -d@1766581517 '+%Y-%m-%d %H:%M:%S'
++ echo 2025-12-24 13:05:17
++ format_date 1766581517
++ local timestamp=1766581517
+++ TZ=UTC
+++ /usr/sbin/date -d@1766581517 '+%Y-%m-%d %H:%M:%S'
++ echo 2025-12-24 13:05:17
+ echo 'Waiting for last oplog chunk (2025-12-24 13:05:17) to be 120 seconds older than starting chunk (2025-12-24 13:05:17)'
Waiting for last oplog chunk (2025-12-24 13:05:17) to be 120 seconds older than starting chunk (2025-12-24 13:05:17)
+ sleep 10
+ [[ 1766581517 -gt 1766581517 ]]
+ [[ 7 -gt 30 ]]
++ get_latest_oplog_chunk_ts some-name
++ local cluster=some-name
+++ kubectl_bin exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ jq '.backups.pitrChunks.pitrChunks | last | .range.end'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.q1CcxlxMtQ
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.C9uA5pbWlh
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in $(seq 0 2)
+++ set +e
+++ kubectl exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.q1CcxlxMtQ
+++ cat /tmp/tmp.C9uA5pbWlh
+++ rm /tmp/tmp.q1CcxlxMtQ /tmp/tmp.C9uA5pbWlh
+++ return 0
++ echo 1766581517
+ latest_ts=1766581517
+ retries=8
++ format_date 1766581517
++ local timestamp=1766581517
+++ TZ=UTC
+++ /usr/sbin/date -d@1766581517 '+%Y-%m-%d %H:%M:%S'
++ echo 2025-12-24 13:05:17
++ format_date 1766581517
++ local timestamp=1766581517
+++ TZ=UTC
+++ /usr/sbin/date -d@1766581517 '+%Y-%m-%d %H:%M:%S'
++ echo 2025-12-24 13:05:17
+ echo 'Waiting for last oplog chunk (2025-12-24 13:05:17) to be 120 seconds older than starting chunk (2025-12-24 13:05:17)'
Waiting for last oplog chunk (2025-12-24 13:05:17) to be 120 seconds older than starting chunk (2025-12-24 13:05:17)
+ sleep 10
+ [[ 1766581517 -gt 1766581517 ]]
+ [[ 8 -gt 30 ]]
++ get_latest_oplog_chunk_ts some-name
++ local cluster=some-name
+++ kubectl_bin exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ jq '.backups.pitrChunks.pitrChunks | last | .range.end'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.bl3kwBMRHu
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.f24auD1cgL
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in $(seq 0 2)
+++ set +e
+++ kubectl exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.bl3kwBMRHu
+++ cat /tmp/tmp.f24auD1cgL
+++ rm /tmp/tmp.bl3kwBMRHu /tmp/tmp.f24auD1cgL
+++ return 0
++ echo 1766581517
+ latest_ts=1766581517
+ retries=9
++ format_date 1766581517
++ local timestamp=1766581517
+++ TZ=UTC
+++ /usr/sbin/date -d@1766581517 '+%Y-%m-%d %H:%M:%S'
++ echo 2025-12-24 13:05:17
++ format_date 1766581517
++ local timestamp=1766581517
+++ TZ=UTC
+++ /usr/sbin/date -d@1766581517 '+%Y-%m-%d %H:%M:%S'
++ echo 2025-12-24 13:05:17
+ echo 'Waiting for last oplog chunk (2025-12-24 13:05:17) to be 120 seconds older than starting chunk (2025-12-24 13:05:17)'
Waiting for last oplog chunk (2025-12-24 13:05:17) to be 120 seconds older than starting chunk (2025-12-24 13:05:17)
+ sleep 10
+ [[ 1766581517 -gt 1766581517 ]]
+ [[ 9 -gt 30 ]]
++ get_latest_oplog_chunk_ts some-name
++ local cluster=some-name
+++ jq '.backups.pitrChunks.pitrChunks | last | .range.end'
+++ kubectl_bin exec some-name-rs0-0 -c backup-agent -- pbm status -o json
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.Zz4fFqeWaf
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.qUbxivnJ6X
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in $(seq 0 2)
+++ set +e
+++ kubectl exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.Zz4fFqeWaf
+++ cat /tmp/tmp.qUbxivnJ6X
+++ rm /tmp/tmp.Zz4fFqeWaf /tmp/tmp.qUbxivnJ6X
+++ return 0
++ echo 1766581517
+ latest_ts=1766581517
+ retries=10
++ format_date 1766581517
++ local timestamp=1766581517
+++ TZ=UTC
+++ /usr/sbin/date -d@1766581517 '+%Y-%m-%d %H:%M:%S'
++ echo 2025-12-24 13:05:17
++ format_date 1766581517
++ local timestamp=1766581517
+++ TZ=UTC
+++ /usr/sbin/date -d@1766581517 '+%Y-%m-%d %H:%M:%S'
++ echo 2025-12-24 13:05:17
+ echo 'Waiting for last oplog chunk (2025-12-24 13:05:17) to be 120 seconds older than starting chunk (2025-12-24 13:05:17)'
Waiting for last oplog chunk (2025-12-24 13:05:17) to be 120 seconds older than starting chunk (2025-12-24 13:05:17)
+ sleep 10
+ [[ 1766581517 -gt 1766581517 ]]
+ [[ 10 -gt 30 ]]
++ get_latest_oplog_chunk_ts some-name
++ local cluster=some-name
+++ kubectl_bin exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ jq '.backups.pitrChunks.pitrChunks | last | .range.end'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.KWToOxg0sv
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.D85cxJfsjK
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in $(seq 0 2)
+++ set +e
+++ kubectl exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.KWToOxg0sv
+++ cat /tmp/tmp.D85cxJfsjK
+++ rm /tmp/tmp.KWToOxg0sv /tmp/tmp.D85cxJfsjK
+++ return 0
++ echo 1766582768
+ latest_ts=1766582768
+ retries=11
++ format_date 1766582768
++ local timestamp=1766582768
+++ TZ=UTC
+++ /usr/sbin/date -d@1766582768 '+%Y-%m-%d %H:%M:%S'
++ echo 2025-12-24 13:26:08
++ format_date 1766581517
++ local timestamp=1766581517
+++ TZ=UTC
+++ /usr/sbin/date -d@1766581517 '+%Y-%m-%d %H:%M:%S'
++ echo 2025-12-24 13:05:17
+ echo 'Waiting for last oplog chunk (2025-12-24 13:26:08) to be 120 seconds older than starting chunk (2025-12-24 13:05:17)'
Waiting for last oplog chunk (2025-12-24 13:26:08) to be 120 seconds older than starting chunk (2025-12-24 13:05:17)
+ sleep 10
+ [[ 1766582768 -gt 1766581517 ]]
+ '[' -z backupSource ']'
+ desc 'check restore by latest backupSource'
+ set +o xtrace


-----------------------------------------------------------------------------------
check restore by latest backupSource
-----------------------------------------------------------------------------------

++ get_backup_dest backup-minio-2
++ local backup_name=backup-minio-2
++ kubectl_bin get psmdb-backup backup-minio-2 -o 'jsonpath={.status.destination}'
++ sed -e 's/.json$//'
++ sed 's|s3://||'
++ sed 's|azure://||'
+++ mktemp
++ sed 's|gs://||'
++ local LAST_OUT=/tmp/tmp.pdEfPWj1bN
+++ mktemp
++ local LAST_ERR=/tmp/tmp.LBCAP8VrRy
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb-backup backup-minio-2 -o 'jsonpath={.status.destination}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.pdEfPWj1bN
++ cat /tmp/tmp.LBCAP8VrRy
++ rm /tmp/tmp.pdEfPWj1bN /tmp/tmp.LBCAP8VrRy
++ return 0
+ backup_dest=operator-testing/2025-12-24T13:23:50Z
+ cat /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2151/e2e-tests/pitr-physical-backup-source/conf/restore.yml
+ /usr/sbin/sed -e 's/name:/name: restore-backup-minio-2/'
+ /usr/sbin/sed -e /backupName/d
+ /usr/sbin/sed -e 's/pitrType:/type: latest/'
+ '[' -z '' ']'
+ /usr/sbin/sed -e /date:/d
+ /usr/sbin/sed -e 's|DESTINATION|operator-testing/2025-12-24T13:23:50Z|'
+ '[' -n '' ']'
+ yq
+ kubectl_bin apply -f -
++ get_bucket_name backup-minio-2
++ local backup_name=backup-minio-2
++ kubectl_bin get psmdb-backup backup-minio-2 -o 'jsonpath={.status.s3.bucket}'
++ mktemp
+++ mktemp
+ local LAST_OUT=/tmp/tmp.0I0aJl4NJW
++ local LAST_OUT=/tmp/tmp.oTAZ7iJbyh
++ mktemp
+++ mktemp
+ local LAST_ERR=/tmp/tmp.w3Miw9f8m2
+ local exit_status=0
+ local timeout=4
++ local LAST_ERR=/tmp/tmp.FqICd7AHj1
++ local exit_status=0
++ local timeout=4
++ seq 0 2
+++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl apply -f -
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb-backup backup-minio-2 -o 'jsonpath={.status.s3.bucket}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.oTAZ7iJbyh
++ cat /tmp/tmp.FqICd7AHj1
++ rm /tmp/tmp.oTAZ7iJbyh /tmp/tmp.FqICd7AHj1
++ return 0
+ /usr/sbin/sed -e 's|BUCKET-NAME|operator-testing|'
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.0I0aJl4NJW
perconaservermongodbrestore.psmdb.percona.com/restore-backup-minio-2 created
+ cat /tmp/tmp.w3Miw9f8m2
+ rm /tmp/tmp.0I0aJl4NJW /tmp/tmp.w3Miw9f8m2
+ return 0
+ wait_restore backup-minio-2 some-name requested 0 1200
+ local backup_name=backup-minio-2
+ local cluster_name=some-name
+ local target_state=requested
+ local wait_cluster_consistency=0
+ local wait_time=1200
+ local ok_if_ready=0
+ set +o xtrace
Waiting for the psmdb-restore/restore-backup-minio-2 object to be created.OK
Waiting psmdb-restore/restore-backup-minio-2 to reach state "requested" .........OK after 8 minutes
+ [[ 0 -eq 1 ]]
+ echo

+ wait_restore backup-minio-2 some-name ready 0 1600
+ local backup_name=backup-minio-2
+ local cluster_name=some-name
+ local target_state=ready
+ local wait_cluster_consistency=0
+ local wait_time=1600
+ local ok_if_ready=0
+ set +o xtrace
Waiting for the psmdb-restore/restore-backup-minio-2 object to be created.OK
Waiting psmdb-restore/restore-backup-minio-2 to reach state "ready" ...OK after 2 minutes
+ [[ 0 -eq 1 ]]
+ echo

+ set -o xtrace
+ wait_for_running some-name-rs0 3
+ local name=some-name-rs0
+ let last_pod=2
+ local check_cluster_readyness=true
+ set_debug
+ [[ 1 == 1 ]]
+ set -o xtrace
+ local rs_name=rs0
+ local cluster_name=some-name
++ seq 0 2
+ for i in $(seq 0 $last_pod)
+ [[ 0 -eq 2 ]]
+ wait_pod some-name-rs0-0
+ local pod=some-name-rs0-0
+ set +o xtrace
waiting for pod/some-name-rs0-0 to be ready.OK
+ for i in $(seq 0 $last_pod)
+ [[ 1 -eq 2 ]]
+ wait_pod some-name-rs0-1
+ local pod=some-name-rs0-1
+ set +o xtrace
waiting for pod/some-name-rs0-1 to be ready.OK
+ for i in $(seq 0 $last_pod)
+ [[ 2 -eq 2 ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].arbiter.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.SG3CyCkV71
+++ mktemp
++ local LAST_ERR=/tmp/tmp.CzewoEhNLM
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].arbiter.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.SG3CyCkV71
++ cat /tmp/tmp.CzewoEhNLM
++ rm /tmp/tmp.SG3CyCkV71 /tmp/tmp.CzewoEhNLM
++ return 0
+ [[ '' == \t\r\u\e ]]
+ wait_pod some-name-rs0-2
+ local pod=some-name-rs0-2
+ set +o xtrace
waiting for pod/some-name-rs0-2 to be ready.OK
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].nonvoting.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.qYGgi2KYlf
+++ mktemp
++ local LAST_ERR=/tmp/tmp.Di29A6Mlz5
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].nonvoting.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.qYGgi2KYlf
++ cat /tmp/tmp.Di29A6Mlz5
++ rm /tmp/tmp.qYGgi2KYlf /tmp/tmp.Di29A6Mlz5
++ return 0
+ [[ '' == \t\r\u\e ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].hidden.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.Ky29Fpxfxj
+++ mktemp
++ local LAST_ERR=/tmp/tmp.lxTHjZfYui
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].hidden.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.Ky29Fpxfxj
++ cat /tmp/tmp.lxTHjZfYui
++ rm /tmp/tmp.Ky29Fpxfxj /tmp/tmp.lxTHjZfYui
++ return 0
+ [[ '' == \t\r\u\e ]]
+ sleep 10
+ [[ true == \t\r\u\e ]]
+ set +x
Waiting for cluster readyness.....................................................................
+ wait_for_running some-name-cfg 3
+ local name=some-name-cfg
+ let last_pod=2
+ local check_cluster_readyness=true
+ set_debug
+ [[ 1 == 1 ]]
+ set -o xtrace
+ local rs_name=cfg
+ local cluster_name=some-name
++ seq 0 2
+ for i in $(seq 0 $last_pod)
+ [[ 0 -eq 2 ]]
+ wait_pod some-name-cfg-0
+ local pod=some-name-cfg-0
+ set +o xtrace
waiting for pod/some-name-cfg-0 to be ready.OK
+ for i in $(seq 0 $last_pod)
+ [[ 1 -eq 2 ]]
+ wait_pod some-name-cfg-1
+ local pod=some-name-cfg-1
+ set +o xtrace
waiting for pod/some-name-cfg-1 to be ready.OK
+ for i in $(seq 0 $last_pod)
+ [[ 2 -eq 2 ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].arbiter.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.aInALs4FEJ
+++ mktemp
++ local LAST_ERR=/tmp/tmp.46v4Iknedg
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].arbiter.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.aInALs4FEJ
++ cat /tmp/tmp.46v4Iknedg
++ rm /tmp/tmp.aInALs4FEJ /tmp/tmp.46v4Iknedg
++ return 0
+ [[ '' == \t\r\u\e ]]
+ wait_pod some-name-cfg-2
+ local pod=some-name-cfg-2
+ set +o xtrace
waiting for pod/some-name-cfg-2 to be ready.OK
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].nonvoting.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.sSLX6kADUh
+++ mktemp
++ local LAST_ERR=/tmp/tmp.4BIVwzkx51
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].nonvoting.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.sSLX6kADUh
++ cat /tmp/tmp.4BIVwzkx51
++ rm /tmp/tmp.sSLX6kADUh /tmp/tmp.4BIVwzkx51
++ return 0
+ [[ '' == \t\r\u\e ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].hidden.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.5mQhuaJRsD
+++ mktemp
++ local LAST_ERR=/tmp/tmp.13HUQUkrdj
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].hidden.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.5mQhuaJRsD
++ cat /tmp/tmp.13HUQUkrdj
++ rm /tmp/tmp.5mQhuaJRsD /tmp/tmp.13HUQUkrdj
++ return 0
+ [[ '' == \t\r\u\e ]]
+ sleep 10
+ [[ true == \t\r\u\e ]]
+ set +x
Waiting for cluster readyness
+ wait_for_running some-name-mongos 3
+ local name=some-name-mongos
+ let last_pod=2
+ local check_cluster_readyness=true
+ set_debug
+ [[ 1 == 1 ]]
+ set -o xtrace
+ local rs_name=mongos
+ local cluster_name=some-name
++ seq 0 2
+ for i in $(seq 0 $last_pod)
+ [[ 0 -eq 2 ]]
+ wait_pod some-name-mongos-0
+ local pod=some-name-mongos-0
+ set +o xtrace
waiting for pod/some-name-mongos-0 to be ready.OK
+ for i in $(seq 0 $last_pod)
+ [[ 1 -eq 2 ]]
+ wait_pod some-name-mongos-1
+ local pod=some-name-mongos-1
+ set +o xtrace
waiting for pod/some-name-mongos-1 to be ready.OK
+ for i in $(seq 0 $last_pod)
+ [[ 2 -eq 2 ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="mongos")].arbiter.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.9ddARevCw0
+++ mktemp
++ local LAST_ERR=/tmp/tmp.Cyh9eaUDZK
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="mongos")].arbiter.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.9ddARevCw0
++ cat /tmp/tmp.Cyh9eaUDZK
++ rm /tmp/tmp.9ddARevCw0 /tmp/tmp.Cyh9eaUDZK
++ return 0
+ [[ '' == \t\r\u\e ]]
+ wait_pod some-name-mongos-2
+ local pod=some-name-mongos-2
+ set +o xtrace
waiting for pod/some-name-mongos-2 to be ready.OK
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="mongos")].nonvoting.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.UZ4TKCIaiG
+++ mktemp
++ local LAST_ERR=/tmp/tmp.1B3JU6huR1
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="mongos")].nonvoting.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.UZ4TKCIaiG
++ cat /tmp/tmp.1B3JU6huR1
++ rm /tmp/tmp.UZ4TKCIaiG /tmp/tmp.1B3JU6huR1
++ return 0
+ [[ '' == \t\r\u\e ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="mongos")].hidden.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.BeJWE1u2Yw
+++ mktemp
++ local LAST_ERR=/tmp/tmp.bC6kURsGUJ
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="mongos")].hidden.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.BeJWE1u2Yw
++ cat /tmp/tmp.bC6kURsGUJ
++ rm /tmp/tmp.BeJWE1u2Yw /tmp/tmp.bC6kURsGUJ
++ return 0
+ [[ '' == \t\r\u\e ]]
+ sleep 10
+ [[ true == \t\r\u\e ]]
+ set +x
Waiting for cluster readyness
+ sleep 10
+ compare_mongos_cmd find myApp:myPass@some-name-mongos.pitr-physical-backup-source-28579 -3rd
+ local command=find
+ local uri=myApp:myPass@some-name-mongos.pitr-physical-backup-source-28579
+ local postfix=-3rd
+ local suffix=
+ local database=myApp
+ local collection=test
+ local port=27017
+ local tls=false
+ [[ false == \t\r\u\e ]]
+ mongos_command=run_mongos
+ log 'running db.test.command() in myApp'
+ set +o xtrace
[2025-12-24T13:40:07+0000] running db.test.command() in myApp
+ run_mongos 'use myApp\n db.test.find()' myApp:myPass@some-name-mongos.pitr-physical-backup-source-28579 mongodb '' '' 27017
+ local 'command=use myApp\n db.test.find()'
+ local uri=myApp:myPass@some-name-mongos.pitr-physical-backup-source-28579
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local port=27017
+ local mongo_bin=mongo
+ egrep -v 'I NETWORK|W NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|Started a new thread for the timer service'
+ /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
++ echo .svc.cluster.local
++ awk -F: '{print $2}'
egrep: warning: egrep is obsolescent; using grep -E
+ suffix_port=
+ [[ -z '' ]]
+ suffix=.svc.cluster.local:27017
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.51OakboU4R
+++ mktemp
++ local LAST_ERR=/tmp/tmp.gSGSK5dyew
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.51OakboU4R
++ cat /tmp/tmp.gSGSK5dyew
++ rm /tmp/tmp.51OakboU4R /tmp/tmp.gSGSK5dyew
++ return 0
+ local client_container=psmdb-client-696897d69b-m5f2h
+ kubectl_bin exec psmdb-client-696897d69b-m5f2h -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.pitr-physical-backup-source-28579.svc.cluster.local:27017/admin '
++ mktemp
+ local LAST_OUT=/tmp/tmp.iWwfMVEl6d
++ mktemp
+ local LAST_ERR=/tmp/tmp.JSd5MlEFda
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-696897d69b-m5f2h -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.pitr-physical-backup-source-28579.svc.cluster.local:27017/admin '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.iWwfMVEl6d
+ cat /tmp/tmp.JSd5MlEFda
+ rm /tmp/tmp.iWwfMVEl6d /tmp/tmp.JSd5MlEFda
+ return 0
+ [[ 0 -eq 0 ]]
+ diff /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2151/e2e-tests/pitr-physical-backup-source/compare/find-3rd.json /tmp/tmp.e7bs9MurNC/find-3rd
+ desc 'disable pitr'
+ set +o xtrace


-----------------------------------------------------------------------------------
disable pitr
-----------------------------------------------------------------------------------

+ kubectl patch psmdb some-name --type=merge --patch '{"spec": {"backup": {"pitr": {"enabled": false}}}}'
perconaservermongodb.psmdb.percona.com/some-name patched
+ sleep 20
+ desc 'delete all backups'
+ set +o xtrace


-----------------------------------------------------------------------------------
delete all backups
-----------------------------------------------------------------------------------

+ kubectl_bin delete psmdb-backup --all
++ mktemp
+ local LAST_OUT=/tmp/tmp.gvAukyrtT3
++ mktemp
+ local LAST_ERR=/tmp/tmp.gGLYzPjkHf
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl delete psmdb-backup --all
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.gvAukyrtT3
perconaservermongodbbackup.psmdb.percona.com "backup-minio-1" deleted from pitr-physical-backup-source-28579 namespace
perconaservermongodbbackup.psmdb.percona.com "backup-minio-2" deleted from pitr-physical-backup-source-28579 namespace
+ cat /tmp/tmp.gGLYzPjkHf
+ rm /tmp/tmp.gvAukyrtT3 /tmp/tmp.gGLYzPjkHf
+ return 0
+ desc 'destroy cluster'
+ set +o xtrace


-----------------------------------------------------------------------------------
destroy cluster
-----------------------------------------------------------------------------------

+ destroy pitr-physical-backup-source-28579
+ local namespace=pitr-physical-backup-source-28579
+ local ignore_logs=true
+ [[ 0 == 1 ]]
+ desc 'destroy cluster/operator and all other resources'
+ set +o xtrace


-----------------------------------------------------------------------------------
destroy cluster/operator and all other resources
-----------------------------------------------------------------------------------

+ '[' true == false ']'
+ delete_backups
+ desc 'Delete psmdb-backup'
+ set +o xtrace


-----------------------------------------------------------------------------------
Delete psmdb-backup
-----------------------------------------------------------------------------------

++ kubectl_bin get psmdb-backup --no-headers
++ wc -l
+++ mktemp
++ local LAST_OUT=/tmp/tmp.jndgwZcJ69
+++ mktemp
++ local LAST_ERR=/tmp/tmp.ipXW1TL9TM
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb-backup --no-headers
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.jndgwZcJ69
++ cat /tmp/tmp.ipXW1TL9TM
No resources found in pitr-physical-backup-source-28579 namespace.
++ rm /tmp/tmp.jndgwZcJ69 /tmp/tmp.ipXW1TL9TM
++ return 0
+ '[' 0 '!=' 0 ']'
+ delete_crd
+ desc 'get and delete old CRDs and RBAC'
+ set +o xtrace


-----------------------------------------------------------------------------------
get and delete old CRDs and RBAC
-----------------------------------------------------------------------------------

+ kubectl_bin delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2151/deploy/crd.yaml --ignore-not-found --wait=false
++ mktemp
+ local LAST_OUT=/tmp/tmp.dNiyN2fXu0
++ mktemp
+ local LAST_ERR=/tmp/tmp.1YZo0PsJOf
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2151/deploy/crd.yaml --ignore-not-found --wait=false
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.dNiyN2fXu0
customresourcedefinition.apiextensions.k8s.io "perconaservermongodbbackups.psmdb.percona.com" deleted
customresourcedefinition.apiextensions.k8s.io "perconaservermongodbrestores.psmdb.percona.com" deleted
customresourcedefinition.apiextensions.k8s.io "perconaservermongodbs.psmdb.percona.com" deleted
+ cat /tmp/tmp.1YZo0PsJOf
+ rm /tmp/tmp.dNiyN2fXu0 /tmp/tmp.1YZo0PsJOf
+ return 0
++ yq eval .metadata.name /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2151/deploy/crd.yaml
++ grep -v '\-\-\-'
grep: warning: stray \ before -
grep: warning: stray \ before -
+ for crd_name in $(yq eval '.metadata.name' "${src_dir}/deploy/crd.yaml" | grep -v '\-\-\-')
+ grep -v NAMESPACE
+ kubectl get perconaservermongodbbackups.psmdb.percona.com --all-namespaces -o wide
+ xargs -L 1 sh -xc 'kubectl patch perconaservermongodbbackups.psmdb.percona.com -n $0 $1 --type=merge -p "{\"metadata\":{\"finalizers\":[]}}"'
error: the server doesn't have a resource type "perconaservermongodbbackups"
+ kubectl patch perconaservermongodbbackups.psmdb.percona.com -n sh --type=merge -p '{"metadata":{"finalizers":[]}}'
error: the server doesn't have a resource type "perconaservermongodbbackups"
+ :
+ kubectl_bin wait --for=delete crd perconaservermongodbbackups.psmdb.percona.com
++ mktemp
+ local LAST_OUT=/tmp/tmp.hxH0PvWiku
++ mktemp
+ local LAST_ERR=/tmp/tmp.X8BjEGbIIM
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl wait --for=delete crd perconaservermongodbbackups.psmdb.percona.com
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.hxH0PvWiku
+ cat /tmp/tmp.X8BjEGbIIM
+ rm /tmp/tmp.hxH0PvWiku /tmp/tmp.X8BjEGbIIM
+ return 0
+ for crd_name in $(yq eval '.metadata.name' "${src_dir}/deploy/crd.yaml" | grep -v '\-\-\-')
+ kubectl get perconaservermongodbrestores.psmdb.percona.com --all-namespaces -o wide
+ grep -v NAMESPACE
+ xargs -L 1 sh -xc 'kubectl patch perconaservermongodbrestores.psmdb.percona.com -n $0 $1 --type=merge -p "{\"metadata\":{\"finalizers\":[]}}"'
error: the server doesn't have a resource type "perconaservermongodbrestores"
+ kubectl patch perconaservermongodbrestores.psmdb.percona.com -n sh --type=merge -p '{"metadata":{"finalizers":[]}}'
error: the server doesn't have a resource type "perconaservermongodbrestores"
+ :
+ kubectl_bin wait --for=delete crd perconaservermongodbrestores.psmdb.percona.com
++ mktemp
+ local LAST_OUT=/tmp/tmp.uRsbIfjXlw
++ mktemp
+ local LAST_ERR=/tmp/tmp.K2FvwqPjz2
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl wait --for=delete crd perconaservermongodbrestores.psmdb.percona.com
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.uRsbIfjXlw
+ cat /tmp/tmp.K2FvwqPjz2
+ rm /tmp/tmp.uRsbIfjXlw /tmp/tmp.K2FvwqPjz2
+ return 0
+ for crd_name in $(yq eval '.metadata.name' "${src_dir}/deploy/crd.yaml" | grep -v '\-\-\-')
+ kubectl get perconaservermongodbs.psmdb.percona.com --all-namespaces -o wide
+ grep -v NAMESPACE
+ xargs -L 1 sh -xc 'kubectl patch perconaservermongodbs.psmdb.percona.com -n $0 $1 --type=merge -p "{\"metadata\":{\"finalizers\":[]}}"'
error: the server doesn't have a resource type "perconaservermongodbs"
+ kubectl patch perconaservermongodbs.psmdb.percona.com -n sh --type=merge -p '{"metadata":{"finalizers":[]}}'
error: the server doesn't have a resource type "perconaservermongodbs"
+ :
+ kubectl_bin wait --for=delete crd perconaservermongodbs.psmdb.percona.com
++ mktemp
+ local LAST_OUT=/tmp/tmp.ex1sM7zzBA
++ mktemp
+ local LAST_ERR=/tmp/tmp.0Uvrsr56iq
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl wait --for=delete crd perconaservermongodbs.psmdb.percona.com
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.ex1sM7zzBA
+ cat /tmp/tmp.0Uvrsr56iq
+ rm /tmp/tmp.ex1sM7zzBA /tmp/tmp.0Uvrsr56iq
+ return 0
+ local rbac_yaml=rbac.yaml
+ '[' -n psmdb-operator ']'
+ rbac_yaml=cw-rbac.yaml
+ kubectl_bin delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2151/deploy/cw-rbac.yaml --ignore-not-found
++ mktemp
+ local LAST_OUT=/tmp/tmp.LWlb1TS6al
++ mktemp
+ local LAST_ERR=/tmp/tmp.CxLW7XaDse
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2151/deploy/cw-rbac.yaml --ignore-not-found
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.LWlb1TS6al
clusterrole.rbac.authorization.k8s.io "percona-server-mongodb-operator" deleted
clusterrolebinding.rbac.authorization.k8s.io "service-account-percona-server-mongodb-operator" deleted
+ cat /tmp/tmp.CxLW7XaDse
+ rm /tmp/tmp.LWlb1TS6al /tmp/tmp.CxLW7XaDse
+ return 0
+ destroy_cert_manager
+ kubectl_bin delete -f https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml
++ mktemp
+ local LAST_OUT=/tmp/tmp.RGF5mERlfp
++ mktemp
+ local LAST_ERR=/tmp/tmp.YGf4bZvEDl
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl delete -f https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml
+ exit_status=1
+ set -e
+ '[' 1 '!=' 0 -a -n 1 ']'
+ cat /tmp/tmp.RGF5mERlfp
+ cat /tmp/tmp.YGf4bZvEDl
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": namespaces "cert-manager" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "challenges.acme.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "orders.acme.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "certificaterequests.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "certificates.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "clusterissuers.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "issuers.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": serviceaccounts "cert-manager-cainjector" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": serviceaccounts "cert-manager" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": serviceaccounts "cert-manager-webhook" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-cainjector" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-issuers" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-clusterissuers" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-certificates" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-orders" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-challenges" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-ingress-shim" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-cluster-view" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-view" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-edit" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-approve:cert-manager-io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-certificatesigningrequests" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-webhook:subjectaccessreviews" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-cainjector" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-issuers" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-clusterissuers" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-certificates" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-orders" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-challenges" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-ingress-shim" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-approve:cert-manager-io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-certificatesigningrequests" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-webhook:subjectaccessreviews" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager-cainjector:leaderelection" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager:leaderelection" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager-tokenrequest" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager-webhook:dynamic-serving" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager-cainjector:leaderelection" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager:leaderelection" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager-tokenrequest" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager-webhook:dynamic-serving" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": services "cert-manager-cainjector" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": services "cert-manager" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": services "cert-manager-webhook" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": deployments.apps "cert-manager-cainjector" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": deployments.apps "cert-manager" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": deployments.apps "cert-manager-webhook" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": mutatingwebhookconfigurations.admissionregistration.k8s.io "cert-manager-webhook" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": validatingwebhookconfigurations.admissionregistration.k8s.io "cert-manager-webhook" not found
+ sleep 0
+ for i in $(seq 0 2)
+ set +e
+ kubectl delete -f https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml
+ exit_status=1
+ set -e
+ '[' 1 '!=' 0 -a -n 1 ']'
+ cat /tmp/tmp.RGF5mERlfp
+ cat /tmp/tmp.YGf4bZvEDl
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": namespaces "cert-manager" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "challenges.acme.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "orders.acme.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "certificaterequests.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "certificates.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "clusterissuers.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "issuers.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": serviceaccounts "cert-manager-cainjector" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": serviceaccounts "cert-manager" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": serviceaccounts "cert-manager-webhook" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-cainjector" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-issuers" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-clusterissuers" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-certificates" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-orders" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-challenges" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-ingress-shim" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-cluster-view" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-view" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-edit" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-approve:cert-manager-io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-certificatesigningrequests" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-webhook:subjectaccessreviews" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-cainjector" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-issuers" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-clusterissuers" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-certificates" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-orders" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-challenges" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-ingress-shim" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-approve:cert-manager-io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-certificatesigningrequests" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-webhook:subjectaccessreviews" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager-cainjector:leaderelection" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager:leaderelection" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager-tokenrequest" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager-webhook:dynamic-serving" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager-cainjector:leaderelection" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager:leaderelection" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager-tokenrequest" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager-webhook:dynamic-serving" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": services "cert-manager-cainjector" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": services "cert-manager" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": services "cert-manager-webhook" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": deployments.apps "cert-manager-cainjector" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": deployments.apps "cert-manager" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": deployments.apps "cert-manager-webhook" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": mutatingwebhookconfigurations.admissionregistration.k8s.io "cert-manager-webhook" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": validatingwebhookconfigurations.admissionregistration.k8s.io "cert-manager-webhook" not found
+ sleep 4
+ for i in $(seq 0 2)
+ set +e
+ kubectl delete -f https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml
+ exit_status=1
+ set -e
+ '[' 1 '!=' 0 -a -n 1 ']'
+ cat /tmp/tmp.RGF5mERlfp
+ cat /tmp/tmp.YGf4bZvEDl
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": namespaces "cert-manager" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "challenges.acme.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "orders.acme.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "certificaterequests.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "certificates.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "clusterissuers.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "issuers.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": serviceaccounts "cert-manager-cainjector" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": serviceaccounts "cert-manager" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": serviceaccounts "cert-manager-webhook" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-cainjector" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-issuers" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-clusterissuers" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-certificates" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-orders" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-challenges" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-ingress-shim" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-cluster-view" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-view" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-edit" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-approve:cert-manager-io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-certificatesigningrequests" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-webhook:subjectaccessreviews" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-cainjector" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-issuers" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-clusterissuers" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-certificates" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-orders" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-challenges" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-ingress-shim" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-approve:cert-manager-io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-certificatesigningrequests" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-webhook:subjectaccessreviews" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager-cainjector:leaderelection" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager:leaderelection" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager-tokenrequest" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager-webhook:dynamic-serving" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager-cainjector:leaderelection" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager:leaderelection" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager-tokenrequest" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager-webhook:dynamic-serving" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": services "cert-manager-cainjector" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": services "cert-manager" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": services "cert-manager-webhook" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": deployments.apps "cert-manager-cainjector" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": deployments.apps "cert-manager" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": deployments.apps "cert-manager-webhook" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": mutatingwebhookconfigurations.admissionregistration.k8s.io "cert-manager-webhook" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": validatingwebhookconfigurations.admissionregistration.k8s.io "cert-manager-webhook" not found
+ sleep 8
+ cat /tmp/tmp.RGF5mERlfp
+ cat /tmp/tmp.YGf4bZvEDl
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": namespaces "cert-manager" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "challenges.acme.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "orders.acme.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "certificaterequests.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "certificates.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "clusterissuers.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "issuers.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": serviceaccounts "cert-manager-cainjector" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": serviceaccounts "cert-manager" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": serviceaccounts "cert-manager-webhook" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-cainjector" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-issuers" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-clusterissuers" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-certificates" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-orders" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-challenges" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-ingress-shim" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-cluster-view" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-view" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-edit" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-approve:cert-manager-io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-certificatesigningrequests" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-webhook:subjectaccessreviews" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-cainjector" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-issuers" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-clusterissuers" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-certificates" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-orders" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-challenges" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-ingress-shim" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-approve:cert-manager-io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-certificatesigningrequests" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-webhook:subjectaccessreviews" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager-cainjector:leaderelection" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager:leaderelection" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager-tokenrequest" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager-webhook:dynamic-serving" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager-cainjector:leaderelection" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager:leaderelection" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager-tokenrequest" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager-webhook:dynamic-serving" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": services "cert-manager-cainjector" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": services "cert-manager" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": services "cert-manager-webhook" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": deployments.apps "cert-manager-cainjector" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": deployments.apps "cert-manager" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": deployments.apps "cert-manager-webhook" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": mutatingwebhookconfigurations.admissionregistration.k8s.io "cert-manager-webhook" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": validatingwebhookconfigurations.admissionregistration.k8s.io "cert-manager-webhook" not found
+ rm /tmp/tmp.RGF5mERlfp /tmp/tmp.YGf4bZvEDl
+ return 1
+ true
+ '[' -n '' ']'
+ '[' -n psmdb-operator ']'
+ kubectl_bin delete --grace-period=0 --force=true namespace pitr-physical-backup-source-28579
+ rm -rf /tmp/tmp.e7bs9MurNC
+ kubectl_bin delete --grace-period=0 --force=true namespace psmdb-operator
++ mktemp
++ mktemp
+ desc 'test passed'
+ set +o xtrace


-----------------------------------------------------------------------------------
test passed
-----------------------------------------------------------------------------------

+ local LAST_OUT=/tmp/tmp.ODYouzfM0u
+ local LAST_OUT=/tmp/tmp.ubY323VCJT
++ mktemp
++ mktemp
+ local LAST_ERR=/tmp/tmp.zzTT8xdPQY
+ local exit_status=0
+ local timeout=4
+ local LAST_ERR=/tmp/tmp.cdQSVbE9JI
+ local exit_status=0
+ local timeout=4
++ seq 0 2
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl delete --grace-period=0 --force=true namespace pitr-physical-backup-source-28579
+ for i in $(seq 0 2)
+ set +e
+ kubectl delete --grace-period=0 --force=true namespace psmdb-operator