Log: /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1850/e2e-tests/logs/pitr.log
grep: warning: stray \ before -
Warning: version difference between client (1.34) and server (1.31) exceeds the supported minor version skew of +/-1
Warning: version difference between client (1.34) and server (1.31) exceeds the supported minor version skew of +/-1
Warning: version difference between client (1.34) and server (1.31) exceeds the supported minor version skew of +/-1
+ main
+ create_infra pitr-32014
+ local ns=pitr-32014
+ [[ 1 == 1 ]]
+ delete_crd
+ desc 'get and delete old CRDs and RBAC'
+ set +o xtrace


-----------------------------------------------------------------------------------
get and delete old CRDs and RBAC
-----------------------------------------------------------------------------------

+ kubectl_bin delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1850/deploy/crd.yaml --ignore-not-found --wait=false
++ mktemp
+ local LAST_OUT=/tmp/tmp.9AzOp8YdJ3
++ mktemp
+ local LAST_ERR=/tmp/tmp.SYYAroev3G
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1850/deploy/crd.yaml --ignore-not-found --wait=false
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.9AzOp8YdJ3
customresourcedefinition.apiextensions.k8s.io "perconaservermongodbbackups.psmdb.percona.com" deleted
customresourcedefinition.apiextensions.k8s.io "perconaservermongodbrestores.psmdb.percona.com" deleted
customresourcedefinition.apiextensions.k8s.io "perconaservermongodbs.psmdb.percona.com" deleted
+ cat /tmp/tmp.SYYAroev3G
+ rm /tmp/tmp.9AzOp8YdJ3 /tmp/tmp.SYYAroev3G
+ return 0
++ yq eval .metadata.name /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1850/deploy/crd.yaml
++ grep -v '\-\-\-'
grep: warning: stray \ before -
grep: warning: stray \ before -
+ for crd_name in $(yq eval '.metadata.name' "${src_dir}/deploy/crd.yaml" | grep -v '\-\-\-')
+ kubectl get perconaservermongodbbackups.psmdb.percona.com --all-namespaces -o wide
+ grep -v NAMESPACE
+ xargs -L 1 sh -xc 'kubectl patch perconaservermongodbbackups.psmdb.percona.com -n $0 $1 --type=merge -p "{\"metadata\":{\"finalizers\":[]}}"'
No resources found
+ kubectl patch perconaservermongodbbackups.psmdb.percona.com -n sh --type=merge -p '{"metadata":{"finalizers":[]}}'
error: the server doesn't have a resource type "perconaservermongodbbackups"
+ :
+ kubectl_bin wait --for=delete crd perconaservermongodbbackups.psmdb.percona.com
++ mktemp
+ local LAST_OUT=/tmp/tmp.T6mY0pTTeF
++ mktemp
+ local LAST_ERR=/tmp/tmp.wLrTJ9Q1Fr
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl wait --for=delete crd perconaservermongodbbackups.psmdb.percona.com
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.T6mY0pTTeF
+ cat /tmp/tmp.wLrTJ9Q1Fr
+ rm /tmp/tmp.T6mY0pTTeF /tmp/tmp.wLrTJ9Q1Fr
+ return 0
+ for crd_name in $(yq eval '.metadata.name' "${src_dir}/deploy/crd.yaml" | grep -v '\-\-\-')
+ kubectl get perconaservermongodbrestores.psmdb.percona.com --all-namespaces -o wide
+ grep -v NAMESPACE
+ xargs -L 1 sh -xc 'kubectl patch perconaservermongodbrestores.psmdb.percona.com -n $0 $1 --type=merge -p "{\"metadata\":{\"finalizers\":[]}}"'
error: the server doesn't have a resource type "perconaservermongodbrestores"
+ kubectl patch perconaservermongodbrestores.psmdb.percona.com -n sh --type=merge -p '{"metadata":{"finalizers":[]}}'
error: the server doesn't have a resource type "perconaservermongodbrestores"
+ :
+ kubectl_bin wait --for=delete crd perconaservermongodbrestores.psmdb.percona.com
++ mktemp
+ local LAST_OUT=/tmp/tmp.DrNCOJsFOt
++ mktemp
+ local LAST_ERR=/tmp/tmp.qcpmovQmzd
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl wait --for=delete crd perconaservermongodbrestores.psmdb.percona.com
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.DrNCOJsFOt
+ cat /tmp/tmp.qcpmovQmzd
+ rm /tmp/tmp.DrNCOJsFOt /tmp/tmp.qcpmovQmzd
+ return 0
+ for crd_name in $(yq eval '.metadata.name' "${src_dir}/deploy/crd.yaml" | grep -v '\-\-\-')
+ kubectl get perconaservermongodbs.psmdb.percona.com --all-namespaces -o wide
+ grep -v NAMESPACE
+ xargs -L 1 sh -xc 'kubectl patch perconaservermongodbs.psmdb.percona.com -n $0 $1 --type=merge -p "{\"metadata\":{\"finalizers\":[]}}"'
error: the server doesn't have a resource type "perconaservermongodbs"
+ kubectl patch perconaservermongodbs.psmdb.percona.com -n sh --type=merge -p '{"metadata":{"finalizers":[]}}'
error: the server doesn't have a resource type "perconaservermongodbs"
+ :
+ kubectl_bin wait --for=delete crd perconaservermongodbs.psmdb.percona.com
++ mktemp
+ local LAST_OUT=/tmp/tmp.gxmabaS6N9
++ mktemp
+ local LAST_ERR=/tmp/tmp.uVDgkPHSDM
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl wait --for=delete crd perconaservermongodbs.psmdb.percona.com
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.gxmabaS6N9
+ cat /tmp/tmp.uVDgkPHSDM
+ rm /tmp/tmp.gxmabaS6N9 /tmp/tmp.uVDgkPHSDM
+ return 0
+ local rbac_yaml=rbac.yaml
+ '[' -n psmdb-operator ']'
+ rbac_yaml=cw-rbac.yaml
+ kubectl_bin delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1850/deploy/cw-rbac.yaml --ignore-not-found
++ mktemp
+ local LAST_OUT=/tmp/tmp.KnFpZsZE1c
++ mktemp
+ local LAST_ERR=/tmp/tmp.cfpmchf9DR
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1850/deploy/cw-rbac.yaml --ignore-not-found
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.KnFpZsZE1c
clusterrole.rbac.authorization.k8s.io "percona-server-mongodb-operator" deleted
clusterrolebinding.rbac.authorization.k8s.io "service-account-percona-server-mongodb-operator" deleted
+ cat /tmp/tmp.cfpmchf9DR
+ rm /tmp/tmp.KnFpZsZE1c /tmp/tmp.cfpmchf9DR
+ return 0
+ check_crd_for_deletion PR-1850-e336896c
+ local git_tag=PR-1850-e336896c
++ curl -s https://raw.githubusercontent.com/percona/percona-server-mongodb-operator/PR-1850-e336896c/deploy/crd.yaml
++ yq eval .metadata.name
++ /usr/sbin/sed s/---//g
++ /usr/sbin/sed ':a;N;$!ba;s/\n/ /g'
+ for crd_name in $(curl -s https://raw.githubusercontent.com/percona/percona-server-mongodb-operator/${git_tag}/deploy/crd.yaml | yq eval '.metadata.name' | $sed 's/---//g' | $sed ':a;N;$!ba;s/\n/ /g')
++ kubectl_bin get crd/null -o 'jsonpath={.status.conditions[-1].type}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.xNj7q9Ozs7
+++ mktemp
++ local LAST_ERR=/tmp/tmp.UMXEjaaYez
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get crd/null -o 'jsonpath={.status.conditions[-1].type}'
++ exit_status=1
++ set -e
++ '[' 1 '!=' 0 -a -n 1 ']'
++ cat /tmp/tmp.xNj7q9Ozs7
++ cat /tmp/tmp.UMXEjaaYez
Error from server (NotFound): customresourcedefinitions.apiextensions.k8s.io "null" not found
++ sleep 0
++ for i in $(seq 0 2)
++ set +e
++ kubectl get crd/null -o 'jsonpath={.status.conditions[-1].type}'
++ exit_status=1
++ set -e
++ '[' 1 '!=' 0 -a -n 1 ']'
++ cat /tmp/tmp.xNj7q9Ozs7
++ cat /tmp/tmp.UMXEjaaYez
Error from server (NotFound): customresourcedefinitions.apiextensions.k8s.io "null" not found
++ sleep 4
++ for i in $(seq 0 2)
++ set +e
++ kubectl get crd/null -o 'jsonpath={.status.conditions[-1].type}'
++ exit_status=1
++ set -e
++ '[' 1 '!=' 0 -a -n 1 ']'
++ cat /tmp/tmp.xNj7q9Ozs7
++ cat /tmp/tmp.UMXEjaaYez
Error from server (NotFound): customresourcedefinitions.apiextensions.k8s.io "null" not found
++ sleep 8
++ cat /tmp/tmp.xNj7q9Ozs7
++ cat /tmp/tmp.UMXEjaaYez
Error from server (NotFound): customresourcedefinitions.apiextensions.k8s.io "null" not found
++ rm /tmp/tmp.xNj7q9Ozs7 /tmp/tmp.UMXEjaaYez
++ return 1
+ [[ '' == \T\e\r\m\i\n\a\t\i\n\g ]]
+ '[' -n psmdb-operator ']'
+ create_namespace psmdb-operator
+ local namespace=psmdb-operator
+ local skip_clean_namespace=
+ [[ 1 == 1 ]]
+ [[ -z '' ]]
+ destroy_chaos_mesh
++ helm list --all-namespaces --filter chaos-mesh
++ tail -n1
++ awk '-F ' '{print $2}'
++ sed s/NAMESPACE//
+ local chaos_mesh_ns=
+ desc 'destroy chaos-mesh'
+ set +o xtrace


-----------------------------------------------------------------------------------
destroy chaos-mesh
-----------------------------------------------------------------------------------

+ '[' -n '' ']'
++ kubectl get MutatingWebhookConfiguration
++ grep chaos-mesh
++ awk '{print $1}'
+ timeout 30 kubectl delete MutatingWebhookConfiguration
error: resource(s) were provided, but no name was specified
+ :
++ kubectl get ValidatingWebhookConfiguration
++ grep chaos-mesh
++ awk '{print $1}'
+ timeout 30 kubectl delete ValidatingWebhookConfiguration
error: resource(s) were provided, but no name was specified
+ :
++ kubectl get ValidatingWebhookConfiguration
++ grep validate-auth
++ awk '{print $1}'
+ timeout 30 kubectl delete ValidatingWebhookConfiguration
error: resource(s) were provided, but no name was specified
+ :
++ kubectl api-resources
++ grep chaos-mesh
++ awk '{print $1}'
++ kubectl get crd
++ grep chaos-mesh.org
++ awk '{print $1}'
+ timeout 30 kubectl delete crd
error: resource(s) were provided, but no name was specified
+ :
++ kubectl get clusterrolebinding
++ grep chaos-mesh
++ awk '{print $1}'
+ timeout 30 kubectl delete clusterrolebinding
error: resource(s) were provided, but no name was specified
+ :
++ kubectl get clusterrole
++ grep chaos-mesh
++ awk '{print $1}'
+ timeout 30 kubectl delete clusterrole
error: resource(s) were provided, but no name was specified
+ :
+ desc 'cleaned up all old namespaces'
+ set +o xtrace


-----------------------------------------------------------------------------------
cleaned up all old namespaces
-----------------------------------------------------------------------------------

+ kubectl_bin get ns
+ egrep -v '^kube-|^default|Terminating|psmdb-operator|openshift|^gke-|^gmp-|^NAME'
+ awk '{print$1}'
+ '[' -n '' ']'
+ desc 'cleaned up old namespaces psmdb-operator'
+ set +o xtrace
+ xargs kubectl delete ns


-----------------------------------------------------------------------------------
cleaned up old namespaces psmdb-operator
-----------------------------------------------------------------------------------

+ kubectl_bin delete namespace psmdb-operator --ignore-not-found
++ mktemp
++ mktemp
egrep: warning: egrep is obsolescent; using grep -E
+ local LAST_OUT=/tmp/tmp.MvHwyBrLNw
+ local LAST_OUT=/tmp/tmp.tN8NwV9vN1
++ mktemp
++ mktemp
+ local LAST_ERR=/tmp/tmp.lQydzqrq6L
+ local exit_status=0
+ local timeout=4
+ local LAST_ERR=/tmp/tmp.0e0a7Lyb77
+ local exit_status=0
+ local timeout=4
++ seq 0 2
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl get ns
+ for i in $(seq 0 2)
+ set +e
+ kubectl delete namespace psmdb-operator --ignore-not-found
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.MvHwyBrLNw
+ cat /tmp/tmp.lQydzqrq6L
+ rm /tmp/tmp.MvHwyBrLNw /tmp/tmp.lQydzqrq6L
+ return 0
namespace "pitr-19442" deleted
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.tN8NwV9vN1
namespace "psmdb-operator" deleted
+ cat /tmp/tmp.0e0a7Lyb77
+ rm /tmp/tmp.tN8NwV9vN1 /tmp/tmp.0e0a7Lyb77
+ return 0
+ kubectl_bin wait --for=delete namespace psmdb-operator
++ mktemp
+ local LAST_OUT=/tmp/tmp.GzUYYADhLU
++ mktemp
+ local LAST_ERR=/tmp/tmp.mCn1bxjNxe
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl wait --for=delete namespace psmdb-operator
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.GzUYYADhLU
+ cat /tmp/tmp.mCn1bxjNxe
+ rm /tmp/tmp.GzUYYADhLU /tmp/tmp.mCn1bxjNxe
+ return 0
+ desc 'create namespace psmdb-operator'
+ set +o xtrace


-----------------------------------------------------------------------------------
create namespace psmdb-operator
-----------------------------------------------------------------------------------

+ kubectl_bin create namespace psmdb-operator
++ mktemp
+ local LAST_OUT=/tmp/tmp.fcPJGyyTjd
++ mktemp
+ local LAST_ERR=/tmp/tmp.N9IBgvkmSZ
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl create namespace psmdb-operator
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.fcPJGyyTjd
namespace/psmdb-operator created
+ cat /tmp/tmp.N9IBgvkmSZ
+ rm /tmp/tmp.fcPJGyyTjd /tmp/tmp.N9IBgvkmSZ
+ return 0
+ set_kube_ctx psmdb-operator
+ local namespace=psmdb-operator
++ kubectl_bin config current-context
+++ mktemp
++ local LAST_OUT=/tmp/tmp.Pd4Dn8AKTa
+++ mktemp
++ local LAST_ERR=/tmp/tmp.fynxXwhiCr
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl config current-context
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.Pd4Dn8AKTa
++ cat /tmp/tmp.fynxXwhiCr
++ rm /tmp/tmp.Pd4Dn8AKTa /tmp/tmp.fynxXwhiCr
++ return 0
+ kubectl_bin config set-context gke_cloud-dev-112233_us-central1-a_jen-psmdb-1850-e336896c-14-cluster4 --namespace=psmdb-operator
++ mktemp
+ local LAST_OUT=/tmp/tmp.UvkrWJE6ea
++ mktemp
+ local LAST_ERR=/tmp/tmp.rJtO1rmzrW
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl config set-context gke_cloud-dev-112233_us-central1-a_jen-psmdb-1850-e336896c-14-cluster4 --namespace=psmdb-operator
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.UvkrWJE6ea
Context "gke_cloud-dev-112233_us-central1-a_jen-psmdb-1850-e336896c-14-cluster4" modified.
+ cat /tmp/tmp.rJtO1rmzrW
+ rm /tmp/tmp.UvkrWJE6ea /tmp/tmp.rJtO1rmzrW
+ return 0
+ deploy_operator
+ desc 'start PSMDB operator: perconalab/percona-server-mongodb-operator:PR-1850-e336896c'
+ set +o xtrace


-----------------------------------------------------------------------------------
start PSMDB operator: perconalab/percona-server-mongodb-operator:PR-1850-e336896c
-----------------------------------------------------------------------------------

+ local cr_file
+ '[' -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1850/e2e-tests/pitr/conf/crd.yaml ']'
+ cr_file=/mnt/jenkins/workspace/cloud-psmdb-operator_PR-1850/deploy/crd.yaml
+ kubectl_bin apply --server-side --force-conflicts -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1850/deploy/crd.yaml
++ mktemp
+ local LAST_OUT=/tmp/tmp.Tl2V29XVzg
++ mktemp
+ local LAST_ERR=/tmp/tmp.WHGyKgPrw3
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl apply --server-side --force-conflicts -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1850/deploy/crd.yaml
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.Tl2V29XVzg
customresourcedefinition.apiextensions.k8s.io/perconaservermongodbbackups.psmdb.percona.com serverside-applied
customresourcedefinition.apiextensions.k8s.io/perconaservermongodbrestores.psmdb.percona.com serverside-applied
customresourcedefinition.apiextensions.k8s.io/perconaservermongodbs.psmdb.percona.com serverside-applied
+ cat /tmp/tmp.WHGyKgPrw3
+ rm /tmp/tmp.Tl2V29XVzg /tmp/tmp.WHGyKgPrw3
+ return 0
+ '[' -n psmdb-operator ']'
+ apply_rbac cw-rbac
+ local operator_namespace=psmdb-operator
+ local rbac=cw-rbac
+ sed -e 's^namespace: .*^namespace: psmdb-operator^'
+ cat /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1850/deploy/cw-rbac.yaml
+ kubectl_bin apply -n psmdb-operator -f -
++ mktemp
+ local LAST_OUT=/tmp/tmp.RzrIN8uKxT
++ mktemp
+ local LAST_ERR=/tmp/tmp.YZyTsyS6Kj
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl apply -n psmdb-operator -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.RzrIN8uKxT
clusterrole.rbac.authorization.k8s.io/percona-server-mongodb-operator created
serviceaccount/percona-server-mongodb-operator created
clusterrolebinding.rbac.authorization.k8s.io/service-account-percona-server-mongodb-operator created
+ cat /tmp/tmp.YZyTsyS6Kj
+ rm /tmp/tmp.RzrIN8uKxT /tmp/tmp.YZyTsyS6Kj
+ return 0
+ yq eval '
			(.spec.template.spec.containers[].image = "perconalab/percona-server-mongodb-operator:PR-1850-e336896c") |
			((.. | select(.[] == "DISABLE_TELEMETRY")) |= .value="true") |
			((.. | select(.[] == "LOG_LEVEL")) |= .value="DEBUG")' /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1850/deploy/cw-operator.yaml
+ kubectl_bin apply -n psmdb-operator -f -
++ mktemp
+ local LAST_OUT=/tmp/tmp.KqbZKP4s9p
++ mktemp
+ local LAST_ERR=/tmp/tmp.6G0L3ZWwri
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl apply -n psmdb-operator -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.KqbZKP4s9p
deployment.apps/percona-server-mongodb-operator created
+ cat /tmp/tmp.6G0L3ZWwri
+ rm /tmp/tmp.KqbZKP4s9p /tmp/tmp.6G0L3ZWwri
+ return 0
+ sleep 20
++ get_operator_pod
++ kubectl_bin get pods --selector=name=percona-server-mongodb-operator -o 'jsonpath={.items[].metadata.name}' -n psmdb-operator
+++ mktemp
++ local LAST_OUT=/tmp/tmp.EMh17Fn2gm
+++ mktemp
++ local LAST_ERR=/tmp/tmp.0rz2eGdbAJ
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=percona-server-mongodb-operator -o 'jsonpath={.items[].metadata.name}' -n psmdb-operator
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.EMh17Fn2gm
++ cat /tmp/tmp.0rz2eGdbAJ
++ rm /tmp/tmp.EMh17Fn2gm /tmp/tmp.0rz2eGdbAJ
++ return 0
+ wait_operator_pod percona-server-mongodb-operator-7dfb9458d-jhx8v
+ local pod=percona-server-mongodb-operator-7dfb9458d-jhx8v
+ set +o xtrace
waiting for pod/percona-server-mongodb-operator-7dfb9458d-jhx8v to be ready.OK
+ echo 'Print operator info from log'
Print operator info from log
+ grep 'Manager starting up'
++ get_operator_pod
++ kubectl_bin get pods --selector=name=percona-server-mongodb-operator -o 'jsonpath={.items[].metadata.name}' -n psmdb-operator
+++ mktemp
++ local LAST_OUT=/tmp/tmp.xsTdmkMUQ5
+++ mktemp
++ local LAST_ERR=/tmp/tmp.PZd4h7kxmi
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=percona-server-mongodb-operator -o 'jsonpath={.items[].metadata.name}' -n psmdb-operator
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.xsTdmkMUQ5
++ cat /tmp/tmp.PZd4h7kxmi
++ rm /tmp/tmp.xsTdmkMUQ5 /tmp/tmp.PZd4h7kxmi
++ return 0
+ kubectl_bin logs -n psmdb-operator percona-server-mongodb-operator-7dfb9458d-jhx8v
++ mktemp
+ local LAST_OUT=/tmp/tmp.ZdT9QZ7de9
++ mktemp
+ local LAST_ERR=/tmp/tmp.NGu8JAh7mV
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl logs -n psmdb-operator percona-server-mongodb-operator-7dfb9458d-jhx8v
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.ZdT9QZ7de9
+ cat /tmp/tmp.NGu8JAh7mV
+ rm /tmp/tmp.ZdT9QZ7de9 /tmp/tmp.NGu8JAh7mV
+ return 0
2025-11-30T20:07:34.556Z	INFO	setup	Manager starting up	{"gitCommit": "e336896c939a5ae7a170c12b1533a636fa716171", "gitBranch": "PR-1850-e336896c", "buildTime": "", "goVersion": "go1.25.4", "os": "linux", "arch": "amd64"}
+ create_namespace pitr-32014
+ local namespace=pitr-32014
+ local skip_clean_namespace=
+ [[ 1 == 1 ]]
+ [[ -z '' ]]
+ destroy_chaos_mesh
++ helm list --all-namespaces --filter chaos-mesh
++ tail -n1
++ awk '-F ' '{print $2}'
++ sed s/NAMESPACE//
+ local chaos_mesh_ns=
+ desc 'destroy chaos-mesh'
+ set +o xtrace


-----------------------------------------------------------------------------------
destroy chaos-mesh
-----------------------------------------------------------------------------------

+ '[' -n '' ']'
++ kubectl get MutatingWebhookConfiguration
++ grep chaos-mesh
++ awk '{print $1}'
+ timeout 30 kubectl delete MutatingWebhookConfiguration
error: resource(s) were provided, but no name was specified
+ :
++ kubectl get ValidatingWebhookConfiguration
++ grep chaos-mesh
++ awk '{print $1}'
+ timeout 30 kubectl delete ValidatingWebhookConfiguration
error: resource(s) were provided, but no name was specified
+ :
++ kubectl get ValidatingWebhookConfiguration
++ grep validate-auth
++ awk '{print $1}'
+ timeout 30 kubectl delete ValidatingWebhookConfiguration
error: resource(s) were provided, but no name was specified
+ :
++ kubectl api-resources
++ grep chaos-mesh
++ awk '{print $1}'
++ kubectl get crd
++ grep chaos-mesh.org
++ awk '{print $1}'
+ timeout 30 kubectl delete crd
error: resource(s) were provided, but no name was specified
+ :
++ kubectl get clusterrolebinding
++ grep chaos-mesh
++ awk '{print $1}'
+ timeout 30 kubectl delete clusterrolebinding
error: resource(s) were provided, but no name was specified
+ :
++ kubectl get clusterrole
++ grep chaos-mesh
++ awk '{print $1}'
+ timeout 30 kubectl delete clusterrole
error: resource(s) were provided, but no name was specified
+ :
+ desc 'cleaned up all old namespaces'
+ set +o xtrace


-----------------------------------------------------------------------------------
cleaned up all old namespaces
-----------------------------------------------------------------------------------

+ kubectl_bin get ns
+ egrep -v '^kube-|^default|Terminating|psmdb-operator|openshift|^gke-|^gmp-|^NAME'
++ mktemp
+ awk '{print$1}'
+ '[' -n '' ']'
+ desc 'cleaned up old namespaces pitr-32014'
+ set +o xtrace


-----------------------------------------------------------------------------------
cleaned up old namespaces pitr-32014
-----------------------------------------------------------------------------------

+ kubectl_bin delete namespace pitr-32014 --ignore-not-found
+ xargs kubectl delete ns
+ local LAST_OUT=/tmp/tmp.bimSEMcV8A
egrep: warning: egrep is obsolescent; using grep -E
++ mktemp
++ mktemp
+ local LAST_OUT=/tmp/tmp.31bQWBm3P3
+ local LAST_ERR=/tmp/tmp.c4j86FS4dG
+ local exit_status=0
+ local timeout=4
++ seq 0 2
++ mktemp
+ for i in $(seq 0 2)
+ set +e
+ kubectl get ns
+ local LAST_ERR=/tmp/tmp.mFvBE7QxeY
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl delete namespace pitr-32014 --ignore-not-found
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.bimSEMcV8A
+ cat /tmp/tmp.c4j86FS4dG
+ rm /tmp/tmp.bimSEMcV8A /tmp/tmp.c4j86FS4dG
+ return 0
error: resource(s) were provided, but no name was specified
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.31bQWBm3P3
+ cat /tmp/tmp.mFvBE7QxeY
+ rm /tmp/tmp.31bQWBm3P3 /tmp/tmp.mFvBE7QxeY
+ return 0
+ kubectl_bin wait --for=delete namespace pitr-32014
++ mktemp
+ local LAST_OUT=/tmp/tmp.BXJyZccVf5
++ mktemp
+ local LAST_ERR=/tmp/tmp.F6GSpvez1S
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl wait --for=delete namespace pitr-32014
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.BXJyZccVf5
+ cat /tmp/tmp.F6GSpvez1S
+ rm /tmp/tmp.BXJyZccVf5 /tmp/tmp.F6GSpvez1S
+ return 0
+ desc 'create namespace pitr-32014'
+ set +o xtrace


-----------------------------------------------------------------------------------
create namespace pitr-32014
-----------------------------------------------------------------------------------

+ kubectl_bin create namespace pitr-32014
++ mktemp
+ local LAST_OUT=/tmp/tmp.VtJjxi4WFW
++ mktemp
+ local LAST_ERR=/tmp/tmp.Zbbw2GsWI3
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl create namespace pitr-32014
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.VtJjxi4WFW
namespace/pitr-32014 created
+ cat /tmp/tmp.Zbbw2GsWI3
+ rm /tmp/tmp.VtJjxi4WFW /tmp/tmp.Zbbw2GsWI3
+ return 0
+ set_kube_ctx pitr-32014
+ local namespace=pitr-32014
++ kubectl_bin config current-context
+++ mktemp
++ local LAST_OUT=/tmp/tmp.VZfyQSbVJV
+++ mktemp
++ local LAST_ERR=/tmp/tmp.PSqyJbIys6
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl config current-context
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.VZfyQSbVJV
++ cat /tmp/tmp.PSqyJbIys6
++ rm /tmp/tmp.VZfyQSbVJV /tmp/tmp.PSqyJbIys6
++ return 0
+ kubectl_bin config set-context gke_cloud-dev-112233_us-central1-a_jen-psmdb-1850-e336896c-14-cluster4 --namespace=pitr-32014
++ mktemp
+ local LAST_OUT=/tmp/tmp.k05ywSUlfb
++ mktemp
+ local LAST_ERR=/tmp/tmp.onUBnJLoP4
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl config set-context gke_cloud-dev-112233_us-central1-a_jen-psmdb-1850-e336896c-14-cluster4 --namespace=pitr-32014
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.k05ywSUlfb
Context "gke_cloud-dev-112233_us-central1-a_jen-psmdb-1850-e336896c-14-cluster4" modified.
+ cat /tmp/tmp.onUBnJLoP4
+ rm /tmp/tmp.k05ywSUlfb /tmp/tmp.onUBnJLoP4
+ return 0
+ deploy_minio
+ desc 'install Minio'
+ set +o xtrace


-----------------------------------------------------------------------------------
install Minio
-----------------------------------------------------------------------------------

+ helm uninstall minio-service
Error: uninstall: Release not loaded: minio-service: release: not found
+ :
+ helm repo remove minio
"minio" has been removed from your repositories
+ helm repo add minio https://charts.min.io/
"minio" has been added to your repositories
+ retry 10 60 helm install minio-service --version 5.4.0 --set replicas=1 --set mode=standalone --set resources.requests.memory=256Mi --set rootUser=rootuser --set rootPassword=rootpass123 --set 'users[0].accessKey=some-access-key' --set 'users[0].secretKey=some-secret-key' --set 'users[0].policy=consoleAdmin' --set service.type=ClusterIP --set configPathmc=/tmp/.minio/ --set persistence.size=2G --set securityContext.enabled=false minio/minio
+ local max=10
+ local delay=60
+ shift 2
+ local n=1
+ helm install minio-service --version 5.4.0 --set replicas=1 --set mode=standalone --set resources.requests.memory=256Mi --set rootUser=rootuser --set rootPassword=rootpass123 --set 'users[0].accessKey=some-access-key' --set 'users[0].secretKey=some-secret-key' --set 'users[0].policy=consoleAdmin' --set service.type=ClusterIP --set configPathmc=/tmp/.minio/ --set persistence.size=2G --set securityContext.enabled=false minio/minio
NAME: minio-service
LAST DEPLOYED: Sun Nov 30 20:08:13 2025
NAMESPACE: pitr-32014
STATUS: deployed
REVISION: 1
TEST SUITE: None
NOTES:
MinIO can be accessed via port 9000 on the following DNS name from within your cluster:
minio-service.pitr-32014.cluster.local

To access MinIO from localhost, run the below commands:

  1. export POD_NAME=$(kubectl get pods --namespace pitr-32014 -l "release=minio-service" -o jsonpath="{.items[0].metadata.name}")

  2. kubectl port-forward $POD_NAME 9000 --namespace pitr-32014

Read more about port forwarding here: http://kubernetes.io/docs/user-guide/kubectl/kubectl_port-forward/

You can now access MinIO server on http://localhost:9000. Follow the below steps to connect to MinIO server with mc client:

  1. Download the MinIO mc client - https://min.io/docs/minio/linux/reference/minio-mc.html#quickstart

  2. export MC_HOST_minio-service-local=http://$(kubectl get secret --namespace pitr-32014 minio-service -o jsonpath="{.data.rootUser}" | base64 --decode):$(kubectl get secret --namespace pitr-32014 minio-service -o jsonpath="{.data.rootPassword}" | base64 --decode)@localhost:9000

  3. mc ls minio-service-local
++ kubectl_bin get pods --selector=release=minio-service -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.LOnOxtPVIb
+++ mktemp
++ local LAST_ERR=/tmp/tmp.aKTp2weMGb
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=release=minio-service -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.LOnOxtPVIb
++ cat /tmp/tmp.aKTp2weMGb
++ rm /tmp/tmp.LOnOxtPVIb /tmp/tmp.aKTp2weMGb
++ return 0
+ MINIO_POD=minio-service-d9589b474-4htd5
+ wait_pod minio-service-d9589b474-4htd5
+ local pod=minio-service-d9589b474-4htd5
+ set +o xtrace
waiting for pod/minio-service-d9589b474-4htd5 to be ready.OK
+ '[' -n psmdb-operator ']'
+ kubectl_bin create svc -n psmdb-operator externalname minio-service --external-name=minio-service.pitr-32014.svc.cluster.local --tcp=9000
++ mktemp
+ local LAST_OUT=/tmp/tmp.Tr80ONXyEL
++ mktemp
+ local LAST_ERR=/tmp/tmp.MuxFiCJScm
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl create svc -n psmdb-operator externalname minio-service --external-name=minio-service.pitr-32014.svc.cluster.local --tcp=9000
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.Tr80ONXyEL
service/minio-service created
+ cat /tmp/tmp.MuxFiCJScm
+ rm /tmp/tmp.Tr80ONXyEL /tmp/tmp.MuxFiCJScm
+ return 0
+ create_minio_bucket operator-testing
+ local bucket=operator-testing
+ kubectl_bin run -i --rm aws-cli --image=perconalab/awscli --restart=Never -- bash -c 'AWS_ACCESS_KEY_ID=some-access-key 		AWS_SECRET_ACCESS_KEY=some-secret-key 		AWS_DEFAULT_REGION=us-east-1         /usr/bin/aws --endpoint-url http://minio-service:9000 s3 mb s3://operator-testing'
++ mktemp
+ local LAST_OUT=/tmp/tmp.zgY8Y5ukuR
++ mktemp
+ local LAST_ERR=/tmp/tmp.oTVlR4XXXG
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl run -i --rm aws-cli --image=perconalab/awscli --restart=Never -- bash -c 'AWS_ACCESS_KEY_ID=some-access-key 		AWS_SECRET_ACCESS_KEY=some-secret-key 		AWS_DEFAULT_REGION=us-east-1         /usr/bin/aws --endpoint-url http://minio-service:9000 s3 mb s3://operator-testing'
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.zgY8Y5ukuR
make_bucket: operator-testing
pod "aws-cli" deleted from pitr-32014 namespace
+ cat /tmp/tmp.oTVlR4XXXG
+ rm /tmp/tmp.zgY8Y5ukuR /tmp/tmp.oTVlR4XXXG
+ return 0
+ desc 'create secrets and start client'
+ set +o xtrace


-----------------------------------------------------------------------------------
create secrets and start client
-----------------------------------------------------------------------------------

+ kubectl_bin apply -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1850/e2e-tests/conf/secrets.yml -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1850/e2e-tests/conf/client.yml -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1850/e2e-tests/conf/minio-secret.yml
++ mktemp
+ local LAST_OUT=/tmp/tmp.F4oK58owKC
++ mktemp
+ local LAST_ERR=/tmp/tmp.Zrt7iYL7j4
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl apply -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1850/e2e-tests/conf/secrets.yml -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1850/e2e-tests/conf/client.yml -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1850/e2e-tests/conf/minio-secret.yml
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.F4oK58owKC
secret/some-users created
deployment.apps/psmdb-client created
secret/minio-secret created
+ cat /tmp/tmp.Zrt7iYL7j4
+ rm /tmp/tmp.F4oK58owKC /tmp/tmp.Zrt7iYL7j4
+ return 0
+ cluster=some-name-rs0
+ desc 'create first PSMDB cluster some-name-rs0'
+ set +o xtrace


-----------------------------------------------------------------------------------
create first PSMDB cluster some-name-rs0
-----------------------------------------------------------------------------------

+ apply_cluster /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1850/e2e-tests/pitr/conf/some-name-rs0.yml
+ '[' -z '' ']'
+ cat_config /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1850/e2e-tests/pitr/conf/some-name-rs0.yml
+ kubectl_bin apply -f -
+ cat /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1850/e2e-tests/pitr/conf/some-name-rs0.yml
+ yq eval '(.spec | select(has("pmm"))).pmm.image = "percona/pmm-client:2.44.1-1"'
+ yq eval '(.spec | select(.image == null)).image = "perconalab/percona-server-mongodb-operator:main-mongod8.0"'
++ mktemp
+ yq eval '(.spec | select(has("initImage"))).initImage = "perconalab/percona-server-mongodb-operator:PR-1850-e336896c"'
+ yq eval '(.spec | select(has("backup"))).backup.image = "percona/percona-backup-mongodb:2.11.0"'
+ yq eval '.spec.upgradeOptions.apply="Never"'
+ local LAST_OUT=/tmp/tmp.Cdiw1mNri2
++ mktemp
+ local LAST_ERR=/tmp/tmp.t9hIjz4g6l
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl apply -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.Cdiw1mNri2
perconaservermongodb.psmdb.percona.com/some-name created
+ cat /tmp/tmp.t9hIjz4g6l
+ rm /tmp/tmp.Cdiw1mNri2 /tmp/tmp.t9hIjz4g6l
+ return 0
+ desc 'check if all 3 Pods started'
+ set +o xtrace


-----------------------------------------------------------------------------------
check if all 3 Pods started
-----------------------------------------------------------------------------------

+ wait_for_running some-name-rs0 3
+ local name=some-name-rs0
+ let last_pod=2
+ local check_cluster_readyness=true
+ set_debug
+ [[ 1 == 1 ]]
+ set -o xtrace
+ local rs_name=rs0
+ local cluster_name=some-name
++ seq 0 2
+ for i in $(seq 0 $last_pod)
+ [[ 0 -eq 2 ]]
+ wait_pod some-name-rs0-0
+ local pod=some-name-rs0-0
+ set +o xtrace
waiting for pod/some-name-rs0-0 to be ready........OK
+ for i in $(seq 0 $last_pod)
+ [[ 1 -eq 2 ]]
+ wait_pod some-name-rs0-1
+ local pod=some-name-rs0-1
+ set +o xtrace
waiting for pod/some-name-rs0-1 to be ready.......OK
+ for i in $(seq 0 $last_pod)
+ [[ 2 -eq 2 ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].arbiter.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.Jk3Jyw1LtP
+++ mktemp
++ local LAST_ERR=/tmp/tmp.4q3Mj9vN2M
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].arbiter.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.Jk3Jyw1LtP
++ cat /tmp/tmp.4q3Mj9vN2M
++ rm /tmp/tmp.Jk3Jyw1LtP /tmp/tmp.4q3Mj9vN2M
++ return 0
+ [[ '' == \t\r\u\e ]]
+ wait_pod some-name-rs0-2
+ local pod=some-name-rs0-2
+ set +o xtrace
waiting for pod/some-name-rs0-2 to be ready.....OK
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].nonvoting.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.9UE4Ecf0NU
+++ mktemp
++ local LAST_ERR=/tmp/tmp.HF8R3gitEA
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].nonvoting.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.9UE4Ecf0NU
++ cat /tmp/tmp.HF8R3gitEA
++ rm /tmp/tmp.9UE4Ecf0NU /tmp/tmp.HF8R3gitEA
++ return 0
+ [[ '' == \t\r\u\e ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].hidden.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.FuobmFFPTE
+++ mktemp
++ local LAST_ERR=/tmp/tmp.iLOkXmp8HY
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].hidden.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.FuobmFFPTE
++ cat /tmp/tmp.iLOkXmp8HY
++ rm /tmp/tmp.FuobmFFPTE /tmp/tmp.iLOkXmp8HY
++ return 0
+ [[ '' == \t\r\u\e ]]
+ sleep 10
+ [[ true == \t\r\u\e ]]
+ set +x
Waiting for cluster readyness..
+ desc 'check if statefulset created with expected config'
+ set +o xtrace


-----------------------------------------------------------------------------------
check if statefulset created with expected config
-----------------------------------------------------------------------------------

+ compare_kubectl statefulset/some-name-rs0
+ local resource=statefulset/some-name-rs0
+ local postfix=
+ local skip_generation_check=
+ local expected_result=/mnt/jenkins/workspace/cloud-psmdb-operator_PR-1850/e2e-tests/pitr/compare/statefulset_some-name-rs0.yml
+ local new_result=/tmp/tmp.1qPVwubFaS/statefulset_some-name-rs0.yml
+ '[' -n '' -a -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1850/e2e-tests/pitr/compare/statefulset_some-name-rs0-oc.yml ']'
+ kubectl_bin get -o yaml statefulset/some-name-rs0
+ yq eval '
			del(.metadata.ownerReferences[].apiVersion) |
			del(.metadata.managedFields) |
			del(.. | select(has("creationTimestamp")).creationTimestamp) |
			del(.. | select(has("namespace")).namespace) |
			del(.. | select(has("uid")).uid) |
			del(.metadata.resourceVersion) |
			del(.spec.template.spec.containers[].env[] | select(.name == "NAMESPACE")) |
			del(.metadata.selfLink) |
			del(.metadata.annotations."cloud.google.com/neg") |
			del(.metadata.annotations."kubectl.kubernetes.io/last-applied-configuration") |
			del(.. | select(has("image")).image) |
			del(.. | select(has("clusterIP")).clusterIP) |
			del(.. | select(has("clusterIPs")).clusterIPs) |
			del(.. | select(has("dataSource")).dataSource) |
			del(.. | select(has("procMount")).procMount) |
			del(.. | select(has("storageClassName")).storageClassName) |
			del(.. | select(has("finalizers")).finalizers) |
			del(.. | select(has("kubernetes.io/pvc-protection"))."kubernetes.io/pvc-protection") |
			del(.. | select(has("volumeName")).volumeName) |
			del(.. | select(has("volume.beta.kubernetes.io/storage-provisioner"))."volume.beta.kubernetes.io/storage-provisioner") |
			del(.. | select(has("volume.kubernetes.io/storage-provisioner"))."volume.kubernetes.io/storage-provisioner") |
			del(.spec.volumeMode) |
			del(.. | select(has("volume.kubernetes.io/selected-node"))."volume.kubernetes.io/selected-node") |
			del(.. | select(has("percona.com/last-config-hash"))."percona.com/last-config-hash") |
			del(.. | select(has("percona.com/configuration-hash"))."percona.com/configuration-hash") |
			del(.. | select(has("percona.com/ssl-hash"))."percona.com/ssl-hash") |
			del(.. | select(has("percona.com/ssl-internal-hash"))."percona.com/ssl-internal-hash") |
			del(.spec.volumeClaimTemplates[].spec.volumeMode | select(. == "Filesystem")) |
			del(.. | select(has("healthCheckNodePort")).healthCheckNodePort) |
			del(.. | select(has("nodePort")).nodePort) |
			del(.status) |
			(.. | select(tag == "!!str")) |= sub("pitr-32014", "NAME_SPACE") |
			del(.spec.volumeClaimTemplates[].apiVersion) |
			del(.spec.volumeClaimTemplates[].kind) |
			del(.spec.ipFamilies) |
			del(.spec.ipFamilyPolicy) |
			(.. | select(. == "extensions/v1beta1")) = "apps/v1" |
			(.. | select(. == "batch/v1beta1")) = "batch/v1" ' -
++ mktemp
+ local LAST_OUT=/tmp/tmp.rBBv3khS89
++ mktemp
+ local LAST_ERR=/tmp/tmp.0KZVlcSHwK
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl get -o yaml statefulset/some-name-rs0
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.rBBv3khS89
+ cat /tmp/tmp.0KZVlcSHwK
+ rm /tmp/tmp.rBBv3khS89 /tmp/tmp.0KZVlcSHwK
+ return 0
+ yq -i eval 'del(.spec.persistentVolumeClaimRetentionPolicy)' /tmp/tmp.1qPVwubFaS/statefulset_some-name-rs0.yml
+ version_gt 1.22
++ echo '1.31 >= 1.22'
++ bc -l
+ '[' 1 -eq 1 ']'
+ return 0
+ yq -i eval 'del(.spec.internalTrafficPolicy)' /tmp/tmp.1qPVwubFaS/statefulset_some-name-rs0.yml
+ yq -i eval 'del(.spec.allocateLoadBalancerNodePorts)' /tmp/tmp.1qPVwubFaS/statefulset_some-name-rs0.yml
+ [[ /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1850/e2e-tests/pitr/compare/statefulset_some-name-rs0.yml == */cronjob* ]]
+ '[' -n '' ']'
+ [[ 0 -eq 0 ]]
+ diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1850/e2e-tests/pitr/compare/statefulset_some-name-rs0.yml /tmp/tmp.1qPVwubFaS/statefulset_some-name-rs0.yml
+ log 'compare_kubectl: statefulset/some-name-rs0 OK'
+ set +o xtrace
[2025-11-30T20:10:24+0000] compare_kubectl: statefulset/some-name-rs0 OK
+ write_initial_data
+ desc 'create user myApp'
+ set +o xtrace


-----------------------------------------------------------------------------------
create user myApp
-----------------------------------------------------------------------------------

+ run_mongo 'db.createUser({user:"myApp",pwd:"myPass",roles:[{db:"myApp",role:"readWrite"}]})' userAdmin:userAdmin123456@some-name-rs0.pitr-32014
+ local 'command=db.createUser({user:"myApp",pwd:"myPass",roles:[{db:"myApp",role:"readWrite"}]})'
+ local uri=userAdmin:userAdmin123456@some-name-rs0.pitr-32014
+ local driver=mongodb+srv
+ local suffix=.svc.cluster.local
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.4vJkpTrwKZ
+++ mktemp
++ local LAST_ERR=/tmp/tmp.nrAwt2NQcS
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.4vJkpTrwKZ
++ cat /tmp/tmp.nrAwt2NQcS
++ rm /tmp/tmp.4vJkpTrwKZ /tmp/tmp.nrAwt2NQcS
++ return 0
+ local client_container=psmdb-client-696897d69b-whdv5
+ local mongo_flag=
+ [[ userAdmin:userAdmin123456@some-name-rs0.pitr-32014 == *cfg* ]]
+ replica_set=rs0
+ kubectl_bin exec psmdb-client-696897d69b-whdv5 -- bash -c 'printf '\''db.createUser({user:"myApp",pwd:"myPass",roles:[{db:"myApp",role:"readWrite"}]})\n'\'' | mongo mongodb+srv://userAdmin:userAdmin123456@some-name-rs0.pitr-32014.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
++ mktemp
+ local LAST_OUT=/tmp/tmp.mIxuLheHYM
++ mktemp
+ local LAST_ERR=/tmp/tmp.VwbQ1n8FPw
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-696897d69b-whdv5 -- bash -c 'printf '\''db.createUser({user:"myApp",pwd:"myPass",roles:[{db:"myApp",role:"readWrite"}]})\n'\'' | mongo mongodb+srv://userAdmin:userAdmin123456@some-name-rs0.pitr-32014.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.mIxuLheHYM
Percona Server for MongoDB shell version v4.4.29-28
connecting to: mongodb://some-name-rs0-2.some-name-rs0.pitr-32014.svc.cluster.local:27017,some-name-rs0-0.some-name-rs0.pitr-32014.svc.cluster.local:27017,some-name-rs0-1.some-name-rs0.pitr-32014.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb&replicaSet=rs0&ssl=false
Implicit session: session { "id" : UUID("7557e333-b9a5-4800-971d-6bb0117fe125") }
Percona Server for MongoDB server version: v8.0.12-4
WARNING: shell and server versions do not match
Successfully added user: {
	"user" : "myApp",
	"roles" : [
		{
			"db" : "myApp",
			"role" : "readWrite"
		}
	]
}
bye
+ cat /tmp/tmp.VwbQ1n8FPw
+ rm /tmp/tmp.mIxuLheHYM /tmp/tmp.VwbQ1n8FPw
+ return 0
+ sleep 2
+ write_document
+ local cmp_postfix=
+ local cluster_name=some-name-rs0
+ desc 'write initial data, read from all'
+ set +o xtrace


-----------------------------------------------------------------------------------
write initial data, read from all
-----------------------------------------------------------------------------------

+ run_mongo 'use myApp\n db.test.insert({ x: 100500 })' myApp:myPass@some-name-rs0.pitr-32014
+ local 'command=use myApp\n db.test.insert({ x: 100500 })'
+ local uri=myApp:myPass@some-name-rs0.pitr-32014
+ local driver=mongodb+srv
+ local suffix=.svc.cluster.local
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.IpV4QiJtHn
+++ mktemp
++ local LAST_ERR=/tmp/tmp.tVT2NXRPtR
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.IpV4QiJtHn
++ cat /tmp/tmp.tVT2NXRPtR
++ rm /tmp/tmp.IpV4QiJtHn /tmp/tmp.tVT2NXRPtR
++ return 0
+ local client_container=psmdb-client-696897d69b-whdv5
+ local mongo_flag=
+ [[ myApp:myPass@some-name-rs0.pitr-32014 == *cfg* ]]
+ replica_set=rs0
+ kubectl_bin exec psmdb-client-696897d69b-whdv5 -- bash -c 'printf '\''use myApp\n db.test.insert({ x: 100500 })\n'\'' | mongo mongodb+srv://myApp:myPass@some-name-rs0.pitr-32014.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
++ mktemp
+ local LAST_OUT=/tmp/tmp.FRXnHYCjlF
++ mktemp
+ local LAST_ERR=/tmp/tmp.1AvkuLNwlg
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-696897d69b-whdv5 -- bash -c 'printf '\''use myApp\n db.test.insert({ x: 100500 })\n'\'' | mongo mongodb+srv://myApp:myPass@some-name-rs0.pitr-32014.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.FRXnHYCjlF
Percona Server for MongoDB shell version v4.4.29-28
connecting to: mongodb://some-name-rs0-2.some-name-rs0.pitr-32014.svc.cluster.local:27017,some-name-rs0-0.some-name-rs0.pitr-32014.svc.cluster.local:27017,some-name-rs0-1.some-name-rs0.pitr-32014.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb&replicaSet=rs0&ssl=false
Implicit session: session { "id" : UUID("3d7ce75f-ce80-4cb4-827f-ac2edbb0641d") }
Percona Server for MongoDB server version: v8.0.12-4
WARNING: shell and server versions do not match
switched to db myApp
WriteResult({ "nInserted" : 1 })
bye
+ cat /tmp/tmp.1AvkuLNwlg
+ rm /tmp/tmp.FRXnHYCjlF /tmp/tmp.1AvkuLNwlg
+ return 0
+ minikube_sleep
+ sleep_time=10
+ [[ '' == 1 ]]
+ compare_mongo_cmd find myApp:myPass@some-name-rs0-0.some-name-rs0.pitr-32014 ''
+ local command=find
+ local uri=myApp:myPass@some-name-rs0-0.some-name-rs0.pitr-32014
+ local postfix=
+ local suffix=
+ local database=myApp
+ local collection=test
+ local sort=
+ local tls=false
+ local 'full_command=db.test.find()'
+ [[ -n '' ]]
+ log 'running db.test.find() in myApp'
+ set +o xtrace
[2025-11-30T20:10:31+0000] running db.test.find() in myApp
+ [[ false == \t\r\u\e ]]
+ mongo_command=run_mongo
+ run_mongo 'use myApp\n db.test.find()' myApp:myPass@some-name-rs0-0.some-name-rs0.pitr-32014 mongodb ''
+ local 'command=use myApp\n db.test.find()'
+ local uri=myApp:myPass@some-name-rs0-0.some-name-rs0.pitr-32014
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ egrep -v 'I NETWORK|W NETWORK|F NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|Started a new thread for the timer service'
+ /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.yQtAqoBx1I
egrep: warning: egrep is obsolescent; using grep -E
+++ mktemp
++ local LAST_ERR=/tmp/tmp.NjcRVhyesB
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.yQtAqoBx1I
++ cat /tmp/tmp.NjcRVhyesB
++ rm /tmp/tmp.yQtAqoBx1I /tmp/tmp.NjcRVhyesB
++ return 0
+ local client_container=psmdb-client-696897d69b-whdv5
+ local mongo_flag=
+ [[ myApp:myPass@some-name-rs0-0.some-name-rs0.pitr-32014 == *cfg* ]]
+ replica_set=rs0
+ kubectl_bin exec psmdb-client-696897d69b-whdv5 -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-0.some-name-rs0.pitr-32014.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
++ mktemp
+ local LAST_OUT=/tmp/tmp.vPB8Hbkh2d
++ mktemp
+ local LAST_ERR=/tmp/tmp.h7STxMI0m8
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-696897d69b-whdv5 -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-0.some-name-rs0.pitr-32014.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.vPB8Hbkh2d
+ cat /tmp/tmp.h7STxMI0m8
+ rm /tmp/tmp.vPB8Hbkh2d /tmp/tmp.h7STxMI0m8
+ return 0
+ [[ 0 -eq 0 ]]
+ diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1850/e2e-tests/pitr/compare/find.json /tmp/tmp.1qPVwubFaS/find
+ compare_mongo_cmd find myApp:myPass@some-name-rs0-1.some-name-rs0.pitr-32014 ''
+ local command=find
+ local uri=myApp:myPass@some-name-rs0-1.some-name-rs0.pitr-32014
+ local postfix=
+ local suffix=
+ local database=myApp
+ local collection=test
+ local sort=
+ local tls=false
+ local 'full_command=db.test.find()'
+ [[ -n '' ]]
+ log 'running db.test.find() in myApp'
+ set +o xtrace
[2025-11-30T20:10:34+0000] running db.test.find() in myApp
+ [[ false == \t\r\u\e ]]
+ mongo_command=run_mongo
+ run_mongo 'use myApp\n db.test.find()' myApp:myPass@some-name-rs0-1.some-name-rs0.pitr-32014 mongodb ''
+ local 'command=use myApp\n db.test.find()'
+ local uri=myApp:myPass@some-name-rs0-1.some-name-rs0.pitr-32014
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ egrep -v 'I NETWORK|W NETWORK|F NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|Started a new thread for the timer service'
+ /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.yD5SBbGB8w
egrep: warning: egrep is obsolescent; using grep -E
+++ mktemp
++ local LAST_ERR=/tmp/tmp.EtXLcOo7cu
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.yD5SBbGB8w
++ cat /tmp/tmp.EtXLcOo7cu
++ rm /tmp/tmp.yD5SBbGB8w /tmp/tmp.EtXLcOo7cu
++ return 0
+ local client_container=psmdb-client-696897d69b-whdv5
+ local mongo_flag=
+ [[ myApp:myPass@some-name-rs0-1.some-name-rs0.pitr-32014 == *cfg* ]]
+ replica_set=rs0
+ kubectl_bin exec psmdb-client-696897d69b-whdv5 -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-1.some-name-rs0.pitr-32014.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
++ mktemp
+ local LAST_OUT=/tmp/tmp.9BKg5VJdbj
++ mktemp
+ local LAST_ERR=/tmp/tmp.jcoSM0wg2i
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-696897d69b-whdv5 -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-1.some-name-rs0.pitr-32014.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.9BKg5VJdbj
+ cat /tmp/tmp.jcoSM0wg2i
+ rm /tmp/tmp.9BKg5VJdbj /tmp/tmp.jcoSM0wg2i
+ return 0
+ [[ 0 -eq 0 ]]
+ diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1850/e2e-tests/pitr/compare/find.json /tmp/tmp.1qPVwubFaS/find
+ compare_mongo_cmd find myApp:myPass@some-name-rs0-2.some-name-rs0.pitr-32014 ''
+ local command=find
+ local uri=myApp:myPass@some-name-rs0-2.some-name-rs0.pitr-32014
+ local postfix=
+ local suffix=
+ local database=myApp
+ local collection=test
+ local sort=
+ local tls=false
+ local 'full_command=db.test.find()'
+ [[ -n '' ]]
+ log 'running db.test.find() in myApp'
+ set +o xtrace
[2025-11-30T20:10:37+0000] running db.test.find() in myApp
+ [[ false == \t\r\u\e ]]
+ mongo_command=run_mongo
+ run_mongo 'use myApp\n db.test.find()' myApp:myPass@some-name-rs0-2.some-name-rs0.pitr-32014 mongodb ''
+ local 'command=use myApp\n db.test.find()'
+ local uri=myApp:myPass@some-name-rs0-2.some-name-rs0.pitr-32014
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ egrep -v 'I NETWORK|W NETWORK|F NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|Started a new thread for the timer service'
+ /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
egrep: warning: egrep is obsolescent; using grep -E
++ local LAST_OUT=/tmp/tmp.kzORI2oMWm
+++ mktemp
++ local LAST_ERR=/tmp/tmp.RXeBwS8X3F
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.kzORI2oMWm
++ cat /tmp/tmp.RXeBwS8X3F
++ rm /tmp/tmp.kzORI2oMWm /tmp/tmp.RXeBwS8X3F
++ return 0
+ local client_container=psmdb-client-696897d69b-whdv5
+ local mongo_flag=
+ [[ myApp:myPass@some-name-rs0-2.some-name-rs0.pitr-32014 == *cfg* ]]
+ replica_set=rs0
+ kubectl_bin exec psmdb-client-696897d69b-whdv5 -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-2.some-name-rs0.pitr-32014.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
++ mktemp
+ local LAST_OUT=/tmp/tmp.3SpTZAxi4F
++ mktemp
+ local LAST_ERR=/tmp/tmp.AfIdNqsEg4
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-696897d69b-whdv5 -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-2.some-name-rs0.pitr-32014.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.3SpTZAxi4F
+ cat /tmp/tmp.AfIdNqsEg4
+ rm /tmp/tmp.3SpTZAxi4F /tmp/tmp.AfIdNqsEg4
+ return 0
+ [[ 0 -eq 0 ]]
+ diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1850/e2e-tests/pitr/compare/find.json /tmp/tmp.1qPVwubFaS/find
+ wait_backup_agent some-name-rs0-0
+ local agent_pod=some-name-rs0-0
+ set +o xtrace
waiting for pbm-agent to be ready in some-name-rs0-0...2025-11-30T20:10:18.000+0000 I listening for the commands

+ wait_backup_agent some-name-rs0-1
+ local agent_pod=some-name-rs0-1
+ set +o xtrace
waiting for pbm-agent to be ready in some-name-rs0-1...2025-11-30T20:10:22.000+0000 I listening for the commands

+ wait_backup_agent some-name-rs0-2
+ local agent_pod=some-name-rs0-2
+ set +o xtrace
waiting for pbm-agent to be ready in some-name-rs0-2...2025-11-30T20:10:24.000+0000 I listening for the commands

+ backup_name_minio=backup-minio
+ run_mongo 'use myApp\n db.test2.insert({ x: 100501 })' myApp:myPass@some-name-rs0.pitr-32014
+ local 'command=use myApp\n db.test2.insert({ x: 100501 })'
+ local uri=myApp:myPass@some-name-rs0.pitr-32014
+ local driver=mongodb+srv
+ local suffix=.svc.cluster.local
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.fsmaK70eFC
+++ mktemp
++ local LAST_ERR=/tmp/tmp.C8OoxeFfqh
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.fsmaK70eFC
++ cat /tmp/tmp.C8OoxeFfqh
++ rm /tmp/tmp.fsmaK70eFC /tmp/tmp.C8OoxeFfqh
++ return 0
+ local client_container=psmdb-client-696897d69b-whdv5
+ local mongo_flag=
+ [[ myApp:myPass@some-name-rs0.pitr-32014 == *cfg* ]]
+ replica_set=rs0
+ kubectl_bin exec psmdb-client-696897d69b-whdv5 -- bash -c 'printf '\''use myApp\n db.test2.insert({ x: 100501 })\n'\'' | mongo mongodb+srv://myApp:myPass@some-name-rs0.pitr-32014.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
++ mktemp
+ local LAST_OUT=/tmp/tmp.c03cvx91sH
++ mktemp
+ local LAST_ERR=/tmp/tmp.mYvVCufgvH
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-696897d69b-whdv5 -- bash -c 'printf '\''use myApp\n db.test2.insert({ x: 100501 })\n'\'' | mongo mongodb+srv://myApp:myPass@some-name-rs0.pitr-32014.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.c03cvx91sH
Percona Server for MongoDB shell version v4.4.29-28
connecting to: mongodb://some-name-rs0-1.some-name-rs0.pitr-32014.svc.cluster.local:27017,some-name-rs0-2.some-name-rs0.pitr-32014.svc.cluster.local:27017,some-name-rs0-0.some-name-rs0.pitr-32014.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb&replicaSet=rs0&ssl=false
Implicit session: session { "id" : UUID("67bef656-59d6-4662-94bf-95882c805f32") }
Percona Server for MongoDB server version: v8.0.12-4
WARNING: shell and server versions do not match
switched to db myApp
WriteResult({ "nInserted" : 1 })
bye
+ cat /tmp/tmp.mYvVCufgvH
+ rm /tmp/tmp.c03cvx91sH /tmp/tmp.mYvVCufgvH
+ return 0
+ run_backup backup-minio 0
+ local name=backup-minio
+ local idx=0
+ desc 'run backup backup-minio-0'
+ set +o xtrace


-----------------------------------------------------------------------------------
run backup backup-minio-0
-----------------------------------------------------------------------------------

+ cat /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1850/e2e-tests/pitr/conf/backup-minio.yml
+ /usr/sbin/sed -e 's/name:/name: backup-minio-0/'
+ kubectl_bin apply -f -
++ mktemp
+ local LAST_OUT=/tmp/tmp.lyKRbl5R1s
++ mktemp
+ local LAST_ERR=/tmp/tmp.6tAwL2pbz0
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl apply -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.lyKRbl5R1s
perconaservermongodbbackup.psmdb.percona.com/backup-minio-0 created
+ cat /tmp/tmp.6tAwL2pbz0
+ rm /tmp/tmp.lyKRbl5R1s /tmp/tmp.6tAwL2pbz0
+ return 0
+ wait_backup backup-minio-0
+ local backup_name=backup-minio-0
+ local target_state=ready
+ set +o xtrace
waiting for backup-minio-0 to reach ready state.......OK
+ write_document -2nd
+ local cmp_postfix=-2nd
+ local cluster_name=some-name-rs0
+ desc 'write initial data, read from all'
+ set +o xtrace


-----------------------------------------------------------------------------------
write initial data, read from all
-----------------------------------------------------------------------------------

+ run_mongo 'use myApp\n db.test.insert({ x: 100500 })' myApp:myPass@some-name-rs0.pitr-32014
+ local 'command=use myApp\n db.test.insert({ x: 100500 })'
+ local uri=myApp:myPass@some-name-rs0.pitr-32014
+ local driver=mongodb+srv
+ local suffix=.svc.cluster.local
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.9ftHahHXmK
+++ mktemp
++ local LAST_ERR=/tmp/tmp.PuAqT2TAxL
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.9ftHahHXmK
++ cat /tmp/tmp.PuAqT2TAxL
++ rm /tmp/tmp.9ftHahHXmK /tmp/tmp.PuAqT2TAxL
++ return 0
+ local client_container=psmdb-client-696897d69b-whdv5
+ local mongo_flag=
+ [[ myApp:myPass@some-name-rs0.pitr-32014 == *cfg* ]]
+ replica_set=rs0
+ kubectl_bin exec psmdb-client-696897d69b-whdv5 -- bash -c 'printf '\''use myApp\n db.test.insert({ x: 100500 })\n'\'' | mongo mongodb+srv://myApp:myPass@some-name-rs0.pitr-32014.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
++ mktemp
+ local LAST_OUT=/tmp/tmp.jJupLID4Us
++ mktemp
+ local LAST_ERR=/tmp/tmp.zipN4uAHjc
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-696897d69b-whdv5 -- bash -c 'printf '\''use myApp\n db.test.insert({ x: 100500 })\n'\'' | mongo mongodb+srv://myApp:myPass@some-name-rs0.pitr-32014.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.jJupLID4Us
Percona Server for MongoDB shell version v4.4.29-28
connecting to: mongodb://some-name-rs0-2.some-name-rs0.pitr-32014.svc.cluster.local:27017,some-name-rs0-0.some-name-rs0.pitr-32014.svc.cluster.local:27017,some-name-rs0-1.some-name-rs0.pitr-32014.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb&replicaSet=rs0&ssl=false
Implicit session: session { "id" : UUID("3fba7322-312b-4861-a556-e903d04d90ba") }
Percona Server for MongoDB server version: v8.0.12-4
WARNING: shell and server versions do not match
switched to db myApp
WriteResult({ "nInserted" : 1 })
bye
+ cat /tmp/tmp.zipN4uAHjc
+ rm /tmp/tmp.jJupLID4Us /tmp/tmp.zipN4uAHjc
+ return 0
+ minikube_sleep
+ sleep_time=10
+ [[ '' == 1 ]]
+ compare_mongo_cmd find myApp:myPass@some-name-rs0-0.some-name-rs0.pitr-32014 -2nd
+ local command=find
+ local uri=myApp:myPass@some-name-rs0-0.some-name-rs0.pitr-32014
+ local postfix=-2nd
+ local suffix=
+ local database=myApp
+ local collection=test
+ local sort=
+ local tls=false
+ local 'full_command=db.test.find()'
+ [[ -n '' ]]
+ log 'running db.test.find() in myApp'
+ set +o xtrace
[2025-11-30T20:11:04+0000] running db.test.find() in myApp
+ [[ false == \t\r\u\e ]]
+ mongo_command=run_mongo
+ run_mongo 'use myApp\n db.test.find()' myApp:myPass@some-name-rs0-0.some-name-rs0.pitr-32014 mongodb ''
+ local 'command=use myApp\n db.test.find()'
+ local uri=myApp:myPass@some-name-rs0-0.some-name-rs0.pitr-32014
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ egrep -v 'I NETWORK|W NETWORK|F NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|Started a new thread for the timer service'
+ /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
egrep: warning: egrep is obsolescent; using grep -E
++ local LAST_OUT=/tmp/tmp.OX7BI2tY0d
+++ mktemp
++ local LAST_ERR=/tmp/tmp.g9eISmXeLX
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.OX7BI2tY0d
++ cat /tmp/tmp.g9eISmXeLX
++ rm /tmp/tmp.OX7BI2tY0d /tmp/tmp.g9eISmXeLX
++ return 0
+ local client_container=psmdb-client-696897d69b-whdv5
+ local mongo_flag=
+ [[ myApp:myPass@some-name-rs0-0.some-name-rs0.pitr-32014 == *cfg* ]]
+ replica_set=rs0
+ kubectl_bin exec psmdb-client-696897d69b-whdv5 -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-0.some-name-rs0.pitr-32014.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
++ mktemp
+ local LAST_OUT=/tmp/tmp.EJmPljzqpq
++ mktemp
+ local LAST_ERR=/tmp/tmp.W98lVDOW40
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-696897d69b-whdv5 -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-0.some-name-rs0.pitr-32014.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.EJmPljzqpq
+ cat /tmp/tmp.W98lVDOW40
+ rm /tmp/tmp.EJmPljzqpq /tmp/tmp.W98lVDOW40
+ return 0
+ [[ 0 -eq 0 ]]
+ diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1850/e2e-tests/pitr/compare/find-2nd.json /tmp/tmp.1qPVwubFaS/find-2nd
+ compare_mongo_cmd find myApp:myPass@some-name-rs0-1.some-name-rs0.pitr-32014 -2nd
+ local command=find
+ local uri=myApp:myPass@some-name-rs0-1.some-name-rs0.pitr-32014
+ local postfix=-2nd
+ local suffix=
+ local database=myApp
+ local collection=test
+ local sort=
+ local tls=false
+ local 'full_command=db.test.find()'
+ [[ -n '' ]]
+ log 'running db.test.find() in myApp'
+ set +o xtrace
[2025-11-30T20:11:06+0000] running db.test.find() in myApp
+ [[ false == \t\r\u\e ]]
+ mongo_command=run_mongo
+ run_mongo 'use myApp\n db.test.find()' myApp:myPass@some-name-rs0-1.some-name-rs0.pitr-32014 mongodb ''
+ egrep -v 'I NETWORK|W NETWORK|F NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|Started a new thread for the timer service'
+ local 'command=use myApp\n db.test.find()'
+ local uri=myApp:myPass@some-name-rs0-1.some-name-rs0.pitr-32014
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
egrep: warning: egrep is obsolescent; using grep -E
++ local LAST_OUT=/tmp/tmp.YhVZIkLrGJ
+++ mktemp
++ local LAST_ERR=/tmp/tmp.ffrFBjVIsx
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.YhVZIkLrGJ
++ cat /tmp/tmp.ffrFBjVIsx
++ rm /tmp/tmp.YhVZIkLrGJ /tmp/tmp.ffrFBjVIsx
++ return 0
+ local client_container=psmdb-client-696897d69b-whdv5
+ local mongo_flag=
+ [[ myApp:myPass@some-name-rs0-1.some-name-rs0.pitr-32014 == *cfg* ]]
+ replica_set=rs0
+ kubectl_bin exec psmdb-client-696897d69b-whdv5 -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-1.some-name-rs0.pitr-32014.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
++ mktemp
+ local LAST_OUT=/tmp/tmp.XujHLdKtA0
++ mktemp
+ local LAST_ERR=/tmp/tmp.Gr3QJhjVml
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-696897d69b-whdv5 -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-1.some-name-rs0.pitr-32014.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.XujHLdKtA0
+ cat /tmp/tmp.Gr3QJhjVml
+ rm /tmp/tmp.XujHLdKtA0 /tmp/tmp.Gr3QJhjVml
+ return 0
+ [[ 0 -eq 0 ]]
+ diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1850/e2e-tests/pitr/compare/find-2nd.json /tmp/tmp.1qPVwubFaS/find-2nd
+ compare_mongo_cmd find myApp:myPass@some-name-rs0-2.some-name-rs0.pitr-32014 -2nd
+ local command=find
+ local uri=myApp:myPass@some-name-rs0-2.some-name-rs0.pitr-32014
+ local postfix=-2nd
+ local suffix=
+ local database=myApp
+ local collection=test
+ local sort=
+ local tls=false
+ local 'full_command=db.test.find()'
+ [[ -n '' ]]
+ log 'running db.test.find() in myApp'
+ set +o xtrace
[2025-11-30T20:11:09+0000] running db.test.find() in myApp
+ [[ false == \t\r\u\e ]]
+ mongo_command=run_mongo
+ run_mongo 'use myApp\n db.test.find()' myApp:myPass@some-name-rs0-2.some-name-rs0.pitr-32014 mongodb ''
+ local 'command=use myApp\n db.test.find()'
+ local uri=myApp:myPass@some-name-rs0-2.some-name-rs0.pitr-32014
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ egrep -v 'I NETWORK|W NETWORK|F NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|Started a new thread for the timer service'
+ /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
egrep: warning: egrep is obsolescent; using grep -E
++ local LAST_OUT=/tmp/tmp.5K6q9Kphqi
+++ mktemp
++ local LAST_ERR=/tmp/tmp.r29YDVxTSY
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.5K6q9Kphqi
++ cat /tmp/tmp.r29YDVxTSY
++ rm /tmp/tmp.5K6q9Kphqi /tmp/tmp.r29YDVxTSY
++ return 0
+ local client_container=psmdb-client-696897d69b-whdv5
+ local mongo_flag=
+ [[ myApp:myPass@some-name-rs0-2.some-name-rs0.pitr-32014 == *cfg* ]]
+ replica_set=rs0
+ kubectl_bin exec psmdb-client-696897d69b-whdv5 -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-2.some-name-rs0.pitr-32014.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
++ mktemp
+ local LAST_OUT=/tmp/tmp.M7bnKXCMsO
++ mktemp
+ local LAST_ERR=/tmp/tmp.jRihLEIYlb
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-696897d69b-whdv5 -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-2.some-name-rs0.pitr-32014.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.M7bnKXCMsO
+ cat /tmp/tmp.jRihLEIYlb
+ rm /tmp/tmp.M7bnKXCMsO /tmp/tmp.jRihLEIYlb
+ return 0
+ [[ 0 -eq 0 ]]
+ diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1850/e2e-tests/pitr/compare/find-2nd.json /tmp/tmp.1qPVwubFaS/find-2nd
+ sleep 2
++ run_mongo 'new Date().toISOString()' myApp:myPass@some-name-rs0.pitr-32014 mongodb '' --quiet
++ local 'command=new Date().toISOString()'
++ local uri=myApp:myPass@some-name-rs0.pitr-32014
++ local driver=mongodb
++ local suffix=.svc.cluster.local
++ grep -E -v 'I NETWORK|W NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match'
++ cut -c1-19
+++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ tr T ' '
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.9N23RVpPQy
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.vZJUJTqzq9
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in $(seq 0 2)
+++ set +e
+++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.9N23RVpPQy
+++ cat /tmp/tmp.vZJUJTqzq9
+++ rm /tmp/tmp.9N23RVpPQy /tmp/tmp.vZJUJTqzq9
+++ return 0
++ local client_container=psmdb-client-696897d69b-whdv5
++ local mongo_flag=--quiet
++ [[ myApp:myPass@some-name-rs0.pitr-32014 == *cfg* ]]
++ replica_set=rs0
++ kubectl_bin exec psmdb-client-696897d69b-whdv5 -- bash -c 'printf '\''new Date().toISOString()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0.pitr-32014.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 --quiet'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.ghSKTo7grS
+++ mktemp
++ local LAST_ERR=/tmp/tmp.DcS5VEmENF
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl exec psmdb-client-696897d69b-whdv5 -- bash -c 'printf '\''new Date().toISOString()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0.pitr-32014.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 --quiet'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.ghSKTo7grS
++ cat /tmp/tmp.DcS5VEmENF
++ rm /tmp/tmp.ghSKTo7grS /tmp/tmp.DcS5VEmENF
++ return 0
+ time_now='2025-11-30 20:11:16'
+ check_recovery backup-minio-0 date '2025-11-30 20:11:16' -2nd '' some-name test2
+ local backup_name=backup-minio-0
+ local restore_type=date
+ local 'restore_date=2025-11-30 20:11:16'
+ local cmp_postfix=-2nd
+ local backupSource=
+ local cluster_name=some-name
+ local selective_collection=test2
+ local restore_name=restore-backup-minio-0
+ local restore_file=restore.yml
+ local cluster
+ cluster=some-name-rs0
+ desc 'write more data before restore by date'
+ set +o xtrace


-----------------------------------------------------------------------------------
write more data before restore by date
-----------------------------------------------------------------------------------

+ sleep 60
+ run_mongo 'use myApp\n db.test.insert({ x: 100501 })' myApp:myPass@some-name-rs0.pitr-32014
+ local 'command=use myApp\n db.test.insert({ x: 100501 })'
+ local uri=myApp:myPass@some-name-rs0.pitr-32014
+ local driver=mongodb+srv
+ local suffix=.svc.cluster.local
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.xAXavbbPeE
+++ mktemp
++ local LAST_ERR=/tmp/tmp.6dx2jodFkx
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.xAXavbbPeE
++ cat /tmp/tmp.6dx2jodFkx
++ rm /tmp/tmp.xAXavbbPeE /tmp/tmp.6dx2jodFkx
++ return 0
+ local client_container=psmdb-client-696897d69b-whdv5
+ local mongo_flag=
+ [[ myApp:myPass@some-name-rs0.pitr-32014 == *cfg* ]]
+ replica_set=rs0
+ kubectl_bin exec psmdb-client-696897d69b-whdv5 -- bash -c 'printf '\''use myApp\n db.test.insert({ x: 100501 })\n'\'' | mongo mongodb+srv://myApp:myPass@some-name-rs0.pitr-32014.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
++ mktemp
+ local LAST_OUT=/tmp/tmp.fc0koQdAFi
++ mktemp
+ local LAST_ERR=/tmp/tmp.DVbdxFUSSv
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-696897d69b-whdv5 -- bash -c 'printf '\''use myApp\n db.test.insert({ x: 100501 })\n'\'' | mongo mongodb+srv://myApp:myPass@some-name-rs0.pitr-32014.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.fc0koQdAFi
Percona Server for MongoDB shell version v4.4.29-28
connecting to: mongodb://some-name-rs0-2.some-name-rs0.pitr-32014.svc.cluster.local:27017,some-name-rs0-0.some-name-rs0.pitr-32014.svc.cluster.local:27017,some-name-rs0-1.some-name-rs0.pitr-32014.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb&replicaSet=rs0&ssl=false
Implicit session: session { "id" : UUID("848aa2c3-c2fa-42b3-9fa4-1148d0783ed3") }
Percona Server for MongoDB server version: v8.0.12-4
WARNING: shell and server versions do not match
switched to db myApp
WriteResult({ "nInserted" : 1 })
bye
+ cat /tmp/tmp.DVbdxFUSSv
+ rm /tmp/tmp.fc0koQdAFi /tmp/tmp.DVbdxFUSSv
+ return 0
+ '[' -n test2 ']'
+ run_mongo 'use myApp\n db.test2.drop()' myApp:myPass@some-name-rs0.pitr-32014
+ local 'command=use myApp\n db.test2.drop()'
+ local uri=myApp:myPass@some-name-rs0.pitr-32014
+ local driver=mongodb+srv
+ local suffix=.svc.cluster.local
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.1CmbRNOLEu
+++ mktemp
++ local LAST_ERR=/tmp/tmp.0Z87qlhBKn
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.1CmbRNOLEu
++ cat /tmp/tmp.0Z87qlhBKn
++ rm /tmp/tmp.1CmbRNOLEu /tmp/tmp.0Z87qlhBKn
++ return 0
+ local client_container=psmdb-client-696897d69b-whdv5
+ local mongo_flag=
+ [[ myApp:myPass@some-name-rs0.pitr-32014 == *cfg* ]]
+ replica_set=rs0
+ kubectl_bin exec psmdb-client-696897d69b-whdv5 -- bash -c 'printf '\''use myApp\n db.test2.drop()\n'\'' | mongo mongodb+srv://myApp:myPass@some-name-rs0.pitr-32014.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
++ mktemp
+ local LAST_OUT=/tmp/tmp.Zs5I5Yxmqq
++ mktemp
+ local LAST_ERR=/tmp/tmp.S9W79UaG61
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-696897d69b-whdv5 -- bash -c 'printf '\''use myApp\n db.test2.drop()\n'\'' | mongo mongodb+srv://myApp:myPass@some-name-rs0.pitr-32014.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.Zs5I5Yxmqq
Percona Server for MongoDB shell version v4.4.29-28
connecting to: mongodb://some-name-rs0-2.some-name-rs0.pitr-32014.svc.cluster.local:27017,some-name-rs0-0.some-name-rs0.pitr-32014.svc.cluster.local:27017,some-name-rs0-1.some-name-rs0.pitr-32014.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb&replicaSet=rs0&ssl=false
Implicit session: session { "id" : UUID("e1c611ad-eba1-4c12-b229-0055a027f83b") }
Percona Server for MongoDB server version: v8.0.12-4
WARNING: shell and server versions do not match
switched to db myApp
true
bye
+ cat /tmp/tmp.S9W79UaG61
+ rm /tmp/tmp.Zs5I5Yxmqq /tmp/tmp.S9W79UaG61
+ return 0
+ desc 'waiting for chunks to be uploaded'
+ set +o xtrace


-----------------------------------------------------------------------------------
waiting for chunks to be uploaded
-----------------------------------------------------------------------------------

+ sleep 120
+ desc 'check restore by date'
+ set +o xtrace


-----------------------------------------------------------------------------------
check restore by date
-----------------------------------------------------------------------------------

+ '[' -z '' ']'
+ cat /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1850/e2e-tests/pitr/conf/restore.yml
+ /usr/sbin/sed -e 's/name:/name: restore-backup-minio-0/'
+ /usr/sbin/sed -e 's/backupName:/backupName: backup-minio-0/'
+ /usr/sbin/sed -e 's/clusterName:/clusterName: some-name/'
+ '[' -n test2 ']'
+ yq eval '.spec.selective = {"namespaces": ["myApp.test"], "withUsersAndRoles": true}'
+ /usr/sbin/sed -e /backupSource/,+2d
+ kubectl_bin apply -f -
+ /usr/sbin/sed -e 's/pitrType:/type: date/'
+ '[' -z '2025-11-30 20:11:16' ']'
+ /usr/sbin/sed -e 's/date:/date: 2025-11-30 20:11:16/'
++ mktemp
+ local LAST_OUT=/tmp/tmp.XVeZyNGRVF
++ mktemp
+ local LAST_ERR=/tmp/tmp.0un4VG0llc
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl apply -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.XVeZyNGRVF
perconaservermongodbrestore.psmdb.percona.com/restore-backup-minio-0 created
+ cat /tmp/tmp.0un4VG0llc
+ rm /tmp/tmp.XVeZyNGRVF /tmp/tmp.0un4VG0llc
+ return 0
+ wait_restore backup-minio-0 some-name
+ local backup_name=backup-minio-0
+ local cluster_name=some-name
+ local target_state=ready
+ local wait_cluster_consistency=1
+ local wait_time=1780
+ local ok_if_ready=0
+ set +o xtrace
Waiting for the psmdb-restore/restore-backup-minio-0 object to be created.OK
Waiting psmdb-restore/restore-backup-minio-0 to reach state "ready" .OK after 0 minutes
+ [[ 1 -eq 1 ]]
+ wait_cluster_consistency some-name
+ local cluster_name=some-name
+ local wait_time=32
+ retry=0
+ sleep 7
+ echo -n 'waiting for cluster readyness'
waiting for cluster readyness++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.IglmKGDi8E
+++ mktemp
++ local LAST_ERR=/tmp/tmp.u0pRxk5sKU
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.IglmKGDi8E
++ cat /tmp/tmp.u0pRxk5sKU
++ rm /tmp/tmp.IglmKGDi8E /tmp/tmp.u0pRxk5sKU
++ return 0
+ [[ ready == \r\e\a\d\y ]]
+ echo .OK
.OK
+ echo

+ set -o xtrace
+ '[' -n test2 ']'
++ collection_exists test2
./e2e-tests/pitr/run: line 108: collection_exists: command not found
+ [[ '' == \t\r\u\e ]]
+ compare_mongo_cmd find myApp:myPass@some-name-rs0-0.some-name-rs0.pitr-32014 -2nd
+ local command=find
+ local uri=myApp:myPass@some-name-rs0-0.some-name-rs0.pitr-32014
+ local postfix=-2nd
+ local suffix=
+ local database=myApp
+ local collection=test
+ local sort=
+ local tls=false
+ local 'full_command=db.test.find()'
+ [[ -n '' ]]
+ log 'running db.test.find() in myApp'
+ set +o xtrace
[2025-11-30T20:14:59+0000] running db.test.find() in myApp
+ [[ false == \t\r\u\e ]]
+ mongo_command=run_mongo
+ run_mongo 'use myApp\n db.test.find()' myApp:myPass@some-name-rs0-0.some-name-rs0.pitr-32014 mongodb ''
+ egrep -v 'I NETWORK|W NETWORK|F NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|Started a new thread for the timer service'
+ local 'command=use myApp\n db.test.find()'
+ local uri=myApp:myPass@some-name-rs0-0.some-name-rs0.pitr-32014
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
egrep: warning: egrep is obsolescent; using grep -E
++ local LAST_OUT=/tmp/tmp.79V3Z1lpqf
+++ mktemp
++ local LAST_ERR=/tmp/tmp.Dxepx8SRuH
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.79V3Z1lpqf
++ cat /tmp/tmp.Dxepx8SRuH
++ rm /tmp/tmp.79V3Z1lpqf /tmp/tmp.Dxepx8SRuH
++ return 0
+ local client_container=psmdb-client-696897d69b-whdv5
+ local mongo_flag=
+ [[ myApp:myPass@some-name-rs0-0.some-name-rs0.pitr-32014 == *cfg* ]]
+ replica_set=rs0
+ kubectl_bin exec psmdb-client-696897d69b-whdv5 -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-0.some-name-rs0.pitr-32014.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
++ mktemp
+ local LAST_OUT=/tmp/tmp.ikM96d5Uzd
++ mktemp
+ local LAST_ERR=/tmp/tmp.x50an6JEs1
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-696897d69b-whdv5 -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-0.some-name-rs0.pitr-32014.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.ikM96d5Uzd
+ cat /tmp/tmp.x50an6JEs1
+ rm /tmp/tmp.ikM96d5Uzd /tmp/tmp.x50an6JEs1
+ return 0
+ [[ 0 -eq 0 ]]
+ diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1850/e2e-tests/pitr/compare/find-2nd.json /tmp/tmp.1qPVwubFaS/find-2nd
+ compare_mongo_cmd find myApp:myPass@some-name-rs0-1.some-name-rs0.pitr-32014 -2nd
+ local command=find
+ local uri=myApp:myPass@some-name-rs0-1.some-name-rs0.pitr-32014
+ local postfix=-2nd
+ local suffix=
+ local database=myApp
+ local collection=test
+ local sort=
+ local tls=false
+ local 'full_command=db.test.find()'
+ [[ -n '' ]]
+ log 'running db.test.find() in myApp'
+ set +o xtrace
[2025-11-30T20:15:01+0000] running db.test.find() in myApp
+ [[ false == \t\r\u\e ]]
+ mongo_command=run_mongo
+ run_mongo 'use myApp\n db.test.find()' myApp:myPass@some-name-rs0-1.some-name-rs0.pitr-32014 mongodb ''
+ local 'command=use myApp\n db.test.find()'
+ local uri=myApp:myPass@some-name-rs0-1.some-name-rs0.pitr-32014
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ egrep -v 'I NETWORK|W NETWORK|F NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|Started a new thread for the timer service'
+ /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.Q11nI7dWrj
egrep: warning: egrep is obsolescent; using grep -E
+++ mktemp
++ local LAST_ERR=/tmp/tmp.m79Z81YKVo
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.Q11nI7dWrj
++ cat /tmp/tmp.m79Z81YKVo
++ rm /tmp/tmp.Q11nI7dWrj /tmp/tmp.m79Z81YKVo
++ return 0
+ local client_container=psmdb-client-696897d69b-whdv5
+ local mongo_flag=
+ [[ myApp:myPass@some-name-rs0-1.some-name-rs0.pitr-32014 == *cfg* ]]
+ replica_set=rs0
+ kubectl_bin exec psmdb-client-696897d69b-whdv5 -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-1.some-name-rs0.pitr-32014.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
++ mktemp
+ local LAST_OUT=/tmp/tmp.JUpQDbi9OJ
++ mktemp
+ local LAST_ERR=/tmp/tmp.gPBwKJvFpw
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-696897d69b-whdv5 -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-1.some-name-rs0.pitr-32014.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.JUpQDbi9OJ
+ cat /tmp/tmp.gPBwKJvFpw
+ rm /tmp/tmp.JUpQDbi9OJ /tmp/tmp.gPBwKJvFpw
+ return 0
+ [[ 0 -eq 0 ]]
+ diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1850/e2e-tests/pitr/compare/find-2nd.json /tmp/tmp.1qPVwubFaS/find-2nd
+ compare_mongo_cmd find myApp:myPass@some-name-rs0-2.some-name-rs0.pitr-32014 -2nd
+ local command=find
+ local uri=myApp:myPass@some-name-rs0-2.some-name-rs0.pitr-32014
+ local postfix=-2nd
+ local suffix=
+ local database=myApp
+ local collection=test
+ local sort=
+ local tls=false
+ local 'full_command=db.test.find()'
+ [[ -n '' ]]
+ log 'running db.test.find() in myApp'
+ set +o xtrace
[2025-11-30T20:15:04+0000] running db.test.find() in myApp
+ [[ false == \t\r\u\e ]]
+ mongo_command=run_mongo
+ run_mongo 'use myApp\n db.test.find()' myApp:myPass@some-name-rs0-2.some-name-rs0.pitr-32014 mongodb ''
+ local 'command=use myApp\n db.test.find()'
+ local uri=myApp:myPass@some-name-rs0-2.some-name-rs0.pitr-32014
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ egrep -v 'I NETWORK|W NETWORK|F NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|Started a new thread for the timer service'
+ /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
egrep: warning: egrep is obsolescent; using grep -E
++ local LAST_OUT=/tmp/tmp.fZ5iPVVYb7
+++ mktemp
++ local LAST_ERR=/tmp/tmp.Zm6OOnJgqH
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.fZ5iPVVYb7
++ cat /tmp/tmp.Zm6OOnJgqH
++ rm /tmp/tmp.fZ5iPVVYb7 /tmp/tmp.Zm6OOnJgqH
++ return 0
+ local client_container=psmdb-client-696897d69b-whdv5
+ local mongo_flag=
+ [[ myApp:myPass@some-name-rs0-2.some-name-rs0.pitr-32014 == *cfg* ]]
+ replica_set=rs0
+ kubectl_bin exec psmdb-client-696897d69b-whdv5 -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-2.some-name-rs0.pitr-32014.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
++ mktemp
+ local LAST_OUT=/tmp/tmp.UvvAxmH5TQ
++ mktemp
+ local LAST_ERR=/tmp/tmp.lgP20Kpl3r
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-696897d69b-whdv5 -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-2.some-name-rs0.pitr-32014.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.UvvAxmH5TQ
+ cat /tmp/tmp.lgP20Kpl3r
+ rm /tmp/tmp.UvvAxmH5TQ /tmp/tmp.lgP20Kpl3r
+ return 0
+ [[ 0 -eq 0 ]]
+ diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1850/e2e-tests/pitr/compare/find-2nd.json /tmp/tmp.1qPVwubFaS/find-2nd
+ run_backup backup-minio 1
+ local name=backup-minio
+ local idx=1
+ desc 'run backup backup-minio-1'
+ set +o xtrace


-----------------------------------------------------------------------------------
run backup backup-minio-1
-----------------------------------------------------------------------------------

+ cat /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1850/e2e-tests/pitr/conf/backup-minio.yml
+ /usr/sbin/sed -e 's/name:/name: backup-minio-1/'
+ kubectl_bin apply -f -
++ mktemp
+ local LAST_OUT=/tmp/tmp.AcUnMpf6Fj
++ mktemp
+ local LAST_ERR=/tmp/tmp.MmZ1oUM5fl
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl apply -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.AcUnMpf6Fj
perconaservermongodbbackup.psmdb.percona.com/backup-minio-1 created
+ cat /tmp/tmp.MmZ1oUM5fl
+ rm /tmp/tmp.AcUnMpf6Fj /tmp/tmp.MmZ1oUM5fl
+ return 0
+ wait_backup backup-minio-1
+ local backup_name=backup-minio-1
+ local target_state=ready
+ set +o xtrace
waiting for backup-minio-1 to reach ready state.......OK
+ compare_latest_restorable_time some-name-rs0 backup-minio-1
+ local cluster=some-name-rs0
+ local backup_name=backup-minio-1
+ local latest_restorable_time
+ local backup_time
++ get_latest_restorable_time some-name-rs0
++ local cluster=some-name-rs0
++ local first_timestamp
++ local second_timestamp
++ local retry=0
++ [[ '' != '' ]]
+++ kubectl_bin exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ jq '.backups.pitrChunks.pitrChunks | last | .range.end'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.TAIekBBWz7
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.4yFcg1yiG7
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in $(seq 0 2)
+++ set +e
+++ kubectl exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.TAIekBBWz7
+++ cat /tmp/tmp.4yFcg1yiG7
+++ rm /tmp/tmp.TAIekBBWz7 /tmp/tmp.4yFcg1yiG7
+++ return 0
++ first_timestamp=1764533678
++ sleep 5
++ [[ 1764533678 != '' ]]
++ [[ 1764533678 != \n\u\l\l ]]
+++ kubectl_bin exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ jq '.backups.pitrChunks.pitrChunks | last | .range.end'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.obStNq86LT
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.yVuUGJIvsv
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in $(seq 0 2)
+++ set +e
+++ kubectl exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.obStNq86LT
+++ cat /tmp/tmp.yVuUGJIvsv
+++ rm /tmp/tmp.obStNq86LT /tmp/tmp.yVuUGJIvsv
+++ return 0
++ second_timestamp=1764533678
++ let retry+=1
++ [[ 1 -gt 30 ]]
++ [[ 1764533678 != '' ]]
++ [[ 1764533678 != \n\u\l\l ]]
++ [[ 1764533678 == 1764533678 ]]
++ /usr/sbin/date -u -d @1764533678 +%Y-%m-%dT%H:%M:%SZ
+ latest_restorable_time=2025-11-30T20:14:38Z
++ get_latest_restorable_time_from_backup_object backup-minio-1
++ local backup_name=backup-minio-1
++ local latestRestorableTime
++ local retry=0
++ [[ '' != '' ]]
++ sleep 5
+++ kubectl_bin get psmdb-backup backup-minio-1 -o 'jsonpath={.status.latestRestorableTime}'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.pWeVi0wFmf
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.v6vhWFio2N
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in $(seq 0 2)
+++ set +e
+++ kubectl get psmdb-backup backup-minio-1 -o 'jsonpath={.status.latestRestorableTime}'
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.pWeVi0wFmf
+++ cat /tmp/tmp.v6vhWFio2N
+++ rm /tmp/tmp.pWeVi0wFmf /tmp/tmp.v6vhWFio2N
+++ return 0
++ latestRestorableTime=2025-11-30T20:14:38Z
++ let retry+=1
++ [[ 1 -gt 30 ]]
++ [[ 2025-11-30T20:14:38Z != '' ]]
++ [[ 2025-11-30T20:14:38Z != \n\u\l\l ]]
++ echo 2025-11-30T20:14:38Z
+ backup_time=2025-11-30T20:14:38Z
+ [[ 2025-11-30T20:14:38Z != \2\0\2\5\-\1\1\-\3\0\T\2\0\:\1\4\:\3\8\Z ]]
+ check_recovery backup-minio-1 latest '' -3rd '' some-name
+ local backup_name=backup-minio-1
+ local restore_type=latest
+ local restore_date=
+ local cmp_postfix=-3rd
+ local backupSource=
+ local cluster_name=some-name
+ local selective_collection=
+ local restore_name=restore-backup-minio-1
+ local restore_file=restore.yml
+ local cluster
+ cluster=some-name-rs0
+ desc 'write more data before restore by latest'
+ set +o xtrace


-----------------------------------------------------------------------------------
write more data before restore by latest
-----------------------------------------------------------------------------------

+ sleep 60
+ run_mongo 'use myApp\n db.test.insert({ x: 100501 })' myApp:myPass@some-name-rs0.pitr-32014
+ local 'command=use myApp\n db.test.insert({ x: 100501 })'
+ local uri=myApp:myPass@some-name-rs0.pitr-32014
+ local driver=mongodb+srv
+ local suffix=.svc.cluster.local
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.vzZxCoPD9X
+++ mktemp
++ local LAST_ERR=/tmp/tmp.liCOKPukx2
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.vzZxCoPD9X
++ cat /tmp/tmp.liCOKPukx2
++ rm /tmp/tmp.vzZxCoPD9X /tmp/tmp.liCOKPukx2
++ return 0
+ local client_container=psmdb-client-696897d69b-whdv5
+ local mongo_flag=
+ [[ myApp:myPass@some-name-rs0.pitr-32014 == *cfg* ]]
+ replica_set=rs0
+ kubectl_bin exec psmdb-client-696897d69b-whdv5 -- bash -c 'printf '\''use myApp\n db.test.insert({ x: 100501 })\n'\'' | mongo mongodb+srv://myApp:myPass@some-name-rs0.pitr-32014.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
++ mktemp
+ local LAST_OUT=/tmp/tmp.19XwWH7X1g
++ mktemp
+ local LAST_ERR=/tmp/tmp.jymvp20KYD
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-696897d69b-whdv5 -- bash -c 'printf '\''use myApp\n db.test.insert({ x: 100501 })\n'\'' | mongo mongodb+srv://myApp:myPass@some-name-rs0.pitr-32014.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.19XwWH7X1g
Percona Server for MongoDB shell version v4.4.29-28
connecting to: mongodb://some-name-rs0-1.some-name-rs0.pitr-32014.svc.cluster.local:27017,some-name-rs0-2.some-name-rs0.pitr-32014.svc.cluster.local:27017,some-name-rs0-0.some-name-rs0.pitr-32014.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb&replicaSet=rs0&ssl=false
Implicit session: session { "id" : UUID("e3c4d41a-0449-40c7-93f4-73b020f21dd7") }
Percona Server for MongoDB server version: v8.0.12-4
WARNING: shell and server versions do not match
switched to db myApp
WriteResult({ "nInserted" : 1 })
bye
+ cat /tmp/tmp.jymvp20KYD
+ rm /tmp/tmp.19XwWH7X1g /tmp/tmp.jymvp20KYD
+ return 0
+ '[' -n '' ']'
+ desc 'waiting for chunks to be uploaded'
+ set +o xtrace


-----------------------------------------------------------------------------------
waiting for chunks to be uploaded
-----------------------------------------------------------------------------------

+ sleep 120
+ desc 'check restore by latest'
+ set +o xtrace


-----------------------------------------------------------------------------------
check restore by latest
-----------------------------------------------------------------------------------

+ '[' -z '' ']'
+ cat /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1850/e2e-tests/pitr/conf/restore.yml
+ /usr/sbin/sed -e 's/name:/name: restore-backup-minio-1/'
+ /usr/sbin/sed -e 's/backupName:/backupName: backup-minio-1/'
+ /usr/sbin/sed -e 's/clusterName:/clusterName: some-name/'
+ /usr/sbin/sed -e 's/pitrType:/type: latest/'
+ '[' -z '' ']'
+ /usr/sbin/sed -e /date:/d
+ kubectl_bin apply -f -
+ /usr/sbin/sed -e /backupSource/,+2d
+ '[' -n '' ']'
+ yq
++ mktemp
+ local LAST_OUT=/tmp/tmp.qVh48wMQpK
++ mktemp
+ local LAST_ERR=/tmp/tmp.CklEsQIEJH
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl apply -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.qVh48wMQpK
perconaservermongodbrestore.psmdb.percona.com/restore-backup-minio-1 created
+ cat /tmp/tmp.CklEsQIEJH
+ rm /tmp/tmp.qVh48wMQpK /tmp/tmp.CklEsQIEJH
+ return 0
+ wait_restore backup-minio-1 some-name
+ local backup_name=backup-minio-1
+ local cluster_name=some-name
+ local target_state=ready
+ local wait_cluster_consistency=1
+ local wait_time=1780
+ local ok_if_ready=0
+ set +o xtrace
Waiting for the psmdb-restore/restore-backup-minio-1 object to be created.OK
Waiting psmdb-restore/restore-backup-minio-1 to reach state "ready" .OK after 0 minutes
+ [[ 1 -eq 1 ]]
+ wait_cluster_consistency some-name
+ local cluster_name=some-name
+ local wait_time=32
+ retry=0
+ sleep 7
+ echo -n 'waiting for cluster readyness'
waiting for cluster readyness++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.9PGO70BTs4
+++ mktemp
++ local LAST_ERR=/tmp/tmp.QIiaRaRo00
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.9PGO70BTs4
++ cat /tmp/tmp.QIiaRaRo00
++ rm /tmp/tmp.9PGO70BTs4 /tmp/tmp.QIiaRaRo00
++ return 0
+ [[ ready == \r\e\a\d\y ]]
+ echo .OK
.OK
+ echo

+ set -o xtrace
+ '[' -n '' ']'
+ compare_mongo_cmd find myApp:myPass@some-name-rs0-0.some-name-rs0.pitr-32014 -3rd
+ local command=find
+ local uri=myApp:myPass@some-name-rs0-0.some-name-rs0.pitr-32014
+ local postfix=-3rd
+ local suffix=
+ local database=myApp
+ local collection=test
+ local sort=
+ local tls=false
+ local 'full_command=db.test.find()'
+ [[ -n '' ]]
+ log 'running db.test.find() in myApp'
+ set +o xtrace
[2025-11-30T20:19:05+0000] running db.test.find() in myApp
+ [[ false == \t\r\u\e ]]
+ mongo_command=run_mongo
+ run_mongo 'use myApp\n db.test.find()' myApp:myPass@some-name-rs0-0.some-name-rs0.pitr-32014 mongodb ''
+ local 'command=use myApp\n db.test.find()'
+ local uri=myApp:myPass@some-name-rs0-0.some-name-rs0.pitr-32014
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ egrep -v 'I NETWORK|W NETWORK|F NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|Started a new thread for the timer service'
+ /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
egrep: warning: egrep is obsolescent; using grep -E
++ local LAST_OUT=/tmp/tmp.oXJ0jiLGXi
+++ mktemp
++ local LAST_ERR=/tmp/tmp.LPR6lo8ffJ
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.oXJ0jiLGXi
++ cat /tmp/tmp.LPR6lo8ffJ
++ rm /tmp/tmp.oXJ0jiLGXi /tmp/tmp.LPR6lo8ffJ
++ return 0
+ local client_container=psmdb-client-696897d69b-whdv5
+ local mongo_flag=
+ [[ myApp:myPass@some-name-rs0-0.some-name-rs0.pitr-32014 == *cfg* ]]
+ replica_set=rs0
+ kubectl_bin exec psmdb-client-696897d69b-whdv5 -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-0.some-name-rs0.pitr-32014.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
++ mktemp
+ local LAST_OUT=/tmp/tmp.On9K8cNKhP
++ mktemp
+ local LAST_ERR=/tmp/tmp.oy69pk5QhW
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-696897d69b-whdv5 -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-0.some-name-rs0.pitr-32014.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.On9K8cNKhP
+ cat /tmp/tmp.oy69pk5QhW
+ rm /tmp/tmp.On9K8cNKhP /tmp/tmp.oy69pk5QhW
+ return 0
+ [[ 0 -eq 0 ]]
+ diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1850/e2e-tests/pitr/compare/find-3rd.json /tmp/tmp.1qPVwubFaS/find-3rd
+ compare_mongo_cmd find myApp:myPass@some-name-rs0-1.some-name-rs0.pitr-32014 -3rd
+ local command=find
+ local uri=myApp:myPass@some-name-rs0-1.some-name-rs0.pitr-32014
+ local postfix=-3rd
+ local suffix=
+ local database=myApp
+ local collection=test
+ local sort=
+ local tls=false
+ local 'full_command=db.test.find()'
+ [[ -n '' ]]
+ log 'running db.test.find() in myApp'
+ set +o xtrace
[2025-11-30T20:19:08+0000] running db.test.find() in myApp
+ [[ false == \t\r\u\e ]]
+ mongo_command=run_mongo
+ run_mongo 'use myApp\n db.test.find()' myApp:myPass@some-name-rs0-1.some-name-rs0.pitr-32014 mongodb ''
+ egrep -v 'I NETWORK|W NETWORK|F NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|Started a new thread for the timer service'
+ local 'command=use myApp\n db.test.find()'
+ local uri=myApp:myPass@some-name-rs0-1.some-name-rs0.pitr-32014
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
egrep: warning: egrep is obsolescent; using grep -E
+++ mktemp
++ local LAST_OUT=/tmp/tmp.ZSsJhRuaS3
+++ mktemp
++ local LAST_ERR=/tmp/tmp.RPdoviFV2q
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.ZSsJhRuaS3
++ cat /tmp/tmp.RPdoviFV2q
++ rm /tmp/tmp.ZSsJhRuaS3 /tmp/tmp.RPdoviFV2q
++ return 0
+ local client_container=psmdb-client-696897d69b-whdv5
+ local mongo_flag=
+ [[ myApp:myPass@some-name-rs0-1.some-name-rs0.pitr-32014 == *cfg* ]]
+ replica_set=rs0
+ kubectl_bin exec psmdb-client-696897d69b-whdv5 -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-1.some-name-rs0.pitr-32014.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
++ mktemp
+ local LAST_OUT=/tmp/tmp.guLfOqfoJr
++ mktemp
+ local LAST_ERR=/tmp/tmp.RCdvEOwdIS
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-696897d69b-whdv5 -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-1.some-name-rs0.pitr-32014.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.guLfOqfoJr
+ cat /tmp/tmp.RCdvEOwdIS
+ rm /tmp/tmp.guLfOqfoJr /tmp/tmp.RCdvEOwdIS
+ return 0
+ [[ 0 -eq 0 ]]
+ diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1850/e2e-tests/pitr/compare/find-3rd.json /tmp/tmp.1qPVwubFaS/find-3rd
+ compare_mongo_cmd find myApp:myPass@some-name-rs0-2.some-name-rs0.pitr-32014 -3rd
+ local command=find
+ local uri=myApp:myPass@some-name-rs0-2.some-name-rs0.pitr-32014
+ local postfix=-3rd
+ local suffix=
+ local database=myApp
+ local collection=test
+ local sort=
+ local tls=false
+ local 'full_command=db.test.find()'
+ [[ -n '' ]]
+ log 'running db.test.find() in myApp'
+ set +o xtrace
[2025-11-30T20:19:10+0000] running db.test.find() in myApp
+ [[ false == \t\r\u\e ]]
+ mongo_command=run_mongo
+ run_mongo 'use myApp\n db.test.find()' myApp:myPass@some-name-rs0-2.some-name-rs0.pitr-32014 mongodb ''
+ local 'command=use myApp\n db.test.find()'
+ egrep -v 'I NETWORK|W NETWORK|F NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|Started a new thread for the timer service'
+ local uri=myApp:myPass@some-name-rs0-2.some-name-rs0.pitr-32014
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
egrep: warning: egrep is obsolescent; using grep -E
++ local LAST_OUT=/tmp/tmp.m821age22p
+++ mktemp
++ local LAST_ERR=/tmp/tmp.u7OpACRfyi
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.m821age22p
++ cat /tmp/tmp.u7OpACRfyi
++ rm /tmp/tmp.m821age22p /tmp/tmp.u7OpACRfyi
++ return 0
+ local client_container=psmdb-client-696897d69b-whdv5
+ local mongo_flag=
+ [[ myApp:myPass@some-name-rs0-2.some-name-rs0.pitr-32014 == *cfg* ]]
+ replica_set=rs0
+ kubectl_bin exec psmdb-client-696897d69b-whdv5 -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-2.some-name-rs0.pitr-32014.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
++ mktemp
+ local LAST_OUT=/tmp/tmp.thz5xMFQdt
++ mktemp
+ local LAST_ERR=/tmp/tmp.UmZhJJXchK
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-696897d69b-whdv5 -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-2.some-name-rs0.pitr-32014.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.thz5xMFQdt
+ cat /tmp/tmp.UmZhJJXchK
+ rm /tmp/tmp.thz5xMFQdt /tmp/tmp.UmZhJJXchK
+ return 0
+ [[ 0 -eq 0 ]]
+ diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1850/e2e-tests/pitr/compare/find-3rd.json /tmp/tmp.1qPVwubFaS/find-3rd
+ desc 'restore pitr using backupSource'
+ set +o xtrace


-----------------------------------------------------------------------------------
restore pitr using backupSource
-----------------------------------------------------------------------------------

+ reset_collection
+ desc 'reset data'
+ set +o xtrace


-----------------------------------------------------------------------------------
reset data
-----------------------------------------------------------------------------------

+ run_mongo 'use myApp\n db.test.remove({})' myApp:myPass@some-name-rs0.pitr-32014
+ local 'command=use myApp\n db.test.remove({})'
+ local uri=myApp:myPass@some-name-rs0.pitr-32014
+ local driver=mongodb+srv
+ local suffix=.svc.cluster.local
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.WfwINaM9rr
+++ mktemp
++ local LAST_ERR=/tmp/tmp.WnIdxR2JIC
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.WfwINaM9rr
++ cat /tmp/tmp.WnIdxR2JIC
++ rm /tmp/tmp.WfwINaM9rr /tmp/tmp.WnIdxR2JIC
++ return 0
+ local client_container=psmdb-client-696897d69b-whdv5
+ local mongo_flag=
+ [[ myApp:myPass@some-name-rs0.pitr-32014 == *cfg* ]]
+ replica_set=rs0
+ kubectl_bin exec psmdb-client-696897d69b-whdv5 -- bash -c 'printf '\''use myApp\n db.test.remove({})\n'\'' | mongo mongodb+srv://myApp:myPass@some-name-rs0.pitr-32014.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
++ mktemp
+ local LAST_OUT=/tmp/tmp.WsSyHYW8FE
++ mktemp
+ local LAST_ERR=/tmp/tmp.jYyGPlWo7l
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-696897d69b-whdv5 -- bash -c 'printf '\''use myApp\n db.test.remove({})\n'\'' | mongo mongodb+srv://myApp:myPass@some-name-rs0.pitr-32014.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.WsSyHYW8FE
Percona Server for MongoDB shell version v4.4.29-28
connecting to: mongodb://some-name-rs0-2.some-name-rs0.pitr-32014.svc.cluster.local:27017,some-name-rs0-0.some-name-rs0.pitr-32014.svc.cluster.local:27017,some-name-rs0-1.some-name-rs0.pitr-32014.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb&replicaSet=rs0&ssl=false
Implicit session: session { "id" : UUID("d263a690-1b94-4574-a285-04ec61a7e68c") }
Percona Server for MongoDB server version: v8.0.12-4
WARNING: shell and server versions do not match
switched to db myApp
WriteResult({ "nRemoved" : 3 })
bye
+ cat /tmp/tmp.jYyGPlWo7l
+ rm /tmp/tmp.WsSyHYW8FE /tmp/tmp.jYyGPlWo7l
+ return 0
+ sleep 2
+ write_document
+ local cmp_postfix=
+ local cluster_name=some-name-rs0
+ desc 'write initial data, read from all'
+ set +o xtrace


-----------------------------------------------------------------------------------
write initial data, read from all
-----------------------------------------------------------------------------------

+ run_mongo 'use myApp\n db.test.insert({ x: 100500 })' myApp:myPass@some-name-rs0.pitr-32014
+ local 'command=use myApp\n db.test.insert({ x: 100500 })'
+ local uri=myApp:myPass@some-name-rs0.pitr-32014
+ local driver=mongodb+srv
+ local suffix=.svc.cluster.local
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.BTdihyGhbJ
+++ mktemp
++ local LAST_ERR=/tmp/tmp.IC27MlrnrS
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.BTdihyGhbJ
++ cat /tmp/tmp.IC27MlrnrS
++ rm /tmp/tmp.BTdihyGhbJ /tmp/tmp.IC27MlrnrS
++ return 0
+ local client_container=psmdb-client-696897d69b-whdv5
+ local mongo_flag=
+ [[ myApp:myPass@some-name-rs0.pitr-32014 == *cfg* ]]
+ replica_set=rs0
+ kubectl_bin exec psmdb-client-696897d69b-whdv5 -- bash -c 'printf '\''use myApp\n db.test.insert({ x: 100500 })\n'\'' | mongo mongodb+srv://myApp:myPass@some-name-rs0.pitr-32014.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
++ mktemp
+ local LAST_OUT=/tmp/tmp.SoCVXzI7OX
++ mktemp
+ local LAST_ERR=/tmp/tmp.Pe38nrPXQh
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-696897d69b-whdv5 -- bash -c 'printf '\''use myApp\n db.test.insert({ x: 100500 })\n'\'' | mongo mongodb+srv://myApp:myPass@some-name-rs0.pitr-32014.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.SoCVXzI7OX
Percona Server for MongoDB shell version v4.4.29-28
connecting to: mongodb://some-name-rs0-2.some-name-rs0.pitr-32014.svc.cluster.local:27017,some-name-rs0-0.some-name-rs0.pitr-32014.svc.cluster.local:27017,some-name-rs0-1.some-name-rs0.pitr-32014.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb&replicaSet=rs0&ssl=false
Implicit session: session { "id" : UUID("fda376a7-b04c-4643-bb91-15a321c95557") }
Percona Server for MongoDB server version: v8.0.12-4
WARNING: shell and server versions do not match
switched to db myApp
WriteResult({ "nInserted" : 1 })
bye
+ cat /tmp/tmp.Pe38nrPXQh
+ rm /tmp/tmp.SoCVXzI7OX /tmp/tmp.Pe38nrPXQh
+ return 0
+ minikube_sleep
+ sleep_time=10
+ [[ '' == 1 ]]
+ compare_mongo_cmd find myApp:myPass@some-name-rs0-0.some-name-rs0.pitr-32014 ''
+ local command=find
+ local uri=myApp:myPass@some-name-rs0-0.some-name-rs0.pitr-32014
+ local postfix=
+ local suffix=
+ local database=myApp
+ local collection=test
+ local sort=
+ local tls=false
+ local 'full_command=db.test.find()'
+ [[ -n '' ]]
+ log 'running db.test.find() in myApp'
+ set +o xtrace
[2025-11-30T20:19:21+0000] running db.test.find() in myApp
+ [[ false == \t\r\u\e ]]
+ mongo_command=run_mongo
+ run_mongo 'use myApp\n db.test.find()' myApp:myPass@some-name-rs0-0.some-name-rs0.pitr-32014 mongodb ''
+ egrep -v 'I NETWORK|W NETWORK|F NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|Started a new thread for the timer service'
+ local 'command=use myApp\n db.test.find()'
+ local uri=myApp:myPass@some-name-rs0-0.some-name-rs0.pitr-32014
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
egrep: warning: egrep is obsolescent; using grep -E
++ local LAST_OUT=/tmp/tmp.HkfPYy2nXz
+++ mktemp
++ local LAST_ERR=/tmp/tmp.LSVO5z28ql
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.HkfPYy2nXz
++ cat /tmp/tmp.LSVO5z28ql
++ rm /tmp/tmp.HkfPYy2nXz /tmp/tmp.LSVO5z28ql
++ return 0
+ local client_container=psmdb-client-696897d69b-whdv5
+ local mongo_flag=
+ [[ myApp:myPass@some-name-rs0-0.some-name-rs0.pitr-32014 == *cfg* ]]
+ replica_set=rs0
+ kubectl_bin exec psmdb-client-696897d69b-whdv5 -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-0.some-name-rs0.pitr-32014.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
++ mktemp
+ local LAST_OUT=/tmp/tmp.MYZNV24rlH
++ mktemp
+ local LAST_ERR=/tmp/tmp.kBUO54LZIo
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-696897d69b-whdv5 -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-0.some-name-rs0.pitr-32014.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.MYZNV24rlH
+ cat /tmp/tmp.kBUO54LZIo
+ rm /tmp/tmp.MYZNV24rlH /tmp/tmp.kBUO54LZIo
+ return 0
+ [[ 0 -eq 0 ]]
+ diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1850/e2e-tests/pitr/compare/find.json /tmp/tmp.1qPVwubFaS/find
+ compare_mongo_cmd find myApp:myPass@some-name-rs0-1.some-name-rs0.pitr-32014 ''
+ local command=find
+ local uri=myApp:myPass@some-name-rs0-1.some-name-rs0.pitr-32014
+ local postfix=
+ local suffix=
+ local database=myApp
+ local collection=test
+ local sort=
+ local tls=false
+ local 'full_command=db.test.find()'
+ [[ -n '' ]]
+ log 'running db.test.find() in myApp'
+ set +o xtrace
[2025-11-30T20:19:24+0000] running db.test.find() in myApp
+ [[ false == \t\r\u\e ]]
+ mongo_command=run_mongo
+ run_mongo 'use myApp\n db.test.find()' myApp:myPass@some-name-rs0-1.some-name-rs0.pitr-32014 mongodb ''
+ egrep -v 'I NETWORK|W NETWORK|F NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|Started a new thread for the timer service'
+ local 'command=use myApp\n db.test.find()'
+ local uri=myApp:myPass@some-name-rs0-1.some-name-rs0.pitr-32014
+ local driver=mongodb
+ /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
+ local suffix=.svc.cluster.local
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
egrep: warning: egrep is obsolescent; using grep -E
++ local LAST_OUT=/tmp/tmp.NgAmziLfHB
+++ mktemp
++ local LAST_ERR=/tmp/tmp.oyYOKb3OPe
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.NgAmziLfHB
++ cat /tmp/tmp.oyYOKb3OPe
++ rm /tmp/tmp.NgAmziLfHB /tmp/tmp.oyYOKb3OPe
++ return 0
+ local client_container=psmdb-client-696897d69b-whdv5
+ local mongo_flag=
+ [[ myApp:myPass@some-name-rs0-1.some-name-rs0.pitr-32014 == *cfg* ]]
+ replica_set=rs0
+ kubectl_bin exec psmdb-client-696897d69b-whdv5 -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-1.some-name-rs0.pitr-32014.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
++ mktemp
+ local LAST_OUT=/tmp/tmp.lzVMZI4L4M
++ mktemp
+ local LAST_ERR=/tmp/tmp.WNAJQk10f4
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-696897d69b-whdv5 -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-1.some-name-rs0.pitr-32014.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.lzVMZI4L4M
+ cat /tmp/tmp.WNAJQk10f4
+ rm /tmp/tmp.lzVMZI4L4M /tmp/tmp.WNAJQk10f4
+ return 0
+ [[ 0 -eq 0 ]]
+ diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1850/e2e-tests/pitr/compare/find.json /tmp/tmp.1qPVwubFaS/find
+ compare_mongo_cmd find myApp:myPass@some-name-rs0-2.some-name-rs0.pitr-32014 ''
+ local command=find
+ local uri=myApp:myPass@some-name-rs0-2.some-name-rs0.pitr-32014
+ local postfix=
+ local suffix=
+ local database=myApp
+ local collection=test
+ local sort=
+ local tls=false
+ local 'full_command=db.test.find()'
+ [[ -n '' ]]
+ log 'running db.test.find() in myApp'
+ set +o xtrace
[2025-11-30T20:19:27+0000] running db.test.find() in myApp
+ [[ false == \t\r\u\e ]]
+ mongo_command=run_mongo
+ run_mongo 'use myApp\n db.test.find()' myApp:myPass@some-name-rs0-2.some-name-rs0.pitr-32014 mongodb ''
+ local 'command=use myApp\n db.test.find()'
+ local uri=myApp:myPass@some-name-rs0-2.some-name-rs0.pitr-32014
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ egrep -v 'I NETWORK|W NETWORK|F NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|Started a new thread for the timer service'
+ /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
egrep: warning: egrep is obsolescent; using grep -E
+++ mktemp
++ local LAST_OUT=/tmp/tmp.eb0EWenejQ
+++ mktemp
++ local LAST_ERR=/tmp/tmp.S0DGze52Ry
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.eb0EWenejQ
++ cat /tmp/tmp.S0DGze52Ry
++ rm /tmp/tmp.eb0EWenejQ /tmp/tmp.S0DGze52Ry
++ return 0
+ local client_container=psmdb-client-696897d69b-whdv5
+ local mongo_flag=
+ [[ myApp:myPass@some-name-rs0-2.some-name-rs0.pitr-32014 == *cfg* ]]
+ replica_set=rs0
+ kubectl_bin exec psmdb-client-696897d69b-whdv5 -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-2.some-name-rs0.pitr-32014.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
++ mktemp
+ local LAST_OUT=/tmp/tmp.EGgElzx5Wv
++ mktemp
+ local LAST_ERR=/tmp/tmp.gTp1M2WLGi
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-696897d69b-whdv5 -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-2.some-name-rs0.pitr-32014.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.EGgElzx5Wv
+ cat /tmp/tmp.gTp1M2WLGi
+ rm /tmp/tmp.EGgElzx5Wv /tmp/tmp.gTp1M2WLGi
+ return 0
+ [[ 0 -eq 0 ]]
+ diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1850/e2e-tests/pitr/compare/find.json /tmp/tmp.1qPVwubFaS/find
+ run_backup backup-minio 2
+ local name=backup-minio
+ local idx=2
+ desc 'run backup backup-minio-2'
+ set +o xtrace


-----------------------------------------------------------------------------------
run backup backup-minio-2
-----------------------------------------------------------------------------------

+ cat /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1850/e2e-tests/pitr/conf/backup-minio.yml
+ /usr/sbin/sed -e 's/name:/name: backup-minio-2/'
+ kubectl_bin apply -f -
++ mktemp
+ local LAST_OUT=/tmp/tmp.aNoe3kNqrM
++ mktemp
+ local LAST_ERR=/tmp/tmp.6jKVLEaPaA
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl apply -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.aNoe3kNqrM
perconaservermongodbbackup.psmdb.percona.com/backup-minio-2 created
+ cat /tmp/tmp.6jKVLEaPaA
+ rm /tmp/tmp.aNoe3kNqrM /tmp/tmp.6jKVLEaPaA
+ return 0
+ wait_backup backup-minio-2
+ local backup_name=backup-minio-2
+ local target_state=ready
+ set +o xtrace
waiting for backup-minio-2 to reach ready state......OK
+ compare_latest_restorable_time some-name-rs0 backup-minio-2
+ local cluster=some-name-rs0
+ local backup_name=backup-minio-2
+ local latest_restorable_time
+ local backup_time
++ get_latest_restorable_time some-name-rs0
++ local cluster=some-name-rs0
++ local first_timestamp
++ local second_timestamp
++ local retry=0
++ [[ '' != '' ]]
+++ kubectl_bin exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ jq '.backups.pitrChunks.pitrChunks | last | .range.end'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.lpoumKxdl0
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.kkQHtogntH
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in $(seq 0 2)
+++ set +e
+++ kubectl exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.lpoumKxdl0
+++ cat /tmp/tmp.kkQHtogntH
+++ rm /tmp/tmp.lpoumKxdl0 /tmp/tmp.kkQHtogntH
+++ return 0
++ first_timestamp=1764533924
++ sleep 5
++ [[ 1764533924 != '' ]]
++ [[ 1764533924 != \n\u\l\l ]]
+++ kubectl_bin exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ jq '.backups.pitrChunks.pitrChunks | last | .range.end'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.NNEqClPPJB
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.78K0TqLeKA
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in $(seq 0 2)
+++ set +e
+++ kubectl exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.NNEqClPPJB
+++ cat /tmp/tmp.78K0TqLeKA
+++ rm /tmp/tmp.NNEqClPPJB /tmp/tmp.78K0TqLeKA
+++ return 0
++ second_timestamp=1764533924
++ let retry+=1
++ [[ 1 -gt 30 ]]
++ [[ 1764533924 != '' ]]
++ [[ 1764533924 != \n\u\l\l ]]
++ [[ 1764533924 == 1764533924 ]]
++ /usr/sbin/date -u -d @1764533924 +%Y-%m-%dT%H:%M:%SZ
+ latest_restorable_time=2025-11-30T20:18:44Z
++ get_latest_restorable_time_from_backup_object backup-minio-2
++ local backup_name=backup-minio-2
++ local latestRestorableTime
++ local retry=0
++ [[ '' != '' ]]
++ sleep 5
+++ kubectl_bin get psmdb-backup backup-minio-2 -o 'jsonpath={.status.latestRestorableTime}'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.wx9FL0AleY
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.1IFAxLvrG7
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in $(seq 0 2)
+++ set +e
+++ kubectl get psmdb-backup backup-minio-2 -o 'jsonpath={.status.latestRestorableTime}'
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.wx9FL0AleY
+++ cat /tmp/tmp.1IFAxLvrG7
+++ rm /tmp/tmp.wx9FL0AleY /tmp/tmp.1IFAxLvrG7
+++ return 0
++ latestRestorableTime=2025-11-30T20:18:44Z
++ let retry+=1
++ [[ 1 -gt 30 ]]
++ [[ 2025-11-30T20:18:44Z != '' ]]
++ [[ 2025-11-30T20:18:44Z != \n\u\l\l ]]
++ echo 2025-11-30T20:18:44Z
+ backup_time=2025-11-30T20:18:44Z
+ [[ 2025-11-30T20:18:44Z != \2\0\2\5\-\1\1\-\3\0\T\2\0\:\1\8\:\4\4\Z ]]
++ run_mongo 'new Date().toISOString()' myApp:myPass@some-name-rs0.pitr-32014 mongodb '' --quiet
++ local 'command=new Date().toISOString()'
++ local uri=myApp:myPass@some-name-rs0.pitr-32014
++ local driver=mongodb
++ local suffix=.svc.cluster.local
++ grep -E -v 'I NETWORK|W NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match'
++ cut -c1-19
++ tr T ' '
+++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.ODW6LSMmFb
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.cadpEONHUz
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in $(seq 0 2)
+++ set +e
+++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.ODW6LSMmFb
+++ cat /tmp/tmp.cadpEONHUz
+++ rm /tmp/tmp.ODW6LSMmFb /tmp/tmp.cadpEONHUz
+++ return 0
++ local client_container=psmdb-client-696897d69b-whdv5
++ local mongo_flag=--quiet
++ [[ myApp:myPass@some-name-rs0.pitr-32014 == *cfg* ]]
++ replica_set=rs0
++ kubectl_bin exec psmdb-client-696897d69b-whdv5 -- bash -c 'printf '\''new Date().toISOString()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0.pitr-32014.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 --quiet'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.N9MePlrXLB
+++ mktemp
++ local LAST_ERR=/tmp/tmp.31rGrYH5gq
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl exec psmdb-client-696897d69b-whdv5 -- bash -c 'printf '\''new Date().toISOString()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0.pitr-32014.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 --quiet'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.N9MePlrXLB
++ cat /tmp/tmp.31rGrYH5gq
++ rm /tmp/tmp.N9MePlrXLB /tmp/tmp.31rGrYH5gq
++ return 0
+ time_now='2025-11-30 20:19:59'
+ check_recovery backup-minio-2 date '2025-11-30 20:19:59' '' backupSource some-name
+ local backup_name=backup-minio-2
+ local restore_type=date
+ local 'restore_date=2025-11-30 20:19:59'
+ local cmp_postfix=
+ local backupSource=backupSource
+ local cluster_name=some-name
+ local selective_collection=
+ local restore_name=restore-backup-minio-2
+ local restore_file=restore.yml
+ local cluster
+ cluster=some-name-rs0
+ desc 'write more data before restore by date'
+ set +o xtrace


-----------------------------------------------------------------------------------
write more data before restore by date
-----------------------------------------------------------------------------------

+ sleep 60
+ run_mongo 'use myApp\n db.test.insert({ x: 100501 })' myApp:myPass@some-name-rs0.pitr-32014
+ local 'command=use myApp\n db.test.insert({ x: 100501 })'
+ local uri=myApp:myPass@some-name-rs0.pitr-32014
+ local driver=mongodb+srv
+ local suffix=.svc.cluster.local
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.DOaMFj6UcB
+++ mktemp
++ local LAST_ERR=/tmp/tmp.FygW4GH7aj
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.DOaMFj6UcB
++ cat /tmp/tmp.FygW4GH7aj
++ rm /tmp/tmp.DOaMFj6UcB /tmp/tmp.FygW4GH7aj
++ return 0
+ local client_container=psmdb-client-696897d69b-whdv5
+ local mongo_flag=
+ [[ myApp:myPass@some-name-rs0.pitr-32014 == *cfg* ]]
+ replica_set=rs0
+ kubectl_bin exec psmdb-client-696897d69b-whdv5 -- bash -c 'printf '\''use myApp\n db.test.insert({ x: 100501 })\n'\'' | mongo mongodb+srv://myApp:myPass@some-name-rs0.pitr-32014.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
++ mktemp
+ local LAST_OUT=/tmp/tmp.xif6MUelkA
++ mktemp
+ local LAST_ERR=/tmp/tmp.GHSdfuNQ8E
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-696897d69b-whdv5 -- bash -c 'printf '\''use myApp\n db.test.insert({ x: 100501 })\n'\'' | mongo mongodb+srv://myApp:myPass@some-name-rs0.pitr-32014.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.xif6MUelkA
Percona Server for MongoDB shell version v4.4.29-28
connecting to: mongodb://some-name-rs0-2.some-name-rs0.pitr-32014.svc.cluster.local:27017,some-name-rs0-0.some-name-rs0.pitr-32014.svc.cluster.local:27017,some-name-rs0-1.some-name-rs0.pitr-32014.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb&replicaSet=rs0&ssl=false
Implicit session: session { "id" : UUID("daa17a24-fbd7-40d9-9727-45048fcd4b1f") }
Percona Server for MongoDB server version: v8.0.12-4
WARNING: shell and server versions do not match
switched to db myApp
WriteResult({ "nInserted" : 1 })
bye
+ cat /tmp/tmp.GHSdfuNQ8E
+ rm /tmp/tmp.xif6MUelkA /tmp/tmp.GHSdfuNQ8E
+ return 0
+ '[' -n '' ']'
+ desc 'waiting for chunks to be uploaded'
+ set +o xtrace


-----------------------------------------------------------------------------------
waiting for chunks to be uploaded
-----------------------------------------------------------------------------------

+ sleep 120
+ desc 'check restore by date'
+ set +o xtrace


-----------------------------------------------------------------------------------
check restore by date
-----------------------------------------------------------------------------------

+ '[' -z backupSource ']'
++ get_backup_dest backup-minio-2
++ local backup_name=backup-minio-2
++ kubectl_bin get psmdb-backup backup-minio-2 -o 'jsonpath={.status.destination}'
++ sed -e 's/.json$//'
++ sed 's|s3://||'
+++ mktemp
++ sed 's|azure://||'
++ sed 's|gs://||'
++ local LAST_OUT=/tmp/tmp.qP9yeWpj9D
+++ mktemp
++ local LAST_ERR=/tmp/tmp.puHiPkdi8E
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb-backup backup-minio-2 -o 'jsonpath={.status.destination}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.qP9yeWpj9D
++ cat /tmp/tmp.puHiPkdi8E
++ rm /tmp/tmp.qP9yeWpj9D /tmp/tmp.puHiPkdi8E
++ return 0
+ backup_dest=operator-testing/pitr-prefix-1/2025-11-30T20:19:31Z
+ cat /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1850/e2e-tests/pitr/conf/restore.yml
+ /usr/sbin/sed -e 's/name:/name: restore-backup-minio-2/'
+ /usr/sbin/sed -e /backupName/d
+ /usr/sbin/sed -e 's/clusterName:/clusterName: some-name/'
+ /usr/sbin/sed -e 's/pitrType:/type: date/'
+ '[' -z '2025-11-30 20:19:59' ']'
+ /usr/sbin/sed -e 's/date:/date: 2025-11-30 20:19:59/'
+ /usr/sbin/sed -e 's|BACKUP-NAME|operator-testing/pitr-prefix-1/2025-11-30T20:19:31Z|'
+ '[' -n '' ']'
+ yq
+ kubectl_bin apply -f -
++ mktemp
+ local LAST_OUT=/tmp/tmp.WxSQR3toAl
++ mktemp
+ local LAST_ERR=/tmp/tmp.MA5muHeLNe
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl apply -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.WxSQR3toAl
perconaservermongodbrestore.psmdb.percona.com/restore-backup-minio-2 created
+ cat /tmp/tmp.MA5muHeLNe
+ rm /tmp/tmp.WxSQR3toAl /tmp/tmp.MA5muHeLNe
+ return 0
+ wait_restore backup-minio-2 some-name
+ local backup_name=backup-minio-2
+ local cluster_name=some-name
+ local target_state=ready
+ local wait_cluster_consistency=1
+ local wait_time=1780
+ local ok_if_ready=0
+ set +o xtrace
Waiting for the psmdb-restore/restore-backup-minio-2 object to be created.OK
Waiting psmdb-restore/restore-backup-minio-2 to reach state "ready" .OK after 0 minutes
+ [[ 1 -eq 1 ]]
+ wait_cluster_consistency some-name
+ local cluster_name=some-name
+ local wait_time=32
+ retry=0
+ sleep 7
+ echo -n 'waiting for cluster readyness'
waiting for cluster readyness++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.wjczBD47c7
+++ mktemp
++ local LAST_ERR=/tmp/tmp.HGrRLEuZrI
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.wjczBD47c7
++ cat /tmp/tmp.HGrRLEuZrI
++ rm /tmp/tmp.wjczBD47c7 /tmp/tmp.HGrRLEuZrI
++ return 0
+ [[ ready == \r\e\a\d\y ]]
+ echo .OK
.OK
+ echo

+ set -o xtrace
+ '[' -n '' ']'
+ compare_mongo_cmd find myApp:myPass@some-name-rs0-0.some-name-rs0.pitr-32014 ''
+ local command=find
+ local uri=myApp:myPass@some-name-rs0-0.some-name-rs0.pitr-32014
+ local postfix=
+ local suffix=
+ local database=myApp
+ local collection=test
+ local sort=
+ local tls=false
+ local 'full_command=db.test.find()'
+ [[ -n '' ]]
+ log 'running db.test.find() in myApp'
+ set +o xtrace
[2025-11-30T20:23:24+0000] running db.test.find() in myApp
+ [[ false == \t\r\u\e ]]
+ mongo_command=run_mongo
+ run_mongo 'use myApp\n db.test.find()' myApp:myPass@some-name-rs0-0.some-name-rs0.pitr-32014 mongodb ''
+ local 'command=use myApp\n db.test.find()'
+ egrep -v 'I NETWORK|W NETWORK|F NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|Started a new thread for the timer service'
+ local uri=myApp:myPass@some-name-rs0-0.some-name-rs0.pitr-32014
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
egrep: warning: egrep is obsolescent; using grep -E
++ local LAST_OUT=/tmp/tmp.b7Vm6qcULp
+++ mktemp
++ local LAST_ERR=/tmp/tmp.TgbyMuhN4i
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.b7Vm6qcULp
++ cat /tmp/tmp.TgbyMuhN4i
++ rm /tmp/tmp.b7Vm6qcULp /tmp/tmp.TgbyMuhN4i
++ return 0
+ local client_container=psmdb-client-696897d69b-whdv5
+ local mongo_flag=
+ [[ myApp:myPass@some-name-rs0-0.some-name-rs0.pitr-32014 == *cfg* ]]
+ replica_set=rs0
+ kubectl_bin exec psmdb-client-696897d69b-whdv5 -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-0.some-name-rs0.pitr-32014.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
++ mktemp
+ local LAST_OUT=/tmp/tmp.Qu1TwZjRvm
++ mktemp
+ local LAST_ERR=/tmp/tmp.Lucl9qlROF
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-696897d69b-whdv5 -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-0.some-name-rs0.pitr-32014.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.Qu1TwZjRvm
+ cat /tmp/tmp.Lucl9qlROF
+ rm /tmp/tmp.Qu1TwZjRvm /tmp/tmp.Lucl9qlROF
+ return 0
+ [[ 0 -eq 0 ]]
+ diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1850/e2e-tests/pitr/compare/find.json /tmp/tmp.1qPVwubFaS/find
+ compare_mongo_cmd find myApp:myPass@some-name-rs0-1.some-name-rs0.pitr-32014 ''
+ local command=find
+ local uri=myApp:myPass@some-name-rs0-1.some-name-rs0.pitr-32014
+ local postfix=
+ local suffix=
+ local database=myApp
+ local collection=test
+ local sort=
+ local tls=false
+ local 'full_command=db.test.find()'
+ [[ -n '' ]]
+ log 'running db.test.find() in myApp'
+ set +o xtrace
[2025-11-30T20:23:26+0000] running db.test.find() in myApp
+ [[ false == \t\r\u\e ]]
+ mongo_command=run_mongo
+ egrep -v 'I NETWORK|W NETWORK|F NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|Started a new thread for the timer service'
+ run_mongo 'use myApp\n db.test.find()' myApp:myPass@some-name-rs0-1.some-name-rs0.pitr-32014 mongodb ''
+ local 'command=use myApp\n db.test.find()'
+ local uri=myApp:myPass@some-name-rs0-1.some-name-rs0.pitr-32014
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
egrep: warning: egrep is obsolescent; using grep -E
++ local LAST_OUT=/tmp/tmp.HFmpokRN6H
+++ mktemp
++ local LAST_ERR=/tmp/tmp.QFne7cwkaA
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.HFmpokRN6H
++ cat /tmp/tmp.QFne7cwkaA
++ rm /tmp/tmp.HFmpokRN6H /tmp/tmp.QFne7cwkaA
++ return 0
+ local client_container=psmdb-client-696897d69b-whdv5
+ local mongo_flag=
+ [[ myApp:myPass@some-name-rs0-1.some-name-rs0.pitr-32014 == *cfg* ]]
+ replica_set=rs0
+ kubectl_bin exec psmdb-client-696897d69b-whdv5 -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-1.some-name-rs0.pitr-32014.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
++ mktemp
+ local LAST_OUT=/tmp/tmp.LF54rp3AA8
++ mktemp
+ local LAST_ERR=/tmp/tmp.YdvgmaHhew
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-696897d69b-whdv5 -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-1.some-name-rs0.pitr-32014.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.LF54rp3AA8
+ cat /tmp/tmp.YdvgmaHhew
+ rm /tmp/tmp.LF54rp3AA8 /tmp/tmp.YdvgmaHhew
+ return 0
+ [[ 0 -eq 0 ]]
+ diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1850/e2e-tests/pitr/compare/find.json /tmp/tmp.1qPVwubFaS/find
+ compare_mongo_cmd find myApp:myPass@some-name-rs0-2.some-name-rs0.pitr-32014 ''
+ local command=find
+ local uri=myApp:myPass@some-name-rs0-2.some-name-rs0.pitr-32014
+ local postfix=
+ local suffix=
+ local database=myApp
+ local collection=test
+ local sort=
+ local tls=false
+ local 'full_command=db.test.find()'
+ [[ -n '' ]]
+ log 'running db.test.find() in myApp'
+ set +o xtrace
[2025-11-30T20:23:29+0000] running db.test.find() in myApp
+ [[ false == \t\r\u\e ]]
+ mongo_command=run_mongo
+ run_mongo 'use myApp\n db.test.find()' myApp:myPass@some-name-rs0-2.some-name-rs0.pitr-32014 mongodb ''
+ local 'command=use myApp\n db.test.find()'
+ local uri=myApp:myPass@some-name-rs0-2.some-name-rs0.pitr-32014
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ egrep -v 'I NETWORK|W NETWORK|F NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|Started a new thread for the timer service'
+ /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
egrep: warning: egrep is obsolescent; using grep -E
++ local LAST_OUT=/tmp/tmp.UYLJECQH9X
+++ mktemp
++ local LAST_ERR=/tmp/tmp.bxGXxBBVdM
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.UYLJECQH9X
++ cat /tmp/tmp.bxGXxBBVdM
++ rm /tmp/tmp.UYLJECQH9X /tmp/tmp.bxGXxBBVdM
++ return 0
+ local client_container=psmdb-client-696897d69b-whdv5
+ local mongo_flag=
+ [[ myApp:myPass@some-name-rs0-2.some-name-rs0.pitr-32014 == *cfg* ]]
+ replica_set=rs0
+ kubectl_bin exec psmdb-client-696897d69b-whdv5 -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-2.some-name-rs0.pitr-32014.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
++ mktemp
+ local LAST_OUT=/tmp/tmp.z68PsTyukh
++ mktemp
+ local LAST_ERR=/tmp/tmp.RpNrWhwQQu
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-696897d69b-whdv5 -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-2.some-name-rs0.pitr-32014.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.z68PsTyukh
+ cat /tmp/tmp.RpNrWhwQQu
+ rm /tmp/tmp.z68PsTyukh /tmp/tmp.RpNrWhwQQu
+ return 0
+ [[ 0 -eq 0 ]]
+ diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1850/e2e-tests/pitr/compare/find.json /tmp/tmp.1qPVwubFaS/find
+ run_backup backup-minio 3
+ local name=backup-minio
+ local idx=3
+ desc 'run backup backup-minio-3'
+ set +o xtrace


-----------------------------------------------------------------------------------
run backup backup-minio-3
-----------------------------------------------------------------------------------

+ cat /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1850/e2e-tests/pitr/conf/backup-minio.yml
+ /usr/sbin/sed -e 's/name:/name: backup-minio-3/'
+ kubectl_bin apply -f -
++ mktemp
+ local LAST_OUT=/tmp/tmp.3ellGsGZte
++ mktemp
+ local LAST_ERR=/tmp/tmp.Zmm5JBDqDf
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl apply -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.3ellGsGZte
perconaservermongodbbackup.psmdb.percona.com/backup-minio-3 created
+ cat /tmp/tmp.Zmm5JBDqDf
+ rm /tmp/tmp.3ellGsGZte /tmp/tmp.Zmm5JBDqDf
+ return 0
+ wait_backup backup-minio-3
+ local backup_name=backup-minio-3
+ local target_state=ready
+ set +o xtrace
waiting for backup-minio-3 to reach ready state.......OK
+ compare_latest_restorable_time some-name-rs0 backup-minio-3
+ local cluster=some-name-rs0
+ local backup_name=backup-minio-3
+ local latest_restorable_time
+ local backup_time
++ get_latest_restorable_time some-name-rs0
++ local cluster=some-name-rs0
++ local first_timestamp
++ local second_timestamp
++ local retry=0
++ [[ '' != '' ]]
+++ kubectl_bin exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ jq '.backups.pitrChunks.pitrChunks | last | .range.end'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.r7xsLWfoWo
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.hd3EEzcAtn
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in $(seq 0 2)
+++ set +e
+++ kubectl exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.r7xsLWfoWo
+++ cat /tmp/tmp.hd3EEzcAtn
+++ rm /tmp/tmp.r7xsLWfoWo /tmp/tmp.hd3EEzcAtn
+++ return 0
++ first_timestamp=1764534185
++ sleep 5
++ [[ 1764534185 != '' ]]
++ [[ 1764534185 != \n\u\l\l ]]
+++ kubectl_bin exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ jq '.backups.pitrChunks.pitrChunks | last | .range.end'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.fFFCABnwqs
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.5QqYMLdATV
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in $(seq 0 2)
+++ set +e
+++ kubectl exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.fFFCABnwqs
+++ cat /tmp/tmp.5QqYMLdATV
+++ rm /tmp/tmp.fFFCABnwqs /tmp/tmp.5QqYMLdATV
+++ return 0
++ second_timestamp=1764534185
++ let retry+=1
++ [[ 1 -gt 30 ]]
++ [[ 1764534185 != '' ]]
++ [[ 1764534185 != \n\u\l\l ]]
++ [[ 1764534185 == 1764534185 ]]
++ /usr/sbin/date -u -d @1764534185 +%Y-%m-%dT%H:%M:%SZ
+ latest_restorable_time=2025-11-30T20:23:05Z
++ get_latest_restorable_time_from_backup_object backup-minio-3
++ local backup_name=backup-minio-3
++ local latestRestorableTime
++ local retry=0
++ [[ '' != '' ]]
++ sleep 5
+++ kubectl_bin get psmdb-backup backup-minio-3 -o 'jsonpath={.status.latestRestorableTime}'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.S8u07WWeoY
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.TqZY10YeZy
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in $(seq 0 2)
+++ set +e
+++ kubectl get psmdb-backup backup-minio-3 -o 'jsonpath={.status.latestRestorableTime}'
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.S8u07WWeoY
+++ cat /tmp/tmp.TqZY10YeZy
+++ rm /tmp/tmp.S8u07WWeoY /tmp/tmp.TqZY10YeZy
+++ return 0
++ latestRestorableTime=2025-11-30T20:23:05Z
++ let retry+=1
++ [[ 1 -gt 30 ]]
++ [[ 2025-11-30T20:23:05Z != '' ]]
++ [[ 2025-11-30T20:23:05Z != \n\u\l\l ]]
++ echo 2025-11-30T20:23:05Z
+ backup_time=2025-11-30T20:23:05Z
+ [[ 2025-11-30T20:23:05Z != \2\0\2\5\-\1\1\-\3\0\T\2\0\:\2\3\:\0\5\Z ]]
+ check_recovery backup-minio-3 latest '' -4th backupSource some-name
+ local backup_name=backup-minio-3
+ local restore_type=latest
+ local restore_date=
+ local cmp_postfix=-4th
+ local backupSource=backupSource
+ local cluster_name=some-name
+ local selective_collection=
+ local restore_name=restore-backup-minio-3
+ local restore_file=restore.yml
+ local cluster
+ cluster=some-name-rs0
+ desc 'write more data before restore by latest'
+ set +o xtrace


-----------------------------------------------------------------------------------
write more data before restore by latest
-----------------------------------------------------------------------------------

+ sleep 60
+ run_mongo 'use myApp\n db.test.insert({ x: 100501 })' myApp:myPass@some-name-rs0.pitr-32014
+ local 'command=use myApp\n db.test.insert({ x: 100501 })'
+ local uri=myApp:myPass@some-name-rs0.pitr-32014
+ local driver=mongodb+srv
+ local suffix=.svc.cluster.local
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.Jy3YLuBGuK
+++ mktemp
++ local LAST_ERR=/tmp/tmp.Q0umZXF8nU
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.Jy3YLuBGuK
++ cat /tmp/tmp.Q0umZXF8nU
++ rm /tmp/tmp.Jy3YLuBGuK /tmp/tmp.Q0umZXF8nU
++ return 0
+ local client_container=psmdb-client-696897d69b-whdv5
+ local mongo_flag=
+ [[ myApp:myPass@some-name-rs0.pitr-32014 == *cfg* ]]
+ replica_set=rs0
+ kubectl_bin exec psmdb-client-696897d69b-whdv5 -- bash -c 'printf '\''use myApp\n db.test.insert({ x: 100501 })\n'\'' | mongo mongodb+srv://myApp:myPass@some-name-rs0.pitr-32014.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
++ mktemp
+ local LAST_OUT=/tmp/tmp.dkQRQE1wVH
++ mktemp
+ local LAST_ERR=/tmp/tmp.Trk11Hz8RH
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-696897d69b-whdv5 -- bash -c 'printf '\''use myApp\n db.test.insert({ x: 100501 })\n'\'' | mongo mongodb+srv://myApp:myPass@some-name-rs0.pitr-32014.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.dkQRQE1wVH
Percona Server for MongoDB shell version v4.4.29-28
connecting to: mongodb://some-name-rs0-2.some-name-rs0.pitr-32014.svc.cluster.local:27017,some-name-rs0-0.some-name-rs0.pitr-32014.svc.cluster.local:27017,some-name-rs0-1.some-name-rs0.pitr-32014.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb&replicaSet=rs0&ssl=false
Implicit session: session { "id" : UUID("01eb4a12-206d-4efd-b417-b44b73347409") }
Percona Server for MongoDB server version: v8.0.12-4
WARNING: shell and server versions do not match
switched to db myApp
WriteResult({ "nInserted" : 1 })
bye
+ cat /tmp/tmp.Trk11Hz8RH
+ rm /tmp/tmp.dkQRQE1wVH /tmp/tmp.Trk11Hz8RH
+ return 0
+ '[' -n '' ']'
+ desc 'waiting for chunks to be uploaded'
+ set +o xtrace


-----------------------------------------------------------------------------------
waiting for chunks to be uploaded
-----------------------------------------------------------------------------------

+ sleep 120
+ desc 'check restore by latest'
+ set +o xtrace


-----------------------------------------------------------------------------------
check restore by latest
-----------------------------------------------------------------------------------

+ '[' -z backupSource ']'
++ get_backup_dest backup-minio-3
++ local backup_name=backup-minio-3
++ kubectl_bin get psmdb-backup backup-minio-3 -o 'jsonpath={.status.destination}'
++ sed -e 's/.json$//'
++ sed 's|s3://||'
+++ mktemp
++ sed 's|azure://||'
++ sed 's|gs://||'
++ local LAST_OUT=/tmp/tmp.qnuFqCiYsl
+++ mktemp
++ local LAST_ERR=/tmp/tmp.9LxY9OBRm9
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb-backup backup-minio-3 -o 'jsonpath={.status.destination}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.qnuFqCiYsl
++ cat /tmp/tmp.9LxY9OBRm9
++ rm /tmp/tmp.qnuFqCiYsl /tmp/tmp.9LxY9OBRm9
++ return 0
+ backup_dest=operator-testing/pitr-prefix-1/2025-11-30T20:23:34Z
+ cat /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1850/e2e-tests/pitr/conf/restore.yml
+ /usr/sbin/sed -e 's/name:/name: restore-backup-minio-3/'
+ /usr/sbin/sed -e /backupName/d
+ /usr/sbin/sed -e 's/pitrType:/type: latest/'
+ '[' -n '' ']'
+ yq
+ /usr/sbin/sed -e 's/clusterName:/clusterName: some-name/'
+ '[' -z '' ']'
+ kubectl_bin apply -f -
+ /usr/sbin/sed -e /date:/d
++ mktemp
+ local LAST_OUT=/tmp/tmp.b03dHDB9rU
+ /usr/sbin/sed -e 's|BACKUP-NAME|operator-testing/pitr-prefix-1/2025-11-30T20:23:34Z|'
++ mktemp
+ local LAST_ERR=/tmp/tmp.MWavZcSKSh
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl apply -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.b03dHDB9rU
perconaservermongodbrestore.psmdb.percona.com/restore-backup-minio-3 created
+ cat /tmp/tmp.MWavZcSKSh
+ rm /tmp/tmp.b03dHDB9rU /tmp/tmp.MWavZcSKSh
+ return 0
+ wait_restore backup-minio-3 some-name
+ local backup_name=backup-minio-3
+ local cluster_name=some-name
+ local target_state=ready
+ local wait_cluster_consistency=1
+ local wait_time=1780
+ local ok_if_ready=0
+ set +o xtrace
Waiting for the psmdb-restore/restore-backup-minio-3 object to be created.OK
Waiting psmdb-restore/restore-backup-minio-3 to reach state "ready" .OK after 0 minutes
+ [[ 1 -eq 1 ]]
+ wait_cluster_consistency some-name
+ local cluster_name=some-name
+ local wait_time=32
+ retry=0
+ sleep 7
+ echo -n 'waiting for cluster readyness'
waiting for cluster readyness++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.s3cFWWZbu0
+++ mktemp
++ local LAST_ERR=/tmp/tmp.quBLjnt3p7
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.s3cFWWZbu0
++ cat /tmp/tmp.quBLjnt3p7
++ rm /tmp/tmp.s3cFWWZbu0 /tmp/tmp.quBLjnt3p7
++ return 0
+ [[ ready == \r\e\a\d\y ]]
+ echo .OK
.OK
+ echo

+ set -o xtrace
+ '[' -n '' ']'
+ compare_mongo_cmd find myApp:myPass@some-name-rs0-0.some-name-rs0.pitr-32014 -4th
+ local command=find
+ local uri=myApp:myPass@some-name-rs0-0.some-name-rs0.pitr-32014
+ local postfix=-4th
+ local suffix=
+ local database=myApp
+ local collection=test
+ local sort=
+ local tls=false
+ local 'full_command=db.test.find()'
+ [[ -n '' ]]
+ log 'running db.test.find() in myApp'
+ set +o xtrace
[2025-11-30T20:27:31+0000] running db.test.find() in myApp
+ [[ false == \t\r\u\e ]]
+ mongo_command=run_mongo
+ run_mongo 'use myApp\n db.test.find()' myApp:myPass@some-name-rs0-0.some-name-rs0.pitr-32014 mongodb ''
+ egrep -v 'I NETWORK|W NETWORK|F NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|Started a new thread for the timer service'
+ local 'command=use myApp\n db.test.find()'
+ local uri=myApp:myPass@some-name-rs0-0.some-name-rs0.pitr-32014
+ /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
+ local driver=mongodb
+ local suffix=.svc.cluster.local
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
egrep: warning: egrep is obsolescent; using grep -E
++ local LAST_OUT=/tmp/tmp.Mc6Cggu3JC
+++ mktemp
++ local LAST_ERR=/tmp/tmp.RvuCpoF7yf
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.Mc6Cggu3JC
++ cat /tmp/tmp.RvuCpoF7yf
++ rm /tmp/tmp.Mc6Cggu3JC /tmp/tmp.RvuCpoF7yf
++ return 0
+ local client_container=psmdb-client-696897d69b-whdv5
+ local mongo_flag=
+ [[ myApp:myPass@some-name-rs0-0.some-name-rs0.pitr-32014 == *cfg* ]]
+ replica_set=rs0
+ kubectl_bin exec psmdb-client-696897d69b-whdv5 -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-0.some-name-rs0.pitr-32014.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
++ mktemp
+ local LAST_OUT=/tmp/tmp.B9RfqMPkeR
++ mktemp
+ local LAST_ERR=/tmp/tmp.gaXG6HMryb
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-696897d69b-whdv5 -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-0.some-name-rs0.pitr-32014.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.B9RfqMPkeR
+ cat /tmp/tmp.gaXG6HMryb
+ rm /tmp/tmp.B9RfqMPkeR /tmp/tmp.gaXG6HMryb
+ return 0
+ [[ 0 -eq 0 ]]
+ diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1850/e2e-tests/pitr/compare/find-4th.json /tmp/tmp.1qPVwubFaS/find-4th
+ compare_mongo_cmd find myApp:myPass@some-name-rs0-1.some-name-rs0.pitr-32014 -4th
+ local command=find
+ local uri=myApp:myPass@some-name-rs0-1.some-name-rs0.pitr-32014
+ local postfix=-4th
+ local suffix=
+ local database=myApp
+ local collection=test
+ local sort=
+ local tls=false
+ local 'full_command=db.test.find()'
+ [[ -n '' ]]
+ log 'running db.test.find() in myApp'
+ set +o xtrace
[2025-11-30T20:27:33+0000] running db.test.find() in myApp
+ [[ false == \t\r\u\e ]]
+ mongo_command=run_mongo
+ run_mongo 'use myApp\n db.test.find()' myApp:myPass@some-name-rs0-1.some-name-rs0.pitr-32014 mongodb ''
+ local 'command=use myApp\n db.test.find()'
+ egrep -v 'I NETWORK|W NETWORK|F NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|Started a new thread for the timer service'
+ local uri=myApp:myPass@some-name-rs0-1.some-name-rs0.pitr-32014
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
egrep: warning: egrep is obsolescent; using grep -E
+++ mktemp
++ local LAST_OUT=/tmp/tmp.7nBbVNil1b
+++ mktemp
++ local LAST_ERR=/tmp/tmp.lcQCjZclIM
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.7nBbVNil1b
++ cat /tmp/tmp.lcQCjZclIM
++ rm /tmp/tmp.7nBbVNil1b /tmp/tmp.lcQCjZclIM
++ return 0
+ local client_container=psmdb-client-696897d69b-whdv5
+ local mongo_flag=
+ [[ myApp:myPass@some-name-rs0-1.some-name-rs0.pitr-32014 == *cfg* ]]
+ replica_set=rs0
+ kubectl_bin exec psmdb-client-696897d69b-whdv5 -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-1.some-name-rs0.pitr-32014.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
++ mktemp
+ local LAST_OUT=/tmp/tmp.xWTpWM3bKn
++ mktemp
+ local LAST_ERR=/tmp/tmp.Q9UBCALf9C
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-696897d69b-whdv5 -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-1.some-name-rs0.pitr-32014.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.xWTpWM3bKn
+ cat /tmp/tmp.Q9UBCALf9C
+ rm /tmp/tmp.xWTpWM3bKn /tmp/tmp.Q9UBCALf9C
+ return 0
+ [[ 0 -eq 0 ]]
+ diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1850/e2e-tests/pitr/compare/find-4th.json /tmp/tmp.1qPVwubFaS/find-4th
+ compare_mongo_cmd find myApp:myPass@some-name-rs0-2.some-name-rs0.pitr-32014 -4th
+ local command=find
+ local uri=myApp:myPass@some-name-rs0-2.some-name-rs0.pitr-32014
+ local postfix=-4th
+ local suffix=
+ local database=myApp
+ local collection=test
+ local sort=
+ local tls=false
+ local 'full_command=db.test.find()'
+ [[ -n '' ]]
+ log 'running db.test.find() in myApp'
+ set +o xtrace
[2025-11-30T20:27:36+0000] running db.test.find() in myApp
+ [[ false == \t\r\u\e ]]
+ mongo_command=run_mongo
+ run_mongo 'use myApp\n db.test.find()' myApp:myPass@some-name-rs0-2.some-name-rs0.pitr-32014 mongodb ''
+ local 'command=use myApp\n db.test.find()'
+ local uri=myApp:myPass@some-name-rs0-2.some-name-rs0.pitr-32014
+ egrep -v 'I NETWORK|W NETWORK|F NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|Started a new thread for the timer service'
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
egrep: warning: egrep is obsolescent; using grep -E
++ local LAST_OUT=/tmp/tmp.OWtupgjFZy
+++ mktemp
++ local LAST_ERR=/tmp/tmp.F2aMCQFWds
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.OWtupgjFZy
++ cat /tmp/tmp.F2aMCQFWds
++ rm /tmp/tmp.OWtupgjFZy /tmp/tmp.F2aMCQFWds
++ return 0
+ local client_container=psmdb-client-696897d69b-whdv5
+ local mongo_flag=
+ [[ myApp:myPass@some-name-rs0-2.some-name-rs0.pitr-32014 == *cfg* ]]
+ replica_set=rs0
+ kubectl_bin exec psmdb-client-696897d69b-whdv5 -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-2.some-name-rs0.pitr-32014.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
++ mktemp
+ local LAST_OUT=/tmp/tmp.oW1ZjGUWSo
++ mktemp
+ local LAST_ERR=/tmp/tmp.rSKAw3M196
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-696897d69b-whdv5 -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-2.some-name-rs0.pitr-32014.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.oW1ZjGUWSo
+ cat /tmp/tmp.rSKAw3M196
+ rm /tmp/tmp.oW1ZjGUWSo /tmp/tmp.rSKAw3M196
+ return 0
+ [[ 0 -eq 0 ]]
+ diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1850/e2e-tests/pitr/compare/find-4th.json /tmp/tmp.1qPVwubFaS/find-4th
+ desc 'restore pitr using backupSource on second cluster with another prefix in storage'
+ set +o xtrace


-----------------------------------------------------------------------------------
restore pitr using backupSource on second cluster with another prefix in storage
-----------------------------------------------------------------------------------

+ second_cluster=cluster2-rs0
+ apply_cluster /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1850/e2e-tests/pitr/conf/cluster2-rs0.yml
+ '[' -z '' ']'
+ cat_config /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1850/e2e-tests/pitr/conf/cluster2-rs0.yml
+ kubectl_bin apply -f -
+ cat /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1850/e2e-tests/pitr/conf/cluster2-rs0.yml
+ yq eval '(.spec | select(.image == null)).image = "perconalab/percona-server-mongodb-operator:main-mongod8.0"'
+ yq eval '(.spec | select(has("pmm"))).pmm.image = "percona/pmm-client:2.44.1-1"'
++ mktemp
+ yq eval '(.spec | select(has("initImage"))).initImage = "perconalab/percona-server-mongodb-operator:PR-1850-e336896c"'
+ yq eval '(.spec | select(has("backup"))).backup.image = "percona/percona-backup-mongodb:2.11.0"'
+ yq eval '.spec.upgradeOptions.apply="Never"'
+ local LAST_OUT=/tmp/tmp.66xHZyReHs
++ mktemp
+ local LAST_ERR=/tmp/tmp.gigOPpUrG1
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl apply -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.66xHZyReHs
perconaservermongodb.psmdb.percona.com/cluster2 created
+ cat /tmp/tmp.gigOPpUrG1
+ rm /tmp/tmp.66xHZyReHs /tmp/tmp.gigOPpUrG1
+ return 0
+ desc 'check if all 3 Pods started'
+ set +o xtrace


-----------------------------------------------------------------------------------
check if all 3 Pods started
-----------------------------------------------------------------------------------

+ wait_for_running cluster2-rs0 3
+ local name=cluster2-rs0
+ let last_pod=2
+ local check_cluster_readyness=true
+ set_debug
+ [[ 1 == 1 ]]
+ set -o xtrace
+ local rs_name=rs0
+ local cluster_name=cluster2
++ seq 0 2
+ for i in $(seq 0 $last_pod)
+ [[ 0 -eq 2 ]]
+ wait_pod cluster2-rs0-0
+ local pod=cluster2-rs0-0
+ set +o xtrace
waiting for pod/cluster2-rs0-0 to be ready........OK
+ for i in $(seq 0 $last_pod)
+ [[ 1 -eq 2 ]]
+ wait_pod cluster2-rs0-1
+ local pod=cluster2-rs0-1
+ set +o xtrace
waiting for pod/cluster2-rs0-1 to be ready....OK
+ for i in $(seq 0 $last_pod)
+ [[ 2 -eq 2 ]]
++ kubectl_bin get psmdb cluster2 -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].arbiter.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.3yw5aKh27C
+++ mktemp
++ local LAST_ERR=/tmp/tmp.D5NJJusVab
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb cluster2 -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].arbiter.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.3yw5aKh27C
++ cat /tmp/tmp.D5NJJusVab
++ rm /tmp/tmp.3yw5aKh27C /tmp/tmp.D5NJJusVab
++ return 0
+ [[ '' == \t\r\u\e ]]
+ wait_pod cluster2-rs0-2
+ local pod=cluster2-rs0-2
+ set +o xtrace
waiting for pod/cluster2-rs0-2 to be ready......OK
++ kubectl_bin get psmdb cluster2 -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].nonvoting.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.zue96mFiML
+++ mktemp
++ local LAST_ERR=/tmp/tmp.s6mztRWHoM
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb cluster2 -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].nonvoting.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.zue96mFiML
++ cat /tmp/tmp.s6mztRWHoM
++ rm /tmp/tmp.zue96mFiML /tmp/tmp.s6mztRWHoM
++ return 0
+ [[ '' == \t\r\u\e ]]
++ kubectl_bin get psmdb cluster2 -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].hidden.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.vKG31prPHm
+++ mktemp
++ local LAST_ERR=/tmp/tmp.tGcyU99eqX
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb cluster2 -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].hidden.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.vKG31prPHm
++ cat /tmp/tmp.tGcyU99eqX
++ rm /tmp/tmp.vKG31prPHm /tmp/tmp.tGcyU99eqX
++ return 0
+ [[ '' == \t\r\u\e ]]
+ sleep 10
+ [[ true == \t\r\u\e ]]
+ set +x
Waiting for cluster readyness
+ wait_backup_agent cluster2-rs0-0
+ local agent_pod=cluster2-rs0-0
+ set +o xtrace
waiting for pbm-agent to be ready in cluster2-rs0-0...2025-11-30T20:28:55.000+0000 I listening for the commands

+ wait_backup_agent cluster2-rs0-1
+ local agent_pod=cluster2-rs0-1
+ set +o xtrace
waiting for pbm-agent to be ready in cluster2-rs0-1...2025-11-30T20:29:01.000+0000 I listening for the commands

+ wait_backup_agent cluster2-rs0-2
+ local agent_pod=cluster2-rs0-2
+ set +o xtrace
waiting for pbm-agent to be ready in cluster2-rs0-2...2025-11-30T20:29:03.000+0000 I listening for the commands

+ desc 'create user myApp'
+ set +o xtrace


-----------------------------------------------------------------------------------
create user myApp
-----------------------------------------------------------------------------------

+ run_mongo 'db.createUser({user:"myApp",pwd:"myPass",roles:[{db:"myApp",role:"readWrite"}]})' userAdmin:userAdmin123456@cluster2-rs0.pitr-32014
+ local 'command=db.createUser({user:"myApp",pwd:"myPass",roles:[{db:"myApp",role:"readWrite"}]})'
+ local uri=userAdmin:userAdmin123456@cluster2-rs0.pitr-32014
+ local driver=mongodb+srv
+ local suffix=.svc.cluster.local
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.2csYqNwas0
+++ mktemp
++ local LAST_ERR=/tmp/tmp.mbbG24E9V2
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.2csYqNwas0
++ cat /tmp/tmp.mbbG24E9V2
++ rm /tmp/tmp.2csYqNwas0 /tmp/tmp.mbbG24E9V2
++ return 0
+ local client_container=psmdb-client-696897d69b-whdv5
+ local mongo_flag=
+ [[ userAdmin:userAdmin123456@cluster2-rs0.pitr-32014 == *cfg* ]]
+ replica_set=rs0
+ kubectl_bin exec psmdb-client-696897d69b-whdv5 -- bash -c 'printf '\''db.createUser({user:"myApp",pwd:"myPass",roles:[{db:"myApp",role:"readWrite"}]})\n'\'' | mongo mongodb+srv://userAdmin:userAdmin123456@cluster2-rs0.pitr-32014.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
++ mktemp
+ local LAST_OUT=/tmp/tmp.jsmFijoVYp
++ mktemp
+ local LAST_ERR=/tmp/tmp.659yGQMi6k
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-696897d69b-whdv5 -- bash -c 'printf '\''db.createUser({user:"myApp",pwd:"myPass",roles:[{db:"myApp",role:"readWrite"}]})\n'\'' | mongo mongodb+srv://userAdmin:userAdmin123456@cluster2-rs0.pitr-32014.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.jsmFijoVYp
Percona Server for MongoDB shell version v4.4.29-28
connecting to: mongodb://cluster2-rs0-2.cluster2-rs0.pitr-32014.svc.cluster.local:27017,cluster2-rs0-0.cluster2-rs0.pitr-32014.svc.cluster.local:27017,cluster2-rs0-1.cluster2-rs0.pitr-32014.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb&replicaSet=rs0&ssl=false
Implicit session: session { "id" : UUID("f4622033-0fd6-466d-9d33-013a220e64ff") }
Percona Server for MongoDB server version: v8.0.12-4
WARNING: shell and server versions do not match
Successfully added user: {
	"user" : "myApp",
	"roles" : [
		{
			"db" : "myApp",
			"role" : "readWrite"
		}
	]
}
bye
+ cat /tmp/tmp.659yGQMi6k
+ rm /tmp/tmp.jsmFijoVYp /tmp/tmp.659yGQMi6k
+ return 0
+ sleep 2
+ check_recovery backup-minio-3 latest '' -4th backupSource cluster2 '' backup-minio-3-second-cluster restore2.yml
+ local backup_name=backup-minio-3
+ local restore_type=latest
+ local restore_date=
+ local cmp_postfix=-4th
+ local backupSource=backupSource
+ local cluster_name=cluster2
+ local selective_collection=
+ local restore_name=restore-backup-minio-3-second-cluster
+ local restore_file=restore2.yml
+ local cluster
+ cluster=cluster2-rs0
+ desc 'write more data before restore by latest'
+ set +o xtrace


-----------------------------------------------------------------------------------
write more data before restore by latest
-----------------------------------------------------------------------------------

+ sleep 60
+ run_mongo 'use myApp\n db.test.insert({ x: 100501 })' myApp:myPass@cluster2-rs0.pitr-32014
+ local 'command=use myApp\n db.test.insert({ x: 100501 })'
+ local uri=myApp:myPass@cluster2-rs0.pitr-32014
+ local driver=mongodb+srv
+ local suffix=.svc.cluster.local
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.2Fv7P19NDY
+++ mktemp
++ local LAST_ERR=/tmp/tmp.BntSOoQslL
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.2Fv7P19NDY
++ cat /tmp/tmp.BntSOoQslL
++ rm /tmp/tmp.2Fv7P19NDY /tmp/tmp.BntSOoQslL
++ return 0
+ local client_container=psmdb-client-696897d69b-whdv5
+ local mongo_flag=
+ [[ myApp:myPass@cluster2-rs0.pitr-32014 == *cfg* ]]
+ replica_set=rs0
+ kubectl_bin exec psmdb-client-696897d69b-whdv5 -- bash -c 'printf '\''use myApp\n db.test.insert({ x: 100501 })\n'\'' | mongo mongodb+srv://myApp:myPass@cluster2-rs0.pitr-32014.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
++ mktemp
+ local LAST_OUT=/tmp/tmp.rJX05jQeCW
++ mktemp
+ local LAST_ERR=/tmp/tmp.SRuVBVnjWJ
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-696897d69b-whdv5 -- bash -c 'printf '\''use myApp\n db.test.insert({ x: 100501 })\n'\'' | mongo mongodb+srv://myApp:myPass@cluster2-rs0.pitr-32014.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.rJX05jQeCW
Percona Server for MongoDB shell version v4.4.29-28
connecting to: mongodb://cluster2-rs0-0.cluster2-rs0.pitr-32014.svc.cluster.local:27017,cluster2-rs0-1.cluster2-rs0.pitr-32014.svc.cluster.local:27017,cluster2-rs0-2.cluster2-rs0.pitr-32014.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb&replicaSet=rs0&ssl=false
Implicit session: session { "id" : UUID("f53f2785-1c7b-407a-9ff4-bddb16dee5f7") }
Percona Server for MongoDB server version: v8.0.12-4
WARNING: shell and server versions do not match
switched to db myApp
WriteResult({ "nInserted" : 1 })
bye
+ cat /tmp/tmp.SRuVBVnjWJ
+ rm /tmp/tmp.rJX05jQeCW /tmp/tmp.SRuVBVnjWJ
+ return 0
+ '[' -n '' ']'
+ desc 'waiting for chunks to be uploaded'
+ set +o xtrace


-----------------------------------------------------------------------------------
waiting for chunks to be uploaded
-----------------------------------------------------------------------------------

+ sleep 120
+ desc 'check restore by latest'
+ set +o xtrace


-----------------------------------------------------------------------------------
check restore by latest
-----------------------------------------------------------------------------------

+ '[' -z backupSource ']'
++ get_backup_dest backup-minio-3
++ local backup_name=backup-minio-3
++ kubectl_bin get psmdb-backup backup-minio-3 -o 'jsonpath={.status.destination}'
++ sed -e 's/.json$//'
++ sed 's|s3://||'
++ sed 's|azure://||'
++ sed 's|gs://||'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.EYcJtgOhEi
+++ mktemp
++ local LAST_ERR=/tmp/tmp.DLpce64auo
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb-backup backup-minio-3 -o 'jsonpath={.status.destination}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.EYcJtgOhEi
++ cat /tmp/tmp.DLpce64auo
++ rm /tmp/tmp.EYcJtgOhEi /tmp/tmp.DLpce64auo
++ return 0
+ backup_dest=operator-testing/pitr-prefix-1/2025-11-30T20:23:34Z
+ cat /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1850/e2e-tests/pitr/conf/restore2.yml
+ /usr/sbin/sed -e 's/name:/name: restore-backup-minio-3-second-cluster/'
+ /usr/sbin/sed -e /backupName/d
+ /usr/sbin/sed -e 's/clusterName:/clusterName: cluster2/'
+ /usr/sbin/sed -e 's/pitrType:/type: latest/'
+ '[' -z '' ']'
+ /usr/sbin/sed -e /date:/d
+ '[' -n '' ']'
+ yq
+ /usr/sbin/sed -e 's|BACKUP-NAME|operator-testing/pitr-prefix-1/2025-11-30T20:23:34Z|'
+ kubectl_bin apply -f -
++ mktemp
+ local LAST_OUT=/tmp/tmp.lQvdebjG4Z
++ mktemp
+ local LAST_ERR=/tmp/tmp.oawr4sfeHf
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl apply -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.lQvdebjG4Z
perconaservermongodbrestore.psmdb.percona.com/restore-backup-minio-3-second-cluster created
+ cat /tmp/tmp.oawr4sfeHf
+ rm /tmp/tmp.lQvdebjG4Z /tmp/tmp.oawr4sfeHf
+ return 0
+ wait_restore backup-minio-3-second-cluster cluster2
+ local backup_name=backup-minio-3-second-cluster
+ local cluster_name=cluster2
+ local target_state=ready
+ local wait_cluster_consistency=1
+ local wait_time=1780
+ local ok_if_ready=0
+ set +o xtrace
Waiting for the psmdb-restore/restore-backup-minio-3-second-cluster object to be created.OK
Waiting psmdb-restore/restore-backup-minio-3-second-cluster to reach state "ready" .OK after 0 minutes
+ [[ 1 -eq 1 ]]
+ wait_cluster_consistency cluster2
+ local cluster_name=cluster2
+ local wait_time=32
+ retry=0
+ sleep 7
+ echo -n 'waiting for cluster readyness'
waiting for cluster readyness++ kubectl_bin get psmdb cluster2 -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.VjuZUu5sKC
+++ mktemp
++ local LAST_ERR=/tmp/tmp.f1Tu5FmbwN
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb cluster2 -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.VjuZUu5sKC
++ cat /tmp/tmp.f1Tu5FmbwN
++ rm /tmp/tmp.VjuZUu5sKC /tmp/tmp.f1Tu5FmbwN
++ return 0
+ [[ ready == \r\e\a\d\y ]]
+ echo .OK
.OK
+ echo

+ set -o xtrace
+ '[' -n '' ']'
+ compare_mongo_cmd find myApp:myPass@cluster2-rs0-0.cluster2-rs0.pitr-32014 -4th
+ local command=find
+ local uri=myApp:myPass@cluster2-rs0-0.cluster2-rs0.pitr-32014
+ local postfix=-4th
+ local suffix=
+ local database=myApp
+ local collection=test
+ local sort=
+ local tls=false
+ local 'full_command=db.test.find()'
+ [[ -n '' ]]
+ log 'running db.test.find() in myApp'
+ set +o xtrace
[2025-11-30T20:32:41+0000] running db.test.find() in myApp
+ [[ false == \t\r\u\e ]]
+ mongo_command=run_mongo
+ run_mongo 'use myApp\n db.test.find()' myApp:myPass@cluster2-rs0-0.cluster2-rs0.pitr-32014 mongodb ''
+ local 'command=use myApp\n db.test.find()'
+ local uri=myApp:myPass@cluster2-rs0-0.cluster2-rs0.pitr-32014
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ egrep -v 'I NETWORK|W NETWORK|F NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|Started a new thread for the timer service'
+ /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
egrep: warning: egrep is obsolescent; using grep -E
++ local LAST_OUT=/tmp/tmp.FzSAsBN6es
+++ mktemp
++ local LAST_ERR=/tmp/tmp.G4lxARs1BN
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.FzSAsBN6es
++ cat /tmp/tmp.G4lxARs1BN
++ rm /tmp/tmp.FzSAsBN6es /tmp/tmp.G4lxARs1BN
++ return 0
+ local client_container=psmdb-client-696897d69b-whdv5
+ local mongo_flag=
+ [[ myApp:myPass@cluster2-rs0-0.cluster2-rs0.pitr-32014 == *cfg* ]]
+ replica_set=rs0
+ kubectl_bin exec psmdb-client-696897d69b-whdv5 -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@cluster2-rs0-0.cluster2-rs0.pitr-32014.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
++ mktemp
+ local LAST_OUT=/tmp/tmp.NIDEL2v9VN
++ mktemp
+ local LAST_ERR=/tmp/tmp.RTyOYNBAbN
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-696897d69b-whdv5 -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@cluster2-rs0-0.cluster2-rs0.pitr-32014.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.NIDEL2v9VN
+ cat /tmp/tmp.RTyOYNBAbN
+ rm /tmp/tmp.NIDEL2v9VN /tmp/tmp.RTyOYNBAbN
+ return 0
+ [[ 0 -eq 0 ]]
+ diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1850/e2e-tests/pitr/compare/find-4th.json /tmp/tmp.1qPVwubFaS/find-4th
+ compare_mongo_cmd find myApp:myPass@cluster2-rs0-1.cluster2-rs0.pitr-32014 -4th
+ local command=find
+ local uri=myApp:myPass@cluster2-rs0-1.cluster2-rs0.pitr-32014
+ local postfix=-4th
+ local suffix=
+ local database=myApp
+ local collection=test
+ local sort=
+ local tls=false
+ local 'full_command=db.test.find()'
+ [[ -n '' ]]
+ log 'running db.test.find() in myApp'
+ set +o xtrace
[2025-11-30T20:32:43+0000] running db.test.find() in myApp
+ [[ false == \t\r\u\e ]]
+ mongo_command=run_mongo
+ run_mongo 'use myApp\n db.test.find()' myApp:myPass@cluster2-rs0-1.cluster2-rs0.pitr-32014 mongodb ''
+ egrep -v 'I NETWORK|W NETWORK|F NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|Started a new thread for the timer service'
+ /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
+ local 'command=use myApp\n db.test.find()'
+ local uri=myApp:myPass@cluster2-rs0-1.cluster2-rs0.pitr-32014
+ local driver=mongodb
+ local suffix=.svc.cluster.local
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
egrep: warning: egrep is obsolescent; using grep -E
+++ mktemp
++ local LAST_OUT=/tmp/tmp.TS31A4fggS
+++ mktemp
++ local LAST_ERR=/tmp/tmp.v10zt9B0ep
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.TS31A4fggS
++ cat /tmp/tmp.v10zt9B0ep
++ rm /tmp/tmp.TS31A4fggS /tmp/tmp.v10zt9B0ep
++ return 0
+ local client_container=psmdb-client-696897d69b-whdv5
+ local mongo_flag=
+ [[ myApp:myPass@cluster2-rs0-1.cluster2-rs0.pitr-32014 == *cfg* ]]
+ replica_set=rs0
+ kubectl_bin exec psmdb-client-696897d69b-whdv5 -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@cluster2-rs0-1.cluster2-rs0.pitr-32014.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
++ mktemp
+ local LAST_OUT=/tmp/tmp.KPN4TqgIZn
++ mktemp
+ local LAST_ERR=/tmp/tmp.daXBSR1Flx
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-696897d69b-whdv5 -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@cluster2-rs0-1.cluster2-rs0.pitr-32014.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.KPN4TqgIZn
+ cat /tmp/tmp.daXBSR1Flx
+ rm /tmp/tmp.KPN4TqgIZn /tmp/tmp.daXBSR1Flx
+ return 0
+ [[ 0 -eq 0 ]]
+ diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1850/e2e-tests/pitr/compare/find-4th.json /tmp/tmp.1qPVwubFaS/find-4th
+ compare_mongo_cmd find myApp:myPass@cluster2-rs0-2.cluster2-rs0.pitr-32014 -4th
+ local command=find
+ local uri=myApp:myPass@cluster2-rs0-2.cluster2-rs0.pitr-32014
+ local postfix=-4th
+ local suffix=
+ local database=myApp
+ local collection=test
+ local sort=
+ local tls=false
+ local 'full_command=db.test.find()'
+ [[ -n '' ]]
+ log 'running db.test.find() in myApp'
+ set +o xtrace
[2025-11-30T20:32:46+0000] running db.test.find() in myApp
+ [[ false == \t\r\u\e ]]
+ mongo_command=run_mongo
+ run_mongo 'use myApp\n db.test.find()' myApp:myPass@cluster2-rs0-2.cluster2-rs0.pitr-32014 mongodb ''
+ local 'command=use myApp\n db.test.find()'
+ egrep -v 'I NETWORK|W NETWORK|F NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|Started a new thread for the timer service'
+ local uri=myApp:myPass@cluster2-rs0-2.cluster2-rs0.pitr-32014
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
egrep: warning: egrep is obsolescent; using grep -E
++ local LAST_OUT=/tmp/tmp.eYZRR5dUqo
+++ mktemp
++ local LAST_ERR=/tmp/tmp.hdrnYThHkL
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.eYZRR5dUqo
++ cat /tmp/tmp.hdrnYThHkL
++ rm /tmp/tmp.eYZRR5dUqo /tmp/tmp.hdrnYThHkL
++ return 0
+ local client_container=psmdb-client-696897d69b-whdv5
+ local mongo_flag=
+ [[ myApp:myPass@cluster2-rs0-2.cluster2-rs0.pitr-32014 == *cfg* ]]
+ replica_set=rs0
+ kubectl_bin exec psmdb-client-696897d69b-whdv5 -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@cluster2-rs0-2.cluster2-rs0.pitr-32014.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
++ mktemp
+ local LAST_OUT=/tmp/tmp.lAhTP2hx02
++ mktemp
+ local LAST_ERR=/tmp/tmp.HI7Afa1sPQ
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-696897d69b-whdv5 -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@cluster2-rs0-2.cluster2-rs0.pitr-32014.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.lAhTP2hx02
+ cat /tmp/tmp.HI7Afa1sPQ
+ rm /tmp/tmp.lAhTP2hx02 /tmp/tmp.HI7Afa1sPQ
+ return 0
+ [[ 0 -eq 0 ]]
+ diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1850/e2e-tests/pitr/compare/find-4th.json /tmp/tmp.1qPVwubFaS/find-4th
+ backup_name_minio_2=backup-minio2
+ run_backup backup-minio2 0
+ local name=backup-minio2
+ local idx=0
+ desc 'run backup backup-minio2-0'
+ set +o xtrace


-----------------------------------------------------------------------------------
run backup backup-minio2-0
-----------------------------------------------------------------------------------

+ cat /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1850/e2e-tests/pitr/conf/backup-minio2.yml
+ /usr/sbin/sed -e 's/name:/name: backup-minio2-0/'
+ kubectl_bin apply -f -
++ mktemp
+ local LAST_OUT=/tmp/tmp.DdzV3kCiJG
++ mktemp
+ local LAST_ERR=/tmp/tmp.bALZ1NUA4V
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl apply -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.DdzV3kCiJG
perconaservermongodbbackup.psmdb.percona.com/backup-minio2-0 created
+ cat /tmp/tmp.bALZ1NUA4V
+ rm /tmp/tmp.DdzV3kCiJG /tmp/tmp.bALZ1NUA4V
+ return 0
+ wait_backup backup-minio2-0
+ local backup_name=backup-minio2-0
+ local target_state=ready
+ set +o xtrace
waiting for backup-minio2-0 to reach ready state......OK
++ run_mongo 'new Date().toISOString()' myApp:myPass@cluster2-rs0.pitr-32014 mongodb '' --quiet
++ local 'command=new Date().toISOString()'
++ local uri=myApp:myPass@cluster2-rs0.pitr-32014
++ local driver=mongodb
++ local suffix=.svc.cluster.local
++ grep -E -v 'I NETWORK|W NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match'
++ cut -c1-19
++ tr T ' '
+++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.Ufs1juolwL
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.bQPQWj3kuD
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in $(seq 0 2)
+++ set +e
+++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.Ufs1juolwL
+++ cat /tmp/tmp.bQPQWj3kuD
+++ rm /tmp/tmp.Ufs1juolwL /tmp/tmp.bQPQWj3kuD
+++ return 0
++ local client_container=psmdb-client-696897d69b-whdv5
++ local mongo_flag=--quiet
++ [[ myApp:myPass@cluster2-rs0.pitr-32014 == *cfg* ]]
++ replica_set=rs0
++ kubectl_bin exec psmdb-client-696897d69b-whdv5 -- bash -c 'printf '\''new Date().toISOString()\n'\'' | mongo mongodb://myApp:myPass@cluster2-rs0.pitr-32014.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 --quiet'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.H8WTTcG2TL
+++ mktemp
++ local LAST_ERR=/tmp/tmp.ioVg1bawNf
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl exec psmdb-client-696897d69b-whdv5 -- bash -c 'printf '\''new Date().toISOString()\n'\'' | mongo mongodb://myApp:myPass@cluster2-rs0.pitr-32014.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 --quiet'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.H8WTTcG2TL
++ cat /tmp/tmp.ioVg1bawNf
++ rm /tmp/tmp.H8WTTcG2TL /tmp/tmp.ioVg1bawNf
++ return 0
+ time_now='2025-11-30 20:33:05'
+ write_document -5th cluster2-rs0
+ local cmp_postfix=-5th
+ local cluster_name=cluster2-rs0
+ desc 'write initial data, read from all'
+ set +o xtrace


-----------------------------------------------------------------------------------
write initial data, read from all
-----------------------------------------------------------------------------------

+ run_mongo 'use myApp\n db.test.insert({ x: 100500 })' myApp:myPass@cluster2-rs0.pitr-32014
+ local 'command=use myApp\n db.test.insert({ x: 100500 })'
+ local uri=myApp:myPass@cluster2-rs0.pitr-32014
+ local driver=mongodb+srv
+ local suffix=.svc.cluster.local
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.xujn6Hjm4B
+++ mktemp
++ local LAST_ERR=/tmp/tmp.BHx1SU3TSD
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.xujn6Hjm4B
++ cat /tmp/tmp.BHx1SU3TSD
++ rm /tmp/tmp.xujn6Hjm4B /tmp/tmp.BHx1SU3TSD
++ return 0
+ local client_container=psmdb-client-696897d69b-whdv5
+ local mongo_flag=
+ [[ myApp:myPass@cluster2-rs0.pitr-32014 == *cfg* ]]
+ replica_set=rs0
+ kubectl_bin exec psmdb-client-696897d69b-whdv5 -- bash -c 'printf '\''use myApp\n db.test.insert({ x: 100500 })\n'\'' | mongo mongodb+srv://myApp:myPass@cluster2-rs0.pitr-32014.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
++ mktemp
+ local LAST_OUT=/tmp/tmp.1M3YNbDef7
++ mktemp
+ local LAST_ERR=/tmp/tmp.sDtHH3V4wh
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-696897d69b-whdv5 -- bash -c 'printf '\''use myApp\n db.test.insert({ x: 100500 })\n'\'' | mongo mongodb+srv://myApp:myPass@cluster2-rs0.pitr-32014.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.1M3YNbDef7
Percona Server for MongoDB shell version v4.4.29-28
connecting to: mongodb://cluster2-rs0-0.cluster2-rs0.pitr-32014.svc.cluster.local:27017,cluster2-rs0-1.cluster2-rs0.pitr-32014.svc.cluster.local:27017,cluster2-rs0-2.cluster2-rs0.pitr-32014.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb&replicaSet=rs0&ssl=false
Implicit session: session { "id" : UUID("a12d91b3-09f5-4a34-bddf-20a4689f019b") }
Percona Server for MongoDB server version: v8.0.12-4
WARNING: shell and server versions do not match
switched to db myApp
WriteResult({ "nInserted" : 1 })
bye
+ cat /tmp/tmp.sDtHH3V4wh
+ rm /tmp/tmp.1M3YNbDef7 /tmp/tmp.sDtHH3V4wh
+ return 0
+ minikube_sleep
+ sleep_time=10
+ [[ '' == 1 ]]
+ compare_mongo_cmd find myApp:myPass@cluster2-rs0-0.cluster2-rs0.pitr-32014 -5th
+ local command=find
+ local uri=myApp:myPass@cluster2-rs0-0.cluster2-rs0.pitr-32014
+ local postfix=-5th
+ local suffix=
+ local database=myApp
+ local collection=test
+ local sort=
+ local tls=false
+ local 'full_command=db.test.find()'
+ [[ -n '' ]]
+ log 'running db.test.find() in myApp'
+ set +o xtrace
[2025-11-30T20:33:08+0000] running db.test.find() in myApp
+ [[ false == \t\r\u\e ]]
+ mongo_command=run_mongo
+ run_mongo 'use myApp\n db.test.find()' myApp:myPass@cluster2-rs0-0.cluster2-rs0.pitr-32014 mongodb ''
+ local 'command=use myApp\n db.test.find()'
+ local uri=myApp:myPass@cluster2-rs0-0.cluster2-rs0.pitr-32014
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ egrep -v 'I NETWORK|W NETWORK|F NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|Started a new thread for the timer service'
+ /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
egrep: warning: egrep is obsolescent; using grep -E
++ local LAST_OUT=/tmp/tmp.0bFq2CH1QV
+++ mktemp
++ local LAST_ERR=/tmp/tmp.ZfPlJc6YnW
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.0bFq2CH1QV
++ cat /tmp/tmp.ZfPlJc6YnW
++ rm /tmp/tmp.0bFq2CH1QV /tmp/tmp.ZfPlJc6YnW
++ return 0
+ local client_container=psmdb-client-696897d69b-whdv5
+ local mongo_flag=
+ [[ myApp:myPass@cluster2-rs0-0.cluster2-rs0.pitr-32014 == *cfg* ]]
+ replica_set=rs0
+ kubectl_bin exec psmdb-client-696897d69b-whdv5 -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@cluster2-rs0-0.cluster2-rs0.pitr-32014.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
++ mktemp
+ local LAST_OUT=/tmp/tmp.kquH3YligQ
++ mktemp
+ local LAST_ERR=/tmp/tmp.8VpSpUyVHM
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-696897d69b-whdv5 -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@cluster2-rs0-0.cluster2-rs0.pitr-32014.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.kquH3YligQ
+ cat /tmp/tmp.8VpSpUyVHM
+ rm /tmp/tmp.kquH3YligQ /tmp/tmp.8VpSpUyVHM
+ return 0
+ [[ 0 -eq 0 ]]
+ diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1850/e2e-tests/pitr/compare/find-5th.json /tmp/tmp.1qPVwubFaS/find-5th
+ compare_mongo_cmd find myApp:myPass@cluster2-rs0-1.cluster2-rs0.pitr-32014 -5th
+ local command=find
+ local uri=myApp:myPass@cluster2-rs0-1.cluster2-rs0.pitr-32014
+ local postfix=-5th
+ local suffix=
+ local database=myApp
+ local collection=test
+ local sort=
+ local tls=false
+ local 'full_command=db.test.find()'
+ [[ -n '' ]]
+ log 'running db.test.find() in myApp'
+ set +o xtrace
[2025-11-30T20:33:11+0000] running db.test.find() in myApp
+ [[ false == \t\r\u\e ]]
+ mongo_command=run_mongo
+ run_mongo 'use myApp\n db.test.find()' myApp:myPass@cluster2-rs0-1.cluster2-rs0.pitr-32014 mongodb ''
+ local 'command=use myApp\n db.test.find()'
+ local uri=myApp:myPass@cluster2-rs0-1.cluster2-rs0.pitr-32014
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ egrep -v 'I NETWORK|W NETWORK|F NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|Started a new thread for the timer service'
+ /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
egrep: warning: egrep is obsolescent; using grep -E
++ local LAST_OUT=/tmp/tmp.4oCfb9iI0b
+++ mktemp
++ local LAST_ERR=/tmp/tmp.jT0w2BZLUN
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.4oCfb9iI0b
++ cat /tmp/tmp.jT0w2BZLUN
++ rm /tmp/tmp.4oCfb9iI0b /tmp/tmp.jT0w2BZLUN
++ return 0
+ local client_container=psmdb-client-696897d69b-whdv5
+ local mongo_flag=
+ [[ myApp:myPass@cluster2-rs0-1.cluster2-rs0.pitr-32014 == *cfg* ]]
+ replica_set=rs0
+ kubectl_bin exec psmdb-client-696897d69b-whdv5 -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@cluster2-rs0-1.cluster2-rs0.pitr-32014.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
++ mktemp
+ local LAST_OUT=/tmp/tmp.YbqXPNWvsS
++ mktemp
+ local LAST_ERR=/tmp/tmp.Rv6xqr9Okg
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-696897d69b-whdv5 -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@cluster2-rs0-1.cluster2-rs0.pitr-32014.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.YbqXPNWvsS
+ cat /tmp/tmp.Rv6xqr9Okg
+ rm /tmp/tmp.YbqXPNWvsS /tmp/tmp.Rv6xqr9Okg
+ return 0
+ [[ 0 -eq 0 ]]
+ diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1850/e2e-tests/pitr/compare/find-5th.json /tmp/tmp.1qPVwubFaS/find-5th
+ compare_mongo_cmd find myApp:myPass@cluster2-rs0-2.cluster2-rs0.pitr-32014 -5th
+ local command=find
+ local uri=myApp:myPass@cluster2-rs0-2.cluster2-rs0.pitr-32014
+ local postfix=-5th
+ local suffix=
+ local database=myApp
+ local collection=test
+ local sort=
+ local tls=false
+ local 'full_command=db.test.find()'
+ [[ -n '' ]]
+ log 'running db.test.find() in myApp'
+ set +o xtrace
[2025-11-30T20:33:14+0000] running db.test.find() in myApp
+ [[ false == \t\r\u\e ]]
+ mongo_command=run_mongo
+ run_mongo 'use myApp\n db.test.find()' myApp:myPass@cluster2-rs0-2.cluster2-rs0.pitr-32014 mongodb ''
+ local 'command=use myApp\n db.test.find()'
+ local uri=myApp:myPass@cluster2-rs0-2.cluster2-rs0.pitr-32014
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ egrep -v 'I NETWORK|W NETWORK|F NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|Started a new thread for the timer service'
+ /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
egrep: warning: egrep is obsolescent; using grep -E
++ local LAST_OUT=/tmp/tmp.SlSbtihamD
+++ mktemp
++ local LAST_ERR=/tmp/tmp.01ESMsdsoH
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.SlSbtihamD
++ cat /tmp/tmp.01ESMsdsoH
++ rm /tmp/tmp.SlSbtihamD /tmp/tmp.01ESMsdsoH
++ return 0
+ local client_container=psmdb-client-696897d69b-whdv5
+ local mongo_flag=
+ [[ myApp:myPass@cluster2-rs0-2.cluster2-rs0.pitr-32014 == *cfg* ]]
+ replica_set=rs0
+ kubectl_bin exec psmdb-client-696897d69b-whdv5 -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@cluster2-rs0-2.cluster2-rs0.pitr-32014.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
++ mktemp
+ local LAST_OUT=/tmp/tmp.mYLzvYLq5f
++ mktemp
+ local LAST_ERR=/tmp/tmp.ifJuMTat2i
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-696897d69b-whdv5 -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@cluster2-rs0-2.cluster2-rs0.pitr-32014.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.mYLzvYLq5f
+ cat /tmp/tmp.ifJuMTat2i
+ rm /tmp/tmp.mYLzvYLq5f /tmp/tmp.ifJuMTat2i
+ return 0
+ [[ 0 -eq 0 ]]
+ diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1850/e2e-tests/pitr/compare/find-5th.json /tmp/tmp.1qPVwubFaS/find-5th
+ sleep 2
+ check_recovery backup-minio2-0 date '2025-11-30 20:33:05' -4th '' cluster2
+ local backup_name=backup-minio2-0
+ local restore_type=date
+ local 'restore_date=2025-11-30 20:33:05'
+ local cmp_postfix=-4th
+ local backupSource=
+ local cluster_name=cluster2
+ local selective_collection=
+ local restore_name=restore-backup-minio2-0
+ local restore_file=restore.yml
+ local cluster
+ cluster=cluster2-rs0
+ desc 'write more data before restore by date'
+ set +o xtrace


-----------------------------------------------------------------------------------
write more data before restore by date
-----------------------------------------------------------------------------------

+ sleep 60
+ run_mongo 'use myApp\n db.test.insert({ x: 100501 })' myApp:myPass@cluster2-rs0.pitr-32014
+ local 'command=use myApp\n db.test.insert({ x: 100501 })'
+ local uri=myApp:myPass@cluster2-rs0.pitr-32014
+ local driver=mongodb+srv
+ local suffix=.svc.cluster.local
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.mPjObGJwZx
+++ mktemp
++ local LAST_ERR=/tmp/tmp.52cUrqnPz7
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.mPjObGJwZx
++ cat /tmp/tmp.52cUrqnPz7
++ rm /tmp/tmp.mPjObGJwZx /tmp/tmp.52cUrqnPz7
++ return 0
+ local client_container=psmdb-client-696897d69b-whdv5
+ local mongo_flag=
+ [[ myApp:myPass@cluster2-rs0.pitr-32014 == *cfg* ]]
+ replica_set=rs0
+ kubectl_bin exec psmdb-client-696897d69b-whdv5 -- bash -c 'printf '\''use myApp\n db.test.insert({ x: 100501 })\n'\'' | mongo mongodb+srv://myApp:myPass@cluster2-rs0.pitr-32014.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
++ mktemp
+ local LAST_OUT=/tmp/tmp.n66a2eaY8z
++ mktemp
+ local LAST_ERR=/tmp/tmp.eyfyZKZwHC
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-696897d69b-whdv5 -- bash -c 'printf '\''use myApp\n db.test.insert({ x: 100501 })\n'\'' | mongo mongodb+srv://myApp:myPass@cluster2-rs0.pitr-32014.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.n66a2eaY8z
Percona Server for MongoDB shell version v4.4.29-28
connecting to: mongodb://cluster2-rs0-1.cluster2-rs0.pitr-32014.svc.cluster.local:27017,cluster2-rs0-2.cluster2-rs0.pitr-32014.svc.cluster.local:27017,cluster2-rs0-0.cluster2-rs0.pitr-32014.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb&replicaSet=rs0&ssl=false
Implicit session: session { "id" : UUID("68ea17ea-d2f7-4dee-b705-5b310a8eb20e") }
Percona Server for MongoDB server version: v8.0.12-4
WARNING: shell and server versions do not match
switched to db myApp
WriteResult({ "nInserted" : 1 })
bye
+ cat /tmp/tmp.eyfyZKZwHC
+ rm /tmp/tmp.n66a2eaY8z /tmp/tmp.eyfyZKZwHC
+ return 0
+ '[' -n '' ']'
+ desc 'waiting for chunks to be uploaded'
+ set +o xtrace


-----------------------------------------------------------------------------------
waiting for chunks to be uploaded
-----------------------------------------------------------------------------------

+ sleep 120
+ desc 'check restore by date'
+ set +o xtrace


-----------------------------------------------------------------------------------
check restore by date
-----------------------------------------------------------------------------------

+ '[' -z '' ']'
+ cat /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1850/e2e-tests/pitr/conf/restore.yml
+ /usr/sbin/sed -e 's/clusterName:/clusterName: cluster2/'
+ /usr/sbin/sed -e 's/pitrType:/type: date/'
+ '[' -n '' ']'
+ yq
+ /usr/sbin/sed -e 's/name:/name: restore-backup-minio2-0/'
+ /usr/sbin/sed -e 's/backupName:/backupName: backup-minio2-0/'
+ '[' -z '2025-11-30 20:33:05' ']'
+ /usr/sbin/sed -e 's/date:/date: 2025-11-30 20:33:05/'
+ /usr/sbin/sed -e /backupSource/,+2d
+ kubectl_bin apply -f -
++ mktemp
+ local LAST_OUT=/tmp/tmp.lnm3WtUCMI
++ mktemp
+ local LAST_ERR=/tmp/tmp.BmvMtQi4Mh
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl apply -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.lnm3WtUCMI
perconaservermongodbrestore.psmdb.percona.com/restore-backup-minio2-0 created
+ cat /tmp/tmp.BmvMtQi4Mh
+ rm /tmp/tmp.lnm3WtUCMI /tmp/tmp.BmvMtQi4Mh
+ return 0
+ wait_restore backup-minio2-0 cluster2
+ local backup_name=backup-minio2-0
+ local cluster_name=cluster2
+ local target_state=ready
+ local wait_cluster_consistency=1
+ local wait_time=1780
+ local ok_if_ready=0
+ set +o xtrace
Waiting for the psmdb-restore/restore-backup-minio2-0 object to be created.OK
Waiting psmdb-restore/restore-backup-minio2-0 to reach state "ready" .OK after 0 minutes
+ [[ 1 -eq 1 ]]
+ wait_cluster_consistency cluster2
+ local cluster_name=cluster2
+ local wait_time=32
+ retry=0
+ sleep 7
+ echo -n 'waiting for cluster readyness'
waiting for cluster readyness++ kubectl_bin get psmdb cluster2 -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.y35WLve9nK
+++ mktemp
++ local LAST_ERR=/tmp/tmp.GPZJ4wRlQP
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb cluster2 -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.y35WLve9nK
++ cat /tmp/tmp.GPZJ4wRlQP
++ rm /tmp/tmp.y35WLve9nK /tmp/tmp.GPZJ4wRlQP
++ return 0
+ [[ ready == \r\e\a\d\y ]]
+ echo .OK
.OK
+ echo

+ set -o xtrace
+ '[' -n '' ']'
+ compare_mongo_cmd find myApp:myPass@cluster2-rs0-0.cluster2-rs0.pitr-32014 -4th
+ local command=find
+ local uri=myApp:myPass@cluster2-rs0-0.cluster2-rs0.pitr-32014
+ local postfix=-4th
+ local suffix=
+ local database=myApp
+ local collection=test
+ local sort=
+ local tls=false
+ local 'full_command=db.test.find()'
+ [[ -n '' ]]
+ log 'running db.test.find() in myApp'
+ set +o xtrace
[2025-11-30T20:36:49+0000] running db.test.find() in myApp
+ [[ false == \t\r\u\e ]]
+ mongo_command=run_mongo
+ run_mongo 'use myApp\n db.test.find()' myApp:myPass@cluster2-rs0-0.cluster2-rs0.pitr-32014 mongodb ''
+ local 'command=use myApp\n db.test.find()'
+ local uri=myApp:myPass@cluster2-rs0-0.cluster2-rs0.pitr-32014
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ egrep -v 'I NETWORK|W NETWORK|F NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|Started a new thread for the timer service'
+ /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
egrep: warning: egrep is obsolescent; using grep -E
++ local LAST_OUT=/tmp/tmp.9WCEY9NGPf
+++ mktemp
++ local LAST_ERR=/tmp/tmp.gEaDmUN46L
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.9WCEY9NGPf
++ cat /tmp/tmp.gEaDmUN46L
++ rm /tmp/tmp.9WCEY9NGPf /tmp/tmp.gEaDmUN46L
++ return 0
+ local client_container=psmdb-client-696897d69b-whdv5
+ local mongo_flag=
+ [[ myApp:myPass@cluster2-rs0-0.cluster2-rs0.pitr-32014 == *cfg* ]]
+ replica_set=rs0
+ kubectl_bin exec psmdb-client-696897d69b-whdv5 -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@cluster2-rs0-0.cluster2-rs0.pitr-32014.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
++ mktemp
+ local LAST_OUT=/tmp/tmp.qhPFA5M2Mh
++ mktemp
+ local LAST_ERR=/tmp/tmp.tcGUjKFXOH
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-696897d69b-whdv5 -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@cluster2-rs0-0.cluster2-rs0.pitr-32014.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.qhPFA5M2Mh
+ cat /tmp/tmp.tcGUjKFXOH
+ rm /tmp/tmp.qhPFA5M2Mh /tmp/tmp.tcGUjKFXOH
+ return 0
+ [[ 0 -eq 0 ]]
+ diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1850/e2e-tests/pitr/compare/find-4th.json /tmp/tmp.1qPVwubFaS/find-4th
+ compare_mongo_cmd find myApp:myPass@cluster2-rs0-1.cluster2-rs0.pitr-32014 -4th
+ local command=find
+ local uri=myApp:myPass@cluster2-rs0-1.cluster2-rs0.pitr-32014
+ local postfix=-4th
+ local suffix=
+ local database=myApp
+ local collection=test
+ local sort=
+ local tls=false
+ local 'full_command=db.test.find()'
+ [[ -n '' ]]
+ log 'running db.test.find() in myApp'
+ set +o xtrace
[2025-11-30T20:36:52+0000] running db.test.find() in myApp
+ [[ false == \t\r\u\e ]]
+ mongo_command=run_mongo
+ run_mongo 'use myApp\n db.test.find()' myApp:myPass@cluster2-rs0-1.cluster2-rs0.pitr-32014 mongodb ''
+ local 'command=use myApp\n db.test.find()'
+ egrep -v 'I NETWORK|W NETWORK|F NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|Started a new thread for the timer service'
+ local uri=myApp:myPass@cluster2-rs0-1.cluster2-rs0.pitr-32014
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
egrep: warning: egrep is obsolescent; using grep -E
++ local LAST_OUT=/tmp/tmp.UxKCCJaJfw
+++ mktemp
++ local LAST_ERR=/tmp/tmp.YTpqXvWS8q
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.UxKCCJaJfw
++ cat /tmp/tmp.YTpqXvWS8q
++ rm /tmp/tmp.UxKCCJaJfw /tmp/tmp.YTpqXvWS8q
++ return 0
+ local client_container=psmdb-client-696897d69b-whdv5
+ local mongo_flag=
+ [[ myApp:myPass@cluster2-rs0-1.cluster2-rs0.pitr-32014 == *cfg* ]]
+ replica_set=rs0
+ kubectl_bin exec psmdb-client-696897d69b-whdv5 -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@cluster2-rs0-1.cluster2-rs0.pitr-32014.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
++ mktemp
+ local LAST_OUT=/tmp/tmp.8kJlO4O0sr
++ mktemp
+ local LAST_ERR=/tmp/tmp.CZjCdFutuC
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-696897d69b-whdv5 -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@cluster2-rs0-1.cluster2-rs0.pitr-32014.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.8kJlO4O0sr
+ cat /tmp/tmp.CZjCdFutuC
+ rm /tmp/tmp.8kJlO4O0sr /tmp/tmp.CZjCdFutuC
+ return 0
+ [[ 0 -eq 0 ]]
+ diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1850/e2e-tests/pitr/compare/find-4th.json /tmp/tmp.1qPVwubFaS/find-4th
+ compare_mongo_cmd find myApp:myPass@cluster2-rs0-2.cluster2-rs0.pitr-32014 -4th
+ local command=find
+ local uri=myApp:myPass@cluster2-rs0-2.cluster2-rs0.pitr-32014
+ local postfix=-4th
+ local suffix=
+ local database=myApp
+ local collection=test
+ local sort=
+ local tls=false
+ local 'full_command=db.test.find()'
+ [[ -n '' ]]
+ log 'running db.test.find() in myApp'
+ set +o xtrace
[2025-11-30T20:36:54+0000] running db.test.find() in myApp
+ [[ false == \t\r\u\e ]]
+ mongo_command=run_mongo
+ run_mongo 'use myApp\n db.test.find()' myApp:myPass@cluster2-rs0-2.cluster2-rs0.pitr-32014 mongodb ''
+ local 'command=use myApp\n db.test.find()'
+ local uri=myApp:myPass@cluster2-rs0-2.cluster2-rs0.pitr-32014
+ egrep -v 'I NETWORK|W NETWORK|F NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|Started a new thread for the timer service'
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
egrep: warning: egrep is obsolescent; using grep -E
++ local LAST_OUT=/tmp/tmp.jrxvpksJbk
+++ mktemp
++ local LAST_ERR=/tmp/tmp.amph2vkorT
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.jrxvpksJbk
++ cat /tmp/tmp.amph2vkorT
++ rm /tmp/tmp.jrxvpksJbk /tmp/tmp.amph2vkorT
++ return 0
+ local client_container=psmdb-client-696897d69b-whdv5
+ local mongo_flag=
+ [[ myApp:myPass@cluster2-rs0-2.cluster2-rs0.pitr-32014 == *cfg* ]]
+ replica_set=rs0
+ kubectl_bin exec psmdb-client-696897d69b-whdv5 -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@cluster2-rs0-2.cluster2-rs0.pitr-32014.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
++ mktemp
+ local LAST_OUT=/tmp/tmp.pkB9guIdIu
++ mktemp
+ local LAST_ERR=/tmp/tmp.wVak90r7KR
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-696897d69b-whdv5 -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@cluster2-rs0-2.cluster2-rs0.pitr-32014.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.pkB9guIdIu
+ cat /tmp/tmp.wVak90r7KR
+ rm /tmp/tmp.pkB9guIdIu /tmp/tmp.wVak90r7KR
+ return 0
+ [[ 0 -eq 0 ]]
+ diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1850/e2e-tests/pitr/compare/find-4th.json /tmp/tmp.1qPVwubFaS/find-4th
+ kubectl patch psmdb some-name --type=merge --patch '{"spec": {"backup": {"pitr": {"enabled": false}}}}'
perconaservermongodb.psmdb.percona.com/some-name patched
+ kubectl patch psmdb cluster2 --type=merge --patch '{"spec": {"backup": {"pitr": {"enabled": false}}}}'
perconaservermongodb.psmdb.percona.com/cluster2 patched
+ sleep 20
+ kubectl_bin delete psmdb-backup --all
++ mktemp
+ local LAST_OUT=/tmp/tmp.I4ahlNfzKD
++ mktemp
+ local LAST_ERR=/tmp/tmp.OqdaTl2I73
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl delete psmdb-backup --all
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.I4ahlNfzKD
perconaservermongodbbackup.psmdb.percona.com "backup-minio-0" deleted from pitr-32014 namespace
perconaservermongodbbackup.psmdb.percona.com "backup-minio-1" deleted from pitr-32014 namespace
perconaservermongodbbackup.psmdb.percona.com "backup-minio-2" deleted from pitr-32014 namespace
perconaservermongodbbackup.psmdb.percona.com "backup-minio-3" deleted from pitr-32014 namespace
perconaservermongodbbackup.psmdb.percona.com "backup-minio2-0" deleted from pitr-32014 namespace
+ cat /tmp/tmp.OqdaTl2I73
+ rm /tmp/tmp.I4ahlNfzKD /tmp/tmp.OqdaTl2I73
+ return 0
+ destroy pitr-32014
+ local namespace=pitr-32014
+ local ignore_logs=true
+ [[ 0 == 1 ]]
+ desc 'destroy cluster/operator and all other resources'
+ set +o xtrace


-----------------------------------------------------------------------------------
destroy cluster/operator and all other resources
-----------------------------------------------------------------------------------

+ '[' true == false ']'
+ delete_backups
+ desc 'Delete psmdb-backup'
+ set +o xtrace


-----------------------------------------------------------------------------------
Delete psmdb-backup
-----------------------------------------------------------------------------------

++ kubectl_bin get psmdb-backup --no-headers
++ wc -l
+++ mktemp
++ local LAST_OUT=/tmp/tmp.Hr2kEKGaOr
+++ mktemp
++ local LAST_ERR=/tmp/tmp.QCGnxsmoBA
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb-backup --no-headers
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.Hr2kEKGaOr
++ cat /tmp/tmp.QCGnxsmoBA
No resources found in pitr-32014 namespace.
++ rm /tmp/tmp.Hr2kEKGaOr /tmp/tmp.QCGnxsmoBA
++ return 0
+ '[' 0 '!=' 0 ']'
+ delete_crd
+ desc 'get and delete old CRDs and RBAC'
+ set +o xtrace


-----------------------------------------------------------------------------------
get and delete old CRDs and RBAC
-----------------------------------------------------------------------------------

+ kubectl_bin delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1850/deploy/crd.yaml --ignore-not-found --wait=false
++ mktemp
+ local LAST_OUT=/tmp/tmp.2dHEVue3tP
++ mktemp
+ local LAST_ERR=/tmp/tmp.opFcolI33R
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1850/deploy/crd.yaml --ignore-not-found --wait=false
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.2dHEVue3tP
customresourcedefinition.apiextensions.k8s.io "perconaservermongodbbackups.psmdb.percona.com" deleted
customresourcedefinition.apiextensions.k8s.io "perconaservermongodbrestores.psmdb.percona.com" deleted
customresourcedefinition.apiextensions.k8s.io "perconaservermongodbs.psmdb.percona.com" deleted
+ cat /tmp/tmp.opFcolI33R
+ rm /tmp/tmp.2dHEVue3tP /tmp/tmp.opFcolI33R
+ return 0
++ yq eval .metadata.name /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1850/deploy/crd.yaml
++ grep -v '\-\-\-'
grep: warning: stray \ before -
grep: warning: stray \ before -
+ for crd_name in $(yq eval '.metadata.name' "${src_dir}/deploy/crd.yaml" | grep -v '\-\-\-')
+ kubectl get perconaservermongodbbackups.psmdb.percona.com --all-namespaces -o wide
+ grep -v NAMESPACE
+ xargs -L 1 sh -xc 'kubectl patch perconaservermongodbbackups.psmdb.percona.com -n $0 $1 --type=merge -p "{\"metadata\":{\"finalizers\":[]}}"'
error: the server doesn't have a resource type "perconaservermongodbbackups"
+ kubectl patch perconaservermongodbbackups.psmdb.percona.com -n sh --type=merge -p '{"metadata":{"finalizers":[]}}'
error: the server doesn't have a resource type "perconaservermongodbbackups"
+ :
+ kubectl_bin wait --for=delete crd perconaservermongodbbackups.psmdb.percona.com
++ mktemp
+ local LAST_OUT=/tmp/tmp.ePSuZ0YnVq
++ mktemp
+ local LAST_ERR=/tmp/tmp.0vKYTSzwXI
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl wait --for=delete crd perconaservermongodbbackups.psmdb.percona.com
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.ePSuZ0YnVq
+ cat /tmp/tmp.0vKYTSzwXI
+ rm /tmp/tmp.ePSuZ0YnVq /tmp/tmp.0vKYTSzwXI
+ return 0
+ for crd_name in $(yq eval '.metadata.name' "${src_dir}/deploy/crd.yaml" | grep -v '\-\-\-')
+ kubectl get perconaservermongodbrestores.psmdb.percona.com --all-namespaces -o wide
+ grep -v NAMESPACE
+ xargs -L 1 sh -xc 'kubectl patch perconaservermongodbrestores.psmdb.percona.com -n $0 $1 --type=merge -p "{\"metadata\":{\"finalizers\":[]}}"'
error: the server doesn't have a resource type "perconaservermongodbrestores"
+ kubectl patch perconaservermongodbrestores.psmdb.percona.com -n sh --type=merge -p '{"metadata":{"finalizers":[]}}'
error: the server doesn't have a resource type "perconaservermongodbrestores"
+ :
+ kubectl_bin wait --for=delete crd perconaservermongodbrestores.psmdb.percona.com
++ mktemp
+ local LAST_OUT=/tmp/tmp.6dGnkCfqCe
++ mktemp
+ local LAST_ERR=/tmp/tmp.4ImsWk2PdB
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl wait --for=delete crd perconaservermongodbrestores.psmdb.percona.com
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.6dGnkCfqCe
+ cat /tmp/tmp.4ImsWk2PdB
+ rm /tmp/tmp.6dGnkCfqCe /tmp/tmp.4ImsWk2PdB
+ return 0
+ for crd_name in $(yq eval '.metadata.name' "${src_dir}/deploy/crd.yaml" | grep -v '\-\-\-')
+ kubectl get perconaservermongodbs.psmdb.percona.com --all-namespaces -o wide
+ grep -v NAMESPACE
+ xargs -L 1 sh -xc 'kubectl patch perconaservermongodbs.psmdb.percona.com -n $0 $1 --type=merge -p "{\"metadata\":{\"finalizers\":[]}}"'
error: the server doesn't have a resource type "perconaservermongodbs"
+ kubectl patch perconaservermongodbs.psmdb.percona.com -n sh --type=merge -p '{"metadata":{"finalizers":[]}}'
error: the server doesn't have a resource type "perconaservermongodbs"
+ :
+ kubectl_bin wait --for=delete crd perconaservermongodbs.psmdb.percona.com
++ mktemp
+ local LAST_OUT=/tmp/tmp.78JZaj1nM7
++ mktemp
+ local LAST_ERR=/tmp/tmp.bxn7rO6gZa
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl wait --for=delete crd perconaservermongodbs.psmdb.percona.com
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.78JZaj1nM7
+ cat /tmp/tmp.bxn7rO6gZa
+ rm /tmp/tmp.78JZaj1nM7 /tmp/tmp.bxn7rO6gZa
+ return 0
+ local rbac_yaml=rbac.yaml
+ '[' -n psmdb-operator ']'
+ rbac_yaml=cw-rbac.yaml
+ kubectl_bin delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1850/deploy/cw-rbac.yaml --ignore-not-found
++ mktemp
+ local LAST_OUT=/tmp/tmp.bHZM2fyXaJ
++ mktemp
+ local LAST_ERR=/tmp/tmp.IHj7i7CFcG
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1850/deploy/cw-rbac.yaml --ignore-not-found
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.bHZM2fyXaJ
clusterrole.rbac.authorization.k8s.io "percona-server-mongodb-operator" deleted
clusterrolebinding.rbac.authorization.k8s.io "service-account-percona-server-mongodb-operator" deleted
+ cat /tmp/tmp.IHj7i7CFcG
+ rm /tmp/tmp.bHZM2fyXaJ /tmp/tmp.IHj7i7CFcG
+ return 0
+ destroy_cert_manager
+ kubectl_bin delete -f https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml
++ mktemp
+ local LAST_OUT=/tmp/tmp.Z95478S9j6
++ mktemp
+ local LAST_ERR=/tmp/tmp.mDLVyCIO1t
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl delete -f https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml
+ exit_status=1
+ set -e
+ '[' 1 '!=' 0 -a -n 1 ']'
+ cat /tmp/tmp.Z95478S9j6
+ cat /tmp/tmp.mDLVyCIO1t
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": namespaces "cert-manager" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "challenges.acme.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "orders.acme.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "certificaterequests.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "certificates.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "clusterissuers.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "issuers.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": serviceaccounts "cert-manager-cainjector" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": serviceaccounts "cert-manager" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": serviceaccounts "cert-manager-webhook" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-cainjector" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-issuers" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-clusterissuers" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-certificates" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-orders" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-challenges" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-ingress-shim" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-cluster-view" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-view" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-edit" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-approve:cert-manager-io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-certificatesigningrequests" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-webhook:subjectaccessreviews" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-cainjector" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-issuers" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-clusterissuers" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-certificates" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-orders" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-challenges" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-ingress-shim" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-approve:cert-manager-io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-certificatesigningrequests" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-webhook:subjectaccessreviews" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager-cainjector:leaderelection" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager:leaderelection" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager-tokenrequest" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager-webhook:dynamic-serving" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager-cainjector:leaderelection" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager:leaderelection" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager-tokenrequest" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager-webhook:dynamic-serving" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": services "cert-manager-cainjector" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": services "cert-manager" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": services "cert-manager-webhook" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": deployments.apps "cert-manager-cainjector" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": deployments.apps "cert-manager" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": deployments.apps "cert-manager-webhook" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": mutatingwebhookconfigurations.admissionregistration.k8s.io "cert-manager-webhook" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": validatingwebhookconfigurations.admissionregistration.k8s.io "cert-manager-webhook" not found
+ sleep 0
+ for i in $(seq 0 2)
+ set +e
+ kubectl delete -f https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml
+ exit_status=1
+ set -e
+ '[' 1 '!=' 0 -a -n 1 ']'
+ cat /tmp/tmp.Z95478S9j6
+ cat /tmp/tmp.mDLVyCIO1t
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": namespaces "cert-manager" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "challenges.acme.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "orders.acme.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "certificaterequests.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "certificates.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "clusterissuers.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "issuers.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": serviceaccounts "cert-manager-cainjector" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": serviceaccounts "cert-manager" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": serviceaccounts "cert-manager-webhook" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-cainjector" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-issuers" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-clusterissuers" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-certificates" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-orders" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-challenges" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-ingress-shim" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-cluster-view" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-view" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-edit" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-approve:cert-manager-io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-certificatesigningrequests" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-webhook:subjectaccessreviews" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-cainjector" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-issuers" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-clusterissuers" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-certificates" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-orders" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-challenges" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-ingress-shim" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-approve:cert-manager-io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-certificatesigningrequests" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-webhook:subjectaccessreviews" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager-cainjector:leaderelection" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager:leaderelection" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager-tokenrequest" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager-webhook:dynamic-serving" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager-cainjector:leaderelection" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager:leaderelection" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager-tokenrequest" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager-webhook:dynamic-serving" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": services "cert-manager-cainjector" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": services "cert-manager" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": services "cert-manager-webhook" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": deployments.apps "cert-manager-cainjector" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": deployments.apps "cert-manager" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": deployments.apps "cert-manager-webhook" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": mutatingwebhookconfigurations.admissionregistration.k8s.io "cert-manager-webhook" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": validatingwebhookconfigurations.admissionregistration.k8s.io "cert-manager-webhook" not found
+ sleep 4
+ for i in $(seq 0 2)
+ set +e
+ kubectl delete -f https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml
+ exit_status=1
+ set -e
+ '[' 1 '!=' 0 -a -n 1 ']'
+ cat /tmp/tmp.Z95478S9j6
+ cat /tmp/tmp.mDLVyCIO1t
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": namespaces "cert-manager" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "challenges.acme.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "orders.acme.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "certificaterequests.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "certificates.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "clusterissuers.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "issuers.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": serviceaccounts "cert-manager-cainjector" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": serviceaccounts "cert-manager" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": serviceaccounts "cert-manager-webhook" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-cainjector" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-issuers" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-clusterissuers" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-certificates" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-orders" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-challenges" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-ingress-shim" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-cluster-view" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-view" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-edit" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-approve:cert-manager-io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-certificatesigningrequests" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-webhook:subjectaccessreviews" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-cainjector" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-issuers" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-clusterissuers" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-certificates" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-orders" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-challenges" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-ingress-shim" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-approve:cert-manager-io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-certificatesigningrequests" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-webhook:subjectaccessreviews" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager-cainjector:leaderelection" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager:leaderelection" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager-tokenrequest" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager-webhook:dynamic-serving" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager-cainjector:leaderelection" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager:leaderelection" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager-tokenrequest" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager-webhook:dynamic-serving" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": services "cert-manager-cainjector" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": services "cert-manager" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": services "cert-manager-webhook" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": deployments.apps "cert-manager-cainjector" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": deployments.apps "cert-manager" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": deployments.apps "cert-manager-webhook" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": mutatingwebhookconfigurations.admissionregistration.k8s.io "cert-manager-webhook" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": validatingwebhookconfigurations.admissionregistration.k8s.io "cert-manager-webhook" not found
+ sleep 8
+ cat /tmp/tmp.Z95478S9j6
+ cat /tmp/tmp.mDLVyCIO1t
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": namespaces "cert-manager" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "challenges.acme.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "orders.acme.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "certificaterequests.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "certificates.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "clusterissuers.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "issuers.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": serviceaccounts "cert-manager-cainjector" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": serviceaccounts "cert-manager" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": serviceaccounts "cert-manager-webhook" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-cainjector" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-issuers" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-clusterissuers" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-certificates" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-orders" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-challenges" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-ingress-shim" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-cluster-view" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-view" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-edit" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-approve:cert-manager-io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-certificatesigningrequests" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-webhook:subjectaccessreviews" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-cainjector" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-issuers" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-clusterissuers" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-certificates" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-orders" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-challenges" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-ingress-shim" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-approve:cert-manager-io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-certificatesigningrequests" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-webhook:subjectaccessreviews" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager-cainjector:leaderelection" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager:leaderelection" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager-tokenrequest" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager-webhook:dynamic-serving" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager-cainjector:leaderelection" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager:leaderelection" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager-tokenrequest" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager-webhook:dynamic-serving" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": services "cert-manager-cainjector" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": services "cert-manager" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": services "cert-manager-webhook" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": deployments.apps "cert-manager-cainjector" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": deployments.apps "cert-manager" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": deployments.apps "cert-manager-webhook" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": mutatingwebhookconfigurations.admissionregistration.k8s.io "cert-manager-webhook" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": validatingwebhookconfigurations.admissionregistration.k8s.io "cert-manager-webhook" not found
+ rm /tmp/tmp.Z95478S9j6 /tmp/tmp.mDLVyCIO1t
+ return 1
+ true
+ '[' -n '' ']'
+ '[' -n psmdb-operator ']'
+ kubectl_bin delete --grace-period=0 --force=true namespace pitr-32014
+ rm -rf /tmp/tmp.1qPVwubFaS
+ kubectl_bin delete --grace-period=0 --force=true namespace psmdb-operator
++ mktemp
++ mktemp
+ desc 'test passed'
+ set +o xtrace


-----------------------------------------------------------------------------------
test passed
-----------------------------------------------------------------------------------

+ local LAST_OUT=/tmp/tmp.QX6Qc0oGG8
+ local LAST_OUT=/tmp/tmp.9GSS3wiQ1X
++ mktemp
++ mktemp
+ local LAST_ERR=/tmp/tmp.pileimUhs5
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ local LAST_ERR=/tmp/tmp.gxVjJ56ylF
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl delete --grace-period=0 --force=true namespace psmdb-operator
+ for i in $(seq 0 2)
+ set +e
+ kubectl delete --grace-period=0 --force=true namespace pitr-32014