Log: /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2144/e2e-tests/logs/pitr-to-new-cluster.log
grep: warning: stray \ before -
Warning: version difference between client (1.35) and server (1.31) exceeds the supported minor version skew of +/-1
Warning: version difference between client (1.35) and server (1.31) exceeds the supported minor version skew of +/-1
Warning: version difference between client (1.35) and server (1.31) exceeds the supported minor version skew of +/-1
+ source_cluster=some-name-source
+ target_cluster=some-name-target
+ backup_name_minio=backup-minio
+ setup_infra
+ create_infra pitr-to-new-cluster-13977
+ local ns=pitr-to-new-cluster-13977
+ [[ 1 == 1 ]]
+ delete_crd
+ desc 'get and delete old CRDs and RBAC'
+ set +o xtrace


-----------------------------------------------------------------------------------
get and delete old CRDs and RBAC
-----------------------------------------------------------------------------------

+ kubectl_bin delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2144/deploy/crd.yaml --ignore-not-found --wait=false
++ mktemp
+ local LAST_OUT=/tmp/tmp.3HZWmvgyoy
++ mktemp
+ local LAST_ERR=/tmp/tmp.awhBUQITVS
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2144/deploy/crd.yaml --ignore-not-found --wait=false
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.3HZWmvgyoy
customresourcedefinition.apiextensions.k8s.io "perconaservermongodbbackups.psmdb.percona.com" deleted
customresourcedefinition.apiextensions.k8s.io "perconaservermongodbrestores.psmdb.percona.com" deleted
customresourcedefinition.apiextensions.k8s.io "perconaservermongodbs.psmdb.percona.com" deleted
+ cat /tmp/tmp.awhBUQITVS
+ rm /tmp/tmp.3HZWmvgyoy /tmp/tmp.awhBUQITVS
+ return 0
++ yq eval .metadata.name /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2144/deploy/crd.yaml
++ grep -v '\-\-\-'
grep: warning: stray \ before -
grep: warning: stray \ before -
+ for crd_name in $(yq eval '.metadata.name' "${src_dir}/deploy/crd.yaml" | grep -v '\-\-\-')
+ kubectl get perconaservermongodbbackups.psmdb.percona.com --all-namespaces -o wide
+ grep -v NAMESPACE
+ xargs -L 1 sh -xc 'kubectl patch perconaservermongodbbackups.psmdb.percona.com -n $0 $1 --type=merge -p "{\"metadata\":{\"finalizers\":[]}}"'
No resources found
+ kubectl patch perconaservermongodbbackups.psmdb.percona.com -n sh --type=merge -p '{"metadata":{"finalizers":[]}}'
error: resource(s) were provided, but no name was specified
+ :
+ kubectl_bin wait --for=delete crd perconaservermongodbbackups.psmdb.percona.com
++ mktemp
+ local LAST_OUT=/tmp/tmp.dP0LqSQFqX
++ mktemp
+ local LAST_ERR=/tmp/tmp.EZMPoAisqU
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl wait --for=delete crd perconaservermongodbbackups.psmdb.percona.com
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.dP0LqSQFqX
+ cat /tmp/tmp.EZMPoAisqU
+ rm /tmp/tmp.dP0LqSQFqX /tmp/tmp.EZMPoAisqU
+ return 0
+ for crd_name in $(yq eval '.metadata.name' "${src_dir}/deploy/crd.yaml" | grep -v '\-\-\-')
+ kubectl get perconaservermongodbrestores.psmdb.percona.com --all-namespaces -o wide
+ grep -v NAMESPACE
+ xargs -L 1 sh -xc 'kubectl patch perconaservermongodbrestores.psmdb.percona.com -n $0 $1 --type=merge -p "{\"metadata\":{\"finalizers\":[]}}"'
error: the server doesn't have a resource type "perconaservermongodbrestores"
+ kubectl patch perconaservermongodbrestores.psmdb.percona.com -n sh --type=merge -p '{"metadata":{"finalizers":[]}}'
error: the server doesn't have a resource type "perconaservermongodbrestores"
+ :
+ kubectl_bin wait --for=delete crd perconaservermongodbrestores.psmdb.percona.com
++ mktemp
+ local LAST_OUT=/tmp/tmp.RVzPpXj9Lt
++ mktemp
+ local LAST_ERR=/tmp/tmp.B0GwrtpoQC
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl wait --for=delete crd perconaservermongodbrestores.psmdb.percona.com
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.RVzPpXj9Lt
+ cat /tmp/tmp.B0GwrtpoQC
+ rm /tmp/tmp.RVzPpXj9Lt /tmp/tmp.B0GwrtpoQC
+ return 0
+ for crd_name in $(yq eval '.metadata.name' "${src_dir}/deploy/crd.yaml" | grep -v '\-\-\-')
+ kubectl get perconaservermongodbs.psmdb.percona.com --all-namespaces -o wide
+ grep -v NAMESPACE
+ xargs -L 1 sh -xc 'kubectl patch perconaservermongodbs.psmdb.percona.com -n $0 $1 --type=merge -p "{\"metadata\":{\"finalizers\":[]}}"'
error: the server doesn't have a resource type "perconaservermongodbs"
+ kubectl patch perconaservermongodbs.psmdb.percona.com -n sh --type=merge -p '{"metadata":{"finalizers":[]}}'
error: the server doesn't have a resource type "perconaservermongodbs"
+ :
+ kubectl_bin wait --for=delete crd perconaservermongodbs.psmdb.percona.com
++ mktemp
+ local LAST_OUT=/tmp/tmp.chxyMQYzGP
++ mktemp
+ local LAST_ERR=/tmp/tmp.zjjdRKhYeT
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl wait --for=delete crd perconaservermongodbs.psmdb.percona.com
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.chxyMQYzGP
+ cat /tmp/tmp.zjjdRKhYeT
+ rm /tmp/tmp.chxyMQYzGP /tmp/tmp.zjjdRKhYeT
+ return 0
+ local rbac_yaml=rbac.yaml
+ '[' -n psmdb-operator ']'
+ rbac_yaml=cw-rbac.yaml
+ kubectl_bin delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2144/deploy/cw-rbac.yaml --ignore-not-found
++ mktemp
+ local LAST_OUT=/tmp/tmp.UjQO9eCEBA
++ mktemp
+ local LAST_ERR=/tmp/tmp.eH9LUzF5fS
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2144/deploy/cw-rbac.yaml --ignore-not-found
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.UjQO9eCEBA
clusterrole.rbac.authorization.k8s.io "percona-server-mongodb-operator" deleted
clusterrolebinding.rbac.authorization.k8s.io "service-account-percona-server-mongodb-operator" deleted
+ cat /tmp/tmp.eH9LUzF5fS
+ rm /tmp/tmp.UjQO9eCEBA /tmp/tmp.eH9LUzF5fS
+ return 0
+ check_crd_for_deletion PR-2144-c278c1c9
+ local git_tag=PR-2144-c278c1c9
++ curl -s https://raw.githubusercontent.com/percona/percona-server-mongodb-operator/PR-2144-c278c1c9/deploy/crd.yaml
++ yq eval .metadata.name
++ /usr/sbin/sed s/---//g
++ /usr/sbin/sed ':a;N;$!ba;s/\n/ /g'
+ for crd_name in $(curl -s https://raw.githubusercontent.com/percona/percona-server-mongodb-operator/${git_tag}/deploy/crd.yaml | yq eval '.metadata.name' | $sed 's/---//g' | $sed ':a;N;$!ba;s/\n/ /g')
++ kubectl_bin get crd/null -o 'jsonpath={.status.conditions[-1].type}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.PdhfGKHa7l
+++ mktemp
++ local LAST_ERR=/tmp/tmp.5mTQWviMrU
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get crd/null -o 'jsonpath={.status.conditions[-1].type}'
++ exit_status=1
++ set -e
++ '[' 1 '!=' 0 -a -n 1 ']'
++ cat /tmp/tmp.PdhfGKHa7l
++ cat /tmp/tmp.5mTQWviMrU
Error from server (NotFound): customresourcedefinitions.apiextensions.k8s.io "null" not found
++ sleep 0
++ for i in $(seq 0 2)
++ set +e
++ kubectl get crd/null -o 'jsonpath={.status.conditions[-1].type}'
++ exit_status=1
++ set -e
++ '[' 1 '!=' 0 -a -n 1 ']'
++ cat /tmp/tmp.PdhfGKHa7l
++ cat /tmp/tmp.5mTQWviMrU
Error from server (NotFound): customresourcedefinitions.apiextensions.k8s.io "null" not found
++ sleep 4
++ for i in $(seq 0 2)
++ set +e
++ kubectl get crd/null -o 'jsonpath={.status.conditions[-1].type}'
++ exit_status=1
++ set -e
++ '[' 1 '!=' 0 -a -n 1 ']'
++ cat /tmp/tmp.PdhfGKHa7l
++ cat /tmp/tmp.5mTQWviMrU
Error from server (NotFound): customresourcedefinitions.apiextensions.k8s.io "null" not found
++ sleep 8
++ cat /tmp/tmp.PdhfGKHa7l
++ cat /tmp/tmp.5mTQWviMrU
Error from server (NotFound): customresourcedefinitions.apiextensions.k8s.io "null" not found
++ rm /tmp/tmp.PdhfGKHa7l /tmp/tmp.5mTQWviMrU
++ return 1
+ [[ '' == \T\e\r\m\i\n\a\t\i\n\g ]]
+ '[' -n psmdb-operator ']'
+ create_namespace psmdb-operator
+ local namespace=psmdb-operator
+ local skip_clean_namespace=
+ [[ 1 == 1 ]]
+ [[ -z '' ]]
+ destroy_chaos_mesh
++ helm list --all-namespaces --filter chaos-mesh
++ tail -n1
++ awk '-F ' '{print $2}'
++ sed s/NAMESPACE//
+ local chaos_mesh_ns=
+ desc 'destroy chaos-mesh'
+ set +o xtrace


-----------------------------------------------------------------------------------
destroy chaos-mesh
-----------------------------------------------------------------------------------

+ '[' -n '' ']'
++ kubectl get MutatingWebhookConfiguration
++ grep chaos-mesh
++ awk '{print $1}'
+ timeout 30 kubectl delete MutatingWebhookConfiguration
error: resource(s) were provided, but no name was specified
+ :
++ kubectl get ValidatingWebhookConfiguration
++ grep chaos-mesh
++ awk '{print $1}'
+ timeout 30 kubectl delete ValidatingWebhookConfiguration
error: resource(s) were provided, but no name was specified
+ :
++ kubectl get ValidatingWebhookConfiguration
++ grep validate-auth
++ awk '{print $1}'
+ timeout 30 kubectl delete ValidatingWebhookConfiguration
error: resource(s) were provided, but no name was specified
+ :
++ kubectl api-resources
++ grep chaos-mesh
++ awk '{print $1}'
++ kubectl get crd
++ grep chaos-mesh.org
++ awk '{print $1}'
+ timeout 30 kubectl delete crd
error: resource(s) were provided, but no name was specified
+ :
++ kubectl get clusterrolebinding
++ grep chaos-mesh
++ awk '{print $1}'
+ timeout 30 kubectl delete clusterrolebinding
error: resource(s) were provided, but no name was specified
+ :
++ kubectl get clusterrole
++ grep chaos-mesh
++ awk '{print $1}'
+ timeout 30 kubectl delete clusterrole
error: resource(s) were provided, but no name was specified
+ :
+ desc 'cleaned up all old namespaces'
+ set +o xtrace


-----------------------------------------------------------------------------------
cleaned up all old namespaces
-----------------------------------------------------------------------------------

+ kubectl_bin get ns
+ egrep -v '^kube-|^default|Terminating|psmdb-operator|openshift|^gke-|^gmp-|^NAME'
++ mktemp
+ awk '{print$1}'
+ '[' -n '' ']'
+ desc 'cleaned up old namespaces psmdb-operator'
+ xargs kubectl delete ns
+ set +o xtrace


-----------------------------------------------------------------------------------
cleaned up old namespaces psmdb-operator
-----------------------------------------------------------------------------------

+ kubectl_bin delete namespace psmdb-operator --ignore-not-found
++ mktemp
+ local LAST_OUT=/tmp/tmp.9ozCApYuhA
egrep: warning: egrep is obsolescent; using grep -E
++ mktemp
+ local LAST_ERR=/tmp/tmp.aAfxlHFjzH
+ local exit_status=0
+ local timeout=4
+ local LAST_OUT=/tmp/tmp.9X8sJai5Lm
++ seq 0 2
++ mktemp
+ for i in $(seq 0 2)
+ set +e
+ kubectl get ns
+ local LAST_ERR=/tmp/tmp.JD4HdGoW7I
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl delete namespace psmdb-operator --ignore-not-found
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.9ozCApYuhA
+ cat /tmp/tmp.aAfxlHFjzH
+ rm /tmp/tmp.9ozCApYuhA /tmp/tmp.aAfxlHFjzH
+ return 0
namespace "pitr-to-new-cluster-12982" deleted
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.9X8sJai5Lm
namespace "psmdb-operator" deleted
+ cat /tmp/tmp.JD4HdGoW7I
+ rm /tmp/tmp.9X8sJai5Lm /tmp/tmp.JD4HdGoW7I
+ return 0
+ kubectl_bin wait --for=delete namespace psmdb-operator
++ mktemp
+ local LAST_OUT=/tmp/tmp.iCTkBIW65C
++ mktemp
+ local LAST_ERR=/tmp/tmp.Xf7hPK2QX2
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl wait --for=delete namespace psmdb-operator
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.iCTkBIW65C
+ cat /tmp/tmp.Xf7hPK2QX2
+ rm /tmp/tmp.iCTkBIW65C /tmp/tmp.Xf7hPK2QX2
+ return 0
+ desc 'create namespace psmdb-operator'
+ set +o xtrace


-----------------------------------------------------------------------------------
create namespace psmdb-operator
-----------------------------------------------------------------------------------

+ kubectl_bin create namespace psmdb-operator
++ mktemp
+ local LAST_OUT=/tmp/tmp.ffQ3LqfhiM
++ mktemp
+ local LAST_ERR=/tmp/tmp.6GYm7nHEcj
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl create namespace psmdb-operator
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.ffQ3LqfhiM
namespace/psmdb-operator created
+ cat /tmp/tmp.6GYm7nHEcj
+ rm /tmp/tmp.ffQ3LqfhiM /tmp/tmp.6GYm7nHEcj
+ return 0
+ set_kube_ctx psmdb-operator
+ local namespace=psmdb-operator
++ kubectl_bin config current-context
+++ mktemp
++ local LAST_OUT=/tmp/tmp.MRJOgZKO45
+++ mktemp
++ local LAST_ERR=/tmp/tmp.XzxyC0d6e3
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl config current-context
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.MRJOgZKO45
++ cat /tmp/tmp.XzxyC0d6e3
++ rm /tmp/tmp.MRJOgZKO45 /tmp/tmp.XzxyC0d6e3
++ return 0
+ kubectl_bin config set-context gke_cloud-dev-112233_us-central1-a_jen-psmdb-2144-c278c1c9-12-cluster6 --namespace=psmdb-operator
++ mktemp
+ local LAST_OUT=/tmp/tmp.vYnaEgKhgR
++ mktemp
+ local LAST_ERR=/tmp/tmp.n9JjpVO2zL
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl config set-context gke_cloud-dev-112233_us-central1-a_jen-psmdb-2144-c278c1c9-12-cluster6 --namespace=psmdb-operator
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.vYnaEgKhgR
Context "gke_cloud-dev-112233_us-central1-a_jen-psmdb-2144-c278c1c9-12-cluster6" modified.
+ cat /tmp/tmp.n9JjpVO2zL
+ rm /tmp/tmp.vYnaEgKhgR /tmp/tmp.n9JjpVO2zL
+ return 0
+ deploy_operator
+ desc 'start PSMDB operator: perconalab/percona-server-mongodb-operator:PR-2144-c278c1c9'
+ set +o xtrace


-----------------------------------------------------------------------------------
start PSMDB operator: perconalab/percona-server-mongodb-operator:PR-2144-c278c1c9
-----------------------------------------------------------------------------------

+ local cr_file
+ '[' -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2144/e2e-tests/pitr-to-new-cluster/conf/crd.yaml ']'
+ cr_file=/mnt/jenkins/workspace/cloud-psmdb-operator_PR-2144/deploy/crd.yaml
+ kubectl_bin apply --server-side --force-conflicts -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2144/deploy/crd.yaml
++ mktemp
+ local LAST_OUT=/tmp/tmp.AX1FikpaoR
++ mktemp
+ local LAST_ERR=/tmp/tmp.Ig85uvVphu
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl apply --server-side --force-conflicts -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2144/deploy/crd.yaml
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.AX1FikpaoR
customresourcedefinition.apiextensions.k8s.io/perconaservermongodbbackups.psmdb.percona.com serverside-applied
customresourcedefinition.apiextensions.k8s.io/perconaservermongodbrestores.psmdb.percona.com serverside-applied
customresourcedefinition.apiextensions.k8s.io/perconaservermongodbs.psmdb.percona.com serverside-applied
+ cat /tmp/tmp.Ig85uvVphu
+ rm /tmp/tmp.AX1FikpaoR /tmp/tmp.Ig85uvVphu
+ return 0
+ '[' -n psmdb-operator ']'
+ apply_rbac cw-rbac
+ local operator_namespace=psmdb-operator
+ local rbac=cw-rbac
+ cat /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2144/deploy/cw-rbac.yaml
+ kubectl_bin apply -n psmdb-operator -f -
+ sed -e 's^namespace: .*^namespace: psmdb-operator^'
++ mktemp
+ local LAST_OUT=/tmp/tmp.XI7Lt5kal4
++ mktemp
+ local LAST_ERR=/tmp/tmp.6XtIzmnNRy
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl apply -n psmdb-operator -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.XI7Lt5kal4
clusterrole.rbac.authorization.k8s.io/percona-server-mongodb-operator created
serviceaccount/percona-server-mongodb-operator created
clusterrolebinding.rbac.authorization.k8s.io/service-account-percona-server-mongodb-operator created
+ cat /tmp/tmp.6XtIzmnNRy
+ rm /tmp/tmp.XI7Lt5kal4 /tmp/tmp.6XtIzmnNRy
+ return 0
+ yq eval '
			(.spec.template.spec.containers[].image = "perconalab/percona-server-mongodb-operator:PR-2144-c278c1c9") |
			((.. | select(.[] == "DISABLE_TELEMETRY")) |= .value="true") |
			((.. | select(.[] == "LOG_LEVEL")) |= .value="DEBUG")' /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2144/deploy/cw-operator.yaml
+ kubectl_bin apply -n psmdb-operator -f -
++ mktemp
+ local LAST_OUT=/tmp/tmp.9O9QsG6wDW
++ mktemp
+ local LAST_ERR=/tmp/tmp.FWql64Fc86
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl apply -n psmdb-operator -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.9O9QsG6wDW
deployment.apps/percona-server-mongodb-operator created
+ cat /tmp/tmp.FWql64Fc86
+ rm /tmp/tmp.9O9QsG6wDW /tmp/tmp.FWql64Fc86
+ return 0
+ sleep 20
++ get_operator_pod
++ kubectl_bin get pods --selector=name=percona-server-mongodb-operator -o 'jsonpath={.items[].metadata.name}' -n psmdb-operator
+++ mktemp
++ local LAST_OUT=/tmp/tmp.9I8W1zXW6Q
+++ mktemp
++ local LAST_ERR=/tmp/tmp.nl4PqieFJ6
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=percona-server-mongodb-operator -o 'jsonpath={.items[].metadata.name}' -n psmdb-operator
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.9I8W1zXW6Q
++ cat /tmp/tmp.nl4PqieFJ6
++ rm /tmp/tmp.9I8W1zXW6Q /tmp/tmp.nl4PqieFJ6
++ return 0
+ wait_operator_pod percona-server-mongodb-operator-6454c8bfb-tnhsz
+ local pod=percona-server-mongodb-operator-6454c8bfb-tnhsz
+ set +o xtrace
waiting for pod/percona-server-mongodb-operator-6454c8bfb-tnhsz to be ready.OK
+ echo 'Print operator info from log'
Print operator info from log
+ grep 'Manager starting up'
++ get_operator_pod
++ kubectl_bin get pods --selector=name=percona-server-mongodb-operator -o 'jsonpath={.items[].metadata.name}' -n psmdb-operator
+++ mktemp
++ local LAST_OUT=/tmp/tmp.DiuwWmS1No
+++ mktemp
++ local LAST_ERR=/tmp/tmp.LzJOCuPY0j
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=percona-server-mongodb-operator -o 'jsonpath={.items[].metadata.name}' -n psmdb-operator
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.DiuwWmS1No
++ cat /tmp/tmp.LzJOCuPY0j
++ rm /tmp/tmp.DiuwWmS1No /tmp/tmp.LzJOCuPY0j
++ return 0
+ kubectl_bin logs -n psmdb-operator percona-server-mongodb-operator-6454c8bfb-tnhsz
++ mktemp
+ local LAST_OUT=/tmp/tmp.ZU0Oqg1OGs
++ mktemp
+ local LAST_ERR=/tmp/tmp.UX4HuDr902
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl logs -n psmdb-operator percona-server-mongodb-operator-6454c8bfb-tnhsz
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.ZU0Oqg1OGs
+ cat /tmp/tmp.UX4HuDr902
+ rm /tmp/tmp.ZU0Oqg1OGs /tmp/tmp.UX4HuDr902
+ return 0
2025-12-23T15:35:35.261Z	INFO	setup	Manager starting up	{"gitCommit": "c278c1c9be58fc863085595b1a3aad8b308ea1a7", "gitBranch": "PR-2144-c278c1c9", "buildTime": "", "goVersion": "go1.25.5", "os": "linux", "arch": "amd64"}
+ create_namespace pitr-to-new-cluster-13977
+ local namespace=pitr-to-new-cluster-13977
+ local skip_clean_namespace=
+ [[ 1 == 1 ]]
+ [[ -z '' ]]
+ destroy_chaos_mesh
++ helm list --all-namespaces --filter chaos-mesh
++ tail -n1
++ awk '-F ' '{print $2}'
++ sed s/NAMESPACE//
+ local chaos_mesh_ns=
+ desc 'destroy chaos-mesh'
+ set +o xtrace


-----------------------------------------------------------------------------------
destroy chaos-mesh
-----------------------------------------------------------------------------------

+ '[' -n '' ']'
++ kubectl get MutatingWebhookConfiguration
++ grep chaos-mesh
++ awk '{print $1}'
+ timeout 30 kubectl delete MutatingWebhookConfiguration
error: resource(s) were provided, but no name was specified
+ :
++ kubectl get ValidatingWebhookConfiguration
++ grep chaos-mesh
++ awk '{print $1}'
+ timeout 30 kubectl delete ValidatingWebhookConfiguration
error: resource(s) were provided, but no name was specified
+ :
++ kubectl get ValidatingWebhookConfiguration
++ grep validate-auth
++ awk '{print $1}'
+ timeout 30 kubectl delete ValidatingWebhookConfiguration
error: resource(s) were provided, but no name was specified
+ :
++ kubectl api-resources
++ grep chaos-mesh
++ awk '{print $1}'
++ kubectl get crd
++ grep chaos-mesh.org
++ awk '{print $1}'
+ timeout 30 kubectl delete crd
error: resource(s) were provided, but no name was specified
+ :
++ kubectl get clusterrolebinding
++ grep chaos-mesh
++ awk '{print $1}'
+ timeout 30 kubectl delete clusterrolebinding
error: resource(s) were provided, but no name was specified
+ :
++ kubectl get clusterrole
++ grep chaos-mesh
++ awk '{print $1}'
+ timeout 30 kubectl delete clusterrole
error: resource(s) were provided, but no name was specified
+ :
+ desc 'cleaned up all old namespaces'
+ set +o xtrace


-----------------------------------------------------------------------------------
cleaned up all old namespaces
-----------------------------------------------------------------------------------

+ kubectl_bin get ns
+ egrep -v '^kube-|^default|Terminating|psmdb-operator|openshift|^gke-|^gmp-|^NAME'
+ '[' -n '' ']'
+ awk '{print$1}'
+ desc 'cleaned up old namespaces pitr-to-new-cluster-13977'
+ set +o xtrace


-----------------------------------------------------------------------------------
cleaned up old namespaces pitr-to-new-cluster-13977
-----------------------------------------------------------------------------------

++ mktemp
+ kubectl_bin delete namespace pitr-to-new-cluster-13977 --ignore-not-found
+ xargs kubectl delete ns
++ mktemp
egrep: warning: egrep is obsolescent; using grep -E
+ local LAST_OUT=/tmp/tmp.cjDiGnYu2e
++ mktemp
+ local LAST_OUT=/tmp/tmp.PeD0c9Mk0T
+ local LAST_ERR=/tmp/tmp.FUUKaM7Vij
+ local exit_status=0
+ local timeout=4
++ mktemp
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl get ns
+ local LAST_ERR=/tmp/tmp.vSSGkSNpzU
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl delete namespace pitr-to-new-cluster-13977 --ignore-not-found
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.cjDiGnYu2e
+ cat /tmp/tmp.FUUKaM7Vij
+ rm /tmp/tmp.cjDiGnYu2e /tmp/tmp.FUUKaM7Vij
+ return 0
error: resource(s) were provided, but no name was specified
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.PeD0c9Mk0T
+ cat /tmp/tmp.vSSGkSNpzU
+ rm /tmp/tmp.PeD0c9Mk0T /tmp/tmp.vSSGkSNpzU
+ return 0
+ kubectl_bin wait --for=delete namespace pitr-to-new-cluster-13977
++ mktemp
+ local LAST_OUT=/tmp/tmp.mPwYPlM2J7
++ mktemp
+ local LAST_ERR=/tmp/tmp.IHJD7xysGk
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl wait --for=delete namespace pitr-to-new-cluster-13977
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.mPwYPlM2J7
+ cat /tmp/tmp.IHJD7xysGk
+ rm /tmp/tmp.mPwYPlM2J7 /tmp/tmp.IHJD7xysGk
+ return 0
+ desc 'create namespace pitr-to-new-cluster-13977'
+ set +o xtrace


-----------------------------------------------------------------------------------
create namespace pitr-to-new-cluster-13977
-----------------------------------------------------------------------------------

+ kubectl_bin create namespace pitr-to-new-cluster-13977
++ mktemp
+ local LAST_OUT=/tmp/tmp.HbhTSTRIlC
++ mktemp
+ local LAST_ERR=/tmp/tmp.pLP6ELylAu
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl create namespace pitr-to-new-cluster-13977
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.HbhTSTRIlC
namespace/pitr-to-new-cluster-13977 created
+ cat /tmp/tmp.pLP6ELylAu
+ rm /tmp/tmp.HbhTSTRIlC /tmp/tmp.pLP6ELylAu
+ return 0
+ set_kube_ctx pitr-to-new-cluster-13977
+ local namespace=pitr-to-new-cluster-13977
++ kubectl_bin config current-context
+++ mktemp
++ local LAST_OUT=/tmp/tmp.CexypKwPgJ
+++ mktemp
++ local LAST_ERR=/tmp/tmp.3rOqaGBrUR
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl config current-context
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.CexypKwPgJ
++ cat /tmp/tmp.3rOqaGBrUR
++ rm /tmp/tmp.CexypKwPgJ /tmp/tmp.3rOqaGBrUR
++ return 0
+ kubectl_bin config set-context gke_cloud-dev-112233_us-central1-a_jen-psmdb-2144-c278c1c9-12-cluster6 --namespace=pitr-to-new-cluster-13977
++ mktemp
+ local LAST_OUT=/tmp/tmp.hVjUTUmDLi
++ mktemp
+ local LAST_ERR=/tmp/tmp.zqiW4v25SD
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl config set-context gke_cloud-dev-112233_us-central1-a_jen-psmdb-2144-c278c1c9-12-cluster6 --namespace=pitr-to-new-cluster-13977
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.hVjUTUmDLi
Context "gke_cloud-dev-112233_us-central1-a_jen-psmdb-2144-c278c1c9-12-cluster6" modified.
+ cat /tmp/tmp.zqiW4v25SD
+ rm /tmp/tmp.hVjUTUmDLi /tmp/tmp.zqiW4v25SD
+ return 0
+ deploy_minio
+ desc 'install Minio'
+ set +o xtrace


-----------------------------------------------------------------------------------
install Minio
-----------------------------------------------------------------------------------

+ helm uninstall minio-service
Error: uninstall: Release not loaded: minio-service: release: not found
+ :
+ helm repo remove minio
"minio" has been removed from your repositories
+ helm repo add minio https://charts.min.io/
"minio" has been added to your repositories
+ retry 10 60 helm install minio-service --version 5.4.0 --set replicas=1 --set mode=standalone --set resources.requests.memory=256Mi --set rootUser=rootuser --set rootPassword=rootpass123 --set 'users[0].accessKey=some-access-key' --set 'users[0].secretKey=some-secret-key' --set 'users[0].policy=consoleAdmin' --set service.type=ClusterIP --set configPathmc=/tmp/.minio/ --set persistence.size=2G --set securityContext.enabled=false minio/minio
+ local max=10
+ local delay=60
+ shift 2
+ local n=1
+ helm install minio-service --version 5.4.0 --set replicas=1 --set mode=standalone --set resources.requests.memory=256Mi --set rootUser=rootuser --set rootPassword=rootpass123 --set 'users[0].accessKey=some-access-key' --set 'users[0].secretKey=some-secret-key' --set 'users[0].policy=consoleAdmin' --set service.type=ClusterIP --set configPathmc=/tmp/.minio/ --set persistence.size=2G --set securityContext.enabled=false minio/minio
NAME: minio-service
LAST DEPLOYED: Tue Dec 23 15:36:12 2025
NAMESPACE: pitr-to-new-cluster-13977
STATUS: deployed
REVISION: 1
TEST SUITE: None
NOTES:
MinIO can be accessed via port 9000 on the following DNS name from within your cluster:
minio-service.pitr-to-new-cluster-13977.cluster.local

To access MinIO from localhost, run the below commands:

  1. export POD_NAME=$(kubectl get pods --namespace pitr-to-new-cluster-13977 -l "release=minio-service" -o jsonpath="{.items[0].metadata.name}")

  2. kubectl port-forward $POD_NAME 9000 --namespace pitr-to-new-cluster-13977

Read more about port forwarding here: http://kubernetes.io/docs/user-guide/kubectl/kubectl_port-forward/

You can now access MinIO server on http://localhost:9000. Follow the below steps to connect to MinIO server with mc client:

  1. Download the MinIO mc client - https://min.io/docs/minio/linux/reference/minio-mc.html#quickstart

  2. export MC_HOST_minio-service-local=http://$(kubectl get secret --namespace pitr-to-new-cluster-13977 minio-service -o jsonpath="{.data.rootUser}" | base64 --decode):$(kubectl get secret --namespace pitr-to-new-cluster-13977 minio-service -o jsonpath="{.data.rootPassword}" | base64 --decode)@localhost:9000

  3. mc ls minio-service-local
++ kubectl_bin get pods --selector=release=minio-service -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.tcNBUPo7uy
+++ mktemp
++ local LAST_ERR=/tmp/tmp.BNVDd6g1NX
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=release=minio-service -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.tcNBUPo7uy
++ cat /tmp/tmp.BNVDd6g1NX
++ rm /tmp/tmp.tcNBUPo7uy /tmp/tmp.BNVDd6g1NX
++ return 0
+ MINIO_POD=minio-service-d9589b474-5cj2c
+ wait_pod minio-service-d9589b474-5cj2c
+ local pod=minio-service-d9589b474-5cj2c
+ set +o xtrace
waiting for pod/minio-service-d9589b474-5cj2c to be ready.OK
+ '[' -n psmdb-operator ']'
+ kubectl_bin create svc -n psmdb-operator externalname minio-service --external-name=minio-service.pitr-to-new-cluster-13977.svc.cluster.local --tcp=9000
++ mktemp
+ local LAST_OUT=/tmp/tmp.j5fkdrgxnx
++ mktemp
+ local LAST_ERR=/tmp/tmp.bP9KOE3gj5
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl create svc -n psmdb-operator externalname minio-service --external-name=minio-service.pitr-to-new-cluster-13977.svc.cluster.local --tcp=9000
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.j5fkdrgxnx
service/minio-service created
+ cat /tmp/tmp.bP9KOE3gj5
+ rm /tmp/tmp.j5fkdrgxnx /tmp/tmp.bP9KOE3gj5
+ return 0
+ create_minio_bucket operator-testing
+ local bucket=operator-testing
+ kubectl_bin run -i --rm aws-cli --image=perconalab/awscli --restart=Never -- bash -c 'AWS_ACCESS_KEY_ID=some-access-key 		AWS_SECRET_ACCESS_KEY=some-secret-key 		AWS_DEFAULT_REGION=us-east-1         /usr/bin/aws --endpoint-url http://minio-service:9000 s3 mb s3://operator-testing'
++ mktemp
+ local LAST_OUT=/tmp/tmp.yJrYUyYwdY
++ mktemp
+ local LAST_ERR=/tmp/tmp.dJBy69Tqi3
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl run -i --rm aws-cli --image=perconalab/awscli --restart=Never -- bash -c 'AWS_ACCESS_KEY_ID=some-access-key 		AWS_SECRET_ACCESS_KEY=some-secret-key 		AWS_DEFAULT_REGION=us-east-1         /usr/bin/aws --endpoint-url http://minio-service:9000 s3 mb s3://operator-testing'
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.yJrYUyYwdY
make_bucket: operator-testing
pod "aws-cli" deleted from pitr-to-new-cluster-13977 namespace
+ cat /tmp/tmp.dJBy69Tqi3
+ rm /tmp/tmp.yJrYUyYwdY /tmp/tmp.dJBy69Tqi3
+ return 0
+ desc 'create secrets and start client'
+ set +o xtrace


-----------------------------------------------------------------------------------
create secrets and start client
-----------------------------------------------------------------------------------

+ kubectl_bin apply -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2144/e2e-tests/conf/secrets.yml -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2144/e2e-tests/conf/client.yml -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2144/e2e-tests/conf/minio-secret.yml
++ mktemp
+ local LAST_OUT=/tmp/tmp.fcVNXN2eRk
++ mktemp
+ local LAST_ERR=/tmp/tmp.1CKO5cJCme
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl apply -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2144/e2e-tests/conf/secrets.yml -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2144/e2e-tests/conf/client.yml -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2144/e2e-tests/conf/minio-secret.yml
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.fcVNXN2eRk
secret/some-users created
deployment.apps/psmdb-client created
secret/minio-secret created
+ cat /tmp/tmp.1CKO5cJCme
+ rm /tmp/tmp.fcVNXN2eRk /tmp/tmp.1CKO5cJCme
+ return 0
+ setup_source_cluster
+ desc 'create PSMDB cluster: some-name-source'
+ set +o xtrace


-----------------------------------------------------------------------------------
create PSMDB cluster: some-name-source
-----------------------------------------------------------------------------------

+ apply_cluster /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2144/e2e-tests/pitr-to-new-cluster/conf/some-name-source.yml
+ '[' -z '' ']'
+ cat_config /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2144/e2e-tests/pitr-to-new-cluster/conf/some-name-source.yml
+ kubectl_bin apply -f -
+ cat /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2144/e2e-tests/pitr-to-new-cluster/conf/some-name-source.yml
+ yq eval '(.spec | select(.image == null)).image = "perconalab/percona-server-mongodb-operator:main-mongod8.0"'
+ yq eval '(.spec | select(has("pmm"))).pmm.image = "percona/pmm-client:2.44.1-1"'
++ mktemp
+ yq eval '(.spec | select(has("initImage"))).initImage = "perconalab/percona-server-mongodb-operator:PR-2144-c278c1c9"'
+ yq eval '(.spec | select(has("backup"))).backup.image = "perconalab/percona-server-mongodb-operator:main-backup"'
+ yq eval '.spec.upgradeOptions.apply="Never"'
+ local LAST_OUT=/tmp/tmp.xcUQqsPQyd
++ mktemp
+ local LAST_ERR=/tmp/tmp.0Ikt4NDx5T
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl apply -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.xcUQqsPQyd
perconaservermongodb.psmdb.percona.com/some-name-source created
+ cat /tmp/tmp.0Ikt4NDx5T
+ rm /tmp/tmp.xcUQqsPQyd /tmp/tmp.0Ikt4NDx5T
+ return 0
+ wait_for_running some-name-source-rs0 3
+ local name=some-name-source-rs0
+ let last_pod=2
+ local check_cluster_readyness=true
+ set_debug
+ [[ 1 == 1 ]]
+ set -o xtrace
+ local rs_name=rs0
+ local cluster_name=some-name-source
++ seq 0 2
+ for i in $(seq 0 $last_pod)
+ [[ 0 -eq 2 ]]
+ wait_pod some-name-source-rs0-0
+ local pod=some-name-source-rs0-0
+ set +o xtrace
waiting for pod/some-name-source-rs0-0 to be ready.......OK
+ for i in $(seq 0 $last_pod)
+ [[ 1 -eq 2 ]]
+ wait_pod some-name-source-rs0-1
+ local pod=some-name-source-rs0-1
+ set +o xtrace
waiting for pod/some-name-source-rs0-1 to be ready...OK
+ for i in $(seq 0 $last_pod)
+ [[ 2 -eq 2 ]]
++ kubectl_bin get psmdb some-name-source -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].arbiter.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.iyUp7UwMgN
+++ mktemp
++ local LAST_ERR=/tmp/tmp.Wmu1aTBBb2
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name-source -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].arbiter.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.iyUp7UwMgN
++ cat /tmp/tmp.Wmu1aTBBb2
++ rm /tmp/tmp.iyUp7UwMgN /tmp/tmp.Wmu1aTBBb2
++ return 0
+ [[ '' == \t\r\u\e ]]
+ wait_pod some-name-source-rs0-2
+ local pod=some-name-source-rs0-2
+ set +o xtrace
waiting for pod/some-name-source-rs0-2 to be ready....OK
++ kubectl_bin get psmdb some-name-source -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].nonvoting.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.d2d1WXH10d
+++ mktemp
++ local LAST_ERR=/tmp/tmp.ShJhm35DaD
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name-source -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].nonvoting.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.d2d1WXH10d
++ cat /tmp/tmp.ShJhm35DaD
++ rm /tmp/tmp.d2d1WXH10d /tmp/tmp.ShJhm35DaD
++ return 0
+ [[ '' == \t\r\u\e ]]
++ kubectl_bin get psmdb some-name-source -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].hidden.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.QH46nRyx5T
+++ mktemp
++ local LAST_ERR=/tmp/tmp.vsaLs4xLsp
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name-source -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].hidden.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.QH46nRyx5T
++ cat /tmp/tmp.vsaLs4xLsp
++ rm /tmp/tmp.QH46nRyx5T /tmp/tmp.vsaLs4xLsp
++ return 0
+ [[ '' == \t\r\u\e ]]
+ sleep 10
+ [[ true == \t\r\u\e ]]
+ set +x
Waiting for cluster readyness...
+ wait_backup_agent some-name-source-rs0-0
+ local agent_pod=some-name-source-rs0-0
+ set +o xtrace
waiting for pbm-agent to be ready in some-name-source-rs0-0...2025-12-23T15:38:05.000+0000 I listening for the commands

+ wait_backup_agent some-name-source-rs0-1
+ local agent_pod=some-name-source-rs0-1
+ set +o xtrace
waiting for pbm-agent to be ready in some-name-source-rs0-1...2025-12-23T15:38:08.000+0000 I listening for the commands

+ wait_backup_agent some-name-source-rs0-2
+ local agent_pod=some-name-source-rs0-2
+ set +o xtrace
waiting for pbm-agent to be ready in some-name-source-rs0-2...2025-12-23T15:38:10.000+0000 I listening for the commands

+ create_user some-name-source
+ local cluster=some-name-source
+ log 'create user myApp'
+ set +o xtrace
[2025-12-23T15:38:13+0000] create user myApp
+ run_mongo 'db.createUser({user:"myApp",pwd:"myPass",roles:[{db:"myApp",role:"readWrite"}]})' userAdmin:userAdmin123456@some-name-source-rs0.pitr-to-new-cluster-13977 mongodb
+ local 'command=db.createUser({user:"myApp",pwd:"myPass",roles:[{db:"myApp",role:"readWrite"}]})'
+ local uri=userAdmin:userAdmin123456@some-name-source-rs0.pitr-to-new-cluster-13977
+ local driver=mongodb
+ local suffix=.svc.cluster.local
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.gkrgbh4cSx
+++ mktemp
++ local LAST_ERR=/tmp/tmp.VUroW6gw6X
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.gkrgbh4cSx
++ cat /tmp/tmp.VUroW6gw6X
++ rm /tmp/tmp.gkrgbh4cSx /tmp/tmp.VUroW6gw6X
++ return 0
+ local client_container=psmdb-client-696897d69b-9r6m5
+ local mongo_flag=
+ [[ userAdmin:userAdmin123456@some-name-source-rs0.pitr-to-new-cluster-13977 == *cfg* ]]
+ replica_set=rs0
+ kubectl_bin exec psmdb-client-696897d69b-9r6m5 -- bash -c 'printf '\''db.createUser({user:"myApp",pwd:"myPass",roles:[{db:"myApp",role:"readWrite"}]})\n'\'' | mongo mongodb://userAdmin:userAdmin123456@some-name-source-rs0.pitr-to-new-cluster-13977.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
++ mktemp
+ local LAST_OUT=/tmp/tmp.YTUtid9ACJ
++ mktemp
+ local LAST_ERR=/tmp/tmp.RWqn3uH9ja
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-696897d69b-9r6m5 -- bash -c 'printf '\''db.createUser({user:"myApp",pwd:"myPass",roles:[{db:"myApp",role:"readWrite"}]})\n'\'' | mongo mongodb://userAdmin:userAdmin123456@some-name-source-rs0.pitr-to-new-cluster-13977.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.YTUtid9ACJ
Percona Server for MongoDB shell version v4.4.29-28
connecting to: mongodb://some-name-source-rs0.pitr-to-new-cluster-13977.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb&replicaSet=rs0&ssl=false
Implicit session: session { "id" : UUID("6a4a1420-99f4-4812-96f8-e43a5eccbeea") }
Percona Server for MongoDB server version: v8.0.16-5
WARNING: shell and server versions do not match
Successfully added user: {
	"user" : "myApp",
	"roles" : [
		{
			"db" : "myApp",
			"role" : "readWrite"
		}
	]
}
bye
+ cat /tmp/tmp.RWqn3uH9ja
+ rm /tmp/tmp.YTUtid9ACJ /tmp/tmp.RWqn3uH9ja
+ return 0
+ write_document some-name-source 100500
+ local cluster=some-name-source
+ local x=100500
+ log 'write document: 100500'
+ set +o xtrace
[2025-12-23T15:38:16+0000] write document: 100500
+ run_mongo 'use myApp\n db.test.insert({ x: 100500 })' myApp:myPass@some-name-source-rs0.pitr-to-new-cluster-13977 mongodb
+ local 'command=use myApp\n db.test.insert({ x: 100500 })'
+ local uri=myApp:myPass@some-name-source-rs0.pitr-to-new-cluster-13977
+ local driver=mongodb
+ local suffix=.svc.cluster.local
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.p7dv6NtnKj
+++ mktemp
++ local LAST_ERR=/tmp/tmp.y5HcnbzorH
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.p7dv6NtnKj
++ cat /tmp/tmp.y5HcnbzorH
++ rm /tmp/tmp.p7dv6NtnKj /tmp/tmp.y5HcnbzorH
++ return 0
+ local client_container=psmdb-client-696897d69b-9r6m5
+ local mongo_flag=
+ [[ myApp:myPass@some-name-source-rs0.pitr-to-new-cluster-13977 == *cfg* ]]
+ replica_set=rs0
+ kubectl_bin exec psmdb-client-696897d69b-9r6m5 -- bash -c 'printf '\''use myApp\n db.test.insert({ x: 100500 })\n'\'' | mongo mongodb://myApp:myPass@some-name-source-rs0.pitr-to-new-cluster-13977.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
++ mktemp
+ local LAST_OUT=/tmp/tmp.xnNK5whMl6
++ mktemp
+ local LAST_ERR=/tmp/tmp.XJMMCRoo5k
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-696897d69b-9r6m5 -- bash -c 'printf '\''use myApp\n db.test.insert({ x: 100500 })\n'\'' | mongo mongodb://myApp:myPass@some-name-source-rs0.pitr-to-new-cluster-13977.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.xnNK5whMl6
Percona Server for MongoDB shell version v4.4.29-28
connecting to: mongodb://some-name-source-rs0.pitr-to-new-cluster-13977.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb&replicaSet=rs0&ssl=false
Implicit session: session { "id" : UUID("e50145b4-496f-46a8-a9a8-585a6539e8f1") }
Percona Server for MongoDB server version: v8.0.16-5
WARNING: shell and server versions do not match
switched to db myApp
WriteResult({ "nInserted" : 1 })
bye
+ cat /tmp/tmp.XJMMCRoo5k
+ rm /tmp/tmp.xnNK5whMl6 /tmp/tmp.XJMMCRoo5k
+ return 0
+ sleep_with_log 360 'wait for PBM-1265 workaround'
+ local d=360
+ local 'msg=wait for PBM-1265 workaround'
+ log 'wait for PBM-1265 workaround. sleeping for 360 seconds'
+ set +o xtrace
[2025-12-23T15:38:19+0000] wait for PBM-1265 workaround. sleeping for 360 seconds
+ sleep 360
+ compare_mongo_cmd find myApp:myPass@some-name-source-rs0.pitr-to-new-cluster-13977
+ local command=find
+ local uri=myApp:myPass@some-name-source-rs0.pitr-to-new-cluster-13977
+ local postfix=
+ local suffix=
+ local database=myApp
+ local collection=test
+ local sort=
+ local tls=false
+ local 'full_command=db.test.find()'
+ [[ -n '' ]]
+ log 'running db.test.find() in myApp'
+ set +o xtrace
[2025-12-23T15:44:19+0000] running db.test.find() in myApp
+ [[ false == \t\r\u\e ]]
+ mongo_command=run_mongo
+ run_mongo 'use myApp\n db.test.find()' myApp:myPass@some-name-source-rs0.pitr-to-new-cluster-13977 mongodb ''
+ egrep -v 'I NETWORK|W NETWORK|F NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|Started a new thread for the timer service'
+ local 'command=use myApp\n db.test.find()'
+ local uri=myApp:myPass@some-name-source-rs0.pitr-to-new-cluster-13977
+ local driver=mongodb
+ /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
+ local suffix=.svc.cluster.local
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
egrep: warning: egrep is obsolescent; using grep -E
++ local LAST_OUT=/tmp/tmp.FQgdovETxB
+++ mktemp
++ local LAST_ERR=/tmp/tmp.pnBzYdzQQQ
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.FQgdovETxB
++ cat /tmp/tmp.pnBzYdzQQQ
++ rm /tmp/tmp.FQgdovETxB /tmp/tmp.pnBzYdzQQQ
++ return 0
+ local client_container=psmdb-client-696897d69b-9r6m5
+ local mongo_flag=
+ [[ myApp:myPass@some-name-source-rs0.pitr-to-new-cluster-13977 == *cfg* ]]
+ replica_set=rs0
+ kubectl_bin exec psmdb-client-696897d69b-9r6m5 -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-source-rs0.pitr-to-new-cluster-13977.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
++ mktemp
+ local LAST_OUT=/tmp/tmp.ZxcxwfhvMI
++ mktemp
+ local LAST_ERR=/tmp/tmp.59TNVOf33M
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-696897d69b-9r6m5 -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-source-rs0.pitr-to-new-cluster-13977.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.ZxcxwfhvMI
+ cat /tmp/tmp.59TNVOf33M
+ rm /tmp/tmp.ZxcxwfhvMI /tmp/tmp.59TNVOf33M
+ return 0
+ [[ 0 -eq 0 ]]
+ diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2144/e2e-tests/pitr-to-new-cluster/compare/find.json /tmp/tmp.pwZ85mBPea/find
+ run_backup some-name-source backup-minio logical
+ local cluster=some-name-source
+ local name=backup-minio
+ local type=logical
+ desc 'run backup backup-minio-logical'
+ set +o xtrace


-----------------------------------------------------------------------------------
run backup backup-minio-logical
-----------------------------------------------------------------------------------

+ cat /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2144/e2e-tests/pitr-to-new-cluster/conf/backup.yml
+ /usr/sbin/sed -e 's/name:/name: backup-minio-logical/'
+ /usr/sbin/sed -e 's/clusterName:/clusterName: some-name-source/'
+ /usr/sbin/sed -e 's/type:/type: logical/'
+ kubectl_bin apply -f -
++ mktemp
+ local LAST_OUT=/tmp/tmp.m8rJtGJ9Or
++ mktemp
+ local LAST_ERR=/tmp/tmp.M57Iko3fV4
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl apply -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.m8rJtGJ9Or
perconaservermongodbbackup.psmdb.percona.com/backup-minio-logical created
+ cat /tmp/tmp.M57Iko3fV4
+ rm /tmp/tmp.m8rJtGJ9Or /tmp/tmp.M57Iko3fV4
+ return 0
+ wait_backup backup-minio-logical
+ local backup_name=backup-minio-logical
+ local target_state=ready
+ set +o xtrace
waiting for backup-minio-logical to reach ready state.......OK
+ sleep_with_log 70 'wait for oplog chunks to be uploaded'
+ local d=70
+ local 'msg=wait for oplog chunks to be uploaded'
+ log 'wait for oplog chunks to be uploaded. sleeping for 70 seconds'
+ set +o xtrace
[2025-12-23T15:44:37+0000] wait for oplog chunks to be uploaded. sleeping for 70 seconds
+ sleep 70
+ run_backup some-name-source backup-minio physical
+ local cluster=some-name-source
+ local name=backup-minio
+ local type=physical
+ desc 'run backup backup-minio-physical'
+ set +o xtrace


-----------------------------------------------------------------------------------
run backup backup-minio-physical
-----------------------------------------------------------------------------------

+ cat /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2144/e2e-tests/pitr-to-new-cluster/conf/backup.yml
+ /usr/sbin/sed -e 's/name:/name: backup-minio-physical/'
+ /usr/sbin/sed -e 's/type:/type: physical/'
+ /usr/sbin/sed -e 's/clusterName:/clusterName: some-name-source/'
+ kubectl_bin apply -f -
++ mktemp
+ local LAST_OUT=/tmp/tmp.BxJaTHKT5z
++ mktemp
+ local LAST_ERR=/tmp/tmp.QiEkoUnEbg
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl apply -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.BxJaTHKT5z
perconaservermongodbbackup.psmdb.percona.com/backup-minio-physical created
+ cat /tmp/tmp.QiEkoUnEbg
+ rm /tmp/tmp.BxJaTHKT5z /tmp/tmp.QiEkoUnEbg
+ return 0
+ wait_backup backup-minio-physical
+ local backup_name=backup-minio-physical
+ local target_state=ready
+ set +o xtrace
waiting for backup-minio-physical to reach ready state.......OK
+ desc 'write data for PiTR'
+ set +o xtrace


-----------------------------------------------------------------------------------
write data for PiTR
-----------------------------------------------------------------------------------

+ write_document some-name-source 100501
+ local cluster=some-name-source
+ local x=100501
+ log 'write document: 100501'
+ set +o xtrace
[2025-12-23T15:46:01+0000] write document: 100501
+ run_mongo 'use myApp\n db.test.insert({ x: 100501 })' myApp:myPass@some-name-source-rs0.pitr-to-new-cluster-13977 mongodb
+ local 'command=use myApp\n db.test.insert({ x: 100501 })'
+ local uri=myApp:myPass@some-name-source-rs0.pitr-to-new-cluster-13977
+ local driver=mongodb
+ local suffix=.svc.cluster.local
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.1OD7eymmGv
+++ mktemp
++ local LAST_ERR=/tmp/tmp.AQaSFnrVK5
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.1OD7eymmGv
++ cat /tmp/tmp.AQaSFnrVK5
++ rm /tmp/tmp.1OD7eymmGv /tmp/tmp.AQaSFnrVK5
++ return 0
+ local client_container=psmdb-client-696897d69b-9r6m5
+ local mongo_flag=
+ [[ myApp:myPass@some-name-source-rs0.pitr-to-new-cluster-13977 == *cfg* ]]
+ replica_set=rs0
+ kubectl_bin exec psmdb-client-696897d69b-9r6m5 -- bash -c 'printf '\''use myApp\n db.test.insert({ x: 100501 })\n'\'' | mongo mongodb://myApp:myPass@some-name-source-rs0.pitr-to-new-cluster-13977.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
++ mktemp
+ local LAST_OUT=/tmp/tmp.xmL04KxQLN
++ mktemp
+ local LAST_ERR=/tmp/tmp.mgLZAWkbKO
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-696897d69b-9r6m5 -- bash -c 'printf '\''use myApp\n db.test.insert({ x: 100501 })\n'\'' | mongo mongodb://myApp:myPass@some-name-source-rs0.pitr-to-new-cluster-13977.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.xmL04KxQLN
Percona Server for MongoDB shell version v4.4.29-28
connecting to: mongodb://some-name-source-rs0.pitr-to-new-cluster-13977.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb&replicaSet=rs0&ssl=false
Implicit session: session { "id" : UUID("db5fad22-63f6-4273-a7a4-3e37cbb0df29") }
Percona Server for MongoDB server version: v8.0.16-5
WARNING: shell and server versions do not match
switched to db myApp
WriteResult({ "nInserted" : 1 })
bye
+ cat /tmp/tmp.mgLZAWkbKO
+ rm /tmp/tmp.xmL04KxQLN /tmp/tmp.mgLZAWkbKO
+ return 0
+ write_document some-name-source 100502
+ local cluster=some-name-source
+ local x=100502
+ log 'write document: 100502'
+ set +o xtrace
[2025-12-23T15:46:03+0000] write document: 100502
+ run_mongo 'use myApp\n db.test.insert({ x: 100502 })' myApp:myPass@some-name-source-rs0.pitr-to-new-cluster-13977 mongodb
+ local 'command=use myApp\n db.test.insert({ x: 100502 })'
+ local uri=myApp:myPass@some-name-source-rs0.pitr-to-new-cluster-13977
+ local driver=mongodb
+ local suffix=.svc.cluster.local
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.VO0SieXFn5
+++ mktemp
++ local LAST_ERR=/tmp/tmp.XoDOzgoZZo
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.VO0SieXFn5
++ cat /tmp/tmp.XoDOzgoZZo
++ rm /tmp/tmp.VO0SieXFn5 /tmp/tmp.XoDOzgoZZo
++ return 0
+ local client_container=psmdb-client-696897d69b-9r6m5
+ local mongo_flag=
+ [[ myApp:myPass@some-name-source-rs0.pitr-to-new-cluster-13977 == *cfg* ]]
+ replica_set=rs0
+ kubectl_bin exec psmdb-client-696897d69b-9r6m5 -- bash -c 'printf '\''use myApp\n db.test.insert({ x: 100502 })\n'\'' | mongo mongodb://myApp:myPass@some-name-source-rs0.pitr-to-new-cluster-13977.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
++ mktemp
+ local LAST_OUT=/tmp/tmp.Rh7LTOTw74
++ mktemp
+ local LAST_ERR=/tmp/tmp.6Q0VvQiIyV
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-696897d69b-9r6m5 -- bash -c 'printf '\''use myApp\n db.test.insert({ x: 100502 })\n'\'' | mongo mongodb://myApp:myPass@some-name-source-rs0.pitr-to-new-cluster-13977.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.Rh7LTOTw74
Percona Server for MongoDB shell version v4.4.29-28
connecting to: mongodb://some-name-source-rs0.pitr-to-new-cluster-13977.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb&replicaSet=rs0&ssl=false
Implicit session: session { "id" : UUID("c294d38c-efae-43a8-9dd3-204eb2409f62") }
Percona Server for MongoDB server version: v8.0.16-5
WARNING: shell and server versions do not match
switched to db myApp
WriteResult({ "nInserted" : 1 })
bye
+ cat /tmp/tmp.6Q0VvQiIyV
+ rm /tmp/tmp.Rh7LTOTw74 /tmp/tmp.6Q0VvQiIyV
+ return 0
+ write_document some-name-source 100503
+ local cluster=some-name-source
+ local x=100503
+ log 'write document: 100503'
+ set +o xtrace
[2025-12-23T15:46:06+0000] write document: 100503
+ run_mongo 'use myApp\n db.test.insert({ x: 100503 })' myApp:myPass@some-name-source-rs0.pitr-to-new-cluster-13977 mongodb
+ local 'command=use myApp\n db.test.insert({ x: 100503 })'
+ local uri=myApp:myPass@some-name-source-rs0.pitr-to-new-cluster-13977
+ local driver=mongodb
+ local suffix=.svc.cluster.local
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.hnbYwKZ3ro
+++ mktemp
++ local LAST_ERR=/tmp/tmp.KxRmkg1oW5
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.hnbYwKZ3ro
++ cat /tmp/tmp.KxRmkg1oW5
++ rm /tmp/tmp.hnbYwKZ3ro /tmp/tmp.KxRmkg1oW5
++ return 0
+ local client_container=psmdb-client-696897d69b-9r6m5
+ local mongo_flag=
+ [[ myApp:myPass@some-name-source-rs0.pitr-to-new-cluster-13977 == *cfg* ]]
+ replica_set=rs0
+ kubectl_bin exec psmdb-client-696897d69b-9r6m5 -- bash -c 'printf '\''use myApp\n db.test.insert({ x: 100503 })\n'\'' | mongo mongodb://myApp:myPass@some-name-source-rs0.pitr-to-new-cluster-13977.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
++ mktemp
+ local LAST_OUT=/tmp/tmp.AYCntiyyyK
++ mktemp
+ local LAST_ERR=/tmp/tmp.ABDTEt6G7A
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-696897d69b-9r6m5 -- bash -c 'printf '\''use myApp\n db.test.insert({ x: 100503 })\n'\'' | mongo mongodb://myApp:myPass@some-name-source-rs0.pitr-to-new-cluster-13977.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.AYCntiyyyK
Percona Server for MongoDB shell version v4.4.29-28
connecting to: mongodb://some-name-source-rs0.pitr-to-new-cluster-13977.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb&replicaSet=rs0&ssl=false
Implicit session: session { "id" : UUID("8b593f69-bc82-4f9c-b700-2b1a7c5df09d") }
Percona Server for MongoDB server version: v8.0.16-5
WARNING: shell and server versions do not match
switched to db myApp
WriteResult({ "nInserted" : 1 })
bye
+ cat /tmp/tmp.ABDTEt6G7A
+ rm /tmp/tmp.AYCntiyyyK /tmp/tmp.ABDTEt6G7A
+ return 0
+ sleep_with_log 70 'wait for oplog chunks to be uploaded'
+ local d=70
+ local 'msg=wait for oplog chunks to be uploaded'
+ log 'wait for oplog chunks to be uploaded. sleeping for 70 seconds'
+ set +o xtrace
[2025-12-23T15:46:09+0000] wait for oplog chunks to be uploaded. sleeping for 70 seconds
+ sleep 70
+ compare_mongo_cmd find myApp:myPass@some-name-source-rs0.pitr-to-new-cluster-13977 -2nd .svc.cluster.local myApp test 'sort( { x: 1 } )'
+ local command=find
+ local uri=myApp:myPass@some-name-source-rs0.pitr-to-new-cluster-13977
+ local postfix=-2nd
+ local suffix=.svc.cluster.local
+ local database=myApp
+ local collection=test
+ local 'sort=sort( { x: 1 } )'
+ local tls=false
+ local 'full_command=db.test.find()'
+ [[ -n sort( { x: 1 } ) ]]
+ full_command='db.test.find().sort( { x: 1 } )'
+ log 'running db.test.find().sort( { x: 1 } ) in myApp'
+ set +o xtrace
[2025-12-23T15:47:19+0000] running db.test.find().sort( { x: 1 } ) in myApp
+ [[ false == \t\r\u\e ]]
+ mongo_command=run_mongo
+ run_mongo 'use myApp\n db.test.find().sort( { x: 1 } )' myApp:myPass@some-name-source-rs0.pitr-to-new-cluster-13977 mongodb .svc.cluster.local
+ local 'command=use myApp\n db.test.find().sort( { x: 1 } )'
+ local uri=myApp:myPass@some-name-source-rs0.pitr-to-new-cluster-13977
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ egrep -v 'I NETWORK|W NETWORK|F NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|Started a new thread for the timer service'
+ /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
egrep: warning: egrep is obsolescent; using grep -E
++ local LAST_OUT=/tmp/tmp.AGv6viu0bo
+++ mktemp
++ local LAST_ERR=/tmp/tmp.oGhpYuBpFw
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.AGv6viu0bo
++ cat /tmp/tmp.oGhpYuBpFw
++ rm /tmp/tmp.AGv6viu0bo /tmp/tmp.oGhpYuBpFw
++ return 0
+ local client_container=psmdb-client-696897d69b-9r6m5
+ local mongo_flag=
+ [[ myApp:myPass@some-name-source-rs0.pitr-to-new-cluster-13977 == *cfg* ]]
+ replica_set=rs0
+ kubectl_bin exec psmdb-client-696897d69b-9r6m5 -- bash -c 'printf '\''use myApp\n db.test.find().sort( { x: 1 } )\n'\'' | mongo mongodb://myApp:myPass@some-name-source-rs0.pitr-to-new-cluster-13977.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
++ mktemp
+ local LAST_OUT=/tmp/tmp.R8DHWBudRR
++ mktemp
+ local LAST_ERR=/tmp/tmp.7V385Nt0u2
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-696897d69b-9r6m5 -- bash -c 'printf '\''use myApp\n db.test.find().sort( { x: 1 } )\n'\'' | mongo mongodb://myApp:myPass@some-name-source-rs0.pitr-to-new-cluster-13977.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.R8DHWBudRR
+ cat /tmp/tmp.7V385Nt0u2
+ rm /tmp/tmp.R8DHWBudRR /tmp/tmp.7V385Nt0u2
+ return 0
+ [[ 0 -eq 0 ]]
+ diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2144/e2e-tests/pitr-to-new-cluster/compare/find-2nd.json /tmp/tmp.pwZ85mBPea/find-2nd
+ log 'PiTR data is ready: OK'
+ set +o xtrace
[2025-12-23T15:47:21+0000] PiTR data is ready: OK
+ setup_target_cluster
+ desc 'create PSMDB cluster: some-name-target'
+ set +o xtrace


-----------------------------------------------------------------------------------
create PSMDB cluster: some-name-target
-----------------------------------------------------------------------------------

+ apply_cluster /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2144/e2e-tests/pitr-to-new-cluster/conf/some-name-target.yml
+ '[' -z '' ']'
+ cat_config /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2144/e2e-tests/pitr-to-new-cluster/conf/some-name-target.yml
+ kubectl_bin apply -f -
+ cat /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2144/e2e-tests/pitr-to-new-cluster/conf/some-name-target.yml
++ mktemp
+ yq eval '(.spec | select(.image == null)).image = "perconalab/percona-server-mongodb-operator:main-mongod8.0"'
+ yq eval '(.spec | select(has("pmm"))).pmm.image = "percona/pmm-client:2.44.1-1"'
+ yq eval '(.spec | select(has("initImage"))).initImage = "perconalab/percona-server-mongodb-operator:PR-2144-c278c1c9"'
+ local LAST_OUT=/tmp/tmp.WQe4Vse6Yi
++ mktemp
+ yq eval '.spec.upgradeOptions.apply="Never"'
+ yq eval '(.spec | select(has("backup"))).backup.image = "perconalab/percona-server-mongodb-operator:main-backup"'
+ local LAST_ERR=/tmp/tmp.ISHJL1Mlzi
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl apply -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.WQe4Vse6Yi
perconaservermongodb.psmdb.percona.com/some-name-target created
+ cat /tmp/tmp.ISHJL1Mlzi
+ rm /tmp/tmp.WQe4Vse6Yi /tmp/tmp.ISHJL1Mlzi
+ return 0
+ wait_for_running some-name-target-rs0 3
+ local name=some-name-target-rs0
+ let last_pod=2
+ local check_cluster_readyness=true
+ set_debug
+ [[ 1 == 1 ]]
+ set -o xtrace
+ local rs_name=rs0
+ local cluster_name=some-name-target
++ seq 0 2
+ for i in $(seq 0 $last_pod)
+ [[ 0 -eq 2 ]]
+ wait_pod some-name-target-rs0-0
+ local pod=some-name-target-rs0-0
+ set +o xtrace
waiting for pod/some-name-target-rs0-0 to be ready.......OK
+ for i in $(seq 0 $last_pod)
+ [[ 1 -eq 2 ]]
+ wait_pod some-name-target-rs0-1
+ local pod=some-name-target-rs0-1
+ set +o xtrace
waiting for pod/some-name-target-rs0-1 to be ready......OK
+ for i in $(seq 0 $last_pod)
+ [[ 2 -eq 2 ]]
++ kubectl_bin get psmdb some-name-target -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].arbiter.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.JBcdN8ATo7
+++ mktemp
++ local LAST_ERR=/tmp/tmp.0FBA7HFIjE
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name-target -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].arbiter.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.JBcdN8ATo7
++ cat /tmp/tmp.0FBA7HFIjE
++ rm /tmp/tmp.JBcdN8ATo7 /tmp/tmp.0FBA7HFIjE
++ return 0
+ [[ '' == \t\r\u\e ]]
+ wait_pod some-name-target-rs0-2
+ local pod=some-name-target-rs0-2
+ set +o xtrace
waiting for pod/some-name-target-rs0-2 to be ready.....OK
++ kubectl_bin get psmdb some-name-target -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].nonvoting.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.RBB9iDDVvS
+++ mktemp
++ local LAST_ERR=/tmp/tmp.s3YI5XLaj1
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name-target -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].nonvoting.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.RBB9iDDVvS
++ cat /tmp/tmp.s3YI5XLaj1
++ rm /tmp/tmp.RBB9iDDVvS /tmp/tmp.s3YI5XLaj1
++ return 0
+ [[ '' == \t\r\u\e ]]
++ kubectl_bin get psmdb some-name-target -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].hidden.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.ZKIQqyJYYP
+++ mktemp
++ local LAST_ERR=/tmp/tmp.sjCupLl8ex
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name-target -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].hidden.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.ZKIQqyJYYP
++ cat /tmp/tmp.sjCupLl8ex
++ rm /tmp/tmp.ZKIQqyJYYP /tmp/tmp.sjCupLl8ex
++ return 0
+ [[ '' == \t\r\u\e ]]
+ sleep 10
+ [[ true == \t\r\u\e ]]
+ set +x
Waiting for cluster readyness..
+ wait_backup_agent some-name-target-rs0-0
+ local agent_pod=some-name-target-rs0-0
+ set +o xtrace
waiting for pbm-agent to be ready in some-name-target-rs0-0...2025-12-23T15:48:38.000+0000 I listening for the commands

+ wait_backup_agent some-name-target-rs0-1
+ local agent_pod=some-name-target-rs0-1
+ set +o xtrace
waiting for pbm-agent to be ready in some-name-target-rs0-1...2025-12-23T15:48:44.000+0000 I listening for the commands

+ wait_backup_agent some-name-target-rs0-2
+ local agent_pod=some-name-target-rs0-2
+ set +o xtrace
waiting for pbm-agent to be ready in some-name-target-rs0-2...2025-12-23T15:48:44.000+0000 I listening for the commands

+ wait_for_pbm_operations some-name-target
+ local cluster=some-name-target
+ set +o xtrace
waiting for PBM operation to finish.OK
+ desc 'Case 1: Logical restore'
+ set +o xtrace


-----------------------------------------------------------------------------------
Case 1: Logical restore
-----------------------------------------------------------------------------------

+ test_case_1
+ run_restore some-name-target backup-minio logical
+ local cluster=some-name-target
+ local backup_name=backup-minio
+ local backup_type=logical
++ get_backup_dest backup-minio-logical
++ local backup_name=backup-minio-logical
++ kubectl_bin get psmdb-backup backup-minio-logical -o 'jsonpath={.status.destination}'
++ sed -e 's/.json$//'
++ sed 's|s3://||'
++ sed 's|azure://||'
+++ mktemp
++ sed 's|gs://||'
++ local LAST_OUT=/tmp/tmp.Flh7BUrZgi
+++ mktemp
++ local LAST_ERR=/tmp/tmp.HYAZTZOyyk
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb-backup backup-minio-logical -o 'jsonpath={.status.destination}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.Flh7BUrZgi
++ cat /tmp/tmp.HYAZTZOyyk
++ rm /tmp/tmp.Flh7BUrZgi /tmp/tmp.HYAZTZOyyk
++ return 0
+ local destination=operator-testing/some-name-source/2025-12-23T15:44:25Z
+ cat /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2144/e2e-tests/pitr-to-new-cluster/conf/restore-backupsource.yml
+ /usr/sbin/sed -e 's/name:/name: restore-backup-minio-logical/'
+ /usr/sbin/sed -e 's/clusterName:/clusterName: some-name-target/'
+ /usr/sbin/sed -e 's/type:/type: logical/'
+ /usr/sbin/sed -e s/pitrType:/type:/
+ /usr/sbin/sed -e 's|DESTINATION|operator-testing/some-name-source/2025-12-23T15:44:25Z|'
+ kubectl_bin apply -f -
++ mktemp
+ local LAST_OUT=/tmp/tmp.VtH7B4OGgZ
++ mktemp
+ local LAST_ERR=/tmp/tmp.DDPI9yVk9i
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl apply -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.VtH7B4OGgZ
perconaservermongodbrestore.psmdb.percona.com/restore-backup-minio-logical created
+ cat /tmp/tmp.DDPI9yVk9i
+ rm /tmp/tmp.VtH7B4OGgZ /tmp/tmp.DDPI9yVk9i
+ return 0
+ wait_restore backup-minio-logical some-name-target requested 0
+ local backup_name=backup-minio-logical
+ local cluster_name=some-name-target
+ local target_state=requested
+ local wait_cluster_consistency=0
+ local wait_time=1780
+ local ok_if_ready=0
+ set +o xtrace
Waiting for the psmdb-restore/restore-backup-minio-logical object to be created.OK
Waiting psmdb-restore/restore-backup-minio-logical to reach state "requested" .OK after 0 minutes
+ [[ 0 -eq 1 ]]
+ compare_pbm_config some-name-target source
+ local cluster=some-name-target
+ local compare_suffix=source
+ local container=backup-agent
+ local pbm_binary=pbm
+ [[ backup-agent == \m\o\n\g\o\d ]]
+ kubectl_bin exec some-name-target-rs0-0 -c backup-agent -- pbm config
++ mktemp
+ local LAST_OUT=/tmp/tmp.AJHP8SGxHy
++ mktemp
+ local LAST_ERR=/tmp/tmp.xfp1N5Yjvp
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec some-name-target-rs0-0 -c backup-agent -- pbm config
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.AJHP8SGxHy
+ cat /tmp/tmp.xfp1N5Yjvp
+ rm /tmp/tmp.AJHP8SGxHy /tmp/tmp.xfp1N5Yjvp
+ return 0
+ diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2144/e2e-tests/pitr-to-new-cluster/compare/pbm_config_source.yml /tmp/tmp.pwZ85mBPea/pbm_config.yml
+ log 'Backup source is set as main storage during restore: OK'
+ set +o xtrace
[2025-12-23T15:49:05+0000] Backup source is set as main storage during restore: OK
++ date +%s
+ local now_ts=1766504945
+ wait_restore backup-minio-logical some-name-target ready
+ local backup_name=backup-minio-logical
+ local cluster_name=some-name-target
+ local target_state=ready
+ local wait_cluster_consistency=1
+ local wait_time=1780
+ local ok_if_ready=0
+ set +o xtrace
Waiting for the psmdb-restore/restore-backup-minio-logical object to be created.OK
Waiting psmdb-restore/restore-backup-minio-logical to reach state "ready" .OK after 0 minutes
+ [[ 1 -eq 1 ]]
+ wait_cluster_consistency some-name-target
+ local cluster_name=some-name-target
+ local wait_time=32
+ retry=0
+ sleep 7
+ echo -n 'waiting for cluster readyness'
waiting for cluster readyness++ kubectl_bin get psmdb some-name-target -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.kPjPxhAXed
+++ mktemp
++ local LAST_ERR=/tmp/tmp.rBmulrU0jF
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name-target -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.kPjPxhAXed
++ cat /tmp/tmp.rBmulrU0jF
++ rm /tmp/tmp.kPjPxhAXed /tmp/tmp.rBmulrU0jF
++ return 0
+ [[ ready == \r\e\a\d\y ]]
+ echo .OK
.OK
+ compare_pbm_config some-name-target target
+ local cluster=some-name-target
+ local compare_suffix=target
+ local container=backup-agent
+ local pbm_binary=pbm
+ [[ backup-agent == \m\o\n\g\o\d ]]
+ kubectl_bin exec some-name-target-rs0-0 -c backup-agent -- pbm config
++ mktemp
+ local LAST_OUT=/tmp/tmp.N0I0Bdrl9Q
++ mktemp
+ local LAST_ERR=/tmp/tmp.WMAv09Av4u
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec some-name-target-rs0-0 -c backup-agent -- pbm config
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.N0I0Bdrl9Q
+ cat /tmp/tmp.WMAv09Av4u
+ rm /tmp/tmp.N0I0Bdrl9Q /tmp/tmp.WMAv09Av4u
+ return 0
+ diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2144/e2e-tests/pitr-to-new-cluster/compare/pbm_config_target.yml /tmp/tmp.pwZ85mBPea/pbm_config.yml
+ log 'Main storage is reverted after backup-minio-logical: OK'
+ set +o xtrace
[2025-12-23T15:49:23+0000] Main storage is reverted after backup-minio-logical: OK
+ wait_for_pbm_operations some-name-target
+ local cluster=some-name-target
+ set +o xtrace
waiting for PBM operation to finish.OK
+ check_if_resync_triggered some-name-target 1766504945
+ local cluster=some-name-target
+ local ts=1766504945
+ log 'checking if resync is triggered after 1766504945'
+ set +o xtrace
[2025-12-23T15:49:25+0000] checking if resync is triggered after 1766504945
++ run_mongo 'use admin\n db.pbmCmd.countDocuments({cmd: '\''resync'\'', ts: { '\''$gt'\'': 1766504945 }})' backup:backup123456@some-name-target-rs0.pitr-to-new-cluster-13977
++ local 'command=use admin\n db.pbmCmd.countDocuments({cmd: '\''resync'\'', ts: { '\''$gt'\'': 1766504945 }})'
++ local uri=backup:backup123456@some-name-target-rs0.pitr-to-new-cluster-13977
++ local driver=mongodb+srv
++ local suffix=.svc.cluster.local
+++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.o7LnRIckj2
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.WBmNGbj3Mx
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in $(seq 0 2)
+++ set +e
+++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.o7LnRIckj2
+++ cat /tmp/tmp.WBmNGbj3Mx
+++ rm /tmp/tmp.o7LnRIckj2 /tmp/tmp.WBmNGbj3Mx
+++ return 0
++ local client_container=psmdb-client-696897d69b-9r6m5
++ local mongo_flag=
++ [[ backup:backup123456@some-name-target-rs0.pitr-to-new-cluster-13977 == *cfg* ]]
++ replica_set=rs0
++ kubectl_bin exec psmdb-client-696897d69b-9r6m5 -- bash -c 'printf '\''use admin\n db.pbmCmd.countDocuments({cmd: '\''resync'\'', ts: { '\''$gt'\'': 1766504945 }})\n'\'' | mongo mongodb+srv://backup:backup123456@some-name-target-rs0.pitr-to-new-cluster-13977.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
+++ mktemp
++ local LAST_OUT=/tmp/tmp.szLNbYHIkq
+++ mktemp
++ local LAST_ERR=/tmp/tmp.GAUtj3XqYm
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl exec psmdb-client-696897d69b-9r6m5 -- bash -c 'printf '\''use admin\n db.pbmCmd.countDocuments({cmd: '\''resync'\'', ts: { '\''$gt'\'': 1766504945 }})\n'\'' | mongo mongodb+srv://backup:backup123456@some-name-target-rs0.pitr-to-new-cluster-13977.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
++ exit_status=1
++ set -e
++ '[' 1 '!=' 0 -a -n 1 ']'
++ cat /tmp/tmp.szLNbYHIkq
++ cat /tmp/tmp.GAUtj3XqYm
command terminated with exit code 1
++ sleep 0
++ for i in $(seq 0 2)
++ set +e
++ kubectl exec psmdb-client-696897d69b-9r6m5 -- bash -c 'printf '\''use admin\n db.pbmCmd.countDocuments({cmd: '\''resync'\'', ts: { '\''$gt'\'': 1766504945 }})\n'\'' | mongo mongodb+srv://backup:backup123456@some-name-target-rs0.pitr-to-new-cluster-13977.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
++ exit_status=1
++ set -e
++ '[' 1 '!=' 0 -a -n 1 ']'
++ cat /tmp/tmp.szLNbYHIkq
++ cat /tmp/tmp.GAUtj3XqYm
command terminated with exit code 1
++ sleep 4
++ for i in $(seq 0 2)
++ set +e
++ kubectl exec psmdb-client-696897d69b-9r6m5 -- bash -c 'printf '\''use admin\n db.pbmCmd.countDocuments({cmd: '\''resync'\'', ts: { '\''$gt'\'': 1766504945 }})\n'\'' | mongo mongodb+srv://backup:backup123456@some-name-target-rs0.pitr-to-new-cluster-13977.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
++ exit_status=1
++ set -e
++ '[' 1 '!=' 0 -a -n 1 ']'
++ cat /tmp/tmp.szLNbYHIkq
++ cat /tmp/tmp.GAUtj3XqYm
command terminated with exit code 1
++ sleep 8
++ cat /tmp/tmp.szLNbYHIkq
++ cat /tmp/tmp.GAUtj3XqYm
command terminated with exit code 1
++ rm /tmp/tmp.szLNbYHIkq /tmp/tmp.GAUtj3XqYm
++ return 1
+ local 'count=Percona Server for MongoDB shell version v4.4.29-28
connecting to: mongodb://some-name-target-rs0-0.some-name-target-rs0.pitr-to-new-cluster-13977.svc.cluster.local:27017,some-name-target-rs0-2.some-name-target-rs0.pitr-to-new-cluster-13977.svc.cluster.local:27017,some-name-target-rs0-1.some-name-target-rs0.pitr-to-new-cluster-13977.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb&replicaSet=rs0&ssl=false
Implicit session: session { "id" : UUID("3b515721-7492-4622-af92-cce560cbff7d") }
Percona Server for MongoDB server version: v8.0.16-5
WARNING: shell and server versions do not match
switched to db admin
uncaught exception: SyntaxError: expected property name, got '\'':'\'' :
@(shell):1:45
bye
Percona Server for MongoDB shell version v4.4.29-28
connecting to: mongodb://some-name-target-rs0-1.some-name-target-rs0.pitr-to-new-cluster-13977.svc.cluster.local:27017,some-name-target-rs0-0.some-name-target-rs0.pitr-to-new-cluster-13977.svc.cluster.local:27017,some-name-target-rs0-2.some-name-target-rs0.pitr-to-new-cluster-13977.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb&replicaSet=rs0&ssl=false
Implicit session: session { "id" : UUID("c985459a-6b77-4e72-a71c-884bf9037e01") }
Percona Server for MongoDB server version: v8.0.16-5
WARNING: shell and server versions do not match
switched to db admin
uncaught exception: SyntaxError: expected property name, got '\'':'\'' :
@(shell):1:45
bye
Percona Server for MongoDB shell version v4.4.29-28
connecting to: mongodb://some-name-target-rs0-0.some-name-target-rs0.pitr-to-new-cluster-13977.svc.cluster.local:27017,some-name-target-rs0-2.some-name-target-rs0.pitr-to-new-cluster-13977.svc.cluster.local:27017,some-name-target-rs0-1.some-name-target-rs0.pitr-to-new-cluster-13977.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb&replicaSet=rs0&ssl=false
Implicit session: session { "id" : UUID("8f926499-362a-41b4-ab37-7e35c7d74109") }
Percona Server for MongoDB server version: v8.0.16-5
WARNING: shell and server versions do not match
switched to db admin
uncaught exception: SyntaxError: expected property name, got '\'':'\'' :
@(shell):1:45
bye
Percona Server for MongoDB shell version v4.4.29-28
connecting to: mongodb://some-name-target-rs0-0.some-name-target-rs0.pitr-to-new-cluster-13977.svc.cluster.local:27017,some-name-target-rs0-2.some-name-target-rs0.pitr-to-new-cluster-13977.svc.cluster.local:27017,some-name-target-rs0-1.some-name-target-rs0.pitr-to-new-cluster-13977.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb&replicaSet=rs0&ssl=false
Implicit session: session { "id" : UUID("8f926499-362a-41b4-ab37-7e35c7d74109") }
Percona Server for MongoDB server version: v8.0.16-5
WARNING: shell and server versions do not match
switched to db admin
uncaught exception: SyntaxError: expected property name, got '\'':'\'' :
@(shell):1:45
bye'
+ [[ Percona Server for MongoDB shell version v4.4.29-28
connecting to: mongodb://some-name-target-rs0-0.some-name-target-rs0.pitr-to-new-cluster-13977.svc.cluster.local:27017,some-name-target-rs0-2.some-name-target-rs0.pitr-to-new-cluster-13977.svc.cluster.local:27017,some-name-target-rs0-1.some-name-target-rs0.pitr-to-new-cluster-13977.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb&replicaSet=rs0&ssl=false
Implicit session: session { "id" : UUID("3b515721-7492-4622-af92-cce560cbff7d") }
Percona Server for MongoDB server version: v8.0.16-5
WARNING: shell and server versions do not match
switched to db admin
uncaught exception: SyntaxError: expected property name, got ':' :
@(shell):1:45
bye
Percona Server for MongoDB shell version v4.4.29-28
connecting to: mongodb://some-name-target-rs0-1.some-name-target-rs0.pitr-to-new-cluster-13977.svc.cluster.local:27017,some-name-target-rs0-0.some-name-target-rs0.pitr-to-new-cluster-13977.svc.cluster.local:27017,some-name-target-rs0-2.some-name-target-rs0.pitr-to-new-cluster-13977.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb&replicaSet=rs0&ssl=false
Implicit session: session { "id" : UUID("c985459a-6b77-4e72-a71c-884bf9037e01") }
Percona Server for MongoDB server version: v8.0.16-5
WARNING: shell and server versions do not match
switched to db admin
uncaught exception: SyntaxError: expected property name, got ':' :
@(shell):1:45
bye
Percona Server for MongoDB shell version v4.4.29-28
connecting to: mongodb://some-name-target-rs0-0.some-name-target-rs0.pitr-to-new-cluster-13977.svc.cluster.local:27017,some-name-target-rs0-2.some-name-target-rs0.pitr-to-new-cluster-13977.svc.cluster.local:27017,some-name-target-rs0-1.some-name-target-rs0.pitr-to-new-cluster-13977.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb&replicaSet=rs0&ssl=false
Implicit session: session { "id" : UUID("8f926499-362a-41b4-ab37-7e35c7d74109") }
Percona Server for MongoDB server version: v8.0.16-5
WARNING: shell and server versions do not match
switched to db admin
uncaught exception: SyntaxError: expected property name, got ':' :
@(shell):1:45
bye
Percona Server for MongoDB shell version v4.4.29-28
connecting to: mongodb://some-name-target-rs0-0.some-name-target-rs0.pitr-to-new-cluster-13977.svc.cluster.local:27017,some-name-target-rs0-2.some-name-target-rs0.pitr-to-new-cluster-13977.svc.cluster.local:27017,some-name-target-rs0-1.some-name-target-rs0.pitr-to-new-cluster-13977.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb&replicaSet=rs0&ssl=false
Implicit session: session { "id" : UUID("8f926499-362a-41b4-ab37-7e35c7d74109") }
Percona Server for MongoDB server version: v8.0.16-5
WARNING: shell and server versions do not match
switched to db admin
uncaught exception: SyntaxError: expected property name, got ':' :
@(shell):1:45
bye -lt 1 ]]
./e2e-tests/pitr-to-new-cluster/run: line 91: [[: Percona Server for MongoDB shell version v4.4.29-28
connecting to: mongodb://some-name-target-rs0-0.some-name-target-rs0.pitr-to-new-cluster-13977.svc.cluster.local:27017,some-name-target-rs0-2.some-name-target-rs0.pitr-to-new-cluster-13977.svc.cluster.local:27017,some-name-target-rs0-1.some-name-target-rs0.pitr-to-new-cluster-13977.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb&replicaSet=rs0&ssl=false
Implicit session: session { "id" : UUID("3b515721-7492-4622-af92-cce560cbff7d") }
Percona Server for MongoDB server version: v8.0.16-5
WARNING: shell and server versions do not match
switched to db admin
uncaught exception: SyntaxError: expected property name, got ':' :
@(shell):1:45
bye
Percona Server for MongoDB shell version v4.4.29-28
connecting to: mongodb://some-name-target-rs0-1.some-name-target-rs0.pitr-to-new-cluster-13977.svc.cluster.local:27017,some-name-target-rs0-0.some-name-target-rs0.pitr-to-new-cluster-13977.svc.cluster.local:27017,some-name-target-rs0-2.some-name-target-rs0.pitr-to-new-cluster-13977.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb&replicaSet=rs0&ssl=false
Implicit session: session { "id" : UUID("c985459a-6b77-4e72-a71c-884bf9037e01") }
Percona Server for MongoDB server version: v8.0.16-5
WARNING: shell and server versions do not match
switched to db admin
uncaught exception: SyntaxError: expected property name, got ':' :
@(shell):1:45
bye
Percona Server for MongoDB shell version v4.4.29-28
connecting to: mongodb://some-name-target-rs0-0.some-name-target-rs0.pitr-to-new-cluster-13977.svc.cluster.local:27017,some-name-target-rs0-2.some-name-target-rs0.pitr-to-new-cluster-13977.svc.cluster.local:27017,some-name-target-rs0-1.some-name-target-rs0.pitr-to-new-cluster-13977.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb&replicaSet=rs0&ssl=false
Implicit session: session { "id" : UUID("8f926499-362a-41b4-ab37-7e35c7d74109") }
Percona Server for MongoDB server version: v8.0.16-5
WARNING: shell and server versions do not match
switched to db admin
uncaught exception: SyntaxError: expected property name, got ':' :
@(shell):1:45
bye
Percona Server for MongoDB shell version v4.4.29-28
connecting to: mongodb://some-name-target-rs0-0.some-name-target-rs0.pitr-to-new-cluster-13977.svc.cluster.local:27017,some-name-target-rs0-2.some-name-target-rs0.pitr-to-new-cluster-13977.svc.cluster.local:27017,some-name-target-rs0-1.some-name-target-rs0.pitr-to-new-cluster-13977.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb&replicaSet=rs0&ssl=false
Implicit session: session { "id" : UUID("8f926499-362a-41b4-ab37-7e35c7d74109") }
Percona Server for MongoDB server version: v8.0.16-5
WARNING: shell and server versions do not match
switched to db admin
uncaught exception: SyntaxError: expected property name, got ':' :
@(shell):1:45
bye: syntax error: invalid arithmetic operator (error token is ".4.29-28
connecting to: mongodb://some-name-target-rs0-0.some-name-target-rs0.pitr-to-new-cluster-13977.svc.cluster.local:27017,some-name-target-rs0-2.some-name-target-rs0.pitr-to-new-cluster-13977.svc.cluster.local:27017,some-name-target-rs0-1.some-name-target-rs0.pitr-to-new-cluster-13977.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb&replicaSet=rs0&ssl=false
Implicit session: session { "id" : UUID("3b515721-7492-4622-af92-cce560cbff7d") }
Percona Server for MongoDB server version: v8.0.16-5
WARNING: shell and server versions do not match
switched to db admin
uncaught exception: SyntaxError: expected property name, got ':' :
@(shell):1:45
bye
Percona Server for MongoDB shell version v4.4.29-28
connecting to: mongodb://some-name-target-rs0-1.some-name-target-rs0.pitr-to-new-cluster-13977.svc.cluster.local:27017,some-name-target-rs0-0.some-name-target-rs0.pitr-to-new-cluster-13977.svc.cluster.local:27017,some-name-target-rs0-2.some-name-target-rs0.pitr-to-new-cluster-13977.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb&replicaSet=rs0&ssl=false
Implicit session: session { "id" : UUID("c985459a-6b77-4e72-a71c-884bf9037e01") }
Percona Server for MongoDB server version: v8.0.16-5
WARNING: shell and server versions do not match
switched to db admin
uncaught exception: SyntaxError: expected property name, got ':' :
@(shell):1:45
bye
Percona Server for MongoDB shell version v4.4.29-28
connecting to: mongodb://some-name-target-rs0-0.some-name-target-rs0.pitr-to-new-cluster-13977.svc.cluster.local:27017,some-name-target-rs0-2.some-name-target-rs0.pitr-to-new-cluster-13977.svc.cluster.local:27017,some-name-target-rs0-1.some-name-target-rs0.pitr-to-new-cluster-13977.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb&replicaSet=rs0&ssl=false
Implicit session: session { "id" : UUID("8f926499-362a-41b4-ab37-7e35c7d74109") }
Percona Server for MongoDB server version: v8.0.16-5
WARNING: shell and server versions do not match
switched to db admin
uncaught exception: SyntaxError: expected property name, got ':' :
@(shell):1:45
bye
Percona Server for MongoDB shell version v4.4.29-28
connecting to: mongodb://some-name-target-rs0-0.some-name-target-rs0.pitr-to-new-cluster-13977.svc.cluster.local:27017,some-name-target-rs0-2.some-name-target-rs0.pitr-to-new-cluster-13977.svc.cluster.local:27017,some-name-target-rs0-1.some-name-target-rs0.pitr-to-new-cluster-13977.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb&replicaSet=rs0&ssl=false
Implicit session: session { "id" : UUID("8f926499-362a-41b4-ab37-7e35c7d74109") }
Percona Server for MongoDB server version: v8.0.16-5
WARNING: shell and server versions do not match
switched to db admin
uncaught exception: SyntaxError: expected property name, got ':' :
@(shell):1:45
bye")
+ compare_mongo_cmd find myApp:myPass@some-name-target-rs0.pitr-to-new-cluster-13977 -2nd .svc.cluster.local myApp test 'sort( { x: 1 } )'
+ local command=find
+ local uri=myApp:myPass@some-name-target-rs0.pitr-to-new-cluster-13977
+ local postfix=-2nd
+ local suffix=.svc.cluster.local
+ local database=myApp
+ local collection=test
+ local 'sort=sort( { x: 1 } )'
+ local tls=false
+ local 'full_command=db.test.find()'
+ [[ -n sort( { x: 1 } ) ]]
+ full_command='db.test.find().sort( { x: 1 } )'
+ log 'running db.test.find().sort( { x: 1 } ) in myApp'
+ set +o xtrace
[2025-12-23T15:49:42+0000] running db.test.find().sort( { x: 1 } ) in myApp
+ [[ false == \t\r\u\e ]]
+ mongo_command=run_mongo
+ run_mongo 'use myApp\n db.test.find().sort( { x: 1 } )' myApp:myPass@some-name-target-rs0.pitr-to-new-cluster-13977 mongodb .svc.cluster.local
+ local 'command=use myApp\n db.test.find().sort( { x: 1 } )'
+ egrep -v 'I NETWORK|W NETWORK|F NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|Started a new thread for the timer service'
+ local uri=myApp:myPass@some-name-target-rs0.pitr-to-new-cluster-13977
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
egrep: warning: egrep is obsolescent; using grep -E
+++ mktemp
++ local LAST_OUT=/tmp/tmp.ZLuxIrEXsm
+++ mktemp
++ local LAST_ERR=/tmp/tmp.FfkCdfcVRq
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.ZLuxIrEXsm
++ cat /tmp/tmp.FfkCdfcVRq
++ rm /tmp/tmp.ZLuxIrEXsm /tmp/tmp.FfkCdfcVRq
++ return 0
+ local client_container=psmdb-client-696897d69b-9r6m5
+ local mongo_flag=
+ [[ myApp:myPass@some-name-target-rs0.pitr-to-new-cluster-13977 == *cfg* ]]
+ replica_set=rs0
+ kubectl_bin exec psmdb-client-696897d69b-9r6m5 -- bash -c 'printf '\''use myApp\n db.test.find().sort( { x: 1 } )\n'\'' | mongo mongodb://myApp:myPass@some-name-target-rs0.pitr-to-new-cluster-13977.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
++ mktemp
+ local LAST_OUT=/tmp/tmp.9tHrsivy4p
++ mktemp
+ local LAST_ERR=/tmp/tmp.drFAxqvDNq
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-696897d69b-9r6m5 -- bash -c 'printf '\''use myApp\n db.test.find().sort( { x: 1 } )\n'\'' | mongo mongodb://myApp:myPass@some-name-target-rs0.pitr-to-new-cluster-13977.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.9tHrsivy4p
+ cat /tmp/tmp.drFAxqvDNq
+ rm /tmp/tmp.9tHrsivy4p /tmp/tmp.drFAxqvDNq
+ return 0
+ [[ 0 -eq 0 ]]
+ diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2144/e2e-tests/pitr-to-new-cluster/compare/find-2nd.json /tmp/tmp.pwZ85mBPea/find-2nd
+ log 'Data is restored from backup-minio-logical: OK'
+ set +o xtrace
[2025-12-23T15:49:44+0000] Data is restored from backup-minio-logical: OK
+ desc 'Case 1: Logical restore: OK'
+ set +o xtrace


-----------------------------------------------------------------------------------
Case 1: Logical restore: OK
-----------------------------------------------------------------------------------

+ desc 'Case 2: Physical restore'
+ set +o xtrace


-----------------------------------------------------------------------------------
Case 2: Physical restore
-----------------------------------------------------------------------------------

+ test_case_2
+ run_restore some-name-target backup-minio physical
+ local cluster=some-name-target
+ local backup_name=backup-minio
+ local backup_type=physical
++ get_backup_dest backup-minio-physical
++ local backup_name=backup-minio-physical
++ kubectl_bin get psmdb-backup backup-minio-physical -o 'jsonpath={.status.destination}'
++ sed -e 's/.json$//'
+++ mktemp
++ sed 's|azure://||'
++ sed 's|s3://||'
++ sed 's|gs://||'
++ local LAST_OUT=/tmp/tmp.Msx0qXW49i
+++ mktemp
++ local LAST_ERR=/tmp/tmp.MH5zRwSDed
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb-backup backup-minio-physical -o 'jsonpath={.status.destination}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.Msx0qXW49i
++ cat /tmp/tmp.MH5zRwSDed
++ rm /tmp/tmp.Msx0qXW49i /tmp/tmp.MH5zRwSDed
++ return 0
+ local destination=operator-testing/some-name-source/2025-12-23T15:45:49Z
+ cat /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2144/e2e-tests/pitr-to-new-cluster/conf/restore-backupsource.yml
+ /usr/sbin/sed -e 's/name:/name: restore-backup-minio-physical/'
+ /usr/sbin/sed -e 's/clusterName:/clusterName: some-name-target/'
+ /usr/sbin/sed -e 's/type:/type: physical/'
+ /usr/sbin/sed -e s/pitrType:/type:/
+ /usr/sbin/sed -e 's|DESTINATION|operator-testing/some-name-source/2025-12-23T15:45:49Z|'
+ kubectl_bin apply -f -
++ mktemp
+ local LAST_OUT=/tmp/tmp.8GGfnwSD57
++ mktemp
+ local LAST_ERR=/tmp/tmp.voFCgz6ToV
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl apply -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.8GGfnwSD57
perconaservermongodbrestore.psmdb.percona.com/restore-backup-minio-physical created
+ cat /tmp/tmp.voFCgz6ToV
+ rm /tmp/tmp.8GGfnwSD57 /tmp/tmp.voFCgz6ToV
+ return 0
+ wait_restore backup-minio-physical some-name-target requested 0
+ local backup_name=backup-minio-physical
+ local cluster_name=some-name-target
+ local target_state=requested
+ local wait_cluster_consistency=0
+ local wait_time=1780
+ local ok_if_ready=0
+ set +o xtrace
Waiting for the psmdb-restore/restore-backup-minio-physical object to be created.OK
Waiting psmdb-restore/restore-backup-minio-physical to reach state "requested" ..OK after 1 minutes
+ [[ 0 -eq 1 ]]
+ compare_pbm_config some-name-target source mongod
+ local cluster=some-name-target
+ local compare_suffix=source
+ local container=mongod
+ local pbm_binary=pbm
+ [[ mongod == \m\o\n\g\o\d ]]
+ pbm_binary=/opt/percona/pbm
+ kubectl_bin exec some-name-target-rs0-0 -c mongod -- /opt/percona/pbm config
++ mktemp
+ local LAST_OUT=/tmp/tmp.vuMfaM8ozk
++ mktemp
+ local LAST_ERR=/tmp/tmp.8AY4fRYTw4
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec some-name-target-rs0-0 -c mongod -- /opt/percona/pbm config
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.vuMfaM8ozk
+ cat /tmp/tmp.8AY4fRYTw4
+ rm /tmp/tmp.vuMfaM8ozk /tmp/tmp.8AY4fRYTw4
+ return 0
+ diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2144/e2e-tests/pitr-to-new-cluster/compare/pbm_config_source.yml /tmp/tmp.pwZ85mBPea/pbm_config.yml
+ log 'Backup source is set as main storage during restore: OK'
+ set +o xtrace
[2025-12-23T15:51:48+0000] Backup source is set as main storage during restore: OK
++ date +%s
+ local now_ts=1766505108
+ wait_restore backup-minio-physical some-name-target ready
+ local backup_name=backup-minio-physical
+ local cluster_name=some-name-target
+ local target_state=ready
+ local wait_cluster_consistency=1
+ local wait_time=1780
+ local ok_if_ready=0
+ set +o xtrace
Waiting for the psmdb-restore/restore-backup-minio-physical object to be created.OK
Waiting psmdb-restore/restore-backup-minio-physical to reach state "ready" ..OK after 1 minutes
+ [[ 1 -eq 1 ]]
+ wait_cluster_consistency some-name-target
+ local cluster_name=some-name-target
+ local wait_time=32
+ retry=0
+ sleep 7
+ echo -n 'waiting for cluster readyness'
waiting for cluster readyness++ kubectl_bin get psmdb some-name-target -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.hSQeQFEk9h
+++ mktemp
++ local LAST_ERR=/tmp/tmp.qqxYtdAMCh
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name-target -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.hSQeQFEk9h
++ cat /tmp/tmp.qqxYtdAMCh
++ rm /tmp/tmp.hSQeQFEk9h /tmp/tmp.qqxYtdAMCh
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 1 -ge 32 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name-target -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.crZxA1ZdNr
+++ mktemp
++ local LAST_ERR=/tmp/tmp.y7MgQzgrte
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name-target -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.crZxA1ZdNr
++ cat /tmp/tmp.y7MgQzgrte
++ rm /tmp/tmp.crZxA1ZdNr /tmp/tmp.y7MgQzgrte
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 2 -ge 32 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name-target -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.NINSViqZpf
+++ mktemp
++ local LAST_ERR=/tmp/tmp.fngSY02PtM
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name-target -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.NINSViqZpf
++ cat /tmp/tmp.fngSY02PtM
++ rm /tmp/tmp.NINSViqZpf /tmp/tmp.fngSY02PtM
++ return 0
+ [[ error == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 3 -ge 32 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name-target -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.2U5tgYGobf
+++ mktemp
++ local LAST_ERR=/tmp/tmp.H87sKTDMLY
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name-target -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.2U5tgYGobf
++ cat /tmp/tmp.H87sKTDMLY
++ rm /tmp/tmp.2U5tgYGobf /tmp/tmp.H87sKTDMLY
++ return 0
+ [[ error == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 4 -ge 32 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name-target -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.8Ka0Uo6Eg8
+++ mktemp
++ local LAST_ERR=/tmp/tmp.zZ1QuSiiz4
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name-target -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.8Ka0Uo6Eg8
++ cat /tmp/tmp.zZ1QuSiiz4
++ rm /tmp/tmp.8Ka0Uo6Eg8 /tmp/tmp.zZ1QuSiiz4
++ return 0
+ [[ error == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 5 -ge 32 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name-target -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.AMTiLay5bD
+++ mktemp
++ local LAST_ERR=/tmp/tmp.Da69TSUHCE
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name-target -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.AMTiLay5bD
++ cat /tmp/tmp.Da69TSUHCE
++ rm /tmp/tmp.AMTiLay5bD /tmp/tmp.Da69TSUHCE
++ return 0
+ [[ error == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 6 -ge 32 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name-target -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.zBPLadicNf
+++ mktemp
++ local LAST_ERR=/tmp/tmp.uxd37HLqRx
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name-target -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.zBPLadicNf
++ cat /tmp/tmp.uxd37HLqRx
++ rm /tmp/tmp.zBPLadicNf /tmp/tmp.uxd37HLqRx
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 7 -ge 32 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name-target -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.Vrr37BJPfr
+++ mktemp
++ local LAST_ERR=/tmp/tmp.uim3rmnUUg
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name-target -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.Vrr37BJPfr
++ cat /tmp/tmp.uim3rmnUUg
++ rm /tmp/tmp.Vrr37BJPfr /tmp/tmp.uim3rmnUUg
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 8 -ge 32 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name-target -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.lYJSHYfJbZ
+++ mktemp
++ local LAST_ERR=/tmp/tmp.nBAA1y8wqk
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name-target -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.lYJSHYfJbZ
++ cat /tmp/tmp.nBAA1y8wqk
++ rm /tmp/tmp.lYJSHYfJbZ /tmp/tmp.nBAA1y8wqk
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 9 -ge 32 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name-target -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.2FuAGOShpc
+++ mktemp
++ local LAST_ERR=/tmp/tmp.2WvBNBvp7U
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name-target -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.2FuAGOShpc
++ cat /tmp/tmp.2WvBNBvp7U
++ rm /tmp/tmp.2FuAGOShpc /tmp/tmp.2WvBNBvp7U
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 10 -ge 32 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name-target -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.swnXhieICk
+++ mktemp
++ local LAST_ERR=/tmp/tmp.KssnWiYF4c
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name-target -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.swnXhieICk
++ cat /tmp/tmp.KssnWiYF4c
++ rm /tmp/tmp.swnXhieICk /tmp/tmp.KssnWiYF4c
++ return 0
+ [[ ready == \r\e\a\d\y ]]
+ echo .OK
.OK
+ compare_pbm_config some-name-target target
+ local cluster=some-name-target
+ local compare_suffix=target
+ local container=backup-agent
+ local pbm_binary=pbm
+ [[ backup-agent == \m\o\n\g\o\d ]]
+ kubectl_bin exec some-name-target-rs0-0 -c backup-agent -- pbm config
++ mktemp
+ local LAST_OUT=/tmp/tmp.MmHWVbLKov
++ mktemp
+ local LAST_ERR=/tmp/tmp.RczVa1OJPf
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec some-name-target-rs0-0 -c backup-agent -- pbm config
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.MmHWVbLKov
+ cat /tmp/tmp.RczVa1OJPf
+ rm /tmp/tmp.MmHWVbLKov /tmp/tmp.RczVa1OJPf
+ return 0
+ diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2144/e2e-tests/pitr-to-new-cluster/compare/pbm_config_target.yml /tmp/tmp.pwZ85mBPea/pbm_config.yml
+ log 'Main storage is reverted after backup-minio-physical: OK'
+ set +o xtrace
[2025-12-23T15:55:29+0000] Main storage is reverted after backup-minio-physical: OK
+ wait_for_pbm_operations some-name-target
+ local cluster=some-name-target
+ set +o xtrace
waiting for PBM operation to finish.OK
+ check_if_resync_triggered some-name-target 1766505108
+ local cluster=some-name-target
+ local ts=1766505108
+ log 'checking if resync is triggered after 1766505108'
+ set +o xtrace
[2025-12-23T15:55:30+0000] checking if resync is triggered after 1766505108
++ run_mongo 'use admin\n db.pbmCmd.countDocuments({cmd: '\''resync'\'', ts: { '\''$gt'\'': 1766505108 }})' backup:backup123456@some-name-target-rs0.pitr-to-new-cluster-13977
++ local 'command=use admin\n db.pbmCmd.countDocuments({cmd: '\''resync'\'', ts: { '\''$gt'\'': 1766505108 }})'
++ local uri=backup:backup123456@some-name-target-rs0.pitr-to-new-cluster-13977
++ local driver=mongodb+srv
++ local suffix=.svc.cluster.local
+++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.JcSqtnmOBY
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.kIPRSyv0Qh
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in $(seq 0 2)
+++ set +e
+++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.JcSqtnmOBY
+++ cat /tmp/tmp.kIPRSyv0Qh
+++ rm /tmp/tmp.JcSqtnmOBY /tmp/tmp.kIPRSyv0Qh
+++ return 0
++ local client_container=psmdb-client-696897d69b-9r6m5
++ local mongo_flag=
++ [[ backup:backup123456@some-name-target-rs0.pitr-to-new-cluster-13977 == *cfg* ]]
++ replica_set=rs0
++ kubectl_bin exec psmdb-client-696897d69b-9r6m5 -- bash -c 'printf '\''use admin\n db.pbmCmd.countDocuments({cmd: '\''resync'\'', ts: { '\''$gt'\'': 1766505108 }})\n'\'' | mongo mongodb+srv://backup:backup123456@some-name-target-rs0.pitr-to-new-cluster-13977.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
+++ mktemp
++ local LAST_OUT=/tmp/tmp.WfHSw34cAL
+++ mktemp
++ local LAST_ERR=/tmp/tmp.bhD4A65mAP
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl exec psmdb-client-696897d69b-9r6m5 -- bash -c 'printf '\''use admin\n db.pbmCmd.countDocuments({cmd: '\''resync'\'', ts: { '\''$gt'\'': 1766505108 }})\n'\'' | mongo mongodb+srv://backup:backup123456@some-name-target-rs0.pitr-to-new-cluster-13977.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
++ exit_status=1
++ set -e
++ '[' 1 '!=' 0 -a -n 1 ']'
++ cat /tmp/tmp.WfHSw34cAL
++ cat /tmp/tmp.bhD4A65mAP
command terminated with exit code 1
++ sleep 0
++ for i in $(seq 0 2)
++ set +e
++ kubectl exec psmdb-client-696897d69b-9r6m5 -- bash -c 'printf '\''use admin\n db.pbmCmd.countDocuments({cmd: '\''resync'\'', ts: { '\''$gt'\'': 1766505108 }})\n'\'' | mongo mongodb+srv://backup:backup123456@some-name-target-rs0.pitr-to-new-cluster-13977.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
++ exit_status=1
++ set -e
++ '[' 1 '!=' 0 -a -n 1 ']'
++ cat /tmp/tmp.WfHSw34cAL
++ cat /tmp/tmp.bhD4A65mAP
command terminated with exit code 1
++ sleep 4
++ for i in $(seq 0 2)
++ set +e
++ kubectl exec psmdb-client-696897d69b-9r6m5 -- bash -c 'printf '\''use admin\n db.pbmCmd.countDocuments({cmd: '\''resync'\'', ts: { '\''$gt'\'': 1766505108 }})\n'\'' | mongo mongodb+srv://backup:backup123456@some-name-target-rs0.pitr-to-new-cluster-13977.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
++ exit_status=1
++ set -e
++ '[' 1 '!=' 0 -a -n 1 ']'
++ cat /tmp/tmp.WfHSw34cAL
++ cat /tmp/tmp.bhD4A65mAP
command terminated with exit code 1
++ sleep 8
++ cat /tmp/tmp.WfHSw34cAL
++ cat /tmp/tmp.bhD4A65mAP
command terminated with exit code 1
++ rm /tmp/tmp.WfHSw34cAL /tmp/tmp.bhD4A65mAP
++ return 1
+ local 'count=Percona Server for MongoDB shell version v4.4.29-28
connecting to: mongodb://some-name-target-rs0-0.some-name-target-rs0.pitr-to-new-cluster-13977.svc.cluster.local:27017,some-name-target-rs0-2.some-name-target-rs0.pitr-to-new-cluster-13977.svc.cluster.local:27017,some-name-target-rs0-1.some-name-target-rs0.pitr-to-new-cluster-13977.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb&replicaSet=rs0&ssl=false
Implicit session: session { "id" : UUID("957b2da1-2c41-49c8-b015-f1ed6490abf9") }
Percona Server for MongoDB server version: v8.0.16-5
WARNING: shell and server versions do not match
switched to db admin
uncaught exception: SyntaxError: expected property name, got '\'':'\'' :
@(shell):1:45
bye
Percona Server for MongoDB shell version v4.4.29-28
connecting to: mongodb://some-name-target-rs0-2.some-name-target-rs0.pitr-to-new-cluster-13977.svc.cluster.local:27017,some-name-target-rs0-1.some-name-target-rs0.pitr-to-new-cluster-13977.svc.cluster.local:27017,some-name-target-rs0-0.some-name-target-rs0.pitr-to-new-cluster-13977.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb&replicaSet=rs0&ssl=false
Implicit session: session { "id" : UUID("c6b92e16-0633-4376-9f81-0cf4b72a7b1e") }
Percona Server for MongoDB server version: v8.0.16-5
WARNING: shell and server versions do not match
switched to db admin
uncaught exception: SyntaxError: expected property name, got '\'':'\'' :
@(shell):1:45
bye
Percona Server for MongoDB shell version v4.4.29-28
connecting to: mongodb://some-name-target-rs0-0.some-name-target-rs0.pitr-to-new-cluster-13977.svc.cluster.local:27017,some-name-target-rs0-2.some-name-target-rs0.pitr-to-new-cluster-13977.svc.cluster.local:27017,some-name-target-rs0-1.some-name-target-rs0.pitr-to-new-cluster-13977.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb&replicaSet=rs0&ssl=false
Implicit session: session { "id" : UUID("b8d94521-36bd-4c5b-bb1d-2a89bae198e3") }
Percona Server for MongoDB server version: v8.0.16-5
WARNING: shell and server versions do not match
switched to db admin
uncaught exception: SyntaxError: expected property name, got '\'':'\'' :
@(shell):1:45
bye
Percona Server for MongoDB shell version v4.4.29-28
connecting to: mongodb://some-name-target-rs0-0.some-name-target-rs0.pitr-to-new-cluster-13977.svc.cluster.local:27017,some-name-target-rs0-2.some-name-target-rs0.pitr-to-new-cluster-13977.svc.cluster.local:27017,some-name-target-rs0-1.some-name-target-rs0.pitr-to-new-cluster-13977.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb&replicaSet=rs0&ssl=false
Implicit session: session { "id" : UUID("b8d94521-36bd-4c5b-bb1d-2a89bae198e3") }
Percona Server for MongoDB server version: v8.0.16-5
WARNING: shell and server versions do not match
switched to db admin
uncaught exception: SyntaxError: expected property name, got '\'':'\'' :
@(shell):1:45
bye'
+ [[ Percona Server for MongoDB shell version v4.4.29-28
connecting to: mongodb://some-name-target-rs0-0.some-name-target-rs0.pitr-to-new-cluster-13977.svc.cluster.local:27017,some-name-target-rs0-2.some-name-target-rs0.pitr-to-new-cluster-13977.svc.cluster.local:27017,some-name-target-rs0-1.some-name-target-rs0.pitr-to-new-cluster-13977.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb&replicaSet=rs0&ssl=false
Implicit session: session { "id" : UUID("957b2da1-2c41-49c8-b015-f1ed6490abf9") }
Percona Server for MongoDB server version: v8.0.16-5
WARNING: shell and server versions do not match
switched to db admin
uncaught exception: SyntaxError: expected property name, got ':' :
@(shell):1:45
bye
Percona Server for MongoDB shell version v4.4.29-28
connecting to: mongodb://some-name-target-rs0-2.some-name-target-rs0.pitr-to-new-cluster-13977.svc.cluster.local:27017,some-name-target-rs0-1.some-name-target-rs0.pitr-to-new-cluster-13977.svc.cluster.local:27017,some-name-target-rs0-0.some-name-target-rs0.pitr-to-new-cluster-13977.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb&replicaSet=rs0&ssl=false
Implicit session: session { "id" : UUID("c6b92e16-0633-4376-9f81-0cf4b72a7b1e") }
Percona Server for MongoDB server version: v8.0.16-5
WARNING: shell and server versions do not match
switched to db admin
uncaught exception: SyntaxError: expected property name, got ':' :
@(shell):1:45
bye
Percona Server for MongoDB shell version v4.4.29-28
connecting to: mongodb://some-name-target-rs0-0.some-name-target-rs0.pitr-to-new-cluster-13977.svc.cluster.local:27017,some-name-target-rs0-2.some-name-target-rs0.pitr-to-new-cluster-13977.svc.cluster.local:27017,some-name-target-rs0-1.some-name-target-rs0.pitr-to-new-cluster-13977.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb&replicaSet=rs0&ssl=false
Implicit session: session { "id" : UUID("b8d94521-36bd-4c5b-bb1d-2a89bae198e3") }
Percona Server for MongoDB server version: v8.0.16-5
WARNING: shell and server versions do not match
switched to db admin
uncaught exception: SyntaxError: expected property name, got ':' :
@(shell):1:45
bye
Percona Server for MongoDB shell version v4.4.29-28
connecting to: mongodb://some-name-target-rs0-0.some-name-target-rs0.pitr-to-new-cluster-13977.svc.cluster.local:27017,some-name-target-rs0-2.some-name-target-rs0.pitr-to-new-cluster-13977.svc.cluster.local:27017,some-name-target-rs0-1.some-name-target-rs0.pitr-to-new-cluster-13977.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb&replicaSet=rs0&ssl=false
Implicit session: session { "id" : UUID("b8d94521-36bd-4c5b-bb1d-2a89bae198e3") }
Percona Server for MongoDB server version: v8.0.16-5
WARNING: shell and server versions do not match
switched to db admin
uncaught exception: SyntaxError: expected property name, got ':' :
@(shell):1:45
bye -lt 1 ]]
./e2e-tests/pitr-to-new-cluster/run: line 91: [[: Percona Server for MongoDB shell version v4.4.29-28
connecting to: mongodb://some-name-target-rs0-0.some-name-target-rs0.pitr-to-new-cluster-13977.svc.cluster.local:27017,some-name-target-rs0-2.some-name-target-rs0.pitr-to-new-cluster-13977.svc.cluster.local:27017,some-name-target-rs0-1.some-name-target-rs0.pitr-to-new-cluster-13977.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb&replicaSet=rs0&ssl=false
Implicit session: session { "id" : UUID("957b2da1-2c41-49c8-b015-f1ed6490abf9") }
Percona Server for MongoDB server version: v8.0.16-5
WARNING: shell and server versions do not match
switched to db admin
uncaught exception: SyntaxError: expected property name, got ':' :
@(shell):1:45
bye
Percona Server for MongoDB shell version v4.4.29-28
connecting to: mongodb://some-name-target-rs0-2.some-name-target-rs0.pitr-to-new-cluster-13977.svc.cluster.local:27017,some-name-target-rs0-1.some-name-target-rs0.pitr-to-new-cluster-13977.svc.cluster.local:27017,some-name-target-rs0-0.some-name-target-rs0.pitr-to-new-cluster-13977.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb&replicaSet=rs0&ssl=false
Implicit session: session { "id" : UUID("c6b92e16-0633-4376-9f81-0cf4b72a7b1e") }
Percona Server for MongoDB server version: v8.0.16-5
WARNING: shell and server versions do not match
switched to db admin
uncaught exception: SyntaxError: expected property name, got ':' :
@(shell):1:45
bye
Percona Server for MongoDB shell version v4.4.29-28
connecting to: mongodb://some-name-target-rs0-0.some-name-target-rs0.pitr-to-new-cluster-13977.svc.cluster.local:27017,some-name-target-rs0-2.some-name-target-rs0.pitr-to-new-cluster-13977.svc.cluster.local:27017,some-name-target-rs0-1.some-name-target-rs0.pitr-to-new-cluster-13977.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb&replicaSet=rs0&ssl=false
Implicit session: session { "id" : UUID("b8d94521-36bd-4c5b-bb1d-2a89bae198e3") }
Percona Server for MongoDB server version: v8.0.16-5
WARNING: shell and server versions do not match
switched to db admin
uncaught exception: SyntaxError: expected property name, got ':' :
@(shell):1:45
bye
Percona Server for MongoDB shell version v4.4.29-28
connecting to: mongodb://some-name-target-rs0-0.some-name-target-rs0.pitr-to-new-cluster-13977.svc.cluster.local:27017,some-name-target-rs0-2.some-name-target-rs0.pitr-to-new-cluster-13977.svc.cluster.local:27017,some-name-target-rs0-1.some-name-target-rs0.pitr-to-new-cluster-13977.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb&replicaSet=rs0&ssl=false
Implicit session: session { "id" : UUID("b8d94521-36bd-4c5b-bb1d-2a89bae198e3") }
Percona Server for MongoDB server version: v8.0.16-5
WARNING: shell and server versions do not match
switched to db admin
uncaught exception: SyntaxError: expected property name, got ':' :
@(shell):1:45
bye: syntax error: invalid arithmetic operator (error token is ".4.29-28
connecting to: mongodb://some-name-target-rs0-0.some-name-target-rs0.pitr-to-new-cluster-13977.svc.cluster.local:27017,some-name-target-rs0-2.some-name-target-rs0.pitr-to-new-cluster-13977.svc.cluster.local:27017,some-name-target-rs0-1.some-name-target-rs0.pitr-to-new-cluster-13977.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb&replicaSet=rs0&ssl=false
Implicit session: session { "id" : UUID("957b2da1-2c41-49c8-b015-f1ed6490abf9") }
Percona Server for MongoDB server version: v8.0.16-5
WARNING: shell and server versions do not match
switched to db admin
uncaught exception: SyntaxError: expected property name, got ':' :
@(shell):1:45
bye
Percona Server for MongoDB shell version v4.4.29-28
connecting to: mongodb://some-name-target-rs0-2.some-name-target-rs0.pitr-to-new-cluster-13977.svc.cluster.local:27017,some-name-target-rs0-1.some-name-target-rs0.pitr-to-new-cluster-13977.svc.cluster.local:27017,some-name-target-rs0-0.some-name-target-rs0.pitr-to-new-cluster-13977.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb&replicaSet=rs0&ssl=false
Implicit session: session { "id" : UUID("c6b92e16-0633-4376-9f81-0cf4b72a7b1e") }
Percona Server for MongoDB server version: v8.0.16-5
WARNING: shell and server versions do not match
switched to db admin
uncaught exception: SyntaxError: expected property name, got ':' :
@(shell):1:45
bye
Percona Server for MongoDB shell version v4.4.29-28
connecting to: mongodb://some-name-target-rs0-0.some-name-target-rs0.pitr-to-new-cluster-13977.svc.cluster.local:27017,some-name-target-rs0-2.some-name-target-rs0.pitr-to-new-cluster-13977.svc.cluster.local:27017,some-name-target-rs0-1.some-name-target-rs0.pitr-to-new-cluster-13977.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb&replicaSet=rs0&ssl=false
Implicit session: session { "id" : UUID("b8d94521-36bd-4c5b-bb1d-2a89bae198e3") }
Percona Server for MongoDB server version: v8.0.16-5
WARNING: shell and server versions do not match
switched to db admin
uncaught exception: SyntaxError: expected property name, got ':' :
@(shell):1:45
bye
Percona Server for MongoDB shell version v4.4.29-28
connecting to: mongodb://some-name-target-rs0-0.some-name-target-rs0.pitr-to-new-cluster-13977.svc.cluster.local:27017,some-name-target-rs0-2.some-name-target-rs0.pitr-to-new-cluster-13977.svc.cluster.local:27017,some-name-target-rs0-1.some-name-target-rs0.pitr-to-new-cluster-13977.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb&replicaSet=rs0&ssl=false
Implicit session: session { "id" : UUID("b8d94521-36bd-4c5b-bb1d-2a89bae198e3") }
Percona Server for MongoDB server version: v8.0.16-5
WARNING: shell and server versions do not match
switched to db admin
uncaught exception: SyntaxError: expected property name, got ':' :
@(shell):1:45
bye")
+ compare_mongo_cmd find myApp:myPass@some-name-target-rs0.pitr-to-new-cluster-13977 -2nd .svc.cluster.local myApp test 'sort( { x: 1 } )'
+ local command=find
+ local uri=myApp:myPass@some-name-target-rs0.pitr-to-new-cluster-13977
+ local postfix=-2nd
+ local suffix=.svc.cluster.local
+ local database=myApp
+ local collection=test
+ local 'sort=sort( { x: 1 } )'
+ local tls=false
+ local 'full_command=db.test.find()'
+ [[ -n sort( { x: 1 } ) ]]
+ full_command='db.test.find().sort( { x: 1 } )'
+ log 'running db.test.find().sort( { x: 1 } ) in myApp'
+ set +o xtrace
[2025-12-23T15:55:48+0000] running db.test.find().sort( { x: 1 } ) in myApp
+ [[ false == \t\r\u\e ]]
+ mongo_command=run_mongo
+ run_mongo 'use myApp\n db.test.find().sort( { x: 1 } )' myApp:myPass@some-name-target-rs0.pitr-to-new-cluster-13977 mongodb .svc.cluster.local
+ local 'command=use myApp\n db.test.find().sort( { x: 1 } )'
+ local uri=myApp:myPass@some-name-target-rs0.pitr-to-new-cluster-13977
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ egrep -v 'I NETWORK|W NETWORK|F NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|Started a new thread for the timer service'
+ /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.dFad2yClmq
egrep: warning: egrep is obsolescent; using grep -E
+++ mktemp
++ local LAST_ERR=/tmp/tmp.ES5qK373SC
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.dFad2yClmq
++ cat /tmp/tmp.ES5qK373SC
++ rm /tmp/tmp.dFad2yClmq /tmp/tmp.ES5qK373SC
++ return 0
+ local client_container=psmdb-client-696897d69b-9r6m5
+ local mongo_flag=
+ [[ myApp:myPass@some-name-target-rs0.pitr-to-new-cluster-13977 == *cfg* ]]
+ replica_set=rs0
+ kubectl_bin exec psmdb-client-696897d69b-9r6m5 -- bash -c 'printf '\''use myApp\n db.test.find().sort( { x: 1 } )\n'\'' | mongo mongodb://myApp:myPass@some-name-target-rs0.pitr-to-new-cluster-13977.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
++ mktemp
+ local LAST_OUT=/tmp/tmp.h7uVQ7qz63
++ mktemp
+ local LAST_ERR=/tmp/tmp.etw1ZxgPmD
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-696897d69b-9r6m5 -- bash -c 'printf '\''use myApp\n db.test.find().sort( { x: 1 } )\n'\'' | mongo mongodb://myApp:myPass@some-name-target-rs0.pitr-to-new-cluster-13977.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.h7uVQ7qz63
+ cat /tmp/tmp.etw1ZxgPmD
+ rm /tmp/tmp.h7uVQ7qz63 /tmp/tmp.etw1ZxgPmD
+ return 0
+ [[ 0 -eq 0 ]]
+ diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2144/e2e-tests/pitr-to-new-cluster/compare/find-2nd.json /tmp/tmp.pwZ85mBPea/find-2nd
+ log 'Data is restored from backup-minio-physical: OK'
+ set +o xtrace
[2025-12-23T15:55:51+0000] Data is restored from backup-minio-physical: OK
+ desc 'Case 2: Physical restore: OK'
+ set +o xtrace


-----------------------------------------------------------------------------------
Case 2: Physical restore: OK
-----------------------------------------------------------------------------------

+ desc 'Case 3: Backup and restore on some-name-target'
+ set +o xtrace


-----------------------------------------------------------------------------------
Case 3: Backup and restore on some-name-target
-----------------------------------------------------------------------------------

+ test_case_3
+ local backup_name=target-backup-minio
+ write_document some-name-target 100504
+ local cluster=some-name-target
+ local x=100504
+ log 'write document: 100504'
+ set +o xtrace
[2025-12-23T15:55:51+0000] write document: 100504
+ run_mongo 'use myApp\n db.test.insert({ x: 100504 })' myApp:myPass@some-name-target-rs0.pitr-to-new-cluster-13977 mongodb
+ local 'command=use myApp\n db.test.insert({ x: 100504 })'
+ local uri=myApp:myPass@some-name-target-rs0.pitr-to-new-cluster-13977
+ local driver=mongodb
+ local suffix=.svc.cluster.local
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.aqEO2bVThL
+++ mktemp
++ local LAST_ERR=/tmp/tmp.UPYUSka4tE
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.aqEO2bVThL
++ cat /tmp/tmp.UPYUSka4tE
++ rm /tmp/tmp.aqEO2bVThL /tmp/tmp.UPYUSka4tE
++ return 0
+ local client_container=psmdb-client-696897d69b-9r6m5
+ local mongo_flag=
+ [[ myApp:myPass@some-name-target-rs0.pitr-to-new-cluster-13977 == *cfg* ]]
+ replica_set=rs0
+ kubectl_bin exec psmdb-client-696897d69b-9r6m5 -- bash -c 'printf '\''use myApp\n db.test.insert({ x: 100504 })\n'\'' | mongo mongodb://myApp:myPass@some-name-target-rs0.pitr-to-new-cluster-13977.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
++ mktemp
+ local LAST_OUT=/tmp/tmp.Lc9Av0SV5G
++ mktemp
+ local LAST_ERR=/tmp/tmp.JwhHtyukvr
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-696897d69b-9r6m5 -- bash -c 'printf '\''use myApp\n db.test.insert({ x: 100504 })\n'\'' | mongo mongodb://myApp:myPass@some-name-target-rs0.pitr-to-new-cluster-13977.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.Lc9Av0SV5G
Percona Server for MongoDB shell version v4.4.29-28
connecting to: mongodb://some-name-target-rs0.pitr-to-new-cluster-13977.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb&replicaSet=rs0&ssl=false
Implicit session: session { "id" : UUID("61c2025a-dba0-4a36-a70e-acbe8d0787de") }
Percona Server for MongoDB server version: v8.0.16-5
WARNING: shell and server versions do not match
switched to db myApp
WriteResult({ "nInserted" : 1 })
bye
+ cat /tmp/tmp.JwhHtyukvr
+ rm /tmp/tmp.Lc9Av0SV5G /tmp/tmp.JwhHtyukvr
+ return 0
+ run_backup some-name-target target-backup-minio logical
+ local cluster=some-name-target
+ local name=target-backup-minio
+ local type=logical
+ desc 'run backup target-backup-minio-logical'
+ set +o xtrace


-----------------------------------------------------------------------------------
run backup target-backup-minio-logical
-----------------------------------------------------------------------------------

+ cat /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2144/e2e-tests/pitr-to-new-cluster/conf/backup.yml
+ /usr/sbin/sed -e 's/name:/name: target-backup-minio-logical/'
+ /usr/sbin/sed -e 's/clusterName:/clusterName: some-name-target/'
+ /usr/sbin/sed -e 's/type:/type: logical/'
+ kubectl_bin apply -f -
++ mktemp
+ local LAST_OUT=/tmp/tmp.e4Wv1SgE9C
++ mktemp
+ local LAST_ERR=/tmp/tmp.IDcPvXRhUO
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl apply -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.e4Wv1SgE9C
perconaservermongodbbackup.psmdb.percona.com/target-backup-minio-logical created
+ cat /tmp/tmp.IDcPvXRhUO
+ rm /tmp/tmp.e4Wv1SgE9C /tmp/tmp.IDcPvXRhUO
+ return 0
+ wait_backup target-backup-minio-logical
+ local backup_name=target-backup-minio-logical
+ local target_state=ready
+ set +o xtrace
waiting for target-backup-minio-logical to reach ready state.......OK
+ log 'checking target-backup-minio-logical in storage'
+ set +o xtrace
[2025-12-23T15:56:08+0000] checking target-backup-minio-logical in storage
+ check_backup_in_storage target-backup-minio-logical minio rs0 myApp.test.gz
+ local backup=target-backup-minio-logical
+ local storage_type=minio
+ local replset=rs0
+ local file=myApp.test.gz
+ local endpoint
++ get_backup_dest target-backup-minio-logical
++ local backup_name=target-backup-minio-logical
++ /usr/sbin/sed 's|https://engk8soperators.blob.core.windows.net/||'
++ kubectl_bin get psmdb-backup target-backup-minio-logical -o 'jsonpath={.status.destination}'
++ sed -e 's/.json$//'
++ sed 's|s3://||'
+++ mktemp
++ sed 's|azure://||'
++ sed 's|gs://||'
++ local LAST_OUT=/tmp/tmp.0Uz1xjXniW
+++ mktemp
++ local LAST_ERR=/tmp/tmp.ZhGrNKRqmI
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb-backup target-backup-minio-logical -o 'jsonpath={.status.destination}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.0Uz1xjXniW
++ cat /tmp/tmp.ZhGrNKRqmI
++ rm /tmp/tmp.0Uz1xjXniW /tmp/tmp.ZhGrNKRqmI
++ return 0
+ backup_dest=operator-testing/some-name-target/2025-12-23T15:55:56Z
+ case ${storage_type} in
+ endpoint=minio-service
+ kubectl_bin run -i --rm aws-cli --image=perconalab/awscli --restart=Never -- /usr/bin/env AWS_ACCESS_KEY_ID=some-access-key AWS_SECRET_ACCESS_KEY=some-secret-key AWS_DEFAULT_REGION=us-east-1 /usr/bin/aws --endpoint-url http://minio-service:9000 s3 ls s3://operator-testing/some-name-target/2025-12-23T15:55:56Z/rs0/myApp.test.gz
+ grep myApp.test.gz
++ mktemp
+ local LAST_OUT=/tmp/tmp.jK99A3hsYw
++ mktemp
+ local LAST_ERR=/tmp/tmp.6KJwf1X4ih
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl run -i --rm aws-cli --image=perconalab/awscli --restart=Never -- /usr/bin/env AWS_ACCESS_KEY_ID=some-access-key AWS_SECRET_ACCESS_KEY=some-secret-key AWS_DEFAULT_REGION=us-east-1 /usr/bin/aws --endpoint-url http://minio-service:9000 s3 ls s3://operator-testing/some-name-target/2025-12-23T15:55:56Z/rs0/myApp.test.gz
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.jK99A3hsYw
+ cat /tmp/tmp.6KJwf1X4ih
+ rm /tmp/tmp.jK99A3hsYw /tmp/tmp.6KJwf1X4ih
+ return 0
2025-12-23 15:55:59        118 myApp.test.gz
+ log 'target-backup-minio is uploaded to correct storage: OK'
+ set +o xtrace
[2025-12-23T15:56:13+0000] target-backup-minio is uploaded to correct storage: OK
+ log 'enabling PiTR on some-name-target'
+ set +o xtrace
[2025-12-23T15:56:13+0000] enabling PiTR on some-name-target
+ kubectl_bin patch psmdb some-name-target --type=merge --patch '{"spec": {"backup": {"pitr": {"enabled": true}}}}'
++ mktemp
+ local LAST_OUT=/tmp/tmp.nyLV2gUgfe
++ mktemp
+ local LAST_ERR=/tmp/tmp.aZNGkwOVru
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl patch psmdb some-name-target --type=merge --patch '{"spec": {"backup": {"pitr": {"enabled": true}}}}'
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.nyLV2gUgfe
perconaservermongodb.psmdb.percona.com/some-name-target patched
+ cat /tmp/tmp.aZNGkwOVru
+ rm /tmp/tmp.nyLV2gUgfe /tmp/tmp.aZNGkwOVru
+ return 0
+ write_document some-name-target 100505
+ local cluster=some-name-target
+ local x=100505
+ log 'write document: 100505'
+ set +o xtrace
[2025-12-23T15:56:14+0000] write document: 100505
+ run_mongo 'use myApp\n db.test.insert({ x: 100505 })' myApp:myPass@some-name-target-rs0.pitr-to-new-cluster-13977 mongodb
+ local 'command=use myApp\n db.test.insert({ x: 100505 })'
+ local uri=myApp:myPass@some-name-target-rs0.pitr-to-new-cluster-13977
+ local driver=mongodb
+ local suffix=.svc.cluster.local
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.vclZDv0TvC
+++ mktemp
++ local LAST_ERR=/tmp/tmp.PoRtQDejQk
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.vclZDv0TvC
++ cat /tmp/tmp.PoRtQDejQk
++ rm /tmp/tmp.vclZDv0TvC /tmp/tmp.PoRtQDejQk
++ return 0
+ local client_container=psmdb-client-696897d69b-9r6m5
+ local mongo_flag=
+ [[ myApp:myPass@some-name-target-rs0.pitr-to-new-cluster-13977 == *cfg* ]]
+ replica_set=rs0
+ kubectl_bin exec psmdb-client-696897d69b-9r6m5 -- bash -c 'printf '\''use myApp\n db.test.insert({ x: 100505 })\n'\'' | mongo mongodb://myApp:myPass@some-name-target-rs0.pitr-to-new-cluster-13977.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
++ mktemp
+ local LAST_OUT=/tmp/tmp.Kxv46aJejw
++ mktemp
+ local LAST_ERR=/tmp/tmp.vFAoi1FpMB
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-696897d69b-9r6m5 -- bash -c 'printf '\''use myApp\n db.test.insert({ x: 100505 })\n'\'' | mongo mongodb://myApp:myPass@some-name-target-rs0.pitr-to-new-cluster-13977.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.Kxv46aJejw
Percona Server for MongoDB shell version v4.4.29-28
connecting to: mongodb://some-name-target-rs0.pitr-to-new-cluster-13977.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb&replicaSet=rs0&ssl=false
Implicit session: session { "id" : UUID("b0d2bbe2-3368-45af-ba1f-6f8e3587be41") }
Percona Server for MongoDB server version: v8.0.16-5
WARNING: shell and server versions do not match
switched to db myApp
WriteResult({ "nInserted" : 1 })
bye
+ cat /tmp/tmp.vFAoi1FpMB
+ rm /tmp/tmp.Kxv46aJejw /tmp/tmp.vFAoi1FpMB
+ return 0
+ write_document some-name-target 100506
+ local cluster=some-name-target
+ local x=100506
+ log 'write document: 100506'
+ set +o xtrace
[2025-12-23T15:56:17+0000] write document: 100506
+ run_mongo 'use myApp\n db.test.insert({ x: 100506 })' myApp:myPass@some-name-target-rs0.pitr-to-new-cluster-13977 mongodb
+ local 'command=use myApp\n db.test.insert({ x: 100506 })'
+ local uri=myApp:myPass@some-name-target-rs0.pitr-to-new-cluster-13977
+ local driver=mongodb
+ local suffix=.svc.cluster.local
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.LTH4Lu8Gar
+++ mktemp
++ local LAST_ERR=/tmp/tmp.UmLluLiGUj
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.LTH4Lu8Gar
++ cat /tmp/tmp.UmLluLiGUj
++ rm /tmp/tmp.LTH4Lu8Gar /tmp/tmp.UmLluLiGUj
++ return 0
+ local client_container=psmdb-client-696897d69b-9r6m5
+ local mongo_flag=
+ [[ myApp:myPass@some-name-target-rs0.pitr-to-new-cluster-13977 == *cfg* ]]
+ replica_set=rs0
+ kubectl_bin exec psmdb-client-696897d69b-9r6m5 -- bash -c 'printf '\''use myApp\n db.test.insert({ x: 100506 })\n'\'' | mongo mongodb://myApp:myPass@some-name-target-rs0.pitr-to-new-cluster-13977.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
++ mktemp
+ local LAST_OUT=/tmp/tmp.uJPSpObvEy
++ mktemp
+ local LAST_ERR=/tmp/tmp.R9vEp2q5S7
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-696897d69b-9r6m5 -- bash -c 'printf '\''use myApp\n db.test.insert({ x: 100506 })\n'\'' | mongo mongodb://myApp:myPass@some-name-target-rs0.pitr-to-new-cluster-13977.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.uJPSpObvEy
Percona Server for MongoDB shell version v4.4.29-28
connecting to: mongodb://some-name-target-rs0.pitr-to-new-cluster-13977.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb&replicaSet=rs0&ssl=false
Implicit session: session { "id" : UUID("9141d6f5-0982-46ce-81dd-27ad5c96e06d") }
Percona Server for MongoDB server version: v8.0.16-5
WARNING: shell and server versions do not match
switched to db myApp
WriteResult({ "nInserted" : 1 })
bye
+ cat /tmp/tmp.R9vEp2q5S7
+ rm /tmp/tmp.uJPSpObvEy /tmp/tmp.R9vEp2q5S7
+ return 0
+ write_document some-name-target 100507
+ local cluster=some-name-target
+ local x=100507
+ log 'write document: 100507'
+ set +o xtrace
[2025-12-23T15:56:19+0000] write document: 100507
+ run_mongo 'use myApp\n db.test.insert({ x: 100507 })' myApp:myPass@some-name-target-rs0.pitr-to-new-cluster-13977 mongodb
+ local 'command=use myApp\n db.test.insert({ x: 100507 })'
+ local uri=myApp:myPass@some-name-target-rs0.pitr-to-new-cluster-13977
+ local driver=mongodb
+ local suffix=.svc.cluster.local
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.Om20Iib51Q
+++ mktemp
++ local LAST_ERR=/tmp/tmp.dMT8m1O2IB
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.Om20Iib51Q
++ cat /tmp/tmp.dMT8m1O2IB
++ rm /tmp/tmp.Om20Iib51Q /tmp/tmp.dMT8m1O2IB
++ return 0
+ local client_container=psmdb-client-696897d69b-9r6m5
+ local mongo_flag=
+ [[ myApp:myPass@some-name-target-rs0.pitr-to-new-cluster-13977 == *cfg* ]]
+ replica_set=rs0
+ kubectl_bin exec psmdb-client-696897d69b-9r6m5 -- bash -c 'printf '\''use myApp\n db.test.insert({ x: 100507 })\n'\'' | mongo mongodb://myApp:myPass@some-name-target-rs0.pitr-to-new-cluster-13977.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
++ mktemp
+ local LAST_OUT=/tmp/tmp.kIOjbePTU0
++ mktemp
+ local LAST_ERR=/tmp/tmp.bLUGDk0tO7
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-696897d69b-9r6m5 -- bash -c 'printf '\''use myApp\n db.test.insert({ x: 100507 })\n'\'' | mongo mongodb://myApp:myPass@some-name-target-rs0.pitr-to-new-cluster-13977.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.kIOjbePTU0
Percona Server for MongoDB shell version v4.4.29-28
connecting to: mongodb://some-name-target-rs0.pitr-to-new-cluster-13977.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb&replicaSet=rs0&ssl=false
Implicit session: session { "id" : UUID("25cafd3d-540d-48ab-8d09-5b2a06c523a7") }
Percona Server for MongoDB server version: v8.0.16-5
WARNING: shell and server versions do not match
switched to db myApp
WriteResult({ "nInserted" : 1 })
bye
+ cat /tmp/tmp.bLUGDk0tO7
+ rm /tmp/tmp.kIOjbePTU0 /tmp/tmp.bLUGDk0tO7
+ return 0
+ sleep_with_log 3 'wait for data to be replicated'
+ local d=3
+ local 'msg=wait for data to be replicated'
+ log 'wait for data to be replicated. sleeping for 3 seconds'
+ set +o xtrace
[2025-12-23T15:56:22+0000] wait for data to be replicated. sleeping for 3 seconds
+ sleep 3
+ compare_mongo_cmd find myApp:myPass@some-name-target-rs0.pitr-to-new-cluster-13977 -3rd .svc.cluster.local myApp test 'sort( { x: 1 } )'
+ local command=find
+ local uri=myApp:myPass@some-name-target-rs0.pitr-to-new-cluster-13977
+ local postfix=-3rd
+ local suffix=.svc.cluster.local
+ local database=myApp
+ local collection=test
+ local 'sort=sort( { x: 1 } )'
+ local tls=false
+ local 'full_command=db.test.find()'
+ [[ -n sort( { x: 1 } ) ]]
+ full_command='db.test.find().sort( { x: 1 } )'
+ log 'running db.test.find().sort( { x: 1 } ) in myApp'
+ set +o xtrace
[2025-12-23T15:56:25+0000] running db.test.find().sort( { x: 1 } ) in myApp
+ [[ false == \t\r\u\e ]]
+ mongo_command=run_mongo
+ run_mongo 'use myApp\n db.test.find().sort( { x: 1 } )' myApp:myPass@some-name-target-rs0.pitr-to-new-cluster-13977 mongodb .svc.cluster.local
+ local 'command=use myApp\n db.test.find().sort( { x: 1 } )'
+ local uri=myApp:myPass@some-name-target-rs0.pitr-to-new-cluster-13977
+ local driver=mongodb
+ egrep -v 'I NETWORK|W NETWORK|F NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|Started a new thread for the timer service'
+ local suffix=.svc.cluster.local
+ /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
egrep: warning: egrep is obsolescent; using grep -E
++ local LAST_OUT=/tmp/tmp.CXsKMvyWrQ
+++ mktemp
++ local LAST_ERR=/tmp/tmp.D6l0KaWiS7
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.CXsKMvyWrQ
++ cat /tmp/tmp.D6l0KaWiS7
++ rm /tmp/tmp.CXsKMvyWrQ /tmp/tmp.D6l0KaWiS7
++ return 0
+ local client_container=psmdb-client-696897d69b-9r6m5
+ local mongo_flag=
+ [[ myApp:myPass@some-name-target-rs0.pitr-to-new-cluster-13977 == *cfg* ]]
+ replica_set=rs0
+ kubectl_bin exec psmdb-client-696897d69b-9r6m5 -- bash -c 'printf '\''use myApp\n db.test.find().sort( { x: 1 } )\n'\'' | mongo mongodb://myApp:myPass@some-name-target-rs0.pitr-to-new-cluster-13977.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
++ mktemp
+ local LAST_OUT=/tmp/tmp.3ZyLsxMUkU
++ mktemp
+ local LAST_ERR=/tmp/tmp.lwL1mrVHk5
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-696897d69b-9r6m5 -- bash -c 'printf '\''use myApp\n db.test.find().sort( { x: 1 } )\n'\'' | mongo mongodb://myApp:myPass@some-name-target-rs0.pitr-to-new-cluster-13977.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.3ZyLsxMUkU
+ cat /tmp/tmp.lwL1mrVHk5
+ rm /tmp/tmp.3ZyLsxMUkU /tmp/tmp.lwL1mrVHk5
+ return 0
+ [[ 0 -eq 0 ]]
+ diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2144/e2e-tests/pitr-to-new-cluster/compare/find-3rd.json /tmp/tmp.pwZ85mBPea/find-3rd
+ log 'Data is ready for PiTR: OK'
+ set +o xtrace
[2025-12-23T15:56:27+0000] Data is ready for PiTR: OK
+ sleep_with_log 70 'wait for oplog chunks to be uploaded'
+ local d=70
+ local 'msg=wait for oplog chunks to be uploaded'
+ log 'wait for oplog chunks to be uploaded. sleeping for 70 seconds'
+ set +o xtrace
[2025-12-23T15:56:27+0000] wait for oplog chunks to be uploaded. sleeping for 70 seconds
+ sleep 70
+++ get_latest_restorable_time some-name-target-rs0
+++ local cluster=some-name-target-rs0
+++ local first_timestamp
+++ local second_timestamp
+++ local retry=0
+++ [[ '' != '' ]]
++++ kubectl_bin exec some-name-target-rs0-0 -c backup-agent -- pbm status -o json
++++ jq '.backups.pitrChunks.pitrChunks | last | .range.end'
+++++ mktemp
++++ local LAST_OUT=/tmp/tmp.Dwz2LGKEqV
+++++ mktemp
++++ local LAST_ERR=/tmp/tmp.nuVKaEElE5
++++ local exit_status=0
++++ local timeout=4
+++++ seq 0 2
++++ for i in $(seq 0 2)
++++ set +e
++++ kubectl exec some-name-target-rs0-0 -c backup-agent -- pbm status -o json
++++ exit_status=0
++++ set -e
++++ '[' 0 '!=' 0 -a -n 1 ']'
++++ break
++++ cat /tmp/tmp.Dwz2LGKEqV
++++ cat /tmp/tmp.nuVKaEElE5
++++ rm /tmp/tmp.Dwz2LGKEqV /tmp/tmp.nuVKaEElE5
++++ return 0
+++ first_timestamp=null
+++ sleep 5
+++ [[ null != '' ]]
+++ [[ null != \n\u\l\l ]]
+++ let retry+=1
+++ [[ 1 -gt 30 ]]
+++ [[ null != '' ]]
+++ [[ null != \n\u\l\l ]]
++++ kubectl_bin exec some-name-target-rs0-0 -c backup-agent -- pbm status -o json
++++ jq '.backups.pitrChunks.pitrChunks | last | .range.end'
+++++ mktemp
++++ local LAST_OUT=/tmp/tmp.ldxan2HM7S
+++++ mktemp
++++ local LAST_ERR=/tmp/tmp.ZvfuAyCD4t
++++ local exit_status=0
++++ local timeout=4
+++++ seq 0 2
++++ for i in $(seq 0 2)
++++ set +e
++++ kubectl exec some-name-target-rs0-0 -c backup-agent -- pbm status -o json
++++ exit_status=0
++++ set -e
++++ '[' 0 '!=' 0 -a -n 1 ']'
++++ break
++++ cat /tmp/tmp.ldxan2HM7S
++++ cat /tmp/tmp.ZvfuAyCD4t
++++ rm /tmp/tmp.ldxan2HM7S /tmp/tmp.ZvfuAyCD4t
++++ return 0
+++ first_timestamp=null
+++ sleep 5
+++ [[ null != '' ]]
+++ [[ null != \n\u\l\l ]]
+++ let retry+=1
+++ [[ 2 -gt 30 ]]
+++ [[ null != '' ]]
+++ [[ null != \n\u\l\l ]]
++++ kubectl_bin exec some-name-target-rs0-0 -c backup-agent -- pbm status -o json
++++ jq '.backups.pitrChunks.pitrChunks | last | .range.end'
+++++ mktemp
++++ local LAST_OUT=/tmp/tmp.RgJUnbqY7k
+++++ mktemp
++++ local LAST_ERR=/tmp/tmp.KVIdsGhuvm
++++ local exit_status=0
++++ local timeout=4
+++++ seq 0 2
++++ for i in $(seq 0 2)
++++ set +e
++++ kubectl exec some-name-target-rs0-0 -c backup-agent -- pbm status -o json
++++ exit_status=0
++++ set -e
++++ '[' 0 '!=' 0 -a -n 1 ']'
++++ break
++++ cat /tmp/tmp.RgJUnbqY7k
++++ cat /tmp/tmp.KVIdsGhuvm
++++ rm /tmp/tmp.RgJUnbqY7k /tmp/tmp.KVIdsGhuvm
++++ return 0
+++ first_timestamp=1766505467
+++ sleep 5
+++ [[ 1766505467 != '' ]]
+++ [[ 1766505467 != \n\u\l\l ]]
++++ kubectl_bin exec some-name-target-rs0-0 -c backup-agent -- pbm status -o json
++++ jq '.backups.pitrChunks.pitrChunks | last | .range.end'
+++++ mktemp
++++ local LAST_OUT=/tmp/tmp.on6Ffsnmt4
+++++ mktemp
++++ local LAST_ERR=/tmp/tmp.sRs6e6Sku2
++++ local exit_status=0
++++ local timeout=4
+++++ seq 0 2
++++ for i in $(seq 0 2)
++++ set +e
++++ kubectl exec some-name-target-rs0-0 -c backup-agent -- pbm status -o json
++++ exit_status=0
++++ set -e
++++ '[' 0 '!=' 0 -a -n 1 ']'
++++ break
++++ cat /tmp/tmp.on6Ffsnmt4
++++ cat /tmp/tmp.sRs6e6Sku2
++++ rm /tmp/tmp.on6Ffsnmt4 /tmp/tmp.sRs6e6Sku2
++++ return 0
+++ second_timestamp=1766505467
+++ let retry+=1
+++ [[ 3 -gt 30 ]]
+++ [[ 1766505467 != '' ]]
+++ [[ 1766505467 != \n\u\l\l ]]
+++ [[ 1766505467 == 1766505467 ]]
+++ /usr/sbin/date -u -d @1766505467 +%Y-%m-%dT%H:%M:%SZ
++ format_pitr_target 2025-12-23T15:57:47Z
++ local target=2025-12-23T15:57:47Z
++ echo 2025-12-23T15:57:47Z
++ sed s/Z//
++ sed 's/T/ /'
+ local 'pitrTarget=2025-12-23 15:57:47'
+ log 'dropping collection: myApp.test'
+ set +o xtrace
[2025-12-23T15:57:59+0000] dropping collection: myApp.test
+ run_mongo 'use myApp\n db.test.drop()' myApp:myPass@some-name-target-rs0.pitr-to-new-cluster-13977
+ local 'command=use myApp\n db.test.drop()'
+ local uri=myApp:myPass@some-name-target-rs0.pitr-to-new-cluster-13977
+ local driver=mongodb+srv
+ local suffix=.svc.cluster.local
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.n8EnlH3EmH
+++ mktemp
++ local LAST_ERR=/tmp/tmp.1Q039o4lWn
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.n8EnlH3EmH
++ cat /tmp/tmp.1Q039o4lWn
++ rm /tmp/tmp.n8EnlH3EmH /tmp/tmp.1Q039o4lWn
++ return 0
+ local client_container=psmdb-client-696897d69b-9r6m5
+ local mongo_flag=
+ [[ myApp:myPass@some-name-target-rs0.pitr-to-new-cluster-13977 == *cfg* ]]
+ replica_set=rs0
+ kubectl_bin exec psmdb-client-696897d69b-9r6m5 -- bash -c 'printf '\''use myApp\n db.test.drop()\n'\'' | mongo mongodb+srv://myApp:myPass@some-name-target-rs0.pitr-to-new-cluster-13977.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
++ mktemp
+ local LAST_OUT=/tmp/tmp.2OPGMNDFEz
++ mktemp
+ local LAST_ERR=/tmp/tmp.xyw8YUBa6p
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-696897d69b-9r6m5 -- bash -c 'printf '\''use myApp\n db.test.drop()\n'\'' | mongo mongodb+srv://myApp:myPass@some-name-target-rs0.pitr-to-new-cluster-13977.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.2OPGMNDFEz
Percona Server for MongoDB shell version v4.4.29-28
connecting to: mongodb://some-name-target-rs0-0.some-name-target-rs0.pitr-to-new-cluster-13977.svc.cluster.local:27017,some-name-target-rs0-2.some-name-target-rs0.pitr-to-new-cluster-13977.svc.cluster.local:27017,some-name-target-rs0-1.some-name-target-rs0.pitr-to-new-cluster-13977.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb&replicaSet=rs0&ssl=false
Implicit session: session { "id" : UUID("23ee1b15-6d63-4c79-8171-a779ffe58308") }
Percona Server for MongoDB server version: v8.0.16-5
WARNING: shell and server versions do not match
switched to db myApp
true
bye
+ cat /tmp/tmp.xyw8YUBa6p
+ rm /tmp/tmp.2OPGMNDFEz /tmp/tmp.xyw8YUBa6p
+ return 0
+ log 'PiTR target is 2025-12-23 15:57:47'
+ set +o xtrace
[2025-12-23T15:58:01+0000] PiTR target is 2025-12-23 15:57:47
+ cat /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2144/e2e-tests/pitr-to-new-cluster/conf/restore-target.yml
+ /usr/sbin/sed -e 's/date:/date: 2025-12-23 15:57:47/'
+ kubectl_bin apply -f -
++ mktemp
+ local LAST_OUT=/tmp/tmp.NbsqGqN2vk
++ mktemp
+ local LAST_ERR=/tmp/tmp.x3uspjSd5Z
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl apply -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.NbsqGqN2vk
perconaservermongodbrestore.psmdb.percona.com/restore-target-backup-minio-logical created
+ cat /tmp/tmp.x3uspjSd5Z
+ rm /tmp/tmp.NbsqGqN2vk /tmp/tmp.x3uspjSd5Z
+ return 0
+ wait_restore target-backup-minio-logical some-name-target ready
+ local backup_name=target-backup-minio-logical
+ local cluster_name=some-name-target
+ local target_state=ready
+ local wait_cluster_consistency=1
+ local wait_time=1780
+ local ok_if_ready=0
+ set +o xtrace
Waiting for the psmdb-restore/restore-target-backup-minio-logical object to be created.OK
Waiting psmdb-restore/restore-target-backup-minio-logical to reach state "ready" .OK after 0 minutes
+ [[ 1 -eq 1 ]]
+ wait_cluster_consistency some-name-target
+ local cluster_name=some-name-target
+ local wait_time=32
+ retry=0
+ sleep 7
+ echo -n 'waiting for cluster readyness'
waiting for cluster readyness++ kubectl_bin get psmdb some-name-target -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.m6nHPFy8CS
+++ mktemp
++ local LAST_ERR=/tmp/tmp.aFyJa50nmN
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name-target -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.m6nHPFy8CS
++ cat /tmp/tmp.aFyJa50nmN
++ rm /tmp/tmp.m6nHPFy8CS /tmp/tmp.aFyJa50nmN
++ return 0
+ [[ ready == \r\e\a\d\y ]]
+ echo .OK
.OK
+ compare_mongo_cmd find myApp:myPass@some-name-target-rs0.pitr-to-new-cluster-13977 -3rd .svc.cluster.local myApp test 'sort( { x: 1 } )'
+ local command=find
+ local uri=myApp:myPass@some-name-target-rs0.pitr-to-new-cluster-13977
+ local postfix=-3rd
+ local suffix=.svc.cluster.local
+ local database=myApp
+ local collection=test
+ local 'sort=sort( { x: 1 } )'
+ local tls=false
+ local 'full_command=db.test.find()'
+ [[ -n sort( { x: 1 } ) ]]
+ full_command='db.test.find().sort( { x: 1 } )'
+ log 'running db.test.find().sort( { x: 1 } ) in myApp'
+ set +o xtrace
[2025-12-23T15:58:39+0000] running db.test.find().sort( { x: 1 } ) in myApp
+ [[ false == \t\r\u\e ]]
+ mongo_command=run_mongo
+ egrep -v 'I NETWORK|W NETWORK|F NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|Started a new thread for the timer service'
+ run_mongo 'use myApp\n db.test.find().sort( { x: 1 } )' myApp:myPass@some-name-target-rs0.pitr-to-new-cluster-13977 mongodb .svc.cluster.local
+ /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
+ local 'command=use myApp\n db.test.find().sort( { x: 1 } )'
+ local uri=myApp:myPass@some-name-target-rs0.pitr-to-new-cluster-13977
+ local driver=mongodb
+ local suffix=.svc.cluster.local
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
egrep: warning: egrep is obsolescent; using grep -E
++ local LAST_OUT=/tmp/tmp.1bU5aEungO
+++ mktemp
++ local LAST_ERR=/tmp/tmp.rG66gPQzDI
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.1bU5aEungO
++ cat /tmp/tmp.rG66gPQzDI
++ rm /tmp/tmp.1bU5aEungO /tmp/tmp.rG66gPQzDI
++ return 0
+ local client_container=psmdb-client-696897d69b-9r6m5
+ local mongo_flag=
+ [[ myApp:myPass@some-name-target-rs0.pitr-to-new-cluster-13977 == *cfg* ]]
+ replica_set=rs0
+ kubectl_bin exec psmdb-client-696897d69b-9r6m5 -- bash -c 'printf '\''use myApp\n db.test.find().sort( { x: 1 } )\n'\'' | mongo mongodb://myApp:myPass@some-name-target-rs0.pitr-to-new-cluster-13977.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
++ mktemp
+ local LAST_OUT=/tmp/tmp.R5SFf7yOD1
++ mktemp
+ local LAST_ERR=/tmp/tmp.vreyHIyIrC
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-696897d69b-9r6m5 -- bash -c 'printf '\''use myApp\n db.test.find().sort( { x: 1 } )\n'\'' | mongo mongodb://myApp:myPass@some-name-target-rs0.pitr-to-new-cluster-13977.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.R5SFf7yOD1
+ cat /tmp/tmp.vreyHIyIrC
+ rm /tmp/tmp.R5SFf7yOD1 /tmp/tmp.vreyHIyIrC
+ return 0
+ [[ 0 -eq 0 ]]
+ diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2144/e2e-tests/pitr-to-new-cluster/compare/find-3rd.json /tmp/tmp.pwZ85mBPea/find-3rd
+ log 'Data is restored from target-backup-minio-logical: OK'
+ set +o xtrace
[2025-12-23T15:58:41+0000] Data is restored from target-backup-minio-logical: OK
+ desc 'Case 3: Backup and restore on some-name-target: OK'
+ set +o xtrace


-----------------------------------------------------------------------------------
Case 3: Backup and restore on some-name-target: OK
-----------------------------------------------------------------------------------

+ desc 'test passed'
+ set +o xtrace


-----------------------------------------------------------------------------------
test passed
-----------------------------------------------------------------------------------

+ destroy pitr-to-new-cluster-13977
+ local namespace=pitr-to-new-cluster-13977
+ local ignore_logs=true
+ [[ 0 == 1 ]]
+ desc 'destroy cluster/operator and all other resources'
+ set +o xtrace


-----------------------------------------------------------------------------------
destroy cluster/operator and all other resources
-----------------------------------------------------------------------------------

+ '[' true == false ']'
+ delete_backups
+ desc 'Delete psmdb-backup'
+ set +o xtrace


-----------------------------------------------------------------------------------
Delete psmdb-backup
-----------------------------------------------------------------------------------

++ kubectl_bin get psmdb-backup --no-headers
++ wc -l
+++ mktemp
++ local LAST_OUT=/tmp/tmp.pjImXjNGhS
+++ mktemp
++ local LAST_ERR=/tmp/tmp.ULdhrodAs2
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb-backup --no-headers
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.pjImXjNGhS
++ cat /tmp/tmp.ULdhrodAs2
++ rm /tmp/tmp.pjImXjNGhS /tmp/tmp.ULdhrodAs2
++ return 0
+ '[' 3 '!=' 0 ']'
+ kubectl_bin get psmdb-backup
++ mktemp
+ local LAST_OUT=/tmp/tmp.b6OKe2fTp1
++ mktemp
+ local LAST_ERR=/tmp/tmp.w6OgdnngCg
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl get psmdb-backup
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.b6OKe2fTp1
NAME                          CLUSTER            STORAGE   DESTINATION                                                   TYPE       SIZE       STATUS   COMPLETED   AGE
backup-minio-logical          some-name-source   minio     s3://operator-testing/some-name-source/2025-12-23T15:44:25Z   logical    53.61KB    ready    14m         14m
backup-minio-physical         some-name-source   minio     s3://operator-testing/some-name-source/2025-12-23T15:45:49Z   physical   598.19KB   ready    12m         12m
target-backup-minio-logical   some-name-target   minio     s3://operator-testing/some-name-target/2025-12-23T15:55:56Z   logical    49.30KB    ready    2m42s       2m48s
+ cat /tmp/tmp.w6OgdnngCg
+ rm /tmp/tmp.b6OKe2fTp1 /tmp/tmp.w6OgdnngCg
+ return 0
+ kubectl_bin delete psmdb-backup --all
++ mktemp
+ local LAST_OUT=/tmp/tmp.sAu4ODBCuC
++ mktemp
+ local LAST_ERR=/tmp/tmp.GpA9BiIqCF
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl delete psmdb-backup --all
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.sAu4ODBCuC
perconaservermongodbbackup.psmdb.percona.com "backup-minio-logical" deleted from pitr-to-new-cluster-13977 namespace
perconaservermongodbbackup.psmdb.percona.com "backup-minio-physical" deleted from pitr-to-new-cluster-13977 namespace
perconaservermongodbbackup.psmdb.percona.com "target-backup-minio-logical" deleted from pitr-to-new-cluster-13977 namespace
+ cat /tmp/tmp.GpA9BiIqCF
+ rm /tmp/tmp.sAu4ODBCuC /tmp/tmp.GpA9BiIqCF
+ return 0
+ delete_crd
+ desc 'get and delete old CRDs and RBAC'
+ set +o xtrace


-----------------------------------------------------------------------------------
get and delete old CRDs and RBAC
-----------------------------------------------------------------------------------

+ kubectl_bin delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2144/deploy/crd.yaml --ignore-not-found --wait=false
++ mktemp
+ local LAST_OUT=/tmp/tmp.7tJMlnf8fJ
++ mktemp
+ local LAST_ERR=/tmp/tmp.L97TRJsXJa
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2144/deploy/crd.yaml --ignore-not-found --wait=false
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.7tJMlnf8fJ
customresourcedefinition.apiextensions.k8s.io "perconaservermongodbbackups.psmdb.percona.com" deleted
customresourcedefinition.apiextensions.k8s.io "perconaservermongodbrestores.psmdb.percona.com" deleted
customresourcedefinition.apiextensions.k8s.io "perconaservermongodbs.psmdb.percona.com" deleted
+ cat /tmp/tmp.L97TRJsXJa
+ rm /tmp/tmp.7tJMlnf8fJ /tmp/tmp.L97TRJsXJa
+ return 0
++ yq eval .metadata.name /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2144/deploy/crd.yaml
++ grep -v '\-\-\-'
grep: warning: stray \ before -
grep: warning: stray \ before -
+ for crd_name in $(yq eval '.metadata.name' "${src_dir}/deploy/crd.yaml" | grep -v '\-\-\-')
+ kubectl get perconaservermongodbbackups.psmdb.percona.com --all-namespaces -o wide
+ grep -v NAMESPACE
+ xargs -L 1 sh -xc 'kubectl patch perconaservermongodbbackups.psmdb.percona.com -n $0 $1 --type=merge -p "{\"metadata\":{\"finalizers\":[]}}"'
error: the server doesn't have a resource type "perconaservermongodbbackups"
+ kubectl patch perconaservermongodbbackups.psmdb.percona.com -n sh --type=merge -p '{"metadata":{"finalizers":[]}}'
error: the server doesn't have a resource type "perconaservermongodbbackups"
+ :
+ kubectl_bin wait --for=delete crd perconaservermongodbbackups.psmdb.percona.com
++ mktemp
+ local LAST_OUT=/tmp/tmp.TMQ7dnzog6
++ mktemp
+ local LAST_ERR=/tmp/tmp.yu3bRt0gs7
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl wait --for=delete crd perconaservermongodbbackups.psmdb.percona.com
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.TMQ7dnzog6
+ cat /tmp/tmp.yu3bRt0gs7
+ rm /tmp/tmp.TMQ7dnzog6 /tmp/tmp.yu3bRt0gs7
+ return 0
+ for crd_name in $(yq eval '.metadata.name' "${src_dir}/deploy/crd.yaml" | grep -v '\-\-\-')
+ kubectl get perconaservermongodbrestores.psmdb.percona.com --all-namespaces -o wide
+ xargs -L 1 sh -xc 'kubectl patch perconaservermongodbrestores.psmdb.percona.com -n $0 $1 --type=merge -p "{\"metadata\":{\"finalizers\":[]}}"'
+ grep -v NAMESPACE
error: the server doesn't have a resource type "perconaservermongodbrestores"
+ kubectl patch perconaservermongodbrestores.psmdb.percona.com -n sh --type=merge -p '{"metadata":{"finalizers":[]}}'
error: the server doesn't have a resource type "perconaservermongodbrestores"
+ :
+ kubectl_bin wait --for=delete crd perconaservermongodbrestores.psmdb.percona.com
++ mktemp
+ local LAST_OUT=/tmp/tmp.uaQ10TWx0b
++ mktemp
+ local LAST_ERR=/tmp/tmp.AeS76Mnoon
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl wait --for=delete crd perconaservermongodbrestores.psmdb.percona.com
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.uaQ10TWx0b
+ cat /tmp/tmp.AeS76Mnoon
+ rm /tmp/tmp.uaQ10TWx0b /tmp/tmp.AeS76Mnoon
+ return 0
+ for crd_name in $(yq eval '.metadata.name' "${src_dir}/deploy/crd.yaml" | grep -v '\-\-\-')
+ kubectl get perconaservermongodbs.psmdb.percona.com --all-namespaces -o wide
+ grep -v NAMESPACE
+ xargs -L 1 sh -xc 'kubectl patch perconaservermongodbs.psmdb.percona.com -n $0 $1 --type=merge -p "{\"metadata\":{\"finalizers\":[]}}"'
error: the server doesn't have a resource type "perconaservermongodbs"
+ kubectl patch perconaservermongodbs.psmdb.percona.com -n sh --type=merge -p '{"metadata":{"finalizers":[]}}'
error: the server doesn't have a resource type "perconaservermongodbs"
+ :
+ kubectl_bin wait --for=delete crd perconaservermongodbs.psmdb.percona.com
++ mktemp
+ local LAST_OUT=/tmp/tmp.HukR1sV4pc
++ mktemp
+ local LAST_ERR=/tmp/tmp.FMwQZMiBeX
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl wait --for=delete crd perconaservermongodbs.psmdb.percona.com
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.HukR1sV4pc
+ cat /tmp/tmp.FMwQZMiBeX
+ rm /tmp/tmp.HukR1sV4pc /tmp/tmp.FMwQZMiBeX
+ return 0
+ local rbac_yaml=rbac.yaml
+ '[' -n psmdb-operator ']'
+ rbac_yaml=cw-rbac.yaml
+ kubectl_bin delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2144/deploy/cw-rbac.yaml --ignore-not-found
++ mktemp
+ local LAST_OUT=/tmp/tmp.l0fC9Z3NOT
++ mktemp
+ local LAST_ERR=/tmp/tmp.vCEegZk7Z5
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2144/deploy/cw-rbac.yaml --ignore-not-found
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.l0fC9Z3NOT
clusterrole.rbac.authorization.k8s.io "percona-server-mongodb-operator" deleted
clusterrolebinding.rbac.authorization.k8s.io "service-account-percona-server-mongodb-operator" deleted
+ cat /tmp/tmp.vCEegZk7Z5
+ rm /tmp/tmp.l0fC9Z3NOT /tmp/tmp.vCEegZk7Z5
+ return 0
+ destroy_cert_manager
+ kubectl_bin delete -f https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml
++ mktemp
+ local LAST_OUT=/tmp/tmp.eS7KLFk2K4
++ mktemp
+ local LAST_ERR=/tmp/tmp.z6P6i0wzLn
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl delete -f https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml
+ exit_status=1
+ set -e
+ '[' 1 '!=' 0 -a -n 1 ']'
+ cat /tmp/tmp.eS7KLFk2K4
+ cat /tmp/tmp.z6P6i0wzLn
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": namespaces "cert-manager" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "challenges.acme.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "orders.acme.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "certificaterequests.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "certificates.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "clusterissuers.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "issuers.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": serviceaccounts "cert-manager-cainjector" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": serviceaccounts "cert-manager" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": serviceaccounts "cert-manager-webhook" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-cainjector" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-issuers" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-clusterissuers" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-certificates" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-orders" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-challenges" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-ingress-shim" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-cluster-view" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-view" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-edit" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-approve:cert-manager-io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-certificatesigningrequests" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-webhook:subjectaccessreviews" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-cainjector" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-issuers" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-clusterissuers" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-certificates" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-orders" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-challenges" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-ingress-shim" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-approve:cert-manager-io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-certificatesigningrequests" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-webhook:subjectaccessreviews" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager-cainjector:leaderelection" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager:leaderelection" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager-tokenrequest" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager-webhook:dynamic-serving" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager-cainjector:leaderelection" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager:leaderelection" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager-tokenrequest" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager-webhook:dynamic-serving" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": services "cert-manager-cainjector" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": services "cert-manager" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": services "cert-manager-webhook" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": deployments.apps "cert-manager-cainjector" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": deployments.apps "cert-manager" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": deployments.apps "cert-manager-webhook" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": mutatingwebhookconfigurations.admissionregistration.k8s.io "cert-manager-webhook" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": validatingwebhookconfigurations.admissionregistration.k8s.io "cert-manager-webhook" not found
+ sleep 0
+ for i in $(seq 0 2)
+ set +e
+ kubectl delete -f https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml
+ exit_status=1
+ set -e
+ '[' 1 '!=' 0 -a -n 1 ']'
+ cat /tmp/tmp.eS7KLFk2K4
+ cat /tmp/tmp.z6P6i0wzLn
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": namespaces "cert-manager" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "challenges.acme.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "orders.acme.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "certificaterequests.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "certificates.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "clusterissuers.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "issuers.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": serviceaccounts "cert-manager-cainjector" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": serviceaccounts "cert-manager" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": serviceaccounts "cert-manager-webhook" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-cainjector" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-issuers" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-clusterissuers" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-certificates" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-orders" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-challenges" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-ingress-shim" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-cluster-view" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-view" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-edit" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-approve:cert-manager-io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-certificatesigningrequests" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-webhook:subjectaccessreviews" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-cainjector" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-issuers" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-clusterissuers" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-certificates" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-orders" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-challenges" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-ingress-shim" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-approve:cert-manager-io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-certificatesigningrequests" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-webhook:subjectaccessreviews" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager-cainjector:leaderelection" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager:leaderelection" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager-tokenrequest" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager-webhook:dynamic-serving" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager-cainjector:leaderelection" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager:leaderelection" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager-tokenrequest" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager-webhook:dynamic-serving" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": services "cert-manager-cainjector" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": services "cert-manager" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": services "cert-manager-webhook" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": deployments.apps "cert-manager-cainjector" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": deployments.apps "cert-manager" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": deployments.apps "cert-manager-webhook" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": mutatingwebhookconfigurations.admissionregistration.k8s.io "cert-manager-webhook" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": validatingwebhookconfigurations.admissionregistration.k8s.io "cert-manager-webhook" not found
+ sleep 4
+ for i in $(seq 0 2)
+ set +e
+ kubectl delete -f https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml
+ exit_status=1
+ set -e
+ '[' 1 '!=' 0 -a -n 1 ']'
+ cat /tmp/tmp.eS7KLFk2K4
+ cat /tmp/tmp.z6P6i0wzLn
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": namespaces "cert-manager" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "challenges.acme.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "orders.acme.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "certificaterequests.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "certificates.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "clusterissuers.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "issuers.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": serviceaccounts "cert-manager-cainjector" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": serviceaccounts "cert-manager" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": serviceaccounts "cert-manager-webhook" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-cainjector" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-issuers" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-clusterissuers" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-certificates" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-orders" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-challenges" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-ingress-shim" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-cluster-view" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-view" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-edit" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-approve:cert-manager-io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-certificatesigningrequests" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-webhook:subjectaccessreviews" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-cainjector" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-issuers" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-clusterissuers" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-certificates" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-orders" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-challenges" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-ingress-shim" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-approve:cert-manager-io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-certificatesigningrequests" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-webhook:subjectaccessreviews" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager-cainjector:leaderelection" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager:leaderelection" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager-tokenrequest" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager-webhook:dynamic-serving" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager-cainjector:leaderelection" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager:leaderelection" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager-tokenrequest" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager-webhook:dynamic-serving" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": services "cert-manager-cainjector" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": services "cert-manager" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": services "cert-manager-webhook" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": deployments.apps "cert-manager-cainjector" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": deployments.apps "cert-manager" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": deployments.apps "cert-manager-webhook" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": mutatingwebhookconfigurations.admissionregistration.k8s.io "cert-manager-webhook" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": validatingwebhookconfigurations.admissionregistration.k8s.io "cert-manager-webhook" not found
+ sleep 8
+ cat /tmp/tmp.eS7KLFk2K4
+ cat /tmp/tmp.z6P6i0wzLn
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": namespaces "cert-manager" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "challenges.acme.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "orders.acme.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "certificaterequests.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "certificates.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "clusterissuers.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "issuers.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": serviceaccounts "cert-manager-cainjector" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": serviceaccounts "cert-manager" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": serviceaccounts "cert-manager-webhook" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-cainjector" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-issuers" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-clusterissuers" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-certificates" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-orders" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-challenges" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-ingress-shim" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-cluster-view" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-view" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-edit" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-approve:cert-manager-io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-certificatesigningrequests" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-webhook:subjectaccessreviews" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-cainjector" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-issuers" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-clusterissuers" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-certificates" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-orders" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-challenges" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-ingress-shim" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-approve:cert-manager-io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-certificatesigningrequests" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-webhook:subjectaccessreviews" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager-cainjector:leaderelection" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager:leaderelection" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager-tokenrequest" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager-webhook:dynamic-serving" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager-cainjector:leaderelection" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager:leaderelection" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager-tokenrequest" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager-webhook:dynamic-serving" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": services "cert-manager-cainjector" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": services "cert-manager" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": services "cert-manager-webhook" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": deployments.apps "cert-manager-cainjector" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": deployments.apps "cert-manager" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": deployments.apps "cert-manager-webhook" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": mutatingwebhookconfigurations.admissionregistration.k8s.io "cert-manager-webhook" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": validatingwebhookconfigurations.admissionregistration.k8s.io "cert-manager-webhook" not found
+ rm /tmp/tmp.eS7KLFk2K4 /tmp/tmp.z6P6i0wzLn
+ return 1
+ true
+ '[' -n '' ']'
+ '[' -n psmdb-operator ']'
+ kubectl_bin delete --grace-period=0 --force=true namespace pitr-to-new-cluster-13977
+ rm -rf /tmp/tmp.pwZ85mBPea
+ kubectl_bin delete --grace-period=0 --force=true namespace psmdb-operator
++ mktemp
++ mktemp
+ local LAST_OUT=/tmp/tmp.J2atR9FJQ3
++ mktemp
+ local LAST_OUT=/tmp/tmp.fUsJ9PZ7FH
++ mktemp
+ local LAST_ERR=/tmp/tmp.Lq7ZXzC9dv
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ local LAST_ERR=/tmp/tmp.Cv99pOrXAd
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl delete --grace-period=0 --force=true namespace pitr-to-new-cluster-13977
+ for i in $(seq 0 2)
+ set +e
+ kubectl delete --grace-period=0 --force=true namespace psmdb-operator