Log: /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1393/e2e-tests/logs/pitr-physical-backup-source.log
grep: warning: stray \ before -
Warning: version difference between client (1.34) and server (1.31) exceeds the supported minor version skew of +/-1
Warning: version difference between client (1.34) and server (1.31) exceeds the supported minor version skew of +/-1
Warning: version difference between client (1.34) and server (1.31) exceeds the supported minor version skew of +/-1
+ main
+ create_infra pitr-physical-backup-source-4895
+ local ns=pitr-physical-backup-source-4895
+ [[ 1 == 1 ]]
+ delete_crd
+ desc 'get and delete old CRDs and RBAC'
+ set +o xtrace


-----------------------------------------------------------------------------------
get and delete old CRDs and RBAC
-----------------------------------------------------------------------------------

+ kubectl_bin delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1393/deploy/crd.yaml --ignore-not-found --wait=false
++ mktemp
+ local LAST_OUT=/tmp/tmp.5w6bggNd2Y
++ mktemp
+ local LAST_ERR=/tmp/tmp.E18YabXJep
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1393/deploy/crd.yaml --ignore-not-found --wait=false
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.5w6bggNd2Y
customresourcedefinition.apiextensions.k8s.io "perconaservermongodbbackups.psmdb.percona.com" deleted
customresourcedefinition.apiextensions.k8s.io "perconaservermongodbrestores.psmdb.percona.com" deleted
customresourcedefinition.apiextensions.k8s.io "perconaservermongodbs.psmdb.percona.com" deleted
+ cat /tmp/tmp.E18YabXJep
+ rm /tmp/tmp.5w6bggNd2Y /tmp/tmp.E18YabXJep
+ return 0
++ yq eval .metadata.name /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1393/deploy/crd.yaml
++ grep -v '\-\-\-'
grep: warning: stray \ before -
grep: warning: stray \ before -
+ for crd_name in $(yq eval '.metadata.name' "${src_dir}/deploy/crd.yaml" | grep -v '\-\-\-')
+ kubectl get perconaservermongodbbackups.psmdb.percona.com --all-namespaces -o wide
+ grep -v NAMESPACE
+ xargs -L 1 sh -xc 'kubectl patch perconaservermongodbbackups.psmdb.percona.com -n $0 $1 --type=merge -p "{\"metadata\":{\"finalizers\":[]}}"'
+ kubectl patch perconaservermongodbbackups.psmdb.percona.com -n pitr-physical-backup-source-15535 backup-minio-1 --type=merge -p '{"metadata":{"finalizers":[]}}'
perconaservermongodbbackup.psmdb.percona.com/backup-minio-1 patched
+ kubectl_bin wait --for=delete crd perconaservermongodbbackups.psmdb.percona.com
++ mktemp
+ local LAST_OUT=/tmp/tmp.cQUiMx9qy0
++ mktemp
+ local LAST_ERR=/tmp/tmp.05buM5n1Fe
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl wait --for=delete crd perconaservermongodbbackups.psmdb.percona.com
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.cQUiMx9qy0
+ cat /tmp/tmp.05buM5n1Fe
+ rm /tmp/tmp.cQUiMx9qy0 /tmp/tmp.05buM5n1Fe
+ return 0
+ for crd_name in $(yq eval '.metadata.name' "${src_dir}/deploy/crd.yaml" | grep -v '\-\-\-')
+ kubectl get perconaservermongodbrestores.psmdb.percona.com --all-namespaces -o wide
+ grep -v NAMESPACE
+ xargs -L 1 sh -xc 'kubectl patch perconaservermongodbrestores.psmdb.percona.com -n $0 $1 --type=merge -p "{\"metadata\":{\"finalizers\":[]}}"'
error: the server doesn't have a resource type "perconaservermongodbrestores"
+ kubectl patch perconaservermongodbrestores.psmdb.percona.com -n sh --type=merge -p '{"metadata":{"finalizers":[]}}'
error: the server doesn't have a resource type "perconaservermongodbrestores"
+ :
+ kubectl_bin wait --for=delete crd perconaservermongodbrestores.psmdb.percona.com
++ mktemp
+ local LAST_OUT=/tmp/tmp.XpddZpjAqz
++ mktemp
+ local LAST_ERR=/tmp/tmp.bqjKKjc7mM
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl wait --for=delete crd perconaservermongodbrestores.psmdb.percona.com
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.XpddZpjAqz
+ cat /tmp/tmp.bqjKKjc7mM
+ rm /tmp/tmp.XpddZpjAqz /tmp/tmp.bqjKKjc7mM
+ return 0
+ for crd_name in $(yq eval '.metadata.name' "${src_dir}/deploy/crd.yaml" | grep -v '\-\-\-')
+ kubectl get perconaservermongodbs.psmdb.percona.com --all-namespaces -o wide
+ grep -v NAMESPACE
+ xargs -L 1 sh -xc 'kubectl patch perconaservermongodbs.psmdb.percona.com -n $0 $1 --type=merge -p "{\"metadata\":{\"finalizers\":[]}}"'
error: the server doesn't have a resource type "perconaservermongodbs"
+ kubectl patch perconaservermongodbs.psmdb.percona.com -n sh --type=merge -p '{"metadata":{"finalizers":[]}}'
error: the server doesn't have a resource type "perconaservermongodbs"
+ :
+ kubectl_bin wait --for=delete crd perconaservermongodbs.psmdb.percona.com
++ mktemp
+ local LAST_OUT=/tmp/tmp.dD4L3xAzP3
++ mktemp
+ local LAST_ERR=/tmp/tmp.TXgCYeHrYN
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl wait --for=delete crd perconaservermongodbs.psmdb.percona.com
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.dD4L3xAzP3
+ cat /tmp/tmp.TXgCYeHrYN
+ rm /tmp/tmp.dD4L3xAzP3 /tmp/tmp.TXgCYeHrYN
+ return 0
+ local rbac_yaml=rbac.yaml
+ '[' -n psmdb-operator ']'
+ rbac_yaml=cw-rbac.yaml
+ kubectl_bin delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1393/deploy/cw-rbac.yaml --ignore-not-found
++ mktemp
+ local LAST_OUT=/tmp/tmp.7R1Nv3VIqy
++ mktemp
+ local LAST_ERR=/tmp/tmp.I4295QXGZx
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1393/deploy/cw-rbac.yaml --ignore-not-found
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.7R1Nv3VIqy
clusterrole.rbac.authorization.k8s.io "percona-server-mongodb-operator" deleted
clusterrolebinding.rbac.authorization.k8s.io "service-account-percona-server-mongodb-operator" deleted
+ cat /tmp/tmp.I4295QXGZx
+ rm /tmp/tmp.7R1Nv3VIqy /tmp/tmp.I4295QXGZx
+ return 0
+ check_crd_for_deletion PR-1393-371bdb64
+ local git_tag=PR-1393-371bdb64
++ curl -s https://raw.githubusercontent.com/percona/percona-server-mongodb-operator/PR-1393-371bdb64/deploy/crd.yaml
++ yq eval .metadata.name
++ /usr/sbin/sed s/---//g
++ /usr/sbin/sed ':a;N;$!ba;s/\n/ /g'
+ for crd_name in $(curl -s https://raw.githubusercontent.com/percona/percona-server-mongodb-operator/${git_tag}/deploy/crd.yaml | yq eval '.metadata.name' | $sed 's/---//g' | $sed ':a;N;$!ba;s/\n/ /g')
++ kubectl_bin get crd/null -o 'jsonpath={.status.conditions[-1].type}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.AuuCUTmxMH
+++ mktemp
++ local LAST_ERR=/tmp/tmp.PDBSqdlNkX
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get crd/null -o 'jsonpath={.status.conditions[-1].type}'
++ exit_status=1
++ set -e
++ '[' 1 '!=' 0 -a -n 1 ']'
++ cat /tmp/tmp.AuuCUTmxMH
++ cat /tmp/tmp.PDBSqdlNkX
Error from server (NotFound): customresourcedefinitions.apiextensions.k8s.io "null" not found
++ sleep 0
++ for i in $(seq 0 2)
++ set +e
++ kubectl get crd/null -o 'jsonpath={.status.conditions[-1].type}'
++ exit_status=1
++ set -e
++ '[' 1 '!=' 0 -a -n 1 ']'
++ cat /tmp/tmp.AuuCUTmxMH
++ cat /tmp/tmp.PDBSqdlNkX
Error from server (NotFound): customresourcedefinitions.apiextensions.k8s.io "null" not found
++ sleep 4
++ for i in $(seq 0 2)
++ set +e
++ kubectl get crd/null -o 'jsonpath={.status.conditions[-1].type}'
++ exit_status=1
++ set -e
++ '[' 1 '!=' 0 -a -n 1 ']'
++ cat /tmp/tmp.AuuCUTmxMH
++ cat /tmp/tmp.PDBSqdlNkX
Error from server (NotFound): customresourcedefinitions.apiextensions.k8s.io "null" not found
++ sleep 8
++ cat /tmp/tmp.AuuCUTmxMH
++ cat /tmp/tmp.PDBSqdlNkX
Error from server (NotFound): customresourcedefinitions.apiextensions.k8s.io "null" not found
++ rm /tmp/tmp.AuuCUTmxMH /tmp/tmp.PDBSqdlNkX
++ return 1
+ [[ '' == \T\e\r\m\i\n\a\t\i\n\g ]]
+ '[' -n psmdb-operator ']'
+ create_namespace psmdb-operator
+ local namespace=psmdb-operator
+ local skip_clean_namespace=
+ [[ 1 == 1 ]]
+ [[ -z '' ]]
+ destroy_chaos_mesh
++ helm list --all-namespaces --filter chaos-mesh
++ tail -n1
++ awk '-F ' '{print $2}'
++ sed s/NAMESPACE//
+ local chaos_mesh_ns=
+ desc 'destroy chaos-mesh'
+ set +o xtrace


-----------------------------------------------------------------------------------
destroy chaos-mesh
-----------------------------------------------------------------------------------

+ '[' -n '' ']'
++ kubectl get MutatingWebhookConfiguration
++ grep chaos-mesh
++ awk '{print $1}'
+ timeout 30 kubectl delete MutatingWebhookConfiguration
error: resource(s) were provided, but no name was specified
+ :
++ kubectl get ValidatingWebhookConfiguration
++ grep chaos-mesh
++ awk '{print $1}'
+ timeout 30 kubectl delete ValidatingWebhookConfiguration
error: resource(s) were provided, but no name was specified
+ :
++ kubectl get ValidatingWebhookConfiguration
++ grep validate-auth
++ awk '{print $1}'
+ timeout 30 kubectl delete ValidatingWebhookConfiguration
error: resource(s) were provided, but no name was specified
+ :
++ kubectl api-resources
++ grep chaos-mesh
++ awk '{print $1}'
++ kubectl get crd
++ grep chaos-mesh.org
++ awk '{print $1}'
+ timeout 30 kubectl delete crd
error: resource(s) were provided, but no name was specified
+ :
++ kubectl get clusterrolebinding
++ grep chaos-mesh
++ awk '{print $1}'
+ timeout 30 kubectl delete clusterrolebinding
error: resource(s) were provided, but no name was specified
+ :
++ kubectl get clusterrole
++ grep chaos-mesh
++ awk '{print $1}'
+ timeout 30 kubectl delete clusterrole
error: resource(s) were provided, but no name was specified
+ :
+ desc 'cleaned up all old namespaces'
+ set +o xtrace


-----------------------------------------------------------------------------------
cleaned up all old namespaces
-----------------------------------------------------------------------------------

+ kubectl_bin get ns
+ egrep -v '^kube-|^default|Terminating|psmdb-operator|openshift|^gke-|^gmp-|^NAME'
+ awk '{print$1}'
+ '[' -n '' ']'
+ desc 'cleaned up old namespaces psmdb-operator'
+ xargs kubectl delete ns
+ set +o xtrace


-----------------------------------------------------------------------------------
++ mktemp
cleaned up old namespaces psmdb-operator
-----------------------------------------------------------------------------------

+ kubectl_bin delete namespace psmdb-operator --ignore-not-found
++ mktemp
+ local LAST_OUT=/tmp/tmp.5eKQsEiTsw
++ mktemp
egrep: warning: egrep is obsolescent; using grep -E
+ local LAST_OUT=/tmp/tmp.BxE7uUeGX6
++ mktemp
+ local LAST_ERR=/tmp/tmp.bbJSLYgVWJ
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ local LAST_ERR=/tmp/tmp.IUPiyp9RRw
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl get ns
+ for i in $(seq 0 2)
+ set +e
+ kubectl delete namespace psmdb-operator --ignore-not-found
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.5eKQsEiTsw
+ cat /tmp/tmp.bbJSLYgVWJ
+ rm /tmp/tmp.5eKQsEiTsw /tmp/tmp.bbJSLYgVWJ
+ return 0
namespace "pitr-physical-backup-source-15535" deleted
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.BxE7uUeGX6
namespace "psmdb-operator" deleted
+ cat /tmp/tmp.IUPiyp9RRw
+ rm /tmp/tmp.BxE7uUeGX6 /tmp/tmp.IUPiyp9RRw
+ return 0
+ kubectl_bin wait --for=delete namespace psmdb-operator
++ mktemp
+ local LAST_OUT=/tmp/tmp.ET9WB4XrAb
++ mktemp
+ local LAST_ERR=/tmp/tmp.IzIq1EHwWN
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl wait --for=delete namespace psmdb-operator
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.ET9WB4XrAb
+ cat /tmp/tmp.IzIq1EHwWN
+ rm /tmp/tmp.ET9WB4XrAb /tmp/tmp.IzIq1EHwWN
+ return 0
+ desc 'create namespace psmdb-operator'
+ set +o xtrace


-----------------------------------------------------------------------------------
create namespace psmdb-operator
-----------------------------------------------------------------------------------

+ kubectl_bin create namespace psmdb-operator
++ mktemp
+ local LAST_OUT=/tmp/tmp.vA8np6MPtn
++ mktemp
+ local LAST_ERR=/tmp/tmp.fSyAbkZ4Vp
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl create namespace psmdb-operator
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.vA8np6MPtn
namespace/psmdb-operator created
+ cat /tmp/tmp.fSyAbkZ4Vp
+ rm /tmp/tmp.vA8np6MPtn /tmp/tmp.fSyAbkZ4Vp
+ return 0
+ set_kube_ctx psmdb-operator
+ local namespace=psmdb-operator
++ kubectl_bin config current-context
+++ mktemp
++ local LAST_OUT=/tmp/tmp.TeUjQif46G
+++ mktemp
++ local LAST_ERR=/tmp/tmp.UgJMsUnkCj
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl config current-context
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.TeUjQif46G
++ cat /tmp/tmp.UgJMsUnkCj
++ rm /tmp/tmp.TeUjQif46G /tmp/tmp.UgJMsUnkCj
++ return 0
+ kubectl_bin config set-context gke_cloud-dev-112233_us-central1-a_jen-psmdb-1393-371bdb64-32-cluster3 --namespace=psmdb-operator
++ mktemp
+ local LAST_OUT=/tmp/tmp.kOxDxDC7eU
++ mktemp
+ local LAST_ERR=/tmp/tmp.hftWCc4AoB
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl config set-context gke_cloud-dev-112233_us-central1-a_jen-psmdb-1393-371bdb64-32-cluster3 --namespace=psmdb-operator
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.kOxDxDC7eU
Context "gke_cloud-dev-112233_us-central1-a_jen-psmdb-1393-371bdb64-32-cluster3" modified.
+ cat /tmp/tmp.hftWCc4AoB
+ rm /tmp/tmp.kOxDxDC7eU /tmp/tmp.hftWCc4AoB
+ return 0
+ deploy_operator
+ desc 'start PSMDB operator: perconalab/percona-server-mongodb-operator:PR-1393-371bdb64'
+ set +o xtrace


-----------------------------------------------------------------------------------
start PSMDB operator: perconalab/percona-server-mongodb-operator:PR-1393-371bdb64
-----------------------------------------------------------------------------------

+ local cr_file
+ '[' -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1393/e2e-tests/pitr-physical-backup-source/conf/crd.yaml ']'
+ cr_file=/mnt/jenkins/workspace/cloud-psmdb-operator_PR-1393/deploy/crd.yaml
+ kubectl_bin apply --server-side --force-conflicts -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1393/deploy/crd.yaml
++ mktemp
+ local LAST_OUT=/tmp/tmp.aoqMceOm6W
++ mktemp
+ local LAST_ERR=/tmp/tmp.ljK50DESFu
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl apply --server-side --force-conflicts -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1393/deploy/crd.yaml
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.aoqMceOm6W
customresourcedefinition.apiextensions.k8s.io/perconaservermongodbbackups.psmdb.percona.com serverside-applied
customresourcedefinition.apiextensions.k8s.io/perconaservermongodbrestores.psmdb.percona.com serverside-applied
customresourcedefinition.apiextensions.k8s.io/perconaservermongodbs.psmdb.percona.com serverside-applied
+ cat /tmp/tmp.ljK50DESFu
+ rm /tmp/tmp.aoqMceOm6W /tmp/tmp.ljK50DESFu
+ return 0
+ '[' -n psmdb-operator ']'
+ apply_rbac cw-rbac
+ local operator_namespace=psmdb-operator
+ local rbac=cw-rbac
+ cat /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1393/deploy/cw-rbac.yaml
+ sed -e 's^namespace: .*^namespace: psmdb-operator^'
+ kubectl_bin apply -n psmdb-operator -f -
++ mktemp
+ local LAST_OUT=/tmp/tmp.y2AlNmyJWF
++ mktemp
+ local LAST_ERR=/tmp/tmp.6vxazsZUqC
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl apply -n psmdb-operator -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.y2AlNmyJWF
clusterrole.rbac.authorization.k8s.io/percona-server-mongodb-operator created
serviceaccount/percona-server-mongodb-operator created
clusterrolebinding.rbac.authorization.k8s.io/service-account-percona-server-mongodb-operator created
+ cat /tmp/tmp.6vxazsZUqC
+ rm /tmp/tmp.y2AlNmyJWF /tmp/tmp.6vxazsZUqC
+ return 0
+ yq eval '
			(.spec.template.spec.containers[].image = "perconalab/percona-server-mongodb-operator:PR-1393-371bdb64") |
			((.. | select(.[] == "DISABLE_TELEMETRY")) |= .value="true") |
			((.. | select(.[] == "LOG_LEVEL")) |= .value="DEBUG")' /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1393/deploy/cw-operator.yaml
+ kubectl_bin apply -n psmdb-operator -f -
++ mktemp
+ local LAST_OUT=/tmp/tmp.sDKv1G1MkB
++ mktemp
+ local LAST_ERR=/tmp/tmp.GKy22Jybwc
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl apply -n psmdb-operator -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.sDKv1G1MkB
deployment.apps/percona-server-mongodb-operator created
+ cat /tmp/tmp.GKy22Jybwc
+ rm /tmp/tmp.sDKv1G1MkB /tmp/tmp.GKy22Jybwc
+ return 0
+ sleep 20
++ get_operator_pod
++ kubectl_bin get pods --selector=name=percona-server-mongodb-operator -o 'jsonpath={.items[].metadata.name}' -n psmdb-operator
+++ mktemp
++ local LAST_OUT=/tmp/tmp.yrIzmkxHeA
+++ mktemp
++ local LAST_ERR=/tmp/tmp.YcpIXiHrZQ
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=percona-server-mongodb-operator -o 'jsonpath={.items[].metadata.name}' -n psmdb-operator
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.yrIzmkxHeA
++ cat /tmp/tmp.YcpIXiHrZQ
++ rm /tmp/tmp.yrIzmkxHeA /tmp/tmp.YcpIXiHrZQ
++ return 0
+ wait_operator_pod percona-server-mongodb-operator-dcddb45fb-wp9d7
+ local pod=percona-server-mongodb-operator-dcddb45fb-wp9d7
+ set +o xtrace
waiting for pod/percona-server-mongodb-operator-dcddb45fb-wp9d7 to be ready.OK
+ echo 'Print operator info from log'
Print operator info from log
+ grep 'Manager starting up'
++ get_operator_pod
++ kubectl_bin get pods --selector=name=percona-server-mongodb-operator -o 'jsonpath={.items[].metadata.name}' -n psmdb-operator
+++ mktemp
++ local LAST_OUT=/tmp/tmp.n2b4Z6I6QW
+++ mktemp
++ local LAST_ERR=/tmp/tmp.tbWForuccp
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=percona-server-mongodb-operator -o 'jsonpath={.items[].metadata.name}' -n psmdb-operator
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.n2b4Z6I6QW
++ cat /tmp/tmp.tbWForuccp
++ rm /tmp/tmp.n2b4Z6I6QW /tmp/tmp.tbWForuccp
++ return 0
+ kubectl_bin logs -n psmdb-operator percona-server-mongodb-operator-dcddb45fb-wp9d7
++ mktemp
+ local LAST_OUT=/tmp/tmp.UUg5R6Kr1D
++ mktemp
+ local LAST_ERR=/tmp/tmp.dhe6aJ66Dj
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl logs -n psmdb-operator percona-server-mongodb-operator-dcddb45fb-wp9d7
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.UUg5R6Kr1D
+ cat /tmp/tmp.dhe6aJ66Dj
+ rm /tmp/tmp.UUg5R6Kr1D /tmp/tmp.dhe6aJ66Dj
+ return 0
2025-12-17T16:28:24.343Z	INFO	setup	Manager starting up	{"gitCommit": "371bdb64640f297e36461f9cff6152f74a37a7bb", "gitBranch": "PR-1393-371bdb64", "buildTime": "", "goVersion": "go1.25.5", "os": "linux", "arch": "amd64"}
+ create_namespace pitr-physical-backup-source-4895
+ local namespace=pitr-physical-backup-source-4895
+ local skip_clean_namespace=
+ [[ 1 == 1 ]]
+ [[ -z '' ]]
+ destroy_chaos_mesh
++ tail -n1
++ helm list --all-namespaces --filter chaos-mesh
++ awk '-F ' '{print $2}'
++ sed s/NAMESPACE//
+ local chaos_mesh_ns=
+ desc 'destroy chaos-mesh'
+ set +o xtrace


-----------------------------------------------------------------------------------
destroy chaos-mesh
-----------------------------------------------------------------------------------

+ '[' -n '' ']'
++ kubectl get MutatingWebhookConfiguration
++ grep chaos-mesh
++ awk '{print $1}'
+ timeout 30 kubectl delete MutatingWebhookConfiguration
error: resource(s) were provided, but no name was specified
+ :
++ kubectl get ValidatingWebhookConfiguration
++ grep chaos-mesh
++ awk '{print $1}'
+ timeout 30 kubectl delete ValidatingWebhookConfiguration
error: resource(s) were provided, but no name was specified
+ :
++ kubectl get ValidatingWebhookConfiguration
++ grep validate-auth
++ awk '{print $1}'
+ timeout 30 kubectl delete ValidatingWebhookConfiguration
error: resource(s) were provided, but no name was specified
+ :
++ kubectl api-resources
++ grep chaos-mesh
++ awk '{print $1}'
++ kubectl get crd
++ grep chaos-mesh.org
++ awk '{print $1}'
+ timeout 30 kubectl delete crd
error: resource(s) were provided, but no name was specified
+ :
++ kubectl get clusterrolebinding
++ grep chaos-mesh
++ awk '{print $1}'
+ timeout 30 kubectl delete clusterrolebinding
error: resource(s) were provided, but no name was specified
+ :
++ kubectl get clusterrole
++ grep chaos-mesh
++ awk '{print $1}'
+ timeout 30 kubectl delete clusterrole
error: resource(s) were provided, but no name was specified
+ :
+ desc 'cleaned up all old namespaces'
+ set +o xtrace


-----------------------------------------------------------------------------------
cleaned up all old namespaces
-----------------------------------------------------------------------------------

+ kubectl_bin get ns
+ egrep -v '^kube-|^default|Terminating|psmdb-operator|openshift|^gke-|^gmp-|^NAME'
+ awk '{print$1}'
+ '[' -n '' ']'
+ desc 'cleaned up old namespaces pitr-physical-backup-source-4895'
+ set +o xtrace


-----------------------------------------------------------------------------------
cleaned up old namespaces pitr-physical-backup-source-4895
-----------------------------------------------------------------------------------

++ mktemp
+ kubectl_bin delete namespace pitr-physical-backup-source-4895 --ignore-not-found
+ xargs kubectl delete ns
egrep: warning: egrep is obsolescent; using grep -E
++ mktemp
+ local LAST_OUT=/tmp/tmp.PmIrpNX417
++ mktemp
+ local LAST_OUT=/tmp/tmp.FSal1zGITj
++ mktemp
+ local LAST_ERR=/tmp/tmp.umxbeLwxQi
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ local LAST_ERR=/tmp/tmp.cQqLX8yvZK
+ local exit_status=0
+ local timeout=4
+ for i in $(seq 0 2)
+ set +e
+ kubectl get ns
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl delete namespace pitr-physical-backup-source-4895 --ignore-not-found
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.PmIrpNX417
+ cat /tmp/tmp.umxbeLwxQi
+ rm /tmp/tmp.PmIrpNX417 /tmp/tmp.umxbeLwxQi
+ return 0
error: resource(s) were provided, but no name was specified
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.FSal1zGITj
+ cat /tmp/tmp.cQqLX8yvZK
+ rm /tmp/tmp.FSal1zGITj /tmp/tmp.cQqLX8yvZK
+ return 0
+ kubectl_bin wait --for=delete namespace pitr-physical-backup-source-4895
++ mktemp
+ local LAST_OUT=/tmp/tmp.fFxLu84k7P
++ mktemp
+ local LAST_ERR=/tmp/tmp.rTMzwdGfxW
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl wait --for=delete namespace pitr-physical-backup-source-4895
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.fFxLu84k7P
+ cat /tmp/tmp.rTMzwdGfxW
+ rm /tmp/tmp.fFxLu84k7P /tmp/tmp.rTMzwdGfxW
+ return 0
+ desc 'create namespace pitr-physical-backup-source-4895'
+ set +o xtrace


-----------------------------------------------------------------------------------
create namespace pitr-physical-backup-source-4895
-----------------------------------------------------------------------------------

+ kubectl_bin create namespace pitr-physical-backup-source-4895
++ mktemp
+ local LAST_OUT=/tmp/tmp.yW6fZH72jw
++ mktemp
+ local LAST_ERR=/tmp/tmp.JQskBA8IWN
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl create namespace pitr-physical-backup-source-4895
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.yW6fZH72jw
namespace/pitr-physical-backup-source-4895 created
+ cat /tmp/tmp.JQskBA8IWN
+ rm /tmp/tmp.yW6fZH72jw /tmp/tmp.JQskBA8IWN
+ return 0
+ set_kube_ctx pitr-physical-backup-source-4895
+ local namespace=pitr-physical-backup-source-4895
++ kubectl_bin config current-context
+++ mktemp
++ local LAST_OUT=/tmp/tmp.lnJzK3fUq7
+++ mktemp
++ local LAST_ERR=/tmp/tmp.1hmoEBE9NQ
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl config current-context
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.lnJzK3fUq7
++ cat /tmp/tmp.1hmoEBE9NQ
++ rm /tmp/tmp.lnJzK3fUq7 /tmp/tmp.1hmoEBE9NQ
++ return 0
+ kubectl_bin config set-context gke_cloud-dev-112233_us-central1-a_jen-psmdb-1393-371bdb64-32-cluster3 --namespace=pitr-physical-backup-source-4895
++ mktemp
+ local LAST_OUT=/tmp/tmp.vJqsF6IQvj
++ mktemp
+ local LAST_ERR=/tmp/tmp.02Mf3zlpGO
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl config set-context gke_cloud-dev-112233_us-central1-a_jen-psmdb-1393-371bdb64-32-cluster3 --namespace=pitr-physical-backup-source-4895
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.vJqsF6IQvj
Context "gke_cloud-dev-112233_us-central1-a_jen-psmdb-1393-371bdb64-32-cluster3" modified.
+ cat /tmp/tmp.02Mf3zlpGO
+ rm /tmp/tmp.vJqsF6IQvj /tmp/tmp.02Mf3zlpGO
+ return 0
+ deploy_minio
+ desc 'install Minio'
+ set +o xtrace


-----------------------------------------------------------------------------------
install Minio
-----------------------------------------------------------------------------------

+ helm uninstall minio-service
Error: uninstall: Release not loaded: minio-service: release: not found
+ :
+ helm repo remove minio
"minio" has been removed from your repositories
+ helm repo add minio https://charts.min.io/
"minio" has been added to your repositories
+ retry 10 60 helm install minio-service --version 5.4.0 --set replicas=1 --set mode=standalone --set resources.requests.memory=256Mi --set rootUser=rootuser --set rootPassword=rootpass123 --set 'users[0].accessKey=some-access-key' --set 'users[0].secretKey=some-secret-key' --set 'users[0].policy=consoleAdmin' --set service.type=ClusterIP --set configPathmc=/tmp/.minio/ --set persistence.size=2G --set securityContext.enabled=false minio/minio
+ local max=10
+ local delay=60
+ shift 2
+ local n=1
+ helm install minio-service --version 5.4.0 --set replicas=1 --set mode=standalone --set resources.requests.memory=256Mi --set rootUser=rootuser --set rootPassword=rootpass123 --set 'users[0].accessKey=some-access-key' --set 'users[0].secretKey=some-secret-key' --set 'users[0].policy=consoleAdmin' --set service.type=ClusterIP --set configPathmc=/tmp/.minio/ --set persistence.size=2G --set securityContext.enabled=false minio/minio
NAME: minio-service
LAST DEPLOYED: Wed Dec 17 16:29:01 2025
NAMESPACE: pitr-physical-backup-source-4895
STATUS: deployed
REVISION: 1
TEST SUITE: None
NOTES:
MinIO can be accessed via port 9000 on the following DNS name from within your cluster:
minio-service.pitr-physical-backup-source-4895.cluster.local

To access MinIO from localhost, run the below commands:

  1. export POD_NAME=$(kubectl get pods --namespace pitr-physical-backup-source-4895 -l "release=minio-service" -o jsonpath="{.items[0].metadata.name}")

  2. kubectl port-forward $POD_NAME 9000 --namespace pitr-physical-backup-source-4895

Read more about port forwarding here: http://kubernetes.io/docs/user-guide/kubectl/kubectl_port-forward/

You can now access MinIO server on http://localhost:9000. Follow the below steps to connect to MinIO server with mc client:

  1. Download the MinIO mc client - https://min.io/docs/minio/linux/reference/minio-mc.html#quickstart

  2. export MC_HOST_minio-service-local=http://$(kubectl get secret --namespace pitr-physical-backup-source-4895 minio-service -o jsonpath="{.data.rootUser}" | base64 --decode):$(kubectl get secret --namespace pitr-physical-backup-source-4895 minio-service -o jsonpath="{.data.rootPassword}" | base64 --decode)@localhost:9000

  3. mc ls minio-service-local
++ kubectl_bin get pods --selector=release=minio-service -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.7KKzmZySZa
+++ mktemp
++ local LAST_ERR=/tmp/tmp.wzorvGOTNt
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=release=minio-service -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.7KKzmZySZa
++ cat /tmp/tmp.wzorvGOTNt
++ rm /tmp/tmp.7KKzmZySZa /tmp/tmp.wzorvGOTNt
++ return 0
+ MINIO_POD=minio-service-d9589b474-2vvlg
+ wait_pod minio-service-d9589b474-2vvlg
+ local pod=minio-service-d9589b474-2vvlg
+ set +o xtrace
waiting for pod/minio-service-d9589b474-2vvlg to be ready.OK
+ '[' -n psmdb-operator ']'
+ kubectl_bin create svc -n psmdb-operator externalname minio-service --external-name=minio-service.pitr-physical-backup-source-4895.svc.cluster.local --tcp=9000
++ mktemp
+ local LAST_OUT=/tmp/tmp.poZ8h5qUcv
++ mktemp
+ local LAST_ERR=/tmp/tmp.SkvKi8WuW5
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl create svc -n psmdb-operator externalname minio-service --external-name=minio-service.pitr-physical-backup-source-4895.svc.cluster.local --tcp=9000
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.poZ8h5qUcv
service/minio-service created
+ cat /tmp/tmp.SkvKi8WuW5
+ rm /tmp/tmp.poZ8h5qUcv /tmp/tmp.SkvKi8WuW5
+ return 0
+ create_minio_bucket operator-testing
+ local bucket=operator-testing
+ kubectl_bin run -i --rm aws-cli --image=perconalab/awscli --restart=Never -- bash -c 'AWS_ACCESS_KEY_ID=some-access-key 		AWS_SECRET_ACCESS_KEY=some-secret-key 		AWS_DEFAULT_REGION=us-east-1         /usr/bin/aws --endpoint-url http://minio-service:9000 s3 mb s3://operator-testing'
++ mktemp
+ local LAST_OUT=/tmp/tmp.JlJxJZZJHp
++ mktemp
+ local LAST_ERR=/tmp/tmp.ekqjQ8gmpX
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl run -i --rm aws-cli --image=perconalab/awscli --restart=Never -- bash -c 'AWS_ACCESS_KEY_ID=some-access-key 		AWS_SECRET_ACCESS_KEY=some-secret-key 		AWS_DEFAULT_REGION=us-east-1         /usr/bin/aws --endpoint-url http://minio-service:9000 s3 mb s3://operator-testing'
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.JlJxJZZJHp
pod "aws-cli" deleted from pitr-physical-backup-source-4895 namespace
+ cat /tmp/tmp.ekqjQ8gmpX
All commands and output from this session will be recorded in container logs, including credentials and sensitive information passed through the command prompt.
If you don't see a command prompt, try pressing enter.
+ rm /tmp/tmp.JlJxJZZJHp /tmp/tmp.ekqjQ8gmpX
+ return 0
+ desc 'create secrets and start client'
+ set +o xtrace


-----------------------------------------------------------------------------------
create secrets and start client
-----------------------------------------------------------------------------------

+ kubectl_bin apply -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1393/e2e-tests/conf/secrets.yml -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1393/e2e-tests/conf/client.yml -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1393/e2e-tests/conf/minio-secret.yml
++ mktemp
+ local LAST_OUT=/tmp/tmp.dSzBkKBQYa
++ mktemp
+ local LAST_ERR=/tmp/tmp.rIHxdCbUHd
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl apply -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1393/e2e-tests/conf/secrets.yml -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1393/e2e-tests/conf/client.yml -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1393/e2e-tests/conf/minio-secret.yml
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.dSzBkKBQYa
secret/some-users created
deployment.apps/psmdb-client created
secret/minio-secret created
+ cat /tmp/tmp.rIHxdCbUHd
+ rm /tmp/tmp.dSzBkKBQYa /tmp/tmp.rIHxdCbUHd
+ return 0
+ cluster=some-name
+ desc 'create first PSMDB cluster some-name'
+ set +o xtrace


-----------------------------------------------------------------------------------
create first PSMDB cluster some-name
-----------------------------------------------------------------------------------

+ apply_cluster /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1393/e2e-tests/pitr-physical-backup-source/conf/some-name-rs0.yml
+ '[' -z '' ']'
+ cat_config /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1393/e2e-tests/pitr-physical-backup-source/conf/some-name-rs0.yml
+ kubectl_bin apply -f -
+ cat /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1393/e2e-tests/pitr-physical-backup-source/conf/some-name-rs0.yml
+ yq eval '(.spec | select(.image == null)).image = "perconalab/percona-server-mongodb-operator:main-mongod8.0"'
++ mktemp
+ yq eval '(.spec | select(has("pmm"))).pmm.image = "percona/pmm-client:2.44.1-1"'
+ yq eval '(.spec | select(has("initImage"))).initImage = "perconalab/percona-server-mongodb-operator:PR-1393-371bdb64"'
+ yq eval '(.spec | select(has("backup"))).backup.image = "perconalab/percona-server-mongodb-operator:main-backup"'
+ local LAST_OUT=/tmp/tmp.eZkCbKptE8
+ yq eval '.spec.upgradeOptions.apply="Never"'
++ mktemp
+ local LAST_ERR=/tmp/tmp.jpN2JJWOlx
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl apply -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.eZkCbKptE8
perconaservermongodb.psmdb.percona.com/some-name created
+ cat /tmp/tmp.jpN2JJWOlx
+ rm /tmp/tmp.eZkCbKptE8 /tmp/tmp.jpN2JJWOlx
+ return 0
+ desc 'check if all 3 Pods started'
+ set +o xtrace


-----------------------------------------------------------------------------------
check if all 3 Pods started
-----------------------------------------------------------------------------------

+ wait_for_running some-name-rs0 3
+ local name=some-name-rs0
+ let last_pod=2
+ local check_cluster_readyness=true
+ set_debug
+ [[ 1 == 1 ]]
+ set -o xtrace
+ local rs_name=rs0
+ local cluster_name=some-name
++ seq 0 2
+ for i in $(seq 0 $last_pod)
+ [[ 0 -eq 2 ]]
+ wait_pod some-name-rs0-0
+ local pod=some-name-rs0-0
+ set +o xtrace
waiting for pod/some-name-rs0-0 to be ready......OK
+ for i in $(seq 0 $last_pod)
+ [[ 1 -eq 2 ]]
+ wait_pod some-name-rs0-1
+ local pod=some-name-rs0-1
+ set +o xtrace
waiting for pod/some-name-rs0-1 to be ready.....OK
+ for i in $(seq 0 $last_pod)
+ [[ 2 -eq 2 ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].arbiter.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.FNf6kPuWYc
+++ mktemp
++ local LAST_ERR=/tmp/tmp.AUiRNFsdHD
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].arbiter.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.FNf6kPuWYc
++ cat /tmp/tmp.AUiRNFsdHD
++ rm /tmp/tmp.FNf6kPuWYc /tmp/tmp.AUiRNFsdHD
++ return 0
+ [[ '' == \t\r\u\e ]]
+ wait_pod some-name-rs0-2
+ local pod=some-name-rs0-2
+ set +o xtrace
waiting for pod/some-name-rs0-2 to be ready....OK
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].nonvoting.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.cYBkUQ0DQU
+++ mktemp
++ local LAST_ERR=/tmp/tmp.h6UTGFfNGV
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].nonvoting.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.cYBkUQ0DQU
++ cat /tmp/tmp.h6UTGFfNGV
++ rm /tmp/tmp.cYBkUQ0DQU /tmp/tmp.h6UTGFfNGV
++ return 0
+ [[ '' == \t\r\u\e ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].hidden.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.Fdk6jppVIH
+++ mktemp
++ local LAST_ERR=/tmp/tmp.i2xzqwEMMu
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].hidden.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.Fdk6jppVIH
++ cat /tmp/tmp.i2xzqwEMMu
++ rm /tmp/tmp.Fdk6jppVIH /tmp/tmp.i2xzqwEMMu
++ return 0
+ [[ '' == \t\r\u\e ]]
+ sleep 10
+ [[ true == \t\r\u\e ]]
+ set +x
Waiting for cluster readyness..............................................
+ wait_for_running some-name-cfg 3 false
+ local name=some-name-cfg
+ let last_pod=2
+ local check_cluster_readyness=false
+ set_debug
+ [[ 1 == 1 ]]
+ set -o xtrace
+ local rs_name=cfg
+ local cluster_name=some-name
++ seq 0 2
+ for i in $(seq 0 $last_pod)
+ [[ 0 -eq 2 ]]
+ wait_pod some-name-cfg-0
+ local pod=some-name-cfg-0
+ set +o xtrace
waiting for pod/some-name-cfg-0 to be ready.OK
+ for i in $(seq 0 $last_pod)
+ [[ 1 -eq 2 ]]
+ wait_pod some-name-cfg-1
+ local pod=some-name-cfg-1
+ set +o xtrace
waiting for pod/some-name-cfg-1 to be ready.OK
+ for i in $(seq 0 $last_pod)
+ [[ 2 -eq 2 ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].arbiter.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.g3u4z0wt4M
+++ mktemp
++ local LAST_ERR=/tmp/tmp.hNv7oaOvyf
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].arbiter.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.g3u4z0wt4M
++ cat /tmp/tmp.hNv7oaOvyf
++ rm /tmp/tmp.g3u4z0wt4M /tmp/tmp.hNv7oaOvyf
++ return 0
+ [[ '' == \t\r\u\e ]]
+ wait_pod some-name-cfg-2
+ local pod=some-name-cfg-2
+ set +o xtrace
waiting for pod/some-name-cfg-2 to be ready.OK
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].nonvoting.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.cAxnKnBNbq
+++ mktemp
++ local LAST_ERR=/tmp/tmp.mnuM9DwX7f
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].nonvoting.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.cAxnKnBNbq
++ cat /tmp/tmp.mnuM9DwX7f
++ rm /tmp/tmp.cAxnKnBNbq /tmp/tmp.mnuM9DwX7f
++ return 0
+ [[ '' == \t\r\u\e ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].hidden.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.b5k7StDbQS
+++ mktemp
++ local LAST_ERR=/tmp/tmp.DzcqJ1bwnj
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].hidden.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.b5k7StDbQS
++ cat /tmp/tmp.DzcqJ1bwnj
++ rm /tmp/tmp.b5k7StDbQS /tmp/tmp.DzcqJ1bwnj
++ return 0
+ [[ '' == \t\r\u\e ]]
+ sleep 10
+ [[ false == \t\r\u\e ]]
+ sleep 10
+ write_initial_data
+ desc 'create user myApp'
+ set +o xtrace


-----------------------------------------------------------------------------------
create user myApp
-----------------------------------------------------------------------------------

+ run_mongos 'db.createUser({user:"myApp",pwd:"myPass",roles:[{db:"myApp",role:"readWrite"}]})' userAdmin:userAdmin123456@some-name-mongos.pitr-physical-backup-source-4895
+ local 'command=db.createUser({user:"myApp",pwd:"myPass",roles:[{db:"myApp",role:"readWrite"}]})'
+ local uri=userAdmin:userAdmin123456@some-name-mongos.pitr-physical-backup-source-4895
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local port=27017
+ local mongo_bin=mongo
++ echo .svc.cluster.local
++ awk -F: '{print $2}'
+ suffix_port=
+ [[ -z '' ]]
+ suffix=.svc.cluster.local:27017
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.fPMVc92BmD
+++ mktemp
++ local LAST_ERR=/tmp/tmp.cQGKSdsDf3
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.fPMVc92BmD
++ cat /tmp/tmp.cQGKSdsDf3
++ rm /tmp/tmp.fPMVc92BmD /tmp/tmp.cQGKSdsDf3
++ return 0
+ local client_container=psmdb-client-696897d69b-7fp87
+ kubectl_bin exec psmdb-client-696897d69b-7fp87 -- bash -c 'printf '\''db.createUser({user:"myApp",pwd:"myPass",roles:[{db:"myApp",role:"readWrite"}]})\n'\'' | mongo mongodb://userAdmin:userAdmin123456@some-name-mongos.pitr-physical-backup-source-4895.svc.cluster.local:27017/admin '
++ mktemp
+ local LAST_OUT=/tmp/tmp.u1SgktnTiN
++ mktemp
+ local LAST_ERR=/tmp/tmp.qpR8EQUF1Z
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-696897d69b-7fp87 -- bash -c 'printf '\''db.createUser({user:"myApp",pwd:"myPass",roles:[{db:"myApp",role:"readWrite"}]})\n'\'' | mongo mongodb://userAdmin:userAdmin123456@some-name-mongos.pitr-physical-backup-source-4895.svc.cluster.local:27017/admin '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.u1SgktnTiN
Percona Server for MongoDB shell version v4.4.29-28
connecting to: mongodb://some-name-mongos.pitr-physical-backup-source-4895.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb
Implicit session: session { "id" : UUID("83c4835d-a8b1-48cf-9c2c-4798510a20ae") }
Percona Server for MongoDB server version: v8.0.16-5
WARNING: shell and server versions do not match
Successfully added user: {
	"user" : "myApp",
	"roles" : [
		{
			"db" : "myApp",
			"role" : "readWrite"
		}
	]
}
bye
+ cat /tmp/tmp.qpR8EQUF1Z
+ rm /tmp/tmp.u1SgktnTiN /tmp/tmp.qpR8EQUF1Z
+ return 0
+ sleep 2
+ write_document
+ local cmp_postfix=
+ local sleep_value=0
+ log 'write initial data, read from all'
+ set +o xtrace
[2025-12-17T16:33:11+0000] write initial data, read from all
+ run_mongos 'use myApp\n db.test.insert({ x: 100500 })' myApp:myPass@some-name-mongos.pitr-physical-backup-source-4895
+ local 'command=use myApp\n db.test.insert({ x: 100500 })'
+ local uri=myApp:myPass@some-name-mongos.pitr-physical-backup-source-4895
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local port=27017
+ local mongo_bin=mongo
++ echo .svc.cluster.local
++ awk -F: '{print $2}'
+ suffix_port=
+ [[ -z '' ]]
+ suffix=.svc.cluster.local:27017
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.948l4dse3O
+++ mktemp
++ local LAST_ERR=/tmp/tmp.PPn6Ha8YW7
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.948l4dse3O
++ cat /tmp/tmp.PPn6Ha8YW7
++ rm /tmp/tmp.948l4dse3O /tmp/tmp.PPn6Ha8YW7
++ return 0
+ local client_container=psmdb-client-696897d69b-7fp87
+ kubectl_bin exec psmdb-client-696897d69b-7fp87 -- bash -c 'printf '\''use myApp\n db.test.insert({ x: 100500 })\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.pitr-physical-backup-source-4895.svc.cluster.local:27017/admin '
++ mktemp
+ local LAST_OUT=/tmp/tmp.si53fr01p4
++ mktemp
+ local LAST_ERR=/tmp/tmp.CO59Q9NdZA
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-696897d69b-7fp87 -- bash -c 'printf '\''use myApp\n db.test.insert({ x: 100500 })\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.pitr-physical-backup-source-4895.svc.cluster.local:27017/admin '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.si53fr01p4
Percona Server for MongoDB shell version v4.4.29-28
connecting to: mongodb://some-name-mongos.pitr-physical-backup-source-4895.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb
Implicit session: session { "id" : UUID("0e1cda88-c1fb-4162-8d7b-ac286207f46a") }
Percona Server for MongoDB server version: v8.0.16-5
WARNING: shell and server versions do not match
switched to db myApp
WriteResult({ "nInserted" : 1 })
bye
+ cat /tmp/tmp.CO59Q9NdZA
+ rm /tmp/tmp.si53fr01p4 /tmp/tmp.CO59Q9NdZA
+ return 0
+ sleep 0
+ compare_mongos_cmd find myApp:myPass@some-name-mongos.pitr-physical-backup-source-4895
+ local command=find
+ local uri=myApp:myPass@some-name-mongos.pitr-physical-backup-source-4895
+ local postfix=
+ local suffix=
+ local database=myApp
+ local collection=test
+ local port=27017
+ local tls=false
+ [[ false == \t\r\u\e ]]
+ mongos_command=run_mongos
+ run_mongos 'use myApp\n db.test.find()' myApp:myPass@some-name-mongos.pitr-physical-backup-source-4895 mongodb '' '' 27017
+ local 'command=use myApp\n db.test.find()'
+ local uri=myApp:myPass@some-name-mongos.pitr-physical-backup-source-4895
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ egrep -v 'I NETWORK|W NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|Started a new thread for the timer service'
+ local mongo_flag=
+ local port=27017
+ local mongo_bin=mongo
+ /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
++ echo .svc.cluster.local
++ awk -F: '{print $2}'
egrep: warning: egrep is obsolescent; using grep -E
+ suffix_port=
+ [[ -z '' ]]
+ suffix=.svc.cluster.local:27017
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.HGTzCg4zIM
+++ mktemp
++ local LAST_ERR=/tmp/tmp.53vVEbYNbA
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.HGTzCg4zIM
++ cat /tmp/tmp.53vVEbYNbA
++ rm /tmp/tmp.HGTzCg4zIM /tmp/tmp.53vVEbYNbA
++ return 0
+ local client_container=psmdb-client-696897d69b-7fp87
+ kubectl_bin exec psmdb-client-696897d69b-7fp87 -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.pitr-physical-backup-source-4895.svc.cluster.local:27017/admin '
++ mktemp
+ local LAST_OUT=/tmp/tmp.60zJEadF7L
++ mktemp
+ local LAST_ERR=/tmp/tmp.OjMjJXeCw0
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-696897d69b-7fp87 -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.pitr-physical-backup-source-4895.svc.cluster.local:27017/admin '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.60zJEadF7L
+ cat /tmp/tmp.OjMjJXeCw0
+ rm /tmp/tmp.60zJEadF7L /tmp/tmp.OjMjJXeCw0
+ return 0
+ [[ 0 -eq 0 ]]
+ diff /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1393/e2e-tests/pitr-physical-backup-source/compare/find.json /tmp/tmp.XdTK1TTa6I/find
+ wait_backup_agent some-name-rs0-0
+ local agent_pod=some-name-rs0-0
+ set +o xtrace
waiting for pbm-agent to be ready in some-name-rs0-0...2025-12-17T16:32:47.000+0000 I listening for the commands

+ wait_backup_agent some-name-rs0-1
+ local agent_pod=some-name-rs0-1
+ set +o xtrace
waiting for pbm-agent to be ready in some-name-rs0-1...2025-12-17T16:32:47.000+0000 I listening for the commands

+ wait_backup_agent some-name-rs0-2
+ local agent_pod=some-name-rs0-2
+ set +o xtrace
waiting for pbm-agent to be ready in some-name-rs0-2...2025-12-17T16:32:48.000+0000 I listening for the commands

+ wait_backup_agent some-name-rs1-0
+ local agent_pod=some-name-rs1-0
+ set +o xtrace
waiting for pbm-agent to be ready in some-name-rs1-0...2025-12-17T16:32:49.000+0000 I listening for the commands

+ wait_backup_agent some-name-rs1-1
+ local agent_pod=some-name-rs1-1
+ set +o xtrace
waiting for pbm-agent to be ready in some-name-rs1-1...2025-12-17T16:32:46.000+0000 I listening for the commands

+ wait_backup_agent some-name-rs1-2
+ local agent_pod=some-name-rs1-2
+ set +o xtrace
waiting for pbm-agent to be ready in some-name-rs1-2...2025-12-17T16:32:49.000+0000 I listening for the commands

+ wait_backup_agent some-name-rs2-0
+ local agent_pod=some-name-rs2-0
+ set +o xtrace
waiting for pbm-agent to be ready in some-name-rs2-0...2025-12-17T16:32:46.000+0000 I listening for the commands

+ wait_backup_agent some-name-rs2-1
+ local agent_pod=some-name-rs2-1
+ set +o xtrace
waiting for pbm-agent to be ready in some-name-rs2-1...2025-12-17T16:32:47.000+0000 I listening for the commands

+ wait_backup_agent some-name-rs2-2
+ local agent_pod=some-name-rs2-2
+ set +o xtrace
waiting for pbm-agent to be ready in some-name-rs2-2...2025-12-17T16:32:50.000+0000 I listening for the commands

+ echo 'Sleeping for 360 seconds'
Sleeping for 360 seconds
+ sleep 360
+ backup_name_minio=backup-minio
+ desc 'restore pitr type date using backupSource'
+ set +o xtrace


-----------------------------------------------------------------------------------
restore pitr type date using backupSource
-----------------------------------------------------------------------------------

+ run_backup backup-minio 1 physical
+ local name=backup-minio
+ local idx=1
+ local type=physical
+ desc 'run backup backup-minio-1'
+ set +o xtrace


-----------------------------------------------------------------------------------
run backup backup-minio-1
-----------------------------------------------------------------------------------

+ cat /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1393/e2e-tests/pitr-physical-backup-source/conf/backup-minio.yml
+ /usr/sbin/sed -e 's/name:/name: backup-minio-1/'
+ /usr/sbin/sed -e 's/type:/type: physical/'
+ kubectl_bin apply -f -
++ mktemp
+ local LAST_OUT=/tmp/tmp.Q3e30oTOT7
++ mktemp
+ local LAST_ERR=/tmp/tmp.cH6xbYjwhz
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl apply -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.Q3e30oTOT7
perconaservermongodbbackup.psmdb.percona.com/backup-minio-1 created
+ cat /tmp/tmp.cH6xbYjwhz
+ rm /tmp/tmp.Q3e30oTOT7 /tmp/tmp.cH6xbYjwhz
+ return 0
+ wait_backup backup-minio-1
+ local backup_name=backup-minio-1
+ local target_state=ready
+ set +o xtrace
waiting for backup-minio-1 to reach ready state..........OK
+ compare_latest_restorable_time some-name-rs0 backup-minio-1
+ local cluster=some-name-rs0
+ local backup_name=backup-minio-1
+ local latest_restorable_time
+ local backup_time
++ get_latest_restorable_time some-name-rs0
++ local cluster=some-name-rs0
++ local first_timestamp
++ local second_timestamp
++ local retry=0
++ [[ '' != '' ]]
+++ kubectl_bin exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ jq '.backups.pitrChunks.pitrChunks | last | .range.end'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.SO2FCyuoVM
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.G8FyDdElHh
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in $(seq 0 2)
+++ set +e
+++ kubectl exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.SO2FCyuoVM
+++ cat /tmp/tmp.G8FyDdElHh
+++ rm /tmp/tmp.SO2FCyuoVM /tmp/tmp.G8FyDdElHh
+++ return 0
++ first_timestamp=null
++ sleep 5
++ [[ null != '' ]]
++ [[ null != \n\u\l\l ]]
++ let retry+=1
++ [[ 1 -gt 30 ]]
++ [[ null != '' ]]
++ [[ null != \n\u\l\l ]]
+++ kubectl_bin exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ jq '.backups.pitrChunks.pitrChunks | last | .range.end'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.ciV0HWMhxI
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.kjFMclrdhU
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in $(seq 0 2)
+++ set +e
+++ kubectl exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.ciV0HWMhxI
+++ cat /tmp/tmp.kjFMclrdhU
+++ rm /tmp/tmp.ciV0HWMhxI /tmp/tmp.kjFMclrdhU
+++ return 0
++ first_timestamp=null
++ sleep 5
++ [[ null != '' ]]
++ [[ null != \n\u\l\l ]]
++ let retry+=1
++ [[ 2 -gt 30 ]]
++ [[ null != '' ]]
++ [[ null != \n\u\l\l ]]
+++ kubectl_bin exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ jq '.backups.pitrChunks.pitrChunks | last | .range.end'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.3Lu8hC1pGd
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.qH3btDeNC3
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in $(seq 0 2)
+++ set +e
+++ kubectl exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.3Lu8hC1pGd
+++ cat /tmp/tmp.qH3btDeNC3
+++ rm /tmp/tmp.3Lu8hC1pGd /tmp/tmp.qH3btDeNC3
+++ return 0
++ first_timestamp=null
++ sleep 5
++ [[ null != '' ]]
++ [[ null != \n\u\l\l ]]
++ let retry+=1
++ [[ 3 -gt 30 ]]
++ [[ null != '' ]]
++ [[ null != \n\u\l\l ]]
+++ kubectl_bin exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ jq '.backups.pitrChunks.pitrChunks | last | .range.end'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.5G2c8UhVpe
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.soiG8UaYEa
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in $(seq 0 2)
+++ set +e
+++ kubectl exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.5G2c8UhVpe
+++ cat /tmp/tmp.soiG8UaYEa
+++ rm /tmp/tmp.5G2c8UhVpe /tmp/tmp.soiG8UaYEa
+++ return 0
++ first_timestamp=null
++ sleep 5
++ [[ null != '' ]]
++ [[ null != \n\u\l\l ]]
++ let retry+=1
++ [[ 4 -gt 30 ]]
++ [[ null != '' ]]
++ [[ null != \n\u\l\l ]]
+++ kubectl_bin exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ jq '.backups.pitrChunks.pitrChunks | last | .range.end'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.vBQw7jFrOw
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.O2kkgRu86U
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in $(seq 0 2)
+++ set +e
+++ kubectl exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.vBQw7jFrOw
+++ cat /tmp/tmp.O2kkgRu86U
+++ rm /tmp/tmp.vBQw7jFrOw /tmp/tmp.O2kkgRu86U
+++ return 0
++ first_timestamp=1765989608
++ sleep 5
++ [[ 1765989608 != '' ]]
++ [[ 1765989608 != \n\u\l\l ]]
+++ kubectl_bin exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ jq '.backups.pitrChunks.pitrChunks | last | .range.end'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.zl0TTJUcvg
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.SX1Wh7BLTX
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in $(seq 0 2)
+++ set +e
+++ kubectl exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.zl0TTJUcvg
+++ cat /tmp/tmp.SX1Wh7BLTX
+++ rm /tmp/tmp.zl0TTJUcvg /tmp/tmp.SX1Wh7BLTX
+++ return 0
++ second_timestamp=1765989608
++ let retry+=1
++ [[ 5 -gt 30 ]]
++ [[ 1765989608 != '' ]]
++ [[ 1765989608 != \n\u\l\l ]]
++ [[ 1765989608 == 1765989608 ]]
++ /usr/sbin/date -u -d @1765989608 +%Y-%m-%dT%H:%M:%SZ
+ latest_restorable_time=2025-12-17T16:40:08Z
++ get_latest_restorable_time_from_backup_object backup-minio-1
++ local backup_name=backup-minio-1
++ local latestRestorableTime
++ local retry=0
++ [[ '' != '' ]]
++ sleep 5
+++ kubectl_bin get psmdb-backup backup-minio-1 -o 'jsonpath={.status.latestRestorableTime}'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.3kbo5M6s5Z
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.IZVkh2BPJS
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in $(seq 0 2)
+++ set +e
+++ kubectl get psmdb-backup backup-minio-1 -o 'jsonpath={.status.latestRestorableTime}'
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.3kbo5M6s5Z
+++ cat /tmp/tmp.IZVkh2BPJS
+++ rm /tmp/tmp.3kbo5M6s5Z /tmp/tmp.IZVkh2BPJS
+++ return 0
++ latestRestorableTime=2025-12-17T16:40:08Z
++ let retry+=1
++ [[ 1 -gt 30 ]]
++ [[ 2025-12-17T16:40:08Z != '' ]]
++ [[ 2025-12-17T16:40:08Z != \n\u\l\l ]]
++ echo 2025-12-17T16:40:08Z
+ backup_time=2025-12-17T16:40:08Z
+ [[ 2025-12-17T16:40:08Z != \2\0\2\5\-\1\2\-\1\7\T\1\6\:\4\0\:\0\8\Z ]]
+ reset_collection
+ desc 'reset data'
+ set +o xtrace


-----------------------------------------------------------------------------------
reset data
-----------------------------------------------------------------------------------

+ run_mongos 'use myApp\n db.test.remove({})' myApp:myPass@some-name-mongos.pitr-physical-backup-source-4895
+ local 'command=use myApp\n db.test.remove({})'
+ local uri=myApp:myPass@some-name-mongos.pitr-physical-backup-source-4895
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local port=27017
+ local mongo_bin=mongo
++ echo .svc.cluster.local
++ awk -F: '{print $2}'
+ suffix_port=
+ [[ -z '' ]]
+ suffix=.svc.cluster.local:27017
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.E2dhTIYq6q
+++ mktemp
++ local LAST_ERR=/tmp/tmp.G3CCkYVPTU
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.E2dhTIYq6q
++ cat /tmp/tmp.G3CCkYVPTU
++ rm /tmp/tmp.E2dhTIYq6q /tmp/tmp.G3CCkYVPTU
++ return 0
+ local client_container=psmdb-client-696897d69b-7fp87
+ kubectl_bin exec psmdb-client-696897d69b-7fp87 -- bash -c 'printf '\''use myApp\n db.test.remove({})\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.pitr-physical-backup-source-4895.svc.cluster.local:27017/admin '
++ mktemp
+ local LAST_OUT=/tmp/tmp.65eIR5FSEt
++ mktemp
+ local LAST_ERR=/tmp/tmp.oeDr8xEhXY
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-696897d69b-7fp87 -- bash -c 'printf '\''use myApp\n db.test.remove({})\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.pitr-physical-backup-source-4895.svc.cluster.local:27017/admin '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.65eIR5FSEt
Percona Server for MongoDB shell version v4.4.29-28
connecting to: mongodb://some-name-mongos.pitr-physical-backup-source-4895.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb
Implicit session: session { "id" : UUID("b7c3fc9a-73e8-4280-a442-914f44e991a1") }
Percona Server for MongoDB server version: v8.0.16-5
WARNING: shell and server versions do not match
switched to db myApp
WriteResult({ "nRemoved" : 1 })
bye
+ cat /tmp/tmp.oeDr8xEhXY
+ rm /tmp/tmp.65eIR5FSEt /tmp/tmp.oeDr8xEhXY
+ return 0
+ sleep 2
+ write_document '' 120
+ local cmp_postfix=
+ local sleep_value=120
+ log 'write initial data, read from all'
+ set +o xtrace
[2025-12-17T16:40:36+0000] write initial data, read from all
+ run_mongos 'use myApp\n db.test.insert({ x: 100500 })' myApp:myPass@some-name-mongos.pitr-physical-backup-source-4895
+ local 'command=use myApp\n db.test.insert({ x: 100500 })'
+ local uri=myApp:myPass@some-name-mongos.pitr-physical-backup-source-4895
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local port=27017
+ local mongo_bin=mongo
++ echo .svc.cluster.local
++ awk -F: '{print $2}'
+ suffix_port=
+ [[ -z '' ]]
+ suffix=.svc.cluster.local:27017
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.k6HQoXU05c
+++ mktemp
++ local LAST_ERR=/tmp/tmp.rDglFFMiPX
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.k6HQoXU05c
++ cat /tmp/tmp.rDglFFMiPX
++ rm /tmp/tmp.k6HQoXU05c /tmp/tmp.rDglFFMiPX
++ return 0
+ local client_container=psmdb-client-696897d69b-7fp87
+ kubectl_bin exec psmdb-client-696897d69b-7fp87 -- bash -c 'printf '\''use myApp\n db.test.insert({ x: 100500 })\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.pitr-physical-backup-source-4895.svc.cluster.local:27017/admin '
++ mktemp
+ local LAST_OUT=/tmp/tmp.jU3X0Sk2qj
++ mktemp
+ local LAST_ERR=/tmp/tmp.1W4eN4NoLz
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-696897d69b-7fp87 -- bash -c 'printf '\''use myApp\n db.test.insert({ x: 100500 })\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.pitr-physical-backup-source-4895.svc.cluster.local:27017/admin '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.jU3X0Sk2qj
Percona Server for MongoDB shell version v4.4.29-28
connecting to: mongodb://some-name-mongos.pitr-physical-backup-source-4895.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb
Implicit session: session { "id" : UUID("cbb64329-3a96-47b2-91f5-fed6a4e4c9de") }
Percona Server for MongoDB server version: v8.0.16-5
WARNING: shell and server versions do not match
switched to db myApp
WriteResult({ "nInserted" : 1 })
bye
+ cat /tmp/tmp.1W4eN4NoLz
+ rm /tmp/tmp.jU3X0Sk2qj /tmp/tmp.1W4eN4NoLz
+ return 0
+ sleep 120
+ compare_mongos_cmd find myApp:myPass@some-name-mongos.pitr-physical-backup-source-4895
+ local command=find
+ local uri=myApp:myPass@some-name-mongos.pitr-physical-backup-source-4895
+ local postfix=
+ local suffix=
+ local database=myApp
+ local collection=test
+ local port=27017
+ local tls=false
+ [[ false == \t\r\u\e ]]
+ mongos_command=run_mongos
+ run_mongos 'use myApp\n db.test.find()' myApp:myPass@some-name-mongos.pitr-physical-backup-source-4895 mongodb '' '' 27017
+ local 'command=use myApp\n db.test.find()'
+ local uri=myApp:myPass@some-name-mongos.pitr-physical-backup-source-4895
+ local driver=mongodb
+ egrep -v 'I NETWORK|W NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|Started a new thread for the timer service'
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local port=27017
+ local mongo_bin=mongo
+ /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
egrep: warning: egrep is obsolescent; using grep -E
++ echo .svc.cluster.local
++ awk -F: '{print $2}'
+ suffix_port=
+ [[ -z '' ]]
+ suffix=.svc.cluster.local:27017
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.LsNxqAA3Y2
+++ mktemp
++ local LAST_ERR=/tmp/tmp.RPp6Smr9Ht
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.LsNxqAA3Y2
++ cat /tmp/tmp.RPp6Smr9Ht
++ rm /tmp/tmp.LsNxqAA3Y2 /tmp/tmp.RPp6Smr9Ht
++ return 0
+ local client_container=psmdb-client-696897d69b-7fp87
+ kubectl_bin exec psmdb-client-696897d69b-7fp87 -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.pitr-physical-backup-source-4895.svc.cluster.local:27017/admin '
++ mktemp
+ local LAST_OUT=/tmp/tmp.una7YsLB3X
++ mktemp
+ local LAST_ERR=/tmp/tmp.TbkiXvtHjZ
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-696897d69b-7fp87 -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.pitr-physical-backup-source-4895.svc.cluster.local:27017/admin '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.una7YsLB3X
+ cat /tmp/tmp.TbkiXvtHjZ
+ rm /tmp/tmp.una7YsLB3X /tmp/tmp.TbkiXvtHjZ
+ return 0
+ [[ 0 -eq 0 ]]
+ diff /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1393/e2e-tests/pitr-physical-backup-source/compare/find.json /tmp/tmp.XdTK1TTa6I/find
++ run_mongos 'new Date().getTime() / 1000' myApp:myPass@some-name-mongos.pitr-physical-backup-source-4895 mongodb '' --quiet
++ local 'command=new Date().getTime() / 1000'
++ local uri=myApp:myPass@some-name-mongos.pitr-physical-backup-source-4895
++ local driver=mongodb
++ local suffix=.svc.cluster.local
++ local mongo_flag=--quiet
++ local port=27017
++ local mongo_bin=mongo
++ grep -E -v 'I NETWORK|W NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match'
++ cut -d. -f1
+++ echo .svc.cluster.local
+++ awk -F: '{print $2}'
++ suffix_port=
++ [[ -z '' ]]
++ suffix=.svc.cluster.local:27017
+++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.yX3QOYTDVg
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.oUE6SMToSQ
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in $(seq 0 2)
+++ set +e
+++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.yX3QOYTDVg
+++ cat /tmp/tmp.oUE6SMToSQ
+++ rm /tmp/tmp.yX3QOYTDVg /tmp/tmp.oUE6SMToSQ
+++ return 0
++ local client_container=psmdb-client-696897d69b-7fp87
++ kubectl_bin exec psmdb-client-696897d69b-7fp87 -- bash -c 'printf '\''new Date().getTime() / 1000\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.pitr-physical-backup-source-4895.svc.cluster.local:27017/admin --quiet'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.qxjwa5fMcV
+++ mktemp
++ local LAST_ERR=/tmp/tmp.9SPD2B9ggk
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl exec psmdb-client-696897d69b-7fp87 -- bash -c 'printf '\''new Date().getTime() / 1000\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.pitr-physical-backup-source-4895.svc.cluster.local:27017/admin --quiet'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.qxjwa5fMcV
++ cat /tmp/tmp.9SPD2B9ggk
++ rm /tmp/tmp.qxjwa5fMcV /tmp/tmp.9SPD2B9ggk
++ return 0
+ time_now=1765989763
+ check_recovery backup-minio-1 date 1765989763 '' some-name backupSource
+ local backup_name=backup-minio-1
+ local restore_type=date
+ local restore_date=1765989763
+ local cmp_postfix=
+ local cluster_name=some-name
+ local backupSource=backupSource
++ get_latest_oplog_chunk_ts some-name
++ local cluster=some-name
+++ kubectl_bin exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ jq '.backups.pitrChunks.pitrChunks | last | .range.end'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.nW01Yb84eX
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.1iSM20uc4h
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in $(seq 0 2)
+++ set +e
+++ kubectl exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.nW01Yb84eX
+++ cat /tmp/tmp.1iSM20uc4h
+++ rm /tmp/tmp.nW01Yb84eX /tmp/tmp.1iSM20uc4h
+++ return 0
++ echo 1765989737
+ local latest_ts=1765989737
+ desc 'write more data before restore by date'
+ set +o xtrace


-----------------------------------------------------------------------------------
write more data before restore by date
-----------------------------------------------------------------------------------

+ run_mongos 'use myApp\n db.test.insert({ x: 100501 })' myApp:myPass@some-name-mongos.pitr-physical-backup-source-4895
+ local 'command=use myApp\n db.test.insert({ x: 100501 })'
+ local uri=myApp:myPass@some-name-mongos.pitr-physical-backup-source-4895
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local port=27017
+ local mongo_bin=mongo
++ echo .svc.cluster.local
++ awk -F: '{print $2}'
+ suffix_port=
+ [[ -z '' ]]
+ suffix=.svc.cluster.local:27017
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.lX0MvhHath
+++ mktemp
++ local LAST_ERR=/tmp/tmp.6hZdOpPGkV
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.lX0MvhHath
++ cat /tmp/tmp.6hZdOpPGkV
++ rm /tmp/tmp.lX0MvhHath /tmp/tmp.6hZdOpPGkV
++ return 0
+ local client_container=psmdb-client-696897d69b-7fp87
+ kubectl_bin exec psmdb-client-696897d69b-7fp87 -- bash -c 'printf '\''use myApp\n db.test.insert({ x: 100501 })\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.pitr-physical-backup-source-4895.svc.cluster.local:27017/admin '
++ mktemp
+ local LAST_OUT=/tmp/tmp.wfo0gSXgPW
++ mktemp
+ local LAST_ERR=/tmp/tmp.WXhfEP48oe
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-696897d69b-7fp87 -- bash -c 'printf '\''use myApp\n db.test.insert({ x: 100501 })\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.pitr-physical-backup-source-4895.svc.cluster.local:27017/admin '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.wfo0gSXgPW
Percona Server for MongoDB shell version v4.4.29-28
connecting to: mongodb://some-name-mongos.pitr-physical-backup-source-4895.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb
Implicit session: session { "id" : UUID("04673585-996e-4e9f-a5f2-1b5c3ed95795") }
Percona Server for MongoDB server version: v8.0.16-5
WARNING: shell and server versions do not match
switched to db myApp
WriteResult({ "nInserted" : 1 })
bye
+ cat /tmp/tmp.WXhfEP48oe
+ rm /tmp/tmp.wfo0gSXgPW /tmp/tmp.WXhfEP48oe
+ return 0
+ [[ -n 1765989763 ]]
++ format_date 1765989763
++ local timestamp=1765989763
+++ TZ=UTC
+++ /usr/sbin/date -d@1765989763 '+%Y-%m-%d %H:%M:%S'
++ echo 2025-12-17 16:42:43
+ desc 'Restoring to time 2025-12-17 16:42:43'
+ set +o xtrace


-----------------------------------------------------------------------------------
Restoring to time 2025-12-17 16:42:43
-----------------------------------------------------------------------------------

+ retries=0
+ [[ 1765989737 -gt 1765989763 ]]
+ [[ 0 -gt 30 ]]
++ get_latest_oplog_chunk_ts some-name
++ local cluster=some-name
+++ kubectl_bin exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ jq '.backups.pitrChunks.pitrChunks | last | .range.end'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.K9CQlCk3yb
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.2PCdrWDufs
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in $(seq 0 2)
+++ set +e
+++ kubectl exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.K9CQlCk3yb
+++ cat /tmp/tmp.2PCdrWDufs
+++ rm /tmp/tmp.K9CQlCk3yb /tmp/tmp.2PCdrWDufs
+++ return 0
++ echo 1765989737
+ latest_ts=1765989737
+ retries=1
++ format_date 1765989737
++ local timestamp=1765989737
+++ TZ=UTC
+++ /usr/sbin/date -d@1765989737 '+%Y-%m-%d %H:%M:%S'
++ echo 2025-12-17 16:42:17
++ format_date 1765989763
++ local timestamp=1765989763
+++ TZ=UTC
+++ /usr/sbin/date -d@1765989763 '+%Y-%m-%d %H:%M:%S'
++ echo 2025-12-17 16:42:43
+ echo 'Waiting for last oplog chunk (2025-12-17 16:42:17) to be greater than restore target (2025-12-17 16:42:43)'
Waiting for last oplog chunk (2025-12-17 16:42:17) to be greater than restore target (2025-12-17 16:42:43)
+ sleep 10
+ [[ 1765989737 -gt 1765989763 ]]
+ [[ 1 -gt 30 ]]
++ get_latest_oplog_chunk_ts some-name
++ local cluster=some-name
+++ kubectl_bin exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ jq '.backups.pitrChunks.pitrChunks | last | .range.end'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.YlsN5iBkQ8
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.3vzOD1XRrb
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in $(seq 0 2)
+++ set +e
+++ kubectl exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.YlsN5iBkQ8
+++ cat /tmp/tmp.3vzOD1XRrb
+++ rm /tmp/tmp.YlsN5iBkQ8 /tmp/tmp.3vzOD1XRrb
+++ return 0
++ echo 1765989737
+ latest_ts=1765989737
+ retries=2
++ format_date 1765989737
++ local timestamp=1765989737
+++ TZ=UTC
+++ /usr/sbin/date -d@1765989737 '+%Y-%m-%d %H:%M:%S'
++ echo 2025-12-17 16:42:17
++ format_date 1765989763
++ local timestamp=1765989763
+++ TZ=UTC
+++ /usr/sbin/date -d@1765989763 '+%Y-%m-%d %H:%M:%S'
++ echo 2025-12-17 16:42:43
+ echo 'Waiting for last oplog chunk (2025-12-17 16:42:17) to be greater than restore target (2025-12-17 16:42:43)'
Waiting for last oplog chunk (2025-12-17 16:42:17) to be greater than restore target (2025-12-17 16:42:43)
+ sleep 10
+ [[ 1765989737 -gt 1765989763 ]]
+ [[ 2 -gt 30 ]]
++ get_latest_oplog_chunk_ts some-name
++ local cluster=some-name
+++ kubectl_bin exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ jq '.backups.pitrChunks.pitrChunks | last | .range.end'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.bQm3w1O9vr
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.hCAYXkHTZJ
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in $(seq 0 2)
+++ set +e
+++ kubectl exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.bQm3w1O9vr
+++ cat /tmp/tmp.hCAYXkHTZJ
+++ rm /tmp/tmp.bQm3w1O9vr /tmp/tmp.hCAYXkHTZJ
+++ return 0
++ echo 1765989737
+ latest_ts=1765989737
+ retries=3
++ format_date 1765989737
++ local timestamp=1765989737
+++ TZ=UTC
+++ /usr/sbin/date -d@1765989737 '+%Y-%m-%d %H:%M:%S'
++ echo 2025-12-17 16:42:17
++ format_date 1765989763
++ local timestamp=1765989763
+++ TZ=UTC
+++ /usr/sbin/date -d@1765989763 '+%Y-%m-%d %H:%M:%S'
++ echo 2025-12-17 16:42:43
+ echo 'Waiting for last oplog chunk (2025-12-17 16:42:17) to be greater than restore target (2025-12-17 16:42:43)'
Waiting for last oplog chunk (2025-12-17 16:42:17) to be greater than restore target (2025-12-17 16:42:43)
+ sleep 10
+ [[ 1765989737 -gt 1765989763 ]]
+ [[ 3 -gt 30 ]]
++ get_latest_oplog_chunk_ts some-name
++ local cluster=some-name
+++ kubectl_bin exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ jq '.backups.pitrChunks.pitrChunks | last | .range.end'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.Blzx4ta8hp
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.R71l9fiOjg
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in $(seq 0 2)
+++ set +e
+++ kubectl exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.Blzx4ta8hp
+++ cat /tmp/tmp.R71l9fiOjg
+++ rm /tmp/tmp.Blzx4ta8hp /tmp/tmp.R71l9fiOjg
+++ return 0
++ echo 1765989737
+ latest_ts=1765989737
+ retries=4
++ format_date 1765989737
++ local timestamp=1765989737
+++ TZ=UTC
+++ /usr/sbin/date -d@1765989737 '+%Y-%m-%d %H:%M:%S'
++ echo 2025-12-17 16:42:17
++ format_date 1765989763
++ local timestamp=1765989763
+++ TZ=UTC
+++ /usr/sbin/date -d@1765989763 '+%Y-%m-%d %H:%M:%S'
++ echo 2025-12-17 16:42:43
+ echo 'Waiting for last oplog chunk (2025-12-17 16:42:17) to be greater than restore target (2025-12-17 16:42:43)'
Waiting for last oplog chunk (2025-12-17 16:42:17) to be greater than restore target (2025-12-17 16:42:43)
+ sleep 10
+ [[ 1765989737 -gt 1765989763 ]]
+ [[ 4 -gt 30 ]]
++ get_latest_oplog_chunk_ts some-name
++ local cluster=some-name
+++ kubectl_bin exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ jq '.backups.pitrChunks.pitrChunks | last | .range.end'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.wL7qqVR7i9
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.gox6F64hbg
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in $(seq 0 2)
+++ set +e
+++ kubectl exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.wL7qqVR7i9
+++ cat /tmp/tmp.gox6F64hbg
+++ rm /tmp/tmp.wL7qqVR7i9 /tmp/tmp.gox6F64hbg
+++ return 0
++ echo 1765989737
+ latest_ts=1765989737
+ retries=5
++ format_date 1765989737
++ local timestamp=1765989737
+++ TZ=UTC
+++ /usr/sbin/date -d@1765989737 '+%Y-%m-%d %H:%M:%S'
++ echo 2025-12-17 16:42:17
++ format_date 1765989763
++ local timestamp=1765989763
+++ TZ=UTC
+++ /usr/sbin/date -d@1765989763 '+%Y-%m-%d %H:%M:%S'
++ echo 2025-12-17 16:42:43
+ echo 'Waiting for last oplog chunk (2025-12-17 16:42:17) to be greater than restore target (2025-12-17 16:42:43)'
Waiting for last oplog chunk (2025-12-17 16:42:17) to be greater than restore target (2025-12-17 16:42:43)
+ sleep 10
+ [[ 1765989737 -gt 1765989763 ]]
+ [[ 5 -gt 30 ]]
++ get_latest_oplog_chunk_ts some-name
++ local cluster=some-name
+++ kubectl_bin exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ jq '.backups.pitrChunks.pitrChunks | last | .range.end'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.yJHlLnZfvw
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.iJtyx4tYxI
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in $(seq 0 2)
+++ set +e
+++ kubectl exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.yJHlLnZfvw
+++ cat /tmp/tmp.iJtyx4tYxI
+++ rm /tmp/tmp.yJHlLnZfvw /tmp/tmp.iJtyx4tYxI
+++ return 0
++ echo 1765989737
+ latest_ts=1765989737
+ retries=6
++ format_date 1765989737
++ local timestamp=1765989737
+++ TZ=UTC
+++ /usr/sbin/date -d@1765989737 '+%Y-%m-%d %H:%M:%S'
++ echo 2025-12-17 16:42:17
++ format_date 1765989763
++ local timestamp=1765989763
+++ TZ=UTC
+++ /usr/sbin/date -d@1765989763 '+%Y-%m-%d %H:%M:%S'
++ echo 2025-12-17 16:42:43
+ echo 'Waiting for last oplog chunk (2025-12-17 16:42:17) to be greater than restore target (2025-12-17 16:42:43)'
Waiting for last oplog chunk (2025-12-17 16:42:17) to be greater than restore target (2025-12-17 16:42:43)
+ sleep 10
+ [[ 1765989737 -gt 1765989763 ]]
+ [[ 6 -gt 30 ]]
++ get_latest_oplog_chunk_ts some-name
++ local cluster=some-name
+++ kubectl_bin exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ jq '.backups.pitrChunks.pitrChunks | last | .range.end'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.RXl5oMkaZm
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.j1YQwVc4bX
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in $(seq 0 2)
+++ set +e
+++ kubectl exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.RXl5oMkaZm
+++ cat /tmp/tmp.j1YQwVc4bX
+++ rm /tmp/tmp.RXl5oMkaZm /tmp/tmp.j1YQwVc4bX
+++ return 0
++ echo 1765989737
+ latest_ts=1765989737
+ retries=7
++ format_date 1765989737
++ local timestamp=1765989737
+++ TZ=UTC
+++ /usr/sbin/date -d@1765989737 '+%Y-%m-%d %H:%M:%S'
++ echo 2025-12-17 16:42:17
++ format_date 1765989763
++ local timestamp=1765989763
+++ TZ=UTC
+++ /usr/sbin/date -d@1765989763 '+%Y-%m-%d %H:%M:%S'
++ echo 2025-12-17 16:42:43
+ echo 'Waiting for last oplog chunk (2025-12-17 16:42:17) to be greater than restore target (2025-12-17 16:42:43)'
Waiting for last oplog chunk (2025-12-17 16:42:17) to be greater than restore target (2025-12-17 16:42:43)
+ sleep 10
+ [[ 1765989737 -gt 1765989763 ]]
+ [[ 7 -gt 30 ]]
++ get_latest_oplog_chunk_ts some-name
++ local cluster=some-name
+++ kubectl_bin exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ jq '.backups.pitrChunks.pitrChunks | last | .range.end'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.zz85SOCX8p
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.wyzPOrEkFt
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in $(seq 0 2)
+++ set +e
+++ kubectl exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.zz85SOCX8p
+++ cat /tmp/tmp.wyzPOrEkFt
+++ rm /tmp/tmp.zz85SOCX8p /tmp/tmp.wyzPOrEkFt
+++ return 0
++ echo 1765989737
+ latest_ts=1765989737
+ retries=8
++ format_date 1765989737
++ local timestamp=1765989737
+++ TZ=UTC
+++ /usr/sbin/date -d@1765989737 '+%Y-%m-%d %H:%M:%S'
++ echo 2025-12-17 16:42:17
++ format_date 1765989763
++ local timestamp=1765989763
+++ TZ=UTC
+++ /usr/sbin/date -d@1765989763 '+%Y-%m-%d %H:%M:%S'
++ echo 2025-12-17 16:42:43
+ echo 'Waiting for last oplog chunk (2025-12-17 16:42:17) to be greater than restore target (2025-12-17 16:42:43)'
Waiting for last oplog chunk (2025-12-17 16:42:17) to be greater than restore target (2025-12-17 16:42:43)
+ sleep 10
+ [[ 1765989737 -gt 1765989763 ]]
+ [[ 8 -gt 30 ]]
++ get_latest_oplog_chunk_ts some-name
++ local cluster=some-name
+++ kubectl_bin exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ jq '.backups.pitrChunks.pitrChunks | last | .range.end'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.EgaSI5GEXc
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.AiMd2OsAR7
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in $(seq 0 2)
+++ set +e
+++ kubectl exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.EgaSI5GEXc
+++ cat /tmp/tmp.AiMd2OsAR7
+++ rm /tmp/tmp.EgaSI5GEXc /tmp/tmp.AiMd2OsAR7
+++ return 0
++ echo 1765989738
+ latest_ts=1765989738
+ retries=9
++ format_date 1765989738
++ local timestamp=1765989738
+++ TZ=UTC
+++ /usr/sbin/date -d@1765989738 '+%Y-%m-%d %H:%M:%S'
++ echo 2025-12-17 16:42:18
++ format_date 1765989763
++ local timestamp=1765989763
+++ TZ=UTC
+++ /usr/sbin/date -d@1765989763 '+%Y-%m-%d %H:%M:%S'
++ echo 2025-12-17 16:42:43
+ echo 'Waiting for last oplog chunk (2025-12-17 16:42:18) to be greater than restore target (2025-12-17 16:42:43)'
Waiting for last oplog chunk (2025-12-17 16:42:18) to be greater than restore target (2025-12-17 16:42:43)
+ sleep 10
+ [[ 1765989738 -gt 1765989763 ]]
+ [[ 9 -gt 30 ]]
++ get_latest_oplog_chunk_ts some-name
++ local cluster=some-name
+++ kubectl_bin exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ jq '.backups.pitrChunks.pitrChunks | last | .range.end'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.2FFAuc3qgo
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.Rctb7qt02D
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in $(seq 0 2)
+++ set +e
+++ kubectl exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.2FFAuc3qgo
+++ cat /tmp/tmp.Rctb7qt02D
+++ rm /tmp/tmp.2FFAuc3qgo /tmp/tmp.Rctb7qt02D
+++ return 0
++ echo 1765989857
+ latest_ts=1765989857
+ retries=10
++ format_date 1765989857
++ local timestamp=1765989857
+++ TZ=UTC
+++ /usr/sbin/date -d@1765989857 '+%Y-%m-%d %H:%M:%S'
++ echo 2025-12-17 16:44:17
++ format_date 1765989763
++ local timestamp=1765989763
+++ TZ=UTC
+++ /usr/sbin/date -d@1765989763 '+%Y-%m-%d %H:%M:%S'
++ echo 2025-12-17 16:42:43
+ echo 'Waiting for last oplog chunk (2025-12-17 16:44:17) to be greater than restore target (2025-12-17 16:42:43)'
Waiting for last oplog chunk (2025-12-17 16:44:17) to be greater than restore target (2025-12-17 16:42:43)
+ sleep 10
+ [[ 1765989857 -gt 1765989763 ]]
+ '[' -z backupSource ']'
+ desc 'check restore by date backupSource'
+ set +o xtrace


-----------------------------------------------------------------------------------
check restore by date backupSource
-----------------------------------------------------------------------------------

++ get_backup_dest backup-minio-1
++ local backup_name=backup-minio-1
++ kubectl_bin get psmdb-backup backup-minio-1 -o 'jsonpath={.status.destination}'
++ sed -e 's/.json$//'
++ sed 's|s3://||'
+++ mktemp
++ sed 's|azure://||'
++ sed 's|gs://||'
++ local LAST_OUT=/tmp/tmp.rn0CUWZZk0
+++ mktemp
++ local LAST_ERR=/tmp/tmp.cUWObEPMGT
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb-backup backup-minio-1 -o 'jsonpath={.status.destination}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.rn0CUWZZk0
++ cat /tmp/tmp.cUWObEPMGT
++ rm /tmp/tmp.rn0CUWZZk0 /tmp/tmp.cUWObEPMGT
++ return 0
+ backup_dest=operator-testing/2025-12-17T16:39:32Z
+ cat /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1393/e2e-tests/pitr-physical-backup-source/conf/restore.yml
+ /usr/sbin/sed -e 's/name:/name: restore-backup-minio-1/'
+ /usr/sbin/sed -e /backupName/d
+ /usr/sbin/sed -e 's/pitrType:/type: date/'
+ /usr/sbin/sed -e 's|DESTINATION|operator-testing/2025-12-17T16:39:32Z|'
+ '[' -z 1765989763 ']'
++ format_date 1765989763
+ '[' -n '' ']'
++ local timestamp=1765989763
+ yq
+ kubectl_bin apply -f -
+++ TZ=UTC
+++ /usr/sbin/date -d@1765989763 '+%Y-%m-%d %H:%M:%S'
++ get_bucket_name backup-minio-1
++ local backup_name=backup-minio-1
++ kubectl_bin get psmdb-backup backup-minio-1 -o 'jsonpath={.status.s3.bucket}'
++ mktemp
+++ mktemp
++ echo 2025-12-17 16:42:43
+ local LAST_OUT=/tmp/tmp.8nnK6pyjtn
++ local LAST_OUT=/tmp/tmp.i0MaH2qMKO
+ /usr/sbin/sed -e 's/date:/date: 2025-12-17 16:42:43/'
++ mktemp
+++ mktemp
++ local LAST_ERR=/tmp/tmp.39gUin7lA2
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
+ local LAST_ERR=/tmp/tmp.BhQMnlrkdP
+ local exit_status=0
+ local timeout=4
++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb-backup backup-minio-1 -o 'jsonpath={.status.s3.bucket}'
+ for i in $(seq 0 2)
+ set +e
+ kubectl apply -f -
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.i0MaH2qMKO
++ cat /tmp/tmp.39gUin7lA2
++ rm /tmp/tmp.i0MaH2qMKO /tmp/tmp.39gUin7lA2
++ return 0
+ /usr/sbin/sed -e 's|BUCKET-NAME|operator-testing|'
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.8nnK6pyjtn
perconaservermongodbrestore.psmdb.percona.com/restore-backup-minio-1 created
+ cat /tmp/tmp.BhQMnlrkdP
+ rm /tmp/tmp.8nnK6pyjtn /tmp/tmp.BhQMnlrkdP
+ return 0
+ wait_restore backup-minio-1 some-name requested 0 1200
+ local backup_name=backup-minio-1
+ local cluster_name=some-name
+ local target_state=requested
+ local wait_cluster_consistency=0
+ local wait_time=1200
+ local ok_if_ready=0
+ set +o xtrace
Waiting for the psmdb-restore/restore-backup-minio-1 object to be created.OK
Waiting psmdb-restore/restore-backup-minio-1 to reach state "requested" .........OK after 8 minutes
+ [[ 0 -eq 1 ]]
+ echo

+ wait_restore backup-minio-1 some-name ready 0 1600
+ local backup_name=backup-minio-1
+ local cluster_name=some-name
+ local target_state=ready
+ local wait_cluster_consistency=0
+ local wait_time=1600
+ local ok_if_ready=0
+ set +o xtrace
Waiting for the psmdb-restore/restore-backup-minio-1 object to be created.OK
Waiting psmdb-restore/restore-backup-minio-1 to reach state "ready" ...OK after 2 minutes
+ [[ 0 -eq 1 ]]
+ echo

+ set -o xtrace
+ wait_for_running some-name-rs0 3
+ local name=some-name-rs0
+ let last_pod=2
+ local check_cluster_readyness=true
+ set_debug
+ [[ 1 == 1 ]]
+ set -o xtrace
+ local rs_name=rs0
+ local cluster_name=some-name
++ seq 0 2
+ for i in $(seq 0 $last_pod)
+ [[ 0 -eq 2 ]]
+ wait_pod some-name-rs0-0
+ local pod=some-name-rs0-0
+ set +o xtrace
waiting for pod/some-name-rs0-0 to be ready.OK
+ for i in $(seq 0 $last_pod)
+ [[ 1 -eq 2 ]]
+ wait_pod some-name-rs0-1
+ local pod=some-name-rs0-1
+ set +o xtrace
waiting for pod/some-name-rs0-1 to be ready.OK
+ for i in $(seq 0 $last_pod)
+ [[ 2 -eq 2 ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].arbiter.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.ouW04skSYF
+++ mktemp
++ local LAST_ERR=/tmp/tmp.UO6TtnTD0P
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].arbiter.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.ouW04skSYF
++ cat /tmp/tmp.UO6TtnTD0P
++ rm /tmp/tmp.ouW04skSYF /tmp/tmp.UO6TtnTD0P
++ return 0
+ [[ '' == \t\r\u\e ]]
+ wait_pod some-name-rs0-2
+ local pod=some-name-rs0-2
+ set +o xtrace
waiting for pod/some-name-rs0-2 to be ready.OK
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].nonvoting.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.JkhdpOKMtE
+++ mktemp
++ local LAST_ERR=/tmp/tmp.jGsTYVAApX
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].nonvoting.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.JkhdpOKMtE
++ cat /tmp/tmp.jGsTYVAApX
++ rm /tmp/tmp.JkhdpOKMtE /tmp/tmp.jGsTYVAApX
++ return 0
+ [[ '' == \t\r\u\e ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].hidden.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.Tmn5GfcYHD
+++ mktemp
++ local LAST_ERR=/tmp/tmp.btYHhzZrok
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].hidden.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.Tmn5GfcYHD
++ cat /tmp/tmp.btYHhzZrok
++ rm /tmp/tmp.Tmn5GfcYHD /tmp/tmp.btYHhzZrok
++ return 0
+ [[ '' == \t\r\u\e ]]
+ sleep 10
+ [[ true == \t\r\u\e ]]
+ set +x
Waiting for cluster readyness.................................................
+ wait_for_running some-name-cfg 3
+ local name=some-name-cfg
+ let last_pod=2
+ local check_cluster_readyness=true
+ set_debug
+ [[ 1 == 1 ]]
+ set -o xtrace
+ local rs_name=cfg
+ local cluster_name=some-name
++ seq 0 2
+ for i in $(seq 0 $last_pod)
+ [[ 0 -eq 2 ]]
+ wait_pod some-name-cfg-0
+ local pod=some-name-cfg-0
+ set +o xtrace
waiting for pod/some-name-cfg-0 to be ready.OK
+ for i in $(seq 0 $last_pod)
+ [[ 1 -eq 2 ]]
+ wait_pod some-name-cfg-1
+ local pod=some-name-cfg-1
+ set +o xtrace
waiting for pod/some-name-cfg-1 to be ready.OK
+ for i in $(seq 0 $last_pod)
+ [[ 2 -eq 2 ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].arbiter.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.S0HYTSTsWA
+++ mktemp
++ local LAST_ERR=/tmp/tmp.I1ZgtVzHAg
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].arbiter.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.S0HYTSTsWA
++ cat /tmp/tmp.I1ZgtVzHAg
++ rm /tmp/tmp.S0HYTSTsWA /tmp/tmp.I1ZgtVzHAg
++ return 0
+ [[ '' == \t\r\u\e ]]
+ wait_pod some-name-cfg-2
+ local pod=some-name-cfg-2
+ set +o xtrace
waiting for pod/some-name-cfg-2 to be ready.OK
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].nonvoting.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.dxhR1Pl779
+++ mktemp
++ local LAST_ERR=/tmp/tmp.k2uEbMdzMw
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].nonvoting.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.dxhR1Pl779
++ cat /tmp/tmp.k2uEbMdzMw
++ rm /tmp/tmp.dxhR1Pl779 /tmp/tmp.k2uEbMdzMw
++ return 0
+ [[ '' == \t\r\u\e ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].hidden.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.6P6RxFrAMU
+++ mktemp
++ local LAST_ERR=/tmp/tmp.GTVE2ZBU3c
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].hidden.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.6P6RxFrAMU
++ cat /tmp/tmp.GTVE2ZBU3c
++ rm /tmp/tmp.6P6RxFrAMU /tmp/tmp.GTVE2ZBU3c
++ return 0
+ [[ '' == \t\r\u\e ]]
+ sleep 10
+ [[ true == \t\r\u\e ]]
+ set +x
Waiting for cluster readyness
+ wait_for_running some-name-mongos 3
+ local name=some-name-mongos
+ let last_pod=2
+ local check_cluster_readyness=true
+ set_debug
+ [[ 1 == 1 ]]
+ set -o xtrace
+ local rs_name=mongos
+ local cluster_name=some-name
++ seq 0 2
+ for i in $(seq 0 $last_pod)
+ [[ 0 -eq 2 ]]
+ wait_pod some-name-mongos-0
+ local pod=some-name-mongos-0
+ set +o xtrace
waiting for pod/some-name-mongos-0 to be ready.OK
+ for i in $(seq 0 $last_pod)
+ [[ 1 -eq 2 ]]
+ wait_pod some-name-mongos-1
+ local pod=some-name-mongos-1
+ set +o xtrace
waiting for pod/some-name-mongos-1 to be ready.OK
+ for i in $(seq 0 $last_pod)
+ [[ 2 -eq 2 ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="mongos")].arbiter.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.NNVgGSMgBd
+++ mktemp
++ local LAST_ERR=/tmp/tmp.iC3WMQGSyd
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="mongos")].arbiter.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.NNVgGSMgBd
++ cat /tmp/tmp.iC3WMQGSyd
++ rm /tmp/tmp.NNVgGSMgBd /tmp/tmp.iC3WMQGSyd
++ return 0
+ [[ '' == \t\r\u\e ]]
+ wait_pod some-name-mongos-2
+ local pod=some-name-mongos-2
+ set +o xtrace
waiting for pod/some-name-mongos-2 to be ready.OK
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="mongos")].nonvoting.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.dWRkNHczCM
+++ mktemp
++ local LAST_ERR=/tmp/tmp.SygGB1KsSy
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="mongos")].nonvoting.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.dWRkNHczCM
++ cat /tmp/tmp.SygGB1KsSy
++ rm /tmp/tmp.dWRkNHczCM /tmp/tmp.SygGB1KsSy
++ return 0
+ [[ '' == \t\r\u\e ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="mongos")].hidden.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.2axQqPDxYa
+++ mktemp
++ local LAST_ERR=/tmp/tmp.VK5WAcQMK4
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="mongos")].hidden.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.2axQqPDxYa
++ cat /tmp/tmp.VK5WAcQMK4
++ rm /tmp/tmp.2axQqPDxYa /tmp/tmp.VK5WAcQMK4
++ return 0
+ [[ '' == \t\r\u\e ]]
+ sleep 10
+ [[ true == \t\r\u\e ]]
+ set +x
Waiting for cluster readyness
+ sleep 10
+ compare_mongos_cmd find myApp:myPass@some-name-mongos.pitr-physical-backup-source-4895 ''
+ local command=find
+ local uri=myApp:myPass@some-name-mongos.pitr-physical-backup-source-4895
+ local postfix=
+ local suffix=
+ local database=myApp
+ local collection=test
+ local port=27017
+ local tls=false
+ [[ false == \t\r\u\e ]]
+ mongos_command=run_mongos
+ run_mongos 'use myApp\n db.test.find()' myApp:myPass@some-name-mongos.pitr-physical-backup-source-4895 mongodb '' '' 27017
+ egrep -v 'I NETWORK|W NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|Started a new thread for the timer service'
+ local 'command=use myApp\n db.test.find()'
+ local uri=myApp:myPass@some-name-mongos.pitr-physical-backup-source-4895
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local port=27017
+ local mongo_bin=mongo
+ /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
++ echo .svc.cluster.local
++ awk -F: '{print $2}'
egrep: warning: egrep is obsolescent; using grep -E
+ suffix_port=
+ [[ -z '' ]]
+ suffix=.svc.cluster.local:27017
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.ch8Y0BBn6f
+++ mktemp
++ local LAST_ERR=/tmp/tmp.AALq7kU9kV
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.ch8Y0BBn6f
++ cat /tmp/tmp.AALq7kU9kV
++ rm /tmp/tmp.ch8Y0BBn6f /tmp/tmp.AALq7kU9kV
++ return 0
+ local client_container=psmdb-client-696897d69b-7fp87
+ kubectl_bin exec psmdb-client-696897d69b-7fp87 -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.pitr-physical-backup-source-4895.svc.cluster.local:27017/admin '
++ mktemp
+ local LAST_OUT=/tmp/tmp.Ym4UUmDN7p
++ mktemp
+ local LAST_ERR=/tmp/tmp.JXr8u6QmfA
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-696897d69b-7fp87 -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.pitr-physical-backup-source-4895.svc.cluster.local:27017/admin '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.Ym4UUmDN7p
+ cat /tmp/tmp.JXr8u6QmfA
+ rm /tmp/tmp.Ym4UUmDN7p /tmp/tmp.JXr8u6QmfA
+ return 0
+ [[ 0 -eq 0 ]]
+ diff /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1393/e2e-tests/pitr-physical-backup-source/compare/find.json /tmp/tmp.XdTK1TTa6I/find
+ desc 'delete PSMDB cluster some-name'
+ set +o xtrace


-----------------------------------------------------------------------------------
delete PSMDB cluster some-name
-----------------------------------------------------------------------------------

+ kubectl_bin delete psmdb some-name
++ mktemp
+ local LAST_OUT=/tmp/tmp.Am7tCUPdEh
++ mktemp
+ local LAST_ERR=/tmp/tmp.xCK8V9yNcu
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl delete psmdb some-name
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.Am7tCUPdEh
perconaservermongodb.psmdb.percona.com "some-name" deleted from pitr-physical-backup-source-4895 namespace
+ cat /tmp/tmp.xCK8V9yNcu
+ rm /tmp/tmp.Am7tCUPdEh /tmp/tmp.xCK8V9yNcu
+ return 0
+ kubectl_bin delete pvc -l app.kubernetes.io/managed-by=percona-server-mongodb-operator
++ mktemp
+ local LAST_OUT=/tmp/tmp.8uTDRx9PWT
++ mktemp
+ local LAST_ERR=/tmp/tmp.hXNVoyHMFC
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl delete pvc -l app.kubernetes.io/managed-by=percona-server-mongodb-operator
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.8uTDRx9PWT
persistentvolumeclaim "mongod-data-some-name-cfg-0" deleted from pitr-physical-backup-source-4895 namespace
persistentvolumeclaim "mongod-data-some-name-cfg-1" deleted from pitr-physical-backup-source-4895 namespace
persistentvolumeclaim "mongod-data-some-name-cfg-2" deleted from pitr-physical-backup-source-4895 namespace
persistentvolumeclaim "mongod-data-some-name-rs0-0" deleted from pitr-physical-backup-source-4895 namespace
persistentvolumeclaim "mongod-data-some-name-rs0-1" deleted from pitr-physical-backup-source-4895 namespace
persistentvolumeclaim "mongod-data-some-name-rs0-2" deleted from pitr-physical-backup-source-4895 namespace
persistentvolumeclaim "mongod-data-some-name-rs1-0" deleted from pitr-physical-backup-source-4895 namespace
persistentvolumeclaim "mongod-data-some-name-rs1-1" deleted from pitr-physical-backup-source-4895 namespace
persistentvolumeclaim "mongod-data-some-name-rs1-2" deleted from pitr-physical-backup-source-4895 namespace
persistentvolumeclaim "mongod-data-some-name-rs2-0" deleted from pitr-physical-backup-source-4895 namespace
persistentvolumeclaim "mongod-data-some-name-rs2-1" deleted from pitr-physical-backup-source-4895 namespace
persistentvolumeclaim "mongod-data-some-name-rs2-2" deleted from pitr-physical-backup-source-4895 namespace
+ cat /tmp/tmp.hXNVoyHMFC
+ rm /tmp/tmp.8uTDRx9PWT /tmp/tmp.hXNVoyHMFC
+ return 0
+ sleep 10
+ desc 'recreate PSMDB cluster some-name'
+ set +o xtrace


-----------------------------------------------------------------------------------
recreate PSMDB cluster some-name
-----------------------------------------------------------------------------------

+ desc 'create second PSMDB cluster some-name'
+ set +o xtrace


-----------------------------------------------------------------------------------
create second PSMDB cluster some-name
-----------------------------------------------------------------------------------

+ apply_cluster /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1393/e2e-tests/pitr-physical-backup-source/conf/some-name-rs0.yml
+ '[' -z '' ']'
+ cat_config /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1393/e2e-tests/pitr-physical-backup-source/conf/some-name-rs0.yml
+ kubectl_bin apply -f -
+ cat /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1393/e2e-tests/pitr-physical-backup-source/conf/some-name-rs0.yml
++ mktemp
+ yq eval '(.spec | select(.image == null)).image = "perconalab/percona-server-mongodb-operator:main-mongod8.0"'
+ yq eval '(.spec | select(has("pmm"))).pmm.image = "percona/pmm-client:2.44.1-1"'
+ yq eval '(.spec | select(has("initImage"))).initImage = "perconalab/percona-server-mongodb-operator:PR-1393-371bdb64"'
+ yq eval '(.spec | select(has("backup"))).backup.image = "perconalab/percona-server-mongodb-operator:main-backup"'
+ yq eval '.spec.upgradeOptions.apply="Never"'
+ local LAST_OUT=/tmp/tmp.pbFNsV7oAi
++ mktemp
+ local LAST_ERR=/tmp/tmp.3Gc6xCjHly
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl apply -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.pbFNsV7oAi
perconaservermongodb.psmdb.percona.com/some-name created
+ cat /tmp/tmp.3Gc6xCjHly
+ rm /tmp/tmp.pbFNsV7oAi /tmp/tmp.3Gc6xCjHly
+ return 0
+ desc 'check if all 3 Pods started'
+ set +o xtrace


-----------------------------------------------------------------------------------
check if all 3 Pods started
-----------------------------------------------------------------------------------

+ wait_for_running some-name-rs0 3
+ local name=some-name-rs0
+ let last_pod=2
+ local check_cluster_readyness=true
+ set_debug
+ [[ 1 == 1 ]]
+ set -o xtrace
+ local rs_name=rs0
+ local cluster_name=some-name
++ seq 0 2
+ for i in $(seq 0 $last_pod)
+ [[ 0 -eq 2 ]]
+ wait_pod some-name-rs0-0
+ local pod=some-name-rs0-0
+ set +o xtrace
waiting for pod/some-name-rs0-0 to be ready.....OK
+ for i in $(seq 0 $last_pod)
+ [[ 1 -eq 2 ]]
+ wait_pod some-name-rs0-1
+ local pod=some-name-rs0-1
+ set +o xtrace
waiting for pod/some-name-rs0-1 to be ready......OK
+ for i in $(seq 0 $last_pod)
+ [[ 2 -eq 2 ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].arbiter.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.gfID2GyP7B
+++ mktemp
++ local LAST_ERR=/tmp/tmp.QCI3POKAgB
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].arbiter.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.gfID2GyP7B
++ cat /tmp/tmp.QCI3POKAgB
++ rm /tmp/tmp.gfID2GyP7B /tmp/tmp.QCI3POKAgB
++ return 0
+ [[ '' == \t\r\u\e ]]
+ wait_pod some-name-rs0-2
+ local pod=some-name-rs0-2
+ set +o xtrace
waiting for pod/some-name-rs0-2 to be ready.......OK
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].nonvoting.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.hxV1L3BhBI
+++ mktemp
++ local LAST_ERR=/tmp/tmp.fpqdHOPZE6
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].nonvoting.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.hxV1L3BhBI
++ cat /tmp/tmp.fpqdHOPZE6
++ rm /tmp/tmp.hxV1L3BhBI /tmp/tmp.fpqdHOPZE6
++ return 0
+ [[ '' == \t\r\u\e ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].hidden.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.TfU0T3eaPW
+++ mktemp
++ local LAST_ERR=/tmp/tmp.kPBYzHZLv8
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].hidden.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.TfU0T3eaPW
++ cat /tmp/tmp.kPBYzHZLv8
++ rm /tmp/tmp.TfU0T3eaPW /tmp/tmp.kPBYzHZLv8
++ return 0
+ [[ '' == \t\r\u\e ]]
+ sleep 10
+ [[ true == \t\r\u\e ]]
+ set +x
Waiting for cluster readyness...................................................
+ wait_for_running some-name-cfg 3 false
+ local name=some-name-cfg
+ let last_pod=2
+ local check_cluster_readyness=false
+ set_debug
+ [[ 1 == 1 ]]
+ set -o xtrace
+ local rs_name=cfg
+ local cluster_name=some-name
++ seq 0 2
+ for i in $(seq 0 $last_pod)
+ [[ 0 -eq 2 ]]
+ wait_pod some-name-cfg-0
+ local pod=some-name-cfg-0
+ set +o xtrace
waiting for pod/some-name-cfg-0 to be ready.OK
+ for i in $(seq 0 $last_pod)
+ [[ 1 -eq 2 ]]
+ wait_pod some-name-cfg-1
+ local pod=some-name-cfg-1
+ set +o xtrace
waiting for pod/some-name-cfg-1 to be ready.OK
+ for i in $(seq 0 $last_pod)
+ [[ 2 -eq 2 ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].arbiter.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.Mt2NfJnHfq
+++ mktemp
++ local LAST_ERR=/tmp/tmp.MmpjvHg3I1
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].arbiter.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.Mt2NfJnHfq
++ cat /tmp/tmp.MmpjvHg3I1
++ rm /tmp/tmp.Mt2NfJnHfq /tmp/tmp.MmpjvHg3I1
++ return 0
+ [[ '' == \t\r\u\e ]]
+ wait_pod some-name-cfg-2
+ local pod=some-name-cfg-2
+ set +o xtrace
waiting for pod/some-name-cfg-2 to be ready.OK
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].nonvoting.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.6J1lZhxUQn
+++ mktemp
++ local LAST_ERR=/tmp/tmp.qGnzAw1Pht
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].nonvoting.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.6J1lZhxUQn
++ cat /tmp/tmp.qGnzAw1Pht
++ rm /tmp/tmp.6J1lZhxUQn /tmp/tmp.qGnzAw1Pht
++ return 0
+ [[ '' == \t\r\u\e ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].hidden.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.4QNvPlBQhi
+++ mktemp
++ local LAST_ERR=/tmp/tmp.Y6da1d9hpk
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].hidden.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.4QNvPlBQhi
++ cat /tmp/tmp.Y6da1d9hpk
++ rm /tmp/tmp.4QNvPlBQhi /tmp/tmp.Y6da1d9hpk
++ return 0
+ [[ '' == \t\r\u\e ]]
+ sleep 10
+ [[ false == \t\r\u\e ]]
+ wait_for_running some-name-rs1 3
+ local name=some-name-rs1
+ let last_pod=2
+ local check_cluster_readyness=true
+ set_debug
+ [[ 1 == 1 ]]
+ set -o xtrace
+ local rs_name=rs1
+ local cluster_name=some-name
++ seq 0 2
+ for i in $(seq 0 $last_pod)
+ [[ 0 -eq 2 ]]
+ wait_pod some-name-rs1-0
+ local pod=some-name-rs1-0
+ set +o xtrace
waiting for pod/some-name-rs1-0 to be ready.OK
+ for i in $(seq 0 $last_pod)
+ [[ 1 -eq 2 ]]
+ wait_pod some-name-rs1-1
+ local pod=some-name-rs1-1
+ set +o xtrace
waiting for pod/some-name-rs1-1 to be ready.OK
+ for i in $(seq 0 $last_pod)
+ [[ 2 -eq 2 ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs1")].arbiter.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.iVUZbfR1zd
+++ mktemp
++ local LAST_ERR=/tmp/tmp.0GlvKju4ER
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs1")].arbiter.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.iVUZbfR1zd
++ cat /tmp/tmp.0GlvKju4ER
++ rm /tmp/tmp.iVUZbfR1zd /tmp/tmp.0GlvKju4ER
++ return 0
+ [[ '' == \t\r\u\e ]]
+ wait_pod some-name-rs1-2
+ local pod=some-name-rs1-2
+ set +o xtrace
waiting for pod/some-name-rs1-2 to be ready.OK
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs1")].nonvoting.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.S1WU70mKaj
+++ mktemp
++ local LAST_ERR=/tmp/tmp.pU866jQGNl
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs1")].nonvoting.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.S1WU70mKaj
++ cat /tmp/tmp.pU866jQGNl
++ rm /tmp/tmp.S1WU70mKaj /tmp/tmp.pU866jQGNl
++ return 0
+ [[ '' == \t\r\u\e ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs1")].hidden.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.m9qtH2tTbq
+++ mktemp
++ local LAST_ERR=/tmp/tmp.3p5EyJVSAC
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs1")].hidden.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.m9qtH2tTbq
++ cat /tmp/tmp.3p5EyJVSAC
++ rm /tmp/tmp.m9qtH2tTbq /tmp/tmp.3p5EyJVSAC
++ return 0
+ [[ '' == \t\r\u\e ]]
+ sleep 10
+ [[ true == \t\r\u\e ]]
+ set +x
Waiting for cluster readyness
+ wait_for_running some-name-rs2 3
+ local name=some-name-rs2
+ let last_pod=2
+ local check_cluster_readyness=true
+ set_debug
+ [[ 1 == 1 ]]
+ set -o xtrace
+ local rs_name=rs2
+ local cluster_name=some-name
++ seq 0 2
+ for i in $(seq 0 $last_pod)
+ [[ 0 -eq 2 ]]
+ wait_pod some-name-rs2-0
+ local pod=some-name-rs2-0
+ set +o xtrace
waiting for pod/some-name-rs2-0 to be ready.OK
+ for i in $(seq 0 $last_pod)
+ [[ 1 -eq 2 ]]
+ wait_pod some-name-rs2-1
+ local pod=some-name-rs2-1
+ set +o xtrace
waiting for pod/some-name-rs2-1 to be ready.OK
+ for i in $(seq 0 $last_pod)
+ [[ 2 -eq 2 ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs2")].arbiter.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.ju3L1B8FBI
+++ mktemp
++ local LAST_ERR=/tmp/tmp.nvS1jv3JV2
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs2")].arbiter.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.ju3L1B8FBI
++ cat /tmp/tmp.nvS1jv3JV2
++ rm /tmp/tmp.ju3L1B8FBI /tmp/tmp.nvS1jv3JV2
++ return 0
+ [[ '' == \t\r\u\e ]]
+ wait_pod some-name-rs2-2
+ local pod=some-name-rs2-2
+ set +o xtrace
waiting for pod/some-name-rs2-2 to be ready.OK
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs2")].nonvoting.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.lO9FX3JF0y
+++ mktemp
++ local LAST_ERR=/tmp/tmp.MqIiJTDJTU
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs2")].nonvoting.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.lO9FX3JF0y
++ cat /tmp/tmp.MqIiJTDJTU
++ rm /tmp/tmp.lO9FX3JF0y /tmp/tmp.MqIiJTDJTU
++ return 0
+ [[ '' == \t\r\u\e ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs2")].hidden.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.YYh1zCc8ei
+++ mktemp
++ local LAST_ERR=/tmp/tmp.orZoVP1yHI
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs2")].hidden.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.YYh1zCc8ei
++ cat /tmp/tmp.orZoVP1yHI
++ rm /tmp/tmp.YYh1zCc8ei /tmp/tmp.orZoVP1yHI
++ return 0
+ [[ '' == \t\r\u\e ]]
+ sleep 10
+ [[ true == \t\r\u\e ]]
+ set +x
Waiting for cluster readyness
+ sleep 10
+ write_initial_data
+ desc 'create user myApp'
+ set +o xtrace


-----------------------------------------------------------------------------------
create user myApp
-----------------------------------------------------------------------------------

+ run_mongos 'db.createUser({user:"myApp",pwd:"myPass",roles:[{db:"myApp",role:"readWrite"}]})' userAdmin:userAdmin123456@some-name-mongos.pitr-physical-backup-source-4895
+ local 'command=db.createUser({user:"myApp",pwd:"myPass",roles:[{db:"myApp",role:"readWrite"}]})'
+ local uri=userAdmin:userAdmin123456@some-name-mongos.pitr-physical-backup-source-4895
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local port=27017
+ local mongo_bin=mongo
++ echo .svc.cluster.local
++ awk -F: '{print $2}'
+ suffix_port=
+ [[ -z '' ]]
+ suffix=.svc.cluster.local:27017
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.gmq5UaVyW7
+++ mktemp
++ local LAST_ERR=/tmp/tmp.aBYlNtWqrn
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.gmq5UaVyW7
++ cat /tmp/tmp.aBYlNtWqrn
++ rm /tmp/tmp.gmq5UaVyW7 /tmp/tmp.aBYlNtWqrn
++ return 0
+ local client_container=psmdb-client-696897d69b-7fp87
+ kubectl_bin exec psmdb-client-696897d69b-7fp87 -- bash -c 'printf '\''db.createUser({user:"myApp",pwd:"myPass",roles:[{db:"myApp",role:"readWrite"}]})\n'\'' | mongo mongodb://userAdmin:userAdmin123456@some-name-mongos.pitr-physical-backup-source-4895.svc.cluster.local:27017/admin '
++ mktemp
+ local LAST_OUT=/tmp/tmp.jVeEo1TCz7
++ mktemp
+ local LAST_ERR=/tmp/tmp.qgmtDZvR6N
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-696897d69b-7fp87 -- bash -c 'printf '\''db.createUser({user:"myApp",pwd:"myPass",roles:[{db:"myApp",role:"readWrite"}]})\n'\'' | mongo mongodb://userAdmin:userAdmin123456@some-name-mongos.pitr-physical-backup-source-4895.svc.cluster.local:27017/admin '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.jVeEo1TCz7
Percona Server for MongoDB shell version v4.4.29-28
connecting to: mongodb://some-name-mongos.pitr-physical-backup-source-4895.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb
Implicit session: session { "id" : UUID("04d445b8-874f-406c-9638-2d9dfef6688f") }
Percona Server for MongoDB server version: v8.0.16-5
WARNING: shell and server versions do not match
Successfully added user: {
	"user" : "myApp",
	"roles" : [
		{
			"db" : "myApp",
			"role" : "readWrite"
		}
	]
}
bye
+ cat /tmp/tmp.qgmtDZvR6N
+ rm /tmp/tmp.jVeEo1TCz7 /tmp/tmp.qgmtDZvR6N
+ return 0
+ sleep 2
+ write_document
+ local cmp_postfix=
+ local sleep_value=0
+ log 'write initial data, read from all'
+ set +o xtrace
[2025-12-17T17:03:12+0000] write initial data, read from all
+ run_mongos 'use myApp\n db.test.insert({ x: 100500 })' myApp:myPass@some-name-mongos.pitr-physical-backup-source-4895
+ local 'command=use myApp\n db.test.insert({ x: 100500 })'
+ local uri=myApp:myPass@some-name-mongos.pitr-physical-backup-source-4895
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local port=27017
+ local mongo_bin=mongo
++ echo .svc.cluster.local
++ awk -F: '{print $2}'
+ suffix_port=
+ [[ -z '' ]]
+ suffix=.svc.cluster.local:27017
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.zdNtcOrPID
+++ mktemp
++ local LAST_ERR=/tmp/tmp.hSdK30Qmuu
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.zdNtcOrPID
++ cat /tmp/tmp.hSdK30Qmuu
++ rm /tmp/tmp.zdNtcOrPID /tmp/tmp.hSdK30Qmuu
++ return 0
+ local client_container=psmdb-client-696897d69b-7fp87
+ kubectl_bin exec psmdb-client-696897d69b-7fp87 -- bash -c 'printf '\''use myApp\n db.test.insert({ x: 100500 })\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.pitr-physical-backup-source-4895.svc.cluster.local:27017/admin '
++ mktemp
+ local LAST_OUT=/tmp/tmp.NnRJoEaQ5l
++ mktemp
+ local LAST_ERR=/tmp/tmp.PHc61ReLgd
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-696897d69b-7fp87 -- bash -c 'printf '\''use myApp\n db.test.insert({ x: 100500 })\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.pitr-physical-backup-source-4895.svc.cluster.local:27017/admin '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.NnRJoEaQ5l
Percona Server for MongoDB shell version v4.4.29-28
connecting to: mongodb://some-name-mongos.pitr-physical-backup-source-4895.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb
Implicit session: session { "id" : UUID("1a1c4791-b2e5-4754-9529-9f27e61e913d") }
Percona Server for MongoDB server version: v8.0.16-5
WARNING: shell and server versions do not match
switched to db myApp
WriteResult({ "nInserted" : 1 })
bye
+ cat /tmp/tmp.PHc61ReLgd
+ rm /tmp/tmp.NnRJoEaQ5l /tmp/tmp.PHc61ReLgd
+ return 0
+ sleep 0
+ compare_mongos_cmd find myApp:myPass@some-name-mongos.pitr-physical-backup-source-4895
+ local command=find
+ local uri=myApp:myPass@some-name-mongos.pitr-physical-backup-source-4895
+ local postfix=
+ local suffix=
+ local database=myApp
+ local collection=test
+ local port=27017
+ local tls=false
+ [[ false == \t\r\u\e ]]
+ mongos_command=run_mongos
+ run_mongos 'use myApp\n db.test.find()' myApp:myPass@some-name-mongos.pitr-physical-backup-source-4895 mongodb '' '' 27017
+ local 'command=use myApp\n db.test.find()'
+ egrep -v 'I NETWORK|W NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|Started a new thread for the timer service'
+ local uri=myApp:myPass@some-name-mongos.pitr-physical-backup-source-4895
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local port=27017
+ local mongo_bin=mongo
+ /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
++ echo .svc.cluster.local
++ awk -F: '{print $2}'
egrep: warning: egrep is obsolescent; using grep -E
+ suffix_port=
+ [[ -z '' ]]
+ suffix=.svc.cluster.local:27017
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.VHTdssf6kn
+++ mktemp
++ local LAST_ERR=/tmp/tmp.EUbq6qrPvb
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.VHTdssf6kn
++ cat /tmp/tmp.EUbq6qrPvb
++ rm /tmp/tmp.VHTdssf6kn /tmp/tmp.EUbq6qrPvb
++ return 0
+ local client_container=psmdb-client-696897d69b-7fp87
+ kubectl_bin exec psmdb-client-696897d69b-7fp87 -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.pitr-physical-backup-source-4895.svc.cluster.local:27017/admin '
++ mktemp
+ local LAST_OUT=/tmp/tmp.F6YYevUCiI
++ mktemp
+ local LAST_ERR=/tmp/tmp.3xXUG3SZ8x
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-696897d69b-7fp87 -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.pitr-physical-backup-source-4895.svc.cluster.local:27017/admin '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.F6YYevUCiI
+ cat /tmp/tmp.3xXUG3SZ8x
+ rm /tmp/tmp.F6YYevUCiI /tmp/tmp.3xXUG3SZ8x
+ return 0
+ [[ 0 -eq 0 ]]
+ diff /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1393/e2e-tests/pitr-physical-backup-source/compare/find.json /tmp/tmp.XdTK1TTa6I/find
+ desc 'restore pitr type latest using backupSource'
+ set +o xtrace


-----------------------------------------------------------------------------------
restore pitr type latest using backupSource
-----------------------------------------------------------------------------------

+ write_document -2nd
+ local cmp_postfix=-2nd
+ local sleep_value=0
+ log 'write initial data, read from all'
+ set +o xtrace
[2025-12-17T17:03:17+0000] write initial data, read from all
+ run_mongos 'use myApp\n db.test.insert({ x: 100500 })' myApp:myPass@some-name-mongos.pitr-physical-backup-source-4895
+ local 'command=use myApp\n db.test.insert({ x: 100500 })'
+ local uri=myApp:myPass@some-name-mongos.pitr-physical-backup-source-4895
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local port=27017
+ local mongo_bin=mongo
++ echo .svc.cluster.local
++ awk -F: '{print $2}'
+ suffix_port=
+ [[ -z '' ]]
+ suffix=.svc.cluster.local:27017
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.zIc0haJXFs
+++ mktemp
++ local LAST_ERR=/tmp/tmp.5qLPVTYSua
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.zIc0haJXFs
++ cat /tmp/tmp.5qLPVTYSua
++ rm /tmp/tmp.zIc0haJXFs /tmp/tmp.5qLPVTYSua
++ return 0
+ local client_container=psmdb-client-696897d69b-7fp87
+ kubectl_bin exec psmdb-client-696897d69b-7fp87 -- bash -c 'printf '\''use myApp\n db.test.insert({ x: 100500 })\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.pitr-physical-backup-source-4895.svc.cluster.local:27017/admin '
++ mktemp
+ local LAST_OUT=/tmp/tmp.MJUgo2QqPA
++ mktemp
+ local LAST_ERR=/tmp/tmp.wcoS90JPlF
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-696897d69b-7fp87 -- bash -c 'printf '\''use myApp\n db.test.insert({ x: 100500 })\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.pitr-physical-backup-source-4895.svc.cluster.local:27017/admin '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.MJUgo2QqPA
Percona Server for MongoDB shell version v4.4.29-28
connecting to: mongodb://some-name-mongos.pitr-physical-backup-source-4895.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb
Implicit session: session { "id" : UUID("0ea3133b-c5db-454e-b4f3-0ee3b901378b") }
Percona Server for MongoDB server version: v8.0.16-5
WARNING: shell and server versions do not match
switched to db myApp
WriteResult({ "nInserted" : 1 })
bye
+ cat /tmp/tmp.wcoS90JPlF
+ rm /tmp/tmp.MJUgo2QqPA /tmp/tmp.wcoS90JPlF
+ return 0
+ sleep 0
+ compare_mongos_cmd find myApp:myPass@some-name-mongos.pitr-physical-backup-source-4895 -2nd
+ local command=find
+ local uri=myApp:myPass@some-name-mongos.pitr-physical-backup-source-4895
+ local postfix=-2nd
+ local suffix=
+ local database=myApp
+ local collection=test
+ local port=27017
+ local tls=false
+ [[ false == \t\r\u\e ]]
+ mongos_command=run_mongos
+ run_mongos 'use myApp\n db.test.find()' myApp:myPass@some-name-mongos.pitr-physical-backup-source-4895 mongodb '' '' 27017
+ local 'command=use myApp\n db.test.find()'
+ local uri=myApp:myPass@some-name-mongos.pitr-physical-backup-source-4895
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local port=27017
+ local mongo_bin=mongo
+ egrep -v 'I NETWORK|W NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|Started a new thread for the timer service'
+ /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
++ echo .svc.cluster.local
++ awk -F: '{print $2}'
egrep: warning: egrep is obsolescent; using grep -E
+ suffix_port=
+ [[ -z '' ]]
+ suffix=.svc.cluster.local:27017
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.oOGz68GlL7
+++ mktemp
++ local LAST_ERR=/tmp/tmp.7iWrgzgSsW
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.oOGz68GlL7
++ cat /tmp/tmp.7iWrgzgSsW
++ rm /tmp/tmp.oOGz68GlL7 /tmp/tmp.7iWrgzgSsW
++ return 0
+ local client_container=psmdb-client-696897d69b-7fp87
+ kubectl_bin exec psmdb-client-696897d69b-7fp87 -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.pitr-physical-backup-source-4895.svc.cluster.local:27017/admin '
++ mktemp
+ local LAST_OUT=/tmp/tmp.wDi0ZcpfPW
++ mktemp
+ local LAST_ERR=/tmp/tmp.3luGppNu6i
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-696897d69b-7fp87 -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.pitr-physical-backup-source-4895.svc.cluster.local:27017/admin '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.wDi0ZcpfPW
+ cat /tmp/tmp.3luGppNu6i
+ rm /tmp/tmp.wDi0ZcpfPW /tmp/tmp.3luGppNu6i
+ return 0
+ [[ 0 -eq 0 ]]
+ diff /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1393/e2e-tests/pitr-physical-backup-source/compare/find-2nd.json /tmp/tmp.XdTK1TTa6I/find-2nd
+ run_backup backup-minio 2 physical
+ local name=backup-minio
+ local idx=2
+ local type=physical
+ desc 'run backup backup-minio-2'
+ set +o xtrace


-----------------------------------------------------------------------------------
run backup backup-minio-2
-----------------------------------------------------------------------------------

+ cat /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1393/e2e-tests/pitr-physical-backup-source/conf/backup-minio.yml
+ /usr/sbin/sed -e 's/name:/name: backup-minio-2/'
+ /usr/sbin/sed -e 's/type:/type: physical/'
+ kubectl_bin apply -f -
++ mktemp
+ local LAST_OUT=/tmp/tmp.rZSiDKOlV9
++ mktemp
+ local LAST_ERR=/tmp/tmp.7FBLopqeiX
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl apply -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.rZSiDKOlV9
perconaservermongodbbackup.psmdb.percona.com/backup-minio-2 created
+ cat /tmp/tmp.7FBLopqeiX
+ rm /tmp/tmp.rZSiDKOlV9 /tmp/tmp.7FBLopqeiX
+ return 0
+ wait_backup backup-minio-2
+ local backup_name=backup-minio-2
+ local target_state=ready
+ set +o xtrace
waiting for backup-minio-2 to reach ready state..........OK
+ compare_latest_restorable_time some-name-rs0 backup-minio-2
+ local cluster=some-name-rs0
+ local backup_name=backup-minio-2
+ local latest_restorable_time
+ local backup_time
++ get_latest_restorable_time some-name-rs0
++ local cluster=some-name-rs0
++ local first_timestamp
++ local second_timestamp
++ local retry=0
++ [[ '' != '' ]]
+++ kubectl_bin exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ jq '.backups.pitrChunks.pitrChunks | last | .range.end'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.LIruqAM6Nq
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.SjVsNflfCa
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in $(seq 0 2)
+++ set +e
+++ kubectl exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.LIruqAM6Nq
+++ cat /tmp/tmp.SjVsNflfCa
+++ rm /tmp/tmp.LIruqAM6Nq /tmp/tmp.SjVsNflfCa
+++ return 0
++ first_timestamp=1765989857
++ sleep 5
++ [[ 1765989857 != '' ]]
++ [[ 1765989857 != \n\u\l\l ]]
+++ kubectl_bin exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ jq '.backups.pitrChunks.pitrChunks | last | .range.end'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.Z8PBvLhMxe
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.XZNrGK4psU
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in $(seq 0 2)
+++ set +e
+++ kubectl exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.Z8PBvLhMxe
+++ cat /tmp/tmp.XZNrGK4psU
+++ rm /tmp/tmp.Z8PBvLhMxe /tmp/tmp.XZNrGK4psU
+++ return 0
++ second_timestamp=1765989857
++ let retry+=1
++ [[ 1 -gt 30 ]]
++ [[ 1765989857 != '' ]]
++ [[ 1765989857 != \n\u\l\l ]]
++ [[ 1765989857 == 1765989857 ]]
++ /usr/sbin/date -u -d @1765989857 +%Y-%m-%dT%H:%M:%SZ
+ latest_restorable_time=2025-12-17T16:44:17Z
++ get_latest_restorable_time_from_backup_object backup-minio-2
++ local backup_name=backup-minio-2
++ local latestRestorableTime
++ local retry=0
++ [[ '' != '' ]]
++ sleep 5
+++ kubectl_bin get psmdb-backup backup-minio-2 -o 'jsonpath={.status.latestRestorableTime}'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.YIf9NQ5id3
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.S7GP5ykhfa
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in $(seq 0 2)
+++ set +e
+++ kubectl get psmdb-backup backup-minio-2 -o 'jsonpath={.status.latestRestorableTime}'
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.YIf9NQ5id3
+++ cat /tmp/tmp.S7GP5ykhfa
+++ rm /tmp/tmp.YIf9NQ5id3 /tmp/tmp.S7GP5ykhfa
+++ return 0
++ latestRestorableTime=2025-12-17T16:44:17Z
++ let retry+=1
++ [[ 1 -gt 30 ]]
++ [[ 2025-12-17T16:44:17Z != '' ]]
++ [[ 2025-12-17T16:44:17Z != \n\u\l\l ]]
++ echo 2025-12-17T16:44:17Z
+ backup_time=2025-12-17T16:44:17Z
+ [[ 2025-12-17T16:44:17Z != \2\0\2\5\-\1\2\-\1\7\T\1\6\:\4\4\:\1\7\Z ]]
+ check_recovery backup-minio-2 latest '' -3rd some-name backupSource
+ local backup_name=backup-minio-2
+ local restore_type=latest
+ local restore_date=
+ local cmp_postfix=-3rd
+ local cluster_name=some-name
+ local backupSource=backupSource
++ get_latest_oplog_chunk_ts some-name
++ local cluster=some-name
+++ kubectl_bin exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ jq '.backups.pitrChunks.pitrChunks | last | .range.end'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.HJ7sqziAGT
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.Vm0PMKOwYp
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in $(seq 0 2)
+++ set +e
+++ kubectl exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.HJ7sqziAGT
+++ cat /tmp/tmp.Vm0PMKOwYp
+++ rm /tmp/tmp.HJ7sqziAGT /tmp/tmp.Vm0PMKOwYp
+++ return 0
++ echo 1765989857
+ local latest_ts=1765989857
+ desc 'write more data before restore by latest'
+ set +o xtrace


-----------------------------------------------------------------------------------
write more data before restore by latest
-----------------------------------------------------------------------------------

+ run_mongos 'use myApp\n db.test.insert({ x: 100501 })' myApp:myPass@some-name-mongos.pitr-physical-backup-source-4895
+ local 'command=use myApp\n db.test.insert({ x: 100501 })'
+ local uri=myApp:myPass@some-name-mongos.pitr-physical-backup-source-4895
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local port=27017
+ local mongo_bin=mongo
++ echo .svc.cluster.local
++ awk -F: '{print $2}'
+ suffix_port=
+ [[ -z '' ]]
+ suffix=.svc.cluster.local:27017
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.G8POodxaSK
+++ mktemp
++ local LAST_ERR=/tmp/tmp.oTq6zmIuqc
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.G8POodxaSK
++ cat /tmp/tmp.oTq6zmIuqc
++ rm /tmp/tmp.G8POodxaSK /tmp/tmp.oTq6zmIuqc
++ return 0
+ local client_container=psmdb-client-696897d69b-7fp87
+ kubectl_bin exec psmdb-client-696897d69b-7fp87 -- bash -c 'printf '\''use myApp\n db.test.insert({ x: 100501 })\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.pitr-physical-backup-source-4895.svc.cluster.local:27017/admin '
++ mktemp
+ local LAST_OUT=/tmp/tmp.QCrOuCvdkc
++ mktemp
+ local LAST_ERR=/tmp/tmp.NvwT37vNgf
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-696897d69b-7fp87 -- bash -c 'printf '\''use myApp\n db.test.insert({ x: 100501 })\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.pitr-physical-backup-source-4895.svc.cluster.local:27017/admin '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.QCrOuCvdkc
Percona Server for MongoDB shell version v4.4.29-28
connecting to: mongodb://some-name-mongos.pitr-physical-backup-source-4895.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb
Implicit session: session { "id" : UUID("cadf2cac-3bcd-4ee8-937f-59dba1402e7f") }
Percona Server for MongoDB server version: v8.0.16-5
WARNING: shell and server versions do not match
switched to db myApp
WriteResult({ "nInserted" : 1 })
bye
+ cat /tmp/tmp.NvwT37vNgf
+ rm /tmp/tmp.QCrOuCvdkc /tmp/tmp.NvwT37vNgf
+ return 0
+ [[ -n '' ]]
+ desc 'Restoring to latest'
+ set +o xtrace


-----------------------------------------------------------------------------------
Restoring to latest
-----------------------------------------------------------------------------------

++ get_latest_oplog_chunk_ts some-name
++ local cluster=some-name
+++ kubectl_bin exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ jq '.backups.pitrChunks.pitrChunks | last | .range.end'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.CkRFTB4oZ9
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.Cjj4zox1Gf
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in $(seq 0 2)
+++ set +e
+++ kubectl exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.CkRFTB4oZ9
+++ cat /tmp/tmp.Cjj4zox1Gf
+++ rm /tmp/tmp.CkRFTB4oZ9 /tmp/tmp.Cjj4zox1Gf
+++ return 0
++ echo 1765989857
+ local current_ts=1765989857
+ retries=0
+ [[ 1765989857 -gt 1765989857 ]]
+ [[ 0 -gt 30 ]]
++ get_latest_oplog_chunk_ts some-name
++ local cluster=some-name
+++ kubectl_bin exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ jq '.backups.pitrChunks.pitrChunks | last | .range.end'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.IWyasb6NYV
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.Zlm8esPFmo
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in $(seq 0 2)
+++ set +e
+++ kubectl exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.IWyasb6NYV
+++ cat /tmp/tmp.Zlm8esPFmo
+++ rm /tmp/tmp.IWyasb6NYV /tmp/tmp.Zlm8esPFmo
+++ return 0
++ echo 1765989857
+ latest_ts=1765989857
+ retries=1
++ format_date 1765989857
++ local timestamp=1765989857
+++ TZ=UTC
+++ /usr/sbin/date -d@1765989857 '+%Y-%m-%d %H:%M:%S'
++ echo 2025-12-17 16:44:17
++ format_date 1765989857
++ local timestamp=1765989857
+++ TZ=UTC
+++ /usr/sbin/date -d@1765989857 '+%Y-%m-%d %H:%M:%S'
++ echo 2025-12-17 16:44:17
+ echo 'Waiting for last oplog chunk (2025-12-17 16:44:17) to be 120 seconds older than starting chunk (2025-12-17 16:44:17)'
Waiting for last oplog chunk (2025-12-17 16:44:17) to be 120 seconds older than starting chunk (2025-12-17 16:44:17)
+ sleep 10
+ [[ 1765989857 -gt 1765989857 ]]
+ [[ 1 -gt 30 ]]
++ get_latest_oplog_chunk_ts some-name
++ local cluster=some-name
+++ jq '.backups.pitrChunks.pitrChunks | last | .range.end'
+++ kubectl_bin exec some-name-rs0-0 -c backup-agent -- pbm status -o json
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.pHAxosb7GB
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.QQYHJFJKP8
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in $(seq 0 2)
+++ set +e
+++ kubectl exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.pHAxosb7GB
+++ cat /tmp/tmp.QQYHJFJKP8
+++ rm /tmp/tmp.pHAxosb7GB /tmp/tmp.QQYHJFJKP8
+++ return 0
++ echo 1765989857
+ latest_ts=1765989857
+ retries=2
++ format_date 1765989857
++ local timestamp=1765989857
+++ TZ=UTC
+++ /usr/sbin/date -d@1765989857 '+%Y-%m-%d %H:%M:%S'
++ echo 2025-12-17 16:44:17
++ format_date 1765989857
++ local timestamp=1765989857
+++ TZ=UTC
+++ /usr/sbin/date -d@1765989857 '+%Y-%m-%d %H:%M:%S'
++ echo 2025-12-17 16:44:17
+ echo 'Waiting for last oplog chunk (2025-12-17 16:44:17) to be 120 seconds older than starting chunk (2025-12-17 16:44:17)'
Waiting for last oplog chunk (2025-12-17 16:44:17) to be 120 seconds older than starting chunk (2025-12-17 16:44:17)
+ sleep 10
+ [[ 1765989857 -gt 1765989857 ]]
+ [[ 2 -gt 30 ]]
++ get_latest_oplog_chunk_ts some-name
++ local cluster=some-name
+++ kubectl_bin exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ jq '.backups.pitrChunks.pitrChunks | last | .range.end'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.ncnWVPjNyP
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.Afh1XOpSXO
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in $(seq 0 2)
+++ set +e
+++ kubectl exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.ncnWVPjNyP
+++ cat /tmp/tmp.Afh1XOpSXO
+++ rm /tmp/tmp.ncnWVPjNyP /tmp/tmp.Afh1XOpSXO
+++ return 0
++ echo 1765989857
+ latest_ts=1765989857
+ retries=3
++ format_date 1765989857
++ local timestamp=1765989857
+++ TZ=UTC
+++ /usr/sbin/date -d@1765989857 '+%Y-%m-%d %H:%M:%S'
++ echo 2025-12-17 16:44:17
++ format_date 1765989857
++ local timestamp=1765989857
+++ TZ=UTC
+++ /usr/sbin/date -d@1765989857 '+%Y-%m-%d %H:%M:%S'
++ echo 2025-12-17 16:44:17
+ echo 'Waiting for last oplog chunk (2025-12-17 16:44:17) to be 120 seconds older than starting chunk (2025-12-17 16:44:17)'
Waiting for last oplog chunk (2025-12-17 16:44:17) to be 120 seconds older than starting chunk (2025-12-17 16:44:17)
+ sleep 10
+ [[ 1765989857 -gt 1765989857 ]]
+ [[ 3 -gt 30 ]]
++ get_latest_oplog_chunk_ts some-name
++ local cluster=some-name
+++ kubectl_bin exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ jq '.backups.pitrChunks.pitrChunks | last | .range.end'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.NOAzNRscjy
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.Kfqo7h16ru
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in $(seq 0 2)
+++ set +e
+++ kubectl exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.NOAzNRscjy
+++ cat /tmp/tmp.Kfqo7h16ru
+++ rm /tmp/tmp.NOAzNRscjy /tmp/tmp.Kfqo7h16ru
+++ return 0
++ echo 1765989857
+ latest_ts=1765989857
+ retries=4
++ format_date 1765989857
++ local timestamp=1765989857
+++ TZ=UTC
+++ /usr/sbin/date -d@1765989857 '+%Y-%m-%d %H:%M:%S'
++ echo 2025-12-17 16:44:17
++ format_date 1765989857
++ local timestamp=1765989857
+++ TZ=UTC
+++ /usr/sbin/date -d@1765989857 '+%Y-%m-%d %H:%M:%S'
++ echo 2025-12-17 16:44:17
+ echo 'Waiting for last oplog chunk (2025-12-17 16:44:17) to be 120 seconds older than starting chunk (2025-12-17 16:44:17)'
Waiting for last oplog chunk (2025-12-17 16:44:17) to be 120 seconds older than starting chunk (2025-12-17 16:44:17)
+ sleep 10
+ [[ 1765989857 -gt 1765989857 ]]
+ [[ 4 -gt 30 ]]
++ get_latest_oplog_chunk_ts some-name
++ local cluster=some-name
+++ kubectl_bin exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ jq '.backups.pitrChunks.pitrChunks | last | .range.end'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.6zJkAJ0qjB
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.ta0MP93uYe
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in $(seq 0 2)
+++ set +e
+++ kubectl exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.6zJkAJ0qjB
+++ cat /tmp/tmp.ta0MP93uYe
+++ rm /tmp/tmp.6zJkAJ0qjB /tmp/tmp.ta0MP93uYe
+++ return 0
++ echo 1765989857
+ latest_ts=1765989857
+ retries=5
++ format_date 1765989857
++ local timestamp=1765989857
+++ TZ=UTC
+++ /usr/sbin/date -d@1765989857 '+%Y-%m-%d %H:%M:%S'
++ echo 2025-12-17 16:44:17
++ format_date 1765989857
++ local timestamp=1765989857
+++ TZ=UTC
+++ /usr/sbin/date -d@1765989857 '+%Y-%m-%d %H:%M:%S'
++ echo 2025-12-17 16:44:17
+ echo 'Waiting for last oplog chunk (2025-12-17 16:44:17) to be 120 seconds older than starting chunk (2025-12-17 16:44:17)'
Waiting for last oplog chunk (2025-12-17 16:44:17) to be 120 seconds older than starting chunk (2025-12-17 16:44:17)
+ sleep 10
+ [[ 1765989857 -gt 1765989857 ]]
+ [[ 5 -gt 30 ]]
++ get_latest_oplog_chunk_ts some-name
++ local cluster=some-name
+++ kubectl_bin exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ jq '.backups.pitrChunks.pitrChunks | last | .range.end'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.tpXSQZMTXm
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.VOgJeqtA9I
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in $(seq 0 2)
+++ set +e
+++ kubectl exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.tpXSQZMTXm
+++ cat /tmp/tmp.VOgJeqtA9I
+++ rm /tmp/tmp.tpXSQZMTXm /tmp/tmp.VOgJeqtA9I
+++ return 0
++ echo 1765989857
+ latest_ts=1765989857
+ retries=6
++ format_date 1765989857
++ local timestamp=1765989857
+++ TZ=UTC
+++ /usr/sbin/date -d@1765989857 '+%Y-%m-%d %H:%M:%S'
++ echo 2025-12-17 16:44:17
++ format_date 1765989857
++ local timestamp=1765989857
+++ TZ=UTC
+++ /usr/sbin/date -d@1765989857 '+%Y-%m-%d %H:%M:%S'
++ echo 2025-12-17 16:44:17
+ echo 'Waiting for last oplog chunk (2025-12-17 16:44:17) to be 120 seconds older than starting chunk (2025-12-17 16:44:17)'
Waiting for last oplog chunk (2025-12-17 16:44:17) to be 120 seconds older than starting chunk (2025-12-17 16:44:17)
+ sleep 10
+ [[ 1765989857 -gt 1765989857 ]]
+ [[ 6 -gt 30 ]]
++ get_latest_oplog_chunk_ts some-name
++ local cluster=some-name
+++ kubectl_bin exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ jq '.backups.pitrChunks.pitrChunks | last | .range.end'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.iNCrRJMt20
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.685E2VIzXl
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in $(seq 0 2)
+++ set +e
+++ kubectl exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.iNCrRJMt20
+++ cat /tmp/tmp.685E2VIzXl
+++ rm /tmp/tmp.iNCrRJMt20 /tmp/tmp.685E2VIzXl
+++ return 0
++ echo 1765989857
+ latest_ts=1765989857
+ retries=7
++ format_date 1765989857
++ local timestamp=1765989857
+++ TZ=UTC
+++ /usr/sbin/date -d@1765989857 '+%Y-%m-%d %H:%M:%S'
++ echo 2025-12-17 16:44:17
++ format_date 1765989857
++ local timestamp=1765989857
+++ TZ=UTC
+++ /usr/sbin/date -d@1765989857 '+%Y-%m-%d %H:%M:%S'
++ echo 2025-12-17 16:44:17
+ echo 'Waiting for last oplog chunk (2025-12-17 16:44:17) to be 120 seconds older than starting chunk (2025-12-17 16:44:17)'
Waiting for last oplog chunk (2025-12-17 16:44:17) to be 120 seconds older than starting chunk (2025-12-17 16:44:17)
+ sleep 10
+ [[ 1765989857 -gt 1765989857 ]]
+ [[ 7 -gt 30 ]]
++ get_latest_oplog_chunk_ts some-name
++ local cluster=some-name
+++ kubectl_bin exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ jq '.backups.pitrChunks.pitrChunks | last | .range.end'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.rmYGi4GLrB
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.isOcq4YTlC
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in $(seq 0 2)
+++ set +e
+++ kubectl exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.rmYGi4GLrB
+++ cat /tmp/tmp.isOcq4YTlC
+++ rm /tmp/tmp.rmYGi4GLrB /tmp/tmp.isOcq4YTlC
+++ return 0
++ echo 1765989857
+ latest_ts=1765989857
+ retries=8
++ format_date 1765989857
++ local timestamp=1765989857
+++ TZ=UTC
+++ /usr/sbin/date -d@1765989857 '+%Y-%m-%d %H:%M:%S'
++ echo 2025-12-17 16:44:17
++ format_date 1765989857
++ local timestamp=1765989857
+++ TZ=UTC
+++ /usr/sbin/date -d@1765989857 '+%Y-%m-%d %H:%M:%S'
++ echo 2025-12-17 16:44:17
+ echo 'Waiting for last oplog chunk (2025-12-17 16:44:17) to be 120 seconds older than starting chunk (2025-12-17 16:44:17)'
Waiting for last oplog chunk (2025-12-17 16:44:17) to be 120 seconds older than starting chunk (2025-12-17 16:44:17)
+ sleep 10
+ [[ 1765989857 -gt 1765989857 ]]
+ [[ 8 -gt 30 ]]
++ get_latest_oplog_chunk_ts some-name
++ local cluster=some-name
+++ kubectl_bin exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ jq '.backups.pitrChunks.pitrChunks | last | .range.end'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.hnN0IXSHV3
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.S1X5C9EMfY
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in $(seq 0 2)
+++ set +e
+++ kubectl exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.hnN0IXSHV3
+++ cat /tmp/tmp.S1X5C9EMfY
+++ rm /tmp/tmp.hnN0IXSHV3 /tmp/tmp.S1X5C9EMfY
+++ return 0
++ echo 1765989857
+ latest_ts=1765989857
+ retries=9
++ format_date 1765989857
++ local timestamp=1765989857
+++ TZ=UTC
+++ /usr/sbin/date -d@1765989857 '+%Y-%m-%d %H:%M:%S'
++ echo 2025-12-17 16:44:17
++ format_date 1765989857
++ local timestamp=1765989857
+++ TZ=UTC
+++ /usr/sbin/date -d@1765989857 '+%Y-%m-%d %H:%M:%S'
++ echo 2025-12-17 16:44:17
+ echo 'Waiting for last oplog chunk (2025-12-17 16:44:17) to be 120 seconds older than starting chunk (2025-12-17 16:44:17)'
Waiting for last oplog chunk (2025-12-17 16:44:17) to be 120 seconds older than starting chunk (2025-12-17 16:44:17)
+ sleep 10
+ [[ 1765989857 -gt 1765989857 ]]
+ [[ 9 -gt 30 ]]
++ get_latest_oplog_chunk_ts some-name
++ local cluster=some-name
+++ kubectl_bin exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ jq '.backups.pitrChunks.pitrChunks | last | .range.end'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.VDqSRXHxis
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.nwzQcBrcFM
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in $(seq 0 2)
+++ set +e
+++ kubectl exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.VDqSRXHxis
+++ cat /tmp/tmp.nwzQcBrcFM
+++ rm /tmp/tmp.VDqSRXHxis /tmp/tmp.nwzQcBrcFM
+++ return 0
++ echo 1765989857
+ latest_ts=1765989857
+ retries=10
++ format_date 1765989857
++ local timestamp=1765989857
+++ TZ=UTC
+++ /usr/sbin/date -d@1765989857 '+%Y-%m-%d %H:%M:%S'
++ echo 2025-12-17 16:44:17
++ format_date 1765989857
++ local timestamp=1765989857
+++ TZ=UTC
+++ /usr/sbin/date -d@1765989857 '+%Y-%m-%d %H:%M:%S'
++ echo 2025-12-17 16:44:17
+ echo 'Waiting for last oplog chunk (2025-12-17 16:44:17) to be 120 seconds older than starting chunk (2025-12-17 16:44:17)'
Waiting for last oplog chunk (2025-12-17 16:44:17) to be 120 seconds older than starting chunk (2025-12-17 16:44:17)
+ sleep 10
+ [[ 1765989857 -gt 1765989857 ]]
+ [[ 10 -gt 30 ]]
++ get_latest_oplog_chunk_ts some-name
++ local cluster=some-name
+++ kubectl_bin exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ jq '.backups.pitrChunks.pitrChunks | last | .range.end'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.1GBhCJDQPr
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.1iUWxYmnzc
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in $(seq 0 2)
+++ set +e
+++ kubectl exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.1GBhCJDQPr
+++ cat /tmp/tmp.1iUWxYmnzc
+++ rm /tmp/tmp.1GBhCJDQPr /tmp/tmp.1iUWxYmnzc
+++ return 0
++ echo 1765991142
+ latest_ts=1765991142
+ retries=11
++ format_date 1765991142
++ local timestamp=1765991142
+++ TZ=UTC
+++ /usr/sbin/date -d@1765991142 '+%Y-%m-%d %H:%M:%S'
++ echo 2025-12-17 17:05:42
++ format_date 1765989857
++ local timestamp=1765989857
+++ TZ=UTC
+++ /usr/sbin/date -d@1765989857 '+%Y-%m-%d %H:%M:%S'
++ echo 2025-12-17 16:44:17
+ echo 'Waiting for last oplog chunk (2025-12-17 17:05:42) to be 120 seconds older than starting chunk (2025-12-17 16:44:17)'
Waiting for last oplog chunk (2025-12-17 17:05:42) to be 120 seconds older than starting chunk (2025-12-17 16:44:17)
+ sleep 10
+ [[ 1765991142 -gt 1765989857 ]]
+ '[' -z backupSource ']'
+ desc 'check restore by latest backupSource'
+ set +o xtrace


-----------------------------------------------------------------------------------
check restore by latest backupSource
-----------------------------------------------------------------------------------

++ get_backup_dest backup-minio-2
++ local backup_name=backup-minio-2
++ kubectl_bin get psmdb-backup backup-minio-2 -o 'jsonpath={.status.destination}'
++ sed -e 's/.json$//'
++ sed 's|s3://||'
++ sed 's|azure://||'
++ sed 's|gs://||'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.o1aNh3uJiq
+++ mktemp
++ local LAST_ERR=/tmp/tmp.tMNwarc2K3
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb-backup backup-minio-2 -o 'jsonpath={.status.destination}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.o1aNh3uJiq
++ cat /tmp/tmp.tMNwarc2K3
++ rm /tmp/tmp.o1aNh3uJiq /tmp/tmp.tMNwarc2K3
++ return 0
+ backup_dest=operator-testing/2025-12-17T17:03:25Z
+ cat /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1393/e2e-tests/pitr-physical-backup-source/conf/restore.yml
+ /usr/sbin/sed -e 's/name:/name: restore-backup-minio-2/'
+ /usr/sbin/sed -e /backupName/d
+ /usr/sbin/sed -e 's/pitrType:/type: latest/'
+ '[' -z '' ']'
+ /usr/sbin/sed -e /date:/d
+ /usr/sbin/sed -e 's|DESTINATION|operator-testing/2025-12-17T17:03:25Z|'
+ kubectl_bin apply -f -
+ '[' -n '' ']'
+ yq
++ get_bucket_name backup-minio-2
++ mktemp
++ local backup_name=backup-minio-2
++ kubectl_bin get psmdb-backup backup-minio-2 -o 'jsonpath={.status.s3.bucket}'
+++ mktemp
+ local LAST_OUT=/tmp/tmp.CUacEwKfrz
++ mktemp
++ local LAST_OUT=/tmp/tmp.M3jDVWEmJE
+++ mktemp
+ local LAST_ERR=/tmp/tmp.YZawtiHoLh
+ local exit_status=0
+ local timeout=4
++ seq 0 2
++ local LAST_ERR=/tmp/tmp.I3pvdHf8mR
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl apply -f -
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb-backup backup-minio-2 -o 'jsonpath={.status.s3.bucket}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.M3jDVWEmJE
++ cat /tmp/tmp.I3pvdHf8mR
++ rm /tmp/tmp.M3jDVWEmJE /tmp/tmp.I3pvdHf8mR
++ return 0
+ /usr/sbin/sed -e 's|BUCKET-NAME|operator-testing|'
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.CUacEwKfrz
perconaservermongodbrestore.psmdb.percona.com/restore-backup-minio-2 created
+ cat /tmp/tmp.YZawtiHoLh
+ rm /tmp/tmp.CUacEwKfrz /tmp/tmp.YZawtiHoLh
+ return 0
+ wait_restore backup-minio-2 some-name requested 0 1200
+ local backup_name=backup-minio-2
+ local cluster_name=some-name
+ local target_state=requested
+ local wait_cluster_consistency=0
+ local wait_time=1200
+ local ok_if_ready=0
+ set +o xtrace
Waiting for the psmdb-restore/restore-backup-minio-2 object to be created.OK
Waiting psmdb-restore/restore-backup-minio-2 to reach state "requested" .........OK after 8 minutes
+ [[ 0 -eq 1 ]]
+ echo

+ wait_restore backup-minio-2 some-name ready 0 1600
+ local backup_name=backup-minio-2
+ local cluster_name=some-name
+ local target_state=ready
+ local wait_cluster_consistency=0
+ local wait_time=1600
+ local ok_if_ready=0
+ set +o xtrace
Waiting for the psmdb-restore/restore-backup-minio-2 object to be created.OK
Waiting psmdb-restore/restore-backup-minio-2 to reach state "ready" ...OK after 2 minutes
+ [[ 0 -eq 1 ]]
+ echo

+ set -o xtrace
+ wait_for_running some-name-rs0 3
+ local name=some-name-rs0
+ let last_pod=2
+ local check_cluster_readyness=true
+ set_debug
+ [[ 1 == 1 ]]
+ set -o xtrace
+ local rs_name=rs0
+ local cluster_name=some-name
++ seq 0 2
+ for i in $(seq 0 $last_pod)
+ [[ 0 -eq 2 ]]
+ wait_pod some-name-rs0-0
+ local pod=some-name-rs0-0
+ set +o xtrace
waiting for pod/some-name-rs0-0 to be ready.OK
+ for i in $(seq 0 $last_pod)
+ [[ 1 -eq 2 ]]
+ wait_pod some-name-rs0-1
+ local pod=some-name-rs0-1
+ set +o xtrace
waiting for pod/some-name-rs0-1 to be ready.OK
+ for i in $(seq 0 $last_pod)
+ [[ 2 -eq 2 ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].arbiter.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.f7QEDwquz6
+++ mktemp
++ local LAST_ERR=/tmp/tmp.ShGYFE4FF7
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].arbiter.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.f7QEDwquz6
++ cat /tmp/tmp.ShGYFE4FF7
++ rm /tmp/tmp.f7QEDwquz6 /tmp/tmp.ShGYFE4FF7
++ return 0
+ [[ '' == \t\r\u\e ]]
+ wait_pod some-name-rs0-2
+ local pod=some-name-rs0-2
+ set +o xtrace
waiting for pod/some-name-rs0-2 to be ready.OK
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].nonvoting.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.D98e2f7auz
+++ mktemp
++ local LAST_ERR=/tmp/tmp.3mFRRJs2Q3
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].nonvoting.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.D98e2f7auz
++ cat /tmp/tmp.3mFRRJs2Q3
++ rm /tmp/tmp.D98e2f7auz /tmp/tmp.3mFRRJs2Q3
++ return 0
+ [[ '' == \t\r\u\e ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].hidden.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.eBR3nFi1OD
+++ mktemp
++ local LAST_ERR=/tmp/tmp.AIppQ7sBcX
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].hidden.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.eBR3nFi1OD
++ cat /tmp/tmp.AIppQ7sBcX
++ rm /tmp/tmp.eBR3nFi1OD /tmp/tmp.AIppQ7sBcX
++ return 0
+ [[ '' == \t\r\u\e ]]
+ sleep 10
+ [[ true == \t\r\u\e ]]
+ set +x
Waiting for cluster readyness........................................................
+ wait_for_running some-name-cfg 3
+ local name=some-name-cfg
+ let last_pod=2
+ local check_cluster_readyness=true
+ set_debug
+ [[ 1 == 1 ]]
+ set -o xtrace
+ local rs_name=cfg
+ local cluster_name=some-name
++ seq 0 2
+ for i in $(seq 0 $last_pod)
+ [[ 0 -eq 2 ]]
+ wait_pod some-name-cfg-0
+ local pod=some-name-cfg-0
+ set +o xtrace
waiting for pod/some-name-cfg-0 to be ready.OK
+ for i in $(seq 0 $last_pod)
+ [[ 1 -eq 2 ]]
+ wait_pod some-name-cfg-1
+ local pod=some-name-cfg-1
+ set +o xtrace
waiting for pod/some-name-cfg-1 to be ready.OK
+ for i in $(seq 0 $last_pod)
+ [[ 2 -eq 2 ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].arbiter.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.d7JNtwdVIc
+++ mktemp
++ local LAST_ERR=/tmp/tmp.Nk55xfXCeY
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].arbiter.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.d7JNtwdVIc
++ cat /tmp/tmp.Nk55xfXCeY
++ rm /tmp/tmp.d7JNtwdVIc /tmp/tmp.Nk55xfXCeY
++ return 0
+ [[ '' == \t\r\u\e ]]
+ wait_pod some-name-cfg-2
+ local pod=some-name-cfg-2
+ set +o xtrace
waiting for pod/some-name-cfg-2 to be ready.OK
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].nonvoting.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.qGgyI51Y7X
+++ mktemp
++ local LAST_ERR=/tmp/tmp.EJCvH2i3mr
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].nonvoting.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.qGgyI51Y7X
++ cat /tmp/tmp.EJCvH2i3mr
++ rm /tmp/tmp.qGgyI51Y7X /tmp/tmp.EJCvH2i3mr
++ return 0
+ [[ '' == \t\r\u\e ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].hidden.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.naXSYz3yjz
+++ mktemp
++ local LAST_ERR=/tmp/tmp.nRWAjQPe46
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].hidden.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.naXSYz3yjz
++ cat /tmp/tmp.nRWAjQPe46
++ rm /tmp/tmp.naXSYz3yjz /tmp/tmp.nRWAjQPe46
++ return 0
+ [[ '' == \t\r\u\e ]]
+ sleep 10
+ [[ true == \t\r\u\e ]]
+ set +x
Waiting for cluster readyness
+ wait_for_running some-name-mongos 3
+ local name=some-name-mongos
+ let last_pod=2
+ local check_cluster_readyness=true
+ set_debug
+ [[ 1 == 1 ]]
+ set -o xtrace
+ local rs_name=mongos
+ local cluster_name=some-name
++ seq 0 2
+ for i in $(seq 0 $last_pod)
+ [[ 0 -eq 2 ]]
+ wait_pod some-name-mongos-0
+ local pod=some-name-mongos-0
+ set +o xtrace
waiting for pod/some-name-mongos-0 to be ready.OK
+ for i in $(seq 0 $last_pod)
+ [[ 1 -eq 2 ]]
+ wait_pod some-name-mongos-1
+ local pod=some-name-mongos-1
+ set +o xtrace
waiting for pod/some-name-mongos-1 to be ready.OK
+ for i in $(seq 0 $last_pod)
+ [[ 2 -eq 2 ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="mongos")].arbiter.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.Lki6Uykbq1
+++ mktemp
++ local LAST_ERR=/tmp/tmp.KTx6wx2Np7
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="mongos")].arbiter.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.Lki6Uykbq1
++ cat /tmp/tmp.KTx6wx2Np7
++ rm /tmp/tmp.Lki6Uykbq1 /tmp/tmp.KTx6wx2Np7
++ return 0
+ [[ '' == \t\r\u\e ]]
+ wait_pod some-name-mongos-2
+ local pod=some-name-mongos-2
+ set +o xtrace
waiting for pod/some-name-mongos-2 to be ready.OK
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="mongos")].nonvoting.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.FWzFL1O8ki
+++ mktemp
++ local LAST_ERR=/tmp/tmp.TKYhhjUyLB
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="mongos")].nonvoting.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.FWzFL1O8ki
++ cat /tmp/tmp.TKYhhjUyLB
++ rm /tmp/tmp.FWzFL1O8ki /tmp/tmp.TKYhhjUyLB
++ return 0
+ [[ '' == \t\r\u\e ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="mongos")].hidden.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.laftcGajQ2
+++ mktemp
++ local LAST_ERR=/tmp/tmp.yptr584skq
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="mongos")].hidden.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.laftcGajQ2
++ cat /tmp/tmp.yptr584skq
++ rm /tmp/tmp.laftcGajQ2 /tmp/tmp.yptr584skq
++ return 0
+ [[ '' == \t\r\u\e ]]
+ sleep 10
+ [[ true == \t\r\u\e ]]
+ set +x
Waiting for cluster readyness
+ sleep 10
+ compare_mongos_cmd find myApp:myPass@some-name-mongos.pitr-physical-backup-source-4895 -3rd
+ local command=find
+ local uri=myApp:myPass@some-name-mongos.pitr-physical-backup-source-4895
+ local postfix=-3rd
+ local suffix=
+ local database=myApp
+ local collection=test
+ local port=27017
+ local tls=false
+ [[ false == \t\r\u\e ]]
+ mongos_command=run_mongos
+ run_mongos 'use myApp\n db.test.find()' myApp:myPass@some-name-mongos.pitr-physical-backup-source-4895 mongodb '' '' 27017
+ local 'command=use myApp\n db.test.find()'
+ local uri=myApp:myPass@some-name-mongos.pitr-physical-backup-source-4895
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local port=27017
+ local mongo_bin=mongo
+ egrep -v 'I NETWORK|W NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|Started a new thread for the timer service'
+ /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
++ echo .svc.cluster.local
++ awk -F: '{print $2}'
egrep: warning: egrep is obsolescent; using grep -E
+ suffix_port=
+ [[ -z '' ]]
+ suffix=.svc.cluster.local:27017
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.1k7QME6HSG
+++ mktemp
++ local LAST_ERR=/tmp/tmp.s6OYgMFBtl
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.1k7QME6HSG
++ cat /tmp/tmp.s6OYgMFBtl
++ rm /tmp/tmp.1k7QME6HSG /tmp/tmp.s6OYgMFBtl
++ return 0
+ local client_container=psmdb-client-696897d69b-7fp87
+ kubectl_bin exec psmdb-client-696897d69b-7fp87 -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.pitr-physical-backup-source-4895.svc.cluster.local:27017/admin '
++ mktemp
+ local LAST_OUT=/tmp/tmp.NysfIHL9TA
++ mktemp
+ local LAST_ERR=/tmp/tmp.7mWsC4hBzi
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-696897d69b-7fp87 -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.pitr-physical-backup-source-4895.svc.cluster.local:27017/admin '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.NysfIHL9TA
+ cat /tmp/tmp.7mWsC4hBzi
+ rm /tmp/tmp.NysfIHL9TA /tmp/tmp.7mWsC4hBzi
+ return 0
+ [[ 0 -eq 0 ]]
+ diff /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1393/e2e-tests/pitr-physical-backup-source/compare/find-3rd.json /tmp/tmp.XdTK1TTa6I/find-3rd
+ desc 'disable pitr'
+ set +o xtrace


-----------------------------------------------------------------------------------
disable pitr
-----------------------------------------------------------------------------------

+ kubectl patch psmdb some-name --type=merge --patch '{"spec": {"backup": {"pitr": {"enabled": false}}}}'
perconaservermongodb.psmdb.percona.com/some-name patched
+ sleep 20
+ desc 'delete all backups'
+ set +o xtrace


-----------------------------------------------------------------------------------
delete all backups
-----------------------------------------------------------------------------------

+ kubectl_bin delete psmdb-backup --all
++ mktemp
+ local LAST_OUT=/tmp/tmp.QYT8dAhedj
++ mktemp
+ local LAST_ERR=/tmp/tmp.Bgbr4IzvFX
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl delete psmdb-backup --all
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.QYT8dAhedj
perconaservermongodbbackup.psmdb.percona.com "backup-minio-1" deleted from pitr-physical-backup-source-4895 namespace
perconaservermongodbbackup.psmdb.percona.com "backup-minio-2" deleted from pitr-physical-backup-source-4895 namespace
+ cat /tmp/tmp.Bgbr4IzvFX
+ rm /tmp/tmp.QYT8dAhedj /tmp/tmp.Bgbr4IzvFX
+ return 0
+ desc 'destroy cluster'
+ set +o xtrace


-----------------------------------------------------------------------------------
destroy cluster
-----------------------------------------------------------------------------------

+ destroy pitr-physical-backup-source-4895
+ local namespace=pitr-physical-backup-source-4895
+ local ignore_logs=true
+ [[ 0 == 1 ]]
+ desc 'destroy cluster/operator and all other resources'
+ set +o xtrace


-----------------------------------------------------------------------------------
destroy cluster/operator and all other resources
-----------------------------------------------------------------------------------

+ '[' true == false ']'
+ delete_backups
+ desc 'Delete psmdb-backup'
+ set +o xtrace


-----------------------------------------------------------------------------------
Delete psmdb-backup
-----------------------------------------------------------------------------------

++ kubectl_bin get psmdb-backup --no-headers
++ wc -l
+++ mktemp
++ local LAST_OUT=/tmp/tmp.JQxlF9YwR8
+++ mktemp
++ local LAST_ERR=/tmp/tmp.wLJ22Pc4VL
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb-backup --no-headers
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.JQxlF9YwR8
++ cat /tmp/tmp.wLJ22Pc4VL
No resources found in pitr-physical-backup-source-4895 namespace.
++ rm /tmp/tmp.JQxlF9YwR8 /tmp/tmp.wLJ22Pc4VL
++ return 0
+ '[' 0 '!=' 0 ']'
+ delete_crd
+ desc 'get and delete old CRDs and RBAC'
+ set +o xtrace


-----------------------------------------------------------------------------------
get and delete old CRDs and RBAC
-----------------------------------------------------------------------------------

+ kubectl_bin delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1393/deploy/crd.yaml --ignore-not-found --wait=false
++ mktemp
+ local LAST_OUT=/tmp/tmp.5XuBdlQyax
++ mktemp
+ local LAST_ERR=/tmp/tmp.1odfURcZtK
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1393/deploy/crd.yaml --ignore-not-found --wait=false
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.5XuBdlQyax
customresourcedefinition.apiextensions.k8s.io "perconaservermongodbbackups.psmdb.percona.com" deleted
customresourcedefinition.apiextensions.k8s.io "perconaservermongodbrestores.psmdb.percona.com" deleted
customresourcedefinition.apiextensions.k8s.io "perconaservermongodbs.psmdb.percona.com" deleted
+ cat /tmp/tmp.1odfURcZtK
+ rm /tmp/tmp.5XuBdlQyax /tmp/tmp.1odfURcZtK
+ return 0
++ yq eval .metadata.name /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1393/deploy/crd.yaml
++ grep -v '\-\-\-'
grep: warning: stray \ before -
grep: warning: stray \ before -
+ for crd_name in $(yq eval '.metadata.name' "${src_dir}/deploy/crd.yaml" | grep -v '\-\-\-')
+ kubectl get perconaservermongodbbackups.psmdb.percona.com --all-namespaces -o wide
+ grep -v NAMESPACE
+ xargs -L 1 sh -xc 'kubectl patch perconaservermongodbbackups.psmdb.percona.com -n $0 $1 --type=merge -p "{\"metadata\":{\"finalizers\":[]}}"'
error: the server doesn't have a resource type "perconaservermongodbbackups"
+ kubectl patch perconaservermongodbbackups.psmdb.percona.com -n sh --type=merge -p '{"metadata":{"finalizers":[]}}'
error: the server doesn't have a resource type "perconaservermongodbbackups"
+ :
+ kubectl_bin wait --for=delete crd perconaservermongodbbackups.psmdb.percona.com
++ mktemp
+ local LAST_OUT=/tmp/tmp.hhpvz31HCt
++ mktemp
+ local LAST_ERR=/tmp/tmp.nJZuevUOaD
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl wait --for=delete crd perconaservermongodbbackups.psmdb.percona.com
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.hhpvz31HCt
+ cat /tmp/tmp.nJZuevUOaD
+ rm /tmp/tmp.hhpvz31HCt /tmp/tmp.nJZuevUOaD
+ return 0
+ for crd_name in $(yq eval '.metadata.name' "${src_dir}/deploy/crd.yaml" | grep -v '\-\-\-')
+ kubectl get perconaservermongodbrestores.psmdb.percona.com --all-namespaces -o wide
+ grep -v NAMESPACE
+ xargs -L 1 sh -xc 'kubectl patch perconaservermongodbrestores.psmdb.percona.com -n $0 $1 --type=merge -p "{\"metadata\":{\"finalizers\":[]}}"'
error: the server doesn't have a resource type "perconaservermongodbrestores"
+ kubectl patch perconaservermongodbrestores.psmdb.percona.com -n sh --type=merge -p '{"metadata":{"finalizers":[]}}'
error: the server doesn't have a resource type "perconaservermongodbrestores"
+ :
+ kubectl_bin wait --for=delete crd perconaservermongodbrestores.psmdb.percona.com
++ mktemp
+ local LAST_OUT=/tmp/tmp.QBCmM4kG3d
++ mktemp
+ local LAST_ERR=/tmp/tmp.Rmy73oUj9f
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl wait --for=delete crd perconaservermongodbrestores.psmdb.percona.com
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.QBCmM4kG3d
+ cat /tmp/tmp.Rmy73oUj9f
+ rm /tmp/tmp.QBCmM4kG3d /tmp/tmp.Rmy73oUj9f
+ return 0
+ for crd_name in $(yq eval '.metadata.name' "${src_dir}/deploy/crd.yaml" | grep -v '\-\-\-')
+ kubectl get perconaservermongodbs.psmdb.percona.com --all-namespaces -o wide
+ grep -v NAMESPACE
+ xargs -L 1 sh -xc 'kubectl patch perconaservermongodbs.psmdb.percona.com -n $0 $1 --type=merge -p "{\"metadata\":{\"finalizers\":[]}}"'
Error from server (NotFound): Unable to list "psmdb.percona.com/v1, Resource=perconaservermongodbs": the server could not find the requested resource (get perconaservermongodbs.psmdb.percona.com)
+ kubectl patch perconaservermongodbs.psmdb.percona.com -n sh --type=merge -p '{"metadata":{"finalizers":[]}}'
error: the server doesn't have a resource type "perconaservermongodbs"
+ :
+ kubectl_bin wait --for=delete crd perconaservermongodbs.psmdb.percona.com
++ mktemp
+ local LAST_OUT=/tmp/tmp.sR87zcqg2R
++ mktemp
+ local LAST_ERR=/tmp/tmp.M79r6kRpwe
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl wait --for=delete crd perconaservermongodbs.psmdb.percona.com
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.sR87zcqg2R
+ cat /tmp/tmp.M79r6kRpwe
+ rm /tmp/tmp.sR87zcqg2R /tmp/tmp.M79r6kRpwe
+ return 0
+ local rbac_yaml=rbac.yaml
+ '[' -n psmdb-operator ']'
+ rbac_yaml=cw-rbac.yaml
+ kubectl_bin delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1393/deploy/cw-rbac.yaml --ignore-not-found
++ mktemp
+ local LAST_OUT=/tmp/tmp.wJGMdFIiYf
++ mktemp
+ local LAST_ERR=/tmp/tmp.BBJIzmTBbg
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1393/deploy/cw-rbac.yaml --ignore-not-found
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.wJGMdFIiYf
clusterrole.rbac.authorization.k8s.io "percona-server-mongodb-operator" deleted
clusterrolebinding.rbac.authorization.k8s.io "service-account-percona-server-mongodb-operator" deleted
+ cat /tmp/tmp.BBJIzmTBbg
+ rm /tmp/tmp.wJGMdFIiYf /tmp/tmp.BBJIzmTBbg
+ return 0
+ destroy_cert_manager
+ kubectl_bin delete -f https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml
++ mktemp
+ local LAST_OUT=/tmp/tmp.ywdFpnuJQT
++ mktemp
+ local LAST_ERR=/tmp/tmp.sDmzmZ4mkF
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl delete -f https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml
+ exit_status=1
+ set -e
+ '[' 1 '!=' 0 -a -n 1 ']'
+ cat /tmp/tmp.ywdFpnuJQT
+ cat /tmp/tmp.sDmzmZ4mkF
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": namespaces "cert-manager" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "challenges.acme.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "orders.acme.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "certificaterequests.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "certificates.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "clusterissuers.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "issuers.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": serviceaccounts "cert-manager-cainjector" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": serviceaccounts "cert-manager" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": serviceaccounts "cert-manager-webhook" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-cainjector" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-issuers" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-clusterissuers" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-certificates" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-orders" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-challenges" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-ingress-shim" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-cluster-view" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-view" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-edit" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-approve:cert-manager-io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-certificatesigningrequests" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-webhook:subjectaccessreviews" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-cainjector" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-issuers" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-clusterissuers" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-certificates" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-orders" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-challenges" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-ingress-shim" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-approve:cert-manager-io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-certificatesigningrequests" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-webhook:subjectaccessreviews" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager-cainjector:leaderelection" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager:leaderelection" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager-tokenrequest" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager-webhook:dynamic-serving" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager-cainjector:leaderelection" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager:leaderelection" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager-tokenrequest" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager-webhook:dynamic-serving" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": services "cert-manager-cainjector" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": services "cert-manager" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": services "cert-manager-webhook" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": deployments.apps "cert-manager-cainjector" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": deployments.apps "cert-manager" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": deployments.apps "cert-manager-webhook" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": mutatingwebhookconfigurations.admissionregistration.k8s.io "cert-manager-webhook" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": validatingwebhookconfigurations.admissionregistration.k8s.io "cert-manager-webhook" not found
+ sleep 0
+ for i in $(seq 0 2)
+ set +e
+ kubectl delete -f https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml
+ exit_status=1
+ set -e
+ '[' 1 '!=' 0 -a -n 1 ']'
+ cat /tmp/tmp.ywdFpnuJQT
+ cat /tmp/tmp.sDmzmZ4mkF
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": namespaces "cert-manager" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "challenges.acme.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "orders.acme.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "certificaterequests.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "certificates.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "clusterissuers.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "issuers.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": serviceaccounts "cert-manager-cainjector" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": serviceaccounts "cert-manager" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": serviceaccounts "cert-manager-webhook" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-cainjector" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-issuers" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-clusterissuers" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-certificates" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-orders" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-challenges" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-ingress-shim" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-cluster-view" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-view" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-edit" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-approve:cert-manager-io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-certificatesigningrequests" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-webhook:subjectaccessreviews" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-cainjector" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-issuers" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-clusterissuers" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-certificates" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-orders" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-challenges" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-ingress-shim" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-approve:cert-manager-io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-certificatesigningrequests" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-webhook:subjectaccessreviews" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager-cainjector:leaderelection" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager:leaderelection" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager-tokenrequest" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager-webhook:dynamic-serving" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager-cainjector:leaderelection" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager:leaderelection" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager-tokenrequest" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager-webhook:dynamic-serving" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": services "cert-manager-cainjector" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": services "cert-manager" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": services "cert-manager-webhook" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": deployments.apps "cert-manager-cainjector" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": deployments.apps "cert-manager" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": deployments.apps "cert-manager-webhook" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": mutatingwebhookconfigurations.admissionregistration.k8s.io "cert-manager-webhook" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": validatingwebhookconfigurations.admissionregistration.k8s.io "cert-manager-webhook" not found
+ sleep 4
+ for i in $(seq 0 2)
+ set +e
+ kubectl delete -f https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml
+ exit_status=1
+ set -e
+ '[' 1 '!=' 0 -a -n 1 ']'
+ cat /tmp/tmp.ywdFpnuJQT
+ cat /tmp/tmp.sDmzmZ4mkF
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": namespaces "cert-manager" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "challenges.acme.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "orders.acme.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "certificaterequests.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "certificates.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "clusterissuers.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "issuers.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": serviceaccounts "cert-manager-cainjector" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": serviceaccounts "cert-manager" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": serviceaccounts "cert-manager-webhook" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-cainjector" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-issuers" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-clusterissuers" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-certificates" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-orders" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-challenges" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-ingress-shim" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-cluster-view" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-view" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-edit" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-approve:cert-manager-io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-certificatesigningrequests" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-webhook:subjectaccessreviews" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-cainjector" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-issuers" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-clusterissuers" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-certificates" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-orders" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-challenges" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-ingress-shim" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-approve:cert-manager-io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-certificatesigningrequests" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-webhook:subjectaccessreviews" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager-cainjector:leaderelection" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager:leaderelection" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager-tokenrequest" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager-webhook:dynamic-serving" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager-cainjector:leaderelection" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager:leaderelection" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager-tokenrequest" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager-webhook:dynamic-serving" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": services "cert-manager-cainjector" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": services "cert-manager" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": services "cert-manager-webhook" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": deployments.apps "cert-manager-cainjector" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": deployments.apps "cert-manager" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": deployments.apps "cert-manager-webhook" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": mutatingwebhookconfigurations.admissionregistration.k8s.io "cert-manager-webhook" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": validatingwebhookconfigurations.admissionregistration.k8s.io "cert-manager-webhook" not found
+ sleep 8
+ cat /tmp/tmp.ywdFpnuJQT
+ cat /tmp/tmp.sDmzmZ4mkF
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": namespaces "cert-manager" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "challenges.acme.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "orders.acme.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "certificaterequests.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "certificates.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "clusterissuers.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "issuers.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": serviceaccounts "cert-manager-cainjector" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": serviceaccounts "cert-manager" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": serviceaccounts "cert-manager-webhook" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-cainjector" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-issuers" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-clusterissuers" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-certificates" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-orders" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-challenges" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-ingress-shim" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-cluster-view" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-view" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-edit" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-approve:cert-manager-io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-certificatesigningrequests" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-webhook:subjectaccessreviews" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-cainjector" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-issuers" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-clusterissuers" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-certificates" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-orders" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-challenges" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-ingress-shim" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-approve:cert-manager-io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-certificatesigningrequests" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-webhook:subjectaccessreviews" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager-cainjector:leaderelection" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager:leaderelection" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager-tokenrequest" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager-webhook:dynamic-serving" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager-cainjector:leaderelection" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager:leaderelection" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager-tokenrequest" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager-webhook:dynamic-serving" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": services "cert-manager-cainjector" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": services "cert-manager" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": services "cert-manager-webhook" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": deployments.apps "cert-manager-cainjector" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": deployments.apps "cert-manager" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": deployments.apps "cert-manager-webhook" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": mutatingwebhookconfigurations.admissionregistration.k8s.io "cert-manager-webhook" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": validatingwebhookconfigurations.admissionregistration.k8s.io "cert-manager-webhook" not found
+ rm /tmp/tmp.ywdFpnuJQT /tmp/tmp.sDmzmZ4mkF
+ return 1
+ true
+ '[' -n '' ']'
+ '[' -n psmdb-operator ']'
+ kubectl_bin delete --grace-period=0 --force=true namespace pitr-physical-backup-source-4895
+ rm -rf /tmp/tmp.XdTK1TTa6I
+ kubectl_bin delete --grace-period=0 --force=true namespace psmdb-operator
++ mktemp
++ mktemp
+ desc 'test passed'
+ set +o xtrace


-----------------------------------------------------------------------------------
test passed
-----------------------------------------------------------------------------------

+ local LAST_OUT=/tmp/tmp.yfDp5Wml8p
+ local LAST_OUT=/tmp/tmp.lZUi0JjaDP
++ mktemp
++ mktemp
+ local LAST_ERR=/tmp/tmp.oIK6H9YA6w
+ local exit_status=0
+ local timeout=4
+ local LAST_ERR=/tmp/tmp.hEDtr4564C
+ local exit_status=0
+ local timeout=4
++ seq 0 2
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl delete --grace-period=0 --force=true namespace psmdb-operator
+ for i in $(seq 0 2)
+ set +e
+ kubectl delete --grace-period=0 --force=true namespace pitr-physical-backup-source-4895