Log: /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2158/e2e-tests/logs/pitr-physical.log
grep: warning: stray \ before -
Warning: version difference between client (1.35) and server (1.31) exceeds the supported minor version skew of +/-1
Warning: version difference between client (1.35) and server (1.31) exceeds the supported minor version skew of +/-1
Warning: version difference between client (1.35) and server (1.31) exceeds the supported minor version skew of +/-1
+ main
+ create_infra pitr-physical-28895
+ local ns=pitr-physical-28895
+ [[ 1 == 1 ]]
+ delete_crd
+ desc 'get and delete old CRDs and RBAC'
+ set +o xtrace


-----------------------------------------------------------------------------------
get and delete old CRDs and RBAC
-----------------------------------------------------------------------------------

+ kubectl_bin delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2158/deploy/crd.yaml --ignore-not-found --wait=false
++ mktemp
+ local LAST_OUT=/tmp/tmp.psDLH0gHo3
++ mktemp
+ local LAST_ERR=/tmp/tmp.6w793kF1GD
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2158/deploy/crd.yaml --ignore-not-found --wait=false
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.psDLH0gHo3
customresourcedefinition.apiextensions.k8s.io "perconaservermongodbbackups.psmdb.percona.com" deleted
customresourcedefinition.apiextensions.k8s.io "perconaservermongodbrestores.psmdb.percona.com" deleted
customresourcedefinition.apiextensions.k8s.io "perconaservermongodbs.psmdb.percona.com" deleted
+ cat /tmp/tmp.6w793kF1GD
+ rm /tmp/tmp.psDLH0gHo3 /tmp/tmp.6w793kF1GD
+ return 0
++ yq eval .metadata.name /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2158/deploy/crd.yaml
++ grep -v '\-\-\-'
grep: warning: stray \ before -
grep: warning: stray \ before -
+ for crd_name in $(yq eval '.metadata.name' "${src_dir}/deploy/crd.yaml" | grep -v '\-\-\-')
+ kubectl get perconaservermongodbbackups.psmdb.percona.com --all-namespaces -o wide
+ grep -v NAMESPACE
+ xargs -L 1 sh -xc 'kubectl patch perconaservermongodbbackups.psmdb.percona.com -n $0 $1 --type=merge -p "{\"metadata\":{\"finalizers\":[]}}"'
No resources found
+ kubectl patch perconaservermongodbbackups.psmdb.percona.com -n sh --type=merge -p '{"metadata":{"finalizers":[]}}'
error: resource(s) were provided, but no name was specified
+ :
+ kubectl_bin wait --for=delete crd perconaservermongodbbackups.psmdb.percona.com
++ mktemp
+ local LAST_OUT=/tmp/tmp.iC9sGN5srY
++ mktemp
+ local LAST_ERR=/tmp/tmp.45t2LMTgc7
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl wait --for=delete crd perconaservermongodbbackups.psmdb.percona.com
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.iC9sGN5srY
+ cat /tmp/tmp.45t2LMTgc7
+ rm /tmp/tmp.iC9sGN5srY /tmp/tmp.45t2LMTgc7
+ return 0
+ for crd_name in $(yq eval '.metadata.name' "${src_dir}/deploy/crd.yaml" | grep -v '\-\-\-')
+ kubectl get perconaservermongodbrestores.psmdb.percona.com --all-namespaces -o wide
+ grep -v NAMESPACE
+ xargs -L 1 sh -xc 'kubectl patch perconaservermongodbrestores.psmdb.percona.com -n $0 $1 --type=merge -p "{\"metadata\":{\"finalizers\":[]}}"'
error: the server doesn't have a resource type "perconaservermongodbrestores"
+ kubectl patch perconaservermongodbrestores.psmdb.percona.com -n sh --type=merge -p '{"metadata":{"finalizers":[]}}'
error: the server doesn't have a resource type "perconaservermongodbrestores"
+ :
+ kubectl_bin wait --for=delete crd perconaservermongodbrestores.psmdb.percona.com
++ mktemp
+ local LAST_OUT=/tmp/tmp.A9cQ01Dw3j
++ mktemp
+ local LAST_ERR=/tmp/tmp.R5E3Aoyd0e
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl wait --for=delete crd perconaservermongodbrestores.psmdb.percona.com
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.A9cQ01Dw3j
+ cat /tmp/tmp.R5E3Aoyd0e
+ rm /tmp/tmp.A9cQ01Dw3j /tmp/tmp.R5E3Aoyd0e
+ return 0
+ for crd_name in $(yq eval '.metadata.name' "${src_dir}/deploy/crd.yaml" | grep -v '\-\-\-')
+ kubectl get perconaservermongodbs.psmdb.percona.com --all-namespaces -o wide
+ grep -v NAMESPACE
+ xargs -L 1 sh -xc 'kubectl patch perconaservermongodbs.psmdb.percona.com -n $0 $1 --type=merge -p "{\"metadata\":{\"finalizers\":[]}}"'
error: the server doesn't have a resource type "perconaservermongodbs"
+ kubectl patch perconaservermongodbs.psmdb.percona.com -n sh --type=merge -p '{"metadata":{"finalizers":[]}}'
error: the server doesn't have a resource type "perconaservermongodbs"
+ :
+ kubectl_bin wait --for=delete crd perconaservermongodbs.psmdb.percona.com
++ mktemp
+ local LAST_OUT=/tmp/tmp.G8dusPtvHu
++ mktemp
+ local LAST_ERR=/tmp/tmp.fuNuhh1qlG
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl wait --for=delete crd perconaservermongodbs.psmdb.percona.com
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.G8dusPtvHu
+ cat /tmp/tmp.fuNuhh1qlG
+ rm /tmp/tmp.G8dusPtvHu /tmp/tmp.fuNuhh1qlG
+ return 0
+ local rbac_yaml=rbac.yaml
+ '[' -n psmdb-operator ']'
+ rbac_yaml=cw-rbac.yaml
+ kubectl_bin delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2158/deploy/cw-rbac.yaml --ignore-not-found
++ mktemp
+ local LAST_OUT=/tmp/tmp.45Gpxb82i0
++ mktemp
+ local LAST_ERR=/tmp/tmp.QIBZfSvZYj
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2158/deploy/cw-rbac.yaml --ignore-not-found
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.45Gpxb82i0
clusterrole.rbac.authorization.k8s.io "percona-server-mongodb-operator" deleted
clusterrolebinding.rbac.authorization.k8s.io "service-account-percona-server-mongodb-operator" deleted
+ cat /tmp/tmp.QIBZfSvZYj
+ rm /tmp/tmp.45Gpxb82i0 /tmp/tmp.QIBZfSvZYj
+ return 0
+ check_crd_for_deletion PR-2158-c70b6c4a
+ local git_tag=PR-2158-c70b6c4a
++ curl -s https://raw.githubusercontent.com/percona/percona-server-mongodb-operator/PR-2158-c70b6c4a/deploy/crd.yaml
++ yq eval .metadata.name
++ /usr/sbin/sed s/---//g
++ /usr/sbin/sed ':a;N;$!ba;s/\n/ /g'
+ for crd_name in $(curl -s https://raw.githubusercontent.com/percona/percona-server-mongodb-operator/${git_tag}/deploy/crd.yaml | yq eval '.metadata.name' | $sed 's/---//g' | $sed ':a;N;$!ba;s/\n/ /g')
++ kubectl_bin get crd/null -o 'jsonpath={.status.conditions[-1].type}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.1CexhQ97oX
+++ mktemp
++ local LAST_ERR=/tmp/tmp.9NODHucrIH
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get crd/null -o 'jsonpath={.status.conditions[-1].type}'
++ exit_status=1
++ set -e
++ '[' 1 '!=' 0 -a -n 1 ']'
++ cat /tmp/tmp.1CexhQ97oX
++ cat /tmp/tmp.9NODHucrIH
Error from server (NotFound): customresourcedefinitions.apiextensions.k8s.io "null" not found
++ sleep 0
++ for i in $(seq 0 2)
++ set +e
++ kubectl get crd/null -o 'jsonpath={.status.conditions[-1].type}'
++ exit_status=1
++ set -e
++ '[' 1 '!=' 0 -a -n 1 ']'
++ cat /tmp/tmp.1CexhQ97oX
++ cat /tmp/tmp.9NODHucrIH
Error from server (NotFound): customresourcedefinitions.apiextensions.k8s.io "null" not found
++ sleep 4
++ for i in $(seq 0 2)
++ set +e
++ kubectl get crd/null -o 'jsonpath={.status.conditions[-1].type}'
++ exit_status=1
++ set -e
++ '[' 1 '!=' 0 -a -n 1 ']'
++ cat /tmp/tmp.1CexhQ97oX
++ cat /tmp/tmp.9NODHucrIH
Error from server (NotFound): customresourcedefinitions.apiextensions.k8s.io "null" not found
++ sleep 8
++ cat /tmp/tmp.1CexhQ97oX
++ cat /tmp/tmp.9NODHucrIH
Error from server (NotFound): customresourcedefinitions.apiextensions.k8s.io "null" not found
++ rm /tmp/tmp.1CexhQ97oX /tmp/tmp.9NODHucrIH
++ return 1
+ [[ '' == \T\e\r\m\i\n\a\t\i\n\g ]]
+ '[' -n psmdb-operator ']'
+ create_namespace psmdb-operator
+ local namespace=psmdb-operator
+ local skip_clean_namespace=
+ [[ 1 == 1 ]]
+ [[ -z '' ]]
+ destroy_chaos_mesh
++ helm list --all-namespaces --filter chaos-mesh
++ tail -n1
++ awk '-F ' '{print $2}'
++ sed s/NAMESPACE//
+ local chaos_mesh_ns=
+ desc 'destroy chaos-mesh'
+ set +o xtrace


-----------------------------------------------------------------------------------
destroy chaos-mesh
-----------------------------------------------------------------------------------

+ '[' -n '' ']'
++ kubectl get MutatingWebhookConfiguration
++ grep chaos-mesh
++ awk '{print $1}'
+ timeout 30 kubectl delete MutatingWebhookConfiguration
error: resource(s) were provided, but no name was specified
+ :
++ kubectl get ValidatingWebhookConfiguration
++ grep chaos-mesh
++ awk '{print $1}'
+ timeout 30 kubectl delete ValidatingWebhookConfiguration
error: resource(s) were provided, but no name was specified
+ :
++ kubectl get ValidatingWebhookConfiguration
++ grep validate-auth
++ awk '{print $1}'
+ timeout 30 kubectl delete ValidatingWebhookConfiguration
error: resource(s) were provided, but no name was specified
+ :
++ kubectl api-resources
++ grep chaos-mesh
++ awk '{print $1}'
++ kubectl get crd
++ grep chaos-mesh.org
++ awk '{print $1}'
+ timeout 30 kubectl delete crd
error: resource(s) were provided, but no name was specified
+ :
++ kubectl get clusterrolebinding
++ grep chaos-mesh
++ awk '{print $1}'
+ timeout 30 kubectl delete clusterrolebinding
error: resource(s) were provided, but no name was specified
+ :
++ kubectl get clusterrole
++ grep chaos-mesh
++ awk '{print $1}'
+ timeout 30 kubectl delete clusterrole
error: resource(s) were provided, but no name was specified
+ :
+ desc 'cleaned up all old namespaces'
+ set +o xtrace


-----------------------------------------------------------------------------------
cleaned up all old namespaces
-----------------------------------------------------------------------------------

+ kubectl_bin get ns
+ egrep -v '^kube-|^default|Terminating|psmdb-operator|openshift|^gke-|^gmp-|^NAME'
+ awk '{print$1}'
+ '[' -n '' ']'
+ desc 'cleaned up old namespaces psmdb-operator'
++ mktemp
+ set +o xtrace
+ xargs kubectl delete ns


-----------------------------------------------------------------------------------
cleaned up old namespaces psmdb-operator
-----------------------------------------------------------------------------------

+ kubectl_bin delete namespace psmdb-operator --ignore-not-found
++ mktemp
egrep: warning: egrep is obsolescent; using grep -E
+ local LAST_OUT=/tmp/tmp.lQqf1sKt9H
++ mktemp
+ local LAST_OUT=/tmp/tmp.ja0YDTg8DD
++ mktemp
+ local LAST_ERR=/tmp/tmp.thglY2J2OP
+ local exit_status=0
+ local timeout=4
+ local LAST_ERR=/tmp/tmp.Q43kO16Ptg
+ local exit_status=0
+ local timeout=4
++ seq 0 2
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl get ns
+ for i in $(seq 0 2)
+ set +e
+ kubectl delete namespace psmdb-operator --ignore-not-found
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.lQqf1sKt9H
+ cat /tmp/tmp.thglY2J2OP
+ rm /tmp/tmp.lQqf1sKt9H /tmp/tmp.thglY2J2OP
+ return 0
namespace "pitr-physical-24291" deleted
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.ja0YDTg8DD
namespace "psmdb-operator" deleted
+ cat /tmp/tmp.Q43kO16Ptg
+ rm /tmp/tmp.ja0YDTg8DD /tmp/tmp.Q43kO16Ptg
+ return 0
+ kubectl_bin wait --for=delete namespace psmdb-operator
++ mktemp
+ local LAST_OUT=/tmp/tmp.ascr8x8lFt
++ mktemp
+ local LAST_ERR=/tmp/tmp.5eqqXLVMHt
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl wait --for=delete namespace psmdb-operator
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.ascr8x8lFt
+ cat /tmp/tmp.5eqqXLVMHt
+ rm /tmp/tmp.ascr8x8lFt /tmp/tmp.5eqqXLVMHt
+ return 0
+ desc 'create namespace psmdb-operator'
+ set +o xtrace


-----------------------------------------------------------------------------------
create namespace psmdb-operator
-----------------------------------------------------------------------------------

+ kubectl_bin create namespace psmdb-operator
++ mktemp
+ local LAST_OUT=/tmp/tmp.J4FO8l3vDM
++ mktemp
+ local LAST_ERR=/tmp/tmp.uZPgDgqy9K
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl create namespace psmdb-operator
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.J4FO8l3vDM
namespace/psmdb-operator created
+ cat /tmp/tmp.uZPgDgqy9K
+ rm /tmp/tmp.J4FO8l3vDM /tmp/tmp.uZPgDgqy9K
+ return 0
+ set_kube_ctx psmdb-operator
+ local namespace=psmdb-operator
++ kubectl_bin config current-context
+++ mktemp
++ local LAST_OUT=/tmp/tmp.CSKWa80nqp
+++ mktemp
++ local LAST_ERR=/tmp/tmp.yFvc5FPXLi
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl config current-context
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.CSKWa80nqp
++ cat /tmp/tmp.yFvc5FPXLi
++ rm /tmp/tmp.CSKWa80nqp /tmp/tmp.yFvc5FPXLi
++ return 0
+ kubectl_bin config set-context gke_cloud-dev-112233_us-central1-a_jen-psmdb-2158-c70b6c4a-2-cluster7 --namespace=psmdb-operator
++ mktemp
+ local LAST_OUT=/tmp/tmp.WLePGwXSqD
++ mktemp
+ local LAST_ERR=/tmp/tmp.ArDr9QuYb2
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl config set-context gke_cloud-dev-112233_us-central1-a_jen-psmdb-2158-c70b6c4a-2-cluster7 --namespace=psmdb-operator
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.WLePGwXSqD
Context "gke_cloud-dev-112233_us-central1-a_jen-psmdb-2158-c70b6c4a-2-cluster7" modified.
+ cat /tmp/tmp.ArDr9QuYb2
+ rm /tmp/tmp.WLePGwXSqD /tmp/tmp.ArDr9QuYb2
+ return 0
+ deploy_operator
+ desc 'start PSMDB operator: perconalab/percona-server-mongodb-operator:PR-2158-c70b6c4a'
+ set +o xtrace


-----------------------------------------------------------------------------------
start PSMDB operator: perconalab/percona-server-mongodb-operator:PR-2158-c70b6c4a
-----------------------------------------------------------------------------------

+ local cr_file
+ '[' -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2158/e2e-tests/pitr-physical/conf/crd.yaml ']'
+ cr_file=/mnt/jenkins/workspace/cloud-psmdb-operator_PR-2158/deploy/crd.yaml
+ kubectl_bin apply --server-side --force-conflicts -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2158/deploy/crd.yaml
++ mktemp
+ local LAST_OUT=/tmp/tmp.46IIMSRzoW
++ mktemp
+ local LAST_ERR=/tmp/tmp.DfAsQu05P5
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl apply --server-side --force-conflicts -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2158/deploy/crd.yaml
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.46IIMSRzoW
customresourcedefinition.apiextensions.k8s.io/perconaservermongodbbackups.psmdb.percona.com serverside-applied
customresourcedefinition.apiextensions.k8s.io/perconaservermongodbrestores.psmdb.percona.com serverside-applied
customresourcedefinition.apiextensions.k8s.io/perconaservermongodbs.psmdb.percona.com serverside-applied
+ cat /tmp/tmp.DfAsQu05P5
+ rm /tmp/tmp.46IIMSRzoW /tmp/tmp.DfAsQu05P5
+ return 0
+ '[' -n psmdb-operator ']'
+ apply_rbac cw-rbac
+ local operator_namespace=psmdb-operator
+ local rbac=cw-rbac
+ cat /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2158/deploy/cw-rbac.yaml
+ sed -e 's^namespace: .*^namespace: psmdb-operator^'
+ kubectl_bin apply -n psmdb-operator -f -
++ mktemp
+ local LAST_OUT=/tmp/tmp.eQNovwiHXz
++ mktemp
+ local LAST_ERR=/tmp/tmp.Oa5ISbFYa8
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl apply -n psmdb-operator -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.eQNovwiHXz
clusterrole.rbac.authorization.k8s.io/percona-server-mongodb-operator created
serviceaccount/percona-server-mongodb-operator created
clusterrolebinding.rbac.authorization.k8s.io/service-account-percona-server-mongodb-operator created
+ cat /tmp/tmp.Oa5ISbFYa8
+ rm /tmp/tmp.eQNovwiHXz /tmp/tmp.Oa5ISbFYa8
+ return 0
+ yq eval '
			(.spec.template.spec.containers[].image = "perconalab/percona-server-mongodb-operator:PR-2158-c70b6c4a") |
			((.. | select(.[] == "DISABLE_TELEMETRY")) |= .value="true") |
			((.. | select(.[] == "LOG_LEVEL")) |= .value="DEBUG")' /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2158/deploy/cw-operator.yaml
+ kubectl_bin apply -n psmdb-operator -f -
++ mktemp
+ local LAST_OUT=/tmp/tmp.AHyPKGI4ee
++ mktemp
+ local LAST_ERR=/tmp/tmp.jPlrtBafWL
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl apply -n psmdb-operator -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.AHyPKGI4ee
deployment.apps/percona-server-mongodb-operator created
+ cat /tmp/tmp.jPlrtBafWL
+ rm /tmp/tmp.AHyPKGI4ee /tmp/tmp.jPlrtBafWL
+ return 0
+ sleep 20
++ get_operator_pod
++ kubectl_bin get pods --selector=name=percona-server-mongodb-operator -o 'jsonpath={.items[].metadata.name}' -n psmdb-operator
+++ mktemp
++ local LAST_OUT=/tmp/tmp.a8m66SqTjC
+++ mktemp
++ local LAST_ERR=/tmp/tmp.sTdcKuAXno
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=percona-server-mongodb-operator -o 'jsonpath={.items[].metadata.name}' -n psmdb-operator
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.a8m66SqTjC
++ cat /tmp/tmp.sTdcKuAXno
++ rm /tmp/tmp.a8m66SqTjC /tmp/tmp.sTdcKuAXno
++ return 0
+ wait_operator_pod percona-server-mongodb-operator-6999c84b9c-ljm57
+ local pod=percona-server-mongodb-operator-6999c84b9c-ljm57
+ set +o xtrace
waiting for pod/percona-server-mongodb-operator-6999c84b9c-ljm57 to be ready.OK
+ echo 'Print operator info from log'
Print operator info from log
+ grep 'Manager starting up'
++ get_operator_pod
++ kubectl_bin get pods --selector=name=percona-server-mongodb-operator -o 'jsonpath={.items[].metadata.name}' -n psmdb-operator
+++ mktemp
++ local LAST_OUT=/tmp/tmp.y6DBAOXCli
+++ mktemp
++ local LAST_ERR=/tmp/tmp.ghIyQzqLB9
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=percona-server-mongodb-operator -o 'jsonpath={.items[].metadata.name}' -n psmdb-operator
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.y6DBAOXCli
++ cat /tmp/tmp.ghIyQzqLB9
++ rm /tmp/tmp.y6DBAOXCli /tmp/tmp.ghIyQzqLB9
++ return 0
+ kubectl_bin logs -n psmdb-operator percona-server-mongodb-operator-6999c84b9c-ljm57
++ mktemp
+ local LAST_OUT=/tmp/tmp.boRa9sW8pl
++ mktemp
+ local LAST_ERR=/tmp/tmp.YtyX8wmG0G
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl logs -n psmdb-operator percona-server-mongodb-operator-6999c84b9c-ljm57
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.boRa9sW8pl
+ cat /tmp/tmp.YtyX8wmG0G
+ rm /tmp/tmp.boRa9sW8pl /tmp/tmp.YtyX8wmG0G
+ return 0
2025-12-23T15:06:45.405Z	INFO	setup	Manager starting up	{"gitCommit": "c70b6c4a288beb141674b1787dd0e83d6ad0098a", "gitBranch": "PR-2158-c70b6c4a", "buildTime": "", "goVersion": "go1.25.5", "os": "linux", "arch": "amd64"}
+ create_namespace pitr-physical-28895
+ local namespace=pitr-physical-28895
+ local skip_clean_namespace=
+ [[ 1 == 1 ]]
+ [[ -z '' ]]
+ destroy_chaos_mesh
++ helm list --all-namespaces --filter chaos-mesh
++ tail -n1
++ awk '-F ' '{print $2}'
++ sed s/NAMESPACE//
+ local chaos_mesh_ns=
+ desc 'destroy chaos-mesh'
+ set +o xtrace


-----------------------------------------------------------------------------------
destroy chaos-mesh
-----------------------------------------------------------------------------------

+ '[' -n '' ']'
++ kubectl get MutatingWebhookConfiguration
++ grep chaos-mesh
++ awk '{print $1}'
+ timeout 30 kubectl delete MutatingWebhookConfiguration
error: resource(s) were provided, but no name was specified
+ :
++ kubectl get ValidatingWebhookConfiguration
++ grep chaos-mesh
++ awk '{print $1}'
+ timeout 30 kubectl delete ValidatingWebhookConfiguration
error: resource(s) were provided, but no name was specified
+ :
++ kubectl get ValidatingWebhookConfiguration
++ grep validate-auth
++ awk '{print $1}'
+ timeout 30 kubectl delete ValidatingWebhookConfiguration
error: resource(s) were provided, but no name was specified
+ :
++ kubectl api-resources
++ grep chaos-mesh
++ awk '{print $1}'
++ kubectl get crd
++ grep chaos-mesh.org
++ awk '{print $1}'
+ timeout 30 kubectl delete crd
error: resource(s) were provided, but no name was specified
+ :
++ kubectl get clusterrolebinding
++ grep chaos-mesh
++ awk '{print $1}'
+ timeout 30 kubectl delete clusterrolebinding
error: resource(s) were provided, but no name was specified
+ :
++ kubectl get clusterrole
++ grep chaos-mesh
++ awk '{print $1}'
+ timeout 30 kubectl delete clusterrole
error: resource(s) were provided, but no name was specified
+ :
+ desc 'cleaned up all old namespaces'
+ set +o xtrace


-----------------------------------------------------------------------------------
cleaned up all old namespaces
-----------------------------------------------------------------------------------

+ kubectl_bin get ns
+ egrep -v '^kube-|^default|Terminating|psmdb-operator|openshift|^gke-|^gmp-|^NAME'
+ '[' -n '' ']'
+ desc 'cleaned up old namespaces pitr-physical-28895'
+ set +o xtrace


-----------------------------------------------------------------------------------
cleaned up old namespaces pitr-physical-28895
-----------------------------------------------------------------------------------

+ awk '{print$1}'
+ kubectl_bin delete namespace pitr-physical-28895 --ignore-not-found
+ xargs kubectl delete ns
++ mktemp
++ mktemp
egrep: warning: egrep is obsolescent; using grep -E
+ local LAST_OUT=/tmp/tmp.JwO4DsRqQ3
+ local LAST_OUT=/tmp/tmp.pI0Gke1BtD
++ mktemp
++ mktemp
+ local LAST_ERR=/tmp/tmp.sd01FBhXxf
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ local LAST_ERR=/tmp/tmp.qDnLaOkKzr
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl get ns
+ for i in $(seq 0 2)
+ set +e
+ kubectl delete namespace pitr-physical-28895 --ignore-not-found
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.pI0Gke1BtD
+ cat /tmp/tmp.sd01FBhXxf
+ rm /tmp/tmp.pI0Gke1BtD /tmp/tmp.sd01FBhXxf
+ return 0
error: resource(s) were provided, but no name was specified
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.JwO4DsRqQ3
+ cat /tmp/tmp.qDnLaOkKzr
+ rm /tmp/tmp.JwO4DsRqQ3 /tmp/tmp.qDnLaOkKzr
+ return 0
+ kubectl_bin wait --for=delete namespace pitr-physical-28895
++ mktemp
+ local LAST_OUT=/tmp/tmp.vwrNHTxPXr
++ mktemp
+ local LAST_ERR=/tmp/tmp.xuc5DztcC2
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl wait --for=delete namespace pitr-physical-28895
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.vwrNHTxPXr
+ cat /tmp/tmp.xuc5DztcC2
+ rm /tmp/tmp.vwrNHTxPXr /tmp/tmp.xuc5DztcC2
+ return 0
+ desc 'create namespace pitr-physical-28895'
+ set +o xtrace


-----------------------------------------------------------------------------------
create namespace pitr-physical-28895
-----------------------------------------------------------------------------------

+ kubectl_bin create namespace pitr-physical-28895
++ mktemp
+ local LAST_OUT=/tmp/tmp.RGiHcO1NLl
++ mktemp
+ local LAST_ERR=/tmp/tmp.Bq7pj8NIhL
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl create namespace pitr-physical-28895
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.RGiHcO1NLl
namespace/pitr-physical-28895 created
+ cat /tmp/tmp.Bq7pj8NIhL
+ rm /tmp/tmp.RGiHcO1NLl /tmp/tmp.Bq7pj8NIhL
+ return 0
+ set_kube_ctx pitr-physical-28895
+ local namespace=pitr-physical-28895
++ kubectl_bin config current-context
+++ mktemp
++ local LAST_OUT=/tmp/tmp.RYvlbbd3ky
+++ mktemp
++ local LAST_ERR=/tmp/tmp.8VN5QAwxWz
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl config current-context
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.RYvlbbd3ky
++ cat /tmp/tmp.8VN5QAwxWz
++ rm /tmp/tmp.RYvlbbd3ky /tmp/tmp.8VN5QAwxWz
++ return 0
+ kubectl_bin config set-context gke_cloud-dev-112233_us-central1-a_jen-psmdb-2158-c70b6c4a-2-cluster7 --namespace=pitr-physical-28895
++ mktemp
+ local LAST_OUT=/tmp/tmp.7LzBBCtHZA
++ mktemp
+ local LAST_ERR=/tmp/tmp.PQssh81JzH
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl config set-context gke_cloud-dev-112233_us-central1-a_jen-psmdb-2158-c70b6c4a-2-cluster7 --namespace=pitr-physical-28895
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.7LzBBCtHZA
Context "gke_cloud-dev-112233_us-central1-a_jen-psmdb-2158-c70b6c4a-2-cluster7" modified.
+ cat /tmp/tmp.PQssh81JzH
+ rm /tmp/tmp.7LzBBCtHZA /tmp/tmp.PQssh81JzH
+ return 0
+ deploy_minio
+ desc 'install Minio'
+ set +o xtrace


-----------------------------------------------------------------------------------
install Minio
-----------------------------------------------------------------------------------

+ helm uninstall minio-service
Error: uninstall: Release not loaded: minio-service: release: not found
+ :
+ helm repo remove minio
"minio" has been removed from your repositories
+ helm repo add minio https://charts.min.io/
"minio" has been added to your repositories
+ retry 10 60 helm install minio-service --version 5.4.0 --set replicas=1 --set mode=standalone --set resources.requests.memory=256Mi --set rootUser=rootuser --set rootPassword=rootpass123 --set 'users[0].accessKey=some-access-key' --set 'users[0].secretKey=some-secret-key' --set 'users[0].policy=consoleAdmin' --set service.type=ClusterIP --set configPathmc=/tmp/.minio/ --set persistence.size=2G --set securityContext.enabled=false minio/minio
+ local max=10
+ local delay=60
+ shift 2
+ local n=1
+ helm install minio-service --version 5.4.0 --set replicas=1 --set mode=standalone --set resources.requests.memory=256Mi --set rootUser=rootuser --set rootPassword=rootpass123 --set 'users[0].accessKey=some-access-key' --set 'users[0].secretKey=some-secret-key' --set 'users[0].policy=consoleAdmin' --set service.type=ClusterIP --set configPathmc=/tmp/.minio/ --set persistence.size=2G --set securityContext.enabled=false minio/minio
NAME: minio-service
LAST DEPLOYED: Tue Dec 23 15:07:27 2025
NAMESPACE: pitr-physical-28895
STATUS: deployed
REVISION: 1
TEST SUITE: None
NOTES:
MinIO can be accessed via port 9000 on the following DNS name from within your cluster:
minio-service.pitr-physical-28895.cluster.local

To access MinIO from localhost, run the below commands:

  1. export POD_NAME=$(kubectl get pods --namespace pitr-physical-28895 -l "release=minio-service" -o jsonpath="{.items[0].metadata.name}")

  2. kubectl port-forward $POD_NAME 9000 --namespace pitr-physical-28895

Read more about port forwarding here: http://kubernetes.io/docs/user-guide/kubectl/kubectl_port-forward/

You can now access MinIO server on http://localhost:9000. Follow the below steps to connect to MinIO server with mc client:

  1. Download the MinIO mc client - https://min.io/docs/minio/linux/reference/minio-mc.html#quickstart

  2. export MC_HOST_minio-service-local=http://$(kubectl get secret --namespace pitr-physical-28895 minio-service -o jsonpath="{.data.rootUser}" | base64 --decode):$(kubectl get secret --namespace pitr-physical-28895 minio-service -o jsonpath="{.data.rootPassword}" | base64 --decode)@localhost:9000

  3. mc ls minio-service-local
++ kubectl_bin get pods --selector=release=minio-service -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.EQRoWSUwg1
+++ mktemp
++ local LAST_ERR=/tmp/tmp.P4JbBYNaCw
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=release=minio-service -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.EQRoWSUwg1
++ cat /tmp/tmp.P4JbBYNaCw
++ rm /tmp/tmp.EQRoWSUwg1 /tmp/tmp.P4JbBYNaCw
++ return 0
+ MINIO_POD=minio-service-d9589b474-t68mt
+ wait_pod minio-service-d9589b474-t68mt
+ local pod=minio-service-d9589b474-t68mt
+ set +o xtrace
waiting for pod/minio-service-d9589b474-t68mt to be ready.OK
+ '[' -n psmdb-operator ']'
+ kubectl_bin create svc -n psmdb-operator externalname minio-service --external-name=minio-service.pitr-physical-28895.svc.cluster.local --tcp=9000
++ mktemp
+ local LAST_OUT=/tmp/tmp.dc8RDX2ba7
++ mktemp
+ local LAST_ERR=/tmp/tmp.EETE5EVNvr
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl create svc -n psmdb-operator externalname minio-service --external-name=minio-service.pitr-physical-28895.svc.cluster.local --tcp=9000
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.dc8RDX2ba7
service/minio-service created
+ cat /tmp/tmp.EETE5EVNvr
+ rm /tmp/tmp.dc8RDX2ba7 /tmp/tmp.EETE5EVNvr
+ return 0
+ create_minio_bucket operator-testing
+ local bucket=operator-testing
+ kubectl_bin run -i --rm aws-cli --image=perconalab/awscli --restart=Never -- bash -c 'AWS_ACCESS_KEY_ID=some-access-key 		AWS_SECRET_ACCESS_KEY=some-secret-key 		AWS_DEFAULT_REGION=us-east-1         /usr/bin/aws --endpoint-url http://minio-service:9000 s3 mb s3://operator-testing'
++ mktemp
+ local LAST_OUT=/tmp/tmp.flTUl3Uwag
++ mktemp
+ local LAST_ERR=/tmp/tmp.wDAeiloywK
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl run -i --rm aws-cli --image=perconalab/awscli --restart=Never -- bash -c 'AWS_ACCESS_KEY_ID=some-access-key 		AWS_SECRET_ACCESS_KEY=some-secret-key 		AWS_DEFAULT_REGION=us-east-1         /usr/bin/aws --endpoint-url http://minio-service:9000 s3 mb s3://operator-testing'
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.flTUl3Uwag
pod "aws-cli" deleted from pitr-physical-28895 namespace
+ cat /tmp/tmp.wDAeiloywK
All commands and output from this session will be recorded in container logs, including credentials and sensitive information passed through the command prompt.
If you don't see a command prompt, try pressing enter.
+ rm /tmp/tmp.flTUl3Uwag /tmp/tmp.wDAeiloywK
+ return 0
+ desc 'create secrets and start client'
+ set +o xtrace


-----------------------------------------------------------------------------------
create secrets and start client
-----------------------------------------------------------------------------------

+ kubectl_bin apply -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2158/e2e-tests/conf/secrets.yml -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2158/e2e-tests/conf/client.yml -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2158/e2e-tests/conf/minio-secret.yml
++ mktemp
+ local LAST_OUT=/tmp/tmp.rTFvWV596o
++ mktemp
+ local LAST_ERR=/tmp/tmp.xK4f99kTjq
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl apply -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2158/e2e-tests/conf/secrets.yml -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2158/e2e-tests/conf/client.yml -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2158/e2e-tests/conf/minio-secret.yml
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.rTFvWV596o
secret/some-users created
deployment.apps/psmdb-client created
secret/minio-secret created
+ cat /tmp/tmp.xK4f99kTjq
+ rm /tmp/tmp.rTFvWV596o /tmp/tmp.xK4f99kTjq
+ return 0
+ cluster=some-name
+ desc 'create first PSMDB cluster some-name'
+ set +o xtrace


-----------------------------------------------------------------------------------
create first PSMDB cluster some-name
-----------------------------------------------------------------------------------

+ apply_cluster /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2158/e2e-tests/pitr-physical/conf/some-name-rs0.yml
+ '[' -z '' ']'
+ cat_config /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2158/e2e-tests/pitr-physical/conf/some-name-rs0.yml
+ kubectl_bin apply -f -
+ cat /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2158/e2e-tests/pitr-physical/conf/some-name-rs0.yml
++ mktemp
+ yq eval '(.spec | select(.image == null)).image = "perconalab/percona-server-mongodb-operator:main-mongod8.0"'
+ yq eval '(.spec | select(has("pmm"))).pmm.image = "percona/pmm-client:2.44.1-1"'
+ yq eval '(.spec | select(has("initImage"))).initImage = "perconalab/percona-server-mongodb-operator:PR-2158-c70b6c4a"'
+ yq eval '(.spec | select(has("backup"))).backup.image = "perconalab/percona-server-mongodb-operator:main-backup"'
+ local LAST_OUT=/tmp/tmp.wkfE8m6pa8
++ mktemp
+ /usr/sbin/sed -e s/NAME_SPACE/pitr-physical-28895/g
+ yq eval '.spec.upgradeOptions.apply="Never"'
+ local LAST_ERR=/tmp/tmp.9XlaGZVDr6
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl apply -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.wkfE8m6pa8
perconaservermongodb.psmdb.percona.com/some-name created
+ cat /tmp/tmp.9XlaGZVDr6
+ rm /tmp/tmp.wkfE8m6pa8 /tmp/tmp.9XlaGZVDr6
+ return 0
+ desc 'check if all 3 Pods started'
+ set +o xtrace


-----------------------------------------------------------------------------------
check if all 3 Pods started
-----------------------------------------------------------------------------------

+ wait_for_running some-name-rs0 3
+ local name=some-name-rs0
+ let last_pod=2
+ local check_cluster_readyness=true
+ set_debug
+ [[ 1 == 1 ]]
+ set -o xtrace
+ local rs_name=rs0
+ local cluster_name=some-name
++ seq 0 2
+ for i in $(seq 0 $last_pod)
+ [[ 0 -eq 2 ]]
+ wait_pod some-name-rs0-0
+ local pod=some-name-rs0-0
+ set +o xtrace
waiting for pod/some-name-rs0-0 to be ready....OK
+ for i in $(seq 0 $last_pod)
+ [[ 1 -eq 2 ]]
+ wait_pod some-name-rs0-1
+ local pod=some-name-rs0-1
+ set +o xtrace
waiting for pod/some-name-rs0-1 to be ready.......OK
+ for i in $(seq 0 $last_pod)
+ [[ 2 -eq 2 ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].arbiter.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.FpoXbrV6cT
+++ mktemp
++ local LAST_ERR=/tmp/tmp.6p3w43LGVk
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].arbiter.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.FpoXbrV6cT
++ cat /tmp/tmp.6p3w43LGVk
++ rm /tmp/tmp.FpoXbrV6cT /tmp/tmp.6p3w43LGVk
++ return 0
+ [[ '' == \t\r\u\e ]]
+ wait_pod some-name-rs0-2
+ local pod=some-name-rs0-2
+ set +o xtrace
waiting for pod/some-name-rs0-2 to be ready......OK
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].nonvoting.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.uLJiYT05VU
+++ mktemp
++ local LAST_ERR=/tmp/tmp.NzWz3rHVyL
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].nonvoting.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.uLJiYT05VU
++ cat /tmp/tmp.NzWz3rHVyL
++ rm /tmp/tmp.uLJiYT05VU /tmp/tmp.NzWz3rHVyL
++ return 0
+ [[ '' == \t\r\u\e ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].hidden.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.BrhooSnHlv
+++ mktemp
++ local LAST_ERR=/tmp/tmp.ra7AGzJoG6
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].hidden.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.BrhooSnHlv
++ cat /tmp/tmp.ra7AGzJoG6
++ rm /tmp/tmp.BrhooSnHlv /tmp/tmp.ra7AGzJoG6
++ return 0
+ [[ '' == \t\r\u\e ]]
+ sleep 10
+ [[ true == \t\r\u\e ]]
+ set +x
Waiting for cluster readyness..................................................
+ wait_for_running some-name-cfg 3 false
+ local name=some-name-cfg
+ let last_pod=2
+ local check_cluster_readyness=false
+ set_debug
+ [[ 1 == 1 ]]
+ set -o xtrace
+ local rs_name=cfg
+ local cluster_name=some-name
++ seq 0 2
+ for i in $(seq 0 $last_pod)
+ [[ 0 -eq 2 ]]
+ wait_pod some-name-cfg-0
+ local pod=some-name-cfg-0
+ set +o xtrace
waiting for pod/some-name-cfg-0 to be ready.OK
+ for i in $(seq 0 $last_pod)
+ [[ 1 -eq 2 ]]
+ wait_pod some-name-cfg-1
+ local pod=some-name-cfg-1
+ set +o xtrace
waiting for pod/some-name-cfg-1 to be ready.OK
+ for i in $(seq 0 $last_pod)
+ [[ 2 -eq 2 ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].arbiter.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.3Nxlo18lHu
+++ mktemp
++ local LAST_ERR=/tmp/tmp.2zzvBumhdX
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].arbiter.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.3Nxlo18lHu
++ cat /tmp/tmp.2zzvBumhdX
++ rm /tmp/tmp.3Nxlo18lHu /tmp/tmp.2zzvBumhdX
++ return 0
+ [[ '' == \t\r\u\e ]]
+ wait_pod some-name-cfg-2
+ local pod=some-name-cfg-2
+ set +o xtrace
waiting for pod/some-name-cfg-2 to be ready.OK
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].nonvoting.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.J2hkTv8R0K
+++ mktemp
++ local LAST_ERR=/tmp/tmp.o182Kg5DmO
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].nonvoting.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.J2hkTv8R0K
++ cat /tmp/tmp.o182Kg5DmO
++ rm /tmp/tmp.J2hkTv8R0K /tmp/tmp.o182Kg5DmO
++ return 0
+ [[ '' == \t\r\u\e ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].hidden.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.PZESjnR1gK
+++ mktemp
++ local LAST_ERR=/tmp/tmp.duXtNMOfuS
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].hidden.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.PZESjnR1gK
++ cat /tmp/tmp.duXtNMOfuS
++ rm /tmp/tmp.PZESjnR1gK /tmp/tmp.duXtNMOfuS
++ return 0
+ [[ '' == \t\r\u\e ]]
+ sleep 10
+ [[ false == \t\r\u\e ]]
+ sleep 10
+ desc 'write data'
+ set +o xtrace


-----------------------------------------------------------------------------------
write data
-----------------------------------------------------------------------------------

+ write_initial_data
+ desc 'create user myApp'
+ set +o xtrace


-----------------------------------------------------------------------------------
create user myApp
-----------------------------------------------------------------------------------

+ run_mongos 'db.createUser({user:"myApp",pwd:"myPass",roles:[{db:"myApp",role:"readWrite"}]})' userAdmin:userAdmin123456@some-name-mongos.pitr-physical-28895
+ local 'command=db.createUser({user:"myApp",pwd:"myPass",roles:[{db:"myApp",role:"readWrite"}]})'
+ local uri=userAdmin:userAdmin123456@some-name-mongos.pitr-physical-28895
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local port=27017
+ local mongo_bin=mongo
++ echo .svc.cluster.local
++ awk -F: '{print $2}'
+ suffix_port=
+ [[ -z '' ]]
+ suffix=.svc.cluster.local:27017
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.ALVznzoSpe
+++ mktemp
++ local LAST_ERR=/tmp/tmp.c4Z6xbVxUP
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.ALVznzoSpe
++ cat /tmp/tmp.c4Z6xbVxUP
++ rm /tmp/tmp.ALVznzoSpe /tmp/tmp.c4Z6xbVxUP
++ return 0
+ local client_container=psmdb-client-696897d69b-47lj5
+ kubectl_bin exec psmdb-client-696897d69b-47lj5 -- bash -c 'printf '\''db.createUser({user:"myApp",pwd:"myPass",roles:[{db:"myApp",role:"readWrite"}]})\n'\'' | mongo mongodb://userAdmin:userAdmin123456@some-name-mongos.pitr-physical-28895.svc.cluster.local:27017/admin '
++ mktemp
+ local LAST_OUT=/tmp/tmp.2kltnZsfbI
++ mktemp
+ local LAST_ERR=/tmp/tmp.VHEvZ6foQp
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-696897d69b-47lj5 -- bash -c 'printf '\''db.createUser({user:"myApp",pwd:"myPass",roles:[{db:"myApp",role:"readWrite"}]})\n'\'' | mongo mongodb://userAdmin:userAdmin123456@some-name-mongos.pitr-physical-28895.svc.cluster.local:27017/admin '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.2kltnZsfbI
Percona Server for MongoDB shell version v4.4.29-28
connecting to: mongodb://some-name-mongos.pitr-physical-28895.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb
Implicit session: session { "id" : UUID("da6b5474-e510-4430-bf83-b3a78ee3afab") }
Percona Server for MongoDB server version: v8.0.16-5
WARNING: shell and server versions do not match
Successfully added user: {
	"user" : "myApp",
	"roles" : [
		{
			"db" : "myApp",
			"role" : "readWrite"
		}
	]
}
bye
+ cat /tmp/tmp.VHEvZ6foQp
+ rm /tmp/tmp.2kltnZsfbI /tmp/tmp.VHEvZ6foQp
+ return 0
+ sleep 2
+ write_document
+ local cmp_postfix=
+ local sleep_value=0
+ log 'write initial data, read from all'
+ set +o xtrace
[2025-12-23T15:11:53+0000] write initial data, read from all
+ run_mongos 'use myApp\n db.test.insert({ x: 100500 })' myApp:myPass@some-name-mongos.pitr-physical-28895
+ local 'command=use myApp\n db.test.insert({ x: 100500 })'
+ local uri=myApp:myPass@some-name-mongos.pitr-physical-28895
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local port=27017
+ local mongo_bin=mongo
++ echo .svc.cluster.local
++ awk -F: '{print $2}'
+ suffix_port=
+ [[ -z '' ]]
+ suffix=.svc.cluster.local:27017
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.7CIOF36BDl
+++ mktemp
++ local LAST_ERR=/tmp/tmp.TufLYPkJW9
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.7CIOF36BDl
++ cat /tmp/tmp.TufLYPkJW9
++ rm /tmp/tmp.7CIOF36BDl /tmp/tmp.TufLYPkJW9
++ return 0
+ local client_container=psmdb-client-696897d69b-47lj5
+ kubectl_bin exec psmdb-client-696897d69b-47lj5 -- bash -c 'printf '\''use myApp\n db.test.insert({ x: 100500 })\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.pitr-physical-28895.svc.cluster.local:27017/admin '
++ mktemp
+ local LAST_OUT=/tmp/tmp.5VjR1qOOYO
++ mktemp
+ local LAST_ERR=/tmp/tmp.Qpy9bQmarx
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-696897d69b-47lj5 -- bash -c 'printf '\''use myApp\n db.test.insert({ x: 100500 })\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.pitr-physical-28895.svc.cluster.local:27017/admin '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.5VjR1qOOYO
Percona Server for MongoDB shell version v4.4.29-28
connecting to: mongodb://some-name-mongos.pitr-physical-28895.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb
Implicit session: session { "id" : UUID("a75bf2b2-0053-434e-9ba1-f5dd5cce2639") }
Percona Server for MongoDB server version: v8.0.16-5
WARNING: shell and server versions do not match
switched to db myApp
WriteResult({ "nInserted" : 1 })
bye
+ cat /tmp/tmp.Qpy9bQmarx
+ rm /tmp/tmp.5VjR1qOOYO /tmp/tmp.Qpy9bQmarx
+ return 0
+ sleep 0
+ compare_mongos_cmd find myApp:myPass@some-name-mongos.pitr-physical-28895
+ local command=find
+ local uri=myApp:myPass@some-name-mongos.pitr-physical-28895
+ local postfix=
+ local suffix=
+ local database=myApp
+ local collection=test
+ local port=27017
+ local tls=false
+ [[ false == \t\r\u\e ]]
+ mongos_command=run_mongos
+ log 'running db.test.command() in myApp'
+ set +o xtrace
[2025-12-23T15:11:57+0000] running db.test.command() in myApp
+ run_mongos 'use myApp\n db.test.find()' myApp:myPass@some-name-mongos.pitr-physical-28895 mongodb '' '' 27017
+ local 'command=use myApp\n db.test.find()'
+ local uri=myApp:myPass@some-name-mongos.pitr-physical-28895
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local port=27017
+ local mongo_bin=mongo
+ egrep -v 'I NETWORK|W NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|Started a new thread for the timer service'
+ /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
++ echo .svc.cluster.local
++ awk -F: '{print $2}'
egrep: warning: egrep is obsolescent; using grep -E
+ suffix_port=
+ [[ -z '' ]]
+ suffix=.svc.cluster.local:27017
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.lifcMZ04ld
+++ mktemp
++ local LAST_ERR=/tmp/tmp.ulGc1wTHdx
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.lifcMZ04ld
++ cat /tmp/tmp.ulGc1wTHdx
++ rm /tmp/tmp.lifcMZ04ld /tmp/tmp.ulGc1wTHdx
++ return 0
+ local client_container=psmdb-client-696897d69b-47lj5
+ kubectl_bin exec psmdb-client-696897d69b-47lj5 -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.pitr-physical-28895.svc.cluster.local:27017/admin '
++ mktemp
+ local LAST_OUT=/tmp/tmp.vujTjX6bAF
++ mktemp
+ local LAST_ERR=/tmp/tmp.3cJ6eFpdfI
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-696897d69b-47lj5 -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.pitr-physical-28895.svc.cluster.local:27017/admin '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.vujTjX6bAF
+ cat /tmp/tmp.3cJ6eFpdfI
+ rm /tmp/tmp.vujTjX6bAF /tmp/tmp.3cJ6eFpdfI
+ return 0
+ [[ 0 -eq 0 ]]
+ diff /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2158/e2e-tests/pitr-physical/compare/find.json /tmp/tmp.83ukH1je1W/find
+ desc 'wait for pbm agents to start'
+ set +o xtrace


-----------------------------------------------------------------------------------
wait for pbm agents to start
-----------------------------------------------------------------------------------

+ wait_backup_agent some-name-rs0-0
+ local agent_pod=some-name-rs0-0
+ set +o xtrace
waiting for pbm-agent to be ready in some-name-rs0-0...2025-12-23T15:11:06.000+0000 I listening for the commands

+ wait_backup_agent some-name-rs0-1
+ local agent_pod=some-name-rs0-1
+ set +o xtrace
waiting for pbm-agent to be ready in some-name-rs0-1...2025-12-23T15:11:08.000+0000 I listening for the commands

+ wait_backup_agent some-name-rs0-2
+ local agent_pod=some-name-rs0-2
+ set +o xtrace
waiting for pbm-agent to be ready in some-name-rs0-2...2025-12-23T15:11:17.000+0000 I listening for the commands

+ wait_backup_agent some-name-rs1-0
+ local agent_pod=some-name-rs1-0
+ set +o xtrace
waiting for pbm-agent to be ready in some-name-rs1-0...2025-12-23T15:11:05.000+0000 I listening for the commands

+ wait_backup_agent some-name-rs1-1
+ local agent_pod=some-name-rs1-1
+ set +o xtrace
waiting for pbm-agent to be ready in some-name-rs1-1...2025-12-23T15:11:10.000+0000 I listening for the commands

+ wait_backup_agent some-name-rs1-2
+ local agent_pod=some-name-rs1-2
+ set +o xtrace
waiting for pbm-agent to be ready in some-name-rs1-2...2025-12-23T15:11:18.000+0000 I listening for the commands

+ wait_backup_agent some-name-rs2-0
+ local agent_pod=some-name-rs2-0
+ set +o xtrace
waiting for pbm-agent to be ready in some-name-rs2-0...2025-12-23T15:11:08.000+0000 I listening for the commands

+ wait_backup_agent some-name-rs2-1
+ local agent_pod=some-name-rs2-1
+ set +o xtrace
waiting for pbm-agent to be ready in some-name-rs2-1...2025-12-23T15:11:08.000+0000 I listening for the commands

+ wait_backup_agent some-name-rs2-2
+ local agent_pod=some-name-rs2-2
+ set +o xtrace
waiting for pbm-agent to be ready in some-name-rs2-2...2025-12-23T15:11:18.000+0000 I listening for the commands

+ log 'Sleeping for 360 seconds to avoid PBM-1265'
+ set +o xtrace
[2025-12-23T15:12:13+0000] Sleeping for 360 seconds to avoid PBM-1265
+ sleep 360
+ backup_name_minio=backup-minio
+ run_backup backup-minio 2 physical
+ local name=backup-minio
+ local idx=2
+ local type=physical
+ desc 'run backup backup-minio-2'
+ set +o xtrace


-----------------------------------------------------------------------------------
run backup backup-minio-2
-----------------------------------------------------------------------------------

+ cat /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2158/e2e-tests/pitr-physical/conf/backup-minio.yml
+ /usr/sbin/sed -e 's/name:/name: backup-minio-2/'
+ /usr/sbin/sed -e 's/type:/type: physical/'
+ kubectl_bin apply -f -
++ mktemp
+ local LAST_OUT=/tmp/tmp.7frbiI59OM
++ mktemp
+ local LAST_ERR=/tmp/tmp.w85Mo0bajw
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl apply -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.7frbiI59OM
perconaservermongodbbackup.psmdb.percona.com/backup-minio-2 created
+ cat /tmp/tmp.w85Mo0bajw
+ rm /tmp/tmp.7frbiI59OM /tmp/tmp.w85Mo0bajw
+ return 0
+ wait_backup backup-minio-2
+ local backup_name=backup-minio-2
+ local target_state=ready
+ set +o xtrace
waiting for backup-minio-2 to reach ready state...........OK
+ sleep 5
+ write_document -2nd 130
+ local cmp_postfix=-2nd
+ local sleep_value=130
+ log 'write initial data, read from all'
+ set +o xtrace
[2025-12-23T15:18:42+0000] write initial data, read from all
+ run_mongos 'use myApp\n db.test.insert({ x: 100500 })' myApp:myPass@some-name-mongos.pitr-physical-28895
+ local 'command=use myApp\n db.test.insert({ x: 100500 })'
+ local uri=myApp:myPass@some-name-mongos.pitr-physical-28895
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local port=27017
+ local mongo_bin=mongo
++ echo .svc.cluster.local
++ awk -F: '{print $2}'
+ suffix_port=
+ [[ -z '' ]]
+ suffix=.svc.cluster.local:27017
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.dL9PAUHxiY
+++ mktemp
++ local LAST_ERR=/tmp/tmp.xIheMoqhTl
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.dL9PAUHxiY
++ cat /tmp/tmp.xIheMoqhTl
++ rm /tmp/tmp.dL9PAUHxiY /tmp/tmp.xIheMoqhTl
++ return 0
+ local client_container=psmdb-client-696897d69b-47lj5
+ kubectl_bin exec psmdb-client-696897d69b-47lj5 -- bash -c 'printf '\''use myApp\n db.test.insert({ x: 100500 })\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.pitr-physical-28895.svc.cluster.local:27017/admin '
++ mktemp
+ local LAST_OUT=/tmp/tmp.olQR8RrjXB
++ mktemp
+ local LAST_ERR=/tmp/tmp.3vVuGCMN0n
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-696897d69b-47lj5 -- bash -c 'printf '\''use myApp\n db.test.insert({ x: 100500 })\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.pitr-physical-28895.svc.cluster.local:27017/admin '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.olQR8RrjXB
Percona Server for MongoDB shell version v4.4.29-28
connecting to: mongodb://some-name-mongos.pitr-physical-28895.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb
Implicit session: session { "id" : UUID("cd9984fc-ceed-4692-af56-ab359a2a6c48") }
Percona Server for MongoDB server version: v8.0.16-5
WARNING: shell and server versions do not match
switched to db myApp
WriteResult({ "nInserted" : 1 })
bye
+ cat /tmp/tmp.3vVuGCMN0n
+ rm /tmp/tmp.olQR8RrjXB /tmp/tmp.3vVuGCMN0n
+ return 0
+ sleep 130
+ compare_mongos_cmd find myApp:myPass@some-name-mongos.pitr-physical-28895 -2nd
+ local command=find
+ local uri=myApp:myPass@some-name-mongos.pitr-physical-28895
+ local postfix=-2nd
+ local suffix=
+ local database=myApp
+ local collection=test
+ local port=27017
+ local tls=false
+ [[ false == \t\r\u\e ]]
+ mongos_command=run_mongos
+ log 'running db.test.command() in myApp'
+ set +o xtrace
[2025-12-23T15:20:56+0000] running db.test.command() in myApp
+ run_mongos 'use myApp\n db.test.find()' myApp:myPass@some-name-mongos.pitr-physical-28895 mongodb '' '' 27017
+ egrep -v 'I NETWORK|W NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|Started a new thread for the timer service'
+ local 'command=use myApp\n db.test.find()'
+ local uri=myApp:myPass@some-name-mongos.pitr-physical-28895
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local port=27017
+ local mongo_bin=mongo
+ /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
++ echo .svc.cluster.local
++ awk -F: '{print $2}'
egrep: warning: egrep is obsolescent; using grep -E
+ suffix_port=
+ [[ -z '' ]]
+ suffix=.svc.cluster.local:27017
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.fiNeXSyHcN
+++ mktemp
++ local LAST_ERR=/tmp/tmp.v46D22qOUQ
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.fiNeXSyHcN
++ cat /tmp/tmp.v46D22qOUQ
++ rm /tmp/tmp.fiNeXSyHcN /tmp/tmp.v46D22qOUQ
++ return 0
+ local client_container=psmdb-client-696897d69b-47lj5
+ kubectl_bin exec psmdb-client-696897d69b-47lj5 -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.pitr-physical-28895.svc.cluster.local:27017/admin '
++ mktemp
+ local LAST_OUT=/tmp/tmp.i6eKCDFvrF
++ mktemp
+ local LAST_ERR=/tmp/tmp.lr5SmYGOPT
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-696897d69b-47lj5 -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.pitr-physical-28895.svc.cluster.local:27017/admin '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.i6eKCDFvrF
+ cat /tmp/tmp.lr5SmYGOPT
+ rm /tmp/tmp.i6eKCDFvrF /tmp/tmp.lr5SmYGOPT
+ return 0
+ [[ 0 -eq 0 ]]
+ diff /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2158/e2e-tests/pitr-physical/compare/find-2nd.json /tmp/tmp.83ukH1je1W/find-2nd
++ date +%s
+ last_write=1766503259
++ get_latest_oplog_chunk_ts some-name
++ local cluster=some-name
+++ kubectl_bin exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ jq '.backups.pitrChunks.pitrChunks | last | .range.end'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.RFeCUj6odM
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.ECcbTaQPs6
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in $(seq 0 2)
+++ set +e
+++ kubectl exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.RFeCUj6odM
+++ cat /tmp/tmp.ECcbTaQPs6
+++ rm /tmp/tmp.RFeCUj6odM /tmp/tmp.ECcbTaQPs6
+++ return 0
++ echo 1766503126
+ last_chunk=1766503126
+ log 'last write: 1766503259'
+ set +o xtrace
[2025-12-23T15:21:00+0000] last write: 1766503259
+ log 'last chunk: 1766503126'
+ set +o xtrace
[2025-12-23T15:21:00+0000] last chunk: 1766503126
+ retries=0
+ [[ 1766503126 -gt 1766503259 ]]
+ [[ 0 -gt 30 ]]
++ get_latest_oplog_chunk_ts some-name
++ local cluster=some-name
+++ kubectl_bin exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ jq '.backups.pitrChunks.pitrChunks | last | .range.end'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.ddQ9N80j81
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.BGTCOIAq7l
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in $(seq 0 2)
+++ set +e
+++ kubectl exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.ddQ9N80j81
+++ cat /tmp/tmp.BGTCOIAq7l
+++ rm /tmp/tmp.ddQ9N80j81 /tmp/tmp.BGTCOIAq7l
+++ return 0
++ echo 1766503126
+ last_chunk=1766503126
+ retries=1
++ format_date 1766503126
++ local timestamp=1766503126
+++ TZ=UTC
+++ /usr/sbin/date -d@1766503126 '+%Y-%m-%d %H:%M:%S'
++ echo 2025-12-23 15:18:46
++ format_date 1766503259
++ local timestamp=1766503259
+++ TZ=UTC
+++ /usr/sbin/date -d@1766503259 '+%Y-%m-%d %H:%M:%S'
++ echo 2025-12-23 15:20:59
+ log 'Waiting for last oplog chunk (2025-12-23 15:18:46) to be greater than last write (2025-12-23 15:20:59)'
+ set +o xtrace
[2025-12-23T15:21:03+0000] Waiting for last oplog chunk (2025-12-23 15:18:46) to be greater than last write (2025-12-23 15:20:59)
+ sleep 10
+ [[ 1766503126 -gt 1766503259 ]]
+ [[ 1 -gt 30 ]]
++ get_latest_oplog_chunk_ts some-name
++ local cluster=some-name
+++ kubectl_bin exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ jq '.backups.pitrChunks.pitrChunks | last | .range.end'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.OlZj6m4szH
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.5yC21miG7D
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in $(seq 0 2)
+++ set +e
+++ kubectl exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.OlZj6m4szH
+++ cat /tmp/tmp.5yC21miG7D
+++ rm /tmp/tmp.OlZj6m4szH /tmp/tmp.5yC21miG7D
+++ return 0
++ echo 1766503132
+ last_chunk=1766503132
+ retries=2
++ format_date 1766503132
++ local timestamp=1766503132
+++ TZ=UTC
+++ /usr/sbin/date -d@1766503132 '+%Y-%m-%d %H:%M:%S'
++ echo 2025-12-23 15:18:52
++ format_date 1766503259
++ local timestamp=1766503259
+++ TZ=UTC
+++ /usr/sbin/date -d@1766503259 '+%Y-%m-%d %H:%M:%S'
++ echo 2025-12-23 15:20:59
+ log 'Waiting for last oplog chunk (2025-12-23 15:18:52) to be greater than last write (2025-12-23 15:20:59)'
+ set +o xtrace
[2025-12-23T15:21:15+0000] Waiting for last oplog chunk (2025-12-23 15:18:52) to be greater than last write (2025-12-23 15:20:59)
+ sleep 10
+ [[ 1766503132 -gt 1766503259 ]]
+ [[ 2 -gt 30 ]]
++ get_latest_oplog_chunk_ts some-name
++ local cluster=some-name
+++ kubectl_bin exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ jq '.backups.pitrChunks.pitrChunks | last | .range.end'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.mVbtJMpUdp
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.Kfm4sR2GAQ
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in $(seq 0 2)
+++ set +e
+++ kubectl exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.mVbtJMpUdp
+++ cat /tmp/tmp.Kfm4sR2GAQ
+++ rm /tmp/tmp.mVbtJMpUdp /tmp/tmp.Kfm4sR2GAQ
+++ return 0
++ echo 1766503257
+ last_chunk=1766503257
+ retries=3
++ format_date 1766503257
++ local timestamp=1766503257
+++ TZ=UTC
+++ /usr/sbin/date -d@1766503257 '+%Y-%m-%d %H:%M:%S'
++ echo 2025-12-23 15:20:57
++ format_date 1766503259
++ local timestamp=1766503259
+++ TZ=UTC
+++ /usr/sbin/date -d@1766503259 '+%Y-%m-%d %H:%M:%S'
++ echo 2025-12-23 15:20:59
+ log 'Waiting for last oplog chunk (2025-12-23 15:20:57) to be greater than last write (2025-12-23 15:20:59)'
+ set +o xtrace
[2025-12-23T15:21:28+0000] Waiting for last oplog chunk (2025-12-23 15:20:57) to be greater than last write (2025-12-23 15:20:59)
+ sleep 10
+ [[ 1766503257 -gt 1766503259 ]]
+ [[ 3 -gt 30 ]]
++ get_latest_oplog_chunk_ts some-name
++ local cluster=some-name
+++ kubectl_bin exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ jq '.backups.pitrChunks.pitrChunks | last | .range.end'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.Q9Ss1wC2V1
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.9yD5hcCpIc
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in $(seq 0 2)
+++ set +e
+++ kubectl exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.Q9Ss1wC2V1
+++ cat /tmp/tmp.9yD5hcCpIc
+++ rm /tmp/tmp.Q9Ss1wC2V1 /tmp/tmp.9yD5hcCpIc
+++ return 0
++ echo 1766503257
+ last_chunk=1766503257
+ retries=4
++ format_date 1766503257
++ local timestamp=1766503257
+++ TZ=UTC
+++ /usr/sbin/date -d@1766503257 '+%Y-%m-%d %H:%M:%S'
++ echo 2025-12-23 15:20:57
++ format_date 1766503259
++ local timestamp=1766503259
+++ TZ=UTC
+++ /usr/sbin/date -d@1766503259 '+%Y-%m-%d %H:%M:%S'
++ echo 2025-12-23 15:20:59
+ log 'Waiting for last oplog chunk (2025-12-23 15:20:57) to be greater than last write (2025-12-23 15:20:59)'
+ set +o xtrace
[2025-12-23T15:21:40+0000] Waiting for last oplog chunk (2025-12-23 15:20:57) to be greater than last write (2025-12-23 15:20:59)
+ sleep 10
+ [[ 1766503257 -gt 1766503259 ]]
+ [[ 4 -gt 30 ]]
++ get_latest_oplog_chunk_ts some-name
++ local cluster=some-name
+++ kubectl_bin exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ jq '.backups.pitrChunks.pitrChunks | last | .range.end'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.qsT2vLK2uu
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.Wg9Mz4TwgI
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in $(seq 0 2)
+++ set +e
+++ kubectl exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.qsT2vLK2uu
+++ cat /tmp/tmp.Wg9Mz4TwgI
+++ rm /tmp/tmp.qsT2vLK2uu /tmp/tmp.Wg9Mz4TwgI
+++ return 0
++ echo 1766503257
+ last_chunk=1766503257
+ retries=5
++ format_date 1766503257
++ local timestamp=1766503257
+++ TZ=UTC
+++ /usr/sbin/date -d@1766503257 '+%Y-%m-%d %H:%M:%S'
++ echo 2025-12-23 15:20:57
++ format_date 1766503259
++ local timestamp=1766503259
+++ TZ=UTC
+++ /usr/sbin/date -d@1766503259 '+%Y-%m-%d %H:%M:%S'
++ echo 2025-12-23 15:20:59
+ log 'Waiting for last oplog chunk (2025-12-23 15:20:57) to be greater than last write (2025-12-23 15:20:59)'
+ set +o xtrace
[2025-12-23T15:21:53+0000] Waiting for last oplog chunk (2025-12-23 15:20:57) to be greater than last write (2025-12-23 15:20:59)
+ sleep 10
+ [[ 1766503257 -gt 1766503259 ]]
+ [[ 5 -gt 30 ]]
++ get_latest_oplog_chunk_ts some-name
++ local cluster=some-name
+++ kubectl_bin exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ jq '.backups.pitrChunks.pitrChunks | last | .range.end'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.qBI8hAQWr0
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.Pp3jXVxZhQ
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in $(seq 0 2)
+++ set +e
+++ kubectl exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.qBI8hAQWr0
+++ cat /tmp/tmp.Pp3jXVxZhQ
+++ rm /tmp/tmp.qBI8hAQWr0 /tmp/tmp.Pp3jXVxZhQ
+++ return 0
++ echo 1766503257
+ last_chunk=1766503257
+ retries=6
++ format_date 1766503257
++ local timestamp=1766503257
+++ TZ=UTC
+++ /usr/sbin/date -d@1766503257 '+%Y-%m-%d %H:%M:%S'
++ echo 2025-12-23 15:20:57
++ format_date 1766503259
++ local timestamp=1766503259
+++ TZ=UTC
+++ /usr/sbin/date -d@1766503259 '+%Y-%m-%d %H:%M:%S'
++ echo 2025-12-23 15:20:59
+ log 'Waiting for last oplog chunk (2025-12-23 15:20:57) to be greater than last write (2025-12-23 15:20:59)'
+ set +o xtrace
[2025-12-23T15:22:05+0000] Waiting for last oplog chunk (2025-12-23 15:20:57) to be greater than last write (2025-12-23 15:20:59)
+ sleep 10
+ [[ 1766503257 -gt 1766503259 ]]
+ [[ 6 -gt 30 ]]
++ get_latest_oplog_chunk_ts some-name
++ local cluster=some-name
+++ kubectl_bin exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ jq '.backups.pitrChunks.pitrChunks | last | .range.end'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.3l06lVSpWF
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.BIyLk7FbxA
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in $(seq 0 2)
+++ set +e
+++ kubectl exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.3l06lVSpWF
+++ cat /tmp/tmp.BIyLk7FbxA
+++ rm /tmp/tmp.3l06lVSpWF /tmp/tmp.BIyLk7FbxA
+++ return 0
++ echo 1766503257
+ last_chunk=1766503257
+ retries=7
++ format_date 1766503257
++ local timestamp=1766503257
+++ TZ=UTC
+++ /usr/sbin/date -d@1766503257 '+%Y-%m-%d %H:%M:%S'
++ echo 2025-12-23 15:20:57
++ format_date 1766503259
++ local timestamp=1766503259
+++ TZ=UTC
+++ /usr/sbin/date -d@1766503259 '+%Y-%m-%d %H:%M:%S'
++ echo 2025-12-23 15:20:59
+ log 'Waiting for last oplog chunk (2025-12-23 15:20:57) to be greater than last write (2025-12-23 15:20:59)'
+ set +o xtrace
[2025-12-23T15:22:17+0000] Waiting for last oplog chunk (2025-12-23 15:20:57) to be greater than last write (2025-12-23 15:20:59)
+ sleep 10
+ [[ 1766503257 -gt 1766503259 ]]
+ [[ 7 -gt 30 ]]
++ get_latest_oplog_chunk_ts some-name
++ local cluster=some-name
+++ kubectl_bin exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ jq '.backups.pitrChunks.pitrChunks | last | .range.end'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.UwkuQEabj2
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.NoK8AZc8Fg
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in $(seq 0 2)
+++ set +e
+++ kubectl exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.UwkuQEabj2
+++ cat /tmp/tmp.NoK8AZc8Fg
+++ rm /tmp/tmp.UwkuQEabj2 /tmp/tmp.NoK8AZc8Fg
+++ return 0
++ echo 1766503257
+ last_chunk=1766503257
+ retries=8
++ format_date 1766503257
++ local timestamp=1766503257
+++ TZ=UTC
+++ /usr/sbin/date -d@1766503257 '+%Y-%m-%d %H:%M:%S'
++ echo 2025-12-23 15:20:57
++ format_date 1766503259
++ local timestamp=1766503259
+++ TZ=UTC
+++ /usr/sbin/date -d@1766503259 '+%Y-%m-%d %H:%M:%S'
++ echo 2025-12-23 15:20:59
+ log 'Waiting for last oplog chunk (2025-12-23 15:20:57) to be greater than last write (2025-12-23 15:20:59)'
+ set +o xtrace
[2025-12-23T15:22:30+0000] Waiting for last oplog chunk (2025-12-23 15:20:57) to be greater than last write (2025-12-23 15:20:59)
+ sleep 10
+ [[ 1766503257 -gt 1766503259 ]]
+ [[ 8 -gt 30 ]]
++ get_latest_oplog_chunk_ts some-name
++ local cluster=some-name
+++ kubectl_bin exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ jq '.backups.pitrChunks.pitrChunks | last | .range.end'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.7soAWJJYSs
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.Iepf8D4Wys
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in $(seq 0 2)
+++ set +e
+++ kubectl exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.7soAWJJYSs
+++ cat /tmp/tmp.Iepf8D4Wys
+++ rm /tmp/tmp.7soAWJJYSs /tmp/tmp.Iepf8D4Wys
+++ return 0
++ echo 1766503257
+ last_chunk=1766503257
+ retries=9
++ format_date 1766503257
++ local timestamp=1766503257
+++ TZ=UTC
+++ /usr/sbin/date -d@1766503257 '+%Y-%m-%d %H:%M:%S'
++ echo 2025-12-23 15:20:57
++ format_date 1766503259
++ local timestamp=1766503259
+++ TZ=UTC
+++ /usr/sbin/date -d@1766503259 '+%Y-%m-%d %H:%M:%S'
++ echo 2025-12-23 15:20:59
+ log 'Waiting for last oplog chunk (2025-12-23 15:20:57) to be greater than last write (2025-12-23 15:20:59)'
+ set +o xtrace
[2025-12-23T15:22:42+0000] Waiting for last oplog chunk (2025-12-23 15:20:57) to be greater than last write (2025-12-23 15:20:59)
+ sleep 10
+ [[ 1766503257 -gt 1766503259 ]]
+ [[ 9 -gt 30 ]]
++ get_latest_oplog_chunk_ts some-name
++ local cluster=some-name
+++ jq '.backups.pitrChunks.pitrChunks | last | .range.end'
+++ kubectl_bin exec some-name-rs0-0 -c backup-agent -- pbm status -o json
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.gHYjJ94vTO
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.hdYrnbQrPC
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in $(seq 0 2)
+++ set +e
+++ kubectl exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.gHYjJ94vTO
+++ cat /tmp/tmp.hdYrnbQrPC
+++ rm /tmp/tmp.gHYjJ94vTO /tmp/tmp.hdYrnbQrPC
+++ return 0
++ echo 1766503257
+ last_chunk=1766503257
+ retries=10
++ format_date 1766503257
++ local timestamp=1766503257
+++ TZ=UTC
+++ /usr/sbin/date -d@1766503257 '+%Y-%m-%d %H:%M:%S'
++ echo 2025-12-23 15:20:57
++ format_date 1766503259
++ local timestamp=1766503259
+++ TZ=UTC
+++ /usr/sbin/date -d@1766503259 '+%Y-%m-%d %H:%M:%S'
++ echo 2025-12-23 15:20:59
+ log 'Waiting for last oplog chunk (2025-12-23 15:20:57) to be greater than last write (2025-12-23 15:20:59)'
+ set +o xtrace
[2025-12-23T15:22:55+0000] Waiting for last oplog chunk (2025-12-23 15:20:57) to be greater than last write (2025-12-23 15:20:59)
+ sleep 10
+ [[ 1766503257 -gt 1766503259 ]]
+ [[ 10 -gt 30 ]]
++ get_latest_oplog_chunk_ts some-name
++ local cluster=some-name
+++ kubectl_bin exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ jq '.backups.pitrChunks.pitrChunks | last | .range.end'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.AKUQh5fGSI
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.8RggQgr1hZ
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in $(seq 0 2)
+++ set +e
+++ kubectl exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.AKUQh5fGSI
+++ cat /tmp/tmp.8RggQgr1hZ
+++ rm /tmp/tmp.AKUQh5fGSI /tmp/tmp.8RggQgr1hZ
+++ return 0
++ echo 1766503258
+ last_chunk=1766503258
+ retries=11
++ format_date 1766503258
++ local timestamp=1766503258
+++ TZ=UTC
+++ /usr/sbin/date -d@1766503258 '+%Y-%m-%d %H:%M:%S'
++ echo 2025-12-23 15:20:58
++ format_date 1766503259
++ local timestamp=1766503259
+++ TZ=UTC
+++ /usr/sbin/date -d@1766503259 '+%Y-%m-%d %H:%M:%S'
++ echo 2025-12-23 15:20:59
+ log 'Waiting for last oplog chunk (2025-12-23 15:20:58) to be greater than last write (2025-12-23 15:20:59)'
+ set +o xtrace
[2025-12-23T15:23:07+0000] Waiting for last oplog chunk (2025-12-23 15:20:58) to be greater than last write (2025-12-23 15:20:59)
+ sleep 10
+ [[ 1766503258 -gt 1766503259 ]]
+ [[ 11 -gt 30 ]]
++ get_latest_oplog_chunk_ts some-name
++ local cluster=some-name
+++ kubectl_bin exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ jq '.backups.pitrChunks.pitrChunks | last | .range.end'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.5uTv3SLbUv
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.aXZ1BkucmC
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in $(seq 0 2)
+++ set +e
+++ kubectl exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.5uTv3SLbUv
+++ cat /tmp/tmp.aXZ1BkucmC
+++ rm /tmp/tmp.5uTv3SLbUv /tmp/tmp.aXZ1BkucmC
+++ return 0
++ echo 1766503377
+ last_chunk=1766503377
+ retries=12
++ format_date 1766503377
++ local timestamp=1766503377
+++ TZ=UTC
+++ /usr/sbin/date -d@1766503377 '+%Y-%m-%d %H:%M:%S'
++ echo 2025-12-23 15:22:57
++ format_date 1766503259
++ local timestamp=1766503259
+++ TZ=UTC
+++ /usr/sbin/date -d@1766503259 '+%Y-%m-%d %H:%M:%S'
++ echo 2025-12-23 15:20:59
+ log 'Waiting for last oplog chunk (2025-12-23 15:22:57) to be greater than last write (2025-12-23 15:20:59)'
+ set +o xtrace
[2025-12-23T15:23:20+0000] Waiting for last oplog chunk (2025-12-23 15:22:57) to be greater than last write (2025-12-23 15:20:59)
+ sleep 10
+ [[ 1766503377 -gt 1766503259 ]]
+ sleep 10
+ check_recovery backup-minio-2 date 1766503377 -2nd some-name
+ local backup_name=backup-minio-2
+ local restore_type=date
+ local restore_date=1766503377
+ local cmp_postfix=-2nd
+ local cluster_name=some-name
+ local backupSource=
++ get_latest_oplog_chunk_ts some-name
++ local cluster=some-name
+++ kubectl_bin exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ jq '.backups.pitrChunks.pitrChunks | last | .range.end'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.kEkqikv57M
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.xBeget6a5L
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in $(seq 0 2)
+++ set +e
+++ kubectl exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.kEkqikv57M
+++ cat /tmp/tmp.xBeget6a5L
+++ rm /tmp/tmp.kEkqikv57M /tmp/tmp.xBeget6a5L
+++ return 0
++ echo 1766503377
+ local latest_ts=1766503377
+ desc 'write more data before restore by date'
+ set +o xtrace


-----------------------------------------------------------------------------------
write more data before restore by date
-----------------------------------------------------------------------------------

+ run_mongos 'use myApp\n db.test.insert({ x: 100501 })' myApp:myPass@some-name-mongos.pitr-physical-28895
+ local 'command=use myApp\n db.test.insert({ x: 100501 })'
+ local uri=myApp:myPass@some-name-mongos.pitr-physical-28895
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local port=27017
+ local mongo_bin=mongo
++ echo .svc.cluster.local
++ awk -F: '{print $2}'
+ suffix_port=
+ [[ -z '' ]]
+ suffix=.svc.cluster.local:27017
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.6PrjagzlDq
+++ mktemp
++ local LAST_ERR=/tmp/tmp.0dfyogL7YX
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.6PrjagzlDq
++ cat /tmp/tmp.0dfyogL7YX
++ rm /tmp/tmp.6PrjagzlDq /tmp/tmp.0dfyogL7YX
++ return 0
+ local client_container=psmdb-client-696897d69b-47lj5
+ kubectl_bin exec psmdb-client-696897d69b-47lj5 -- bash -c 'printf '\''use myApp\n db.test.insert({ x: 100501 })\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.pitr-physical-28895.svc.cluster.local:27017/admin '
++ mktemp
+ local LAST_OUT=/tmp/tmp.QNQEeJp8yy
++ mktemp
+ local LAST_ERR=/tmp/tmp.IjmwBjnGbu
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-696897d69b-47lj5 -- bash -c 'printf '\''use myApp\n db.test.insert({ x: 100501 })\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.pitr-physical-28895.svc.cluster.local:27017/admin '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.QNQEeJp8yy
Percona Server for MongoDB shell version v4.4.29-28
connecting to: mongodb://some-name-mongos.pitr-physical-28895.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb
Implicit session: session { "id" : UUID("969bd713-1e97-4590-a1a2-920a53073bd7") }
Percona Server for MongoDB server version: v8.0.16-5
WARNING: shell and server versions do not match
switched to db myApp
WriteResult({ "nInserted" : 1 })
bye
+ cat /tmp/tmp.IjmwBjnGbu
+ rm /tmp/tmp.QNQEeJp8yy /tmp/tmp.IjmwBjnGbu
+ return 0
+ [[ -n 1766503377 ]]
++ format_date 1766503377
++ local timestamp=1766503377
+++ TZ=UTC
+++ /usr/sbin/date -d@1766503377 '+%Y-%m-%d %H:%M:%S'
++ echo 2025-12-23 15:22:57
+ desc 'Restoring to time 2025-12-23 15:22:57'
+ set +o xtrace


-----------------------------------------------------------------------------------
Restoring to time 2025-12-23 15:22:57
-----------------------------------------------------------------------------------

+ retries=0
+ [[ 1766503377 -gt 1766503377 ]]
+ [[ 0 -gt 30 ]]
++ get_latest_oplog_chunk_ts some-name
++ local cluster=some-name
+++ kubectl_bin exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ jq '.backups.pitrChunks.pitrChunks | last | .range.end'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.v0mkxJVyCe
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.AKuQEBBbT9
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in $(seq 0 2)
+++ set +e
+++ kubectl exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.v0mkxJVyCe
+++ cat /tmp/tmp.AKuQEBBbT9
+++ rm /tmp/tmp.v0mkxJVyCe /tmp/tmp.AKuQEBBbT9
+++ return 0
++ echo 1766503377
+ latest_ts=1766503377
+ retries=1
++ format_date 1766503377
++ local timestamp=1766503377
+++ TZ=UTC
+++ /usr/sbin/date -d@1766503377 '+%Y-%m-%d %H:%M:%S'
++ echo 2025-12-23 15:22:57
++ format_date 1766503377
++ local timestamp=1766503377
+++ TZ=UTC
+++ /usr/sbin/date -d@1766503377 '+%Y-%m-%d %H:%M:%S'
++ echo 2025-12-23 15:22:57
+ echo 'Waiting for last oplog chunk (2025-12-23 15:22:57) to be greater than restore target (2025-12-23 15:22:57)'
Waiting for last oplog chunk (2025-12-23 15:22:57) to be greater than restore target (2025-12-23 15:22:57)
+ sleep 10
+ [[ 1766503377 -gt 1766503377 ]]
+ [[ 1 -gt 30 ]]
++ get_latest_oplog_chunk_ts some-name
++ local cluster=some-name
+++ kubectl_bin exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ jq '.backups.pitrChunks.pitrChunks | last | .range.end'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.eYo0oCF6lZ
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.IcquQOej2e
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in $(seq 0 2)
+++ set +e
+++ kubectl exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.eYo0oCF6lZ
+++ cat /tmp/tmp.IcquQOej2e
+++ rm /tmp/tmp.eYo0oCF6lZ /tmp/tmp.IcquQOej2e
+++ return 0
++ echo 1766503377
+ latest_ts=1766503377
+ retries=2
++ format_date 1766503377
++ local timestamp=1766503377
+++ TZ=UTC
+++ /usr/sbin/date -d@1766503377 '+%Y-%m-%d %H:%M:%S'
++ echo 2025-12-23 15:22:57
++ format_date 1766503377
++ local timestamp=1766503377
+++ TZ=UTC
+++ /usr/sbin/date -d@1766503377 '+%Y-%m-%d %H:%M:%S'
++ echo 2025-12-23 15:22:57
+ echo 'Waiting for last oplog chunk (2025-12-23 15:22:57) to be greater than restore target (2025-12-23 15:22:57)'
Waiting for last oplog chunk (2025-12-23 15:22:57) to be greater than restore target (2025-12-23 15:22:57)
+ sleep 10
+ [[ 1766503377 -gt 1766503377 ]]
+ [[ 2 -gt 30 ]]
++ get_latest_oplog_chunk_ts some-name
++ local cluster=some-name
+++ kubectl_bin exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ jq '.backups.pitrChunks.pitrChunks | last | .range.end'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.MV6DSBHmZW
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.GLDTbQaV99
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in $(seq 0 2)
+++ set +e
+++ kubectl exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.MV6DSBHmZW
+++ cat /tmp/tmp.GLDTbQaV99
+++ rm /tmp/tmp.MV6DSBHmZW /tmp/tmp.GLDTbQaV99
+++ return 0
++ echo 1766503377
+ latest_ts=1766503377
+ retries=3
++ format_date 1766503377
++ local timestamp=1766503377
+++ TZ=UTC
+++ /usr/sbin/date -d@1766503377 '+%Y-%m-%d %H:%M:%S'
++ echo 2025-12-23 15:22:57
++ format_date 1766503377
++ local timestamp=1766503377
+++ TZ=UTC
+++ /usr/sbin/date -d@1766503377 '+%Y-%m-%d %H:%M:%S'
++ echo 2025-12-23 15:22:57
+ echo 'Waiting for last oplog chunk (2025-12-23 15:22:57) to be greater than restore target (2025-12-23 15:22:57)'
Waiting for last oplog chunk (2025-12-23 15:22:57) to be greater than restore target (2025-12-23 15:22:57)
+ sleep 10
+ [[ 1766503377 -gt 1766503377 ]]
+ [[ 3 -gt 30 ]]
++ get_latest_oplog_chunk_ts some-name
++ local cluster=some-name
+++ kubectl_bin exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ jq '.backups.pitrChunks.pitrChunks | last | .range.end'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.XR4S0p11xp
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.HCovbLKD2T
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in $(seq 0 2)
+++ set +e
+++ kubectl exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.XR4S0p11xp
+++ cat /tmp/tmp.HCovbLKD2T
+++ rm /tmp/tmp.XR4S0p11xp /tmp/tmp.HCovbLKD2T
+++ return 0
++ echo 1766503377
+ latest_ts=1766503377
+ retries=4
++ format_date 1766503377
++ local timestamp=1766503377
+++ TZ=UTC
+++ /usr/sbin/date -d@1766503377 '+%Y-%m-%d %H:%M:%S'
++ echo 2025-12-23 15:22:57
++ format_date 1766503377
++ local timestamp=1766503377
+++ TZ=UTC
+++ /usr/sbin/date -d@1766503377 '+%Y-%m-%d %H:%M:%S'
++ echo 2025-12-23 15:22:57
+ echo 'Waiting for last oplog chunk (2025-12-23 15:22:57) to be greater than restore target (2025-12-23 15:22:57)'
Waiting for last oplog chunk (2025-12-23 15:22:57) to be greater than restore target (2025-12-23 15:22:57)
+ sleep 10
+ [[ 1766503377 -gt 1766503377 ]]
+ [[ 4 -gt 30 ]]
++ get_latest_oplog_chunk_ts some-name
++ local cluster=some-name
+++ kubectl_bin exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ jq '.backups.pitrChunks.pitrChunks | last | .range.end'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.KIkvHcJePS
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.2iZYEhsE4S
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in $(seq 0 2)
+++ set +e
+++ kubectl exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.KIkvHcJePS
+++ cat /tmp/tmp.2iZYEhsE4S
+++ rm /tmp/tmp.KIkvHcJePS /tmp/tmp.2iZYEhsE4S
+++ return 0
++ echo 1766503377
+ latest_ts=1766503377
+ retries=5
++ format_date 1766503377
++ local timestamp=1766503377
+++ TZ=UTC
+++ /usr/sbin/date -d@1766503377 '+%Y-%m-%d %H:%M:%S'
++ echo 2025-12-23 15:22:57
++ format_date 1766503377
++ local timestamp=1766503377
+++ TZ=UTC
+++ /usr/sbin/date -d@1766503377 '+%Y-%m-%d %H:%M:%S'
++ echo 2025-12-23 15:22:57
+ echo 'Waiting for last oplog chunk (2025-12-23 15:22:57) to be greater than restore target (2025-12-23 15:22:57)'
Waiting for last oplog chunk (2025-12-23 15:22:57) to be greater than restore target (2025-12-23 15:22:57)
+ sleep 10
+ [[ 1766503377 -gt 1766503377 ]]
+ [[ 5 -gt 30 ]]
++ get_latest_oplog_chunk_ts some-name
++ local cluster=some-name
+++ kubectl_bin exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ jq '.backups.pitrChunks.pitrChunks | last | .range.end'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.HJGfwj25J7
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.3kj1NWj3AW
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in $(seq 0 2)
+++ set +e
+++ kubectl exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.HJGfwj25J7
+++ cat /tmp/tmp.3kj1NWj3AW
+++ rm /tmp/tmp.HJGfwj25J7 /tmp/tmp.3kj1NWj3AW
+++ return 0
++ echo 1766503377
+ latest_ts=1766503377
+ retries=6
++ format_date 1766503377
++ local timestamp=1766503377
+++ TZ=UTC
+++ /usr/sbin/date -d@1766503377 '+%Y-%m-%d %H:%M:%S'
++ echo 2025-12-23 15:22:57
++ format_date 1766503377
++ local timestamp=1766503377
+++ TZ=UTC
+++ /usr/sbin/date -d@1766503377 '+%Y-%m-%d %H:%M:%S'
++ echo 2025-12-23 15:22:57
+ echo 'Waiting for last oplog chunk (2025-12-23 15:22:57) to be greater than restore target (2025-12-23 15:22:57)'
Waiting for last oplog chunk (2025-12-23 15:22:57) to be greater than restore target (2025-12-23 15:22:57)
+ sleep 10
+ [[ 1766503377 -gt 1766503377 ]]
+ [[ 6 -gt 30 ]]
++ get_latest_oplog_chunk_ts some-name
++ local cluster=some-name
+++ jq '.backups.pitrChunks.pitrChunks | last | .range.end'
+++ kubectl_bin exec some-name-rs0-0 -c backup-agent -- pbm status -o json
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.tVkT5LH80H
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.w96HKREEnt
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in $(seq 0 2)
+++ set +e
+++ kubectl exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.tVkT5LH80H
+++ cat /tmp/tmp.w96HKREEnt
+++ rm /tmp/tmp.tVkT5LH80H /tmp/tmp.w96HKREEnt
+++ return 0
++ echo 1766503378
+ latest_ts=1766503378
+ retries=7
++ format_date 1766503378
++ local timestamp=1766503378
+++ TZ=UTC
+++ /usr/sbin/date -d@1766503378 '+%Y-%m-%d %H:%M:%S'
++ echo 2025-12-23 15:22:58
++ format_date 1766503377
++ local timestamp=1766503377
+++ TZ=UTC
+++ /usr/sbin/date -d@1766503377 '+%Y-%m-%d %H:%M:%S'
++ echo 2025-12-23 15:22:57
+ echo 'Waiting for last oplog chunk (2025-12-23 15:22:58) to be greater than restore target (2025-12-23 15:22:57)'
Waiting for last oplog chunk (2025-12-23 15:22:58) to be greater than restore target (2025-12-23 15:22:57)
+ sleep 10
+ [[ 1766503378 -gt 1766503377 ]]
+ '[' -z '' ']'
+ desc 'check restore by date'
+ set +o xtrace


-----------------------------------------------------------------------------------
check restore by date
-----------------------------------------------------------------------------------

+ cat /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2158/e2e-tests/pitr-physical/conf/restore.yml
+ /usr/sbin/sed -e 's/name:/name: restore-backup-minio-2/'
+ /usr/sbin/sed -e /backupSource/,+8d
+ /usr/sbin/sed -e 's/backupName:/backupName: backup-minio-2/'
+ /usr/sbin/sed -e 's/pitrType:/type: date/'
+ '[' -z 1766503377 ']'
+ kubectl_bin apply -f -
++ mktemp
++ format_date 1766503377
++ local timestamp=1766503377
+++ TZ=UTC
+++ /usr/sbin/date -d@1766503377 '+%Y-%m-%d %H:%M:%S'
+ local LAST_OUT=/tmp/tmp.oGPcqUUvE9
++ mktemp
++ echo 2025-12-23 15:22:57
+ /usr/sbin/sed -e 's/date:/date: 2025-12-23 15:22:57/'
+ local LAST_ERR=/tmp/tmp.2H35OjQDId
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl apply -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.oGPcqUUvE9
perconaservermongodbrestore.psmdb.percona.com/restore-backup-minio-2 created
+ cat /tmp/tmp.2H35OjQDId
+ rm /tmp/tmp.oGPcqUUvE9 /tmp/tmp.2H35OjQDId
+ return 0
+ wait_restore backup-minio-2 some-name requested 0 1200
+ local backup_name=backup-minio-2
+ local cluster_name=some-name
+ local target_state=requested
+ local wait_cluster_consistency=0
+ local wait_time=1200
+ local ok_if_ready=0
+ set +o xtrace
Waiting for the psmdb-restore/restore-backup-minio-2 object to be created.OK
Waiting psmdb-restore/restore-backup-minio-2 to reach state "requested" ..............OK after 13 minutes
+ [[ 0 -eq 1 ]]
+ echo

+ wait_restore backup-minio-2 some-name ready 0 1600
+ local backup_name=backup-minio-2
+ local cluster_name=some-name
+ local target_state=ready
+ local wait_cluster_consistency=0
+ local wait_time=1600
+ local ok_if_ready=0
+ set +o xtrace
Waiting for the psmdb-restore/restore-backup-minio-2 object to be created.OK
Waiting psmdb-restore/restore-backup-minio-2 to reach state "ready" ...OK after 2 minutes
+ [[ 0 -eq 1 ]]
+ echo

+ set -o xtrace
+ wait_for_running some-name-rs0 3
+ local name=some-name-rs0
+ let last_pod=2
+ local check_cluster_readyness=true
+ set_debug
+ [[ 1 == 1 ]]
+ set -o xtrace
+ local rs_name=rs0
+ local cluster_name=some-name
++ seq 0 2
+ for i in $(seq 0 $last_pod)
+ [[ 0 -eq 2 ]]
+ wait_pod some-name-rs0-0
+ local pod=some-name-rs0-0
+ set +o xtrace
waiting for pod/some-name-rs0-0 to be ready.OK
+ for i in $(seq 0 $last_pod)
+ [[ 1 -eq 2 ]]
+ wait_pod some-name-rs0-1
+ local pod=some-name-rs0-1
+ set +o xtrace
waiting for pod/some-name-rs0-1 to be ready.OK
+ for i in $(seq 0 $last_pod)
+ [[ 2 -eq 2 ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].arbiter.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.Foef67866S
+++ mktemp
++ local LAST_ERR=/tmp/tmp.lzmTwPu2S4
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].arbiter.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.Foef67866S
++ cat /tmp/tmp.lzmTwPu2S4
++ rm /tmp/tmp.Foef67866S /tmp/tmp.lzmTwPu2S4
++ return 0
+ [[ '' == \t\r\u\e ]]
+ wait_pod some-name-rs0-2
+ local pod=some-name-rs0-2
+ set +o xtrace
waiting for pod/some-name-rs0-2 to be ready.OK
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].nonvoting.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.TRtKisntMP
+++ mktemp
++ local LAST_ERR=/tmp/tmp.1NzXVTGoJ6
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].nonvoting.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.TRtKisntMP
++ cat /tmp/tmp.1NzXVTGoJ6
++ rm /tmp/tmp.TRtKisntMP /tmp/tmp.1NzXVTGoJ6
++ return 0
+ [[ '' == \t\r\u\e ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].hidden.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.n89scx9L0p
+++ mktemp
++ local LAST_ERR=/tmp/tmp.PWGBzIfiOM
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].hidden.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.n89scx9L0p
++ cat /tmp/tmp.PWGBzIfiOM
++ rm /tmp/tmp.n89scx9L0p /tmp/tmp.PWGBzIfiOM
++ return 0
+ [[ '' == \t\r\u\e ]]
+ sleep 10
+ [[ true == \t\r\u\e ]]
+ set +x
Waiting for cluster readyness..........................................................................
+ wait_for_running some-name-cfg 3
+ local name=some-name-cfg
+ let last_pod=2
+ local check_cluster_readyness=true
+ set_debug
+ [[ 1 == 1 ]]
+ set -o xtrace
+ local rs_name=cfg
+ local cluster_name=some-name
++ seq 0 2
+ for i in $(seq 0 $last_pod)
+ [[ 0 -eq 2 ]]
+ wait_pod some-name-cfg-0
+ local pod=some-name-cfg-0
+ set +o xtrace
waiting for pod/some-name-cfg-0 to be ready.OK
+ for i in $(seq 0 $last_pod)
+ [[ 1 -eq 2 ]]
+ wait_pod some-name-cfg-1
+ local pod=some-name-cfg-1
+ set +o xtrace
waiting for pod/some-name-cfg-1 to be ready.OK
+ for i in $(seq 0 $last_pod)
+ [[ 2 -eq 2 ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].arbiter.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.bqoGrUxkVg
+++ mktemp
++ local LAST_ERR=/tmp/tmp.rdCml80Ayu
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].arbiter.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.bqoGrUxkVg
++ cat /tmp/tmp.rdCml80Ayu
++ rm /tmp/tmp.bqoGrUxkVg /tmp/tmp.rdCml80Ayu
++ return 0
+ [[ '' == \t\r\u\e ]]
+ wait_pod some-name-cfg-2
+ local pod=some-name-cfg-2
+ set +o xtrace
waiting for pod/some-name-cfg-2 to be ready.OK
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].nonvoting.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.ivFTziz0bI
+++ mktemp
++ local LAST_ERR=/tmp/tmp.PjQCbizjZi
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].nonvoting.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.ivFTziz0bI
++ cat /tmp/tmp.PjQCbizjZi
++ rm /tmp/tmp.ivFTziz0bI /tmp/tmp.PjQCbizjZi
++ return 0
+ [[ '' == \t\r\u\e ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].hidden.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.etl0fsOHTI
+++ mktemp
++ local LAST_ERR=/tmp/tmp.iA9bMTDDZ6
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].hidden.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.etl0fsOHTI
++ cat /tmp/tmp.iA9bMTDDZ6
++ rm /tmp/tmp.etl0fsOHTI /tmp/tmp.iA9bMTDDZ6
++ return 0
+ [[ '' == \t\r\u\e ]]
+ sleep 10
+ [[ true == \t\r\u\e ]]
+ set +x
Waiting for cluster readyness
+ wait_for_running some-name-mongos 3
+ local name=some-name-mongos
+ let last_pod=2
+ local check_cluster_readyness=true
+ set_debug
+ [[ 1 == 1 ]]
+ set -o xtrace
+ local rs_name=mongos
+ local cluster_name=some-name
++ seq 0 2
+ for i in $(seq 0 $last_pod)
+ [[ 0 -eq 2 ]]
+ wait_pod some-name-mongos-0
+ local pod=some-name-mongos-0
+ set +o xtrace
waiting for pod/some-name-mongos-0 to be ready.OK
+ for i in $(seq 0 $last_pod)
+ [[ 1 -eq 2 ]]
+ wait_pod some-name-mongos-1
+ local pod=some-name-mongos-1
+ set +o xtrace
waiting for pod/some-name-mongos-1 to be ready.OK
+ for i in $(seq 0 $last_pod)
+ [[ 2 -eq 2 ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="mongos")].arbiter.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.QFKWQ01Ae6
+++ mktemp
++ local LAST_ERR=/tmp/tmp.RA8jCL8MHD
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="mongos")].arbiter.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.QFKWQ01Ae6
++ cat /tmp/tmp.RA8jCL8MHD
++ rm /tmp/tmp.QFKWQ01Ae6 /tmp/tmp.RA8jCL8MHD
++ return 0
+ [[ '' == \t\r\u\e ]]
+ wait_pod some-name-mongos-2
+ local pod=some-name-mongos-2
+ set +o xtrace
waiting for pod/some-name-mongos-2 to be ready.OK
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="mongos")].nonvoting.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.B8pgBTi88I
+++ mktemp
++ local LAST_ERR=/tmp/tmp.JfstwufnRS
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="mongos")].nonvoting.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.B8pgBTi88I
++ cat /tmp/tmp.JfstwufnRS
++ rm /tmp/tmp.B8pgBTi88I /tmp/tmp.JfstwufnRS
++ return 0
+ [[ '' == \t\r\u\e ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="mongos")].hidden.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.R9t8hLPlMn
+++ mktemp
++ local LAST_ERR=/tmp/tmp.QiSUFJylya
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="mongos")].hidden.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.R9t8hLPlMn
++ cat /tmp/tmp.QiSUFJylya
++ rm /tmp/tmp.R9t8hLPlMn /tmp/tmp.QiSUFJylya
++ return 0
+ [[ '' == \t\r\u\e ]]
+ sleep 10
+ [[ true == \t\r\u\e ]]
+ set +x
Waiting for cluster readyness
+ sleep 10
+ compare_mongos_cmd find myApp:myPass@some-name-mongos.pitr-physical-28895 -2nd
+ local command=find
+ local uri=myApp:myPass@some-name-mongos.pitr-physical-28895
+ local postfix=-2nd
+ local suffix=
+ local database=myApp
+ local collection=test
+ local port=27017
+ local tls=false
+ [[ false == \t\r\u\e ]]
+ mongos_command=run_mongos
+ log 'running db.test.command() in myApp'
+ set +o xtrace
[2025-12-23T15:44:49+0000] running db.test.command() in myApp
+ run_mongos 'use myApp\n db.test.find()' myApp:myPass@some-name-mongos.pitr-physical-28895 mongodb '' '' 27017
+ local 'command=use myApp\n db.test.find()'
+ local uri=myApp:myPass@some-name-mongos.pitr-physical-28895
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local port=27017
+ local mongo_bin=mongo
+ egrep -v 'I NETWORK|W NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|Started a new thread for the timer service'
+ /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
++ echo .svc.cluster.local
++ awk -F: '{print $2}'
egrep: warning: egrep is obsolescent; using grep -E
+ suffix_port=
+ [[ -z '' ]]
+ suffix=.svc.cluster.local:27017
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.58cPKXS3um
+++ mktemp
++ local LAST_ERR=/tmp/tmp.2KBwRQ1FWJ
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.58cPKXS3um
++ cat /tmp/tmp.2KBwRQ1FWJ
++ rm /tmp/tmp.58cPKXS3um /tmp/tmp.2KBwRQ1FWJ
++ return 0
+ local client_container=psmdb-client-696897d69b-47lj5
+ kubectl_bin exec psmdb-client-696897d69b-47lj5 -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.pitr-physical-28895.svc.cluster.local:27017/admin '
++ mktemp
+ local LAST_OUT=/tmp/tmp.QGyBT0GQjk
++ mktemp
+ local LAST_ERR=/tmp/tmp.YfaF8kasid
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-696897d69b-47lj5 -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.pitr-physical-28895.svc.cluster.local:27017/admin '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.QGyBT0GQjk
+ cat /tmp/tmp.YfaF8kasid
+ rm /tmp/tmp.QGyBT0GQjk /tmp/tmp.YfaF8kasid
+ return 0
+ [[ 0 -eq 0 ]]
+ diff /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2158/e2e-tests/pitr-physical/compare/find-2nd.json /tmp/tmp.83ukH1je1W/find-2nd
+ run_backup backup-minio 3 physical
+ local name=backup-minio
+ local idx=3
+ local type=physical
+ desc 'run backup backup-minio-3'
+ set +o xtrace


-----------------------------------------------------------------------------------
run backup backup-minio-3
-----------------------------------------------------------------------------------

+ cat /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2158/e2e-tests/pitr-physical/conf/backup-minio.yml
+ /usr/sbin/sed -e 's/name:/name: backup-minio-3/'
+ /usr/sbin/sed -e 's/type:/type: physical/'
+ kubectl_bin apply -f -
++ mktemp
+ local LAST_OUT=/tmp/tmp.Ohw7z5XwdW
++ mktemp
+ local LAST_ERR=/tmp/tmp.DtQGi7yOrs
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl apply -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.Ohw7z5XwdW
perconaservermongodbbackup.psmdb.percona.com/backup-minio-3 created
+ cat /tmp/tmp.DtQGi7yOrs
+ rm /tmp/tmp.Ohw7z5XwdW /tmp/tmp.DtQGi7yOrs
+ return 0
+ wait_backup backup-minio-3
+ local backup_name=backup-minio-3
+ local target_state=ready
+ set +o xtrace
waiting for backup-minio-3 to reach ready state..............OK
+ sleep 5
+ compare_latest_restorable_time some-name-rs0 backup-minio-3
+ local cluster=some-name-rs0
+ local backup_name=backup-minio-3
+ local latest_restorable_time
+ local backup_time
++ get_latest_restorable_time some-name-rs0
++ local cluster=some-name-rs0
++ local first_timestamp
++ local second_timestamp
++ local retry=0
++ [[ '' != '' ]]
+++ kubectl_bin exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ jq '.backups.pitrChunks.pitrChunks | last | .range.end'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.Qp03POoxFq
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.kcYUvA3KcM
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in $(seq 0 2)
+++ set +e
+++ kubectl exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.Qp03POoxFq
+++ cat /tmp/tmp.kcYUvA3KcM
+++ rm /tmp/tmp.Qp03POoxFq /tmp/tmp.kcYUvA3KcM
+++ return 0
++ first_timestamp=1766503578
++ sleep 5
++ [[ 1766503578 != '' ]]
++ [[ 1766503578 != \n\u\l\l ]]
+++ kubectl_bin exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ jq '.backups.pitrChunks.pitrChunks | last | .range.end'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.c4hUlTqT5H
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.8CBUadIUxS
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in $(seq 0 2)
+++ set +e
+++ kubectl exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.c4hUlTqT5H
+++ cat /tmp/tmp.8CBUadIUxS
+++ rm /tmp/tmp.c4hUlTqT5H /tmp/tmp.8CBUadIUxS
+++ return 0
++ second_timestamp=1766503578
++ let retry+=1
++ [[ 1 -gt 30 ]]
++ [[ 1766503578 != '' ]]
++ [[ 1766503578 != \n\u\l\l ]]
++ [[ 1766503578 == 1766503578 ]]
++ /usr/sbin/date -u -d @1766503578 +%Y-%m-%dT%H:%M:%SZ
+ latest_restorable_time=2025-12-23T15:26:18Z
++ get_latest_restorable_time_from_backup_object backup-minio-3
++ local backup_name=backup-minio-3
++ local latestRestorableTime
++ local retry=0
++ [[ '' != '' ]]
++ sleep 5
+++ kubectl_bin get psmdb-backup backup-minio-3 -o 'jsonpath={.status.latestRestorableTime}'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.xwNvLuNA1Z
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.KKhNVgVxVy
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in $(seq 0 2)
+++ set +e
+++ kubectl get psmdb-backup backup-minio-3 -o 'jsonpath={.status.latestRestorableTime}'
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.xwNvLuNA1Z
+++ cat /tmp/tmp.KKhNVgVxVy
+++ rm /tmp/tmp.xwNvLuNA1Z /tmp/tmp.KKhNVgVxVy
+++ return 0
++ latestRestorableTime=2025-12-23T15:26:18Z
++ let retry+=1
++ [[ 1 -gt 30 ]]
++ [[ 2025-12-23T15:26:18Z != '' ]]
++ [[ 2025-12-23T15:26:18Z != \n\u\l\l ]]
++ echo 2025-12-23T15:26:18Z
+ backup_time=2025-12-23T15:26:18Z
+ [[ 2025-12-23T15:26:18Z != \2\0\2\5\-\1\2\-\2\3\T\1\5\:\2\6\:\1\8\Z ]]
+ check_recovery backup-minio-3 latest '' -3rd some-name
+ local backup_name=backup-minio-3
+ local restore_type=latest
+ local restore_date=
+ local cmp_postfix=-3rd
+ local cluster_name=some-name
+ local backupSource=
++ get_latest_oplog_chunk_ts some-name
++ local cluster=some-name
+++ kubectl_bin exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ jq '.backups.pitrChunks.pitrChunks | last | .range.end'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.5aKNRGKrms
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.gcfKozqKHZ
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in $(seq 0 2)
+++ set +e
+++ kubectl exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.5aKNRGKrms
+++ cat /tmp/tmp.gcfKozqKHZ
+++ rm /tmp/tmp.5aKNRGKrms /tmp/tmp.gcfKozqKHZ
+++ return 0
++ echo 1766503578
+ local latest_ts=1766503578
+ desc 'write more data before restore by latest'
+ set +o xtrace


-----------------------------------------------------------------------------------
write more data before restore by latest
-----------------------------------------------------------------------------------

+ run_mongos 'use myApp\n db.test.insert({ x: 100501 })' myApp:myPass@some-name-mongos.pitr-physical-28895
+ local 'command=use myApp\n db.test.insert({ x: 100501 })'
+ local uri=myApp:myPass@some-name-mongos.pitr-physical-28895
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local port=27017
+ local mongo_bin=mongo
++ echo .svc.cluster.local
++ awk -F: '{print $2}'
+ suffix_port=
+ [[ -z '' ]]
+ suffix=.svc.cluster.local:27017
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.HLr5PrQTRV
+++ mktemp
++ local LAST_ERR=/tmp/tmp.d2KQdvTqBZ
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.HLr5PrQTRV
++ cat /tmp/tmp.d2KQdvTqBZ
++ rm /tmp/tmp.HLr5PrQTRV /tmp/tmp.d2KQdvTqBZ
++ return 0
+ local client_container=psmdb-client-696897d69b-47lj5
+ kubectl_bin exec psmdb-client-696897d69b-47lj5 -- bash -c 'printf '\''use myApp\n db.test.insert({ x: 100501 })\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.pitr-physical-28895.svc.cluster.local:27017/admin '
++ mktemp
+ local LAST_OUT=/tmp/tmp.X9ilKCgvxR
++ mktemp
+ local LAST_ERR=/tmp/tmp.eQwWfcAaBl
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-696897d69b-47lj5 -- bash -c 'printf '\''use myApp\n db.test.insert({ x: 100501 })\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.pitr-physical-28895.svc.cluster.local:27017/admin '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.X9ilKCgvxR
Percona Server for MongoDB shell version v4.4.29-28
connecting to: mongodb://some-name-mongos.pitr-physical-28895.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb
Implicit session: session { "id" : UUID("ad252de0-ca97-4b03-8a52-61f8695fb268") }
Percona Server for MongoDB server version: v8.0.16-5
WARNING: shell and server versions do not match
switched to db myApp
WriteResult({ "nInserted" : 1 })
bye
+ cat /tmp/tmp.eQwWfcAaBl
+ rm /tmp/tmp.X9ilKCgvxR /tmp/tmp.eQwWfcAaBl
+ return 0
+ [[ -n '' ]]
+ desc 'Restoring to latest'
+ set +o xtrace


-----------------------------------------------------------------------------------
Restoring to latest
-----------------------------------------------------------------------------------

++ get_latest_oplog_chunk_ts some-name
++ local cluster=some-name
+++ kubectl_bin exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ jq '.backups.pitrChunks.pitrChunks | last | .range.end'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.bNrT7eQRkv
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.2sydkhnisy
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in $(seq 0 2)
+++ set +e
+++ kubectl exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.bNrT7eQRkv
+++ cat /tmp/tmp.2sydkhnisy
+++ rm /tmp/tmp.bNrT7eQRkv /tmp/tmp.2sydkhnisy
+++ return 0
++ echo 1766503578
+ local current_ts=1766503578
+ retries=0
+ [[ 1766503578 -gt 1766503578 ]]
+ [[ 0 -gt 30 ]]
++ get_latest_oplog_chunk_ts some-name
++ local cluster=some-name
+++ kubectl_bin exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ jq '.backups.pitrChunks.pitrChunks | last | .range.end'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.uO7x8hfmrt
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.LFUbz7TnvP
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in $(seq 0 2)
+++ set +e
+++ kubectl exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.uO7x8hfmrt
+++ cat /tmp/tmp.LFUbz7TnvP
+++ rm /tmp/tmp.uO7x8hfmrt /tmp/tmp.LFUbz7TnvP
+++ return 0
++ echo 1766503578
+ latest_ts=1766503578
+ retries=1
++ format_date 1766503578
++ local timestamp=1766503578
+++ TZ=UTC
+++ /usr/sbin/date -d@1766503578 '+%Y-%m-%d %H:%M:%S'
++ echo 2025-12-23 15:26:18
++ format_date 1766503578
++ local timestamp=1766503578
+++ TZ=UTC
+++ /usr/sbin/date -d@1766503578 '+%Y-%m-%d %H:%M:%S'
++ echo 2025-12-23 15:26:18
+ echo 'Waiting for last oplog chunk (2025-12-23 15:26:18) to be 120 seconds older than starting chunk (2025-12-23 15:26:18)'
Waiting for last oplog chunk (2025-12-23 15:26:18) to be 120 seconds older than starting chunk (2025-12-23 15:26:18)
+ sleep 10
+ [[ 1766503578 -gt 1766503578 ]]
+ [[ 1 -gt 30 ]]
++ get_latest_oplog_chunk_ts some-name
++ local cluster=some-name
+++ kubectl_bin exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ jq '.backups.pitrChunks.pitrChunks | last | .range.end'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.sne3Crb09w
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.iieJ8bFXx5
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in $(seq 0 2)
+++ set +e
+++ kubectl exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.sne3Crb09w
+++ cat /tmp/tmp.iieJ8bFXx5
+++ rm /tmp/tmp.sne3Crb09w /tmp/tmp.iieJ8bFXx5
+++ return 0
++ echo 1766503578
+ latest_ts=1766503578
+ retries=2
++ format_date 1766503578
++ local timestamp=1766503578
+++ TZ=UTC
+++ /usr/sbin/date -d@1766503578 '+%Y-%m-%d %H:%M:%S'
++ echo 2025-12-23 15:26:18
++ format_date 1766503578
++ local timestamp=1766503578
+++ TZ=UTC
+++ /usr/sbin/date -d@1766503578 '+%Y-%m-%d %H:%M:%S'
++ echo 2025-12-23 15:26:18
+ echo 'Waiting for last oplog chunk (2025-12-23 15:26:18) to be 120 seconds older than starting chunk (2025-12-23 15:26:18)'
Waiting for last oplog chunk (2025-12-23 15:26:18) to be 120 seconds older than starting chunk (2025-12-23 15:26:18)
+ sleep 10
+ [[ 1766503578 -gt 1766503578 ]]
+ [[ 2 -gt 30 ]]
++ get_latest_oplog_chunk_ts some-name
++ local cluster=some-name
+++ kubectl_bin exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ jq '.backups.pitrChunks.pitrChunks | last | .range.end'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.AMxy8Kbun5
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.zCzWaJfYiK
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in $(seq 0 2)
+++ set +e
+++ kubectl exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.AMxy8Kbun5
+++ cat /tmp/tmp.zCzWaJfYiK
+++ rm /tmp/tmp.AMxy8Kbun5 /tmp/tmp.zCzWaJfYiK
+++ return 0
++ echo 1766503578
+ latest_ts=1766503578
+ retries=3
++ format_date 1766503578
++ local timestamp=1766503578
+++ TZ=UTC
+++ /usr/sbin/date -d@1766503578 '+%Y-%m-%d %H:%M:%S'
++ echo 2025-12-23 15:26:18
++ format_date 1766503578
++ local timestamp=1766503578
+++ TZ=UTC
+++ /usr/sbin/date -d@1766503578 '+%Y-%m-%d %H:%M:%S'
++ echo 2025-12-23 15:26:18
+ echo 'Waiting for last oplog chunk (2025-12-23 15:26:18) to be 120 seconds older than starting chunk (2025-12-23 15:26:18)'
Waiting for last oplog chunk (2025-12-23 15:26:18) to be 120 seconds older than starting chunk (2025-12-23 15:26:18)
+ sleep 10
+ [[ 1766503578 -gt 1766503578 ]]
+ [[ 3 -gt 30 ]]
++ get_latest_oplog_chunk_ts some-name
++ local cluster=some-name
+++ kubectl_bin exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ jq '.backups.pitrChunks.pitrChunks | last | .range.end'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.NcshXzHOEO
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.5jw9QeTAxe
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in $(seq 0 2)
+++ set +e
+++ kubectl exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.NcshXzHOEO
+++ cat /tmp/tmp.5jw9QeTAxe
+++ rm /tmp/tmp.NcshXzHOEO /tmp/tmp.5jw9QeTAxe
+++ return 0
++ echo 1766503578
+ latest_ts=1766503578
+ retries=4
++ format_date 1766503578
++ local timestamp=1766503578
+++ TZ=UTC
+++ /usr/sbin/date -d@1766503578 '+%Y-%m-%d %H:%M:%S'
++ echo 2025-12-23 15:26:18
++ format_date 1766503578
++ local timestamp=1766503578
+++ TZ=UTC
+++ /usr/sbin/date -d@1766503578 '+%Y-%m-%d %H:%M:%S'
++ echo 2025-12-23 15:26:18
+ echo 'Waiting for last oplog chunk (2025-12-23 15:26:18) to be 120 seconds older than starting chunk (2025-12-23 15:26:18)'
Waiting for last oplog chunk (2025-12-23 15:26:18) to be 120 seconds older than starting chunk (2025-12-23 15:26:18)
+ sleep 10
+ [[ 1766503578 -gt 1766503578 ]]
+ [[ 4 -gt 30 ]]
++ get_latest_oplog_chunk_ts some-name
++ local cluster=some-name
+++ kubectl_bin exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ jq '.backups.pitrChunks.pitrChunks | last | .range.end'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.T28lkzP2KQ
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.NpngSqHz2P
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in $(seq 0 2)
+++ set +e
+++ kubectl exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.T28lkzP2KQ
+++ cat /tmp/tmp.NpngSqHz2P
+++ rm /tmp/tmp.T28lkzP2KQ /tmp/tmp.NpngSqHz2P
+++ return 0
++ echo 1766503578
+ latest_ts=1766503578
+ retries=5
++ format_date 1766503578
++ local timestamp=1766503578
+++ TZ=UTC
+++ /usr/sbin/date -d@1766503578 '+%Y-%m-%d %H:%M:%S'
++ echo 2025-12-23 15:26:18
++ format_date 1766503578
++ local timestamp=1766503578
+++ TZ=UTC
+++ /usr/sbin/date -d@1766503578 '+%Y-%m-%d %H:%M:%S'
++ echo 2025-12-23 15:26:18
+ echo 'Waiting for last oplog chunk (2025-12-23 15:26:18) to be 120 seconds older than starting chunk (2025-12-23 15:26:18)'
Waiting for last oplog chunk (2025-12-23 15:26:18) to be 120 seconds older than starting chunk (2025-12-23 15:26:18)
+ sleep 10
+ [[ 1766503578 -gt 1766503578 ]]
+ [[ 5 -gt 30 ]]
++ get_latest_oplog_chunk_ts some-name
++ local cluster=some-name
+++ kubectl_bin exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ jq '.backups.pitrChunks.pitrChunks | last | .range.end'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.LSJ7PjRUoa
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.4Rr1pzmAAu
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in $(seq 0 2)
+++ set +e
+++ kubectl exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.LSJ7PjRUoa
+++ cat /tmp/tmp.4Rr1pzmAAu
+++ rm /tmp/tmp.LSJ7PjRUoa /tmp/tmp.4Rr1pzmAAu
+++ return 0
++ echo 1766503578
+ latest_ts=1766503578
+ retries=6
++ format_date 1766503578
++ local timestamp=1766503578
+++ TZ=UTC
+++ /usr/sbin/date -d@1766503578 '+%Y-%m-%d %H:%M:%S'
++ echo 2025-12-23 15:26:18
++ format_date 1766503578
++ local timestamp=1766503578
+++ TZ=UTC
+++ /usr/sbin/date -d@1766503578 '+%Y-%m-%d %H:%M:%S'
++ echo 2025-12-23 15:26:18
+ echo 'Waiting for last oplog chunk (2025-12-23 15:26:18) to be 120 seconds older than starting chunk (2025-12-23 15:26:18)'
Waiting for last oplog chunk (2025-12-23 15:26:18) to be 120 seconds older than starting chunk (2025-12-23 15:26:18)
+ sleep 10
+ [[ 1766503578 -gt 1766503578 ]]
+ [[ 6 -gt 30 ]]
++ get_latest_oplog_chunk_ts some-name
++ local cluster=some-name
+++ kubectl_bin exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ jq '.backups.pitrChunks.pitrChunks | last | .range.end'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.10XD9Mo31W
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.E8cEVNk11S
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in $(seq 0 2)
+++ set +e
+++ kubectl exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.10XD9Mo31W
+++ cat /tmp/tmp.E8cEVNk11S
+++ rm /tmp/tmp.10XD9Mo31W /tmp/tmp.E8cEVNk11S
+++ return 0
++ echo 1766503578
+ latest_ts=1766503578
+ retries=7
++ format_date 1766503578
++ local timestamp=1766503578
+++ TZ=UTC
+++ /usr/sbin/date -d@1766503578 '+%Y-%m-%d %H:%M:%S'
++ echo 2025-12-23 15:26:18
++ format_date 1766503578
++ local timestamp=1766503578
+++ TZ=UTC
+++ /usr/sbin/date -d@1766503578 '+%Y-%m-%d %H:%M:%S'
++ echo 2025-12-23 15:26:18
+ echo 'Waiting for last oplog chunk (2025-12-23 15:26:18) to be 120 seconds older than starting chunk (2025-12-23 15:26:18)'
Waiting for last oplog chunk (2025-12-23 15:26:18) to be 120 seconds older than starting chunk (2025-12-23 15:26:18)
+ sleep 10
+ [[ 1766503578 -gt 1766503578 ]]
+ [[ 7 -gt 30 ]]
++ get_latest_oplog_chunk_ts some-name
++ local cluster=some-name
+++ kubectl_bin exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ jq '.backups.pitrChunks.pitrChunks | last | .range.end'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.KlumuFDtw7
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.1VQWidZ1sv
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in $(seq 0 2)
+++ set +e
+++ kubectl exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.KlumuFDtw7
+++ cat /tmp/tmp.1VQWidZ1sv
+++ rm /tmp/tmp.KlumuFDtw7 /tmp/tmp.1VQWidZ1sv
+++ return 0
++ echo 1766503578
+ latest_ts=1766503578
+ retries=8
++ format_date 1766503578
++ local timestamp=1766503578
+++ TZ=UTC
+++ /usr/sbin/date -d@1766503578 '+%Y-%m-%d %H:%M:%S'
++ echo 2025-12-23 15:26:18
++ format_date 1766503578
++ local timestamp=1766503578
+++ TZ=UTC
+++ /usr/sbin/date -d@1766503578 '+%Y-%m-%d %H:%M:%S'
++ echo 2025-12-23 15:26:18
+ echo 'Waiting for last oplog chunk (2025-12-23 15:26:18) to be 120 seconds older than starting chunk (2025-12-23 15:26:18)'
Waiting for last oplog chunk (2025-12-23 15:26:18) to be 120 seconds older than starting chunk (2025-12-23 15:26:18)
+ sleep 10
+ [[ 1766503578 -gt 1766503578 ]]
+ [[ 8 -gt 30 ]]
++ get_latest_oplog_chunk_ts some-name
++ local cluster=some-name
+++ kubectl_bin exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ jq '.backups.pitrChunks.pitrChunks | last | .range.end'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.0AjS5muVU8
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.FFLXjvqDsO
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in $(seq 0 2)
+++ set +e
+++ kubectl exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.0AjS5muVU8
+++ cat /tmp/tmp.FFLXjvqDsO
+++ rm /tmp/tmp.0AjS5muVU8 /tmp/tmp.FFLXjvqDsO
+++ return 0
++ echo 1766503578
+ latest_ts=1766503578
+ retries=9
++ format_date 1766503578
++ local timestamp=1766503578
+++ TZ=UTC
+++ /usr/sbin/date -d@1766503578 '+%Y-%m-%d %H:%M:%S'
++ echo 2025-12-23 15:26:18
++ format_date 1766503578
++ local timestamp=1766503578
+++ TZ=UTC
+++ /usr/sbin/date -d@1766503578 '+%Y-%m-%d %H:%M:%S'
++ echo 2025-12-23 15:26:18
+ echo 'Waiting for last oplog chunk (2025-12-23 15:26:18) to be 120 seconds older than starting chunk (2025-12-23 15:26:18)'
Waiting for last oplog chunk (2025-12-23 15:26:18) to be 120 seconds older than starting chunk (2025-12-23 15:26:18)
+ sleep 10
+ [[ 1766503578 -gt 1766503578 ]]
+ [[ 9 -gt 30 ]]
++ get_latest_oplog_chunk_ts some-name
++ local cluster=some-name
+++ kubectl_bin exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ jq '.backups.pitrChunks.pitrChunks | last | .range.end'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.Gc46roRHLo
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.ALtPwPRgBy
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in $(seq 0 2)
+++ set +e
+++ kubectl exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.Gc46roRHLo
+++ cat /tmp/tmp.ALtPwPRgBy
+++ rm /tmp/tmp.Gc46roRHLo /tmp/tmp.ALtPwPRgBy
+++ return 0
++ echo 1766504835
+ latest_ts=1766504835
+ retries=10
++ format_date 1766504835
++ local timestamp=1766504835
+++ TZ=UTC
+++ /usr/sbin/date -d@1766504835 '+%Y-%m-%d %H:%M:%S'
++ echo 2025-12-23 15:47:15
++ format_date 1766503578
++ local timestamp=1766503578
+++ TZ=UTC
+++ /usr/sbin/date -d@1766503578 '+%Y-%m-%d %H:%M:%S'
++ echo 2025-12-23 15:26:18
+ echo 'Waiting for last oplog chunk (2025-12-23 15:47:15) to be 120 seconds older than starting chunk (2025-12-23 15:26:18)'
Waiting for last oplog chunk (2025-12-23 15:47:15) to be 120 seconds older than starting chunk (2025-12-23 15:26:18)
+ sleep 10
+ [[ 1766504835 -gt 1766503578 ]]
+ '[' -z '' ']'
+ desc 'check restore by latest'
+ set +o xtrace


-----------------------------------------------------------------------------------
check restore by latest
-----------------------------------------------------------------------------------

+ cat /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2158/e2e-tests/pitr-physical/conf/restore.yml
+ /usr/sbin/sed -e 's/name:/name: restore-backup-minio-3/'
+ /usr/sbin/sed -e 's/backupName:/backupName: backup-minio-3/'
+ /usr/sbin/sed -e /backupSource/,+8d
+ /usr/sbin/sed -e 's/pitrType:/type: latest/'
+ '[' -z '' ']'
+ /usr/sbin/sed -e /date:/d
+ kubectl_bin apply -f -
++ mktemp
+ local LAST_OUT=/tmp/tmp.JaMSfkeBkt
++ mktemp
+ local LAST_ERR=/tmp/tmp.l8j92U83pf
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl apply -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.JaMSfkeBkt
perconaservermongodbrestore.psmdb.percona.com/restore-backup-minio-3 created
+ cat /tmp/tmp.l8j92U83pf
+ rm /tmp/tmp.JaMSfkeBkt /tmp/tmp.l8j92U83pf
+ return 0
+ wait_restore backup-minio-3 some-name requested 0 1200
+ local backup_name=backup-minio-3
+ local cluster_name=some-name
+ local target_state=requested
+ local wait_cluster_consistency=0
+ local wait_time=1200
+ local ok_if_ready=0
+ set +o xtrace
Waiting for the psmdb-restore/restore-backup-minio-3 object to be created.OK
Waiting psmdb-restore/restore-backup-minio-3 to reach state "requested" ..............OK after 13 minutes
+ [[ 0 -eq 1 ]]
+ echo

+ wait_restore backup-minio-3 some-name ready 0 1600
+ local backup_name=backup-minio-3
+ local cluster_name=some-name
+ local target_state=ready
+ local wait_cluster_consistency=0
+ local wait_time=1600
+ local ok_if_ready=0
+ set +o xtrace
Waiting for the psmdb-restore/restore-backup-minio-3 object to be created.OK
Waiting psmdb-restore/restore-backup-minio-3 to reach state "ready" ...OK after 2 minutes
+ [[ 0 -eq 1 ]]
+ echo

+ set -o xtrace
+ wait_for_running some-name-rs0 3
+ local name=some-name-rs0
+ let last_pod=2
+ local check_cluster_readyness=true
+ set_debug
+ [[ 1 == 1 ]]
+ set -o xtrace
+ local rs_name=rs0
+ local cluster_name=some-name
++ seq 0 2
+ for i in $(seq 0 $last_pod)
+ [[ 0 -eq 2 ]]
+ wait_pod some-name-rs0-0
+ local pod=some-name-rs0-0
+ set +o xtrace
waiting for pod/some-name-rs0-0 to be ready.OK
+ for i in $(seq 0 $last_pod)
+ [[ 1 -eq 2 ]]
+ wait_pod some-name-rs0-1
+ local pod=some-name-rs0-1
+ set +o xtrace
waiting for pod/some-name-rs0-1 to be ready.OK
+ for i in $(seq 0 $last_pod)
+ [[ 2 -eq 2 ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].arbiter.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.XkAEHY18J1
+++ mktemp
++ local LAST_ERR=/tmp/tmp.bHeKLoew4A
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].arbiter.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.XkAEHY18J1
++ cat /tmp/tmp.bHeKLoew4A
++ rm /tmp/tmp.XkAEHY18J1 /tmp/tmp.bHeKLoew4A
++ return 0
+ [[ '' == \t\r\u\e ]]
+ wait_pod some-name-rs0-2
+ local pod=some-name-rs0-2
+ set +o xtrace
waiting for pod/some-name-rs0-2 to be ready.OK
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].nonvoting.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.XN7VPirHL7
+++ mktemp
++ local LAST_ERR=/tmp/tmp.8ChTL9zS5F
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].nonvoting.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.XN7VPirHL7
++ cat /tmp/tmp.8ChTL9zS5F
++ rm /tmp/tmp.XN7VPirHL7 /tmp/tmp.8ChTL9zS5F
++ return 0
+ [[ '' == \t\r\u\e ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].hidden.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.KXTQmpsJZY
+++ mktemp
++ local LAST_ERR=/tmp/tmp.6XlCTZTrd3
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].hidden.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.KXTQmpsJZY
++ cat /tmp/tmp.6XlCTZTrd3
++ rm /tmp/tmp.KXTQmpsJZY /tmp/tmp.6XlCTZTrd3
++ return 0
+ [[ '' == \t\r\u\e ]]
+ sleep 10
+ [[ true == \t\r\u\e ]]
+ set +x
Waiting for cluster readyness..............................................................................................................
+ wait_for_running some-name-cfg 3
+ local name=some-name-cfg
+ let last_pod=2
+ local check_cluster_readyness=true
+ set_debug
+ [[ 1 == 1 ]]
+ set -o xtrace
+ local rs_name=cfg
+ local cluster_name=some-name
++ seq 0 2
+ for i in $(seq 0 $last_pod)
+ [[ 0 -eq 2 ]]
+ wait_pod some-name-cfg-0
+ local pod=some-name-cfg-0
+ set +o xtrace
waiting for pod/some-name-cfg-0 to be ready.OK
+ for i in $(seq 0 $last_pod)
+ [[ 1 -eq 2 ]]
+ wait_pod some-name-cfg-1
+ local pod=some-name-cfg-1
+ set +o xtrace
waiting for pod/some-name-cfg-1 to be ready.OK
+ for i in $(seq 0 $last_pod)
+ [[ 2 -eq 2 ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].arbiter.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.4ND4cNz08l
+++ mktemp
++ local LAST_ERR=/tmp/tmp.oAR0mYtze3
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].arbiter.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.4ND4cNz08l
++ cat /tmp/tmp.oAR0mYtze3
++ rm /tmp/tmp.4ND4cNz08l /tmp/tmp.oAR0mYtze3
++ return 0
+ [[ '' == \t\r\u\e ]]
+ wait_pod some-name-cfg-2
+ local pod=some-name-cfg-2
+ set +o xtrace
waiting for pod/some-name-cfg-2 to be ready.OK
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].nonvoting.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.9wkGTMsdPS
+++ mktemp
++ local LAST_ERR=/tmp/tmp.RMMdaHrz6l
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].nonvoting.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.9wkGTMsdPS
++ cat /tmp/tmp.RMMdaHrz6l
++ rm /tmp/tmp.9wkGTMsdPS /tmp/tmp.RMMdaHrz6l
++ return 0
+ [[ '' == \t\r\u\e ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].hidden.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.kV6HUcqFAZ
+++ mktemp
++ local LAST_ERR=/tmp/tmp.p21Ligra99
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].hidden.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.kV6HUcqFAZ
++ cat /tmp/tmp.p21Ligra99
++ rm /tmp/tmp.kV6HUcqFAZ /tmp/tmp.p21Ligra99
++ return 0
+ [[ '' == \t\r\u\e ]]
+ sleep 10
+ [[ true == \t\r\u\e ]]
+ set +x
Waiting for cluster readyness
+ wait_for_running some-name-mongos 3
+ local name=some-name-mongos
+ let last_pod=2
+ local check_cluster_readyness=true
+ set_debug
+ [[ 1 == 1 ]]
+ set -o xtrace
+ local rs_name=mongos
+ local cluster_name=some-name
++ seq 0 2
+ for i in $(seq 0 $last_pod)
+ [[ 0 -eq 2 ]]
+ wait_pod some-name-mongos-0
+ local pod=some-name-mongos-0
+ set +o xtrace
waiting for pod/some-name-mongos-0 to be ready.OK
+ for i in $(seq 0 $last_pod)
+ [[ 1 -eq 2 ]]
+ wait_pod some-name-mongos-1
+ local pod=some-name-mongos-1
+ set +o xtrace
waiting for pod/some-name-mongos-1 to be ready.OK
+ for i in $(seq 0 $last_pod)
+ [[ 2 -eq 2 ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="mongos")].arbiter.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.zuFCgYaHgF
+++ mktemp
++ local LAST_ERR=/tmp/tmp.IVw5dcPTWc
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="mongos")].arbiter.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.zuFCgYaHgF
++ cat /tmp/tmp.IVw5dcPTWc
++ rm /tmp/tmp.zuFCgYaHgF /tmp/tmp.IVw5dcPTWc
++ return 0
+ [[ '' == \t\r\u\e ]]
+ wait_pod some-name-mongos-2
+ local pod=some-name-mongos-2
+ set +o xtrace
waiting for pod/some-name-mongos-2 to be ready.OK
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="mongos")].nonvoting.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.m9JsJ6urIf
+++ mktemp
++ local LAST_ERR=/tmp/tmp.rtDFzn4zqV
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="mongos")].nonvoting.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.m9JsJ6urIf
++ cat /tmp/tmp.rtDFzn4zqV
++ rm /tmp/tmp.m9JsJ6urIf /tmp/tmp.rtDFzn4zqV
++ return 0
+ [[ '' == \t\r\u\e ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="mongos")].hidden.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.aUkfnn8GwN
+++ mktemp
++ local LAST_ERR=/tmp/tmp.2dYhT9I0uw
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="mongos")].hidden.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.aUkfnn8GwN
++ cat /tmp/tmp.2dYhT9I0uw
++ rm /tmp/tmp.aUkfnn8GwN /tmp/tmp.2dYhT9I0uw
++ return 0
+ [[ '' == \t\r\u\e ]]
+ sleep 10
+ [[ true == \t\r\u\e ]]
+ set +x
Waiting for cluster readyness
+ sleep 10
+ compare_mongos_cmd find myApp:myPass@some-name-mongos.pitr-physical-28895 -3rd
+ local command=find
+ local uri=myApp:myPass@some-name-mongos.pitr-physical-28895
+ local postfix=-3rd
+ local suffix=
+ local database=myApp
+ local collection=test
+ local port=27017
+ local tls=false
+ [[ false == \t\r\u\e ]]
+ mongos_command=run_mongos
+ log 'running db.test.command() in myApp'
+ set +o xtrace
[2025-12-23T16:07:58+0000] running db.test.command() in myApp
+ run_mongos 'use myApp\n db.test.find()' myApp:myPass@some-name-mongos.pitr-physical-28895 mongodb '' '' 27017
+ egrep -v 'I NETWORK|W NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|Started a new thread for the timer service'
+ local 'command=use myApp\n db.test.find()'
+ local uri=myApp:myPass@some-name-mongos.pitr-physical-28895
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local port=27017
+ local mongo_bin=mongo
+ /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
++ echo .svc.cluster.local
++ awk -F: '{print $2}'
egrep: warning: egrep is obsolescent; using grep -E
+ suffix_port=
+ [[ -z '' ]]
+ suffix=.svc.cluster.local:27017
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.ZqwwYkJvJM
+++ mktemp
++ local LAST_ERR=/tmp/tmp.c2JwwZ10MX
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.ZqwwYkJvJM
++ cat /tmp/tmp.c2JwwZ10MX
++ rm /tmp/tmp.ZqwwYkJvJM /tmp/tmp.c2JwwZ10MX
++ return 0
+ local client_container=psmdb-client-696897d69b-47lj5
+ kubectl_bin exec psmdb-client-696897d69b-47lj5 -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.pitr-physical-28895.svc.cluster.local:27017/admin '
++ mktemp
+ local LAST_OUT=/tmp/tmp.gRwWKliMGd
++ mktemp
+ local LAST_ERR=/tmp/tmp.IotD7H18nM
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-696897d69b-47lj5 -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.pitr-physical-28895.svc.cluster.local:27017/admin '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.gRwWKliMGd
+ cat /tmp/tmp.IotD7H18nM
+ rm /tmp/tmp.gRwWKliMGd /tmp/tmp.IotD7H18nM
+ return 0
+ [[ 0 -eq 0 ]]
+ diff /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2158/e2e-tests/pitr-physical/compare/find-3rd.json /tmp/tmp.83ukH1je1W/find-3rd
+ destroy pitr-physical-28895
+ local namespace=pitr-physical-28895
+ local ignore_logs=true
+ [[ 0 == 1 ]]
+ desc 'destroy cluster/operator and all other resources'
+ set +o xtrace


-----------------------------------------------------------------------------------
destroy cluster/operator and all other resources
-----------------------------------------------------------------------------------

+ '[' true == false ']'
+ delete_backups
+ desc 'Delete psmdb-backup'
+ set +o xtrace


-----------------------------------------------------------------------------------
Delete psmdb-backup
-----------------------------------------------------------------------------------

++ kubectl_bin get psmdb-backup --no-headers
++ wc -l
+++ mktemp
++ local LAST_OUT=/tmp/tmp.yJg0oDfV72
+++ mktemp
++ local LAST_ERR=/tmp/tmp.NW9g23pHug
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb-backup --no-headers
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.yJg0oDfV72
++ cat /tmp/tmp.NW9g23pHug
++ rm /tmp/tmp.yJg0oDfV72 /tmp/tmp.NW9g23pHug
++ return 0
+ '[' 2 '!=' 0 ']'
+ kubectl_bin get psmdb-backup
++ mktemp
+ local LAST_OUT=/tmp/tmp.HntDvK0Hq6
++ mktemp
+ local LAST_ERR=/tmp/tmp.TFCaJFU9qI
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl get psmdb-backup
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.HntDvK0Hq6
NAME             CLUSTER     STORAGE   DESTINATION                                  TYPE       SIZE     STATUS   COMPLETED   AGE
backup-minio-2   some-name   minio     s3://operator-testing/2025-12-23T15:18:16Z   physical   5.94MB   ready    49m         49m
backup-minio-3   some-name   minio     s3://operator-testing/2025-12-23T15:44:54Z   physical   3.69MB   ready    22m         23m
+ cat /tmp/tmp.TFCaJFU9qI
+ rm /tmp/tmp.HntDvK0Hq6 /tmp/tmp.TFCaJFU9qI
+ return 0
+ kubectl_bin delete psmdb-backup --all
++ mktemp
+ local LAST_OUT=/tmp/tmp.Pj2AC9NGkX
++ mktemp
+ local LAST_ERR=/tmp/tmp.38MqlcD7lK
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl delete psmdb-backup --all
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.Pj2AC9NGkX
perconaservermongodbbackup.psmdb.percona.com "backup-minio-2" deleted from pitr-physical-28895 namespace
perconaservermongodbbackup.psmdb.percona.com "backup-minio-3" deleted from pitr-physical-28895 namespace
+ cat /tmp/tmp.38MqlcD7lK
+ rm /tmp/tmp.Pj2AC9NGkX /tmp/tmp.38MqlcD7lK
+ return 0
+ delete_crd
+ desc 'get and delete old CRDs and RBAC'
+ set +o xtrace


-----------------------------------------------------------------------------------
get and delete old CRDs and RBAC
-----------------------------------------------------------------------------------

+ kubectl_bin delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2158/deploy/crd.yaml --ignore-not-found --wait=false
++ mktemp
+ local LAST_OUT=/tmp/tmp.qugTz0JCSO
++ mktemp
+ local LAST_ERR=/tmp/tmp.SNCsuvliTN
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2158/deploy/crd.yaml --ignore-not-found --wait=false
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.qugTz0JCSO
customresourcedefinition.apiextensions.k8s.io "perconaservermongodbbackups.psmdb.percona.com" deleted
customresourcedefinition.apiextensions.k8s.io "perconaservermongodbrestores.psmdb.percona.com" deleted
customresourcedefinition.apiextensions.k8s.io "perconaservermongodbs.psmdb.percona.com" deleted
+ cat /tmp/tmp.SNCsuvliTN
+ rm /tmp/tmp.qugTz0JCSO /tmp/tmp.SNCsuvliTN
+ return 0
++ yq eval .metadata.name /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2158/deploy/crd.yaml
++ grep -v '\-\-\-'
grep: warning: stray \ before -
grep: warning: stray \ before -
+ for crd_name in $(yq eval '.metadata.name' "${src_dir}/deploy/crd.yaml" | grep -v '\-\-\-')
+ kubectl get perconaservermongodbbackups.psmdb.percona.com --all-namespaces -o wide
+ grep -v NAMESPACE
+ xargs -L 1 sh -xc 'kubectl patch perconaservermongodbbackups.psmdb.percona.com -n $0 $1 --type=merge -p "{\"metadata\":{\"finalizers\":[]}}"'
error: the server doesn't have a resource type "perconaservermongodbbackups"
+ kubectl patch perconaservermongodbbackups.psmdb.percona.com -n sh --type=merge -p '{"metadata":{"finalizers":[]}}'
error: the server doesn't have a resource type "perconaservermongodbbackups"
+ :
+ kubectl_bin wait --for=delete crd perconaservermongodbbackups.psmdb.percona.com
++ mktemp
+ local LAST_OUT=/tmp/tmp.tL6PoTBQKF
++ mktemp
+ local LAST_ERR=/tmp/tmp.gWauIrQioG
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl wait --for=delete crd perconaservermongodbbackups.psmdb.percona.com
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.tL6PoTBQKF
+ cat /tmp/tmp.gWauIrQioG
+ rm /tmp/tmp.tL6PoTBQKF /tmp/tmp.gWauIrQioG
+ return 0
+ for crd_name in $(yq eval '.metadata.name' "${src_dir}/deploy/crd.yaml" | grep -v '\-\-\-')
+ kubectl get perconaservermongodbrestores.psmdb.percona.com --all-namespaces -o wide
+ grep -v NAMESPACE
+ xargs -L 1 sh -xc 'kubectl patch perconaservermongodbrestores.psmdb.percona.com -n $0 $1 --type=merge -p "{\"metadata\":{\"finalizers\":[]}}"'
error: the server doesn't have a resource type "perconaservermongodbrestores"
+ kubectl patch perconaservermongodbrestores.psmdb.percona.com -n sh --type=merge -p '{"metadata":{"finalizers":[]}}'
error: the server doesn't have a resource type "perconaservermongodbrestores"
+ :
+ kubectl_bin wait --for=delete crd perconaservermongodbrestores.psmdb.percona.com
++ mktemp
+ local LAST_OUT=/tmp/tmp.4bW6aX1XoL
++ mktemp
+ local LAST_ERR=/tmp/tmp.8s63HFPvWW
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl wait --for=delete crd perconaservermongodbrestores.psmdb.percona.com
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.4bW6aX1XoL
+ cat /tmp/tmp.8s63HFPvWW
+ rm /tmp/tmp.4bW6aX1XoL /tmp/tmp.8s63HFPvWW
+ return 0
+ for crd_name in $(yq eval '.metadata.name' "${src_dir}/deploy/crd.yaml" | grep -v '\-\-\-')
+ kubectl get perconaservermongodbs.psmdb.percona.com --all-namespaces -o wide
+ grep -v NAMESPACE
+ xargs -L 1 sh -xc 'kubectl patch perconaservermongodbs.psmdb.percona.com -n $0 $1 --type=merge -p "{\"metadata\":{\"finalizers\":[]}}"'
error: the server doesn't have a resource type "perconaservermongodbs"
+ kubectl patch perconaservermongodbs.psmdb.percona.com -n sh --type=merge -p '{"metadata":{"finalizers":[]}}'
error: the server doesn't have a resource type "perconaservermongodbs"
+ :
+ kubectl_bin wait --for=delete crd perconaservermongodbs.psmdb.percona.com
++ mktemp
+ local LAST_OUT=/tmp/tmp.1KJtIk5VF1
++ mktemp
+ local LAST_ERR=/tmp/tmp.znYtVVRtgw
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl wait --for=delete crd perconaservermongodbs.psmdb.percona.com
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.1KJtIk5VF1
+ cat /tmp/tmp.znYtVVRtgw
+ rm /tmp/tmp.1KJtIk5VF1 /tmp/tmp.znYtVVRtgw
+ return 0
+ local rbac_yaml=rbac.yaml
+ '[' -n psmdb-operator ']'
+ rbac_yaml=cw-rbac.yaml
+ kubectl_bin delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2158/deploy/cw-rbac.yaml --ignore-not-found
++ mktemp
+ local LAST_OUT=/tmp/tmp.RloSQXGtiD
++ mktemp
+ local LAST_ERR=/tmp/tmp.6dXSja0moE
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2158/deploy/cw-rbac.yaml --ignore-not-found
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.RloSQXGtiD
clusterrole.rbac.authorization.k8s.io "percona-server-mongodb-operator" deleted
clusterrolebinding.rbac.authorization.k8s.io "service-account-percona-server-mongodb-operator" deleted
+ cat /tmp/tmp.6dXSja0moE
+ rm /tmp/tmp.RloSQXGtiD /tmp/tmp.6dXSja0moE
+ return 0
+ destroy_cert_manager
+ kubectl_bin delete -f https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml
++ mktemp
+ local LAST_OUT=/tmp/tmp.HFAxy8DBoC
++ mktemp
+ local LAST_ERR=/tmp/tmp.DpzkS8za3e
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl delete -f https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml
+ exit_status=1
+ set -e
+ '[' 1 '!=' 0 -a -n 1 ']'
+ cat /tmp/tmp.HFAxy8DBoC
+ cat /tmp/tmp.DpzkS8za3e
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": namespaces "cert-manager" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "challenges.acme.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "orders.acme.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "certificaterequests.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "certificates.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "clusterissuers.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "issuers.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": serviceaccounts "cert-manager-cainjector" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": serviceaccounts "cert-manager" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": serviceaccounts "cert-manager-webhook" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-cainjector" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-issuers" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-clusterissuers" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-certificates" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-orders" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-challenges" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-ingress-shim" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-cluster-view" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-view" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-edit" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-approve:cert-manager-io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-certificatesigningrequests" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-webhook:subjectaccessreviews" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-cainjector" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-issuers" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-clusterissuers" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-certificates" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-orders" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-challenges" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-ingress-shim" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-approve:cert-manager-io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-certificatesigningrequests" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-webhook:subjectaccessreviews" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager-cainjector:leaderelection" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager:leaderelection" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager-tokenrequest" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager-webhook:dynamic-serving" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager-cainjector:leaderelection" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager:leaderelection" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager-tokenrequest" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager-webhook:dynamic-serving" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": services "cert-manager-cainjector" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": services "cert-manager" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": services "cert-manager-webhook" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": deployments.apps "cert-manager-cainjector" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": deployments.apps "cert-manager" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": deployments.apps "cert-manager-webhook" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": mutatingwebhookconfigurations.admissionregistration.k8s.io "cert-manager-webhook" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": validatingwebhookconfigurations.admissionregistration.k8s.io "cert-manager-webhook" not found
+ sleep 0
+ for i in $(seq 0 2)
+ set +e
+ kubectl delete -f https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml
+ exit_status=1
+ set -e
+ '[' 1 '!=' 0 -a -n 1 ']'
+ cat /tmp/tmp.HFAxy8DBoC
+ cat /tmp/tmp.DpzkS8za3e
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": namespaces "cert-manager" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "challenges.acme.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "orders.acme.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "certificaterequests.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "certificates.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "clusterissuers.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "issuers.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": serviceaccounts "cert-manager-cainjector" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": serviceaccounts "cert-manager" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": serviceaccounts "cert-manager-webhook" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-cainjector" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-issuers" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-clusterissuers" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-certificates" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-orders" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-challenges" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-ingress-shim" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-cluster-view" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-view" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-edit" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-approve:cert-manager-io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-certificatesigningrequests" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-webhook:subjectaccessreviews" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-cainjector" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-issuers" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-clusterissuers" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-certificates" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-orders" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-challenges" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-ingress-shim" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-approve:cert-manager-io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-certificatesigningrequests" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-webhook:subjectaccessreviews" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager-cainjector:leaderelection" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager:leaderelection" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager-tokenrequest" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager-webhook:dynamic-serving" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager-cainjector:leaderelection" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager:leaderelection" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager-tokenrequest" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager-webhook:dynamic-serving" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": services "cert-manager-cainjector" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": services "cert-manager" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": services "cert-manager-webhook" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": deployments.apps "cert-manager-cainjector" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": deployments.apps "cert-manager" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": deployments.apps "cert-manager-webhook" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": mutatingwebhookconfigurations.admissionregistration.k8s.io "cert-manager-webhook" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": validatingwebhookconfigurations.admissionregistration.k8s.io "cert-manager-webhook" not found
+ sleep 4
+ for i in $(seq 0 2)
+ set +e
+ kubectl delete -f https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml
+ exit_status=1
+ set -e
+ '[' 1 '!=' 0 -a -n 1 ']'
+ cat /tmp/tmp.HFAxy8DBoC
+ cat /tmp/tmp.DpzkS8za3e
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": namespaces "cert-manager" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "challenges.acme.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "orders.acme.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "certificaterequests.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "certificates.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "clusterissuers.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "issuers.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": serviceaccounts "cert-manager-cainjector" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": serviceaccounts "cert-manager" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": serviceaccounts "cert-manager-webhook" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-cainjector" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-issuers" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-clusterissuers" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-certificates" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-orders" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-challenges" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-ingress-shim" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-cluster-view" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-view" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-edit" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-approve:cert-manager-io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-certificatesigningrequests" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-webhook:subjectaccessreviews" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-cainjector" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-issuers" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-clusterissuers" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-certificates" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-orders" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-challenges" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-ingress-shim" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-approve:cert-manager-io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-certificatesigningrequests" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-webhook:subjectaccessreviews" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager-cainjector:leaderelection" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager:leaderelection" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager-tokenrequest" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager-webhook:dynamic-serving" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager-cainjector:leaderelection" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager:leaderelection" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager-tokenrequest" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager-webhook:dynamic-serving" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": services "cert-manager-cainjector" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": services "cert-manager" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": services "cert-manager-webhook" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": deployments.apps "cert-manager-cainjector" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": deployments.apps "cert-manager" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": deployments.apps "cert-manager-webhook" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": mutatingwebhookconfigurations.admissionregistration.k8s.io "cert-manager-webhook" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": validatingwebhookconfigurations.admissionregistration.k8s.io "cert-manager-webhook" not found
+ sleep 8
+ cat /tmp/tmp.HFAxy8DBoC
+ cat /tmp/tmp.DpzkS8za3e
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": namespaces "cert-manager" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "challenges.acme.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "orders.acme.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "certificaterequests.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "certificates.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "clusterissuers.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "issuers.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": serviceaccounts "cert-manager-cainjector" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": serviceaccounts "cert-manager" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": serviceaccounts "cert-manager-webhook" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-cainjector" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-issuers" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-clusterissuers" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-certificates" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-orders" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-challenges" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-ingress-shim" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-cluster-view" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-view" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-edit" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-approve:cert-manager-io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-certificatesigningrequests" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-webhook:subjectaccessreviews" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-cainjector" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-issuers" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-clusterissuers" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-certificates" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-orders" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-challenges" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-ingress-shim" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-approve:cert-manager-io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-certificatesigningrequests" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-webhook:subjectaccessreviews" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager-cainjector:leaderelection" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager:leaderelection" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager-tokenrequest" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager-webhook:dynamic-serving" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager-cainjector:leaderelection" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager:leaderelection" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager-tokenrequest" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager-webhook:dynamic-serving" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": services "cert-manager-cainjector" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": services "cert-manager" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": services "cert-manager-webhook" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": deployments.apps "cert-manager-cainjector" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": deployments.apps "cert-manager" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": deployments.apps "cert-manager-webhook" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": mutatingwebhookconfigurations.admissionregistration.k8s.io "cert-manager-webhook" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml": validatingwebhookconfigurations.admissionregistration.k8s.io "cert-manager-webhook" not found
+ rm /tmp/tmp.HFAxy8DBoC /tmp/tmp.DpzkS8za3e
+ return 1
+ true
+ '[' -n '' ']'
+ '[' -n psmdb-operator ']'
+ kubectl_bin delete --grace-period=0 --force=true namespace pitr-physical-28895
+ rm -rf /tmp/tmp.83ukH1je1W
+ kubectl_bin delete --grace-period=0 --force=true namespace psmdb-operator
++ mktemp
++ mktemp
+ local LAST_OUT=/tmp/tmp.PXQokBWYiL
+ desc 'test passed'
+ local LAST_OUT=/tmp/tmp.bFo2wCUR8W
+ set +o xtrace


-----------------------------------------------------------------------------------
test passed
-----------------------------------------------------------------------------------

++ mktemp
++ mktemp
+ local LAST_ERR=/tmp/tmp.FlHO0UDfRA
+ local exit_status=0
+ local timeout=4
+ local LAST_ERR=/tmp/tmp.qwHz2dAGwH
+ local exit_status=0
+ local timeout=4
++ seq 0 2
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl delete --grace-period=0 --force=true namespace psmdb-operator
+ for i in $(seq 0 2)
+ set +e
+ kubectl delete --grace-period=0 --force=true namespace pitr-physical-28895