Log: /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2039/e2e-tests/logs/pitr-sharded.log
grep: warning: stray \ before -
Warning: version difference between client (1.34) and server (1.30) exceeds the supported minor version skew of +/-1
Warning: version difference between client (1.34) and server (1.30) exceeds the supported minor version skew of +/-1
Warning: version difference between client (1.34) and server (1.30) exceeds the supported minor version skew of +/-1
+ main
+ create_infra pitr-sharded-3057
+ local ns=pitr-sharded-3057
+ [[ 1 == 1 ]]
+ delete_crd
+ desc 'get and delete old CRDs and RBAC'
+ set +o xtrace


-----------------------------------------------------------------------------------
get and delete old CRDs and RBAC
-----------------------------------------------------------------------------------

+ kubectl_bin delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2039/deploy/crd.yaml --ignore-not-found --wait=false
++ mktemp
+ local LAST_OUT=/tmp/tmp.nwXtTiTsQT
++ mktemp
+ local LAST_ERR=/tmp/tmp.0YAfLxPlEJ
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2039/deploy/crd.yaml --ignore-not-found --wait=false
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.nwXtTiTsQT
customresourcedefinition.apiextensions.k8s.io "perconaservermongodbbackups.psmdb.percona.com" deleted
customresourcedefinition.apiextensions.k8s.io "perconaservermongodbrestores.psmdb.percona.com" deleted
customresourcedefinition.apiextensions.k8s.io "perconaservermongodbs.psmdb.percona.com" deleted
+ cat /tmp/tmp.0YAfLxPlEJ
+ rm /tmp/tmp.nwXtTiTsQT /tmp/tmp.0YAfLxPlEJ
+ return 0
++ yq eval .metadata.name /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2039/deploy/crd.yaml
++ grep -v '\-\-\-'
grep: warning: stray \ before -
grep: warning: stray \ before -
+ for crd_name in $(yq eval '.metadata.name' "${src_dir}/deploy/crd.yaml" | grep -v '\-\-\-')
+ kubectl get perconaservermongodbbackups.psmdb.percona.com --all-namespaces -o wide
+ grep -v NAMESPACE
+ xargs -L 1 sh -xc 'kubectl patch perconaservermongodbbackups.psmdb.percona.com -n $0 $1 --type=merge -p "{\"metadata\":{\"finalizers\":[]}}"'
No resources found
+ kubectl patch perconaservermongodbbackups.psmdb.percona.com -n sh --type=merge -p '{"metadata":{"finalizers":[]}}'
error: resource(s) were provided, but no name was specified
+ :
+ kubectl_bin wait --for=delete crd perconaservermongodbbackups.psmdb.percona.com
++ mktemp
+ local LAST_OUT=/tmp/tmp.Mkg7uD4LGU
++ mktemp
+ local LAST_ERR=/tmp/tmp.aQXsaurShf
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl wait --for=delete crd perconaservermongodbbackups.psmdb.percona.com
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.Mkg7uD4LGU
+ cat /tmp/tmp.aQXsaurShf
+ rm /tmp/tmp.Mkg7uD4LGU /tmp/tmp.aQXsaurShf
+ return 0
+ for crd_name in $(yq eval '.metadata.name' "${src_dir}/deploy/crd.yaml" | grep -v '\-\-\-')
+ kubectl get perconaservermongodbrestores.psmdb.percona.com --all-namespaces -o wide
+ grep -v NAMESPACE
+ xargs -L 1 sh -xc 'kubectl patch perconaservermongodbrestores.psmdb.percona.com -n $0 $1 --type=merge -p "{\"metadata\":{\"finalizers\":[]}}"'
error: the server doesn't have a resource type "perconaservermongodbrestores"
+ kubectl patch perconaservermongodbrestores.psmdb.percona.com -n sh --type=merge -p '{"metadata":{"finalizers":[]}}'
error: the server doesn't have a resource type "perconaservermongodbrestores"
+ :
+ kubectl_bin wait --for=delete crd perconaservermongodbrestores.psmdb.percona.com
++ mktemp
+ local LAST_OUT=/tmp/tmp.WhdSAYI2fr
++ mktemp
+ local LAST_ERR=/tmp/tmp.5ErAnIvBkG
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl wait --for=delete crd perconaservermongodbrestores.psmdb.percona.com
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.WhdSAYI2fr
+ cat /tmp/tmp.5ErAnIvBkG
+ rm /tmp/tmp.WhdSAYI2fr /tmp/tmp.5ErAnIvBkG
+ return 0
+ for crd_name in $(yq eval '.metadata.name' "${src_dir}/deploy/crd.yaml" | grep -v '\-\-\-')
+ kubectl get perconaservermongodbs.psmdb.percona.com --all-namespaces -o wide
+ grep -v NAMESPACE
+ xargs -L 1 sh -xc 'kubectl patch perconaservermongodbs.psmdb.percona.com -n $0 $1 --type=merge -p "{\"metadata\":{\"finalizers\":[]}}"'
error: the server doesn't have a resource type "perconaservermongodbs"
+ kubectl patch perconaservermongodbs.psmdb.percona.com -n sh --type=merge -p '{"metadata":{"finalizers":[]}}'
error: the server doesn't have a resource type "perconaservermongodbs"
+ :
+ kubectl_bin wait --for=delete crd perconaservermongodbs.psmdb.percona.com
++ mktemp
+ local LAST_OUT=/tmp/tmp.8SFBx1u1ba
++ mktemp
+ local LAST_ERR=/tmp/tmp.aPcJtTE2GT
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl wait --for=delete crd perconaservermongodbs.psmdb.percona.com
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.8SFBx1u1ba
+ cat /tmp/tmp.aPcJtTE2GT
+ rm /tmp/tmp.8SFBx1u1ba /tmp/tmp.aPcJtTE2GT
+ return 0
+ local rbac_yaml=rbac.yaml
+ '[' -n psmdb-operator ']'
+ rbac_yaml=cw-rbac.yaml
+ kubectl_bin delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2039/deploy/cw-rbac.yaml --ignore-not-found
++ mktemp
+ local LAST_OUT=/tmp/tmp.sJvZmoglBV
++ mktemp
+ local LAST_ERR=/tmp/tmp.rG4twzKuyh
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2039/deploy/cw-rbac.yaml --ignore-not-found
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.sJvZmoglBV
clusterrole.rbac.authorization.k8s.io "percona-server-mongodb-operator" deleted
clusterrolebinding.rbac.authorization.k8s.io "service-account-percona-server-mongodb-operator" deleted
+ cat /tmp/tmp.rG4twzKuyh
+ rm /tmp/tmp.sJvZmoglBV /tmp/tmp.rG4twzKuyh
+ return 0
+ check_crd_for_deletion PR-2039-368d0b21
+ local git_tag=PR-2039-368d0b21
++ curl -s https://raw.githubusercontent.com/percona/percona-server-mongodb-operator/PR-2039-368d0b21/deploy/crd.yaml
++ /usr/sbin/sed s/---//g
++ yq eval .metadata.name
++ /usr/sbin/sed ':a;N;$!ba;s/\n/ /g'
+ for crd_name in $(curl -s https://raw.githubusercontent.com/percona/percona-server-mongodb-operator/${git_tag}/deploy/crd.yaml | yq eval '.metadata.name' | $sed 's/---//g' | $sed ':a;N;$!ba;s/\n/ /g')
++ kubectl_bin get crd/null -o 'jsonpath={.status.conditions[-1].type}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.qQLk6ii5fH
+++ mktemp
++ local LAST_ERR=/tmp/tmp.t02b9I609H
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get crd/null -o 'jsonpath={.status.conditions[-1].type}'
++ exit_status=1
++ set -e
++ '[' 1 '!=' 0 -a -n 1 ']'
++ cat /tmp/tmp.qQLk6ii5fH
++ cat /tmp/tmp.t02b9I609H
Error from server (NotFound): customresourcedefinitions.apiextensions.k8s.io "null" not found
++ sleep 0
++ for i in $(seq 0 2)
++ set +e
++ kubectl get crd/null -o 'jsonpath={.status.conditions[-1].type}'
++ exit_status=1
++ set -e
++ '[' 1 '!=' 0 -a -n 1 ']'
++ cat /tmp/tmp.qQLk6ii5fH
++ cat /tmp/tmp.t02b9I609H
Error from server (NotFound): customresourcedefinitions.apiextensions.k8s.io "null" not found
++ sleep 4
++ for i in $(seq 0 2)
++ set +e
++ kubectl get crd/null -o 'jsonpath={.status.conditions[-1].type}'
++ exit_status=1
++ set -e
++ '[' 1 '!=' 0 -a -n 1 ']'
++ cat /tmp/tmp.qQLk6ii5fH
++ cat /tmp/tmp.t02b9I609H
Error from server (NotFound): customresourcedefinitions.apiextensions.k8s.io "null" not found
++ sleep 8
++ cat /tmp/tmp.qQLk6ii5fH
++ cat /tmp/tmp.t02b9I609H
Error from server (NotFound): customresourcedefinitions.apiextensions.k8s.io "null" not found
++ rm /tmp/tmp.qQLk6ii5fH /tmp/tmp.t02b9I609H
++ return 1
+ [[ '' == \T\e\r\m\i\n\a\t\i\n\g ]]
+ '[' -n psmdb-operator ']'
+ create_namespace psmdb-operator
+ local namespace=psmdb-operator
+ local skip_clean_namespace=
+ [[ 1 == 1 ]]
+ [[ -z '' ]]
+ destroy_chaos_mesh
++ helm list --all-namespaces --filter chaos-mesh
++ tail -n1
++ awk '-F ' '{print $2}'
++ sed s/NAMESPACE//
+ local chaos_mesh_ns=
+ desc 'destroy chaos-mesh'
+ set +o xtrace


-----------------------------------------------------------------------------------
destroy chaos-mesh
-----------------------------------------------------------------------------------

+ '[' -n '' ']'
++ kubectl get MutatingWebhookConfiguration
++ grep chaos-mesh
++ awk '{print $1}'
+ timeout 30 kubectl delete MutatingWebhookConfiguration
error: resource(s) were provided, but no name was specified
+ :
++ kubectl get ValidatingWebhookConfiguration
++ grep chaos-mesh
++ awk '{print $1}'
+ timeout 30 kubectl delete ValidatingWebhookConfiguration
error: resource(s) were provided, but no name was specified
+ :
++ kubectl get ValidatingWebhookConfiguration
++ grep validate-auth
++ awk '{print $1}'
+ timeout 30 kubectl delete ValidatingWebhookConfiguration
error: resource(s) were provided, but no name was specified
+ :
++ kubectl api-resources
++ grep chaos-mesh
++ awk '{print $1}'
++ kubectl get crd
++ grep chaos-mesh.org
++ awk '{print $1}'
+ timeout 30 kubectl delete crd
error: resource(s) were provided, but no name was specified
+ :
++ kubectl get clusterrolebinding
++ grep chaos-mesh
++ awk '{print $1}'
+ timeout 30 kubectl delete clusterrolebinding
error: resource(s) were provided, but no name was specified
+ :
++ kubectl get clusterrole
++ grep chaos-mesh
++ awk '{print $1}'
+ timeout 30 kubectl delete clusterrole
error: resource(s) were provided, but no name was specified
+ :
+ desc 'cleaned up all old namespaces'
+ set +o xtrace


-----------------------------------------------------------------------------------
cleaned up all old namespaces
-----------------------------------------------------------------------------------

+ kubectl_bin get ns
+ egrep -v '^kube-|^default|Terminating|psmdb-operator|openshift|^gke-|^gmp-|^NAME'
+ awk '{print$1}'
+ '[' -n '' ']'
+ xargs kubectl delete ns
++ mktemp
+ desc 'cleaned up old namespaces psmdb-operator'
+ set +o xtrace


-----------------------------------------------------------------------------------
cleaned up old namespaces psmdb-operator
-----------------------------------------------------------------------------------

+ kubectl_bin delete namespace psmdb-operator --ignore-not-found
++ mktemp
+ local LAST_OUT=/tmp/tmp.k5qsN8hdkC
egrep: warning: egrep is obsolescent; using grep -E
+ local LAST_OUT=/tmp/tmp.lSu0ujr81l
++ mktemp
++ mktemp
+ local LAST_ERR=/tmp/tmp.H77kFqzE3b
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ local LAST_ERR=/tmp/tmp.yhLvFLCNGG
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl get ns
+ for i in $(seq 0 2)
+ set +e
+ kubectl delete namespace psmdb-operator --ignore-not-found
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.k5qsN8hdkC
+ cat /tmp/tmp.H77kFqzE3b
+ rm /tmp/tmp.k5qsN8hdkC /tmp/tmp.H77kFqzE3b
+ return 0
namespace "pitr-sharded-9526" deleted
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.lSu0ujr81l
namespace "psmdb-operator" deleted
+ cat /tmp/tmp.yhLvFLCNGG
+ rm /tmp/tmp.lSu0ujr81l /tmp/tmp.yhLvFLCNGG
+ return 0
+ kubectl_bin wait --for=delete namespace psmdb-operator
++ mktemp
+ local LAST_OUT=/tmp/tmp.01pAWBum3l
++ mktemp
+ local LAST_ERR=/tmp/tmp.WzA3LPHAbJ
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl wait --for=delete namespace psmdb-operator
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.01pAWBum3l
+ cat /tmp/tmp.WzA3LPHAbJ
+ rm /tmp/tmp.01pAWBum3l /tmp/tmp.WzA3LPHAbJ
+ return 0
+ desc 'create namespace psmdb-operator'
+ set +o xtrace


-----------------------------------------------------------------------------------
create namespace psmdb-operator
-----------------------------------------------------------------------------------

+ kubectl_bin create namespace psmdb-operator
++ mktemp
+ local LAST_OUT=/tmp/tmp.kaAetm3k1Q
++ mktemp
+ local LAST_ERR=/tmp/tmp.gpey7sGz0r
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl create namespace psmdb-operator
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.kaAetm3k1Q
namespace/psmdb-operator created
+ cat /tmp/tmp.gpey7sGz0r
+ rm /tmp/tmp.kaAetm3k1Q /tmp/tmp.gpey7sGz0r
+ return 0
+ set_kube_ctx psmdb-operator
+ local namespace=psmdb-operator
++ kubectl_bin config current-context
+++ mktemp
++ local LAST_OUT=/tmp/tmp.pCPZZUpsyb
+++ mktemp
++ local LAST_ERR=/tmp/tmp.571r4ABSOw
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl config current-context
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.pCPZZUpsyb
++ cat /tmp/tmp.571r4ABSOw
++ rm /tmp/tmp.pCPZZUpsyb /tmp/tmp.571r4ABSOw
++ return 0
+ kubectl_bin config set-context gke_cloud-dev-112233_us-central1-a_jen-psmdb-2039-368d0b21-10-cluster2 --namespace=psmdb-operator
++ mktemp
+ local LAST_OUT=/tmp/tmp.MNI1l90aOa
++ mktemp
+ local LAST_ERR=/tmp/tmp.Pj5I5oOjiM
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl config set-context gke_cloud-dev-112233_us-central1-a_jen-psmdb-2039-368d0b21-10-cluster2 --namespace=psmdb-operator
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.MNI1l90aOa
Context "gke_cloud-dev-112233_us-central1-a_jen-psmdb-2039-368d0b21-10-cluster2" modified.
+ cat /tmp/tmp.Pj5I5oOjiM
+ rm /tmp/tmp.MNI1l90aOa /tmp/tmp.Pj5I5oOjiM
+ return 0
+ deploy_operator
+ desc 'start PSMDB operator'
+ set +o xtrace


-----------------------------------------------------------------------------------
start PSMDB operator
-----------------------------------------------------------------------------------

+ local cr_file
+ '[' -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2039/e2e-tests/pitr-sharded/conf/crd.yaml ']'
+ cr_file=/mnt/jenkins/workspace/cloud-psmdb-operator_PR-2039/deploy/crd.yaml
+ kubectl_bin apply --server-side --force-conflicts -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2039/deploy/crd.yaml
++ mktemp
+ local LAST_OUT=/tmp/tmp.AoQ3UKOpjm
++ mktemp
+ local LAST_ERR=/tmp/tmp.WEIHtCfLDG
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl apply --server-side --force-conflicts -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2039/deploy/crd.yaml
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.AoQ3UKOpjm
customresourcedefinition.apiextensions.k8s.io/perconaservermongodbbackups.psmdb.percona.com serverside-applied
customresourcedefinition.apiextensions.k8s.io/perconaservermongodbrestores.psmdb.percona.com serverside-applied
customresourcedefinition.apiextensions.k8s.io/perconaservermongodbs.psmdb.percona.com serverside-applied
+ cat /tmp/tmp.WEIHtCfLDG
+ rm /tmp/tmp.AoQ3UKOpjm /tmp/tmp.WEIHtCfLDG
+ return 0
+ '[' -n psmdb-operator ']'
+ apply_rbac cw-rbac
+ local operator_namespace=psmdb-operator
+ local rbac=cw-rbac
+ cat /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2039/deploy/cw-rbac.yaml
+ sed -e 's^namespace: .*^namespace: psmdb-operator^'
+ kubectl_bin apply -n psmdb-operator -f -
++ mktemp
+ local LAST_OUT=/tmp/tmp.IZiS1cqlxA
++ mktemp
+ local LAST_ERR=/tmp/tmp.wh7NSwFWS9
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl apply -n psmdb-operator -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.IZiS1cqlxA
clusterrole.rbac.authorization.k8s.io/percona-server-mongodb-operator created
serviceaccount/percona-server-mongodb-operator created
clusterrolebinding.rbac.authorization.k8s.io/service-account-percona-server-mongodb-operator created
+ cat /tmp/tmp.wh7NSwFWS9
+ rm /tmp/tmp.IZiS1cqlxA /tmp/tmp.wh7NSwFWS9
+ return 0
+ yq eval '
			(.spec.template.spec.containers[].image = "perconalab/percona-server-mongodb-operator:PR-2039-368d0b21") |
			((.. | select(.[] == "DISABLE_TELEMETRY")) |= .value="true") |
			((.. | select(.[] == "LOG_LEVEL")) |= .value="DEBUG")' /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2039/deploy/cw-operator.yaml
+ kubectl_bin apply -f -
++ mktemp
+ local LAST_OUT=/tmp/tmp.kSZt4uiMKu
++ mktemp
+ local LAST_ERR=/tmp/tmp.WIjs6Mh0mJ
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl apply -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.kSZt4uiMKu
deployment.apps/percona-server-mongodb-operator created
+ cat /tmp/tmp.WIjs6Mh0mJ
+ rm /tmp/tmp.kSZt4uiMKu /tmp/tmp.WIjs6Mh0mJ
+ return 0
+ sleep 2
++ get_operator_pod
++ kubectl_bin get pods --selector=name=percona-server-mongodb-operator -o 'jsonpath={.items[].metadata.name}' -n psmdb-operator
+++ mktemp
++ local LAST_OUT=/tmp/tmp.ednVA9XNHm
+++ mktemp
++ local LAST_ERR=/tmp/tmp.02VWIbMA5z
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=percona-server-mongodb-operator -o 'jsonpath={.items[].metadata.name}' -n psmdb-operator
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.ednVA9XNHm
++ cat /tmp/tmp.02VWIbMA5z
++ rm /tmp/tmp.ednVA9XNHm /tmp/tmp.02VWIbMA5z
++ return 0
+ wait_pod percona-server-mongodb-operator-6979d84f9b-tw9tx
+ local pod=percona-server-mongodb-operator-6979d84f9b-tw9tx
+ set +o xtrace
waiting for pod/percona-server-mongodb-operator-6979d84f9b-tw9tx to be ready.OK
+ echo 'Print operator info from log'
Print operator info from log
+ grep 'Manager starting up'
++ get_operator_pod
++ kubectl_bin get pods --selector=name=percona-server-mongodb-operator -o 'jsonpath={.items[].metadata.name}' -n psmdb-operator
+++ mktemp
++ local LAST_OUT=/tmp/tmp.PTGFUlWxsn
+++ mktemp
++ local LAST_ERR=/tmp/tmp.od2yxgigyA
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=percona-server-mongodb-operator -o 'jsonpath={.items[].metadata.name}' -n psmdb-operator
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.PTGFUlWxsn
++ cat /tmp/tmp.od2yxgigyA
++ rm /tmp/tmp.PTGFUlWxsn /tmp/tmp.od2yxgigyA
++ return 0
+ kubectl_bin logs percona-server-mongodb-operator-6979d84f9b-tw9tx
++ mktemp
+ local LAST_OUT=/tmp/tmp.3jGRA5tKG7
++ mktemp
+ local LAST_ERR=/tmp/tmp.dlpw6lYM11
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl logs percona-server-mongodb-operator-6979d84f9b-tw9tx
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.3jGRA5tKG7
+ cat /tmp/tmp.dlpw6lYM11
+ rm /tmp/tmp.3jGRA5tKG7 /tmp/tmp.dlpw6lYM11
+ return 0
2025-09-15T17:37:43.216Z	INFO	setup	Manager starting up	{"gitCommit": "368d0b21092c714d412877d1e9d99df11f58c7cd", "gitBranch": "PR-2039-368d0b21", "buildTime": "", "goVersion": "go1.25.1", "os": "linux", "arch": "amd64"}
+ create_namespace pitr-sharded-3057
+ local namespace=pitr-sharded-3057
+ local skip_clean_namespace=
+ [[ 1 == 1 ]]
+ [[ -z '' ]]
+ destroy_chaos_mesh
++ helm list --all-namespaces --filter chaos-mesh
++ tail -n1
++ awk '-F ' '{print $2}'
++ sed s/NAMESPACE//
+ local chaos_mesh_ns=
+ desc 'destroy chaos-mesh'
+ set +o xtrace


-----------------------------------------------------------------------------------
destroy chaos-mesh
-----------------------------------------------------------------------------------

+ '[' -n '' ']'
++ kubectl get MutatingWebhookConfiguration
++ grep chaos-mesh
++ awk '{print $1}'
+ timeout 30 kubectl delete MutatingWebhookConfiguration
error: resource(s) were provided, but no name was specified
+ :
++ kubectl get ValidatingWebhookConfiguration
++ grep chaos-mesh
++ awk '{print $1}'
+ timeout 30 kubectl delete ValidatingWebhookConfiguration
error: resource(s) were provided, but no name was specified
+ :
++ kubectl get ValidatingWebhookConfiguration
++ grep validate-auth
++ awk '{print $1}'
+ timeout 30 kubectl delete ValidatingWebhookConfiguration
error: resource(s) were provided, but no name was specified
+ :
++ kubectl api-resources
++ grep chaos-mesh
++ awk '{print $1}'
++ kubectl get crd
++ grep chaos-mesh.org
++ awk '{print $1}'
+ timeout 30 kubectl delete crd
error: resource(s) were provided, but no name was specified
+ :
++ kubectl get clusterrolebinding
++ grep chaos-mesh
++ awk '{print $1}'
+ timeout 30 kubectl delete clusterrolebinding
error: resource(s) were provided, but no name was specified
+ :
++ kubectl get clusterrole
++ grep chaos-mesh
++ awk '{print $1}'
+ timeout 30 kubectl delete clusterrole
error: resource(s) were provided, but no name was specified
+ :
+ desc 'cleaned up all old namespaces'
+ set +o xtrace


-----------------------------------------------------------------------------------
cleaned up all old namespaces
-----------------------------------------------------------------------------------

+ kubectl_bin get ns
+ egrep -v '^kube-|^default|Terminating|psmdb-operator|openshift|^gke-|^gmp-|^NAME'
+ '[' -n '' ']'
+ desc 'cleaned up old namespaces pitr-sharded-3057'
+ set +o xtrace


-----------------------------------------------------------------------------------
cleaned up old namespaces pitr-sharded-3057
-----------------------------------------------------------------------------------
+ awk '{print$1}'

+ kubectl_bin delete namespace pitr-sharded-3057 --ignore-not-found
+ xargs kubectl delete ns
++ mktemp
++ mktemp
egrep: warning: egrep is obsolescent; using grep -E
+ local LAST_OUT=/tmp/tmp.nYJOTupICH
++ mktemp
+ local LAST_OUT=/tmp/tmp.neMlrKGFwv
++ mktemp
+ local LAST_ERR=/tmp/tmp.Kgq3UMPEPc
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ local LAST_ERR=/tmp/tmp.7B973PURfF
+ local exit_status=0
+ local timeout=4
+ for i in $(seq 0 2)
+ set +e
+ kubectl get ns
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl delete namespace pitr-sharded-3057 --ignore-not-found
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.nYJOTupICH
+ cat /tmp/tmp.Kgq3UMPEPc
+ rm /tmp/tmp.nYJOTupICH /tmp/tmp.Kgq3UMPEPc
+ return 0
error: resource(s) were provided, but no name was specified
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.neMlrKGFwv
+ cat /tmp/tmp.7B973PURfF
+ rm /tmp/tmp.neMlrKGFwv /tmp/tmp.7B973PURfF
+ return 0
+ kubectl_bin wait --for=delete namespace pitr-sharded-3057
++ mktemp
+ local LAST_OUT=/tmp/tmp.19zP8fTWwq
++ mktemp
+ local LAST_ERR=/tmp/tmp.DW0CocrxFH
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl wait --for=delete namespace pitr-sharded-3057
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.19zP8fTWwq
+ cat /tmp/tmp.DW0CocrxFH
+ rm /tmp/tmp.19zP8fTWwq /tmp/tmp.DW0CocrxFH
+ return 0
+ desc 'create namespace pitr-sharded-3057'
+ set +o xtrace


-----------------------------------------------------------------------------------
create namespace pitr-sharded-3057
-----------------------------------------------------------------------------------

+ kubectl_bin create namespace pitr-sharded-3057
++ mktemp
+ local LAST_OUT=/tmp/tmp.VjhmgdKAI8
++ mktemp
+ local LAST_ERR=/tmp/tmp.lQmuHZbLFh
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl create namespace pitr-sharded-3057
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.VjhmgdKAI8
namespace/pitr-sharded-3057 created
+ cat /tmp/tmp.lQmuHZbLFh
+ rm /tmp/tmp.VjhmgdKAI8 /tmp/tmp.lQmuHZbLFh
+ return 0
+ set_kube_ctx pitr-sharded-3057
+ local namespace=pitr-sharded-3057
++ kubectl_bin config current-context
+++ mktemp
++ local LAST_OUT=/tmp/tmp.1awplWvSiD
+++ mktemp
++ local LAST_ERR=/tmp/tmp.EKPXqOqUQT
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl config current-context
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.1awplWvSiD
++ cat /tmp/tmp.EKPXqOqUQT
++ rm /tmp/tmp.1awplWvSiD /tmp/tmp.EKPXqOqUQT
++ return 0
+ kubectl_bin config set-context gke_cloud-dev-112233_us-central1-a_jen-psmdb-2039-368d0b21-10-cluster2 --namespace=pitr-sharded-3057
++ mktemp
+ local LAST_OUT=/tmp/tmp.bhArBenDsI
++ mktemp
+ local LAST_ERR=/tmp/tmp.BKKF0jUYVF
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl config set-context gke_cloud-dev-112233_us-central1-a_jen-psmdb-2039-368d0b21-10-cluster2 --namespace=pitr-sharded-3057
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.bhArBenDsI
Context "gke_cloud-dev-112233_us-central1-a_jen-psmdb-2039-368d0b21-10-cluster2" modified.
+ cat /tmp/tmp.BKKF0jUYVF
+ rm /tmp/tmp.bhArBenDsI /tmp/tmp.BKKF0jUYVF
+ return 0
+ deploy_minio
+ desc 'install Minio'
+ set +o xtrace


-----------------------------------------------------------------------------------
install Minio
-----------------------------------------------------------------------------------

+ helm uninstall minio-service
Error: uninstall: Release not loaded: minio-service: release: not found
+ :
+ helm repo remove minio
"minio" has been removed from your repositories
+ helm repo add minio https://charts.min.io/
"minio" has been added to your repositories
+ retry 10 60 helm install minio-service --version 5.4.0 --set replicas=1 --set mode=standalone --set resources.requests.memory=256Mi --set rootUser=rootuser --set rootPassword=rootpass123 --set 'users[0].accessKey=some-access-key' --set 'users[0].secretKey=some-secret-key' --set 'users[0].policy=consoleAdmin' --set service.type=ClusterIP --set configPathmc=/tmp/.minio/ --set persistence.size=2G --set securityContext.enabled=false minio/minio
+ local max=10
+ local delay=60
+ shift 2
+ local n=1
+ helm install minio-service --version 5.4.0 --set replicas=1 --set mode=standalone --set resources.requests.memory=256Mi --set rootUser=rootuser --set rootPassword=rootpass123 --set 'users[0].accessKey=some-access-key' --set 'users[0].secretKey=some-secret-key' --set 'users[0].policy=consoleAdmin' --set service.type=ClusterIP --set configPathmc=/tmp/.minio/ --set persistence.size=2G --set securityContext.enabled=false minio/minio
NAME: minio-service
LAST DEPLOYED: Mon Sep 15 17:38:01 2025
NAMESPACE: pitr-sharded-3057
STATUS: deployed
REVISION: 1
TEST SUITE: None
NOTES:
MinIO can be accessed via port 9000 on the following DNS name from within your cluster:
minio-service.pitr-sharded-3057.cluster.local

To access MinIO from localhost, run the below commands:

  1. export POD_NAME=$(kubectl get pods --namespace pitr-sharded-3057 -l "release=minio-service" -o jsonpath="{.items[0].metadata.name}")

  2. kubectl port-forward $POD_NAME 9000 --namespace pitr-sharded-3057

Read more about port forwarding here: http://kubernetes.io/docs/user-guide/kubectl/kubectl_port-forward/

You can now access MinIO server on http://localhost:9000. Follow the below steps to connect to MinIO server with mc client:

  1. Download the MinIO mc client - https://min.io/docs/minio/linux/reference/minio-mc.html#quickstart

  2. export MC_HOST_minio-service-local=http://$(kubectl get secret --namespace pitr-sharded-3057 minio-service -o jsonpath="{.data.rootUser}" | base64 --decode):$(kubectl get secret --namespace pitr-sharded-3057 minio-service -o jsonpath="{.data.rootPassword}" | base64 --decode)@localhost:9000

  3. mc ls minio-service-local
++ kubectl_bin get pods --selector=release=minio-service -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.E9UrisWDg8
+++ mktemp
++ local LAST_ERR=/tmp/tmp.jm7kpd4aBf
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=release=minio-service -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.E9UrisWDg8
++ cat /tmp/tmp.jm7kpd4aBf
++ rm /tmp/tmp.E9UrisWDg8 /tmp/tmp.jm7kpd4aBf
++ return 0
+ MINIO_POD=minio-service-86dfccd949-jhqxr
+ wait_pod minio-service-86dfccd949-jhqxr
+ local pod=minio-service-86dfccd949-jhqxr
+ set +o xtrace
waiting for pod/minio-service-86dfccd949-jhqxr to be ready.OK
+ '[' -n psmdb-operator ']'
+ kubectl_bin create svc -n psmdb-operator externalname minio-service --external-name=minio-service.pitr-sharded-3057.svc.cluster.local --tcp=9000
++ mktemp
+ local LAST_OUT=/tmp/tmp.rJ5cSdTqzd
++ mktemp
+ local LAST_ERR=/tmp/tmp.a4crFrZDQw
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl create svc -n psmdb-operator externalname minio-service --external-name=minio-service.pitr-sharded-3057.svc.cluster.local --tcp=9000
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.rJ5cSdTqzd
service/minio-service created
+ cat /tmp/tmp.a4crFrZDQw
+ rm /tmp/tmp.rJ5cSdTqzd /tmp/tmp.a4crFrZDQw
+ return 0
+ create_minio_bucket operator-testing
+ local bucket=operator-testing
+ kubectl_bin run -i --rm aws-cli --image=perconalab/awscli --restart=Never -- bash -c 'AWS_ACCESS_KEY_ID=some-access-key 		AWS_SECRET_ACCESS_KEY=some-secret-key 		AWS_DEFAULT_REGION=us-east-1         /usr/bin/aws --endpoint-url http://minio-service:9000 s3 mb s3://operator-testing'
++ mktemp
+ local LAST_OUT=/tmp/tmp.hs2dklHsGQ
++ mktemp
+ local LAST_ERR=/tmp/tmp.COTckZux8z
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl run -i --rm aws-cli --image=perconalab/awscli --restart=Never -- bash -c 'AWS_ACCESS_KEY_ID=some-access-key 		AWS_SECRET_ACCESS_KEY=some-secret-key 		AWS_DEFAULT_REGION=us-east-1         /usr/bin/aws --endpoint-url http://minio-service:9000 s3 mb s3://operator-testing'
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.hs2dklHsGQ
make_bucket: operator-testing
pod "aws-cli" deleted from pitr-sharded-3057 namespace
+ cat /tmp/tmp.COTckZux8z
All commands and output from this session will be recorded in container logs, including credentials and sensitive information passed through the command prompt.
If you don't see a command prompt, try pressing enter.
warning: couldn't attach to pod/aws-cli, falling back to streaming logs: Internal error occurred: Internal error occurred: error attaching to container: failed to load task: no running task found: task e9efb2384a736cb36507af2748dd43f9049820573343f005d1dd328cb24f395f not found: not found
+ rm /tmp/tmp.hs2dklHsGQ /tmp/tmp.COTckZux8z
+ return 0
+ desc 'create secrets and start client'
+ set +o xtrace


-----------------------------------------------------------------------------------
create secrets and start client
-----------------------------------------------------------------------------------

+ kubectl_bin apply -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2039/e2e-tests/conf/secrets.yml -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2039/e2e-tests/conf/client.yml
++ mktemp
+ local LAST_OUT=/tmp/tmp.dolaBnRc4T
++ mktemp
+ local LAST_ERR=/tmp/tmp.Fhuv7kcOcz
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl apply -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2039/e2e-tests/conf/secrets.yml -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2039/e2e-tests/conf/client.yml
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.dolaBnRc4T
secret/some-users created
deployment.apps/psmdb-client created
+ cat /tmp/tmp.Fhuv7kcOcz
+ rm /tmp/tmp.dolaBnRc4T /tmp/tmp.Fhuv7kcOcz
+ return 0
+ apply_s3_storage_secrets
+ desc 'create secrets for cloud storages'
+ set +o xtrace


-----------------------------------------------------------------------------------
create secrets for cloud storages
-----------------------------------------------------------------------------------

+ '[' -z '' ']'
+ kubectl_bin apply -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2039/e2e-tests/conf/minio-secret.yml -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2039/e2e-tests/conf/cloud-secret.yml
++ mktemp
+ local LAST_OUT=/tmp/tmp.RIzMRjWgvg
++ mktemp
+ local LAST_ERR=/tmp/tmp.KoFq1zrhMH
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl apply -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2039/e2e-tests/conf/minio-secret.yml -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2039/e2e-tests/conf/cloud-secret.yml
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.RIzMRjWgvg
secret/minio-secret created
secret/aws-s3-secret created
secret/gcp-cs-secret created
secret/azure-secret created
+ cat /tmp/tmp.KoFq1zrhMH
+ rm /tmp/tmp.RIzMRjWgvg /tmp/tmp.KoFq1zrhMH
+ return 0
+ desc 'create custom RuntimeClass'
+ set +o xtrace


-----------------------------------------------------------------------------------
create custom RuntimeClass
-----------------------------------------------------------------------------------

+ version_gt 1.19
++ echo '1.30 >= 1.19'
++ bc -l
+ '[' 1 -eq 1 ']'
+ return 0
+ '[' 0 -ne 1 ']'
+ cat /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2039/e2e-tests/conf/container-rc.yaml
+ /usr/sbin/sed s/docker/runc/g
+ kubectl_bin apply -f -
++ mktemp
+ local LAST_OUT=/tmp/tmp.mxYhmKJZDg
++ mktemp
+ local LAST_ERR=/tmp/tmp.gLNltTSwAS
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl apply -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.mxYhmKJZDg
runtimeclass.node.k8s.io/container-rc unchanged
+ cat /tmp/tmp.gLNltTSwAS
+ rm /tmp/tmp.mxYhmKJZDg /tmp/tmp.gLNltTSwAS
+ return 0
+ cluster=some-name
+ desc 'create first PSMDB cluster some-name'
+ set +o xtrace


-----------------------------------------------------------------------------------
create first PSMDB cluster some-name
-----------------------------------------------------------------------------------

+ '[' 0 -eq 0 ']'
+ apply_cluster /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2039/e2e-tests/pitr-sharded/conf/some-name-rs0.yml
+ '[' -z '' ']'
+ cat_config /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2039/e2e-tests/pitr-sharded/conf/some-name-rs0.yml
+ kubectl_bin apply -f -
+ cat /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2039/e2e-tests/pitr-sharded/conf/some-name-rs0.yml
+ yq eval '(.spec | select(.image == null)).image = "perconalab/percona-server-mongodb-operator:main-mongod7.0"'
++ mktemp
+ yq eval '(.spec | select(has("pmm"))).pmm.image = "percona/pmm-client:2.44.1-1"'
+ yq eval '(.spec | select(has("initImage"))).initImage = "perconalab/percona-server-mongodb-operator:PR-2039-368d0b21"'
+ yq eval '(.spec | select(has("backup"))).backup.image = "perconalab/percona-server-mongodb-operator:main-backup"'
+ yq eval '.spec.upgradeOptions.apply="Never"'
+ local LAST_OUT=/tmp/tmp.EujywsNXsQ
++ mktemp
+ local LAST_ERR=/tmp/tmp.tldS9OpXIK
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl apply -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.EujywsNXsQ
perconaservermongodb.psmdb.percona.com/some-name created
+ cat /tmp/tmp.tldS9OpXIK
+ rm /tmp/tmp.EujywsNXsQ /tmp/tmp.tldS9OpXIK
+ return 0
+ desc 'check if all 3 Pods started'
+ set +o xtrace


-----------------------------------------------------------------------------------
check if all 3 Pods started
-----------------------------------------------------------------------------------

+ wait_for_running some-name-rs0 3
+ local name=some-name-rs0
+ let last_pod=2
+ local check_cluster_readyness=true
+ set_debug
+ [[ 1 == 1 ]]
+ set -o xtrace
+ local rs_name=rs0
+ local cluster_name=some-name
++ seq 0 2
+ for i in $(seq 0 $last_pod)
+ [[ 0 -eq 2 ]]
+ wait_pod some-name-rs0-0
+ local pod=some-name-rs0-0
+ set +o xtrace
waiting for pod/some-name-rs0-0 to be ready......OK
+ for i in $(seq 0 $last_pod)
+ [[ 1 -eq 2 ]]
+ wait_pod some-name-rs0-1
+ local pod=some-name-rs0-1
+ set +o xtrace
waiting for pod/some-name-rs0-1 to be ready.......OK
+ for i in $(seq 0 $last_pod)
+ [[ 2 -eq 2 ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].arbiter.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.FQyppbfypb
+++ mktemp
++ local LAST_ERR=/tmp/tmp.aTHgcUBfti
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].arbiter.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.FQyppbfypb
++ cat /tmp/tmp.aTHgcUBfti
++ rm /tmp/tmp.FQyppbfypb /tmp/tmp.aTHgcUBfti
++ return 0
+ [[ '' == \t\r\u\e ]]
+ wait_pod some-name-rs0-2
+ local pod=some-name-rs0-2
+ set +o xtrace
waiting for pod/some-name-rs0-2 to be ready........OK
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].non_voting.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.Ewo4VRsa8I
+++ mktemp
++ local LAST_ERR=/tmp/tmp.eZWy5ehZYC
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].non_voting.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.Ewo4VRsa8I
++ cat /tmp/tmp.eZWy5ehZYC
++ rm /tmp/tmp.Ewo4VRsa8I /tmp/tmp.eZWy5ehZYC
++ return 0
+ [[ '' == \t\r\u\e ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].hidden.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.TsTsLkRfKz
+++ mktemp
++ local LAST_ERR=/tmp/tmp.npq62fhBz3
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].hidden.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.TsTsLkRfKz
++ cat /tmp/tmp.npq62fhBz3
++ rm /tmp/tmp.TsTsLkRfKz /tmp/tmp.npq62fhBz3
++ return 0
+ [[ '' == \t\r\u\e ]]
+ sleep 10
+ [[ true == \t\r\u\e ]]
+ set +x
Waiting for cluster readyness.........................................
+ wait_for_running some-name-cfg 3 false
+ local name=some-name-cfg
+ let last_pod=2
+ local check_cluster_readyness=false
+ set_debug
+ [[ 1 == 1 ]]
+ set -o xtrace
+ local rs_name=cfg
+ local cluster_name=some-name
++ seq 0 2
+ for i in $(seq 0 $last_pod)
+ [[ 0 -eq 2 ]]
+ wait_pod some-name-cfg-0
+ local pod=some-name-cfg-0
+ set +o xtrace
waiting for pod/some-name-cfg-0 to be ready.OK
+ for i in $(seq 0 $last_pod)
+ [[ 1 -eq 2 ]]
+ wait_pod some-name-cfg-1
+ local pod=some-name-cfg-1
+ set +o xtrace
waiting for pod/some-name-cfg-1 to be ready.OK
+ for i in $(seq 0 $last_pod)
+ [[ 2 -eq 2 ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].arbiter.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.Zzru53Ms30
+++ mktemp
++ local LAST_ERR=/tmp/tmp.yWCA9qoUqg
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].arbiter.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.Zzru53Ms30
++ cat /tmp/tmp.yWCA9qoUqg
++ rm /tmp/tmp.Zzru53Ms30 /tmp/tmp.yWCA9qoUqg
++ return 0
+ [[ '' == \t\r\u\e ]]
+ wait_pod some-name-cfg-2
+ local pod=some-name-cfg-2
+ set +o xtrace
waiting for pod/some-name-cfg-2 to be ready.OK
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].non_voting.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.FcbD41TKNo
+++ mktemp
++ local LAST_ERR=/tmp/tmp.VdempcymOd
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].non_voting.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.FcbD41TKNo
++ cat /tmp/tmp.VdempcymOd
++ rm /tmp/tmp.FcbD41TKNo /tmp/tmp.VdempcymOd
++ return 0
+ [[ '' == \t\r\u\e ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].hidden.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.VuFSMtx39K
+++ mktemp
++ local LAST_ERR=/tmp/tmp.YMpCsf6TQV
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].hidden.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.VuFSMtx39K
++ cat /tmp/tmp.YMpCsf6TQV
++ rm /tmp/tmp.VuFSMtx39K /tmp/tmp.YMpCsf6TQV
++ return 0
+ [[ '' == \t\r\u\e ]]
+ sleep 10
+ [[ false == \t\r\u\e ]]
+ wait_cluster_consistency some-name
+ local cluster_name=some-name
+ local wait_time=32
+ retry=0
+ sleep 7
+ echo -n 'waiting for cluster readyness'
waiting for cluster readyness++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.1sWC9mK10x
+++ mktemp
++ local LAST_ERR=/tmp/tmp.JmRejuWp7x
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.1sWC9mK10x
++ cat /tmp/tmp.JmRejuWp7x
++ rm /tmp/tmp.1sWC9mK10x /tmp/tmp.JmRejuWp7x
++ return 0
+ [[ ready == \r\e\a\d\y ]]
+ echo

+ desc 'check if statefulset created with expected config'
+ set +o xtrace


-----------------------------------------------------------------------------------
check if statefulset created with expected config
-----------------------------------------------------------------------------------

+ compare_kubectl statefulset/some-name-rs0
+ local resource=statefulset/some-name-rs0
+ local postfix=
+ local skip_generation_check=
+ local expected_result=/mnt/jenkins/workspace/cloud-psmdb-operator_PR-2039/e2e-tests/pitr-sharded/compare/statefulset_some-name-rs0.yml
+ local new_result=/tmp/tmp.J3Hag6NYlY/statefulset_some-name-rs0.yml
+ '[' -n '' -a -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2039/e2e-tests/pitr-sharded/compare/statefulset_some-name-rs0-oc.yml ']'
+ kubectl_bin get -o yaml statefulset/some-name-rs0
+ yq eval '
			del(.metadata.ownerReferences[].apiVersion) |
			del(.metadata.managedFields) |
			del(.. | select(has("creationTimestamp")).creationTimestamp) |
			del(.. | select(has("namespace")).namespace) |
			del(.. | select(has("uid")).uid) |
			del(.metadata.resourceVersion) |
			del(.spec.template.spec.containers[].env[] | select(.name == "NAMESPACE")) |
			del(.metadata.selfLink) |
			del(.metadata.annotations."cloud.google.com/neg") |
			del(.metadata.annotations."kubectl.kubernetes.io/last-applied-configuration") |
			del(.. | select(has("image")).image) |
			del(.. | select(has("clusterIP")).clusterIP) |
			del(.. | select(has("clusterIPs")).clusterIPs) |
			del(.. | select(has("dataSource")).dataSource) |
			del(.. | select(has("procMount")).procMount) |
			del(.. | select(has("storageClassName")).storageClassName) |
			del(.. | select(has("finalizers")).finalizers) |
			del(.. | select(has("kubernetes.io/pvc-protection"))."kubernetes.io/pvc-protection") |
			del(.. | select(has("volumeName")).volumeName) |
			del(.. | select(has("volume.beta.kubernetes.io/storage-provisioner"))."volume.beta.kubernetes.io/storage-provisioner") |
			del(.. | select(has("volume.kubernetes.io/storage-provisioner"))."volume.kubernetes.io/storage-provisioner") |
			del(.spec.volumeMode) |
			del(.. | select(has("volume.kubernetes.io/selected-node"))."volume.kubernetes.io/selected-node") |
			del(.. | select(has("percona.com/last-config-hash"))."percona.com/last-config-hash") |
			del(.. | select(has("percona.com/configuration-hash"))."percona.com/configuration-hash") |
			del(.. | select(has("percona.com/ssl-hash"))."percona.com/ssl-hash") |
			del(.. | select(has("percona.com/ssl-internal-hash"))."percona.com/ssl-internal-hash") |
			del(.spec.volumeClaimTemplates[].spec.volumeMode | select(. == "Filesystem")) |
			del(.. | select(has("healthCheckNodePort")).healthCheckNodePort) |
			del(.. | select(has("nodePort")).nodePort) |
			del(.status) |
			(.. | select(tag == "!!str")) |= sub("pitr-sharded-3057", "NAME_SPACE") |
			del(.spec.volumeClaimTemplates[].apiVersion) |
			del(.spec.volumeClaimTemplates[].kind) |
			del(.spec.ipFamilies) |
			del(.spec.ipFamilyPolicy) |
			(.. | select(. == "extensions/v1beta1")) = "apps/v1" |
			(.. | select(. == "batch/v1beta1")) = "batch/v1" ' -
++ mktemp
+ local LAST_OUT=/tmp/tmp.clTLlGDQQG
++ mktemp
+ local LAST_ERR=/tmp/tmp.YwGgQg9EHg
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl get -o yaml statefulset/some-name-rs0
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.clTLlGDQQG
+ cat /tmp/tmp.YwGgQg9EHg
+ rm /tmp/tmp.clTLlGDQQG /tmp/tmp.YwGgQg9EHg
+ return 0
+ yq -i eval 'del(.spec.persistentVolumeClaimRetentionPolicy)' /tmp/tmp.J3Hag6NYlY/statefulset_some-name-rs0.yml
+ version_gt 1.22
++ echo '1.30 >= 1.22'
++ bc -l
+ '[' 1 -eq 1 ']'
+ return 0
+ yq -i eval 'del(.spec.internalTrafficPolicy)' /tmp/tmp.J3Hag6NYlY/statefulset_some-name-rs0.yml
+ yq -i eval 'del(.spec.allocateLoadBalancerNodePorts)' /tmp/tmp.J3Hag6NYlY/statefulset_some-name-rs0.yml
+ [[ /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2039/e2e-tests/pitr-sharded/compare/statefulset_some-name-rs0.yml == */cronjob* ]]
+ '[' -n '' ']'
+ [[ 0 -eq 0 ]]
+ diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2039/e2e-tests/pitr-sharded/compare/statefulset_some-name-rs0.yml /tmp/tmp.J3Hag6NYlY/statefulset_some-name-rs0.yml
+ compare_kubectl statefulset/some-name-rs1
+ local resource=statefulset/some-name-rs1
+ local postfix=
+ local skip_generation_check=
+ local expected_result=/mnt/jenkins/workspace/cloud-psmdb-operator_PR-2039/e2e-tests/pitr-sharded/compare/statefulset_some-name-rs1.yml
+ local new_result=/tmp/tmp.J3Hag6NYlY/statefulset_some-name-rs1.yml
+ '[' -n '' -a -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2039/e2e-tests/pitr-sharded/compare/statefulset_some-name-rs1-oc.yml ']'
+ kubectl_bin get -o yaml statefulset/some-name-rs1
+ yq eval '
			del(.metadata.ownerReferences[].apiVersion) |
			del(.metadata.managedFields) |
			del(.. | select(has("creationTimestamp")).creationTimestamp) |
			del(.. | select(has("namespace")).namespace) |
			del(.. | select(has("uid")).uid) |
			del(.metadata.resourceVersion) |
			del(.spec.template.spec.containers[].env[] | select(.name == "NAMESPACE")) |
			del(.metadata.selfLink) |
			del(.metadata.annotations."cloud.google.com/neg") |
			del(.metadata.annotations."kubectl.kubernetes.io/last-applied-configuration") |
			del(.. | select(has("image")).image) |
			del(.. | select(has("clusterIP")).clusterIP) |
			del(.. | select(has("clusterIPs")).clusterIPs) |
			del(.. | select(has("dataSource")).dataSource) |
			del(.. | select(has("procMount")).procMount) |
			del(.. | select(has("storageClassName")).storageClassName) |
			del(.. | select(has("finalizers")).finalizers) |
			del(.. | select(has("kubernetes.io/pvc-protection"))."kubernetes.io/pvc-protection") |
			del(.. | select(has("volumeName")).volumeName) |
			del(.. | select(has("volume.beta.kubernetes.io/storage-provisioner"))."volume.beta.kubernetes.io/storage-provisioner") |
			del(.. | select(has("volume.kubernetes.io/storage-provisioner"))."volume.kubernetes.io/storage-provisioner") |
			del(.spec.volumeMode) |
			del(.. | select(has("volume.kubernetes.io/selected-node"))."volume.kubernetes.io/selected-node") |
			del(.. | select(has("percona.com/last-config-hash"))."percona.com/last-config-hash") |
			del(.. | select(has("percona.com/configuration-hash"))."percona.com/configuration-hash") |
			del(.. | select(has("percona.com/ssl-hash"))."percona.com/ssl-hash") |
			del(.. | select(has("percona.com/ssl-internal-hash"))."percona.com/ssl-internal-hash") |
			del(.spec.volumeClaimTemplates[].spec.volumeMode | select(. == "Filesystem")) |
			del(.. | select(has("healthCheckNodePort")).healthCheckNodePort) |
			del(.. | select(has("nodePort")).nodePort) |
			del(.status) |
			(.. | select(tag == "!!str")) |= sub("pitr-sharded-3057", "NAME_SPACE") |
			del(.spec.volumeClaimTemplates[].apiVersion) |
			del(.spec.volumeClaimTemplates[].kind) |
			del(.spec.ipFamilies) |
			del(.spec.ipFamilyPolicy) |
			(.. | select(. == "extensions/v1beta1")) = "apps/v1" |
			(.. | select(. == "batch/v1beta1")) = "batch/v1" ' -
++ mktemp
+ local LAST_OUT=/tmp/tmp.YcFnlQofwh
++ mktemp
+ local LAST_ERR=/tmp/tmp.6VDeGJGwq2
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl get -o yaml statefulset/some-name-rs1
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.YcFnlQofwh
+ cat /tmp/tmp.6VDeGJGwq2
+ rm /tmp/tmp.YcFnlQofwh /tmp/tmp.6VDeGJGwq2
+ return 0
+ yq -i eval 'del(.spec.persistentVolumeClaimRetentionPolicy)' /tmp/tmp.J3Hag6NYlY/statefulset_some-name-rs1.yml
+ version_gt 1.22
++ echo '1.30 >= 1.22'
++ bc -l
+ '[' 1 -eq 1 ']'
+ return 0
+ yq -i eval 'del(.spec.internalTrafficPolicy)' /tmp/tmp.J3Hag6NYlY/statefulset_some-name-rs1.yml
+ yq -i eval 'del(.spec.allocateLoadBalancerNodePorts)' /tmp/tmp.J3Hag6NYlY/statefulset_some-name-rs1.yml
+ [[ /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2039/e2e-tests/pitr-sharded/compare/statefulset_some-name-rs1.yml == */cronjob* ]]
+ '[' -n '' ']'
+ [[ 0 -eq 0 ]]
+ diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2039/e2e-tests/pitr-sharded/compare/statefulset_some-name-rs1.yml /tmp/tmp.J3Hag6NYlY/statefulset_some-name-rs1.yml
+ compare_kubectl statefulset/some-name-rs2
+ local resource=statefulset/some-name-rs2
+ local postfix=
+ local skip_generation_check=
+ local expected_result=/mnt/jenkins/workspace/cloud-psmdb-operator_PR-2039/e2e-tests/pitr-sharded/compare/statefulset_some-name-rs2.yml
+ local new_result=/tmp/tmp.J3Hag6NYlY/statefulset_some-name-rs2.yml
+ '[' -n '' -a -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2039/e2e-tests/pitr-sharded/compare/statefulset_some-name-rs2-oc.yml ']'
+ kubectl_bin get -o yaml statefulset/some-name-rs2
+ yq eval '
			del(.metadata.ownerReferences[].apiVersion) |
			del(.metadata.managedFields) |
			del(.. | select(has("creationTimestamp")).creationTimestamp) |
			del(.. | select(has("namespace")).namespace) |
			del(.. | select(has("uid")).uid) |
			del(.metadata.resourceVersion) |
			del(.spec.template.spec.containers[].env[] | select(.name == "NAMESPACE")) |
			del(.metadata.selfLink) |
			del(.metadata.annotations."cloud.google.com/neg") |
			del(.metadata.annotations."kubectl.kubernetes.io/last-applied-configuration") |
			del(.. | select(has("image")).image) |
			del(.. | select(has("clusterIP")).clusterIP) |
			del(.. | select(has("clusterIPs")).clusterIPs) |
			del(.. | select(has("dataSource")).dataSource) |
			del(.. | select(has("procMount")).procMount) |
			del(.. | select(has("storageClassName")).storageClassName) |
			del(.. | select(has("finalizers")).finalizers) |
			del(.. | select(has("kubernetes.io/pvc-protection"))."kubernetes.io/pvc-protection") |
			del(.. | select(has("volumeName")).volumeName) |
			del(.. | select(has("volume.beta.kubernetes.io/storage-provisioner"))."volume.beta.kubernetes.io/storage-provisioner") |
			del(.. | select(has("volume.kubernetes.io/storage-provisioner"))."volume.kubernetes.io/storage-provisioner") |
			del(.spec.volumeMode) |
			del(.. | select(has("volume.kubernetes.io/selected-node"))."volume.kubernetes.io/selected-node") |
			del(.. | select(has("percona.com/last-config-hash"))."percona.com/last-config-hash") |
			del(.. | select(has("percona.com/configuration-hash"))."percona.com/configuration-hash") |
			del(.. | select(has("percona.com/ssl-hash"))."percona.com/ssl-hash") |
			del(.. | select(has("percona.com/ssl-internal-hash"))."percona.com/ssl-internal-hash") |
			del(.spec.volumeClaimTemplates[].spec.volumeMode | select(. == "Filesystem")) |
			del(.. | select(has("healthCheckNodePort")).healthCheckNodePort) |
			del(.. | select(has("nodePort")).nodePort) |
			del(.status) |
			(.. | select(tag == "!!str")) |= sub("pitr-sharded-3057", "NAME_SPACE") |
			del(.spec.volumeClaimTemplates[].apiVersion) |
			del(.spec.volumeClaimTemplates[].kind) |
			del(.spec.ipFamilies) |
			del(.spec.ipFamilyPolicy) |
			(.. | select(. == "extensions/v1beta1")) = "apps/v1" |
			(.. | select(. == "batch/v1beta1")) = "batch/v1" ' -
++ mktemp
+ local LAST_OUT=/tmp/tmp.gdM7QsOeXC
++ mktemp
+ local LAST_ERR=/tmp/tmp.cxpQnWqjvT
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl get -o yaml statefulset/some-name-rs2
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.gdM7QsOeXC
+ cat /tmp/tmp.cxpQnWqjvT
+ rm /tmp/tmp.gdM7QsOeXC /tmp/tmp.cxpQnWqjvT
+ return 0
+ yq -i eval 'del(.spec.persistentVolumeClaimRetentionPolicy)' /tmp/tmp.J3Hag6NYlY/statefulset_some-name-rs2.yml
+ version_gt 1.22
++ echo '1.30 >= 1.22'
++ bc -l
+ '[' 1 -eq 1 ']'
+ return 0
+ yq -i eval 'del(.spec.internalTrafficPolicy)' /tmp/tmp.J3Hag6NYlY/statefulset_some-name-rs2.yml
+ yq -i eval 'del(.spec.allocateLoadBalancerNodePorts)' /tmp/tmp.J3Hag6NYlY/statefulset_some-name-rs2.yml
+ [[ /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2039/e2e-tests/pitr-sharded/compare/statefulset_some-name-rs2.yml == */cronjob* ]]
+ '[' -n '' ']'
+ [[ 0 -eq 0 ]]
+ diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2039/e2e-tests/pitr-sharded/compare/statefulset_some-name-rs2.yml /tmp/tmp.J3Hag6NYlY/statefulset_some-name-rs2.yml
+ compare_kubectl statefulset/some-name-cfg
+ local resource=statefulset/some-name-cfg
+ local postfix=
+ local skip_generation_check=
+ local expected_result=/mnt/jenkins/workspace/cloud-psmdb-operator_PR-2039/e2e-tests/pitr-sharded/compare/statefulset_some-name-cfg.yml
+ local new_result=/tmp/tmp.J3Hag6NYlY/statefulset_some-name-cfg.yml
+ '[' -n '' -a -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2039/e2e-tests/pitr-sharded/compare/statefulset_some-name-cfg-oc.yml ']'
+ kubectl_bin get -o yaml statefulset/some-name-cfg
+ yq eval '
			del(.metadata.ownerReferences[].apiVersion) |
			del(.metadata.managedFields) |
			del(.. | select(has("creationTimestamp")).creationTimestamp) |
			del(.. | select(has("namespace")).namespace) |
			del(.. | select(has("uid")).uid) |
			del(.metadata.resourceVersion) |
			del(.spec.template.spec.containers[].env[] | select(.name == "NAMESPACE")) |
			del(.metadata.selfLink) |
			del(.metadata.annotations."cloud.google.com/neg") |
			del(.metadata.annotations."kubectl.kubernetes.io/last-applied-configuration") |
			del(.. | select(has("image")).image) |
			del(.. | select(has("clusterIP")).clusterIP) |
			del(.. | select(has("clusterIPs")).clusterIPs) |
			del(.. | select(has("dataSource")).dataSource) |
			del(.. | select(has("procMount")).procMount) |
			del(.. | select(has("storageClassName")).storageClassName) |
			del(.. | select(has("finalizers")).finalizers) |
			del(.. | select(has("kubernetes.io/pvc-protection"))."kubernetes.io/pvc-protection") |
			del(.. | select(has("volumeName")).volumeName) |
			del(.. | select(has("volume.beta.kubernetes.io/storage-provisioner"))."volume.beta.kubernetes.io/storage-provisioner") |
			del(.. | select(has("volume.kubernetes.io/storage-provisioner"))."volume.kubernetes.io/storage-provisioner") |
			del(.spec.volumeMode) |
			del(.. | select(has("volume.kubernetes.io/selected-node"))."volume.kubernetes.io/selected-node") |
			del(.. | select(has("percona.com/last-config-hash"))."percona.com/last-config-hash") |
			del(.. | select(has("percona.com/configuration-hash"))."percona.com/configuration-hash") |
			del(.. | select(has("percona.com/ssl-hash"))."percona.com/ssl-hash") |
			del(.. | select(has("percona.com/ssl-internal-hash"))."percona.com/ssl-internal-hash") |
			del(.spec.volumeClaimTemplates[].spec.volumeMode | select(. == "Filesystem")) |
			del(.. | select(has("healthCheckNodePort")).healthCheckNodePort) |
			del(.. | select(has("nodePort")).nodePort) |
			del(.status) |
			(.. | select(tag == "!!str")) |= sub("pitr-sharded-3057", "NAME_SPACE") |
			del(.spec.volumeClaimTemplates[].apiVersion) |
			del(.spec.volumeClaimTemplates[].kind) |
			del(.spec.ipFamilies) |
			del(.spec.ipFamilyPolicy) |
			(.. | select(. == "extensions/v1beta1")) = "apps/v1" |
			(.. | select(. == "batch/v1beta1")) = "batch/v1" ' -
++ mktemp
+ local LAST_OUT=/tmp/tmp.kwfpW4pzGN
++ mktemp
+ local LAST_ERR=/tmp/tmp.ds3wT5RI0t
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl get -o yaml statefulset/some-name-cfg
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.kwfpW4pzGN
+ cat /tmp/tmp.ds3wT5RI0t
+ rm /tmp/tmp.kwfpW4pzGN /tmp/tmp.ds3wT5RI0t
+ return 0
+ yq -i eval 'del(.spec.persistentVolumeClaimRetentionPolicy)' /tmp/tmp.J3Hag6NYlY/statefulset_some-name-cfg.yml
+ version_gt 1.22
++ echo '1.30 >= 1.22'
++ bc -l
+ '[' 1 -eq 1 ']'
+ return 0
+ yq -i eval 'del(.spec.internalTrafficPolicy)' /tmp/tmp.J3Hag6NYlY/statefulset_some-name-cfg.yml
+ yq -i eval 'del(.spec.allocateLoadBalancerNodePorts)' /tmp/tmp.J3Hag6NYlY/statefulset_some-name-cfg.yml
+ [[ /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2039/e2e-tests/pitr-sharded/compare/statefulset_some-name-cfg.yml == */cronjob* ]]
+ '[' -n '' ']'
+ [[ 0 -eq 0 ]]
+ diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2039/e2e-tests/pitr-sharded/compare/statefulset_some-name-cfg.yml /tmp/tmp.J3Hag6NYlY/statefulset_some-name-cfg.yml
+ compare_kubectl statefulset/some-name-mongos ''
+ local resource=statefulset/some-name-mongos
+ local postfix=
+ local skip_generation_check=
+ local expected_result=/mnt/jenkins/workspace/cloud-psmdb-operator_PR-2039/e2e-tests/pitr-sharded/compare/statefulset_some-name-mongos.yml
+ local new_result=/tmp/tmp.J3Hag6NYlY/statefulset_some-name-mongos.yml
+ '[' -n '' -a -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2039/e2e-tests/pitr-sharded/compare/statefulset_some-name-mongos-oc.yml ']'
+ kubectl_bin get -o yaml statefulset/some-name-mongos
+ yq eval '
			del(.metadata.ownerReferences[].apiVersion) |
			del(.metadata.managedFields) |
			del(.. | select(has("creationTimestamp")).creationTimestamp) |
			del(.. | select(has("namespace")).namespace) |
			del(.. | select(has("uid")).uid) |
			del(.metadata.resourceVersion) |
			del(.spec.template.spec.containers[].env[] | select(.name == "NAMESPACE")) |
			del(.metadata.selfLink) |
			del(.metadata.annotations."cloud.google.com/neg") |
			del(.metadata.annotations."kubectl.kubernetes.io/last-applied-configuration") |
			del(.. | select(has("image")).image) |
			del(.. | select(has("clusterIP")).clusterIP) |
			del(.. | select(has("clusterIPs")).clusterIPs) |
			del(.. | select(has("dataSource")).dataSource) |
			del(.. | select(has("procMount")).procMount) |
			del(.. | select(has("storageClassName")).storageClassName) |
			del(.. | select(has("finalizers")).finalizers) |
			del(.. | select(has("kubernetes.io/pvc-protection"))."kubernetes.io/pvc-protection") |
			del(.. | select(has("volumeName")).volumeName) |
			del(.. | select(has("volume.beta.kubernetes.io/storage-provisioner"))."volume.beta.kubernetes.io/storage-provisioner") |
			del(.. | select(has("volume.kubernetes.io/storage-provisioner"))."volume.kubernetes.io/storage-provisioner") |
			del(.spec.volumeMode) |
			del(.. | select(has("volume.kubernetes.io/selected-node"))."volume.kubernetes.io/selected-node") |
			del(.. | select(has("percona.com/last-config-hash"))."percona.com/last-config-hash") |
			del(.. | select(has("percona.com/configuration-hash"))."percona.com/configuration-hash") |
			del(.. | select(has("percona.com/ssl-hash"))."percona.com/ssl-hash") |
			del(.. | select(has("percona.com/ssl-internal-hash"))."percona.com/ssl-internal-hash") |
			del(.spec.volumeClaimTemplates[].spec.volumeMode | select(. == "Filesystem")) |
			del(.. | select(has("healthCheckNodePort")).healthCheckNodePort) |
			del(.. | select(has("nodePort")).nodePort) |
			del(.status) |
			(.. | select(tag == "!!str")) |= sub("pitr-sharded-3057", "NAME_SPACE") |
			del(.spec.volumeClaimTemplates[].apiVersion) |
			del(.spec.volumeClaimTemplates[].kind) |
			del(.spec.ipFamilies) |
			del(.spec.ipFamilyPolicy) |
			(.. | select(. == "extensions/v1beta1")) = "apps/v1" |
			(.. | select(. == "batch/v1beta1")) = "batch/v1" ' -
++ mktemp
+ local LAST_OUT=/tmp/tmp.TbxnsVOh1L
++ mktemp
+ local LAST_ERR=/tmp/tmp.oEbZ9oYNce
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl get -o yaml statefulset/some-name-mongos
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.TbxnsVOh1L
+ cat /tmp/tmp.oEbZ9oYNce
+ rm /tmp/tmp.TbxnsVOh1L /tmp/tmp.oEbZ9oYNce
+ return 0
+ yq -i eval 'del(.spec.persistentVolumeClaimRetentionPolicy)' /tmp/tmp.J3Hag6NYlY/statefulset_some-name-mongos.yml
+ version_gt 1.22
++ echo '1.30 >= 1.22'
++ bc -l
+ '[' 1 -eq 1 ']'
+ return 0
+ yq -i eval 'del(.spec.internalTrafficPolicy)' /tmp/tmp.J3Hag6NYlY/statefulset_some-name-mongos.yml
+ yq -i eval 'del(.spec.allocateLoadBalancerNodePorts)' /tmp/tmp.J3Hag6NYlY/statefulset_some-name-mongos.yml
+ [[ /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2039/e2e-tests/pitr-sharded/compare/statefulset_some-name-mongos.yml == */cronjob* ]]
+ '[' -n '' ']'
+ [[ 0 -eq 0 ]]
+ diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2039/e2e-tests/pitr-sharded/compare/statefulset_some-name-mongos.yml /tmp/tmp.J3Hag6NYlY/statefulset_some-name-mongos.yml
+ write_initial_data
+ desc 'create user myApp'
+ set +o xtrace


-----------------------------------------------------------------------------------
create user myApp
-----------------------------------------------------------------------------------

+ run_mongos 'db.createUser({user:"myApp",pwd:"myPass",roles:[{db:"myApp",role:"readWrite"}]})' userAdmin:userAdmin123456@some-name-mongos.pitr-sharded-3057
+ local 'command=db.createUser({user:"myApp",pwd:"myPass",roles:[{db:"myApp",role:"readWrite"}]})'
+ local uri=userAdmin:userAdmin123456@some-name-mongos.pitr-sharded-3057
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local port=27017
+ local mongo_bin=mongo
++ echo .svc.cluster.local
++ awk -F: '{print $2}'
+ suffix_port=
+ [[ -z '' ]]
+ suffix=.svc.cluster.local:27017
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.RLLRk2qpj6
+++ mktemp
++ local LAST_ERR=/tmp/tmp.cCGpQRqgBw
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.RLLRk2qpj6
++ cat /tmp/tmp.cCGpQRqgBw
++ rm /tmp/tmp.RLLRk2qpj6 /tmp/tmp.cCGpQRqgBw
++ return 0
+ local client_container=psmdb-client-66f577db5f-ckxq9
+ kubectl_bin exec psmdb-client-66f577db5f-ckxq9 -- bash -c 'printf '\''db.createUser({user:"myApp",pwd:"myPass",roles:[{db:"myApp",role:"readWrite"}]})\n'\'' | mongo mongodb://userAdmin:userAdmin123456@some-name-mongos.pitr-sharded-3057.svc.cluster.local:27017/admin '
++ mktemp
+ local LAST_OUT=/tmp/tmp.JiP20aavFB
++ mktemp
+ local LAST_ERR=/tmp/tmp.WW3Z7YM6cd
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-66f577db5f-ckxq9 -- bash -c 'printf '\''db.createUser({user:"myApp",pwd:"myPass",roles:[{db:"myApp",role:"readWrite"}]})\n'\'' | mongo mongodb://userAdmin:userAdmin123456@some-name-mongos.pitr-sharded-3057.svc.cluster.local:27017/admin '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.JiP20aavFB
Percona Server for MongoDB shell version v4.4.29-28
connecting to: mongodb://some-name-mongos.pitr-sharded-3057.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb
Implicit session: session { "id" : UUID("df8303ee-2aa3-42a3-9c0d-8b81212dcba2") }
Percona Server for MongoDB server version: v7.0.24-13
WARNING: shell and server versions do not match
Successfully added user: {
	"user" : "myApp",
	"roles" : [
		{
			"db" : "myApp",
			"role" : "readWrite"
		}
	]
}
bye
+ cat /tmp/tmp.WW3Z7YM6cd
+ rm /tmp/tmp.JiP20aavFB /tmp/tmp.WW3Z7YM6cd
+ return 0
+ sleep 2
+ write_document
+ local cmp_postfix=
+ local sleep_value=0
+ log 'write initial data, read from all'
+ set +o xtrace
[2025-09-15T17:42:24+0000] write initial data, read from all
+ run_mongos 'use myApp\n db.test.insert({ x: 100500 })' myApp:myPass@some-name-mongos.pitr-sharded-3057
+ local 'command=use myApp\n db.test.insert({ x: 100500 })'
+ local uri=myApp:myPass@some-name-mongos.pitr-sharded-3057
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local port=27017
+ local mongo_bin=mongo
++ echo .svc.cluster.local
++ awk -F: '{print $2}'
+ suffix_port=
+ [[ -z '' ]]
+ suffix=.svc.cluster.local:27017
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.BKiFqWmUk5
+++ mktemp
++ local LAST_ERR=/tmp/tmp.PnCL1WZcc9
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.BKiFqWmUk5
++ cat /tmp/tmp.PnCL1WZcc9
++ rm /tmp/tmp.BKiFqWmUk5 /tmp/tmp.PnCL1WZcc9
++ return 0
+ local client_container=psmdb-client-66f577db5f-ckxq9
+ kubectl_bin exec psmdb-client-66f577db5f-ckxq9 -- bash -c 'printf '\''use myApp\n db.test.insert({ x: 100500 })\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.pitr-sharded-3057.svc.cluster.local:27017/admin '
++ mktemp
+ local LAST_OUT=/tmp/tmp.f65zXEI7BJ
++ mktemp
+ local LAST_ERR=/tmp/tmp.XR7CCw3Afn
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-66f577db5f-ckxq9 -- bash -c 'printf '\''use myApp\n db.test.insert({ x: 100500 })\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.pitr-sharded-3057.svc.cluster.local:27017/admin '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.f65zXEI7BJ
Percona Server for MongoDB shell version v4.4.29-28
connecting to: mongodb://some-name-mongos.pitr-sharded-3057.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb
Implicit session: session { "id" : UUID("834dceb5-768a-4e38-a5c1-0acebbf9ffe0") }
Percona Server for MongoDB server version: v7.0.24-13
WARNING: shell and server versions do not match
switched to db myApp
WriteResult({ "nInserted" : 1 })
bye
+ cat /tmp/tmp.XR7CCw3Afn
+ rm /tmp/tmp.f65zXEI7BJ /tmp/tmp.XR7CCw3Afn
+ return 0
+ sleep 0
+ compare_mongos_cmd find myApp:myPass@some-name-mongos.pitr-sharded-3057
+ local command=find
+ local uri=myApp:myPass@some-name-mongos.pitr-sharded-3057
+ local postfix=
+ local suffix=
+ local database=myApp
+ local collection=test
+ local port=27017
+ local tls=false
+ [[ false == \t\r\u\e ]]
+ mongos_command=run_mongos
+ run_mongos 'use myApp\n db.test.find()' myApp:myPass@some-name-mongos.pitr-sharded-3057 mongodb '' '' 27017
+ local 'command=use myApp\n db.test.find()'
+ local uri=myApp:myPass@some-name-mongos.pitr-sharded-3057
+ egrep -v 'I NETWORK|W NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|Started a new thread for the timer service'
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local port=27017
+ local mongo_bin=mongo
+ /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
++ echo .svc.cluster.local
++ awk -F: '{print $2}'
egrep: warning: egrep is obsolescent; using grep -E
+ suffix_port=
+ [[ -z '' ]]
+ suffix=.svc.cluster.local:27017
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.yZV7IxSj1G
+++ mktemp
++ local LAST_ERR=/tmp/tmp.g0CyoJUuy6
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.yZV7IxSj1G
++ cat /tmp/tmp.g0CyoJUuy6
++ rm /tmp/tmp.yZV7IxSj1G /tmp/tmp.g0CyoJUuy6
++ return 0
+ local client_container=psmdb-client-66f577db5f-ckxq9
+ kubectl_bin exec psmdb-client-66f577db5f-ckxq9 -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.pitr-sharded-3057.svc.cluster.local:27017/admin '
++ mktemp
+ local LAST_OUT=/tmp/tmp.rxLRj0v9LA
++ mktemp
+ local LAST_ERR=/tmp/tmp.Wqw1BFX8Ab
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-66f577db5f-ckxq9 -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.pitr-sharded-3057.svc.cluster.local:27017/admin '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.rxLRj0v9LA
+ cat /tmp/tmp.Wqw1BFX8Ab
+ rm /tmp/tmp.rxLRj0v9LA /tmp/tmp.Wqw1BFX8Ab
+ return 0
+ [[ 0 -eq 0 ]]
+ diff /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2039/e2e-tests/pitr-sharded/compare/find.json /tmp/tmp.J3Hag6NYlY/find
+ wait_backup_agent some-name-rs0-0
+ local agent_pod=some-name-rs0-0
+ set +o xtrace
waiting for pbm-agent to be ready in some-name-rs0-0...2025-09-15T17:41:52.000+0000 I listening for the commands

+ wait_backup_agent some-name-rs0-1
+ local agent_pod=some-name-rs0-1
+ set +o xtrace
waiting for pbm-agent to be ready in some-name-rs0-1...2025-09-15T17:41:54.000+0000 I listening for the commands

+ wait_backup_agent some-name-rs0-2
+ local agent_pod=some-name-rs0-2
+ set +o xtrace
waiting for pbm-agent to be ready in some-name-rs0-2...2025-09-15T17:41:54.000+0000 I listening for the commands

+ wait_backup_agent some-name-rs1-0
+ local agent_pod=some-name-rs1-0
+ set +o xtrace
waiting for pbm-agent to be ready in some-name-rs1-0...2025-09-15T17:41:53.000+0000 I listening for the commands

+ wait_backup_agent some-name-rs1-1
+ local agent_pod=some-name-rs1-1
+ set +o xtrace
waiting for pbm-agent to be ready in some-name-rs1-1...2025-09-15T17:41:55.000+0000 I listening for the commands

+ wait_backup_agent some-name-rs1-2
+ local agent_pod=some-name-rs1-2
+ set +o xtrace
waiting for pbm-agent to be ready in some-name-rs1-2...2025-09-15T17:41:55.000+0000 I listening for the commands

+ wait_backup_agent some-name-rs2-0
+ local agent_pod=some-name-rs2-0
+ set +o xtrace
waiting for pbm-agent to be ready in some-name-rs2-0...2025-09-15T17:41:55.000+0000 I listening for the commands

+ wait_backup_agent some-name-rs2-1
+ local agent_pod=some-name-rs2-1
+ set +o xtrace
waiting for pbm-agent to be ready in some-name-rs2-1...2025-09-15T17:41:55.000+0000 I listening for the commands

+ wait_backup_agent some-name-rs2-2
+ local agent_pod=some-name-rs2-2
+ set +o xtrace
waiting for pbm-agent to be ready in some-name-rs2-2...2025-09-15T17:41:56.000+0000 I listening for the commands

+ local backup_name
+ '[' 0 -eq 0 ']'
+ backup_name=backup-minio
+ run_backup backup-minio 0
+ local name=backup-minio
+ local idx=0
+ desc 'run backup backup-minio-0'
+ set +o xtrace


-----------------------------------------------------------------------------------
run backup backup-minio-0
-----------------------------------------------------------------------------------

+ cat /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2039/e2e-tests/pitr-sharded/conf/backup-minio.yml
+ /usr/sbin/sed -e 's/name:/name: backup-minio-0/'
+ kubectl_bin apply -f -
++ mktemp
+ local LAST_OUT=/tmp/tmp.gtz8bonwiN
++ mktemp
+ local LAST_ERR=/tmp/tmp.JNPEaM453k
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl apply -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.gtz8bonwiN
perconaservermongodbbackup.psmdb.percona.com/backup-minio-0 created
+ cat /tmp/tmp.JNPEaM453k
+ rm /tmp/tmp.gtz8bonwiN /tmp/tmp.JNPEaM453k
+ return 0
+ wait_backup backup-minio-0
+ local backup_name=backup-minio-0
+ local target_state=ready
+ set +o xtrace
waiting for backup-minio-0 to reach ready state.............
+ sleep 5
+ compare_latest_restorable_time some-name-rs0 backup-minio-0
+ local cluster=some-name-rs0
+ local backup_name=backup-minio-0
+ local latest_restorable_time
+ local backup_time
++ get_latest_restorable_time some-name-rs0
++ local cluster=some-name-rs0
++ local first_timestamp
++ local second_timestamp
++ local retry=0
++ [[ '' != '' ]]
+++ kubectl_bin exec some-name-rs0-0 -c backup-agent -- pbm status -o json
++++ mktemp
+++ jq '.backups.pitrChunks.pitrChunks | last | .range.end'
+++ local LAST_OUT=/tmp/tmp.2s5EdAm5vO
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.BxDBXheMUY
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in $(seq 0 2)
+++ set +e
+++ kubectl exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.2s5EdAm5vO
+++ cat /tmp/tmp.BxDBXheMUY
+++ rm /tmp/tmp.2s5EdAm5vO /tmp/tmp.BxDBXheMUY
+++ return 0
++ first_timestamp=null
++ sleep 5
++ [[ null != '' ]]
++ [[ null != \n\u\l\l ]]
++ let retry+=1
++ [[ 1 -gt 30 ]]
++ [[ null != '' ]]
++ [[ null != \n\u\l\l ]]
+++ kubectl_bin exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ jq '.backups.pitrChunks.pitrChunks | last | .range.end'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.9YfYUUULVh
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.KG2WBiqNno
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in $(seq 0 2)
+++ set +e
+++ kubectl exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.9YfYUUULVh
+++ cat /tmp/tmp.KG2WBiqNno
+++ rm /tmp/tmp.9YfYUUULVh /tmp/tmp.KG2WBiqNno
+++ return 0
++ first_timestamp=null
++ sleep 5
++ [[ null != '' ]]
++ [[ null != \n\u\l\l ]]
++ let retry+=1
++ [[ 2 -gt 30 ]]
++ [[ null != '' ]]
++ [[ null != \n\u\l\l ]]
+++ kubectl_bin exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ jq '.backups.pitrChunks.pitrChunks | last | .range.end'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.XUPiYDgI9f
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.3OyyeRhBO1
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in $(seq 0 2)
+++ set +e
+++ kubectl exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.XUPiYDgI9f
+++ cat /tmp/tmp.3OyyeRhBO1
+++ rm /tmp/tmp.XUPiYDgI9f /tmp/tmp.3OyyeRhBO1
+++ return 0
++ first_timestamp=null
++ sleep 5
++ [[ null != '' ]]
++ [[ null != \n\u\l\l ]]
++ let retry+=1
++ [[ 3 -gt 30 ]]
++ [[ null != '' ]]
++ [[ null != \n\u\l\l ]]
+++ kubectl_bin exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ jq '.backups.pitrChunks.pitrChunks | last | .range.end'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.48KEMul5SZ
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.eCdNMjqAUs
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in $(seq 0 2)
+++ set +e
+++ kubectl exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.48KEMul5SZ
+++ cat /tmp/tmp.eCdNMjqAUs
+++ rm /tmp/tmp.48KEMul5SZ /tmp/tmp.eCdNMjqAUs
+++ return 0
++ first_timestamp=null
++ sleep 5
++ [[ null != '' ]]
++ [[ null != \n\u\l\l ]]
++ let retry+=1
++ [[ 4 -gt 30 ]]
++ [[ null != '' ]]
++ [[ null != \n\u\l\l ]]
+++ kubectl_bin exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ jq '.backups.pitrChunks.pitrChunks | last | .range.end'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.tlHunqM2T2
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.h1zXdTXqJS
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in $(seq 0 2)
+++ set +e
+++ kubectl exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.tlHunqM2T2
+++ cat /tmp/tmp.h1zXdTXqJS
+++ rm /tmp/tmp.tlHunqM2T2 /tmp/tmp.h1zXdTXqJS
+++ return 0
++ first_timestamp=null
++ sleep 5
++ [[ null != '' ]]
++ [[ null != \n\u\l\l ]]
++ let retry+=1
++ [[ 5 -gt 30 ]]
++ [[ null != '' ]]
++ [[ null != \n\u\l\l ]]
+++ kubectl_bin exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ jq '.backups.pitrChunks.pitrChunks | last | .range.end'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.16iopL63LU
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.wca22qPyOW
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in $(seq 0 2)
+++ set +e
+++ kubectl exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.16iopL63LU
+++ cat /tmp/tmp.wca22qPyOW
+++ rm /tmp/tmp.16iopL63LU /tmp/tmp.wca22qPyOW
+++ return 0
++ first_timestamp=null
++ sleep 5
++ [[ null != '' ]]
++ [[ null != \n\u\l\l ]]
++ let retry+=1
++ [[ 6 -gt 30 ]]
++ [[ null != '' ]]
++ [[ null != \n\u\l\l ]]
+++ kubectl_bin exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ jq '.backups.pitrChunks.pitrChunks | last | .range.end'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.HzlPKYaroN
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.cCDa9hW12L
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in $(seq 0 2)
+++ set +e
+++ kubectl exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.HzlPKYaroN
+++ cat /tmp/tmp.cCDa9hW12L
+++ rm /tmp/tmp.HzlPKYaroN /tmp/tmp.cCDa9hW12L
+++ return 0
++ first_timestamp=null
++ sleep 5
++ [[ null != '' ]]
++ [[ null != \n\u\l\l ]]
++ let retry+=1
++ [[ 7 -gt 30 ]]
++ [[ null != '' ]]
++ [[ null != \n\u\l\l ]]
+++ kubectl_bin exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ jq '.backups.pitrChunks.pitrChunks | last | .range.end'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.oTEqBdSDXs
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.TV8vubhs4F
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in $(seq 0 2)
+++ set +e
+++ kubectl exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.oTEqBdSDXs
+++ cat /tmp/tmp.TV8vubhs4F
+++ rm /tmp/tmp.oTEqBdSDXs /tmp/tmp.TV8vubhs4F
+++ return 0
++ first_timestamp=null
++ sleep 5
++ [[ null != '' ]]
++ [[ null != \n\u\l\l ]]
++ let retry+=1
++ [[ 8 -gt 30 ]]
++ [[ null != '' ]]
++ [[ null != \n\u\l\l ]]
+++ kubectl_bin exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ jq '.backups.pitrChunks.pitrChunks | last | .range.end'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.mOfxZbRUNM
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.VGxW8oW1O6
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in $(seq 0 2)
+++ set +e
+++ kubectl exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.mOfxZbRUNM
+++ cat /tmp/tmp.VGxW8oW1O6
+++ rm /tmp/tmp.mOfxZbRUNM /tmp/tmp.VGxW8oW1O6
+++ return 0
++ first_timestamp=null
++ sleep 5
++ [[ null != '' ]]
++ [[ null != \n\u\l\l ]]
++ let retry+=1
++ [[ 9 -gt 30 ]]
++ [[ null != '' ]]
++ [[ null != \n\u\l\l ]]
+++ kubectl_bin exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ jq '.backups.pitrChunks.pitrChunks | last | .range.end'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.ijxgOCwHPX
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.qJRESbntmk
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in $(seq 0 2)
+++ set +e
+++ kubectl exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.ijxgOCwHPX
+++ cat /tmp/tmp.qJRESbntmk
+++ rm /tmp/tmp.ijxgOCwHPX /tmp/tmp.qJRESbntmk
+++ return 0
++ first_timestamp=null
++ sleep 5
++ [[ null != '' ]]
++ [[ null != \n\u\l\l ]]
++ let retry+=1
++ [[ 10 -gt 30 ]]
++ [[ null != '' ]]
++ [[ null != \n\u\l\l ]]
+++ kubectl_bin exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ jq '.backups.pitrChunks.pitrChunks | last | .range.end'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.arFdZTBUUj
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.wcbwrdX9tQ
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in $(seq 0 2)
+++ set +e
+++ kubectl exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.arFdZTBUUj
+++ cat /tmp/tmp.wcbwrdX9tQ
+++ rm /tmp/tmp.arFdZTBUUj /tmp/tmp.wcbwrdX9tQ
+++ return 0
++ first_timestamp=null
++ sleep 5
++ [[ null != '' ]]
++ [[ null != \n\u\l\l ]]
++ let retry+=1
++ [[ 11 -gt 30 ]]
++ [[ null != '' ]]
++ [[ null != \n\u\l\l ]]
+++ kubectl_bin exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ jq '.backups.pitrChunks.pitrChunks | last | .range.end'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.wSZaPBZxMb
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.L1HriLBZM5
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in $(seq 0 2)
+++ set +e
+++ kubectl exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.wSZaPBZxMb
+++ cat /tmp/tmp.L1HriLBZM5
+++ rm /tmp/tmp.wSZaPBZxMb /tmp/tmp.L1HriLBZM5
+++ return 0
++ first_timestamp=null
++ sleep 5
++ [[ null != '' ]]
++ [[ null != \n\u\l\l ]]
++ let retry+=1
++ [[ 12 -gt 30 ]]
++ [[ null != '' ]]
++ [[ null != \n\u\l\l ]]
+++ kubectl_bin exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ jq '.backups.pitrChunks.pitrChunks | last | .range.end'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.DZ9AIZYm71
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.nLlQ1iFdj7
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in $(seq 0 2)
+++ set +e
+++ kubectl exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.DZ9AIZYm71
+++ cat /tmp/tmp.nLlQ1iFdj7
+++ rm /tmp/tmp.DZ9AIZYm71 /tmp/tmp.nLlQ1iFdj7
+++ return 0
++ first_timestamp=null
++ sleep 5
++ [[ null != '' ]]
++ [[ null != \n\u\l\l ]]
++ let retry+=1
++ [[ 13 -gt 30 ]]
++ [[ null != '' ]]
++ [[ null != \n\u\l\l ]]
+++ kubectl_bin exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ jq '.backups.pitrChunks.pitrChunks | last | .range.end'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.sW3FQcWLr2
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.DObA9roqaQ
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in $(seq 0 2)
+++ set +e
+++ kubectl exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.sW3FQcWLr2
+++ cat /tmp/tmp.DObA9roqaQ
+++ rm /tmp/tmp.sW3FQcWLr2 /tmp/tmp.DObA9roqaQ
+++ return 0
++ first_timestamp=null
++ sleep 5
++ [[ null != '' ]]
++ [[ null != \n\u\l\l ]]
++ let retry+=1
++ [[ 14 -gt 30 ]]
++ [[ null != '' ]]
++ [[ null != \n\u\l\l ]]
+++ kubectl_bin exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ jq '.backups.pitrChunks.pitrChunks | last | .range.end'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.fqjmHsk6Qs
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.fmdU8XLwhO
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in $(seq 0 2)
+++ set +e
+++ kubectl exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.fqjmHsk6Qs
+++ cat /tmp/tmp.fmdU8XLwhO
+++ rm /tmp/tmp.fqjmHsk6Qs /tmp/tmp.fmdU8XLwhO
+++ return 0
++ first_timestamp=null
++ sleep 5
++ [[ null != '' ]]
++ [[ null != \n\u\l\l ]]
++ let retry+=1
++ [[ 15 -gt 30 ]]
++ [[ null != '' ]]
++ [[ null != \n\u\l\l ]]
+++ kubectl_bin exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ jq '.backups.pitrChunks.pitrChunks | last | .range.end'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.WCc6oDVlfQ
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.Bws7Rq37UN
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in $(seq 0 2)
+++ set +e
+++ kubectl exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.WCc6oDVlfQ
+++ cat /tmp/tmp.Bws7Rq37UN
+++ rm /tmp/tmp.WCc6oDVlfQ /tmp/tmp.Bws7Rq37UN
+++ return 0
++ first_timestamp=null
++ sleep 5
++ [[ null != '' ]]
++ [[ null != \n\u\l\l ]]
++ let retry+=1
++ [[ 16 -gt 30 ]]
++ [[ null != '' ]]
++ [[ null != \n\u\l\l ]]
+++ kubectl_bin exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ jq '.backups.pitrChunks.pitrChunks | last | .range.end'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.1OikrqVWV0
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.ryDV6ZHzeZ
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in $(seq 0 2)
+++ set +e
+++ kubectl exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.1OikrqVWV0
+++ cat /tmp/tmp.ryDV6ZHzeZ
+++ rm /tmp/tmp.1OikrqVWV0 /tmp/tmp.ryDV6ZHzeZ
+++ return 0
++ first_timestamp=null
++ sleep 5
++ [[ null != '' ]]
++ [[ null != \n\u\l\l ]]
++ let retry+=1
++ [[ 17 -gt 30 ]]
++ [[ null != '' ]]
++ [[ null != \n\u\l\l ]]
+++ kubectl_bin exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ jq '.backups.pitrChunks.pitrChunks | last | .range.end'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.6402TnKjv4
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.pDAfbsom43
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in $(seq 0 2)
+++ set +e
+++ kubectl exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.6402TnKjv4
+++ cat /tmp/tmp.pDAfbsom43
+++ rm /tmp/tmp.6402TnKjv4 /tmp/tmp.pDAfbsom43
+++ return 0
++ first_timestamp=null
++ sleep 5
++ [[ null != '' ]]
++ [[ null != \n\u\l\l ]]
++ let retry+=1
++ [[ 18 -gt 30 ]]
++ [[ null != '' ]]
++ [[ null != \n\u\l\l ]]
+++ kubectl_bin exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ jq '.backups.pitrChunks.pitrChunks | last | .range.end'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.0hmuT9CYfm
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.NihfROfBm1
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in $(seq 0 2)
+++ set +e
+++ kubectl exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.0hmuT9CYfm
+++ cat /tmp/tmp.NihfROfBm1
+++ rm /tmp/tmp.0hmuT9CYfm /tmp/tmp.NihfROfBm1
+++ return 0
++ first_timestamp=null
++ sleep 5
++ [[ null != '' ]]
++ [[ null != \n\u\l\l ]]
++ let retry+=1
++ [[ 19 -gt 30 ]]
++ [[ null != '' ]]
++ [[ null != \n\u\l\l ]]
+++ kubectl_bin exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ jq '.backups.pitrChunks.pitrChunks | last | .range.end'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.wIUbsBvfjv
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.IdjTvFRHc0
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in $(seq 0 2)
+++ set +e
+++ kubectl exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.wIUbsBvfjv
+++ cat /tmp/tmp.IdjTvFRHc0
+++ rm /tmp/tmp.wIUbsBvfjv /tmp/tmp.IdjTvFRHc0
+++ return 0
++ first_timestamp=null
++ sleep 5
++ [[ null != '' ]]
++ [[ null != \n\u\l\l ]]
++ let retry+=1
++ [[ 20 -gt 30 ]]
++ [[ null != '' ]]
++ [[ null != \n\u\l\l ]]
+++ kubectl_bin exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ jq '.backups.pitrChunks.pitrChunks | last | .range.end'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.Eot42tZbTZ
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.rEKI6NmND3
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in $(seq 0 2)
+++ set +e
+++ kubectl exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.Eot42tZbTZ
+++ cat /tmp/tmp.rEKI6NmND3
+++ rm /tmp/tmp.Eot42tZbTZ /tmp/tmp.rEKI6NmND3
+++ return 0
++ first_timestamp=1757958330
++ sleep 5
++ [[ 1757958330 != '' ]]
++ [[ 1757958330 != \n\u\l\l ]]
+++ kubectl_bin exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ jq '.backups.pitrChunks.pitrChunks | last | .range.end'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.D2sHelH6Qh
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.63G85b7Kfh
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in $(seq 0 2)
+++ set +e
+++ kubectl exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.D2sHelH6Qh
+++ cat /tmp/tmp.63G85b7Kfh
+++ rm /tmp/tmp.D2sHelH6Qh /tmp/tmp.63G85b7Kfh
+++ return 0
++ second_timestamp=1757958330
++ let retry+=1
++ [[ 21 -gt 30 ]]
++ [[ 1757958330 != '' ]]
++ [[ 1757958330 != \n\u\l\l ]]
++ [[ 1757958330 == 1757958330 ]]
++ /usr/sbin/date -u -d @1757958330 +%Y-%m-%dT%H:%M:%SZ
+ latest_restorable_time=2025-09-15T17:45:30Z
++ get_latest_restorable_time_from_backup_object backup-minio-0
++ local backup_name=backup-minio-0
++ local latestRestorableTime
++ local retry=0
++ [[ '' != '' ]]
++ sleep 5
+++ kubectl_bin get psmdb-backup backup-minio-0 -o 'jsonpath={.status.latestRestorableTime}'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.GlfMZhGLyM
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.ls8Vgjb2R9
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in $(seq 0 2)
+++ set +e
+++ kubectl get psmdb-backup backup-minio-0 -o 'jsonpath={.status.latestRestorableTime}'
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.GlfMZhGLyM
+++ cat /tmp/tmp.ls8Vgjb2R9
+++ rm /tmp/tmp.GlfMZhGLyM /tmp/tmp.ls8Vgjb2R9
+++ return 0
++ latestRestorableTime=2025-09-15T17:45:30Z
++ let retry+=1
++ [[ 1 -gt 30 ]]
++ [[ 2025-09-15T17:45:30Z != '' ]]
++ [[ 2025-09-15T17:45:30Z != \n\u\l\l ]]
++ echo 2025-09-15T17:45:30Z
+ backup_time=2025-09-15T17:45:30Z
+ [[ 2025-09-15T17:45:30Z != \2\0\2\5\-\0\9\-\1\5\T\1\7\:\4\5\:\3\0\Z ]]
+ write_document -2nd
+ local cmp_postfix=-2nd
+ local sleep_value=0
+ log 'write initial data, read from all'
+ set +o xtrace
[2025-09-15T17:45:56+0000] write initial data, read from all
+ run_mongos 'use myApp\n db.test.insert({ x: 100500 })' myApp:myPass@some-name-mongos.pitr-sharded-3057
+ local 'command=use myApp\n db.test.insert({ x: 100500 })'
+ local uri=myApp:myPass@some-name-mongos.pitr-sharded-3057
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local port=27017
+ local mongo_bin=mongo
++ echo .svc.cluster.local
++ awk -F: '{print $2}'
+ suffix_port=
+ [[ -z '' ]]
+ suffix=.svc.cluster.local:27017
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.KkzNHaICdy
+++ mktemp
++ local LAST_ERR=/tmp/tmp.x9lACYRF4z
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.KkzNHaICdy
++ cat /tmp/tmp.x9lACYRF4z
++ rm /tmp/tmp.KkzNHaICdy /tmp/tmp.x9lACYRF4z
++ return 0
+ local client_container=psmdb-client-66f577db5f-ckxq9
+ kubectl_bin exec psmdb-client-66f577db5f-ckxq9 -- bash -c 'printf '\''use myApp\n db.test.insert({ x: 100500 })\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.pitr-sharded-3057.svc.cluster.local:27017/admin '
++ mktemp
+ local LAST_OUT=/tmp/tmp.VoBC6RDlPH
++ mktemp
+ local LAST_ERR=/tmp/tmp.l6X9EpC9Td
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-66f577db5f-ckxq9 -- bash -c 'printf '\''use myApp\n db.test.insert({ x: 100500 })\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.pitr-sharded-3057.svc.cluster.local:27017/admin '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.VoBC6RDlPH
Percona Server for MongoDB shell version v4.4.29-28
connecting to: mongodb://some-name-mongos.pitr-sharded-3057.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb
Implicit session: session { "id" : UUID("efc5aa2a-ff94-40e4-9d92-01e35c7b8027") }
Percona Server for MongoDB server version: v7.0.24-13
WARNING: shell and server versions do not match
switched to db myApp
WriteResult({ "nInserted" : 1 })
bye
+ cat /tmp/tmp.l6X9EpC9Td
+ rm /tmp/tmp.VoBC6RDlPH /tmp/tmp.l6X9EpC9Td
+ return 0
+ sleep 0
+ compare_mongos_cmd find myApp:myPass@some-name-mongos.pitr-sharded-3057 -2nd
+ local command=find
+ local uri=myApp:myPass@some-name-mongos.pitr-sharded-3057
+ local postfix=-2nd
+ local suffix=
+ local database=myApp
+ local collection=test
+ local port=27017
+ local tls=false
+ [[ false == \t\r\u\e ]]
+ mongos_command=run_mongos
+ run_mongos 'use myApp\n db.test.find()' myApp:myPass@some-name-mongos.pitr-sharded-3057 mongodb '' '' 27017
+ local 'command=use myApp\n db.test.find()'
+ local uri=myApp:myPass@some-name-mongos.pitr-sharded-3057
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local port=27017
+ local mongo_bin=mongo
+ egrep -v 'I NETWORK|W NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|Started a new thread for the timer service'
+ /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
++ echo .svc.cluster.local
++ awk -F: '{print $2}'
egrep: warning: egrep is obsolescent; using grep -E
+ suffix_port=
+ [[ -z '' ]]
+ suffix=.svc.cluster.local:27017
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.fT3Mw54oMy
+++ mktemp
++ local LAST_ERR=/tmp/tmp.sC9qV963xB
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.fT3Mw54oMy
++ cat /tmp/tmp.sC9qV963xB
++ rm /tmp/tmp.fT3Mw54oMy /tmp/tmp.sC9qV963xB
++ return 0
+ local client_container=psmdb-client-66f577db5f-ckxq9
+ kubectl_bin exec psmdb-client-66f577db5f-ckxq9 -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.pitr-sharded-3057.svc.cluster.local:27017/admin '
++ mktemp
+ local LAST_OUT=/tmp/tmp.M1XWlwB79a
++ mktemp
+ local LAST_ERR=/tmp/tmp.QZ8fm960Gk
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-66f577db5f-ckxq9 -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.pitr-sharded-3057.svc.cluster.local:27017/admin '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.M1XWlwB79a
+ cat /tmp/tmp.QZ8fm960Gk
+ rm /tmp/tmp.M1XWlwB79a /tmp/tmp.QZ8fm960Gk
+ return 0
+ [[ 0 -eq 0 ]]
+ diff /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2039/e2e-tests/pitr-sharded/compare/find-2nd.json /tmp/tmp.J3Hag6NYlY/find-2nd
+ sleep 2
++ run_mongos 'new Date().getTime() / 1000' myApp:myPass@some-name-mongos.pitr-sharded-3057 mongodb '' --quiet
++ local 'command=new Date().getTime() / 1000'
++ local uri=myApp:myPass@some-name-mongos.pitr-sharded-3057
++ local driver=mongodb
++ grep -E -v 'I NETWORK|W NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:'
++ local suffix=.svc.cluster.local
++ local mongo_flag=--quiet
++ local port=27017
++ local mongo_bin=mongo
++ cut -d. -f1
+++ echo .svc.cluster.local
+++ awk -F: '{print $2}'
++ suffix_port=
++ [[ -z '' ]]
++ suffix=.svc.cluster.local:27017
+++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.5kmOZPHBa4
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.hjsLsDrO3o
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in $(seq 0 2)
+++ set +e
+++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.5kmOZPHBa4
+++ cat /tmp/tmp.hjsLsDrO3o
+++ rm /tmp/tmp.5kmOZPHBa4 /tmp/tmp.hjsLsDrO3o
+++ return 0
++ local client_container=psmdb-client-66f577db5f-ckxq9
++ kubectl_bin exec psmdb-client-66f577db5f-ckxq9 -- bash -c 'printf '\''new Date().getTime() / 1000\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.pitr-sharded-3057.svc.cluster.local:27017/admin --quiet'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.THdWERoSNR
+++ mktemp
++ local LAST_ERR=/tmp/tmp.wjQWCKYMbg
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl exec psmdb-client-66f577db5f-ckxq9 -- bash -c 'printf '\''new Date().getTime() / 1000\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.pitr-sharded-3057.svc.cluster.local:27017/admin --quiet'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.THdWERoSNR
++ cat /tmp/tmp.wjQWCKYMbg
++ rm /tmp/tmp.THdWERoSNR /tmp/tmp.wjQWCKYMbg
++ return 0
+ time_now=1757958367
+ check_recovery backup-minio-0 date 1757958367 -2nd some-name
+ local backup_name=backup-minio-0
+ local restore_type=date
+ local restore_date=1757958367
+ local cmp_postfix=-2nd
+ local cluster_name=some-name
+ local backupSource=
++ get_latest_oplog_chunk_ts some-name
++ local cluster=some-name
+++ kubectl_bin exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ jq '.backups.pitrChunks.pitrChunks | last | .range.end'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.Q8noNnVCAo
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.Ar98RDx3p9
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in $(seq 0 2)
+++ set +e
+++ kubectl exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.Q8noNnVCAo
+++ cat /tmp/tmp.Ar98RDx3p9
+++ rm /tmp/tmp.Q8noNnVCAo /tmp/tmp.Ar98RDx3p9
+++ return 0
++ echo 1757958330
+ local latest_ts=1757958330
+ desc 'write more data before restore by date'
+ set +o xtrace


-----------------------------------------------------------------------------------
write more data before restore by date
-----------------------------------------------------------------------------------

+ run_mongos 'use myApp\n db.test.insert({ x: 100501 })' myApp:myPass@some-name-mongos.pitr-sharded-3057
+ local 'command=use myApp\n db.test.insert({ x: 100501 })'
+ local uri=myApp:myPass@some-name-mongos.pitr-sharded-3057
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local port=27017
+ local mongo_bin=mongo
++ echo .svc.cluster.local
++ awk -F: '{print $2}'
+ suffix_port=
+ [[ -z '' ]]
+ suffix=.svc.cluster.local:27017
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.Kzb6wieNwv
+++ mktemp
++ local LAST_ERR=/tmp/tmp.Edd8GzKoqB
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.Kzb6wieNwv
++ cat /tmp/tmp.Edd8GzKoqB
++ rm /tmp/tmp.Kzb6wieNwv /tmp/tmp.Edd8GzKoqB
++ return 0
+ local client_container=psmdb-client-66f577db5f-ckxq9
+ kubectl_bin exec psmdb-client-66f577db5f-ckxq9 -- bash -c 'printf '\''use myApp\n db.test.insert({ x: 100501 })\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.pitr-sharded-3057.svc.cluster.local:27017/admin '
++ mktemp
+ local LAST_OUT=/tmp/tmp.ftpBNfGu0s
++ mktemp
+ local LAST_ERR=/tmp/tmp.YehPaLnIIG
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-66f577db5f-ckxq9 -- bash -c 'printf '\''use myApp\n db.test.insert({ x: 100501 })\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.pitr-sharded-3057.svc.cluster.local:27017/admin '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.ftpBNfGu0s
Percona Server for MongoDB shell version v4.4.29-28
connecting to: mongodb://some-name-mongos.pitr-sharded-3057.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb
Implicit session: session { "id" : UUID("7880e7b4-5684-43b1-bc92-2b381f09373e") }
Percona Server for MongoDB server version: v7.0.24-13
WARNING: shell and server versions do not match
switched to db myApp
WriteResult({ "nInserted" : 1 })
bye
+ cat /tmp/tmp.YehPaLnIIG
+ rm /tmp/tmp.ftpBNfGu0s /tmp/tmp.YehPaLnIIG
+ return 0
+ [[ -n 1757958367 ]]
++ format_date 1757958367
++ local timestamp=1757958367
+++ TZ=UTC
+++ /usr/sbin/date -d@1757958367 '+%Y-%m-%d %H:%M:%S'
++ echo 2025-09-15 17:46:07
+ desc 'Restoring to time 2025-09-15 17:46:07'
+ set +o xtrace


-----------------------------------------------------------------------------------
Restoring to time 2025-09-15 17:46:07
-----------------------------------------------------------------------------------

+ retries=0
+ [[ 1757958330 -gt 1757958367 ]]
+ [[ 0 -gt 30 ]]
++ get_latest_oplog_chunk_ts some-name
++ local cluster=some-name
+++ kubectl_bin exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ jq '.backups.pitrChunks.pitrChunks | last | .range.end'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.rdDMRezxRI
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.nBhNyDS2Tx
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in $(seq 0 2)
+++ set +e
+++ kubectl exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.rdDMRezxRI
+++ cat /tmp/tmp.nBhNyDS2Tx
+++ rm /tmp/tmp.rdDMRezxRI /tmp/tmp.nBhNyDS2Tx
+++ return 0
++ echo 1757958330
+ latest_ts=1757958330
+ retries=1
++ format_date 1757958330
++ local timestamp=1757958330
+++ TZ=UTC
+++ /usr/sbin/date -d@1757958330 '+%Y-%m-%d %H:%M:%S'
++ echo 2025-09-15 17:45:30
++ format_date 1757958367
++ local timestamp=1757958367
+++ TZ=UTC
+++ /usr/sbin/date -d@1757958367 '+%Y-%m-%d %H:%M:%S'
++ echo 2025-09-15 17:46:07
+ echo 'Waiting for last oplog chunk (2025-09-15 17:45:30) to be greater than restore target (2025-09-15 17:46:07)'
Waiting for last oplog chunk (2025-09-15 17:45:30) to be greater than restore target (2025-09-15 17:46:07)
+ sleep 10
+ [[ 1757958330 -gt 1757958367 ]]
+ [[ 1 -gt 30 ]]
++ get_latest_oplog_chunk_ts some-name
++ local cluster=some-name
+++ kubectl_bin exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ jq '.backups.pitrChunks.pitrChunks | last | .range.end'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.fKJvXdn6J5
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.75wHz4scxk
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in $(seq 0 2)
+++ set +e
+++ kubectl exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.fKJvXdn6J5
+++ cat /tmp/tmp.75wHz4scxk
+++ rm /tmp/tmp.fKJvXdn6J5 /tmp/tmp.75wHz4scxk
+++ return 0
++ echo 1757958330
+ latest_ts=1757958330
+ retries=2
++ format_date 1757958330
++ local timestamp=1757958330
+++ TZ=UTC
+++ /usr/sbin/date -d@1757958330 '+%Y-%m-%d %H:%M:%S'
++ echo 2025-09-15 17:45:30
++ format_date 1757958367
++ local timestamp=1757958367
+++ TZ=UTC
+++ /usr/sbin/date -d@1757958367 '+%Y-%m-%d %H:%M:%S'
++ echo 2025-09-15 17:46:07
+ echo 'Waiting for last oplog chunk (2025-09-15 17:45:30) to be greater than restore target (2025-09-15 17:46:07)'
Waiting for last oplog chunk (2025-09-15 17:45:30) to be greater than restore target (2025-09-15 17:46:07)
+ sleep 10
+ [[ 1757958330 -gt 1757958367 ]]
+ [[ 2 -gt 30 ]]
++ get_latest_oplog_chunk_ts some-name
++ local cluster=some-name
+++ kubectl_bin exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ jq '.backups.pitrChunks.pitrChunks | last | .range.end'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.gzG6yOg6yr
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.hC3a7HCi42
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in $(seq 0 2)
+++ set +e
+++ kubectl exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.gzG6yOg6yr
+++ cat /tmp/tmp.hC3a7HCi42
+++ rm /tmp/tmp.gzG6yOg6yr /tmp/tmp.hC3a7HCi42
+++ return 0
++ echo 1757958330
+ latest_ts=1757958330
+ retries=3
++ format_date 1757958330
++ local timestamp=1757958330
+++ TZ=UTC
+++ /usr/sbin/date -d@1757958330 '+%Y-%m-%d %H:%M:%S'
++ echo 2025-09-15 17:45:30
++ format_date 1757958367
++ local timestamp=1757958367
+++ TZ=UTC
+++ /usr/sbin/date -d@1757958367 '+%Y-%m-%d %H:%M:%S'
++ echo 2025-09-15 17:46:07
+ echo 'Waiting for last oplog chunk (2025-09-15 17:45:30) to be greater than restore target (2025-09-15 17:46:07)'
Waiting for last oplog chunk (2025-09-15 17:45:30) to be greater than restore target (2025-09-15 17:46:07)
+ sleep 10
+ [[ 1757958330 -gt 1757958367 ]]
+ [[ 3 -gt 30 ]]
++ get_latest_oplog_chunk_ts some-name
++ local cluster=some-name
+++ kubectl_bin exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ jq '.backups.pitrChunks.pitrChunks | last | .range.end'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.Lwxj9NfkN1
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.YuGc6RAN2y
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in $(seq 0 2)
+++ set +e
+++ kubectl exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.Lwxj9NfkN1
+++ cat /tmp/tmp.YuGc6RAN2y
+++ rm /tmp/tmp.Lwxj9NfkN1 /tmp/tmp.YuGc6RAN2y
+++ return 0
++ echo 1757958330
+ latest_ts=1757958330
+ retries=4
++ format_date 1757958330
++ local timestamp=1757958330
+++ TZ=UTC
+++ /usr/sbin/date -d@1757958330 '+%Y-%m-%d %H:%M:%S'
++ echo 2025-09-15 17:45:30
++ format_date 1757958367
++ local timestamp=1757958367
+++ TZ=UTC
+++ /usr/sbin/date -d@1757958367 '+%Y-%m-%d %H:%M:%S'
++ echo 2025-09-15 17:46:07
+ echo 'Waiting for last oplog chunk (2025-09-15 17:45:30) to be greater than restore target (2025-09-15 17:46:07)'
Waiting for last oplog chunk (2025-09-15 17:45:30) to be greater than restore target (2025-09-15 17:46:07)
+ sleep 10
+ [[ 1757958330 -gt 1757958367 ]]
+ [[ 4 -gt 30 ]]
++ get_latest_oplog_chunk_ts some-name
++ local cluster=some-name
+++ kubectl_bin exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ jq '.backups.pitrChunks.pitrChunks | last | .range.end'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.Ei31Xx7Hrw
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.7Kkd90N6mv
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in $(seq 0 2)
+++ set +e
+++ kubectl exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.Ei31Xx7Hrw
+++ cat /tmp/tmp.7Kkd90N6mv
+++ rm /tmp/tmp.Ei31Xx7Hrw /tmp/tmp.7Kkd90N6mv
+++ return 0
++ echo 1757958330
+ latest_ts=1757958330
+ retries=5
++ format_date 1757958330
++ local timestamp=1757958330
+++ TZ=UTC
+++ /usr/sbin/date -d@1757958330 '+%Y-%m-%d %H:%M:%S'
++ echo 2025-09-15 17:45:30
++ format_date 1757958367
++ local timestamp=1757958367
+++ TZ=UTC
+++ /usr/sbin/date -d@1757958367 '+%Y-%m-%d %H:%M:%S'
++ echo 2025-09-15 17:46:07
+ echo 'Waiting for last oplog chunk (2025-09-15 17:45:30) to be greater than restore target (2025-09-15 17:46:07)'
Waiting for last oplog chunk (2025-09-15 17:45:30) to be greater than restore target (2025-09-15 17:46:07)
+ sleep 10
+ [[ 1757958330 -gt 1757958367 ]]
+ [[ 5 -gt 30 ]]
++ get_latest_oplog_chunk_ts some-name
++ local cluster=some-name
+++ kubectl_bin exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ jq '.backups.pitrChunks.pitrChunks | last | .range.end'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.cFfG9EfCVu
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.6VEHP6czFJ
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in $(seq 0 2)
+++ set +e
+++ kubectl exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.cFfG9EfCVu
+++ cat /tmp/tmp.6VEHP6czFJ
+++ rm /tmp/tmp.cFfG9EfCVu /tmp/tmp.6VEHP6czFJ
+++ return 0
++ echo 1757958330
+ latest_ts=1757958330
+ retries=6
++ format_date 1757958330
++ local timestamp=1757958330
+++ TZ=UTC
+++ /usr/sbin/date -d@1757958330 '+%Y-%m-%d %H:%M:%S'
++ echo 2025-09-15 17:45:30
++ format_date 1757958367
++ local timestamp=1757958367
+++ TZ=UTC
+++ /usr/sbin/date -d@1757958367 '+%Y-%m-%d %H:%M:%S'
++ echo 2025-09-15 17:46:07
+ echo 'Waiting for last oplog chunk (2025-09-15 17:45:30) to be greater than restore target (2025-09-15 17:46:07)'
Waiting for last oplog chunk (2025-09-15 17:45:30) to be greater than restore target (2025-09-15 17:46:07)
+ sleep 10
+ [[ 1757958330 -gt 1757958367 ]]
+ [[ 6 -gt 30 ]]
++ get_latest_oplog_chunk_ts some-name
++ local cluster=some-name
+++ kubectl_bin exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ jq '.backups.pitrChunks.pitrChunks | last | .range.end'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.FY94qovuXw
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.YTWQYYKrKc
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in $(seq 0 2)
+++ set +e
+++ kubectl exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.FY94qovuXw
+++ cat /tmp/tmp.YTWQYYKrKc
+++ rm /tmp/tmp.FY94qovuXw /tmp/tmp.YTWQYYKrKc
+++ return 0
++ echo 1757958330
+ latest_ts=1757958330
+ retries=7
++ format_date 1757958330
++ local timestamp=1757958330
+++ TZ=UTC
+++ /usr/sbin/date -d@1757958330 '+%Y-%m-%d %H:%M:%S'
++ echo 2025-09-15 17:45:30
++ format_date 1757958367
++ local timestamp=1757958367
+++ TZ=UTC
+++ /usr/sbin/date -d@1757958367 '+%Y-%m-%d %H:%M:%S'
++ echo 2025-09-15 17:46:07
+ echo 'Waiting for last oplog chunk (2025-09-15 17:45:30) to be greater than restore target (2025-09-15 17:46:07)'
Waiting for last oplog chunk (2025-09-15 17:45:30) to be greater than restore target (2025-09-15 17:46:07)
+ sleep 10
+ [[ 1757958330 -gt 1757958367 ]]
+ [[ 7 -gt 30 ]]
++ get_latest_oplog_chunk_ts some-name
++ local cluster=some-name
+++ kubectl_bin exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ jq '.backups.pitrChunks.pitrChunks | last | .range.end'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.hGOlaYpEJu
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.pgp1cLhw7O
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in $(seq 0 2)
+++ set +e
+++ kubectl exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.hGOlaYpEJu
+++ cat /tmp/tmp.pgp1cLhw7O
+++ rm /tmp/tmp.hGOlaYpEJu /tmp/tmp.pgp1cLhw7O
+++ return 0
++ echo 1757958330
+ latest_ts=1757958330
+ retries=8
++ format_date 1757958330
++ local timestamp=1757958330
+++ TZ=UTC
+++ /usr/sbin/date -d@1757958330 '+%Y-%m-%d %H:%M:%S'
++ echo 2025-09-15 17:45:30
++ format_date 1757958367
++ local timestamp=1757958367
+++ TZ=UTC
+++ /usr/sbin/date -d@1757958367 '+%Y-%m-%d %H:%M:%S'
++ echo 2025-09-15 17:46:07
+ echo 'Waiting for last oplog chunk (2025-09-15 17:45:30) to be greater than restore target (2025-09-15 17:46:07)'
Waiting for last oplog chunk (2025-09-15 17:45:30) to be greater than restore target (2025-09-15 17:46:07)
+ sleep 10
+ [[ 1757958330 -gt 1757958367 ]]
+ [[ 8 -gt 30 ]]
++ get_latest_oplog_chunk_ts some-name
++ local cluster=some-name
+++ kubectl_bin exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ jq '.backups.pitrChunks.pitrChunks | last | .range.end'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.H877LDRcvq
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.qtQdiKwBpl
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in $(seq 0 2)
+++ set +e
+++ kubectl exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.H877LDRcvq
+++ cat /tmp/tmp.qtQdiKwBpl
+++ rm /tmp/tmp.H877LDRcvq /tmp/tmp.qtQdiKwBpl
+++ return 0
++ echo 1757958450
+ latest_ts=1757958450
+ retries=9
++ format_date 1757958450
++ local timestamp=1757958450
+++ TZ=UTC
+++ /usr/sbin/date -d@1757958450 '+%Y-%m-%d %H:%M:%S'
++ echo 2025-09-15 17:47:30
++ format_date 1757958367
++ local timestamp=1757958367
+++ TZ=UTC
+++ /usr/sbin/date -d@1757958367 '+%Y-%m-%d %H:%M:%S'
++ echo 2025-09-15 17:46:07
+ echo 'Waiting for last oplog chunk (2025-09-15 17:47:30) to be greater than restore target (2025-09-15 17:46:07)'
Waiting for last oplog chunk (2025-09-15 17:47:30) to be greater than restore target (2025-09-15 17:46:07)
+ sleep 10
+ [[ 1757958450 -gt 1757958367 ]]
+ '[' -z '' ']'
+ desc 'check restore by date'
+ set +o xtrace


-----------------------------------------------------------------------------------
check restore by date
-----------------------------------------------------------------------------------

+ cat /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2039/e2e-tests/pitr-sharded/conf/restore.yml
+ /usr/sbin/sed -e 's/name:/name: restore-backup-minio-0/'
+ /usr/sbin/sed -e 's/backupName:/backupName: backup-minio-0/'
+ /usr/sbin/sed -e /backupSource/,+8d
+ /usr/sbin/sed -e 's/pitrType:/type: date/'
+ '[' -z 1757958367 ']'
+ kubectl_bin apply -f -
++ mktemp
++ format_date 1757958367
++ local timestamp=1757958367
+++ TZ=UTC
+++ /usr/sbin/date -d@1757958367 '+%Y-%m-%d %H:%M:%S'
+ local LAST_OUT=/tmp/tmp.moxZanSxt7
++ mktemp
++ echo 2025-09-15 17:46:07
+ /usr/sbin/sed -e 's/date:/date: 2025-09-15 17:46:07/'
+ local LAST_ERR=/tmp/tmp.AX4AVQM6XO
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl apply -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.moxZanSxt7
perconaservermongodbrestore.psmdb.percona.com/restore-backup-minio-0 created
+ cat /tmp/tmp.AX4AVQM6XO
+ rm /tmp/tmp.moxZanSxt7 /tmp/tmp.AX4AVQM6XO
+ return 0
+ wait_restore backup-minio-0 some-name requested 0 1200
+ local backup_name=backup-minio-0
+ local cluster_name=some-name
+ local target_state=requested
+ local wait_cluster_consistency=0
+ local wait_time=1200
+ local ok_if_ready=0
+ set +o xtrace
Waiting for the psmdb-restore/restore-backup-minio-0 object to be createdOK
Waiting psmdb-restore/restore-backup-minio-0 to reach state "requested" .OK after 1 minutes
+ [[ 0 -eq 1 ]]
+ echo

+ wait_restore backup-minio-0 some-name ready 0 1600
+ local backup_name=backup-minio-0
+ local cluster_name=some-name
+ local target_state=ready
+ local wait_cluster_consistency=0
+ local wait_time=1600
+ local ok_if_ready=0
+ set +o xtrace
Waiting for the psmdb-restore/restore-backup-minio-0 object to be createdOK
Waiting psmdb-restore/restore-backup-minio-0 to reach state "ready" OK after 0 minutes
+ [[ 0 -eq 1 ]]
+ echo

+ set -o xtrace
+ wait_for_running some-name-rs0 3
+ local name=some-name-rs0
+ let last_pod=2
+ local check_cluster_readyness=true
+ set_debug
+ [[ 1 == 1 ]]
+ set -o xtrace
+ local rs_name=rs0
+ local cluster_name=some-name
++ seq 0 2
+ for i in $(seq 0 $last_pod)
+ [[ 0 -eq 2 ]]
+ wait_pod some-name-rs0-0
+ local pod=some-name-rs0-0
+ set +o xtrace
waiting for pod/some-name-rs0-0 to be ready.OK
+ for i in $(seq 0 $last_pod)
+ [[ 1 -eq 2 ]]
+ wait_pod some-name-rs0-1
+ local pod=some-name-rs0-1
+ set +o xtrace
waiting for pod/some-name-rs0-1 to be ready.OK
+ for i in $(seq 0 $last_pod)
+ [[ 2 -eq 2 ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].arbiter.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.ulZ5ueOoQO
+++ mktemp
++ local LAST_ERR=/tmp/tmp.YG2A0nOyKF
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].arbiter.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.ulZ5ueOoQO
++ cat /tmp/tmp.YG2A0nOyKF
++ rm /tmp/tmp.ulZ5ueOoQO /tmp/tmp.YG2A0nOyKF
++ return 0
+ [[ '' == \t\r\u\e ]]
+ wait_pod some-name-rs0-2
+ local pod=some-name-rs0-2
+ set +o xtrace
waiting for pod/some-name-rs0-2 to be ready.OK
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].non_voting.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.WVmeq9Z2V7
+++ mktemp
++ local LAST_ERR=/tmp/tmp.T8od5Dewfm
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].non_voting.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.WVmeq9Z2V7
++ cat /tmp/tmp.T8od5Dewfm
++ rm /tmp/tmp.WVmeq9Z2V7 /tmp/tmp.T8od5Dewfm
++ return 0
+ [[ '' == \t\r\u\e ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].hidden.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.qX3DXiMeZn
+++ mktemp
++ local LAST_ERR=/tmp/tmp.fvCAFxrYy6
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].hidden.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.qX3DXiMeZn
++ cat /tmp/tmp.fvCAFxrYy6
++ rm /tmp/tmp.qX3DXiMeZn /tmp/tmp.fvCAFxrYy6
++ return 0
+ [[ '' == \t\r\u\e ]]
+ sleep 10
+ [[ true == \t\r\u\e ]]
+ set +x
Waiting for cluster readyness..........
+ wait_for_running some-name-cfg 3
+ local name=some-name-cfg
+ let last_pod=2
+ local check_cluster_readyness=true
+ set_debug
+ [[ 1 == 1 ]]
+ set -o xtrace
+ local rs_name=cfg
+ local cluster_name=some-name
++ seq 0 2
+ for i in $(seq 0 $last_pod)
+ [[ 0 -eq 2 ]]
+ wait_pod some-name-cfg-0
+ local pod=some-name-cfg-0
+ set +o xtrace
waiting for pod/some-name-cfg-0 to be ready.OK
+ for i in $(seq 0 $last_pod)
+ [[ 1 -eq 2 ]]
+ wait_pod some-name-cfg-1
+ local pod=some-name-cfg-1
+ set +o xtrace
waiting for pod/some-name-cfg-1 to be ready.OK
+ for i in $(seq 0 $last_pod)
+ [[ 2 -eq 2 ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].arbiter.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.6puQNRNx5V
+++ mktemp
++ local LAST_ERR=/tmp/tmp.xRS3rXG9Xb
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].arbiter.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.6puQNRNx5V
++ cat /tmp/tmp.xRS3rXG9Xb
++ rm /tmp/tmp.6puQNRNx5V /tmp/tmp.xRS3rXG9Xb
++ return 0
+ [[ '' == \t\r\u\e ]]
+ wait_pod some-name-cfg-2
+ local pod=some-name-cfg-2
+ set +o xtrace
waiting for pod/some-name-cfg-2 to be ready.OK
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].non_voting.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.jJlbUyrwZ5
+++ mktemp
++ local LAST_ERR=/tmp/tmp.wLenjw35yy
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].non_voting.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.jJlbUyrwZ5
++ cat /tmp/tmp.wLenjw35yy
++ rm /tmp/tmp.jJlbUyrwZ5 /tmp/tmp.wLenjw35yy
++ return 0
+ [[ '' == \t\r\u\e ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].hidden.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.DFWU0k6Q7f
+++ mktemp
++ local LAST_ERR=/tmp/tmp.wtK2b15EBH
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].hidden.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.DFWU0k6Q7f
++ cat /tmp/tmp.wtK2b15EBH
++ rm /tmp/tmp.DFWU0k6Q7f /tmp/tmp.wtK2b15EBH
++ return 0
+ [[ '' == \t\r\u\e ]]
+ sleep 10
+ [[ true == \t\r\u\e ]]
+ set +x
Waiting for cluster readyness
+ wait_for_running some-name-mongos 3
+ local name=some-name-mongos
+ let last_pod=2
+ local check_cluster_readyness=true
+ set_debug
+ [[ 1 == 1 ]]
+ set -o xtrace
+ local rs_name=mongos
+ local cluster_name=some-name
++ seq 0 2
+ for i in $(seq 0 $last_pod)
+ [[ 0 -eq 2 ]]
+ wait_pod some-name-mongos-0
+ local pod=some-name-mongos-0
+ set +o xtrace
waiting for pod/some-name-mongos-0 to be ready.OK
+ for i in $(seq 0 $last_pod)
+ [[ 1 -eq 2 ]]
+ wait_pod some-name-mongos-1
+ local pod=some-name-mongos-1
+ set +o xtrace
waiting for pod/some-name-mongos-1 to be ready.OK
+ for i in $(seq 0 $last_pod)
+ [[ 2 -eq 2 ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="mongos")].arbiter.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.IpewLZYqSv
+++ mktemp
++ local LAST_ERR=/tmp/tmp.xgYQ4Tvlhl
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="mongos")].arbiter.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.IpewLZYqSv
++ cat /tmp/tmp.xgYQ4Tvlhl
++ rm /tmp/tmp.IpewLZYqSv /tmp/tmp.xgYQ4Tvlhl
++ return 0
+ [[ '' == \t\r\u\e ]]
+ wait_pod some-name-mongos-2
+ local pod=some-name-mongos-2
+ set +o xtrace
waiting for pod/some-name-mongos-2 to be ready.OK
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="mongos")].non_voting.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.9uheyGPcYX
+++ mktemp
++ local LAST_ERR=/tmp/tmp.kfHCU00yDV
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="mongos")].non_voting.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.9uheyGPcYX
++ cat /tmp/tmp.kfHCU00yDV
++ rm /tmp/tmp.9uheyGPcYX /tmp/tmp.kfHCU00yDV
++ return 0
+ [[ '' == \t\r\u\e ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="mongos")].hidden.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.WCZwBzAuMe
+++ mktemp
++ local LAST_ERR=/tmp/tmp.Bi5rqlrC1y
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="mongos")].hidden.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.WCZwBzAuMe
++ cat /tmp/tmp.Bi5rqlrC1y
++ rm /tmp/tmp.WCZwBzAuMe /tmp/tmp.Bi5rqlrC1y
++ return 0
+ [[ '' == \t\r\u\e ]]
+ sleep 10
+ [[ true == \t\r\u\e ]]
+ set +x
Waiting for cluster readyness
+ sleep 10
+ compare_mongos_cmd find myApp:myPass@some-name-mongos.pitr-sharded-3057 -2nd
+ local command=find
+ local uri=myApp:myPass@some-name-mongos.pitr-sharded-3057
+ local postfix=-2nd
+ local suffix=
+ local database=myApp
+ local collection=test
+ local port=27017
+ local tls=false
+ [[ false == \t\r\u\e ]]
+ mongos_command=run_mongos
+ run_mongos 'use myApp\n db.test.find()' myApp:myPass@some-name-mongos.pitr-sharded-3057 mongodb '' '' 27017
+ local 'command=use myApp\n db.test.find()'
+ local uri=myApp:myPass@some-name-mongos.pitr-sharded-3057
+ egrep -v 'I NETWORK|W NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|Started a new thread for the timer service'
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local port=27017
+ local mongo_bin=mongo
+ /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
++ echo .svc.cluster.local
++ awk -F: '{print $2}'
egrep: warning: egrep is obsolescent; using grep -E
+ suffix_port=
+ [[ -z '' ]]
+ suffix=.svc.cluster.local:27017
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.Oplbu0qOXQ
+++ mktemp
++ local LAST_ERR=/tmp/tmp.spHD96Qvau
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.Oplbu0qOXQ
++ cat /tmp/tmp.spHD96Qvau
++ rm /tmp/tmp.Oplbu0qOXQ /tmp/tmp.spHD96Qvau
++ return 0
+ local client_container=psmdb-client-66f577db5f-ckxq9
+ kubectl_bin exec psmdb-client-66f577db5f-ckxq9 -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.pitr-sharded-3057.svc.cluster.local:27017/admin '
++ mktemp
+ local LAST_OUT=/tmp/tmp.egPNexnjU8
++ mktemp
+ local LAST_ERR=/tmp/tmp.a8snYoJv1W
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-66f577db5f-ckxq9 -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.pitr-sharded-3057.svc.cluster.local:27017/admin '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.egPNexnjU8
+ cat /tmp/tmp.a8snYoJv1W
+ rm /tmp/tmp.egPNexnjU8 /tmp/tmp.a8snYoJv1W
+ return 0
+ [[ 0 -eq 0 ]]
+ diff /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2039/e2e-tests/pitr-sharded/compare/find-2nd.json /tmp/tmp.J3Hag6NYlY/find-2nd
+ run_backup backup-minio 1
+ local name=backup-minio
+ local idx=1
+ desc 'run backup backup-minio-1'
+ set +o xtrace


-----------------------------------------------------------------------------------
run backup backup-minio-1
-----------------------------------------------------------------------------------

+ cat /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2039/e2e-tests/pitr-sharded/conf/backup-minio.yml
+ /usr/sbin/sed -e 's/name:/name: backup-minio-1/'
+ kubectl_bin apply -f -
++ mktemp
+ local LAST_OUT=/tmp/tmp.1CNeSnKPFa
++ mktemp
+ local LAST_ERR=/tmp/tmp.FbgRJAglB8
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl apply -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.1CNeSnKPFa
perconaservermongodbbackup.psmdb.percona.com/backup-minio-1 created
+ cat /tmp/tmp.FbgRJAglB8
+ rm /tmp/tmp.1CNeSnKPFa /tmp/tmp.FbgRJAglB8
+ return 0
+ wait_backup backup-minio-1
+ local backup_name=backup-minio-1
+ local target_state=ready
+ set +o xtrace
waiting for backup-minio-1 to reach ready state...............
+ sleep 5
+ check_recovery backup-minio-1 latest '' -3rd some-name
+ local backup_name=backup-minio-1
+ local restore_type=latest
+ local restore_date=
+ local cmp_postfix=-3rd
+ local cluster_name=some-name
+ local backupSource=
++ get_latest_oplog_chunk_ts some-name
++ local cluster=some-name
+++ kubectl_bin exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ jq '.backups.pitrChunks.pitrChunks | last | .range.end'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.v36waNMfr2
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.AaXni403mj
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in $(seq 0 2)
+++ set +e
+++ kubectl exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.v36waNMfr2
+++ cat /tmp/tmp.AaXni403mj
+++ rm /tmp/tmp.v36waNMfr2 /tmp/tmp.AaXni403mj
+++ return 0
++ echo 1757958552
+ local latest_ts=1757958552
+ desc 'write more data before restore by latest'
+ set +o xtrace


-----------------------------------------------------------------------------------
write more data before restore by latest
-----------------------------------------------------------------------------------

+ run_mongos 'use myApp\n db.test.insert({ x: 100501 })' myApp:myPass@some-name-mongos.pitr-sharded-3057
+ local 'command=use myApp\n db.test.insert({ x: 100501 })'
+ local uri=myApp:myPass@some-name-mongos.pitr-sharded-3057
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local port=27017
+ local mongo_bin=mongo
++ echo .svc.cluster.local
++ awk -F: '{print $2}'
+ suffix_port=
+ [[ -z '' ]]
+ suffix=.svc.cluster.local:27017
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.6Jgu7ntB6m
+++ mktemp
++ local LAST_ERR=/tmp/tmp.aClai5ndNw
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.6Jgu7ntB6m
++ cat /tmp/tmp.aClai5ndNw
++ rm /tmp/tmp.6Jgu7ntB6m /tmp/tmp.aClai5ndNw
++ return 0
+ local client_container=psmdb-client-66f577db5f-ckxq9
+ kubectl_bin exec psmdb-client-66f577db5f-ckxq9 -- bash -c 'printf '\''use myApp\n db.test.insert({ x: 100501 })\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.pitr-sharded-3057.svc.cluster.local:27017/admin '
++ mktemp
+ local LAST_OUT=/tmp/tmp.jyG9Uzlzr9
++ mktemp
+ local LAST_ERR=/tmp/tmp.aQnNxlFvFC
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-66f577db5f-ckxq9 -- bash -c 'printf '\''use myApp\n db.test.insert({ x: 100501 })\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.pitr-sharded-3057.svc.cluster.local:27017/admin '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.jyG9Uzlzr9
Percona Server for MongoDB shell version v4.4.29-28
connecting to: mongodb://some-name-mongos.pitr-sharded-3057.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb
Implicit session: session { "id" : UUID("bc20c5d1-70b1-4ab7-9325-8fc2587af215") }
Percona Server for MongoDB server version: v7.0.24-13
WARNING: shell and server versions do not match
switched to db myApp
WriteResult({ "nInserted" : 1 })
bye
+ cat /tmp/tmp.aQnNxlFvFC
+ rm /tmp/tmp.jyG9Uzlzr9 /tmp/tmp.aQnNxlFvFC
+ return 0
+ [[ -n '' ]]
+ desc 'Restoring to latest'
+ set +o xtrace


-----------------------------------------------------------------------------------
Restoring to latest
-----------------------------------------------------------------------------------

++ get_latest_oplog_chunk_ts some-name
++ local cluster=some-name
+++ kubectl_bin exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ jq '.backups.pitrChunks.pitrChunks | last | .range.end'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.GUk4aE3DyA
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.7WPqLzOX9K
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in $(seq 0 2)
+++ set +e
+++ kubectl exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.GUk4aE3DyA
+++ cat /tmp/tmp.7WPqLzOX9K
+++ rm /tmp/tmp.GUk4aE3DyA /tmp/tmp.7WPqLzOX9K
+++ return 0
++ echo 1757958552
+ local current_ts=1757958552
+ retries=0
+ [[ 1757958552 -gt 1757958552 ]]
+ [[ 0 -gt 30 ]]
++ get_latest_oplog_chunk_ts some-name
++ local cluster=some-name
+++ kubectl_bin exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ jq '.backups.pitrChunks.pitrChunks | last | .range.end'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.3rRajS5IIM
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.am2NYPoTPE
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in $(seq 0 2)
+++ set +e
+++ kubectl exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.3rRajS5IIM
+++ cat /tmp/tmp.am2NYPoTPE
+++ rm /tmp/tmp.3rRajS5IIM /tmp/tmp.am2NYPoTPE
+++ return 0
++ echo 1757958552
+ latest_ts=1757958552
+ retries=1
++ format_date 1757958552
++ local timestamp=1757958552
+++ TZ=UTC
+++ /usr/sbin/date -d@1757958552 '+%Y-%m-%d %H:%M:%S'
++ echo 2025-09-15 17:49:12
++ format_date 1757958552
++ local timestamp=1757958552
+++ TZ=UTC
+++ /usr/sbin/date -d@1757958552 '+%Y-%m-%d %H:%M:%S'
++ echo 2025-09-15 17:49:12
+ echo 'Waiting for last oplog chunk (2025-09-15 17:49:12) to be 120 seconds older than starting chunk (2025-09-15 17:49:12)'
Waiting for last oplog chunk (2025-09-15 17:49:12) to be 120 seconds older than starting chunk (2025-09-15 17:49:12)
+ sleep 10
+ [[ 1757958552 -gt 1757958552 ]]
+ [[ 1 -gt 30 ]]
++ get_latest_oplog_chunk_ts some-name
++ local cluster=some-name
+++ kubectl_bin exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ jq '.backups.pitrChunks.pitrChunks | last | .range.end'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.2re6ahtjvS
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.6z7KQ8TM58
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in $(seq 0 2)
+++ set +e
+++ kubectl exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.2re6ahtjvS
+++ cat /tmp/tmp.6z7KQ8TM58
+++ rm /tmp/tmp.2re6ahtjvS /tmp/tmp.6z7KQ8TM58
+++ return 0
++ echo 1757958552
+ latest_ts=1757958552
+ retries=2
++ format_date 1757958552
++ local timestamp=1757958552
+++ TZ=UTC
+++ /usr/sbin/date -d@1757958552 '+%Y-%m-%d %H:%M:%S'
++ echo 2025-09-15 17:49:12
++ format_date 1757958552
++ local timestamp=1757958552
+++ TZ=UTC
+++ /usr/sbin/date -d@1757958552 '+%Y-%m-%d %H:%M:%S'
++ echo 2025-09-15 17:49:12
+ echo 'Waiting for last oplog chunk (2025-09-15 17:49:12) to be 120 seconds older than starting chunk (2025-09-15 17:49:12)'
Waiting for last oplog chunk (2025-09-15 17:49:12) to be 120 seconds older than starting chunk (2025-09-15 17:49:12)
+ sleep 10
+ [[ 1757958552 -gt 1757958552 ]]
+ [[ 2 -gt 30 ]]
++ get_latest_oplog_chunk_ts some-name
++ local cluster=some-name
+++ kubectl_bin exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ jq '.backups.pitrChunks.pitrChunks | last | .range.end'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.waCmVdwXsg
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.p3hXdu7aUY
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in $(seq 0 2)
+++ set +e
+++ kubectl exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.waCmVdwXsg
+++ cat /tmp/tmp.p3hXdu7aUY
+++ rm /tmp/tmp.waCmVdwXsg /tmp/tmp.p3hXdu7aUY
+++ return 0
++ echo 1757958552
+ latest_ts=1757958552
+ retries=3
++ format_date 1757958552
++ local timestamp=1757958552
+++ TZ=UTC
+++ /usr/sbin/date -d@1757958552 '+%Y-%m-%d %H:%M:%S'
++ echo 2025-09-15 17:49:12
++ format_date 1757958552
++ local timestamp=1757958552
+++ TZ=UTC
+++ /usr/sbin/date -d@1757958552 '+%Y-%m-%d %H:%M:%S'
++ echo 2025-09-15 17:49:12
+ echo 'Waiting for last oplog chunk (2025-09-15 17:49:12) to be 120 seconds older than starting chunk (2025-09-15 17:49:12)'
Waiting for last oplog chunk (2025-09-15 17:49:12) to be 120 seconds older than starting chunk (2025-09-15 17:49:12)
+ sleep 10
+ [[ 1757958552 -gt 1757958552 ]]
+ [[ 3 -gt 30 ]]
++ get_latest_oplog_chunk_ts some-name
++ local cluster=some-name
+++ kubectl_bin exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ jq '.backups.pitrChunks.pitrChunks | last | .range.end'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.9aQpeOE2H3
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.LS6fX0sBsl
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in $(seq 0 2)
+++ set +e
+++ kubectl exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.9aQpeOE2H3
+++ cat /tmp/tmp.LS6fX0sBsl
+++ rm /tmp/tmp.9aQpeOE2H3 /tmp/tmp.LS6fX0sBsl
+++ return 0
++ echo 1757958552
+ latest_ts=1757958552
+ retries=4
++ format_date 1757958552
++ local timestamp=1757958552
+++ TZ=UTC
+++ /usr/sbin/date -d@1757958552 '+%Y-%m-%d %H:%M:%S'
++ echo 2025-09-15 17:49:12
++ format_date 1757958552
++ local timestamp=1757958552
+++ TZ=UTC
+++ /usr/sbin/date -d@1757958552 '+%Y-%m-%d %H:%M:%S'
++ echo 2025-09-15 17:49:12
+ echo 'Waiting for last oplog chunk (2025-09-15 17:49:12) to be 120 seconds older than starting chunk (2025-09-15 17:49:12)'
Waiting for last oplog chunk (2025-09-15 17:49:12) to be 120 seconds older than starting chunk (2025-09-15 17:49:12)
+ sleep 10
+ [[ 1757958552 -gt 1757958552 ]]
+ [[ 4 -gt 30 ]]
++ get_latest_oplog_chunk_ts some-name
++ local cluster=some-name
+++ kubectl_bin exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ jq '.backups.pitrChunks.pitrChunks | last | .range.end'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.tOjynWZfKb
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.GULyNWDCwP
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in $(seq 0 2)
+++ set +e
+++ kubectl exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.tOjynWZfKb
+++ cat /tmp/tmp.GULyNWDCwP
+++ rm /tmp/tmp.tOjynWZfKb /tmp/tmp.GULyNWDCwP
+++ return 0
++ echo 1757958552
+ latest_ts=1757958552
+ retries=5
++ format_date 1757958552
++ local timestamp=1757958552
+++ TZ=UTC
+++ /usr/sbin/date -d@1757958552 '+%Y-%m-%d %H:%M:%S'
++ echo 2025-09-15 17:49:12
++ format_date 1757958552
++ local timestamp=1757958552
+++ TZ=UTC
+++ /usr/sbin/date -d@1757958552 '+%Y-%m-%d %H:%M:%S'
++ echo 2025-09-15 17:49:12
+ echo 'Waiting for last oplog chunk (2025-09-15 17:49:12) to be 120 seconds older than starting chunk (2025-09-15 17:49:12)'
Waiting for last oplog chunk (2025-09-15 17:49:12) to be 120 seconds older than starting chunk (2025-09-15 17:49:12)
+ sleep 10
+ [[ 1757958552 -gt 1757958552 ]]
+ [[ 5 -gt 30 ]]
++ get_latest_oplog_chunk_ts some-name
++ local cluster=some-name
+++ kubectl_bin exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ jq '.backups.pitrChunks.pitrChunks | last | .range.end'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.5jDH4gIoeV
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.JYCfImHTdo
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in $(seq 0 2)
+++ set +e
+++ kubectl exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.5jDH4gIoeV
+++ cat /tmp/tmp.JYCfImHTdo
+++ rm /tmp/tmp.5jDH4gIoeV /tmp/tmp.JYCfImHTdo
+++ return 0
++ echo 1757958552
+ latest_ts=1757958552
+ retries=6
++ format_date 1757958552
++ local timestamp=1757958552
+++ TZ=UTC
+++ /usr/sbin/date -d@1757958552 '+%Y-%m-%d %H:%M:%S'
++ echo 2025-09-15 17:49:12
++ format_date 1757958552
++ local timestamp=1757958552
+++ TZ=UTC
+++ /usr/sbin/date -d@1757958552 '+%Y-%m-%d %H:%M:%S'
++ echo 2025-09-15 17:49:12
+ echo 'Waiting for last oplog chunk (2025-09-15 17:49:12) to be 120 seconds older than starting chunk (2025-09-15 17:49:12)'
Waiting for last oplog chunk (2025-09-15 17:49:12) to be 120 seconds older than starting chunk (2025-09-15 17:49:12)
+ sleep 10
+ [[ 1757958552 -gt 1757958552 ]]
+ [[ 6 -gt 30 ]]
++ get_latest_oplog_chunk_ts some-name
++ local cluster=some-name
+++ kubectl_bin exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ jq '.backups.pitrChunks.pitrChunks | last | .range.end'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.1X0k4ALVoQ
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.ms3X1A7U5B
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in $(seq 0 2)
+++ set +e
+++ kubectl exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.1X0k4ALVoQ
+++ cat /tmp/tmp.ms3X1A7U5B
+++ rm /tmp/tmp.1X0k4ALVoQ /tmp/tmp.ms3X1A7U5B
+++ return 0
++ echo 1757958552
+ latest_ts=1757958552
+ retries=7
++ format_date 1757958552
++ local timestamp=1757958552
+++ TZ=UTC
+++ /usr/sbin/date -d@1757958552 '+%Y-%m-%d %H:%M:%S'
++ echo 2025-09-15 17:49:12
++ format_date 1757958552
++ local timestamp=1757958552
+++ TZ=UTC
+++ /usr/sbin/date -d@1757958552 '+%Y-%m-%d %H:%M:%S'
++ echo 2025-09-15 17:49:12
+ echo 'Waiting for last oplog chunk (2025-09-15 17:49:12) to be 120 seconds older than starting chunk (2025-09-15 17:49:12)'
Waiting for last oplog chunk (2025-09-15 17:49:12) to be 120 seconds older than starting chunk (2025-09-15 17:49:12)
+ sleep 10
+ [[ 1757958552 -gt 1757958552 ]]
+ [[ 7 -gt 30 ]]
++ get_latest_oplog_chunk_ts some-name
++ local cluster=some-name
+++ kubectl_bin exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ jq '.backups.pitrChunks.pitrChunks | last | .range.end'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.4bopPF7rlg
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.bmr06NVcRp
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in $(seq 0 2)
+++ set +e
+++ kubectl exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.4bopPF7rlg
+++ cat /tmp/tmp.bmr06NVcRp
+++ rm /tmp/tmp.4bopPF7rlg /tmp/tmp.bmr06NVcRp
+++ return 0
++ echo 1757958552
+ latest_ts=1757958552
+ retries=8
++ format_date 1757958552
++ local timestamp=1757958552
+++ TZ=UTC
+++ /usr/sbin/date -d@1757958552 '+%Y-%m-%d %H:%M:%S'
++ echo 2025-09-15 17:49:12
++ format_date 1757958552
++ local timestamp=1757958552
+++ TZ=UTC
+++ /usr/sbin/date -d@1757958552 '+%Y-%m-%d %H:%M:%S'
++ echo 2025-09-15 17:49:12
+ echo 'Waiting for last oplog chunk (2025-09-15 17:49:12) to be 120 seconds older than starting chunk (2025-09-15 17:49:12)'
Waiting for last oplog chunk (2025-09-15 17:49:12) to be 120 seconds older than starting chunk (2025-09-15 17:49:12)
+ sleep 10
+ [[ 1757958552 -gt 1757958552 ]]
+ [[ 8 -gt 30 ]]
++ get_latest_oplog_chunk_ts some-name
++ local cluster=some-name
+++ kubectl_bin exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ jq '.backups.pitrChunks.pitrChunks | last | .range.end'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.T1VeC97ePi
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.y0rnsdoyQD
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in $(seq 0 2)
+++ set +e
+++ kubectl exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.T1VeC97ePi
+++ cat /tmp/tmp.y0rnsdoyQD
+++ rm /tmp/tmp.T1VeC97ePi /tmp/tmp.y0rnsdoyQD
+++ return 0
++ echo 1757958552
+ latest_ts=1757958552
+ retries=9
++ format_date 1757958552
++ local timestamp=1757958552
+++ TZ=UTC
+++ /usr/sbin/date -d@1757958552 '+%Y-%m-%d %H:%M:%S'
++ echo 2025-09-15 17:49:12
++ format_date 1757958552
++ local timestamp=1757958552
+++ TZ=UTC
+++ /usr/sbin/date -d@1757958552 '+%Y-%m-%d %H:%M:%S'
++ echo 2025-09-15 17:49:12
+ echo 'Waiting for last oplog chunk (2025-09-15 17:49:12) to be 120 seconds older than starting chunk (2025-09-15 17:49:12)'
Waiting for last oplog chunk (2025-09-15 17:49:12) to be 120 seconds older than starting chunk (2025-09-15 17:49:12)
+ sleep 10
+ [[ 1757958552 -gt 1757958552 ]]
+ [[ 9 -gt 30 ]]
++ get_latest_oplog_chunk_ts some-name
++ local cluster=some-name
+++ kubectl_bin exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ jq '.backups.pitrChunks.pitrChunks | last | .range.end'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.3KH5Klx84L
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.4JYlXDZEeW
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in $(seq 0 2)
+++ set +e
+++ kubectl exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.3KH5Klx84L
+++ cat /tmp/tmp.4JYlXDZEeW
+++ rm /tmp/tmp.3KH5Klx84L /tmp/tmp.4JYlXDZEeW
+++ return 0
++ echo 1757958552
+ latest_ts=1757958552
+ retries=10
++ format_date 1757958552
++ local timestamp=1757958552
+++ TZ=UTC
+++ /usr/sbin/date -d@1757958552 '+%Y-%m-%d %H:%M:%S'
++ echo 2025-09-15 17:49:12
++ format_date 1757958552
++ local timestamp=1757958552
+++ TZ=UTC
+++ /usr/sbin/date -d@1757958552 '+%Y-%m-%d %H:%M:%S'
++ echo 2025-09-15 17:49:12
+ echo 'Waiting for last oplog chunk (2025-09-15 17:49:12) to be 120 seconds older than starting chunk (2025-09-15 17:49:12)'
Waiting for last oplog chunk (2025-09-15 17:49:12) to be 120 seconds older than starting chunk (2025-09-15 17:49:12)
+ sleep 10
+ [[ 1757958552 -gt 1757958552 ]]
+ [[ 10 -gt 30 ]]
++ get_latest_oplog_chunk_ts some-name
++ local cluster=some-name
+++ kubectl_bin exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ jq '.backups.pitrChunks.pitrChunks | last | .range.end'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.w5SfWfLwDq
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.Nt59ddipeL
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in $(seq 0 2)
+++ set +e
+++ kubectl exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.w5SfWfLwDq
+++ cat /tmp/tmp.Nt59ddipeL
+++ rm /tmp/tmp.w5SfWfLwDq /tmp/tmp.Nt59ddipeL
+++ return 0
++ echo 1757958552
+ latest_ts=1757958552
+ retries=11
++ format_date 1757958552
++ local timestamp=1757958552
+++ TZ=UTC
+++ /usr/sbin/date -d@1757958552 '+%Y-%m-%d %H:%M:%S'
++ echo 2025-09-15 17:49:12
++ format_date 1757958552
++ local timestamp=1757958552
+++ TZ=UTC
+++ /usr/sbin/date -d@1757958552 '+%Y-%m-%d %H:%M:%S'
++ echo 2025-09-15 17:49:12
+ echo 'Waiting for last oplog chunk (2025-09-15 17:49:12) to be 120 seconds older than starting chunk (2025-09-15 17:49:12)'
Waiting for last oplog chunk (2025-09-15 17:49:12) to be 120 seconds older than starting chunk (2025-09-15 17:49:12)
+ sleep 10
+ [[ 1757958552 -gt 1757958552 ]]
+ [[ 11 -gt 30 ]]
++ get_latest_oplog_chunk_ts some-name
++ local cluster=some-name
+++ kubectl_bin exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ jq '.backups.pitrChunks.pitrChunks | last | .range.end'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.K0olGpaNRq
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.OzrIZJNnyG
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in $(seq 0 2)
+++ set +e
+++ kubectl exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.K0olGpaNRq
+++ cat /tmp/tmp.OzrIZJNnyG
+++ rm /tmp/tmp.K0olGpaNRq /tmp/tmp.OzrIZJNnyG
+++ return 0
++ echo 1757958552
+ latest_ts=1757958552
+ retries=12
++ format_date 1757958552
++ local timestamp=1757958552
+++ TZ=UTC
+++ /usr/sbin/date -d@1757958552 '+%Y-%m-%d %H:%M:%S'
++ echo 2025-09-15 17:49:12
++ format_date 1757958552
++ local timestamp=1757958552
+++ TZ=UTC
+++ /usr/sbin/date -d@1757958552 '+%Y-%m-%d %H:%M:%S'
++ echo 2025-09-15 17:49:12
+ echo 'Waiting for last oplog chunk (2025-09-15 17:49:12) to be 120 seconds older than starting chunk (2025-09-15 17:49:12)'
Waiting for last oplog chunk (2025-09-15 17:49:12) to be 120 seconds older than starting chunk (2025-09-15 17:49:12)
+ sleep 10
+ [[ 1757958552 -gt 1757958552 ]]
+ [[ 12 -gt 30 ]]
++ get_latest_oplog_chunk_ts some-name
++ local cluster=some-name
+++ kubectl_bin exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ jq '.backups.pitrChunks.pitrChunks | last | .range.end'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.HUICBsMycJ
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.NxZHJJp6hn
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in $(seq 0 2)
+++ set +e
+++ kubectl exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.HUICBsMycJ
+++ cat /tmp/tmp.NxZHJJp6hn
+++ rm /tmp/tmp.HUICBsMycJ /tmp/tmp.NxZHJJp6hn
+++ return 0
++ echo 1757958842
+ latest_ts=1757958842
+ retries=13
++ format_date 1757958842
++ local timestamp=1757958842
+++ TZ=UTC
+++ /usr/sbin/date -d@1757958842 '+%Y-%m-%d %H:%M:%S'
++ echo 2025-09-15 17:54:02
++ format_date 1757958552
++ local timestamp=1757958552
+++ TZ=UTC
+++ /usr/sbin/date -d@1757958552 '+%Y-%m-%d %H:%M:%S'
++ echo 2025-09-15 17:49:12
+ echo 'Waiting for last oplog chunk (2025-09-15 17:54:02) to be 120 seconds older than starting chunk (2025-09-15 17:49:12)'
Waiting for last oplog chunk (2025-09-15 17:54:02) to be 120 seconds older than starting chunk (2025-09-15 17:49:12)
+ sleep 10
+ [[ 1757958842 -gt 1757958552 ]]
+ '[' -z '' ']'
+ desc 'check restore by latest'
+ set +o xtrace


-----------------------------------------------------------------------------------
check restore by latest
-----------------------------------------------------------------------------------

+ cat /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2039/e2e-tests/pitr-sharded/conf/restore.yml
+ /usr/sbin/sed -e 's/name:/name: restore-backup-minio-1/'
+ /usr/sbin/sed -e 's/backupName:/backupName: backup-minio-1/'
+ /usr/sbin/sed -e /backupSource/,+8d
+ /usr/sbin/sed -e 's/pitrType:/type: latest/'
+ '[' -z '' ']'
+ /usr/sbin/sed -e /date:/d
+ kubectl_bin apply -f -
++ mktemp
+ local LAST_OUT=/tmp/tmp.FrEF5w8j3m
++ mktemp
+ local LAST_ERR=/tmp/tmp.OdV6f2ndEr
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl apply -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.FrEF5w8j3m
perconaservermongodbrestore.psmdb.percona.com/restore-backup-minio-1 created
+ cat /tmp/tmp.OdV6f2ndEr
+ rm /tmp/tmp.FrEF5w8j3m /tmp/tmp.OdV6f2ndEr
+ return 0
+ wait_restore backup-minio-1 some-name requested 0 1200
+ local backup_name=backup-minio-1
+ local cluster_name=some-name
+ local target_state=requested
+ local wait_cluster_consistency=0
+ local wait_time=1200
+ local ok_if_ready=0
+ set +o xtrace
Waiting for the psmdb-restore/restore-backup-minio-1 object to be createdOK
Waiting psmdb-restore/restore-backup-minio-1 to reach state "requested" .OK after 1 minutes
+ [[ 0 -eq 1 ]]
+ echo

+ wait_restore backup-minio-1 some-name ready 0 1600
+ local backup_name=backup-minio-1
+ local cluster_name=some-name
+ local target_state=ready
+ local wait_cluster_consistency=0
+ local wait_time=1600
+ local ok_if_ready=0
+ set +o xtrace
Waiting for the psmdb-restore/restore-backup-minio-1 object to be createdOK
Waiting psmdb-restore/restore-backup-minio-1 to reach state "ready" OK after 0 minutes
+ [[ 0 -eq 1 ]]
+ echo

+ set -o xtrace
+ wait_for_running some-name-rs0 3
+ local name=some-name-rs0
+ let last_pod=2
+ local check_cluster_readyness=true
+ set_debug
+ [[ 1 == 1 ]]
+ set -o xtrace
+ local rs_name=rs0
+ local cluster_name=some-name
++ seq 0 2
+ for i in $(seq 0 $last_pod)
+ [[ 0 -eq 2 ]]
+ wait_pod some-name-rs0-0
+ local pod=some-name-rs0-0
+ set +o xtrace
waiting for pod/some-name-rs0-0 to be ready.OK
+ for i in $(seq 0 $last_pod)
+ [[ 1 -eq 2 ]]
+ wait_pod some-name-rs0-1
+ local pod=some-name-rs0-1
+ set +o xtrace
waiting for pod/some-name-rs0-1 to be ready.OK
+ for i in $(seq 0 $last_pod)
+ [[ 2 -eq 2 ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].arbiter.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.f06zNU4Ys9
+++ mktemp
++ local LAST_ERR=/tmp/tmp.fXfn1Olk89
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].arbiter.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.f06zNU4Ys9
++ cat /tmp/tmp.fXfn1Olk89
++ rm /tmp/tmp.f06zNU4Ys9 /tmp/tmp.fXfn1Olk89
++ return 0
+ [[ '' == \t\r\u\e ]]
+ wait_pod some-name-rs0-2
+ local pod=some-name-rs0-2
+ set +o xtrace
waiting for pod/some-name-rs0-2 to be ready.OK
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].non_voting.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.hxRwyA6WJv
+++ mktemp
++ local LAST_ERR=/tmp/tmp.kQ1chBaYaH
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].non_voting.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.hxRwyA6WJv
++ cat /tmp/tmp.kQ1chBaYaH
++ rm /tmp/tmp.hxRwyA6WJv /tmp/tmp.kQ1chBaYaH
++ return 0
+ [[ '' == \t\r\u\e ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].hidden.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.mGFumFe9c6
+++ mktemp
++ local LAST_ERR=/tmp/tmp.B6lbHY8f4r
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].hidden.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.mGFumFe9c6
++ cat /tmp/tmp.B6lbHY8f4r
++ rm /tmp/tmp.mGFumFe9c6 /tmp/tmp.B6lbHY8f4r
++ return 0
+ [[ '' == \t\r\u\e ]]
+ sleep 10
+ [[ true == \t\r\u\e ]]
+ set +x
Waiting for cluster readyness.........
+ wait_for_running some-name-cfg 3
+ local name=some-name-cfg
+ let last_pod=2
+ local check_cluster_readyness=true
+ set_debug
+ [[ 1 == 1 ]]
+ set -o xtrace
+ local rs_name=cfg
+ local cluster_name=some-name
++ seq 0 2
+ for i in $(seq 0 $last_pod)
+ [[ 0 -eq 2 ]]
+ wait_pod some-name-cfg-0
+ local pod=some-name-cfg-0
+ set +o xtrace
waiting for pod/some-name-cfg-0 to be ready.OK
+ for i in $(seq 0 $last_pod)
+ [[ 1 -eq 2 ]]
+ wait_pod some-name-cfg-1
+ local pod=some-name-cfg-1
+ set +o xtrace
waiting for pod/some-name-cfg-1 to be ready.OK
+ for i in $(seq 0 $last_pod)
+ [[ 2 -eq 2 ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].arbiter.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.XsD2oZ1Teu
+++ mktemp
++ local LAST_ERR=/tmp/tmp.hPbQCrvVJV
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].arbiter.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.XsD2oZ1Teu
++ cat /tmp/tmp.hPbQCrvVJV
++ rm /tmp/tmp.XsD2oZ1Teu /tmp/tmp.hPbQCrvVJV
++ return 0
+ [[ '' == \t\r\u\e ]]
+ wait_pod some-name-cfg-2
+ local pod=some-name-cfg-2
+ set +o xtrace
waiting for pod/some-name-cfg-2 to be ready.OK
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].non_voting.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.cKy7RLTlP8
+++ mktemp
++ local LAST_ERR=/tmp/tmp.nBelChj9Q7
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].non_voting.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.cKy7RLTlP8
++ cat /tmp/tmp.nBelChj9Q7
++ rm /tmp/tmp.cKy7RLTlP8 /tmp/tmp.nBelChj9Q7
++ return 0
+ [[ '' == \t\r\u\e ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].hidden.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.xyNMuolSkF
+++ mktemp
++ local LAST_ERR=/tmp/tmp.FXNIZENtFF
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].hidden.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.xyNMuolSkF
++ cat /tmp/tmp.FXNIZENtFF
++ rm /tmp/tmp.xyNMuolSkF /tmp/tmp.FXNIZENtFF
++ return 0
+ [[ '' == \t\r\u\e ]]
+ sleep 10
+ [[ true == \t\r\u\e ]]
+ set +x
Waiting for cluster readyness
+ wait_for_running some-name-mongos 3
+ local name=some-name-mongos
+ let last_pod=2
+ local check_cluster_readyness=true
+ set_debug
+ [[ 1 == 1 ]]
+ set -o xtrace
+ local rs_name=mongos
+ local cluster_name=some-name
++ seq 0 2
+ for i in $(seq 0 $last_pod)
+ [[ 0 -eq 2 ]]
+ wait_pod some-name-mongos-0
+ local pod=some-name-mongos-0
+ set +o xtrace
waiting for pod/some-name-mongos-0 to be ready.OK
+ for i in $(seq 0 $last_pod)
+ [[ 1 -eq 2 ]]
+ wait_pod some-name-mongos-1
+ local pod=some-name-mongos-1
+ set +o xtrace
waiting for pod/some-name-mongos-1 to be ready.OK
+ for i in $(seq 0 $last_pod)
+ [[ 2 -eq 2 ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="mongos")].arbiter.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.Byefil4rYF
+++ mktemp
++ local LAST_ERR=/tmp/tmp.QDlBmFtbJv
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="mongos")].arbiter.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.Byefil4rYF
++ cat /tmp/tmp.QDlBmFtbJv
++ rm /tmp/tmp.Byefil4rYF /tmp/tmp.QDlBmFtbJv
++ return 0
+ [[ '' == \t\r\u\e ]]
+ wait_pod some-name-mongos-2
+ local pod=some-name-mongos-2
+ set +o xtrace
waiting for pod/some-name-mongos-2 to be ready.OK
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="mongos")].non_voting.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.j44LTHHXHe
+++ mktemp
++ local LAST_ERR=/tmp/tmp.VXsQ7Rze22
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="mongos")].non_voting.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.j44LTHHXHe
++ cat /tmp/tmp.VXsQ7Rze22
++ rm /tmp/tmp.j44LTHHXHe /tmp/tmp.VXsQ7Rze22
++ return 0
+ [[ '' == \t\r\u\e ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="mongos")].hidden.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.h1G1CREx2l
+++ mktemp
++ local LAST_ERR=/tmp/tmp.Moj7f9IVPU
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="mongos")].hidden.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.h1G1CREx2l
++ cat /tmp/tmp.Moj7f9IVPU
++ rm /tmp/tmp.h1G1CREx2l /tmp/tmp.Moj7f9IVPU
++ return 0
+ [[ '' == \t\r\u\e ]]
+ sleep 10
+ [[ true == \t\r\u\e ]]
+ set +x
Waiting for cluster readyness
+ sleep 10
+ compare_mongos_cmd find myApp:myPass@some-name-mongos.pitr-sharded-3057 -3rd
+ local command=find
+ local uri=myApp:myPass@some-name-mongos.pitr-sharded-3057
+ local postfix=-3rd
+ local suffix=
+ local database=myApp
+ local collection=test
+ local port=27017
+ local tls=false
+ [[ false == \t\r\u\e ]]
+ mongos_command=run_mongos
+ run_mongos 'use myApp\n db.test.find()' myApp:myPass@some-name-mongos.pitr-sharded-3057 mongodb '' '' 27017
+ egrep -v 'I NETWORK|W NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|Started a new thread for the timer service'
+ /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
+ local 'command=use myApp\n db.test.find()'
+ local uri=myApp:myPass@some-name-mongos.pitr-sharded-3057
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local port=27017
+ local mongo_bin=mongo
++ echo .svc.cluster.local
++ awk -F: '{print $2}'
egrep: warning: egrep is obsolescent; using grep -E
+ suffix_port=
+ [[ -z '' ]]
+ suffix=.svc.cluster.local:27017
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.NKeXm8IbxP
+++ mktemp
++ local LAST_ERR=/tmp/tmp.FwbC8zmbtB
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.NKeXm8IbxP
++ cat /tmp/tmp.FwbC8zmbtB
++ rm /tmp/tmp.NKeXm8IbxP /tmp/tmp.FwbC8zmbtB
++ return 0
+ local client_container=psmdb-client-66f577db5f-ckxq9
+ kubectl_bin exec psmdb-client-66f577db5f-ckxq9 -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.pitr-sharded-3057.svc.cluster.local:27017/admin '
++ mktemp
+ local LAST_OUT=/tmp/tmp.yFZmqB5wmu
++ mktemp
+ local LAST_ERR=/tmp/tmp.eyFAAjjsGk
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-66f577db5f-ckxq9 -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.pitr-sharded-3057.svc.cluster.local:27017/admin '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.yFZmqB5wmu
+ cat /tmp/tmp.eyFAAjjsGk
+ rm /tmp/tmp.yFZmqB5wmu /tmp/tmp.eyFAAjjsGk
+ return 0
+ [[ 0 -eq 0 ]]
+ diff /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2039/e2e-tests/pitr-sharded/compare/find-3rd.json /tmp/tmp.J3Hag6NYlY/find-3rd
+ compare_latest_restorable_time some-name-rs0 backup-minio-1
+ local cluster=some-name-rs0
+ local backup_name=backup-minio-1
+ local latest_restorable_time
+ local backup_time
++ get_latest_restorable_time some-name-rs0
++ local cluster=some-name-rs0
++ local first_timestamp
++ local second_timestamp
++ local retry=0
++ [[ '' != '' ]]
+++ kubectl_bin exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ jq '.backups.pitrChunks.pitrChunks | last | .range.end'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.S1fmMvRJTi
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.q28woKsQXE
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in $(seq 0 2)
+++ set +e
+++ kubectl exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.S1fmMvRJTi
+++ cat /tmp/tmp.q28woKsQXE
+++ rm /tmp/tmp.S1fmMvRJTi /tmp/tmp.q28woKsQXE
+++ return 0
++ first_timestamp=1757958950
++ sleep 5
++ [[ 1757958950 != '' ]]
++ [[ 1757958950 != \n\u\l\l ]]
+++ kubectl_bin exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ jq '.backups.pitrChunks.pitrChunks | last | .range.end'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.cHk6MV7ERK
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.kOzkIjJmyE
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in $(seq 0 2)
+++ set +e
+++ kubectl exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.cHk6MV7ERK
+++ cat /tmp/tmp.kOzkIjJmyE
+++ rm /tmp/tmp.cHk6MV7ERK /tmp/tmp.kOzkIjJmyE
+++ return 0
++ second_timestamp=1757958950
++ let retry+=1
++ [[ 1 -gt 30 ]]
++ [[ 1757958950 != '' ]]
++ [[ 1757958950 != \n\u\l\l ]]
++ [[ 1757958950 == 1757958950 ]]
++ /usr/sbin/date -u -d @1757958950 +%Y-%m-%dT%H:%M:%SZ
+ latest_restorable_time=2025-09-15T17:55:50Z
++ get_latest_restorable_time_from_backup_object backup-minio-1
++ local backup_name=backup-minio-1
++ local latestRestorableTime
++ local retry=0
++ [[ '' != '' ]]
++ sleep 5
+++ kubectl_bin get psmdb-backup backup-minio-1 -o 'jsonpath={.status.latestRestorableTime}'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.u9y4JWp43B
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.enU4GtIeHB
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in $(seq 0 2)
+++ set +e
+++ kubectl get psmdb-backup backup-minio-1 -o 'jsonpath={.status.latestRestorableTime}'
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.u9y4JWp43B
+++ cat /tmp/tmp.enU4GtIeHB
+++ rm /tmp/tmp.u9y4JWp43B /tmp/tmp.enU4GtIeHB
+++ return 0
++ latestRestorableTime=2025-09-15T17:55:50Z
++ let retry+=1
++ [[ 1 -gt 30 ]]
++ [[ 2025-09-15T17:55:50Z != '' ]]
++ [[ 2025-09-15T17:55:50Z != \n\u\l\l ]]
++ echo 2025-09-15T17:55:50Z
+ backup_time=2025-09-15T17:55:50Z
+ [[ 2025-09-15T17:55:50Z != \2\0\2\5\-\0\9\-\1\5\T\1\7\:\5\5\:\5\0\Z ]]
+ desc 'delete custom RuntimeClass'
+ set +o xtrace


-----------------------------------------------------------------------------------
delete custom RuntimeClass
-----------------------------------------------------------------------------------

+ kubectl_bin delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2039/e2e-tests/conf/container-rc.yaml
++ mktemp
+ local LAST_OUT=/tmp/tmp.0XIxzwAtZu
++ mktemp
+ local LAST_ERR=/tmp/tmp.QezymJQIKY
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2039/e2e-tests/conf/container-rc.yaml
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.0XIxzwAtZu
runtimeclass.node.k8s.io "container-rc" deleted
+ cat /tmp/tmp.QezymJQIKY
+ rm /tmp/tmp.0XIxzwAtZu /tmp/tmp.QezymJQIKY
+ return 0
+ destroy pitr-sharded-3057
+ local namespace=pitr-sharded-3057
+ local ignore_logs=true
+ [[ 0 == 1 ]]
+ desc 'destroy cluster/operator and all other resources'
+ set +o xtrace


-----------------------------------------------------------------------------------
destroy cluster/operator and all other resources
-----------------------------------------------------------------------------------

+ '[' true == false ']'
+ delete_backups
+ desc 'Delete psmdb-backup'
+ set +o xtrace


-----------------------------------------------------------------------------------
Delete psmdb-backup
-----------------------------------------------------------------------------------

++ kubectl_bin get psmdb-backup --no-headers
++ wc -l
+++ mktemp
++ local LAST_OUT=/tmp/tmp.BA4Krrp8Eg
+++ mktemp
++ local LAST_ERR=/tmp/tmp.SFSL4vgCzR
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb-backup --no-headers
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.BA4Krrp8Eg
++ cat /tmp/tmp.SFSL4vgCzR
++ rm /tmp/tmp.BA4Krrp8Eg /tmp/tmp.SFSL4vgCzR
++ return 0
+ '[' 2 '!=' 0 ']'
+ kubectl_bin get psmdb-backup
++ mktemp
+ local LAST_OUT=/tmp/tmp.5nn39m4C6J
++ mktemp
+ local LAST_ERR=/tmp/tmp.jnT1qPi5Jg
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl get psmdb-backup
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.5nn39m4C6J
NAME             CLUSTER     STORAGE   DESTINATION                                  TYPE      SIZE       STATUS   COMPLETED   AGE
backup-minio-0   some-name   minio     s3://operator-testing/2025-09-15T17:42:43Z   logical   150.69KB   ready    15m         15m
backup-minio-1   some-name   minio     s3://operator-testing/2025-09-15T17:51:16Z   logical   218.42KB   ready    6m33s       7m
+ cat /tmp/tmp.jnT1qPi5Jg
+ rm /tmp/tmp.5nn39m4C6J /tmp/tmp.jnT1qPi5Jg
+ return 0
+ kubectl_bin delete psmdb-backup --all
++ mktemp
+ local LAST_OUT=/tmp/tmp.8DO7iVLdyv
++ mktemp
+ local LAST_ERR=/tmp/tmp.6KmjE7LPKN
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl delete psmdb-backup --all
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.8DO7iVLdyv
perconaservermongodbbackup.psmdb.percona.com "backup-minio-0" deleted from pitr-sharded-3057 namespace
perconaservermongodbbackup.psmdb.percona.com "backup-minio-1" deleted from pitr-sharded-3057 namespace
+ cat /tmp/tmp.6KmjE7LPKN
+ rm /tmp/tmp.8DO7iVLdyv /tmp/tmp.6KmjE7LPKN
+ return 0
+ delete_crd
+ desc 'get and delete old CRDs and RBAC'
+ set +o xtrace


-----------------------------------------------------------------------------------
get and delete old CRDs and RBAC
-----------------------------------------------------------------------------------

+ kubectl_bin delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2039/deploy/crd.yaml --ignore-not-found --wait=false
++ mktemp
+ local LAST_OUT=/tmp/tmp.pTLoh845Qb
++ mktemp
+ local LAST_ERR=/tmp/tmp.7iD8zNlJNi
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2039/deploy/crd.yaml --ignore-not-found --wait=false
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.pTLoh845Qb
customresourcedefinition.apiextensions.k8s.io "perconaservermongodbbackups.psmdb.percona.com" deleted
customresourcedefinition.apiextensions.k8s.io "perconaservermongodbrestores.psmdb.percona.com" deleted
customresourcedefinition.apiextensions.k8s.io "perconaservermongodbs.psmdb.percona.com" deleted
+ cat /tmp/tmp.7iD8zNlJNi
+ rm /tmp/tmp.pTLoh845Qb /tmp/tmp.7iD8zNlJNi
+ return 0
++ yq eval .metadata.name /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2039/deploy/crd.yaml
++ grep -v '\-\-\-'
grep: warning: stray \ before -
grep: warning: stray \ before -
+ for crd_name in $(yq eval '.metadata.name' "${src_dir}/deploy/crd.yaml" | grep -v '\-\-\-')
+ kubectl get perconaservermongodbbackups.psmdb.percona.com --all-namespaces -o wide
+ grep -v NAMESPACE
+ xargs -L 1 sh -xc 'kubectl patch perconaservermongodbbackups.psmdb.percona.com -n $0 $1 --type=merge -p "{\"metadata\":{\"finalizers\":[]}}"'
error: the server doesn't have a resource type "perconaservermongodbbackups"
+ kubectl patch perconaservermongodbbackups.psmdb.percona.com -n sh --type=merge -p '{"metadata":{"finalizers":[]}}'
error: the server doesn't have a resource type "perconaservermongodbbackups"
+ :
+ kubectl_bin wait --for=delete crd perconaservermongodbbackups.psmdb.percona.com
++ mktemp
+ local LAST_OUT=/tmp/tmp.unzc9Wmr7h
++ mktemp
+ local LAST_ERR=/tmp/tmp.F86Pdx8F68
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl wait --for=delete crd perconaservermongodbbackups.psmdb.percona.com
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.unzc9Wmr7h
+ cat /tmp/tmp.F86Pdx8F68
+ rm /tmp/tmp.unzc9Wmr7h /tmp/tmp.F86Pdx8F68
+ return 0
+ for crd_name in $(yq eval '.metadata.name' "${src_dir}/deploy/crd.yaml" | grep -v '\-\-\-')
+ kubectl get perconaservermongodbrestores.psmdb.percona.com --all-namespaces -o wide
+ grep -v NAMESPACE
+ xargs -L 1 sh -xc 'kubectl patch perconaservermongodbrestores.psmdb.percona.com -n $0 $1 --type=merge -p "{\"metadata\":{\"finalizers\":[]}}"'
error: the server doesn't have a resource type "perconaservermongodbrestores"
+ kubectl patch perconaservermongodbrestores.psmdb.percona.com -n sh --type=merge -p '{"metadata":{"finalizers":[]}}'
error: the server doesn't have a resource type "perconaservermongodbrestores"
+ :
+ kubectl_bin wait --for=delete crd perconaservermongodbrestores.psmdb.percona.com
++ mktemp
+ local LAST_OUT=/tmp/tmp.RQZgyihSe5
++ mktemp
+ local LAST_ERR=/tmp/tmp.HE3VzC9cHN
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl wait --for=delete crd perconaservermongodbrestores.psmdb.percona.com
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.RQZgyihSe5
+ cat /tmp/tmp.HE3VzC9cHN
+ rm /tmp/tmp.RQZgyihSe5 /tmp/tmp.HE3VzC9cHN
+ return 0
+ for crd_name in $(yq eval '.metadata.name' "${src_dir}/deploy/crd.yaml" | grep -v '\-\-\-')
+ kubectl get perconaservermongodbs.psmdb.percona.com --all-namespaces -o wide
+ grep -v NAMESPACE
+ xargs -L 1 sh -xc 'kubectl patch perconaservermongodbs.psmdb.percona.com -n $0 $1 --type=merge -p "{\"metadata\":{\"finalizers\":[]}}"'
error: the server doesn't have a resource type "perconaservermongodbs"
+ kubectl patch perconaservermongodbs.psmdb.percona.com -n sh --type=merge -p '{"metadata":{"finalizers":[]}}'
error: the server doesn't have a resource type "perconaservermongodbs"
+ :
+ kubectl_bin wait --for=delete crd perconaservermongodbs.psmdb.percona.com
++ mktemp
+ local LAST_OUT=/tmp/tmp.MN8YhUeYlQ
++ mktemp
+ local LAST_ERR=/tmp/tmp.KhO4y3CXc0
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl wait --for=delete crd perconaservermongodbs.psmdb.percona.com
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.MN8YhUeYlQ
+ cat /tmp/tmp.KhO4y3CXc0
+ rm /tmp/tmp.MN8YhUeYlQ /tmp/tmp.KhO4y3CXc0
+ return 0
+ local rbac_yaml=rbac.yaml
+ '[' -n psmdb-operator ']'
+ rbac_yaml=cw-rbac.yaml
+ kubectl_bin delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2039/deploy/cw-rbac.yaml --ignore-not-found
++ mktemp
+ local LAST_OUT=/tmp/tmp.2Ar7WM2gkG
++ mktemp
+ local LAST_ERR=/tmp/tmp.FJgIMclecD
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2039/deploy/cw-rbac.yaml --ignore-not-found
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.2Ar7WM2gkG
clusterrole.rbac.authorization.k8s.io "percona-server-mongodb-operator" deleted
clusterrolebinding.rbac.authorization.k8s.io "service-account-percona-server-mongodb-operator" deleted
+ cat /tmp/tmp.FJgIMclecD
+ rm /tmp/tmp.2Ar7WM2gkG /tmp/tmp.FJgIMclecD
+ return 0
+ destroy_cert_manager
+ kubectl_bin delete -f https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml
++ mktemp
+ local LAST_OUT=/tmp/tmp.7hdYmYKHY2
++ mktemp
+ local LAST_ERR=/tmp/tmp.5Qua4SKFzA
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl delete -f https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml
+ exit_status=1
+ set -e
+ '[' 1 '!=' 0 -a -n 1 ']'
+ cat /tmp/tmp.7hdYmYKHY2
+ cat /tmp/tmp.5Qua4SKFzA
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": namespaces "cert-manager" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "certificaterequests.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "certificates.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "challenges.acme.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "clusterissuers.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "issuers.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "orders.acme.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": serviceaccounts "cert-manager-cainjector" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": serviceaccounts "cert-manager" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": serviceaccounts "cert-manager-webhook" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-cainjector" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-issuers" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-clusterissuers" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-certificates" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-orders" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-challenges" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-ingress-shim" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-cluster-view" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-view" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-edit" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-approve:cert-manager-io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-certificatesigningrequests" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-webhook:subjectaccessreviews" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-cainjector" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-issuers" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-clusterissuers" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-certificates" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-orders" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-challenges" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-ingress-shim" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-approve:cert-manager-io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-certificatesigningrequests" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-webhook:subjectaccessreviews" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager-cainjector:leaderelection" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager:leaderelection" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager-tokenrequest" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager-webhook:dynamic-serving" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager-cainjector:leaderelection" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager:leaderelection" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager-cert-manager-tokenrequest" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager-webhook:dynamic-serving" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": services "cert-manager-cainjector" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": services "cert-manager" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": services "cert-manager-webhook" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": deployments.apps "cert-manager-cainjector" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": deployments.apps "cert-manager" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": deployments.apps "cert-manager-webhook" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": mutatingwebhookconfigurations.admissionregistration.k8s.io "cert-manager-webhook" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": validatingwebhookconfigurations.admissionregistration.k8s.io "cert-manager-webhook" not found
+ sleep 0
+ for i in $(seq 0 2)
+ set +e
+ kubectl delete -f https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml
+ exit_status=1
+ set -e
+ '[' 1 '!=' 0 -a -n 1 ']'
+ cat /tmp/tmp.7hdYmYKHY2
+ cat /tmp/tmp.5Qua4SKFzA
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": namespaces "cert-manager" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "certificaterequests.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "certificates.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "challenges.acme.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "clusterissuers.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "issuers.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "orders.acme.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": serviceaccounts "cert-manager-cainjector" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": serviceaccounts "cert-manager" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": serviceaccounts "cert-manager-webhook" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-cainjector" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-issuers" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-clusterissuers" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-certificates" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-orders" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-challenges" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-ingress-shim" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-cluster-view" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-view" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-edit" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-approve:cert-manager-io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-certificatesigningrequests" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-webhook:subjectaccessreviews" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-cainjector" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-issuers" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-clusterissuers" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-certificates" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-orders" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-challenges" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-ingress-shim" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-approve:cert-manager-io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-certificatesigningrequests" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-webhook:subjectaccessreviews" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager-cainjector:leaderelection" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager:leaderelection" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager-tokenrequest" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager-webhook:dynamic-serving" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager-cainjector:leaderelection" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager:leaderelection" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager-cert-manager-tokenrequest" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager-webhook:dynamic-serving" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": services "cert-manager-cainjector" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": services "cert-manager" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": services "cert-manager-webhook" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": deployments.apps "cert-manager-cainjector" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": deployments.apps "cert-manager" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": deployments.apps "cert-manager-webhook" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": mutatingwebhookconfigurations.admissionregistration.k8s.io "cert-manager-webhook" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": validatingwebhookconfigurations.admissionregistration.k8s.io "cert-manager-webhook" not found
+ sleep 4
+ for i in $(seq 0 2)
+ set +e
+ kubectl delete -f https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml
+ exit_status=1
+ set -e
+ '[' 1 '!=' 0 -a -n 1 ']'
+ cat /tmp/tmp.7hdYmYKHY2
+ cat /tmp/tmp.5Qua4SKFzA
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": namespaces "cert-manager" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "certificaterequests.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "certificates.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "challenges.acme.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "clusterissuers.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "issuers.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "orders.acme.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": serviceaccounts "cert-manager-cainjector" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": serviceaccounts "cert-manager" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": serviceaccounts "cert-manager-webhook" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-cainjector" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-issuers" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-clusterissuers" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-certificates" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-orders" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-challenges" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-ingress-shim" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-cluster-view" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-view" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-edit" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-approve:cert-manager-io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-certificatesigningrequests" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-webhook:subjectaccessreviews" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-cainjector" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-issuers" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-clusterissuers" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-certificates" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-orders" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-challenges" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-ingress-shim" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-approve:cert-manager-io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-certificatesigningrequests" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-webhook:subjectaccessreviews" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager-cainjector:leaderelection" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager:leaderelection" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager-tokenrequest" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager-webhook:dynamic-serving" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager-cainjector:leaderelection" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager:leaderelection" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager-cert-manager-tokenrequest" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager-webhook:dynamic-serving" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": services "cert-manager-cainjector" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": services "cert-manager" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": services "cert-manager-webhook" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": deployments.apps "cert-manager-cainjector" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": deployments.apps "cert-manager" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": deployments.apps "cert-manager-webhook" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": mutatingwebhookconfigurations.admissionregistration.k8s.io "cert-manager-webhook" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": validatingwebhookconfigurations.admissionregistration.k8s.io "cert-manager-webhook" not found
+ sleep 8
+ cat /tmp/tmp.7hdYmYKHY2
+ cat /tmp/tmp.5Qua4SKFzA
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": namespaces "cert-manager" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "certificaterequests.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "certificates.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "challenges.acme.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "clusterissuers.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "issuers.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "orders.acme.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": serviceaccounts "cert-manager-cainjector" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": serviceaccounts "cert-manager" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": serviceaccounts "cert-manager-webhook" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-cainjector" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-issuers" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-clusterissuers" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-certificates" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-orders" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-challenges" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-ingress-shim" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-cluster-view" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-view" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-edit" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-approve:cert-manager-io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-certificatesigningrequests" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-webhook:subjectaccessreviews" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-cainjector" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-issuers" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-clusterissuers" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-certificates" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-orders" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-challenges" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-ingress-shim" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-approve:cert-manager-io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-certificatesigningrequests" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-webhook:subjectaccessreviews" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager-cainjector:leaderelection" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager:leaderelection" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager-tokenrequest" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager-webhook:dynamic-serving" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager-cainjector:leaderelection" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager:leaderelection" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager-cert-manager-tokenrequest" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager-webhook:dynamic-serving" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": services "cert-manager-cainjector" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": services "cert-manager" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": services "cert-manager-webhook" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": deployments.apps "cert-manager-cainjector" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": deployments.apps "cert-manager" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": deployments.apps "cert-manager-webhook" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": mutatingwebhookconfigurations.admissionregistration.k8s.io "cert-manager-webhook" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": validatingwebhookconfigurations.admissionregistration.k8s.io "cert-manager-webhook" not found
+ rm /tmp/tmp.7hdYmYKHY2 /tmp/tmp.5Qua4SKFzA
+ return 1
+ true
+ '[' -n '' ']'
+ '[' -n psmdb-operator ']'
+ kubectl_bin delete --grace-period=0 --force=true namespace pitr-sharded-3057
+ rm -rf /tmp/tmp.J3Hag6NYlY
+ kubectl_bin delete --grace-period=0 --force=true namespace psmdb-operator
++ mktemp
++ mktemp
+ desc 'test passed'
+ set +o xtrace


-----------------------------------------------------------------------------------
test passed
-----------------------------------------------------------------------------------

+ local LAST_OUT=/tmp/tmp.uKlRzLnAvo
+ local LAST_OUT=/tmp/tmp.ubaBNArPxN
++ mktemp
++ mktemp
+ local LAST_ERR=/tmp/tmp.1Mr2W9gtky
+ local LAST_ERR=/tmp/tmp.AT96UipXXu
+ local exit_status=0
+ local timeout=4
+ local exit_status=0
+ local timeout=4
++ seq 0 2
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ for i in $(seq 0 2)
+ kubectl delete --grace-period=0 --force=true namespace pitr-sharded-3057
+ set +e
+ kubectl delete --grace-period=0 --force=true namespace psmdb-operator