Log: /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2219/e2e-tests/logs/multi-storage.log
Warning: version difference between client (1.35) and server (1.32) exceeds the supported minor version skew of +/-1
Warning: version difference between client (1.35) and server (1.32) exceeds the supported minor version skew of +/-1
Warning: version difference between client (1.35) and server (1.32) exceeds the supported minor version skew of +/-1
+ cluster=some-name
+ create_infra multi-storage-10969
+ local ns=multi-storage-10969
+ [[ 1 == 1 ]]
+ delete_crd
+ desc 'get and delete old CRDs and RBAC'
+ set +o xtrace


-----------------------------------------------------------------------------------
get and delete old CRDs and RBAC
-----------------------------------------------------------------------------------

+ kubectl_bin delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2219/deploy/crd.yaml --ignore-not-found --wait=false
++ mktemp
+ local LAST_OUT=/tmp/tmp.6utpc2QBL6
++ mktemp
+ local LAST_ERR=/tmp/tmp.vsOIg1q8O9
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2219/deploy/crd.yaml --ignore-not-found --wait=false
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.6utpc2QBL6
+ cat /tmp/tmp.vsOIg1q8O9
+ rm /tmp/tmp.6utpc2QBL6 /tmp/tmp.vsOIg1q8O9
+ return 0
++ yq eval .metadata.name /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2219/deploy/crd.yaml
++ grep -v '\-\-\-'
grep: warning: stray \ before -
grep: warning: stray \ before -
+ for crd_name in $(yq eval '.metadata.name' "${src_dir}/deploy/crd.yaml" | grep -v '\-\-\-')
+ kubectl get perconaservermongodbbackups.psmdb.percona.com --all-namespaces -o wide
+ grep -v NAMESPACE
+ xargs -L 1 sh -xc 'kubectl patch perconaservermongodbbackups.psmdb.percona.com -n $0 $1 --type=merge -p "{\"metadata\":{\"finalizers\":[]}}"'
error: the server doesn't have a resource type "perconaservermongodbbackups"
+ kubectl patch perconaservermongodbbackups.psmdb.percona.com -n sh --type=merge -p '{"metadata":{"finalizers":[]}}'
error: the server doesn't have a resource type "perconaservermongodbbackups"
+ :
+ kubectl_bin wait --for=delete crd perconaservermongodbbackups.psmdb.percona.com
++ mktemp
+ local LAST_OUT=/tmp/tmp.8hO76RJ8lN
++ mktemp
+ local LAST_ERR=/tmp/tmp.GZ79ZDV7Ol
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl wait --for=delete crd perconaservermongodbbackups.psmdb.percona.com
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.8hO76RJ8lN
+ cat /tmp/tmp.GZ79ZDV7Ol
+ rm /tmp/tmp.8hO76RJ8lN /tmp/tmp.GZ79ZDV7Ol
+ return 0
+ for crd_name in $(yq eval '.metadata.name' "${src_dir}/deploy/crd.yaml" | grep -v '\-\-\-')
+ kubectl get perconaservermongodbrestores.psmdb.percona.com --all-namespaces -o wide
+ grep -v NAMESPACE
+ xargs -L 1 sh -xc 'kubectl patch perconaservermongodbrestores.psmdb.percona.com -n $0 $1 --type=merge -p "{\"metadata\":{\"finalizers\":[]}}"'
error: the server doesn't have a resource type "perconaservermongodbrestores"
+ kubectl patch perconaservermongodbrestores.psmdb.percona.com -n sh --type=merge -p '{"metadata":{"finalizers":[]}}'
error: the server doesn't have a resource type "perconaservermongodbrestores"
+ :
+ kubectl_bin wait --for=delete crd perconaservermongodbrestores.psmdb.percona.com
++ mktemp
+ local LAST_OUT=/tmp/tmp.piDTcUg4NQ
++ mktemp
+ local LAST_ERR=/tmp/tmp.X6qTtTTC0J
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl wait --for=delete crd perconaservermongodbrestores.psmdb.percona.com
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.piDTcUg4NQ
+ cat /tmp/tmp.X6qTtTTC0J
+ rm /tmp/tmp.piDTcUg4NQ /tmp/tmp.X6qTtTTC0J
+ return 0
+ for crd_name in $(yq eval '.metadata.name' "${src_dir}/deploy/crd.yaml" | grep -v '\-\-\-')
+ kubectl get perconaservermongodbs.psmdb.percona.com --all-namespaces -o wide
+ grep -v NAMESPACE
+ xargs -L 1 sh -xc 'kubectl patch perconaservermongodbs.psmdb.percona.com -n $0 $1 --type=merge -p "{\"metadata\":{\"finalizers\":[]}}"'
error: the server doesn't have a resource type "perconaservermongodbs"
+ kubectl patch perconaservermongodbs.psmdb.percona.com -n sh --type=merge -p '{"metadata":{"finalizers":[]}}'
error: the server doesn't have a resource type "perconaservermongodbs"
+ :
+ kubectl_bin wait --for=delete crd perconaservermongodbs.psmdb.percona.com
++ mktemp
+ local LAST_OUT=/tmp/tmp.3cWnRJuwJj
++ mktemp
+ local LAST_ERR=/tmp/tmp.nRq8XrV4cm
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl wait --for=delete crd perconaservermongodbs.psmdb.percona.com
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.3cWnRJuwJj
+ cat /tmp/tmp.nRq8XrV4cm
+ rm /tmp/tmp.3cWnRJuwJj /tmp/tmp.nRq8XrV4cm
+ return 0
+ local rbac_yaml=rbac.yaml
+ '[' -n psmdb-operator ']'
+ rbac_yaml=cw-rbac.yaml
+ kubectl_bin delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2219/deploy/cw-rbac.yaml --ignore-not-found
++ mktemp
+ local LAST_OUT=/tmp/tmp.UKeByDmPoW
++ mktemp
+ local LAST_ERR=/tmp/tmp.E5qUPQLG5z
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2219/deploy/cw-rbac.yaml --ignore-not-found
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.UKeByDmPoW
+ cat /tmp/tmp.E5qUPQLG5z
+ rm /tmp/tmp.UKeByDmPoW /tmp/tmp.E5qUPQLG5z
+ return 0
+ check_crd_for_deletion PR-2219-d7e802db
+ local git_tag=PR-2219-d7e802db
++ curl -s https://raw.githubusercontent.com/percona/percona-server-mongodb-operator/PR-2219-d7e802db/deploy/crd.yaml
++ yq eval .metadata.name
++ /usr/sbin/sed ':a;N;$!ba;s/\n/ /g'
++ /usr/sbin/sed s/---//g
+ for crd_name in $(curl -s https://raw.githubusercontent.com/percona/percona-server-mongodb-operator/${git_tag}/deploy/crd.yaml | yq eval '.metadata.name' | $sed 's/---//g' | $sed ':a;N;$!ba;s/\n/ /g')
++ kubectl_bin get crd/null -o 'jsonpath={.status.conditions[-1].type}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.uXJ9lUeepW
+++ mktemp
++ local LAST_ERR=/tmp/tmp.aJKGo1a4Nl
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get crd/null -o 'jsonpath={.status.conditions[-1].type}'
++ exit_status=1
++ set -e
++ '[' 1 '!=' 0 -a -n 1 ']'
++ cat /tmp/tmp.uXJ9lUeepW
++ cat /tmp/tmp.aJKGo1a4Nl
Error from server (NotFound): customresourcedefinitions.apiextensions.k8s.io "null" not found
++ sleep 0
++ for i in $(seq 0 2)
++ set +e
++ kubectl get crd/null -o 'jsonpath={.status.conditions[-1].type}'
++ exit_status=1
++ set -e
++ '[' 1 '!=' 0 -a -n 1 ']'
++ cat /tmp/tmp.uXJ9lUeepW
++ cat /tmp/tmp.aJKGo1a4Nl
Error from server (NotFound): customresourcedefinitions.apiextensions.k8s.io "null" not found
++ sleep 4
++ for i in $(seq 0 2)
++ set +e
++ kubectl get crd/null -o 'jsonpath={.status.conditions[-1].type}'
++ exit_status=1
++ set -e
++ '[' 1 '!=' 0 -a -n 1 ']'
++ cat /tmp/tmp.uXJ9lUeepW
++ cat /tmp/tmp.aJKGo1a4Nl
Error from server (NotFound): customresourcedefinitions.apiextensions.k8s.io "null" not found
++ sleep 8
++ cat /tmp/tmp.uXJ9lUeepW
++ cat /tmp/tmp.aJKGo1a4Nl
Error from server (NotFound): customresourcedefinitions.apiextensions.k8s.io "null" not found
++ rm /tmp/tmp.uXJ9lUeepW /tmp/tmp.aJKGo1a4Nl
++ return 1
+ [[ '' == \T\e\r\m\i\n\a\t\i\n\g ]]
+ '[' -n psmdb-operator ']'
+ create_namespace psmdb-operator
+ local namespace=psmdb-operator
+ local skip_clean_namespace=
+ [[ 1 == 1 ]]
+ [[ -z '' ]]
+ destroy_chaos_mesh
++ helm list --all-namespaces --filter chaos-mesh
++ tail -n1
++ awk '-F ' '{print $2}'
++ sed s/NAMESPACE//
+ local chaos_mesh_ns=
+ desc 'destroy chaos-mesh'
+ set +o xtrace


-----------------------------------------------------------------------------------
destroy chaos-mesh
-----------------------------------------------------------------------------------

+ '[' -n '' ']'
++ kubectl get MutatingWebhookConfiguration
++ grep chaos-mesh
++ awk '{print $1}'
+ timeout 30 kubectl delete MutatingWebhookConfiguration
error: resource(s) were provided, but no name was specified
+ :
++ kubectl get ValidatingWebhookConfiguration
++ grep chaos-mesh
++ awk '{print $1}'
+ timeout 30 kubectl delete ValidatingWebhookConfiguration
error: resource(s) were provided, but no name was specified
+ :
++ kubectl get ValidatingWebhookConfiguration
++ grep validate-auth
++ awk '{print $1}'
+ timeout 30 kubectl delete ValidatingWebhookConfiguration
error: resource(s) were provided, but no name was specified
+ :
++ kubectl api-resources
++ grep chaos-mesh
++ awk '{print $1}'
++ grep chaos-mesh.org
++ kubectl get crd
++ awk '{print $1}'
+ timeout 30 kubectl delete crd
error: resource(s) were provided, but no name was specified
+ :
++ kubectl get clusterrolebinding
++ grep chaos-mesh
++ awk '{print $1}'
+ timeout 30 kubectl delete clusterrolebinding
error: resource(s) were provided, but no name was specified
+ :
++ kubectl get clusterrole
++ grep chaos-mesh
++ awk '{print $1}'
+ timeout 30 kubectl delete clusterrole
error: resource(s) were provided, but no name was specified
+ :
+ desc 'cleaned up all old namespaces'
+ set +o xtrace


-----------------------------------------------------------------------------------
cleaned up all old namespaces
-----------------------------------------------------------------------------------

+ kubectl_bin get ns
+ grep -E -v '^kube-|^default|Terminating|psmdb-operator|openshift|^gke-|^gmp-|^NAME'
+ awk '{print$1}'
+ xargs kubectl delete ns
+ '[' -n '' ']'
++ mktemp
+ desc 'cleaned up old namespaces psmdb-operator'
+ set +o xtrace


-----------------------------------------------------------------------------------
cleaned up old namespaces psmdb-operator
-----------------------------------------------------------------------------------

+ kubectl_bin delete namespace psmdb-operator --ignore-not-found
++ mktemp
+ local LAST_OUT=/tmp/tmp.G32l1ZX0f4
+ local LAST_OUT=/tmp/tmp.GT728PcD18
++ mktemp
++ mktemp
+ local LAST_ERR=/tmp/tmp.ip7PDuNLU2
+ local exit_status=0
+ local timeout=4
+ local LAST_ERR=/tmp/tmp.4v6imDFmIM
+ local exit_status=0
+ local timeout=4
++ seq 0 2
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl get ns
+ for i in $(seq 0 2)
+ set +e
+ kubectl delete namespace psmdb-operator --ignore-not-found
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.GT728PcD18
+ cat /tmp/tmp.4v6imDFmIM
+ rm /tmp/tmp.GT728PcD18 /tmp/tmp.4v6imDFmIM
+ return 0
error: resource(s) were provided, but no name was specified
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.G32l1ZX0f4
+ cat /tmp/tmp.ip7PDuNLU2
+ rm /tmp/tmp.G32l1ZX0f4 /tmp/tmp.ip7PDuNLU2
+ return 0
+ kubectl_bin wait --for=delete namespace psmdb-operator
++ mktemp
+ local LAST_OUT=/tmp/tmp.6LadaReEGc
++ mktemp
+ local LAST_ERR=/tmp/tmp.QyLCvhuLK5
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl wait --for=delete namespace psmdb-operator
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.6LadaReEGc
+ cat /tmp/tmp.QyLCvhuLK5
+ rm /tmp/tmp.6LadaReEGc /tmp/tmp.QyLCvhuLK5
+ return 0
+ desc 'create namespace psmdb-operator'
+ set +o xtrace


-----------------------------------------------------------------------------------
create namespace psmdb-operator
-----------------------------------------------------------------------------------

+ kubectl_bin create namespace psmdb-operator
++ mktemp
+ local LAST_OUT=/tmp/tmp.NXrBPbf6oy
++ mktemp
+ local LAST_ERR=/tmp/tmp.2yYFPvfPyU
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl create namespace psmdb-operator
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.NXrBPbf6oy
namespace/psmdb-operator created
+ cat /tmp/tmp.2yYFPvfPyU
+ rm /tmp/tmp.NXrBPbf6oy /tmp/tmp.2yYFPvfPyU
+ return 0
+ set_kube_ctx psmdb-operator
+ local namespace=psmdb-operator
++ kubectl_bin config current-context
+++ mktemp
++ local LAST_OUT=/tmp/tmp.tEk81CnUyy
+++ mktemp
++ local LAST_ERR=/tmp/tmp.QSoCPW4yEb
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl config current-context
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.tEk81CnUyy
++ cat /tmp/tmp.QSoCPW4yEb
++ rm /tmp/tmp.tEk81CnUyy /tmp/tmp.QSoCPW4yEb
++ return 0
+ kubectl_bin config set-context gke_cloud-dev-112233_us-central1-a_jen-psmdb-2219-d7e802db-7-cluster14 --namespace=psmdb-operator
++ mktemp
+ local LAST_OUT=/tmp/tmp.HSSVU0Thzc
++ mktemp
+ local LAST_ERR=/tmp/tmp.NmG4SPwpwF
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl config set-context gke_cloud-dev-112233_us-central1-a_jen-psmdb-2219-d7e802db-7-cluster14 --namespace=psmdb-operator
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.HSSVU0Thzc
Context "gke_cloud-dev-112233_us-central1-a_jen-psmdb-2219-d7e802db-7-cluster14" modified.
+ cat /tmp/tmp.NmG4SPwpwF
+ rm /tmp/tmp.HSSVU0Thzc /tmp/tmp.NmG4SPwpwF
+ return 0
+ deploy_operator
+ desc 'start PSMDB operator: docker.io/perconalab/percona-server-mongodb-operator:PR-2219-d7e802db'
+ set +o xtrace


-----------------------------------------------------------------------------------
start PSMDB operator: docker.io/perconalab/percona-server-mongodb-operator:PR-2219-d7e802db
-----------------------------------------------------------------------------------

+ local cr_file
+ '[' -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2219/e2e-tests/multi-storage/conf/crd.yaml ']'
+ cr_file=/mnt/jenkins/workspace/cloud-psmdb-operator_PR-2219/deploy/crd.yaml
+ kubectl_bin apply --server-side --force-conflicts -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2219/deploy/crd.yaml
++ mktemp
+ local LAST_OUT=/tmp/tmp.Z1evDtspXo
++ mktemp
+ local LAST_ERR=/tmp/tmp.N6BufpaTCC
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl apply --server-side --force-conflicts -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2219/deploy/crd.yaml
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.Z1evDtspXo
customresourcedefinition.apiextensions.k8s.io/perconaservermongodbbackups.psmdb.percona.com serverside-applied
customresourcedefinition.apiextensions.k8s.io/perconaservermongodbrestores.psmdb.percona.com serverside-applied
customresourcedefinition.apiextensions.k8s.io/perconaservermongodbs.psmdb.percona.com serverside-applied
+ cat /tmp/tmp.N6BufpaTCC
+ rm /tmp/tmp.Z1evDtspXo /tmp/tmp.N6BufpaTCC
+ return 0
+ '[' -n psmdb-operator ']'
+ apply_rbac cw-rbac
+ local operator_namespace=psmdb-operator
+ local rbac=cw-rbac
+ cat /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2219/deploy/cw-rbac.yaml
+ sed -e 's^namespace: .*^namespace: psmdb-operator^'
+ kubectl_bin apply -n psmdb-operator -f -
++ mktemp
+ local LAST_OUT=/tmp/tmp.G7L5rmAEMU
++ mktemp
+ local LAST_ERR=/tmp/tmp.dZJEUoFnff
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl apply -n psmdb-operator -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.G7L5rmAEMU
clusterrole.rbac.authorization.k8s.io/percona-server-mongodb-operator created
serviceaccount/percona-server-mongodb-operator created
clusterrolebinding.rbac.authorization.k8s.io/service-account-percona-server-mongodb-operator created
+ cat /tmp/tmp.dZJEUoFnff
+ rm /tmp/tmp.G7L5rmAEMU /tmp/tmp.dZJEUoFnff
+ return 0
+ yq eval '
			(.spec.template.spec.containers[].image = "docker.io/perconalab/percona-server-mongodb-operator:PR-2219-d7e802db") |
			((.. | select(.[] == "DISABLE_TELEMETRY")) |= .value="true") |
			((.. | select(.[] == "LOG_LEVEL")) |= .value="DEBUG")' /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2219/deploy/cw-operator.yaml
+ kubectl_bin apply -n psmdb-operator -f -
++ mktemp
+ local LAST_OUT=/tmp/tmp.9lcsQoJv28
++ mktemp
+ local LAST_ERR=/tmp/tmp.25wRYQwNvl
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl apply -n psmdb-operator -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.9lcsQoJv28
deployment.apps/percona-server-mongodb-operator created
+ cat /tmp/tmp.25wRYQwNvl
+ rm /tmp/tmp.9lcsQoJv28 /tmp/tmp.25wRYQwNvl
+ return 0
+ sleep 20
++ get_operator_pod
++ kubectl_bin get pods --selector=name=percona-server-mongodb-operator -o 'jsonpath={.items[].metadata.name}' -n psmdb-operator
+++ mktemp
++ local LAST_OUT=/tmp/tmp.MgYC7Fz9Zr
+++ mktemp
++ local LAST_ERR=/tmp/tmp.dmjtLvG64k
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=percona-server-mongodb-operator -o 'jsonpath={.items[].metadata.name}' -n psmdb-operator
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.MgYC7Fz9Zr
++ cat /tmp/tmp.dmjtLvG64k
++ rm /tmp/tmp.MgYC7Fz9Zr /tmp/tmp.dmjtLvG64k
++ return 0
+ wait_operator_pod percona-server-mongodb-operator-76758dcf9-mzsr2
+ local pod=percona-server-mongodb-operator-76758dcf9-mzsr2
+ set +o xtrace
waiting for pod/percona-server-mongodb-operator-76758dcf9-mzsr2 to be ready.OK
+ echo 'Print operator info from log'
Print operator info from log
+ grep 'Manager starting up'
++ get_operator_pod
++ kubectl_bin get pods --selector=name=percona-server-mongodb-operator -o 'jsonpath={.items[].metadata.name}' -n psmdb-operator
+++ mktemp
++ local LAST_OUT=/tmp/tmp.oTTg8MbEu0
+++ mktemp
++ local LAST_ERR=/tmp/tmp.9O7xRYe8Cw
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=percona-server-mongodb-operator -o 'jsonpath={.items[].metadata.name}' -n psmdb-operator
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.oTTg8MbEu0
++ cat /tmp/tmp.9O7xRYe8Cw
++ rm /tmp/tmp.oTTg8MbEu0 /tmp/tmp.9O7xRYe8Cw
++ return 0
+ kubectl_bin logs -n psmdb-operator percona-server-mongodb-operator-76758dcf9-mzsr2
++ mktemp
+ local LAST_OUT=/tmp/tmp.tlKwfDabAH
++ mktemp
+ local LAST_ERR=/tmp/tmp.BepfPlxbkF
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl logs -n psmdb-operator percona-server-mongodb-operator-76758dcf9-mzsr2
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.tlKwfDabAH
+ cat /tmp/tmp.BepfPlxbkF
+ rm /tmp/tmp.tlKwfDabAH /tmp/tmp.BepfPlxbkF
+ return 0
2026-03-10T22:29:54.771Z	INFO	setup	Manager starting up	{"gitCommit": "d7e802db10c9b2b2028f56c7b5227cb276fe5878", "gitBranch": "PR-2219-d7e802db", "buildTime": "", "goVersion": "go1.25.8", "os": "linux", "arch": "amd64"}
+ create_namespace multi-storage-10969
+ local namespace=multi-storage-10969
+ local skip_clean_namespace=
+ [[ 1 == 1 ]]
+ [[ -z '' ]]
+ destroy_chaos_mesh
++ helm list --all-namespaces --filter chaos-mesh
++ tail -n1
++ awk '-F ' '{print $2}'
++ sed s/NAMESPACE//
+ local chaos_mesh_ns=
+ desc 'destroy chaos-mesh'
+ set +o xtrace


-----------------------------------------------------------------------------------
destroy chaos-mesh
-----------------------------------------------------------------------------------

+ '[' -n '' ']'
++ kubectl get MutatingWebhookConfiguration
++ grep chaos-mesh
++ awk '{print $1}'
+ timeout 30 kubectl delete MutatingWebhookConfiguration
error: resource(s) were provided, but no name was specified
+ :
++ kubectl get ValidatingWebhookConfiguration
++ grep chaos-mesh
++ awk '{print $1}'
+ timeout 30 kubectl delete ValidatingWebhookConfiguration
error: resource(s) were provided, but no name was specified
+ :
++ kubectl get ValidatingWebhookConfiguration
++ grep validate-auth
++ awk '{print $1}'
+ timeout 30 kubectl delete ValidatingWebhookConfiguration
error: resource(s) were provided, but no name was specified
+ :
++ kubectl api-resources
++ grep chaos-mesh
++ awk '{print $1}'
++ kubectl get crd
++ grep chaos-mesh.org
++ awk '{print $1}'
+ timeout 30 kubectl delete crd
error: resource(s) were provided, but no name was specified
+ :
++ kubectl get clusterrolebinding
++ grep chaos-mesh
++ awk '{print $1}'
+ timeout 30 kubectl delete clusterrolebinding
error: resource(s) were provided, but no name was specified
+ :
++ kubectl get clusterrole
++ grep chaos-mesh
++ awk '{print $1}'
+ timeout 30 kubectl delete clusterrole
error: resource(s) were provided, but no name was specified
+ :
+ desc 'cleaned up all old namespaces'
+ set +o xtrace


-----------------------------------------------------------------------------------
cleaned up all old namespaces
-----------------------------------------------------------------------------------

+ kubectl_bin get ns
+ grep -E -v '^kube-|^default|Terminating|psmdb-operator|openshift|^gke-|^gmp-|^NAME'
+ awk '{print$1}'
++ mktemp
+ '[' -n '' ']'
+ xargs kubectl delete ns
+ desc 'cleaned up old namespaces multi-storage-10969'
+ set +o xtrace


-----------------------------------------------------------------------------------
cleaned up old namespaces multi-storage-10969
-----------------------------------------------------------------------------------

+ kubectl_bin delete namespace multi-storage-10969 --ignore-not-found
+ local LAST_OUT=/tmp/tmp.kCMxXdZU1G
++ mktemp
++ mktemp
+ local LAST_OUT=/tmp/tmp.U8OSB8b0AN
+ local LAST_ERR=/tmp/tmp.qJtfeudP7x
+ local exit_status=0
+ local timeout=4
++ mktemp
++ seq 0 2
+ local LAST_ERR=/tmp/tmp.9G9NwHQH2O
+ local exit_status=0
+ local timeout=4
+ for i in $(seq 0 2)
+ set +e
+ kubectl get ns
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl delete namespace multi-storage-10969 --ignore-not-found
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.kCMxXdZU1G
+ cat /tmp/tmp.qJtfeudP7x
+ rm /tmp/tmp.kCMxXdZU1G /tmp/tmp.qJtfeudP7x
+ return 0
error: resource(s) were provided, but no name was specified
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.U8OSB8b0AN
+ cat /tmp/tmp.9G9NwHQH2O
+ rm /tmp/tmp.U8OSB8b0AN /tmp/tmp.9G9NwHQH2O
+ return 0
+ kubectl_bin wait --for=delete namespace multi-storage-10969
++ mktemp
+ local LAST_OUT=/tmp/tmp.iAcy7UqggV
++ mktemp
+ local LAST_ERR=/tmp/tmp.ytozEqEZMz
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl wait --for=delete namespace multi-storage-10969
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.iAcy7UqggV
+ cat /tmp/tmp.ytozEqEZMz
+ rm /tmp/tmp.iAcy7UqggV /tmp/tmp.ytozEqEZMz
+ return 0
+ desc 'create namespace multi-storage-10969'
+ set +o xtrace


-----------------------------------------------------------------------------------
create namespace multi-storage-10969
-----------------------------------------------------------------------------------

+ kubectl_bin create namespace multi-storage-10969
++ mktemp
+ local LAST_OUT=/tmp/tmp.VWWbNyCae9
++ mktemp
+ local LAST_ERR=/tmp/tmp.cSRq2PYm3A
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl create namespace multi-storage-10969
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.VWWbNyCae9
namespace/multi-storage-10969 created
+ cat /tmp/tmp.cSRq2PYm3A
+ rm /tmp/tmp.VWWbNyCae9 /tmp/tmp.cSRq2PYm3A
+ return 0
+ set_kube_ctx multi-storage-10969
+ local namespace=multi-storage-10969
++ kubectl_bin config current-context
+++ mktemp
++ local LAST_OUT=/tmp/tmp.JKcpkqXDV2
+++ mktemp
++ local LAST_ERR=/tmp/tmp.CFFEWdkUvi
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl config current-context
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.JKcpkqXDV2
++ cat /tmp/tmp.CFFEWdkUvi
++ rm /tmp/tmp.JKcpkqXDV2 /tmp/tmp.CFFEWdkUvi
++ return 0
+ kubectl_bin config set-context gke_cloud-dev-112233_us-central1-a_jen-psmdb-2219-d7e802db-7-cluster14 --namespace=multi-storage-10969
++ mktemp
+ local LAST_OUT=/tmp/tmp.1HW4U6Rhwt
++ mktemp
+ local LAST_ERR=/tmp/tmp.dTtcabMtjl
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl config set-context gke_cloud-dev-112233_us-central1-a_jen-psmdb-2219-d7e802db-7-cluster14 --namespace=multi-storage-10969
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.1HW4U6Rhwt
Context "gke_cloud-dev-112233_us-central1-a_jen-psmdb-2219-d7e802db-7-cluster14" modified.
+ cat /tmp/tmp.dTtcabMtjl
+ rm /tmp/tmp.1HW4U6Rhwt /tmp/tmp.dTtcabMtjl
+ return 0
+ deploy_minio
+ local cert_secret=
+ local service_name=minio-service
+ desc 'install MinIO: minio-service'
+ set +o xtrace


-----------------------------------------------------------------------------------
install MinIO: minio-service
-----------------------------------------------------------------------------------

+ helm uninstall minio-service
+ :
+ helm repo remove minio
"minio" has been removed from your repositories
+ helm repo add minio https://charts.min.io/
"minio" has been added to your repositories
+ local endpoint=http://minio-service:9000
+ minio_args=('--version' '5.4.0' '--set' 'replicas=1' '--set' 'mode=standalone' '--set' 'resources.requests.memory=256Mi' '--set' 'rootUser=rootuser' '--set' 'rootPassword=rootpass123' '--set' 'users[0].accessKey=some-access-key' '--set' 'users[0].secretKey=some-secret-key' '--set' 'users[0].policy=consoleAdmin' '--set' 'service.type=ClusterIP' '--set' 'configPathmc=/tmp/' '--set' 'securityContext.enabled=false' '--set' 'persistence.size=2G' '--set' 'fullnameOverride=minio-service' '--set' 'serviceAccount.create=true' '--set' 'serviceAccount.name=minio-service-sa')
+ local minio_args
+ [[ -n '' ]]
+ retry 10 60 helm install minio-service --version 5.4.0 --set replicas=1 --set mode=standalone --set resources.requests.memory=256Mi --set rootUser=rootuser --set rootPassword=rootpass123 --set 'users[0].accessKey=some-access-key' --set 'users[0].secretKey=some-secret-key' --set 'users[0].policy=consoleAdmin' --set service.type=ClusterIP --set configPathmc=/tmp/ --set securityContext.enabled=false --set persistence.size=2G --set fullnameOverride=minio-service --set serviceAccount.create=true --set serviceAccount.name=minio-service-sa minio/minio
+ local max=10
+ local delay=60
+ shift 2
+ local n=1
+ helm install minio-service --version 5.4.0 --set replicas=1 --set mode=standalone --set resources.requests.memory=256Mi --set rootUser=rootuser --set rootPassword=rootpass123 --set 'users[0].accessKey=some-access-key' --set 'users[0].secretKey=some-secret-key' --set 'users[0].policy=consoleAdmin' --set service.type=ClusterIP --set configPathmc=/tmp/ --set securityContext.enabled=false --set persistence.size=2G --set fullnameOverride=minio-service --set serviceAccount.create=true --set serviceAccount.name=minio-service-sa minio/minio
NAME: minio-service
LAST DEPLOYED: Tue Mar 10 22:30:31 2026
NAMESPACE: multi-storage-10969
STATUS: deployed
REVISION: 1
TEST SUITE: None
NOTES:
MinIO can be accessed via port 9000 on the following DNS name from within your cluster:
minio-service.multi-storage-10969.cluster.local

To access MinIO from localhost, run the below commands:

  1. export POD_NAME=$(kubectl get pods --namespace multi-storage-10969 -l "release=minio-service" -o jsonpath="{.items[0].metadata.name}")

  2. kubectl port-forward $POD_NAME 9000 --namespace multi-storage-10969

Read more about port forwarding here: http://kubernetes.io/docs/user-guide/kubectl/kubectl_port-forward/

You can now access MinIO server on http://localhost:9000. Follow the below steps to connect to MinIO server with mc client:

  1. Download the MinIO mc client - https://min.io/docs/minio/linux/reference/minio-mc.html#quickstart

  2. export MC_HOST_minio-service-local=http://$(kubectl get secret --namespace multi-storage-10969 minio-service -o jsonpath="{.data.rootUser}" | base64 --decode):$(kubectl get secret --namespace multi-storage-10969 minio-service -o jsonpath="{.data.rootPassword}" | base64 --decode)@localhost:9000

  3. mc ls minio-service-local
++ kubectl_bin get pods --selector=release=minio-service -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.kR7wHeKAro
+++ mktemp
++ local LAST_ERR=/tmp/tmp.oewu78lazk
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=release=minio-service -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.kR7wHeKAro
++ cat /tmp/tmp.oewu78lazk
++ rm /tmp/tmp.kR7wHeKAro /tmp/tmp.oewu78lazk
++ return 0
+ local MINIO_POD=minio-service-6d5f646cdc-h5fvr
+ wait_pod minio-service-6d5f646cdc-h5fvr
+ local pod=minio-service-6d5f646cdc-h5fvr
+ set +o xtrace
waiting for pod/minio-service-6d5f646cdc-h5fvr to be ready.OK
+ '[' -n psmdb-operator ']'
+ kubectl_bin create svc -n psmdb-operator externalname minio-service --external-name=minio-service.multi-storage-10969.svc.cluster.local --tcp=9000
service/minio-service created
+ create_minio_bucket operator-testing http://minio-service:9000
+ local bucket=operator-testing
+ local endpoint=http://minio-service:9000
+ kubectl_bin run -i --rm aws-cli --image=docker.io/perconalab/awscli --restart=Never -- bash -c 'AWS_ACCESS_KEY_ID=some-access-key 		AWS_SECRET_ACCESS_KEY=some-secret-key 		AWS_DEFAULT_REGION=us-east-1         /usr/bin/aws --no-verify-ssl --endpoint-url http://minio-service:9000 s3 mb s3://operator-testing'
++ mktemp
+ local LAST_OUT=/tmp/tmp.AIZCcFd2aI
++ mktemp
+ local LAST_ERR=/tmp/tmp.OKsGOcrW7I
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl run -i --rm aws-cli --image=docker.io/perconalab/awscli --restart=Never -- bash -c 'AWS_ACCESS_KEY_ID=some-access-key 		AWS_SECRET_ACCESS_KEY=some-secret-key 		AWS_DEFAULT_REGION=us-east-1         /usr/bin/aws --no-verify-ssl --endpoint-url http://minio-service:9000 s3 mb s3://operator-testing'
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.AIZCcFd2aI
make_bucket: operator-testing
pod "aws-cli" deleted from multi-storage-10969 namespace
+ cat /tmp/tmp.OKsGOcrW7I
All commands and output from this session will be recorded in container logs, including credentials and sensitive information passed through the command prompt.
If you don't see a command prompt, try pressing enter.
warning: couldn't attach to pod/aws-cli, falling back to streaming logs: Internal error occurred: unable to upgrade connection: container aws-cli not found in pod aws-cli_multi-storage-10969
+ rm /tmp/tmp.AIZCcFd2aI /tmp/tmp.OKsGOcrW7I
+ return 0
+ endpoint=http://minio-service:9000
+ create_minio_bucket operator-testing-2 http://minio-service:9000
+ local bucket=operator-testing-2
+ local endpoint=http://minio-service:9000
+ kubectl_bin run -i --rm aws-cli --image=docker.io/perconalab/awscli --restart=Never -- bash -c 'AWS_ACCESS_KEY_ID=some-access-key 		AWS_SECRET_ACCESS_KEY=some-secret-key 		AWS_DEFAULT_REGION=us-east-1         /usr/bin/aws --no-verify-ssl --endpoint-url http://minio-service:9000 s3 mb s3://operator-testing-2'
++ mktemp
+ local LAST_OUT=/tmp/tmp.2Dl4JGqq3L
++ mktemp
+ local LAST_ERR=/tmp/tmp.8C1dAQy5gr
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl run -i --rm aws-cli --image=docker.io/perconalab/awscli --restart=Never -- bash -c 'AWS_ACCESS_KEY_ID=some-access-key 		AWS_SECRET_ACCESS_KEY=some-secret-key 		AWS_DEFAULT_REGION=us-east-1         /usr/bin/aws --no-verify-ssl --endpoint-url http://minio-service:9000 s3 mb s3://operator-testing-2'
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.2Dl4JGqq3L
make_bucket: operator-testing-2
pod "aws-cli" deleted from multi-storage-10969 namespace
+ cat /tmp/tmp.8C1dAQy5gr
+ rm /tmp/tmp.2Dl4JGqq3L /tmp/tmp.8C1dAQy5gr
+ return 0
+ create_minio_bucket operator-testing-3 http://minio-service:9000
+ local bucket=operator-testing-3
+ local endpoint=http://minio-service:9000
+ kubectl_bin run -i --rm aws-cli --image=docker.io/perconalab/awscli --restart=Never -- bash -c 'AWS_ACCESS_KEY_ID=some-access-key 		AWS_SECRET_ACCESS_KEY=some-secret-key 		AWS_DEFAULT_REGION=us-east-1         /usr/bin/aws --no-verify-ssl --endpoint-url http://minio-service:9000 s3 mb s3://operator-testing-3'
++ mktemp
+ local LAST_OUT=/tmp/tmp.G08AHAye9y
++ mktemp
+ local LAST_ERR=/tmp/tmp.5IUmMwFTqW
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl run -i --rm aws-cli --image=docker.io/perconalab/awscli --restart=Never -- bash -c 'AWS_ACCESS_KEY_ID=some-access-key 		AWS_SECRET_ACCESS_KEY=some-secret-key 		AWS_DEFAULT_REGION=us-east-1         /usr/bin/aws --no-verify-ssl --endpoint-url http://minio-service:9000 s3 mb s3://operator-testing-3'
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.G08AHAye9y
make_bucket: operator-testing-3
pod "aws-cli" deleted from multi-storage-10969 namespace
+ cat /tmp/tmp.5IUmMwFTqW
All commands and output from this session will be recorded in container logs, including credentials and sensitive information passed through the command prompt.
If you don't see a command prompt, try pressing enter.
warning: couldn't attach to pod/aws-cli, falling back to streaming logs: Internal error occurred: unable to upgrade connection: container aws-cli not found in pod aws-cli_multi-storage-10969
+ rm /tmp/tmp.G08AHAye9y /tmp/tmp.5IUmMwFTqW
+ return 0
+ log 'create secrets and start client'
+ set +o xtrace
[2026-03-10T22:31:16+0000] create secrets and start client
+ kubectl_bin apply -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2219/e2e-tests/conf/secrets.yml -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2219/e2e-tests/conf/client.yml -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2219/e2e-tests/conf/minio-secret.yml
++ mktemp
+ local LAST_OUT=/tmp/tmp.NX5eFb8Ez5
++ mktemp
+ local LAST_ERR=/tmp/tmp.aQuNXJARjC
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl apply -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2219/e2e-tests/conf/secrets.yml -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2219/e2e-tests/conf/client.yml -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2219/e2e-tests/conf/minio-secret.yml
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.NX5eFb8Ez5
secret/some-users created
deployment.apps/psmdb-client created
secret/minio-secret created
+ cat /tmp/tmp.aQuNXJARjC
+ rm /tmp/tmp.NX5eFb8Ez5 /tmp/tmp.aQuNXJARjC
+ return 0
+ log 'create PSMDB cluster: some-name'
+ set +o xtrace
[2026-03-10T22:31:19+0000] create PSMDB cluster: some-name
+ apply_cluster /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2219/e2e-tests/multi-storage/conf/some-name.yml
+ '[' -z '' ']'
+ cat_config /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2219/e2e-tests/multi-storage/conf/some-name.yml
+ kubectl_bin apply -f -
+ cat /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2219/e2e-tests/multi-storage/conf/some-name.yml
++ mktemp
+ yq eval '(.spec | select(.image == null)).image = "docker.io/perconalab/percona-server-mongodb-operator:main-mongod8.0"'
+ yq eval '(.spec | select(has("pmm"))).pmm.image = "docker.io/percona/pmm-client:2.44.1-1"'
+ yq eval '(.spec | select(has("initImage"))).initImage = "docker.io/perconalab/percona-server-mongodb-operator:PR-2219-d7e802db"'
+ local LAST_OUT=/tmp/tmp.LqtT89PQYj
+ yq eval '(.spec | select(has("backup"))).backup.image = "docker.io/perconalab/percona-server-mongodb-operator:main-backup"'
++ mktemp
+ local LAST_ERR=/tmp/tmp.kgXyFY9fvz
+ local exit_status=0
+ local timeout=4
+ /usr/sbin/sed -e s/NAME_SPACE/multi-storage-10969/g
++ seq 0 2
+ yq eval '.spec.upgradeOptions.apply="Never"'
+ for i in $(seq 0 2)
+ set +e
+ kubectl apply -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.LqtT89PQYj
perconaservermongodb.psmdb.percona.com/some-name created
+ cat /tmp/tmp.kgXyFY9fvz
+ rm /tmp/tmp.LqtT89PQYj /tmp/tmp.kgXyFY9fvz
+ return 0
+ log 'check if all 3 Pods started'
+ set +o xtrace
[2026-03-10T22:31:21+0000] check if all 3 Pods started
+ wait_for_running some-name-rs0 3
+ local name=some-name-rs0
+ let last_pod=2
+ local check_cluster_readyness=true
+ set_debug
+ [[ 1 == 1 ]]
+ set -o xtrace
+ local rs_name=rs0
+ local cluster_name=some-name
++ seq 0 2
+ for i in $(seq 0 $last_pod)
+ [[ 0 -eq 2 ]]
+ wait_pod some-name-rs0-0
+ local pod=some-name-rs0-0
+ set +o xtrace
waiting for pod/some-name-rs0-0 to be ready........OK
+ for i in $(seq 0 $last_pod)
+ [[ 1 -eq 2 ]]
+ wait_pod some-name-rs0-1
+ local pod=some-name-rs0-1
+ set +o xtrace
waiting for pod/some-name-rs0-1 to be ready........OK
+ for i in $(seq 0 $last_pod)
+ [[ 2 -eq 2 ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].arbiter.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.hm6eCfCqDI
+++ mktemp
++ local LAST_ERR=/tmp/tmp.LtFmB5ECWJ
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].arbiter.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.hm6eCfCqDI
++ cat /tmp/tmp.LtFmB5ECWJ
++ rm /tmp/tmp.hm6eCfCqDI /tmp/tmp.LtFmB5ECWJ
++ return 0
+ [[ '' == \t\r\u\e ]]
+ wait_pod some-name-rs0-2
+ local pod=some-name-rs0-2
+ set +o xtrace
waiting for pod/some-name-rs0-2 to be ready.....OK
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].nonvoting.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.UcpgOXWnfT
+++ mktemp
++ local LAST_ERR=/tmp/tmp.v3vjm3wnuK
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].nonvoting.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.UcpgOXWnfT
++ cat /tmp/tmp.v3vjm3wnuK
++ rm /tmp/tmp.UcpgOXWnfT /tmp/tmp.v3vjm3wnuK
++ return 0
+ [[ '' == \t\r\u\e ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].hidden.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.nBXmryYy0U
+++ mktemp
++ local LAST_ERR=/tmp/tmp.FZReGLIhMD
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].hidden.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.nBXmryYy0U
++ cat /tmp/tmp.FZReGLIhMD
++ rm /tmp/tmp.nBXmryYy0U /tmp/tmp.FZReGLIhMD
++ return 0
+ [[ '' == \t\r\u\e ]]
+ sleep 10
+ [[ true == \t\r\u\e ]]
+ set +x
Waiting for cluster readyness....
+ wait_backup_agent some-name-rs0-0
+ local agent_pod=some-name-rs0-0
+ set +o xtrace
waiting for pbm-agent to be ready in some-name-rs0-0...2026-03-10T22:32:44.000+0000 I listening for the commands

+ wait_backup_agent some-name-rs0-1
+ local agent_pod=some-name-rs0-1
+ set +o xtrace
waiting for pbm-agent to be ready in some-name-rs0-1...2026-03-10T22:32:53.000+0000 I listening for the commands

+ wait_backup_agent some-name-rs0-2
+ local agent_pod=some-name-rs0-2
+ set +o xtrace
waiting for pbm-agent to be ready in some-name-rs0-2...2026-03-10T22:32:56.000+0000 I listening for the commands

+ create_user
+ log 'create user myApp'
+ set +o xtrace
[2026-03-10T22:32:59+0000] create user myApp
+ run_mongo 'db.createUser({user:"myApp",pwd:"myPass",roles:[{db:"myApp",role:"readWrite"}]})' userAdmin:userAdmin123456@some-name-rs0.multi-storage-10969 mongodb
+ local 'command=db.createUser({user:"myApp",pwd:"myPass",roles:[{db:"myApp",role:"readWrite"}]})'
+ local uri=userAdmin:userAdmin123456@some-name-rs0.multi-storage-10969
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local replica_set=rs0
+ [[ userAdmin:userAdmin123456@some-name-rs0.multi-storage-10969 == *cfg* ]]
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.13mUCW9HQq
+++ mktemp
++ local LAST_ERR=/tmp/tmp.Ko6SDM85Yc
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.13mUCW9HQq
++ cat /tmp/tmp.Ko6SDM85Yc
++ rm /tmp/tmp.13mUCW9HQq /tmp/tmp.Ko6SDM85Yc
++ return 0
+ local client_container=psmdb-client-bb8b97679-7rpbx
+ kubectl_bin exec psmdb-client-bb8b97679-7rpbx -- bash -c 'printf '\''db.createUser({user:"myApp",pwd:"myPass",roles:[{db:"myApp",role:"readWrite"}]})\n'\'' | mongo mongodb://userAdmin:userAdmin123456@some-name-rs0.multi-storage-10969.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
++ mktemp
+ local LAST_OUT=/tmp/tmp.sv1vTuzoXN
++ mktemp
+ local LAST_ERR=/tmp/tmp.FdooydEiF1
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-bb8b97679-7rpbx -- bash -c 'printf '\''db.createUser({user:"myApp",pwd:"myPass",roles:[{db:"myApp",role:"readWrite"}]})\n'\'' | mongo mongodb://userAdmin:userAdmin123456@some-name-rs0.multi-storage-10969.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.sv1vTuzoXN
Percona Server for MongoDB shell version v4.4.29-28
connecting to: mongodb://some-name-rs0.multi-storage-10969.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb&replicaSet=rs0&ssl=false
Implicit session: session { "id" : UUID("e46ac323-0173-4773-aae3-77fb7433d4b8") }
Percona Server for MongoDB server version: v8.0.19-7
WARNING: shell and server versions do not match
Successfully added user: {
	"user" : "myApp",
	"roles" : [
		{
			"db" : "myApp",
			"role" : "readWrite"
		}
	]
}
bye
+ cat /tmp/tmp.FdooydEiF1
+ rm /tmp/tmp.sv1vTuzoXN /tmp/tmp.FdooydEiF1
+ return 0
+ write_document 100500
+ local x=100500
+ local cmp_postfix=
+ log 'write document: 100500'
+ set +o xtrace
[2026-03-10T22:33:01+0000] write document: 100500
+ run_mongo 'use myApp\n db.test.insert({ x: 100500 })' myApp:myPass@some-name-rs0.multi-storage-10969 mongodb
+ local 'command=use myApp\n db.test.insert({ x: 100500 })'
+ local uri=myApp:myPass@some-name-rs0.multi-storage-10969
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local replica_set=rs0
+ [[ myApp:myPass@some-name-rs0.multi-storage-10969 == *cfg* ]]
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.EpnCbkzDQI
+++ mktemp
++ local LAST_ERR=/tmp/tmp.zWXyUxbCMU
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.EpnCbkzDQI
++ cat /tmp/tmp.zWXyUxbCMU
++ rm /tmp/tmp.EpnCbkzDQI /tmp/tmp.zWXyUxbCMU
++ return 0
+ local client_container=psmdb-client-bb8b97679-7rpbx
+ kubectl_bin exec psmdb-client-bb8b97679-7rpbx -- bash -c 'printf '\''use myApp\n db.test.insert({ x: 100500 })\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0.multi-storage-10969.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
++ mktemp
+ local LAST_OUT=/tmp/tmp.G0La03Sosg
++ mktemp
+ local LAST_ERR=/tmp/tmp.Wvzawq1DEY
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-bb8b97679-7rpbx -- bash -c 'printf '\''use myApp\n db.test.insert({ x: 100500 })\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0.multi-storage-10969.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.G0La03Sosg
Percona Server for MongoDB shell version v4.4.29-28
connecting to: mongodb://some-name-rs0.multi-storage-10969.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb&replicaSet=rs0&ssl=false
Implicit session: session { "id" : UUID("3b6fbac1-2b9b-486f-b720-147e5764cb59") }
Percona Server for MongoDB server version: v8.0.19-7
WARNING: shell and server versions do not match
switched to db myApp
WriteResult({ "nInserted" : 1 })
bye
+ cat /tmp/tmp.Wvzawq1DEY
+ rm /tmp/tmp.G0La03Sosg /tmp/tmp.Wvzawq1DEY
+ return 0
+ run_backup minio-1 backup-minio-1 logical
+ local storage=minio-1
+ local name=backup-minio-1
+ local type=logical
+ log 'creating backup: backup-minio-1 (logical)'
+ set +o xtrace
[2026-03-10T22:33:03+0000] creating backup: backup-minio-1 (logical)
+ yq eval '.metadata.name = "backup-minio-1"
			| .spec.storageName = "minio-1"
			| .spec.type = "logical"' /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2219/e2e-tests/multi-storage/conf/backup.yml
+ kubectl_bin apply -f -
++ mktemp
+ local LAST_OUT=/tmp/tmp.X2cRLz5gWy
++ mktemp
+ local LAST_ERR=/tmp/tmp.jgoKWHkhAB
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl apply -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.X2cRLz5gWy
perconaservermongodbbackup.psmdb.percona.com/backup-minio-1 created
+ cat /tmp/tmp.jgoKWHkhAB
+ rm /tmp/tmp.X2cRLz5gWy /tmp/tmp.jgoKWHkhAB
+ return 0
+ wait_backup backup-minio-1
+ local backup_name=backup-minio-1
+ local target_state=ready
+ set +o xtrace
waiting for backup-minio-1 to reach ready state.......OK
+ run_backup minio-2 backup-minio-2 logical
+ local storage=minio-2
+ local name=backup-minio-2
+ local type=logical
+ log 'creating backup: backup-minio-2 (logical)'
+ set +o xtrace
[2026-03-10T22:33:18+0000] creating backup: backup-minio-2 (logical)
+ yq eval '.metadata.name = "backup-minio-2"
			| .spec.storageName = "minio-2"
			| .spec.type = "logical"' /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2219/e2e-tests/multi-storage/conf/backup.yml
+ kubectl_bin apply -f -
++ mktemp
+ local LAST_OUT=/tmp/tmp.NiURP6Zc0u
++ mktemp
+ local LAST_ERR=/tmp/tmp.fF01tDIahY
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl apply -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.NiURP6Zc0u
perconaservermongodbbackup.psmdb.percona.com/backup-minio-2 created
+ cat /tmp/tmp.fF01tDIahY
+ rm /tmp/tmp.NiURP6Zc0u /tmp/tmp.fF01tDIahY
+ return 0
+ wait_backup backup-minio-2
+ local backup_name=backup-minio-2
+ local target_state=ready
+ set +o xtrace
waiting for backup-minio-2 to reach ready state.......OK
+ run_backup minio-3 backup-minio-3 physical
+ local storage=minio-3
+ local name=backup-minio-3
+ local type=physical
+ log 'creating backup: backup-minio-3 (physical)'
+ set +o xtrace
[2026-03-10T22:33:32+0000] creating backup: backup-minio-3 (physical)
+ yq eval '.metadata.name = "backup-minio-3"
			| .spec.storageName = "minio-3"
			| .spec.type = "physical"' /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2219/e2e-tests/multi-storage/conf/backup.yml
+ kubectl_bin apply -f -
++ mktemp
+ local LAST_OUT=/tmp/tmp.wloTbhxrrX
++ mktemp
+ local LAST_ERR=/tmp/tmp.9EWpCJoZLK
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl apply -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.wloTbhxrrX
perconaservermongodbbackup.psmdb.percona.com/backup-minio-3 created
+ cat /tmp/tmp.9EWpCJoZLK
+ rm /tmp/tmp.wloTbhxrrX /tmp/tmp.9EWpCJoZLK
+ return 0
+ wait_backup backup-minio-3
+ local backup_name=backup-minio-3
+ local target_state=ready
+ set +o xtrace
waiting for backup-minio-3 to reach ready state.......OK
+ write_document 100501
+ local x=100501
+ local cmp_postfix=
+ log 'write document: 100501'
+ set +o xtrace
[2026-03-10T22:33:46+0000] write document: 100501
+ run_mongo 'use myApp\n db.test.insert({ x: 100501 })' myApp:myPass@some-name-rs0.multi-storage-10969 mongodb
+ local 'command=use myApp\n db.test.insert({ x: 100501 })'
+ local uri=myApp:myPass@some-name-rs0.multi-storage-10969
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local replica_set=rs0
+ [[ myApp:myPass@some-name-rs0.multi-storage-10969 == *cfg* ]]
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.yE9Pg7KRhg
+++ mktemp
++ local LAST_ERR=/tmp/tmp.7PFZozRSvF
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.yE9Pg7KRhg
++ cat /tmp/tmp.7PFZozRSvF
++ rm /tmp/tmp.yE9Pg7KRhg /tmp/tmp.7PFZozRSvF
++ return 0
+ local client_container=psmdb-client-bb8b97679-7rpbx
+ kubectl_bin exec psmdb-client-bb8b97679-7rpbx -- bash -c 'printf '\''use myApp\n db.test.insert({ x: 100501 })\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0.multi-storage-10969.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
++ mktemp
+ local LAST_OUT=/tmp/tmp.MeP5Rngasq
++ mktemp
+ local LAST_ERR=/tmp/tmp.eKIk4UT4MW
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-bb8b97679-7rpbx -- bash -c 'printf '\''use myApp\n db.test.insert({ x: 100501 })\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0.multi-storage-10969.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.MeP5Rngasq
Percona Server for MongoDB shell version v4.4.29-28
connecting to: mongodb://some-name-rs0.multi-storage-10969.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb&replicaSet=rs0&ssl=false
Implicit session: session { "id" : UUID("555e7ea3-313b-4ad5-ac6b-054caa6cfe5c") }
Percona Server for MongoDB server version: v8.0.19-7
WARNING: shell and server versions do not match
switched to db myApp
WriteResult({ "nInserted" : 1 })
bye
+ cat /tmp/tmp.eKIk4UT4MW
+ rm /tmp/tmp.MeP5Rngasq /tmp/tmp.eKIk4UT4MW
+ return 0
+ write_document 100502
+ local x=100502
+ local cmp_postfix=
+ log 'write document: 100502'
+ set +o xtrace
[2026-03-10T22:33:49+0000] write document: 100502
+ run_mongo 'use myApp\n db.test.insert({ x: 100502 })' myApp:myPass@some-name-rs0.multi-storage-10969 mongodb
+ local 'command=use myApp\n db.test.insert({ x: 100502 })'
+ local uri=myApp:myPass@some-name-rs0.multi-storage-10969
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local replica_set=rs0
+ [[ myApp:myPass@some-name-rs0.multi-storage-10969 == *cfg* ]]
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.LrWyKYNTMI
+++ mktemp
++ local LAST_ERR=/tmp/tmp.8G49EuDV5x
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.LrWyKYNTMI
++ cat /tmp/tmp.8G49EuDV5x
++ rm /tmp/tmp.LrWyKYNTMI /tmp/tmp.8G49EuDV5x
++ return 0
+ local client_container=psmdb-client-bb8b97679-7rpbx
+ kubectl_bin exec psmdb-client-bb8b97679-7rpbx -- bash -c 'printf '\''use myApp\n db.test.insert({ x: 100502 })\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0.multi-storage-10969.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
++ mktemp
+ local LAST_OUT=/tmp/tmp.DLbDusOo3Q
++ mktemp
+ local LAST_ERR=/tmp/tmp.dMaBBNPFsl
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-bb8b97679-7rpbx -- bash -c 'printf '\''use myApp\n db.test.insert({ x: 100502 })\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0.multi-storage-10969.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.DLbDusOo3Q
Percona Server for MongoDB shell version v4.4.29-28
connecting to: mongodb://some-name-rs0.multi-storage-10969.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb&replicaSet=rs0&ssl=false
Implicit session: session { "id" : UUID("e2662fa7-b96a-4c5b-ac6b-804fa6b5d106") }
Percona Server for MongoDB server version: v8.0.19-7
WARNING: shell and server versions do not match
switched to db myApp
WriteResult({ "nInserted" : 1 })
bye
+ cat /tmp/tmp.dMaBBNPFsl
+ rm /tmp/tmp.DLbDusOo3Q /tmp/tmp.dMaBBNPFsl
+ return 0
+ write_document 100503
+ local x=100503
+ local cmp_postfix=
+ log 'write document: 100503'
+ set +o xtrace
[2026-03-10T22:33:52+0000] write document: 100503
+ run_mongo 'use myApp\n db.test.insert({ x: 100503 })' myApp:myPass@some-name-rs0.multi-storage-10969 mongodb
+ local 'command=use myApp\n db.test.insert({ x: 100503 })'
+ local uri=myApp:myPass@some-name-rs0.multi-storage-10969
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local replica_set=rs0
+ [[ myApp:myPass@some-name-rs0.multi-storage-10969 == *cfg* ]]
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.7r6Wy0zF2g
+++ mktemp
++ local LAST_ERR=/tmp/tmp.MlF56suLcj
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.7r6Wy0zF2g
++ cat /tmp/tmp.MlF56suLcj
++ rm /tmp/tmp.7r6Wy0zF2g /tmp/tmp.MlF56suLcj
++ return 0
+ local client_container=psmdb-client-bb8b97679-7rpbx
+ kubectl_bin exec psmdb-client-bb8b97679-7rpbx -- bash -c 'printf '\''use myApp\n db.test.insert({ x: 100503 })\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0.multi-storage-10969.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
++ mktemp
+ local LAST_OUT=/tmp/tmp.HnQCu8tWDA
++ mktemp
+ local LAST_ERR=/tmp/tmp.RXRJ24NwaG
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-bb8b97679-7rpbx -- bash -c 'printf '\''use myApp\n db.test.insert({ x: 100503 })\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0.multi-storage-10969.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.HnQCu8tWDA
Percona Server for MongoDB shell version v4.4.29-28
connecting to: mongodb://some-name-rs0.multi-storage-10969.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb&replicaSet=rs0&ssl=false
Implicit session: session { "id" : UUID("72770a3f-6630-416c-adc7-5495aec6833d") }
Percona Server for MongoDB server version: v8.0.19-7
WARNING: shell and server versions do not match
switched to db myApp
WriteResult({ "nInserted" : 1 })
bye
+ cat /tmp/tmp.RXRJ24NwaG
+ rm /tmp/tmp.HnQCu8tWDA /tmp/tmp.RXRJ24NwaG
+ return 0
+ write_document 100504
+ local x=100504
+ local cmp_postfix=
+ log 'write document: 100504'
+ set +o xtrace
[2026-03-10T22:33:55+0000] write document: 100504
+ run_mongo 'use myApp\n db.test.insert({ x: 100504 })' myApp:myPass@some-name-rs0.multi-storage-10969 mongodb
+ local 'command=use myApp\n db.test.insert({ x: 100504 })'
+ local uri=myApp:myPass@some-name-rs0.multi-storage-10969
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local replica_set=rs0
+ [[ myApp:myPass@some-name-rs0.multi-storage-10969 == *cfg* ]]
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.u3rrGwt9Pp
+++ mktemp
++ local LAST_ERR=/tmp/tmp.Wt8tZ4nWtJ
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.u3rrGwt9Pp
++ cat /tmp/tmp.Wt8tZ4nWtJ
++ rm /tmp/tmp.u3rrGwt9Pp /tmp/tmp.Wt8tZ4nWtJ
++ return 0
+ local client_container=psmdb-client-bb8b97679-7rpbx
+ kubectl_bin exec psmdb-client-bb8b97679-7rpbx -- bash -c 'printf '\''use myApp\n db.test.insert({ x: 100504 })\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0.multi-storage-10969.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
++ mktemp
+ local LAST_OUT=/tmp/tmp.Pb2tcthyZR
++ mktemp
+ local LAST_ERR=/tmp/tmp.Kfm8mdsaNA
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-bb8b97679-7rpbx -- bash -c 'printf '\''use myApp\n db.test.insert({ x: 100504 })\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0.multi-storage-10969.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.Pb2tcthyZR
Percona Server for MongoDB shell version v4.4.29-28
connecting to: mongodb://some-name-rs0.multi-storage-10969.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb&replicaSet=rs0&ssl=false
Implicit session: session { "id" : UUID("0757324d-294f-4fb6-a803-50b1268c9a3b") }
Percona Server for MongoDB server version: v8.0.19-7
WARNING: shell and server versions do not match
switched to db myApp
WriteResult({ "nInserted" : 1 })
bye
+ cat /tmp/tmp.Kfm8mdsaNA
+ rm /tmp/tmp.Pb2tcthyZR /tmp/tmp.Kfm8mdsaNA
+ return 0
+ write_document 100505
+ local x=100505
+ local cmp_postfix=
+ log 'write document: 100505'
+ set +o xtrace
[2026-03-10T22:33:57+0000] write document: 100505
+ run_mongo 'use myApp\n db.test.insert({ x: 100505 })' myApp:myPass@some-name-rs0.multi-storage-10969 mongodb
+ local 'command=use myApp\n db.test.insert({ x: 100505 })'
+ local uri=myApp:myPass@some-name-rs0.multi-storage-10969
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local replica_set=rs0
+ [[ myApp:myPass@some-name-rs0.multi-storage-10969 == *cfg* ]]
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.b1O97bICMQ
+++ mktemp
++ local LAST_ERR=/tmp/tmp.9xliMljrOI
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.b1O97bICMQ
++ cat /tmp/tmp.9xliMljrOI
++ rm /tmp/tmp.b1O97bICMQ /tmp/tmp.9xliMljrOI
++ return 0
+ local client_container=psmdb-client-bb8b97679-7rpbx
+ kubectl_bin exec psmdb-client-bb8b97679-7rpbx -- bash -c 'printf '\''use myApp\n db.test.insert({ x: 100505 })\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0.multi-storage-10969.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
++ mktemp
+ local LAST_OUT=/tmp/tmp.fZP9gd780K
++ mktemp
+ local LAST_ERR=/tmp/tmp.bWE0wdIh6g
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-bb8b97679-7rpbx -- bash -c 'printf '\''use myApp\n db.test.insert({ x: 100505 })\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0.multi-storage-10969.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.fZP9gd780K
Percona Server for MongoDB shell version v4.4.29-28
connecting to: mongodb://some-name-rs0.multi-storage-10969.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb&replicaSet=rs0&ssl=false
Implicit session: session { "id" : UUID("e62e0f47-fb7e-44c4-accc-ceb50a3b29f4") }
Percona Server for MongoDB server version: v8.0.19-7
WARNING: shell and server versions do not match
switched to db myApp
WriteResult({ "nInserted" : 1 })
bye
+ cat /tmp/tmp.bWE0wdIh6g
+ rm /tmp/tmp.fZP9gd780K /tmp/tmp.bWE0wdIh6g
+ return 0
+ compare_mongo_cmd find myApp:myPass@some-name-rs0.multi-storage-10969
+ local command=find
+ local uri=myApp:myPass@some-name-rs0.multi-storage-10969
+ local postfix=
+ local suffix=
+ local database=myApp
+ local collection=test
+ local sort=
+ local tls=false
+ local replicaset=
+ local 'full_command=db.test.find()'
+ [[ -n '' ]]
+ log 'running db.test.find() in myApp'
+ set +o xtrace
[2026-03-10T22:34:00+0000] running db.test.find() in myApp
+ [[ false == \t\r\u\e ]]
+ mongo_command=run_mongo
+ run_mongo 'use myApp\n db.test.find()' myApp:myPass@some-name-rs0.multi-storage-10969 mongodb '' ''
+ local 'command=use myApp\n db.test.find()'
+ local uri=myApp:myPass@some-name-rs0.multi-storage-10969
+ local driver=mongodb
+ grep -E -v 'I NETWORK|W NETWORK|F NETWORK|"c":"NETWORK"|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|Started a new thread for the timer service'
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local replica_set=rs0
+ [[ myApp:myPass@some-name-rs0.multi-storage-10969 == *cfg* ]]
+ /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.17DuVRLbsR
+++ mktemp
++ local LAST_ERR=/tmp/tmp.QEbDaNGDFS
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.17DuVRLbsR
++ cat /tmp/tmp.QEbDaNGDFS
++ rm /tmp/tmp.17DuVRLbsR /tmp/tmp.QEbDaNGDFS
++ return 0
+ local client_container=psmdb-client-bb8b97679-7rpbx
+ kubectl_bin exec psmdb-client-bb8b97679-7rpbx -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0.multi-storage-10969.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
++ mktemp
+ local LAST_OUT=/tmp/tmp.2XCDWCEOWo
++ mktemp
+ local LAST_ERR=/tmp/tmp.T9mgQFZ7Nv
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-bb8b97679-7rpbx -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0.multi-storage-10969.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.2XCDWCEOWo
+ cat /tmp/tmp.T9mgQFZ7Nv
+ rm /tmp/tmp.2XCDWCEOWo /tmp/tmp.T9mgQFZ7Nv
+ return 0
+ [[ 0 -eq 0 ]]
+ diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2219/e2e-tests/multi-storage/compare/find.json /tmp/tmp.iK1Y0vUPYD/find
+ sleep_with_log 70 'wait for oplog chunks to be uploaded'
+ local d=70
+ local 'msg=wait for oplog chunks to be uploaded'
+ log 'wait for oplog chunks to be uploaded. sleeping for 70 seconds'
+ set +o xtrace
[2026-03-10T22:34:03+0000] wait for oplog chunks to be uploaded. sleeping for 70 seconds
+ sleep 70
+ wait_for_restorable_time backup-minio-3
+ local backup_name=backup-minio-3
++ get_latest_restorable_time backup-minio-3
++ local backup_name=backup-minio-3
+++ yq .status.latestRestorableTime
+++ kubectl_bin get psmdb-backup backup-minio-3 -o yaml
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.Tjj5MLljPb
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.s8wPAsEzai
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in $(seq 0 2)
+++ set +e
+++ kubectl get psmdb-backup backup-minio-3 -o yaml
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.Tjj5MLljPb
+++ cat /tmp/tmp.s8wPAsEzai
+++ rm /tmp/tmp.Tjj5MLljPb /tmp/tmp.s8wPAsEzai
+++ return 0
++ local time=2026-03-10T22:34:32Z
++ [[ 2026-03-10T22:34:32Z == \n\u\l\l ]]
++ echo -n 2026-03-10T22:34:32Z
+ local latest_restorable_time=2026-03-10T22:34:32Z
+ local retries=0
+ echo -n 'waiting for psmdb-backup/backup-minio-3'\''s latest restorable time'
waiting for psmdb-backup/backup-minio-3's latest restorable time+ [[ 2026-03-10T22:34:32Z != \n\u\l\l ]]
+ echo '.OK latestRestorableTime: 2026-03-10T22:34:32Z'
.OK latestRestorableTime: 2026-03-10T22:34:32Z
+ wait_for_advanced_restorable_time backup-minio-3
+ local backup_name=backup-minio-3
+++ get_latest_restorable_time backup-minio-3
+++ local backup_name=backup-minio-3
++++ kubectl_bin get psmdb-backup backup-minio-3 -o yaml
++++ yq .status.latestRestorableTime
+++++ mktemp
++++ local LAST_OUT=/tmp/tmp.8MyQNWLN7N
+++++ mktemp
++++ local LAST_ERR=/tmp/tmp.aWiZL903Gd
++++ local exit_status=0
++++ local timeout=4
+++++ seq 0 2
++++ for i in $(seq 0 2)
++++ set +e
++++ kubectl get psmdb-backup backup-minio-3 -o yaml
++++ exit_status=0
++++ set -e
++++ '[' 0 '!=' 0 -a -n 1 ']'
++++ break
++++ cat /tmp/tmp.8MyQNWLN7N
++++ cat /tmp/tmp.aWiZL903Gd
++++ rm /tmp/tmp.8MyQNWLN7N /tmp/tmp.aWiZL903Gd
++++ return 0
+++ local time=2026-03-10T22:34:32Z
+++ [[ 2026-03-10T22:34:32Z == \n\u\l\l ]]
+++ echo -n 2026-03-10T22:34:32Z
++ datetime_to_timestamp 2026-03-10T22:34:32Z
++ local datetime=2026-03-10T22:34:32Z
++ TZ=UTC
++ /usr/sbin/date -d2026-03-10T22:34:32Z +%s
+ local latest_restorable_time=1773182072
++ get_backup_last_write backup-minio-3
++ local backup_name=backup-minio-3
+++ kubectl_bin get psmdb-backup backup-minio-3 -o yaml
+++ yq .status.pbmName
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.tzkDD8YMon
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.b6O4Ern29F
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in $(seq 0 2)
+++ set +e
+++ kubectl get psmdb-backup backup-minio-3 -o yaml
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.tzkDD8YMon
+++ cat /tmp/tmp.b6O4Ern29F
+++ rm /tmp/tmp.tzkDD8YMon /tmp/tmp.b6O4Ern29F
+++ return 0
++ local pbm_name=2026-03-10T22:33:35Z
++ kubectl_bin exec some-name-rs0-0 -c backup-agent -- pbm list -o json
++ jq --arg v 2026-03-10T22:33:35Z '.snapshots[] | select(.name==$v) | .restoreTo'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.oDR7JHO16u
+++ mktemp
++ local LAST_ERR=/tmp/tmp.TAQM3UZLXu
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl exec some-name-rs0-0 -c backup-agent -- pbm list -o json
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.oDR7JHO16u
++ cat /tmp/tmp.TAQM3UZLXu
++ rm /tmp/tmp.oDR7JHO16u /tmp/tmp.TAQM3UZLXu
++ return 0
+ local latest_write=1773182016
+ local retries=0
++ timestamp_to_datetime 1773182016
++ local ts=1773182016
++ TZ=UTC
++ /usr/sbin/date -d@1773182016 +%Y-%m-%dT%H:%M:%S%Z
+ echo -n 'waiting for psmdb-backup/backup-minio-3'\''s latest restorable time to advance its latest write (2026-03-10T22:33:36UTC)'
waiting for psmdb-backup/backup-minio-3's latest restorable time to advance its latest write (2026-03-10T22:33:36UTC)+ [[ 1773182072 -gt 1773182016 ]]
++ timestamp_to_datetime 1773182072
++ local ts=1773182072
++ TZ=UTC
++ /usr/sbin/date -d@1773182072 +%Y-%m-%dT%H:%M:%S%Z
+ echo '.OK latestRestorableTime: 2026-03-10T22:34:32UTC'
.OK latestRestorableTime: 2026-03-10T22:34:32UTC
++ get_latest_restorable_time backup-minio-3
++ local backup_name=backup-minio-3
+++ kubectl_bin get psmdb-backup backup-minio-3 -o yaml
+++ yq .status.latestRestorableTime
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.h62BGy5iD6
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.fGv5NeCJuf
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in $(seq 0 2)
+++ set +e
+++ kubectl get psmdb-backup backup-minio-3 -o yaml
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.h62BGy5iD6
+++ cat /tmp/tmp.fGv5NeCJuf
+++ rm /tmp/tmp.h62BGy5iD6 /tmp/tmp.fGv5NeCJuf
+++ return 0
++ local time=2026-03-10T22:34:32Z
++ [[ 2026-03-10T22:34:32Z == \n\u\l\l ]]
++ echo -n 2026-03-10T22:34:32Z
+ restore_time=2026-03-10T22:34:32Z
++ format_pitr_target 2026-03-10T22:34:32Z
++ local target=2026-03-10T22:34:32Z
++ echo 2026-03-10T22:34:32Z
++ sed 's/T/ /'
++ sed s/Z//
+ check_recovery backup-minio-3 '2026-03-10 22:34:32'
+ local backup_name=backup-minio-3
+ local 'restore_time=2026-03-10 22:34:32'
+ local cmp_postfix=
+ drop_collection myApp test myApp myPass
+ local db=myApp
+ local collection=test
+ local user=myApp
+ local pass=myPass
+ log 'dropping collection: myApp.test'
+ set +o xtrace
[2026-03-10T22:35:18+0000] dropping collection: myApp.test
+ run_mongo 'use myApp\n db.test.drop()' myApp:myPass@some-name-rs0.multi-storage-10969 mongodb
+ local 'command=use myApp\n db.test.drop()'
+ local uri=myApp:myPass@some-name-rs0.multi-storage-10969
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local replica_set=rs0
+ [[ myApp:myPass@some-name-rs0.multi-storage-10969 == *cfg* ]]
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.791wwC5qsu
+++ mktemp
++ local LAST_ERR=/tmp/tmp.7W5SDoMyK8
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.791wwC5qsu
++ cat /tmp/tmp.7W5SDoMyK8
++ rm /tmp/tmp.791wwC5qsu /tmp/tmp.7W5SDoMyK8
++ return 0
+ local client_container=psmdb-client-bb8b97679-7rpbx
+ kubectl_bin exec psmdb-client-bb8b97679-7rpbx -- bash -c 'printf '\''use myApp\n db.test.drop()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0.multi-storage-10969.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
++ mktemp
+ local LAST_OUT=/tmp/tmp.5bpcFHKUgx
++ mktemp
+ local LAST_ERR=/tmp/tmp.etyaNPGKca
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-bb8b97679-7rpbx -- bash -c 'printf '\''use myApp\n db.test.drop()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0.multi-storage-10969.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.5bpcFHKUgx
Percona Server for MongoDB shell version v4.4.29-28
connecting to: mongodb://some-name-rs0.multi-storage-10969.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb&replicaSet=rs0&ssl=false
Implicit session: session { "id" : UUID("be410644-1cde-4f8c-988e-eda448f2e917") }
Percona Server for MongoDB server version: v8.0.19-7
WARNING: shell and server versions do not match
switched to db myApp
true
bye
+ cat /tmp/tmp.etyaNPGKca
+ rm /tmp/tmp.5bpcFHKUgx /tmp/tmp.etyaNPGKca
+ return 0
+ log 'creating restore: restore-backup-minio-3 restore_to: 2026-03-10 22:34:32'
+ set +o xtrace
[2026-03-10T22:35:21+0000] creating restore: restore-backup-minio-3 restore_to: 2026-03-10 22:34:32
+ cat /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2219/e2e-tests/multi-storage/conf/restore.yml
+ /usr/sbin/sed -e 's/name:/name: restore-backup-minio-3/'
+ /usr/sbin/sed -e 's/backupName:/backupName: backup-minio-3/'
+ /usr/sbin/sed -e 's/date:/date: 2026-03-10 22:34:32/'
+ kubectl_bin apply -f -
++ mktemp
+ local LAST_OUT=/tmp/tmp.jDXlPyfm5b
++ mktemp
+ local LAST_ERR=/tmp/tmp.lOqfWJZqAN
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl apply -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.jDXlPyfm5b
perconaservermongodbrestore.psmdb.percona.com/restore-backup-minio-3 created
+ cat /tmp/tmp.lOqfWJZqAN
+ rm /tmp/tmp.jDXlPyfm5b /tmp/tmp.lOqfWJZqAN
+ return 0
+ wait_restore backup-minio-3 some-name requested 0 900 1
+ local backup_name=backup-minio-3
+ local cluster_name=some-name
+ local target_state=requested
+ local wait_cluster_consistency=0
+ local wait_time=900
+ local ok_if_ready=1
+ set +o xtrace
Waiting for the psmdb-restore/restore-backup-minio-3 object to be created.OK
Waiting psmdb-restore/restore-backup-minio-3 to reach state "requested" ...OK after 2 minutes
+ [[ 0 -eq 1 ]]
+ wait_restore backup-minio-3 some-name ready 0 1600
+ local backup_name=backup-minio-3
+ local cluster_name=some-name
+ local target_state=ready
+ local wait_cluster_consistency=0
+ local wait_time=1600
+ local ok_if_ready=0
+ set +o xtrace
Waiting for the psmdb-restore/restore-backup-minio-3 object to be created.OK
Waiting psmdb-restore/restore-backup-minio-3 to reach state "ready" ..OK after 1 minutes
+ [[ 0 -eq 1 ]]
+ wait_for_running some-name-rs0 3
+ local name=some-name-rs0
+ let last_pod=2
+ local check_cluster_readyness=true
+ set_debug
+ [[ 1 == 1 ]]
+ set -o xtrace
+ local rs_name=rs0
+ local cluster_name=some-name
++ seq 0 2
+ for i in $(seq 0 $last_pod)
+ [[ 0 -eq 2 ]]
+ wait_pod some-name-rs0-0
+ local pod=some-name-rs0-0
+ set +o xtrace
waiting for pod/some-name-rs0-0 to be ready.OK
+ for i in $(seq 0 $last_pod)
+ [[ 1 -eq 2 ]]
+ wait_pod some-name-rs0-1
+ local pod=some-name-rs0-1
+ set +o xtrace
waiting for pod/some-name-rs0-1 to be ready.OK
+ for i in $(seq 0 $last_pod)
+ [[ 2 -eq 2 ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].arbiter.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.a1D1tKmYkV
+++ mktemp
++ local LAST_ERR=/tmp/tmp.VYqkKIaLKz
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].arbiter.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.a1D1tKmYkV
++ cat /tmp/tmp.VYqkKIaLKz
++ rm /tmp/tmp.a1D1tKmYkV /tmp/tmp.VYqkKIaLKz
++ return 0
+ [[ '' == \t\r\u\e ]]
+ wait_pod some-name-rs0-2
+ local pod=some-name-rs0-2
+ set +o xtrace
waiting for pod/some-name-rs0-2 to be ready........OK
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].nonvoting.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.o9GdzjkSCs
+++ mktemp
++ local LAST_ERR=/tmp/tmp.aaeLOAnwuX
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].nonvoting.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.o9GdzjkSCs
++ cat /tmp/tmp.aaeLOAnwuX
++ rm /tmp/tmp.o9GdzjkSCs /tmp/tmp.aaeLOAnwuX
++ return 0
+ [[ '' == \t\r\u\e ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].hidden.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.48hzzqpTot
+++ mktemp
++ local LAST_ERR=/tmp/tmp.AUNg8PjEEB
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].hidden.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.48hzzqpTot
++ cat /tmp/tmp.AUNg8PjEEB
++ rm /tmp/tmp.48hzzqpTot /tmp/tmp.AUNg8PjEEB
++ return 0
+ [[ '' == \t\r\u\e ]]
+ sleep 10
+ [[ true == \t\r\u\e ]]
+ set +x
Waiting for cluster readyness
+ compare_mongo_cmd find myApp:myPass@some-name-rs0.multi-storage-10969 '' .svc.cluster.local myApp test 'sort( { x: 1 } )'
+ local command=find
+ local uri=myApp:myPass@some-name-rs0.multi-storage-10969
+ local postfix=
+ local suffix=.svc.cluster.local
+ local database=myApp
+ local collection=test
+ local 'sort=sort( { x: 1 } )'
+ local tls=false
+ local replicaset=
+ local 'full_command=db.test.find()'
+ [[ -n sort( { x: 1 } ) ]]
+ full_command='db.test.find().sort( { x: 1 } )'
+ log 'running db.test.find().sort( { x: 1 } ) in myApp'
+ set +o xtrace
[2026-03-10T22:40:57+0000] running db.test.find().sort( { x: 1 } ) in myApp
+ [[ false == \t\r\u\e ]]
+ mongo_command=run_mongo
+ run_mongo 'use myApp\n db.test.find().sort( { x: 1 } )' myApp:myPass@some-name-rs0.multi-storage-10969 mongodb .svc.cluster.local ''
+ local 'command=use myApp\n db.test.find().sort( { x: 1 } )'
+ local uri=myApp:myPass@some-name-rs0.multi-storage-10969
+ grep -E -v 'I NETWORK|W NETWORK|F NETWORK|"c":"NETWORK"|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|Started a new thread for the timer service'
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local replica_set=rs0
+ [[ myApp:myPass@some-name-rs0.multi-storage-10969 == *cfg* ]]
+ /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.JkwsYFecB3
+++ mktemp
++ local LAST_ERR=/tmp/tmp.W9iz9f50wP
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.JkwsYFecB3
++ cat /tmp/tmp.W9iz9f50wP
++ rm /tmp/tmp.JkwsYFecB3 /tmp/tmp.W9iz9f50wP
++ return 0
+ local client_container=psmdb-client-bb8b97679-7rpbx
+ kubectl_bin exec psmdb-client-bb8b97679-7rpbx -- bash -c 'printf '\''use myApp\n db.test.find().sort( { x: 1 } )\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0.multi-storage-10969.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
++ mktemp
+ local LAST_OUT=/tmp/tmp.bJ2tO6aVHQ
++ mktemp
+ local LAST_ERR=/tmp/tmp.sKJP1Cv6Fy
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-bb8b97679-7rpbx -- bash -c 'printf '\''use myApp\n db.test.find().sort( { x: 1 } )\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0.multi-storage-10969.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.bJ2tO6aVHQ
+ cat /tmp/tmp.sKJP1Cv6Fy
+ rm /tmp/tmp.bJ2tO6aVHQ /tmp/tmp.sKJP1Cv6Fy
+ return 0
+ [[ 0 -eq 0 ]]
+ diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2219/e2e-tests/multi-storage/compare/find.json /tmp/tmp.iK1Y0vUPYD/find
+ sleep_with_log 60 'wait resync to finish'
+ local d=60
+ local 'msg=wait resync to finish'
+ log 'wait resync to finish. sleeping for 60 seconds'
+ set +o xtrace
[2026-03-10T22:41:00+0000] wait resync to finish. sleeping for 60 seconds
+ sleep 60
+ log 'changing main storage from minio-1 to minio-2'
+ set +o xtrace
[2026-03-10T22:42:00+0000] changing main storage from minio-1 to minio-2
+ kubectl patch psmdb some-name --type=json '-p=[
  {"op": "remove", "path": "/spec/backup/storages/minio-1/main"},
  {"op": "add", "path": "/spec/backup/storages/minio-2/main", "value": true}
]'
perconaservermongodb.psmdb.percona.com/some-name patched
+ sleep_with_log 60 'wait resync to finish'
+ local d=60
+ local 'msg=wait resync to finish'
+ log 'wait resync to finish. sleeping for 60 seconds'
+ set +o xtrace
[2026-03-10T22:42:01+0000] wait resync to finish. sleeping for 60 seconds
+ sleep 60
+ run_backup minio-2 backup-minio-2-1 logical
+ local storage=minio-2
+ local name=backup-minio-2-1
+ local type=logical
+ log 'creating backup: backup-minio-2-1 (logical)'
+ set +o xtrace
[2026-03-10T22:43:01+0000] creating backup: backup-minio-2-1 (logical)
+ yq eval '.metadata.name = "backup-minio-2-1"
			| .spec.storageName = "minio-2"
			| .spec.type = "logical"' /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2219/e2e-tests/multi-storage/conf/backup.yml
+ kubectl_bin apply -f -
++ mktemp
+ local LAST_OUT=/tmp/tmp.bZMaAuaUxZ
++ mktemp
+ local LAST_ERR=/tmp/tmp.stdbvcOfGW
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl apply -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.bZMaAuaUxZ
perconaservermongodbbackup.psmdb.percona.com/backup-minio-2-1 created
+ cat /tmp/tmp.stdbvcOfGW
+ rm /tmp/tmp.bZMaAuaUxZ /tmp/tmp.stdbvcOfGW
+ return 0
+ wait_backup backup-minio-2-1
+ local backup_name=backup-minio-2-1
+ local target_state=ready
+ set +o xtrace
waiting for backup-minio-2-1 to reach ready state.......OK
+ write_document 100506
+ local x=100506
+ local cmp_postfix=
+ log 'write document: 100506'
+ set +o xtrace
[2026-03-10T22:43:16+0000] write document: 100506
+ run_mongo 'use myApp\n db.test.insert({ x: 100506 })' myApp:myPass@some-name-rs0.multi-storage-10969 mongodb
+ local 'command=use myApp\n db.test.insert({ x: 100506 })'
+ local uri=myApp:myPass@some-name-rs0.multi-storage-10969
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local replica_set=rs0
+ [[ myApp:myPass@some-name-rs0.multi-storage-10969 == *cfg* ]]
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.E4rNQqmopG
+++ mktemp
++ local LAST_ERR=/tmp/tmp.WKQqZHZj46
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.E4rNQqmopG
++ cat /tmp/tmp.WKQqZHZj46
++ rm /tmp/tmp.E4rNQqmopG /tmp/tmp.WKQqZHZj46
++ return 0
+ local client_container=psmdb-client-bb8b97679-7rpbx
+ kubectl_bin exec psmdb-client-bb8b97679-7rpbx -- bash -c 'printf '\''use myApp\n db.test.insert({ x: 100506 })\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0.multi-storage-10969.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
++ mktemp
+ local LAST_OUT=/tmp/tmp.c0RnZGf11U
++ mktemp
+ local LAST_ERR=/tmp/tmp.2hjIb3lDDP
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-bb8b97679-7rpbx -- bash -c 'printf '\''use myApp\n db.test.insert({ x: 100506 })\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0.multi-storage-10969.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.c0RnZGf11U
Percona Server for MongoDB shell version v4.4.29-28
connecting to: mongodb://some-name-rs0.multi-storage-10969.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb&replicaSet=rs0&ssl=false
Implicit session: session { "id" : UUID("b1a48c0c-658f-44b4-a226-f1a403e222a5") }
Percona Server for MongoDB server version: v8.0.19-7
WARNING: shell and server versions do not match
switched to db myApp
WriteResult({ "nInserted" : 1 })
bye
+ cat /tmp/tmp.2hjIb3lDDP
+ rm /tmp/tmp.c0RnZGf11U /tmp/tmp.2hjIb3lDDP
+ return 0
+ write_document 100507
+ local x=100507
+ local cmp_postfix=
+ log 'write document: 100507'
+ set +o xtrace
[2026-03-10T22:43:18+0000] write document: 100507
+ run_mongo 'use myApp\n db.test.insert({ x: 100507 })' myApp:myPass@some-name-rs0.multi-storage-10969 mongodb
+ local 'command=use myApp\n db.test.insert({ x: 100507 })'
+ local uri=myApp:myPass@some-name-rs0.multi-storage-10969
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local replica_set=rs0
+ [[ myApp:myPass@some-name-rs0.multi-storage-10969 == *cfg* ]]
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.O5qun5gvW3
+++ mktemp
++ local LAST_ERR=/tmp/tmp.6D6mc72T4a
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.O5qun5gvW3
++ cat /tmp/tmp.6D6mc72T4a
++ rm /tmp/tmp.O5qun5gvW3 /tmp/tmp.6D6mc72T4a
++ return 0
+ local client_container=psmdb-client-bb8b97679-7rpbx
+ kubectl_bin exec psmdb-client-bb8b97679-7rpbx -- bash -c 'printf '\''use myApp\n db.test.insert({ x: 100507 })\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0.multi-storage-10969.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
++ mktemp
+ local LAST_OUT=/tmp/tmp.Mgz2XPf6W8
++ mktemp
+ local LAST_ERR=/tmp/tmp.DSjWncsaRW
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-bb8b97679-7rpbx -- bash -c 'printf '\''use myApp\n db.test.insert({ x: 100507 })\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0.multi-storage-10969.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.Mgz2XPf6W8
Percona Server for MongoDB shell version v4.4.29-28
connecting to: mongodb://some-name-rs0.multi-storage-10969.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb&replicaSet=rs0&ssl=false
Implicit session: session { "id" : UUID("c892695c-0c20-4997-978f-b87147b52657") }
Percona Server for MongoDB server version: v8.0.19-7
WARNING: shell and server versions do not match
switched to db myApp
WriteResult({ "nInserted" : 1 })
bye
+ cat /tmp/tmp.DSjWncsaRW
+ rm /tmp/tmp.Mgz2XPf6W8 /tmp/tmp.DSjWncsaRW
+ return 0
+ write_document 100508
+ local x=100508
+ local cmp_postfix=
+ log 'write document: 100508'
+ set +o xtrace
[2026-03-10T22:43:21+0000] write document: 100508
+ run_mongo 'use myApp\n db.test.insert({ x: 100508 })' myApp:myPass@some-name-rs0.multi-storage-10969 mongodb
+ local 'command=use myApp\n db.test.insert({ x: 100508 })'
+ local uri=myApp:myPass@some-name-rs0.multi-storage-10969
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local replica_set=rs0
+ [[ myApp:myPass@some-name-rs0.multi-storage-10969 == *cfg* ]]
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.tsTxflouF5
+++ mktemp
++ local LAST_ERR=/tmp/tmp.Imp67fckOq
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.tsTxflouF5
++ cat /tmp/tmp.Imp67fckOq
++ rm /tmp/tmp.tsTxflouF5 /tmp/tmp.Imp67fckOq
++ return 0
+ local client_container=psmdb-client-bb8b97679-7rpbx
+ kubectl_bin exec psmdb-client-bb8b97679-7rpbx -- bash -c 'printf '\''use myApp\n db.test.insert({ x: 100508 })\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0.multi-storage-10969.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
++ mktemp
+ local LAST_OUT=/tmp/tmp.I98QdiSakL
++ mktemp
+ local LAST_ERR=/tmp/tmp.MU2wx0MKt8
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-bb8b97679-7rpbx -- bash -c 'printf '\''use myApp\n db.test.insert({ x: 100508 })\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0.multi-storage-10969.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.I98QdiSakL
Percona Server for MongoDB shell version v4.4.29-28
connecting to: mongodb://some-name-rs0.multi-storage-10969.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb&replicaSet=rs0&ssl=false
Implicit session: session { "id" : UUID("469fb408-80ed-4fca-8232-551eb085d43d") }
Percona Server for MongoDB server version: v8.0.19-7
WARNING: shell and server versions do not match
switched to db myApp
WriteResult({ "nInserted" : 1 })
bye
+ cat /tmp/tmp.MU2wx0MKt8
+ rm /tmp/tmp.I98QdiSakL /tmp/tmp.MU2wx0MKt8
+ return 0
+ write_document 100509
+ local x=100509
+ local cmp_postfix=
+ log 'write document: 100509'
+ set +o xtrace
[2026-03-10T22:43:24+0000] write document: 100509
+ run_mongo 'use myApp\n db.test.insert({ x: 100509 })' myApp:myPass@some-name-rs0.multi-storage-10969 mongodb
+ local 'command=use myApp\n db.test.insert({ x: 100509 })'
+ local uri=myApp:myPass@some-name-rs0.multi-storage-10969
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local replica_set=rs0
+ [[ myApp:myPass@some-name-rs0.multi-storage-10969 == *cfg* ]]
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.bmMDjxGOij
+++ mktemp
++ local LAST_ERR=/tmp/tmp.LRYlqQcT1d
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.bmMDjxGOij
++ cat /tmp/tmp.LRYlqQcT1d
++ rm /tmp/tmp.bmMDjxGOij /tmp/tmp.LRYlqQcT1d
++ return 0
+ local client_container=psmdb-client-bb8b97679-7rpbx
+ kubectl_bin exec psmdb-client-bb8b97679-7rpbx -- bash -c 'printf '\''use myApp\n db.test.insert({ x: 100509 })\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0.multi-storage-10969.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
++ mktemp
+ local LAST_OUT=/tmp/tmp.ibU4YRyOK9
++ mktemp
+ local LAST_ERR=/tmp/tmp.opBp2ufWkY
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-bb8b97679-7rpbx -- bash -c 'printf '\''use myApp\n db.test.insert({ x: 100509 })\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0.multi-storage-10969.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.ibU4YRyOK9
Percona Server for MongoDB shell version v4.4.29-28
connecting to: mongodb://some-name-rs0.multi-storage-10969.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb&replicaSet=rs0&ssl=false
Implicit session: session { "id" : UUID("d5e90b99-c0a1-42cc-bbf1-202a6a91b9e1") }
Percona Server for MongoDB server version: v8.0.19-7
WARNING: shell and server versions do not match
switched to db myApp
WriteResult({ "nInserted" : 1 })
bye
+ cat /tmp/tmp.opBp2ufWkY
+ rm /tmp/tmp.ibU4YRyOK9 /tmp/tmp.opBp2ufWkY
+ return 0
+ write_document 100510
+ local x=100510
+ local cmp_postfix=
+ log 'write document: 100510'
+ set +o xtrace
[2026-03-10T22:43:27+0000] write document: 100510
+ run_mongo 'use myApp\n db.test.insert({ x: 100510 })' myApp:myPass@some-name-rs0.multi-storage-10969 mongodb
+ local 'command=use myApp\n db.test.insert({ x: 100510 })'
+ local uri=myApp:myPass@some-name-rs0.multi-storage-10969
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local replica_set=rs0
+ [[ myApp:myPass@some-name-rs0.multi-storage-10969 == *cfg* ]]
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.Kko6MW1iu5
+++ mktemp
++ local LAST_ERR=/tmp/tmp.dqWFlGNonX
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.Kko6MW1iu5
++ cat /tmp/tmp.dqWFlGNonX
++ rm /tmp/tmp.Kko6MW1iu5 /tmp/tmp.dqWFlGNonX
++ return 0
+ local client_container=psmdb-client-bb8b97679-7rpbx
+ kubectl_bin exec psmdb-client-bb8b97679-7rpbx -- bash -c 'printf '\''use myApp\n db.test.insert({ x: 100510 })\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0.multi-storage-10969.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
++ mktemp
+ local LAST_OUT=/tmp/tmp.DaE41YhbxY
++ mktemp
+ local LAST_ERR=/tmp/tmp.NUblXaRz46
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-bb8b97679-7rpbx -- bash -c 'printf '\''use myApp\n db.test.insert({ x: 100510 })\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0.multi-storage-10969.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.DaE41YhbxY
Percona Server for MongoDB shell version v4.4.29-28
connecting to: mongodb://some-name-rs0.multi-storage-10969.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb&replicaSet=rs0&ssl=false
Implicit session: session { "id" : UUID("c49d6baa-71d3-47c0-8fbb-646a96e13dbb") }
Percona Server for MongoDB server version: v8.0.19-7
WARNING: shell and server versions do not match
switched to db myApp
WriteResult({ "nInserted" : 1 })
bye
+ cat /tmp/tmp.NUblXaRz46
+ rm /tmp/tmp.DaE41YhbxY /tmp/tmp.NUblXaRz46
+ return 0
+ compare_mongo_cmd find myApp:myPass@some-name-rs0.multi-storage-10969 -2nd .svc.cluster.local myApp test 'sort( { x: 1 } )'
+ local command=find
+ local uri=myApp:myPass@some-name-rs0.multi-storage-10969
+ local postfix=-2nd
+ local suffix=.svc.cluster.local
+ local database=myApp
+ local collection=test
+ local 'sort=sort( { x: 1 } )'
+ local tls=false
+ local replicaset=
+ local 'full_command=db.test.find()'
+ [[ -n sort( { x: 1 } ) ]]
+ full_command='db.test.find().sort( { x: 1 } )'
+ log 'running db.test.find().sort( { x: 1 } ) in myApp'
+ set +o xtrace
[2026-03-10T22:43:29+0000] running db.test.find().sort( { x: 1 } ) in myApp
+ [[ false == \t\r\u\e ]]
+ mongo_command=run_mongo
+ run_mongo 'use myApp\n db.test.find().sort( { x: 1 } )' myApp:myPass@some-name-rs0.multi-storage-10969 mongodb .svc.cluster.local ''
+ /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
+ grep -E -v 'I NETWORK|W NETWORK|F NETWORK|"c":"NETWORK"|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|Started a new thread for the timer service'
+ local 'command=use myApp\n db.test.find().sort( { x: 1 } )'
+ local uri=myApp:myPass@some-name-rs0.multi-storage-10969
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local replica_set=rs0
+ [[ myApp:myPass@some-name-rs0.multi-storage-10969 == *cfg* ]]
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.jJqyvJmeaM
+++ mktemp
++ local LAST_ERR=/tmp/tmp.X5JjdT9AeF
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.jJqyvJmeaM
++ cat /tmp/tmp.X5JjdT9AeF
++ rm /tmp/tmp.jJqyvJmeaM /tmp/tmp.X5JjdT9AeF
++ return 0
+ local client_container=psmdb-client-bb8b97679-7rpbx
+ kubectl_bin exec psmdb-client-bb8b97679-7rpbx -- bash -c 'printf '\''use myApp\n db.test.find().sort( { x: 1 } )\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0.multi-storage-10969.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
++ mktemp
+ local LAST_OUT=/tmp/tmp.dB5WhIAQxx
++ mktemp
+ local LAST_ERR=/tmp/tmp.2NS0FpYrmA
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-bb8b97679-7rpbx -- bash -c 'printf '\''use myApp\n db.test.find().sort( { x: 1 } )\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0.multi-storage-10969.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.dB5WhIAQxx
+ cat /tmp/tmp.2NS0FpYrmA
+ rm /tmp/tmp.dB5WhIAQxx /tmp/tmp.2NS0FpYrmA
+ return 0
+ [[ 0 -eq 0 ]]
+ diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2219/e2e-tests/multi-storage/compare/find-2nd.json /tmp/tmp.iK1Y0vUPYD/find-2nd
+ wait_for_restorable_time backup-minio-2-1
+ local backup_name=backup-minio-2-1
++ get_latest_restorable_time backup-minio-2-1
++ local backup_name=backup-minio-2-1
+++ kubectl_bin get psmdb-backup backup-minio-2-1 -o yaml
+++ yq .status.latestRestorableTime
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.CMFE9A2z00
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.XLH1YVzSxQ
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in $(seq 0 2)
+++ set +e
+++ kubectl get psmdb-backup backup-minio-2-1 -o yaml
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.CMFE9A2z00
+++ cat /tmp/tmp.XLH1YVzSxQ
+++ rm /tmp/tmp.CMFE9A2z00 /tmp/tmp.XLH1YVzSxQ
+++ return 0
++ local time=2026-03-10T22:43:09Z
++ [[ 2026-03-10T22:43:09Z == \n\u\l\l ]]
++ echo -n 2026-03-10T22:43:09Z
+ local latest_restorable_time=2026-03-10T22:43:09Z
+ local retries=0
+ echo -n 'waiting for psmdb-backup/backup-minio-2-1'\''s latest restorable time'
waiting for psmdb-backup/backup-minio-2-1's latest restorable time+ [[ 2026-03-10T22:43:09Z != \n\u\l\l ]]
+ echo '.OK latestRestorableTime: 2026-03-10T22:43:09Z'
.OK latestRestorableTime: 2026-03-10T22:43:09Z
+ wait_for_advanced_restorable_time backup-minio-2-1
+ local backup_name=backup-minio-2-1
+++ get_latest_restorable_time backup-minio-2-1
+++ local backup_name=backup-minio-2-1
++++ kubectl_bin get psmdb-backup backup-minio-2-1 -o yaml
++++ yq .status.latestRestorableTime
+++++ mktemp
++++ local LAST_OUT=/tmp/tmp.vt8RiTkunX
+++++ mktemp
++++ local LAST_ERR=/tmp/tmp.reEkVemPHz
++++ local exit_status=0
++++ local timeout=4
+++++ seq 0 2
++++ for i in $(seq 0 2)
++++ set +e
++++ kubectl get psmdb-backup backup-minio-2-1 -o yaml
++++ exit_status=0
++++ set -e
++++ '[' 0 '!=' 0 -a -n 1 ']'
++++ break
++++ cat /tmp/tmp.vt8RiTkunX
++++ cat /tmp/tmp.reEkVemPHz
++++ rm /tmp/tmp.vt8RiTkunX /tmp/tmp.reEkVemPHz
++++ return 0
+++ local time=2026-03-10T22:43:09Z
+++ [[ 2026-03-10T22:43:09Z == \n\u\l\l ]]
+++ echo -n 2026-03-10T22:43:09Z
++ datetime_to_timestamp 2026-03-10T22:43:09Z
++ local datetime=2026-03-10T22:43:09Z
++ TZ=UTC
++ /usr/sbin/date -d2026-03-10T22:43:09Z +%s
+ local latest_restorable_time=1773182589
++ get_backup_last_write backup-minio-2-1
++ local backup_name=backup-minio-2-1
+++ yq .status.pbmName
+++ kubectl_bin get psmdb-backup backup-minio-2-1 -o yaml
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.AxxVPUX5V0
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.SxRD2DkiHg
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in $(seq 0 2)
+++ set +e
+++ kubectl get psmdb-backup backup-minio-2-1 -o yaml
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.AxxVPUX5V0
+++ cat /tmp/tmp.SxRD2DkiHg
+++ rm /tmp/tmp.AxxVPUX5V0 /tmp/tmp.SxRD2DkiHg
+++ return 0
++ local pbm_name=2026-03-10T22:43:04Z
++ kubectl_bin exec some-name-rs0-0 -c backup-agent -- pbm list -o json
++ jq --arg v 2026-03-10T22:43:04Z '.snapshots[] | select(.name==$v) | .restoreTo'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.8eAfYzhaPm
+++ mktemp
++ local LAST_ERR=/tmp/tmp.99LpgeTMp0
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl exec some-name-rs0-0 -c backup-agent -- pbm list -o json
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.8eAfYzhaPm
++ cat /tmp/tmp.99LpgeTMp0
++ rm /tmp/tmp.8eAfYzhaPm /tmp/tmp.99LpgeTMp0
++ return 0
+ local latest_write=1773182589
+ local retries=0
++ timestamp_to_datetime 1773182589
++ local ts=1773182589
++ TZ=UTC
++ /usr/sbin/date -d@1773182589 +%Y-%m-%dT%H:%M:%S%Z
+ echo -n 'waiting for psmdb-backup/backup-minio-2-1'\''s latest restorable time to advance its latest write (2026-03-10T22:43:09UTC)'
waiting for psmdb-backup/backup-minio-2-1's latest restorable time to advance its latest write (2026-03-10T22:43:09UTC)+ [[ 1773182589 -gt 1773182589 ]]
+++ get_latest_restorable_time backup-minio-2-1
+++ local backup_name=backup-minio-2-1
++++ kubectl_bin get psmdb-backup backup-minio-2-1 -o yaml
++++ yq .status.latestRestorableTime
+++++ mktemp
++++ local LAST_OUT=/tmp/tmp.5gDJhTZaCo
+++++ mktemp
++++ local LAST_ERR=/tmp/tmp.7FZpCDyFF1
++++ local exit_status=0
++++ local timeout=4
+++++ seq 0 2
++++ for i in $(seq 0 2)
++++ set +e
++++ kubectl get psmdb-backup backup-minio-2-1 -o yaml
++++ exit_status=0
++++ set -e
++++ '[' 0 '!=' 0 -a -n 1 ']'
++++ break
++++ cat /tmp/tmp.5gDJhTZaCo
++++ cat /tmp/tmp.7FZpCDyFF1
++++ rm /tmp/tmp.5gDJhTZaCo /tmp/tmp.7FZpCDyFF1
++++ return 0
+++ local time=2026-03-10T22:43:09Z
+++ [[ 2026-03-10T22:43:09Z == \n\u\l\l ]]
+++ echo -n 2026-03-10T22:43:09Z
++ datetime_to_timestamp 2026-03-10T22:43:09Z
++ local datetime=2026-03-10T22:43:09Z
++ TZ=UTC
++ /usr/sbin/date -d2026-03-10T22:43:09Z +%s
+ latest_restorable_time=1773182589
+ [[ 0 -gt 5 ]]
+ let retries+=1
+ echo -n .
.+ sleep_with_log 70 'wait for oplog span min + 10 sec'
+ local d=70
+ local 'msg=wait for oplog span min + 10 sec'
+ log 'wait for oplog span min + 10 sec. sleeping for 70 seconds'
+ set +o xtrace
[2026-03-10T22:43:38+0000] wait for oplog span min + 10 sec. sleeping for 70 seconds
+ sleep 70
+ [[ 1773182589 -gt 1773182589 ]]
+++ get_latest_restorable_time backup-minio-2-1
+++ local backup_name=backup-minio-2-1
++++ kubectl_bin get psmdb-backup backup-minio-2-1 -o yaml
++++ yq .status.latestRestorableTime
+++++ mktemp
++++ local LAST_OUT=/tmp/tmp.MNot1wtKCZ
+++++ mktemp
++++ local LAST_ERR=/tmp/tmp.iR8CSxOwyg
++++ local exit_status=0
++++ local timeout=4
+++++ seq 0 2
++++ for i in $(seq 0 2)
++++ set +e
++++ kubectl get psmdb-backup backup-minio-2-1 -o yaml
++++ exit_status=0
++++ set -e
++++ '[' 0 '!=' 0 -a -n 1 ']'
++++ break
++++ cat /tmp/tmp.MNot1wtKCZ
++++ cat /tmp/tmp.iR8CSxOwyg
++++ rm /tmp/tmp.MNot1wtKCZ /tmp/tmp.iR8CSxOwyg
++++ return 0
+++ local time=2026-03-10T22:44:28Z
+++ [[ 2026-03-10T22:44:28Z == \n\u\l\l ]]
+++ echo -n 2026-03-10T22:44:28Z
++ datetime_to_timestamp 2026-03-10T22:44:28Z
++ local datetime=2026-03-10T22:44:28Z
++ TZ=UTC
++ /usr/sbin/date -d2026-03-10T22:44:28Z +%s
+ latest_restorable_time=1773182668
+ [[ 1 -gt 5 ]]
+ let retries+=1
+ echo -n .
.+ sleep_with_log 70 'wait for oplog span min + 10 sec'
+ local d=70
+ local 'msg=wait for oplog span min + 10 sec'
+ log 'wait for oplog span min + 10 sec. sleeping for 70 seconds'
+ set +o xtrace
[2026-03-10T22:44:49+0000] wait for oplog span min + 10 sec. sleeping for 70 seconds
+ sleep 70
+ [[ 1773182668 -gt 1773182589 ]]
++ timestamp_to_datetime 1773182668
++ local ts=1773182668
++ TZ=UTC
++ /usr/sbin/date -d@1773182668 +%Y-%m-%dT%H:%M:%S%Z
+ echo '.OK latestRestorableTime: 2026-03-10T22:44:28UTC'
.OK latestRestorableTime: 2026-03-10T22:44:28UTC
++ get_latest_restorable_time backup-minio-2-1
++ local backup_name=backup-minio-2-1
+++ kubectl_bin get psmdb-backup backup-minio-2-1 -o yaml
+++ yq .status.latestRestorableTime
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.S6vDTcJZia
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.iHoS0WCj4I
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in $(seq 0 2)
+++ set +e
+++ kubectl get psmdb-backup backup-minio-2-1 -o yaml
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.S6vDTcJZia
+++ cat /tmp/tmp.iHoS0WCj4I
+++ rm /tmp/tmp.S6vDTcJZia /tmp/tmp.iHoS0WCj4I
+++ return 0
++ local time=2026-03-10T22:45:28Z
++ [[ 2026-03-10T22:45:28Z == \n\u\l\l ]]
++ echo -n 2026-03-10T22:45:28Z
+ restore_time=2026-03-10T22:45:28Z
++ format_pitr_target 2026-03-10T22:45:28Z
++ local target=2026-03-10T22:45:28Z
++ echo 2026-03-10T22:45:28Z
++ sed 's/T/ /'
++ sed s/Z//
+ check_recovery backup-minio-2-1 '2026-03-10 22:45:28' -2nd
+ local backup_name=backup-minio-2-1
+ local 'restore_time=2026-03-10 22:45:28'
+ local cmp_postfix=-2nd
+ drop_collection myApp test myApp myPass
+ local db=myApp
+ local collection=test
+ local user=myApp
+ local pass=myPass
+ log 'dropping collection: myApp.test'
+ set +o xtrace
[2026-03-10T22:46:00+0000] dropping collection: myApp.test
+ run_mongo 'use myApp\n db.test.drop()' myApp:myPass@some-name-rs0.multi-storage-10969 mongodb
+ local 'command=use myApp\n db.test.drop()'
+ local uri=myApp:myPass@some-name-rs0.multi-storage-10969
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local replica_set=rs0
+ [[ myApp:myPass@some-name-rs0.multi-storage-10969 == *cfg* ]]
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.Z1Px2Ks8J3
+++ mktemp
++ local LAST_ERR=/tmp/tmp.szUucwyQON
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.Z1Px2Ks8J3
++ cat /tmp/tmp.szUucwyQON
++ rm /tmp/tmp.Z1Px2Ks8J3 /tmp/tmp.szUucwyQON
++ return 0
+ local client_container=psmdb-client-bb8b97679-7rpbx
+ kubectl_bin exec psmdb-client-bb8b97679-7rpbx -- bash -c 'printf '\''use myApp\n db.test.drop()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0.multi-storage-10969.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
++ mktemp
+ local LAST_OUT=/tmp/tmp.jqCAg1IylL
++ mktemp
+ local LAST_ERR=/tmp/tmp.nkFzrYZihS
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-bb8b97679-7rpbx -- bash -c 'printf '\''use myApp\n db.test.drop()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0.multi-storage-10969.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.jqCAg1IylL
Percona Server for MongoDB shell version v4.4.29-28
connecting to: mongodb://some-name-rs0.multi-storage-10969.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb&replicaSet=rs0&ssl=false
Implicit session: session { "id" : UUID("d44e0e5f-43fe-49c5-bc77-0da5b1189f8c") }
Percona Server for MongoDB server version: v8.0.19-7
WARNING: shell and server versions do not match
switched to db myApp
true
bye
+ cat /tmp/tmp.nkFzrYZihS
+ rm /tmp/tmp.jqCAg1IylL /tmp/tmp.nkFzrYZihS
+ return 0
+ log 'creating restore: restore-backup-minio-2-1 restore_to: 2026-03-10 22:45:28'
+ set +o xtrace
[2026-03-10T22:46:02+0000] creating restore: restore-backup-minio-2-1 restore_to: 2026-03-10 22:45:28
+ cat /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2219/e2e-tests/multi-storage/conf/restore.yml
+ /usr/sbin/sed -e 's/name:/name: restore-backup-minio-2-1/'
+ /usr/sbin/sed -e 's/backupName:/backupName: backup-minio-2-1/'
+ /usr/sbin/sed -e 's/date:/date: 2026-03-10 22:45:28/'
+ kubectl_bin apply -f -
++ mktemp
+ local LAST_OUT=/tmp/tmp.LCUF9KcRQo
++ mktemp
+ local LAST_ERR=/tmp/tmp.JNxmC2kZFa
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl apply -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.LCUF9KcRQo
perconaservermongodbrestore.psmdb.percona.com/restore-backup-minio-2-1 created
+ cat /tmp/tmp.JNxmC2kZFa
+ rm /tmp/tmp.LCUF9KcRQo /tmp/tmp.JNxmC2kZFa
+ return 0
+ wait_restore backup-minio-2-1 some-name requested 0 900 1
+ local backup_name=backup-minio-2-1
+ local cluster_name=some-name
+ local target_state=requested
+ local wait_cluster_consistency=0
+ local wait_time=900
+ local ok_if_ready=1
+ set +o xtrace
Waiting for the psmdb-restore/restore-backup-minio-2-1 object to be created.OK
Waiting psmdb-restore/restore-backup-minio-2-1 to reach state "requested" .OK after 0 minutes
+ [[ 0 -eq 1 ]]
+ wait_restore backup-minio-2-1 some-name ready 0 1600
+ local backup_name=backup-minio-2-1
+ local cluster_name=some-name
+ local target_state=ready
+ local wait_cluster_consistency=0
+ local wait_time=1600
+ local ok_if_ready=0
+ set +o xtrace
Waiting for the psmdb-restore/restore-backup-minio-2-1 object to be created.OK
Waiting psmdb-restore/restore-backup-minio-2-1 to reach state "ready" .OK after 0 minutes
+ [[ 0 -eq 1 ]]
+ wait_for_running some-name-rs0 3
+ local name=some-name-rs0
+ let last_pod=2
+ local check_cluster_readyness=true
+ set_debug
+ [[ 1 == 1 ]]
+ set -o xtrace
+ local rs_name=rs0
+ local cluster_name=some-name
++ seq 0 2
+ for i in $(seq 0 $last_pod)
+ [[ 0 -eq 2 ]]
+ wait_pod some-name-rs0-0
+ local pod=some-name-rs0-0
+ set +o xtrace
waiting for pod/some-name-rs0-0 to be ready.OK
+ for i in $(seq 0 $last_pod)
+ [[ 1 -eq 2 ]]
+ wait_pod some-name-rs0-1
+ local pod=some-name-rs0-1
+ set +o xtrace
waiting for pod/some-name-rs0-1 to be ready.OK
+ for i in $(seq 0 $last_pod)
+ [[ 2 -eq 2 ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].arbiter.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.Esbg6d4Ksl
+++ mktemp
++ local LAST_ERR=/tmp/tmp.b9v5XX91Eu
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].arbiter.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.Esbg6d4Ksl
++ cat /tmp/tmp.b9v5XX91Eu
++ rm /tmp/tmp.Esbg6d4Ksl /tmp/tmp.b9v5XX91Eu
++ return 0
+ [[ '' == \t\r\u\e ]]
+ wait_pod some-name-rs0-2
+ local pod=some-name-rs0-2
+ set +o xtrace
waiting for pod/some-name-rs0-2 to be ready.OK
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].nonvoting.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.vxiZ9Q0lWY
+++ mktemp
++ local LAST_ERR=/tmp/tmp.CAq7jN8Nbm
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].nonvoting.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.vxiZ9Q0lWY
++ cat /tmp/tmp.CAq7jN8Nbm
++ rm /tmp/tmp.vxiZ9Q0lWY /tmp/tmp.CAq7jN8Nbm
++ return 0
+ [[ '' == \t\r\u\e ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].hidden.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.viMMxuFpzX
+++ mktemp
++ local LAST_ERR=/tmp/tmp.WdMUHTd6iH
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].hidden.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.viMMxuFpzX
++ cat /tmp/tmp.WdMUHTd6iH
++ rm /tmp/tmp.viMMxuFpzX /tmp/tmp.WdMUHTd6iH
++ return 0
+ [[ '' == \t\r\u\e ]]
+ sleep 10
+ [[ true == \t\r\u\e ]]
+ set +x
Waiting for cluster readyness
+ compare_mongo_cmd find myApp:myPass@some-name-rs0.multi-storage-10969 -2nd .svc.cluster.local myApp test 'sort( { x: 1 } )'
+ local command=find
+ local uri=myApp:myPass@some-name-rs0.multi-storage-10969
+ local postfix=-2nd
+ local suffix=.svc.cluster.local
+ local database=myApp
+ local collection=test
+ local 'sort=sort( { x: 1 } )'
+ local tls=false
+ local replicaset=
+ local 'full_command=db.test.find()'
+ [[ -n sort( { x: 1 } ) ]]
+ full_command='db.test.find().sort( { x: 1 } )'
+ log 'running db.test.find().sort( { x: 1 } ) in myApp'
+ set +o xtrace
[2026-03-10T22:46:45+0000] running db.test.find().sort( { x: 1 } ) in myApp
+ [[ false == \t\r\u\e ]]
+ mongo_command=run_mongo
+ run_mongo 'use myApp\n db.test.find().sort( { x: 1 } )' myApp:myPass@some-name-rs0.multi-storage-10969 mongodb .svc.cluster.local ''
+ local 'command=use myApp\n db.test.find().sort( { x: 1 } )'
+ local uri=myApp:myPass@some-name-rs0.multi-storage-10969
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local replica_set=rs0
+ [[ myApp:myPass@some-name-rs0.multi-storage-10969 == *cfg* ]]
+ grep -E -v 'I NETWORK|W NETWORK|F NETWORK|"c":"NETWORK"|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|Started a new thread for the timer service'
+ /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.lyBlrDYk4V
+++ mktemp
++ local LAST_ERR=/tmp/tmp.tWY7Kgjage
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.lyBlrDYk4V
++ cat /tmp/tmp.tWY7Kgjage
++ rm /tmp/tmp.lyBlrDYk4V /tmp/tmp.tWY7Kgjage
++ return 0
+ local client_container=psmdb-client-bb8b97679-7rpbx
+ kubectl_bin exec psmdb-client-bb8b97679-7rpbx -- bash -c 'printf '\''use myApp\n db.test.find().sort( { x: 1 } )\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0.multi-storage-10969.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
++ mktemp
+ local LAST_OUT=/tmp/tmp.5IA2la2ma2
++ mktemp
+ local LAST_ERR=/tmp/tmp.F2oDviYnb0
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-bb8b97679-7rpbx -- bash -c 'printf '\''use myApp\n db.test.find().sort( { x: 1 } )\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0.multi-storage-10969.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.5IA2la2ma2
+ cat /tmp/tmp.F2oDviYnb0
+ rm /tmp/tmp.5IA2la2ma2 /tmp/tmp.F2oDviYnb0
+ return 0
+ [[ 0 -eq 0 ]]
+ diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2219/e2e-tests/multi-storage/compare/find-2nd.json /tmp/tmp.iK1Y0vUPYD/find-2nd
+ log 'dropping pbmBackups collection and starting resync'
+ set +o xtrace
[2026-03-10T22:46:48+0000] dropping pbmBackups collection and starting resync
+ drop_metadata_and_resync
++ kubectl get psmdb-backup --no-headers
++ wc -l
+ local psmdb_backup_count=4
+ log 'there are 4 psmdb-backups in multi-storage-10969'
+ set +o xtrace
[2026-03-10T22:46:49+0000] there are 4 psmdb-backups in multi-storage-10969
+ drop_collection admin pbmBackups backup backup123456
+ local db=admin
+ local collection=pbmBackups
+ local user=backup
+ local pass=backup123456
+ log 'dropping collection: admin.pbmBackups'
+ set +o xtrace
[2026-03-10T22:46:49+0000] dropping collection: admin.pbmBackups
+ run_mongo 'use admin\n db.pbmBackups.drop()' backup:backup123456@some-name-rs0.multi-storage-10969 mongodb
+ local 'command=use admin\n db.pbmBackups.drop()'
+ local uri=backup:backup123456@some-name-rs0.multi-storage-10969
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local replica_set=rs0
+ [[ backup:backup123456@some-name-rs0.multi-storage-10969 == *cfg* ]]
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.S3bs1V9XT0
+++ mktemp
++ local LAST_ERR=/tmp/tmp.6yyB06nU8k
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.S3bs1V9XT0
++ cat /tmp/tmp.6yyB06nU8k
++ rm /tmp/tmp.S3bs1V9XT0 /tmp/tmp.6yyB06nU8k
++ return 0
+ local client_container=psmdb-client-bb8b97679-7rpbx
+ kubectl_bin exec psmdb-client-bb8b97679-7rpbx -- bash -c 'printf '\''use admin\n db.pbmBackups.drop()\n'\'' | mongo mongodb://backup:backup123456@some-name-rs0.multi-storage-10969.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
++ mktemp
+ local LAST_OUT=/tmp/tmp.GZLGmH7BdN
++ mktemp
+ local LAST_ERR=/tmp/tmp.1N5Gm0fcpe
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-bb8b97679-7rpbx -- bash -c 'printf '\''use admin\n db.pbmBackups.drop()\n'\'' | mongo mongodb://backup:backup123456@some-name-rs0.multi-storage-10969.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.GZLGmH7BdN
Percona Server for MongoDB shell version v4.4.29-28
connecting to: mongodb://some-name-rs0.multi-storage-10969.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb&replicaSet=rs0&ssl=false
Implicit session: session { "id" : UUID("5b5fb6d4-4bfc-4072-959c-336bbe377438") }
Percona Server for MongoDB server version: v8.0.19-7
WARNING: shell and server versions do not match
switched to db admin
true
bye
+ cat /tmp/tmp.1N5Gm0fcpe
+ rm /tmp/tmp.GZLGmH7BdN /tmp/tmp.1N5Gm0fcpe
+ return 0
+ kubectl_bin annotate psmdb some-name percona.com/resync-pbm=true
++ mktemp
+ local LAST_OUT=/tmp/tmp.7hejHIDW5n
++ mktemp
+ local LAST_ERR=/tmp/tmp.Afw4IPTEkY
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl annotate psmdb some-name percona.com/resync-pbm=true
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.7hejHIDW5n
perconaservermongodb.psmdb.percona.com/some-name annotated
+ cat /tmp/tmp.Afw4IPTEkY
+ rm /tmp/tmp.7hejHIDW5n /tmp/tmp.Afw4IPTEkY
+ return 0
+ sleep_with_log 10 'wait resync to start'
+ local d=10
+ local 'msg=wait resync to start'
+ log 'wait resync to start. sleeping for 10 seconds'
+ set +o xtrace
[2026-03-10T22:46:53+0000] wait resync to start. sleeping for 10 seconds
+ sleep 10
+ local retries=0
+ echo -n 'waiting for resync to finish'
waiting for resync to finish++ kubectl_bin get psmdb some-name -o yaml
++ yq '.metadata.annotations."percona.com/resync-in-progress"'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.E6nD2aCRgs
+++ mktemp
++ local LAST_ERR=/tmp/tmp.W1XTC47k7Y
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o yaml
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.E6nD2aCRgs
++ cat /tmp/tmp.W1XTC47k7Y
++ rm /tmp/tmp.E6nD2aCRgs /tmp/tmp.W1XTC47k7Y
++ return 0
+ local resync_running=null
+ [[ null == \n\u\l\l ]]
+ echo .OK
.OK
++ kubectl exec some-name-rs0-0 -c backup-agent -- pbm list -o json
++ jq '.snapshots | length'
+ local backup_count=4
+ log 'there are 4 backups in pbmBackups collection after resync'
+ set +o xtrace
[2026-03-10T22:47:06+0000] there are 4 backups in pbmBackups collection after resync
++ echo 4
++ xargs
++ echo 4
++ xargs
+ [[ 4 != 4 ]]
+ log 'resync finished successfully'
+ set +o xtrace
[2026-03-10T22:47:06+0000] resync finished successfully
+ log 'checking backup deletion'
+ set +o xtrace
[2026-03-10T22:47:06+0000] checking backup deletion
+ check_backup_deletion
++ kubectl_bin exec some-name-rs0-0 -c backup-agent -- pbm status -o json
++ jq '.backups.pitrChunks.pitrChunks[0].range.start'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.VuCqBqv7EH
+++ mktemp
++ local LAST_ERR=/tmp/tmp.680vSBoQ5N
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl exec some-name-rs0-0 -c backup-agent -- pbm status -o json
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.VuCqBqv7EH
++ cat /tmp/tmp.680vSBoQ5N
++ rm /tmp/tmp.VuCqBqv7EH /tmp/tmp.680vSBoQ5N
++ return 0
+ local initial_chunk_start=1773182585
+ log 'deleting psmdb-backup/backup-minio-3'
+ set +o xtrace
[2026-03-10T22:47:07+0000] deleting psmdb-backup/backup-minio-3
+ kubectl_bin delete psmdb-backup backup-minio-3
++ mktemp
+ local LAST_OUT=/tmp/tmp.FjzyuaEPCM
++ mktemp
+ local LAST_ERR=/tmp/tmp.ICAAVc8gwq
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl delete psmdb-backup backup-minio-3
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.FjzyuaEPCM
perconaservermongodbbackup.psmdb.percona.com "backup-minio-3" deleted from multi-storage-10969 namespace
+ cat /tmp/tmp.ICAAVc8gwq
+ rm /tmp/tmp.FjzyuaEPCM /tmp/tmp.ICAAVc8gwq
+ return 0
+ log 'checking if chunks are deleted'
+ set +o xtrace
[2026-03-10T22:47:09+0000] checking if chunks are deleted
++ kubectl_bin exec some-name-rs0-0 -c backup-agent -- pbm status -o json
++ jq '.backups.pitrChunks.pitrChunks[0].range.start'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.TdrgVsZXg5
+++ mktemp
++ local LAST_ERR=/tmp/tmp.PaA66q4nQx
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl exec some-name-rs0-0 -c backup-agent -- pbm status -o json
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.TdrgVsZXg5
++ cat /tmp/tmp.PaA66q4nQx
++ rm /tmp/tmp.TdrgVsZXg5 /tmp/tmp.PaA66q4nQx
++ return 0
+ local chunk_start=1773182585
+ [[ 1773182585 != 1773182585 ]]
+ log 'chunks are not deleted, OK.'
+ set +o xtrace
[2026-03-10T22:47:11+0000] chunks are not deleted, OK.
+ log 'deleting psmdb-backup/backup-minio-2-1'
+ set +o xtrace
[2026-03-10T22:47:11+0000] deleting psmdb-backup/backup-minio-2-1
+ kubectl_bin delete psmdb-backup backup-minio-2-1
++ mktemp
+ local LAST_OUT=/tmp/tmp.CvAa9HloSl
++ mktemp
+ local LAST_ERR=/tmp/tmp.P9CnAA7LlN
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl delete psmdb-backup backup-minio-2-1
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.CvAa9HloSl
perconaservermongodbbackup.psmdb.percona.com "backup-minio-2-1" deleted from multi-storage-10969 namespace
+ cat /tmp/tmp.P9CnAA7LlN
+ rm /tmp/tmp.CvAa9HloSl /tmp/tmp.P9CnAA7LlN
+ return 0
+ log 'checking if chunks are deleted'
+ set +o xtrace
[2026-03-10T22:47:12+0000] checking if chunks are deleted
++ kubectl_bin exec some-name-rs0-0 -c backup-agent -- pbm status -o json
++ jq '.backups.pitrChunks.pitrChunks[0].range.start'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.qgdxiaWGqW
+++ mktemp
++ local LAST_ERR=/tmp/tmp.yIisFSMj0i
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl exec some-name-rs0-0 -c backup-agent -- pbm status -o json
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.qgdxiaWGqW
++ cat /tmp/tmp.yIisFSMj0i
++ rm /tmp/tmp.qgdxiaWGqW /tmp/tmp.yIisFSMj0i
++ return 0
+ chunk_start=1773182668
+ [[ 1773182585 == 1773182668 ]]
+ log 'chunks are deleted, OK.'
+ set +o xtrace
[2026-03-10T22:47:14+0000] chunks are deleted, OK.
+ destroy multi-storage-10969
+ local namespace=multi-storage-10969
+ local ignore_logs=true
+ [[ 0 == 1 ]]
+ desc 'destroy cluster/operator and all other resources'
+ set +o xtrace


-----------------------------------------------------------------------------------
destroy cluster/operator and all other resources
-----------------------------------------------------------------------------------

+ '[' true == false ']'
+ delete_backups
+ desc 'Delete psmdb-backup'
+ set +o xtrace


-----------------------------------------------------------------------------------
Delete psmdb-backup
-----------------------------------------------------------------------------------

++ kubectl_bin get psmdb-backup --no-headers
++ wc -l
+++ mktemp
++ local LAST_OUT=/tmp/tmp.jKTs3r4s7I
+++ mktemp
++ local LAST_ERR=/tmp/tmp.hcGgRebooQ
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb-backup --no-headers
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.jKTs3r4s7I
++ cat /tmp/tmp.hcGgRebooQ
++ rm /tmp/tmp.jKTs3r4s7I /tmp/tmp.hcGgRebooQ
++ return 0
+ '[' 2 '!=' 0 ']'
+ kubectl_bin get psmdb-backup
++ mktemp
+ local LAST_OUT=/tmp/tmp.T42BxBbHPU
++ mktemp
+ local LAST_ERR=/tmp/tmp.z2CpD98Rly
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl get psmdb-backup
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.T42BxBbHPU
NAME             CLUSTER     STORAGE   DESTINATION                                    TYPE      SIZE      STATUS   COMPLETED   AGE
backup-minio-1   some-name   minio-1   s3://operator-testing/2026-03-10T22:33:06Z     logical   48.20KB   ready    14m         14m
backup-minio-2   some-name   minio-2   s3://operator-testing-2/2026-03-10T22:33:20Z   logical   52.95KB   ready    13m         13m
+ cat /tmp/tmp.z2CpD98Rly
+ rm /tmp/tmp.T42BxBbHPU /tmp/tmp.z2CpD98Rly
+ return 0
+ kubectl_bin delete psmdb-backup --all
++ mktemp
+ local LAST_OUT=/tmp/tmp.bOeo2PWvfK
++ mktemp
+ local LAST_ERR=/tmp/tmp.K1Sl5UkF6T
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl delete psmdb-backup --all
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.bOeo2PWvfK
perconaservermongodbbackup.psmdb.percona.com "backup-minio-1" deleted from multi-storage-10969 namespace
perconaservermongodbbackup.psmdb.percona.com "backup-minio-2" deleted from multi-storage-10969 namespace
+ cat /tmp/tmp.K1Sl5UkF6T
+ rm /tmp/tmp.bOeo2PWvfK /tmp/tmp.K1Sl5UkF6T
+ return 0
+ delete_crd
+ desc 'get and delete old CRDs and RBAC'
+ set +o xtrace


-----------------------------------------------------------------------------------
get and delete old CRDs and RBAC
-----------------------------------------------------------------------------------

+ kubectl_bin delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2219/deploy/crd.yaml --ignore-not-found --wait=false
++ mktemp
+ local LAST_OUT=/tmp/tmp.N4MPofLZXm
++ mktemp
+ local LAST_ERR=/tmp/tmp.LlGjXs2SW2
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2219/deploy/crd.yaml --ignore-not-found --wait=false
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.N4MPofLZXm
customresourcedefinition.apiextensions.k8s.io "perconaservermongodbbackups.psmdb.percona.com" deleted
customresourcedefinition.apiextensions.k8s.io "perconaservermongodbrestores.psmdb.percona.com" deleted
customresourcedefinition.apiextensions.k8s.io "perconaservermongodbs.psmdb.percona.com" deleted
+ cat /tmp/tmp.LlGjXs2SW2
+ rm /tmp/tmp.N4MPofLZXm /tmp/tmp.LlGjXs2SW2
+ return 0
++ yq eval .metadata.name /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2219/deploy/crd.yaml
++ grep -v '\-\-\-'
grep: warning: stray \ before -
grep: warning: stray \ before -
+ for crd_name in $(yq eval '.metadata.name' "${src_dir}/deploy/crd.yaml" | grep -v '\-\-\-')
+ kubectl get perconaservermongodbbackups.psmdb.percona.com --all-namespaces -o wide
+ grep -v NAMESPACE
+ xargs -L 1 sh -xc 'kubectl patch perconaservermongodbbackups.psmdb.percona.com -n $0 $1 --type=merge -p "{\"metadata\":{\"finalizers\":[]}}"'
error: the server doesn't have a resource type "perconaservermongodbbackups"
+ kubectl patch perconaservermongodbbackups.psmdb.percona.com -n sh --type=merge -p '{"metadata":{"finalizers":[]}}'
error: the server doesn't have a resource type "perconaservermongodbbackups"
+ :
+ kubectl_bin wait --for=delete crd perconaservermongodbbackups.psmdb.percona.com
++ mktemp
+ local LAST_OUT=/tmp/tmp.Z1LjMEOpMR
++ mktemp
+ local LAST_ERR=/tmp/tmp.KyBPnR0qtC
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl wait --for=delete crd perconaservermongodbbackups.psmdb.percona.com
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.Z1LjMEOpMR
+ cat /tmp/tmp.KyBPnR0qtC
+ rm /tmp/tmp.Z1LjMEOpMR /tmp/tmp.KyBPnR0qtC
+ return 0
+ for crd_name in $(yq eval '.metadata.name' "${src_dir}/deploy/crd.yaml" | grep -v '\-\-\-')
+ kubectl get perconaservermongodbrestores.psmdb.percona.com --all-namespaces -o wide
+ grep -v NAMESPACE
+ xargs -L 1 sh -xc 'kubectl patch perconaservermongodbrestores.psmdb.percona.com -n $0 $1 --type=merge -p "{\"metadata\":{\"finalizers\":[]}}"'
error: the server doesn't have a resource type "perconaservermongodbrestores"
+ kubectl patch perconaservermongodbrestores.psmdb.percona.com -n sh --type=merge -p '{"metadata":{"finalizers":[]}}'
error: the server doesn't have a resource type "perconaservermongodbrestores"
+ :
+ kubectl_bin wait --for=delete crd perconaservermongodbrestores.psmdb.percona.com
++ mktemp
+ local LAST_OUT=/tmp/tmp.ToRJpZz3AZ
++ mktemp
+ local LAST_ERR=/tmp/tmp.SEiL33Fs7V
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl wait --for=delete crd perconaservermongodbrestores.psmdb.percona.com
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.ToRJpZz3AZ
+ cat /tmp/tmp.SEiL33Fs7V
+ rm /tmp/tmp.ToRJpZz3AZ /tmp/tmp.SEiL33Fs7V
+ return 0
+ for crd_name in $(yq eval '.metadata.name' "${src_dir}/deploy/crd.yaml" | grep -v '\-\-\-')
+ kubectl get perconaservermongodbs.psmdb.percona.com --all-namespaces -o wide
+ grep -v NAMESPACE
+ xargs -L 1 sh -xc 'kubectl patch perconaservermongodbs.psmdb.percona.com -n $0 $1 --type=merge -p "{\"metadata\":{\"finalizers\":[]}}"'
error: the server doesn't have a resource type "perconaservermongodbs"
+ kubectl patch perconaservermongodbs.psmdb.percona.com -n sh --type=merge -p '{"metadata":{"finalizers":[]}}'
error: the server doesn't have a resource type "perconaservermongodbs"
+ :
+ kubectl_bin wait --for=delete crd perconaservermongodbs.psmdb.percona.com
++ mktemp
+ local LAST_OUT=/tmp/tmp.wQJ4oskez7
++ mktemp
+ local LAST_ERR=/tmp/tmp.hCASu1fQw8
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl wait --for=delete crd perconaservermongodbs.psmdb.percona.com
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.wQJ4oskez7
+ cat /tmp/tmp.hCASu1fQw8
+ rm /tmp/tmp.wQJ4oskez7 /tmp/tmp.hCASu1fQw8
+ return 0
+ local rbac_yaml=rbac.yaml
+ '[' -n psmdb-operator ']'
+ rbac_yaml=cw-rbac.yaml
+ kubectl_bin delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2219/deploy/cw-rbac.yaml --ignore-not-found
++ mktemp
+ local LAST_OUT=/tmp/tmp.VZDSTBDs3G
++ mktemp
+ local LAST_ERR=/tmp/tmp.FDELFiGRZE
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2219/deploy/cw-rbac.yaml --ignore-not-found
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.VZDSTBDs3G
clusterrole.rbac.authorization.k8s.io "percona-server-mongodb-operator" deleted
clusterrolebinding.rbac.authorization.k8s.io "service-account-percona-server-mongodb-operator" deleted
+ cat /tmp/tmp.FDELFiGRZE
+ rm /tmp/tmp.VZDSTBDs3G /tmp/tmp.FDELFiGRZE
+ return 0
+ destroy_cert_manager
+ kubectl_bin delete -f https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml
++ mktemp
+ local LAST_OUT=/tmp/tmp.swcFalCW8Z
++ mktemp
+ local LAST_ERR=/tmp/tmp.eSTXRkq7rT
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl delete -f https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml
+ exit_status=1
+ set -e
+ '[' 1 '!=' 0 -a -n 1 ']'
+ cat /tmp/tmp.swcFalCW8Z
+ cat /tmp/tmp.eSTXRkq7rT
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": namespaces "cert-manager" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "challenges.acme.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "orders.acme.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "certificaterequests.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "certificates.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "clusterissuers.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "issuers.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": serviceaccounts "cert-manager-cainjector" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": serviceaccounts "cert-manager" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": serviceaccounts "cert-manager-webhook" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-cainjector" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-issuers" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-clusterissuers" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-certificates" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-orders" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-challenges" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-ingress-shim" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-cluster-view" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-view" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-edit" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-approve:cert-manager-io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-certificatesigningrequests" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-webhook:subjectaccessreviews" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-cainjector" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-issuers" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-clusterissuers" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-certificates" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-orders" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-challenges" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-ingress-shim" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-approve:cert-manager-io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-certificatesigningrequests" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-webhook:subjectaccessreviews" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager-cainjector:leaderelection" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager:leaderelection" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager-tokenrequest" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager-webhook:dynamic-serving" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager-cainjector:leaderelection" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager:leaderelection" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager-tokenrequest" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager-webhook:dynamic-serving" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": services "cert-manager-cainjector" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": services "cert-manager" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": services "cert-manager-webhook" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": deployments.apps "cert-manager-cainjector" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": deployments.apps "cert-manager" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": deployments.apps "cert-manager-webhook" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": mutatingwebhookconfigurations.admissionregistration.k8s.io "cert-manager-webhook" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": validatingwebhookconfigurations.admissionregistration.k8s.io "cert-manager-webhook" not found
+ sleep 0
+ for i in $(seq 0 2)
+ set +e
+ kubectl delete -f https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml
+ exit_status=1
+ set -e
+ '[' 1 '!=' 0 -a -n 1 ']'
+ cat /tmp/tmp.swcFalCW8Z
+ cat /tmp/tmp.eSTXRkq7rT
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": namespaces "cert-manager" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "challenges.acme.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "orders.acme.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "certificaterequests.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "certificates.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "clusterissuers.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "issuers.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": serviceaccounts "cert-manager-cainjector" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": serviceaccounts "cert-manager" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": serviceaccounts "cert-manager-webhook" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-cainjector" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-issuers" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-clusterissuers" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-certificates" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-orders" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-challenges" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-ingress-shim" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-cluster-view" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-view" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-edit" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-approve:cert-manager-io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-certificatesigningrequests" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-webhook:subjectaccessreviews" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-cainjector" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-issuers" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-clusterissuers" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-certificates" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-orders" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-challenges" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-ingress-shim" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-approve:cert-manager-io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-certificatesigningrequests" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-webhook:subjectaccessreviews" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager-cainjector:leaderelection" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager:leaderelection" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager-tokenrequest" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager-webhook:dynamic-serving" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager-cainjector:leaderelection" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager:leaderelection" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager-tokenrequest" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager-webhook:dynamic-serving" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": services "cert-manager-cainjector" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": services "cert-manager" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": services "cert-manager-webhook" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": deployments.apps "cert-manager-cainjector" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": deployments.apps "cert-manager" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": deployments.apps "cert-manager-webhook" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": mutatingwebhookconfigurations.admissionregistration.k8s.io "cert-manager-webhook" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": validatingwebhookconfigurations.admissionregistration.k8s.io "cert-manager-webhook" not found
+ sleep 4
+ for i in $(seq 0 2)
+ set +e
+ kubectl delete -f https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml
+ exit_status=1
+ set -e
+ '[' 1 '!=' 0 -a -n 1 ']'
+ cat /tmp/tmp.swcFalCW8Z
+ cat /tmp/tmp.eSTXRkq7rT
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": namespaces "cert-manager" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "challenges.acme.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "orders.acme.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "certificaterequests.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "certificates.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "clusterissuers.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "issuers.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": serviceaccounts "cert-manager-cainjector" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": serviceaccounts "cert-manager" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": serviceaccounts "cert-manager-webhook" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-cainjector" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-issuers" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-clusterissuers" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-certificates" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-orders" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-challenges" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-ingress-shim" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-cluster-view" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-view" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-edit" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-approve:cert-manager-io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-certificatesigningrequests" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-webhook:subjectaccessreviews" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-cainjector" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-issuers" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-clusterissuers" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-certificates" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-orders" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-challenges" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-ingress-shim" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-approve:cert-manager-io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-certificatesigningrequests" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-webhook:subjectaccessreviews" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager-cainjector:leaderelection" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager:leaderelection" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager-tokenrequest" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager-webhook:dynamic-serving" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager-cainjector:leaderelection" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager:leaderelection" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager-tokenrequest" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager-webhook:dynamic-serving" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": services "cert-manager-cainjector" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": services "cert-manager" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": services "cert-manager-webhook" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": deployments.apps "cert-manager-cainjector" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": deployments.apps "cert-manager" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": deployments.apps "cert-manager-webhook" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": mutatingwebhookconfigurations.admissionregistration.k8s.io "cert-manager-webhook" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": validatingwebhookconfigurations.admissionregistration.k8s.io "cert-manager-webhook" not found
+ sleep 8
+ cat /tmp/tmp.swcFalCW8Z
+ cat /tmp/tmp.eSTXRkq7rT
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": namespaces "cert-manager" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "challenges.acme.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "orders.acme.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "certificaterequests.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "certificates.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "clusterissuers.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "issuers.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": serviceaccounts "cert-manager-cainjector" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": serviceaccounts "cert-manager" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": serviceaccounts "cert-manager-webhook" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-cainjector" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-issuers" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-clusterissuers" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-certificates" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-orders" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-challenges" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-ingress-shim" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-cluster-view" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-view" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-edit" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-approve:cert-manager-io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-certificatesigningrequests" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-webhook:subjectaccessreviews" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-cainjector" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-issuers" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-clusterissuers" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-certificates" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-orders" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-challenges" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-ingress-shim" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-approve:cert-manager-io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-certificatesigningrequests" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-webhook:subjectaccessreviews" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager-cainjector:leaderelection" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager:leaderelection" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager-tokenrequest" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager-webhook:dynamic-serving" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager-cainjector:leaderelection" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager:leaderelection" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager-tokenrequest" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager-webhook:dynamic-serving" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": services "cert-manager-cainjector" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": services "cert-manager" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": services "cert-manager-webhook" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": deployments.apps "cert-manager-cainjector" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": deployments.apps "cert-manager" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": deployments.apps "cert-manager-webhook" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": mutatingwebhookconfigurations.admissionregistration.k8s.io "cert-manager-webhook" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": validatingwebhookconfigurations.admissionregistration.k8s.io "cert-manager-webhook" not found
+ rm /tmp/tmp.swcFalCW8Z /tmp/tmp.eSTXRkq7rT
+ return 1
+ true
+ '[' -n '' ']'
+ '[' -n psmdb-operator ']'
+ kubectl_bin delete --grace-period=0 --force=true namespace multi-storage-10969
+ kubectl_bin delete --grace-period=0 --force=true namespace psmdb-operator
+ rm -rf /tmp/tmp.iK1Y0vUPYD
++ mktemp
++ mktemp
+ desc 'test passed'
+ set +o xtrace


-----------------------------------------------------------------------------------
test passed
-----------------------------------------------------------------------------------

+ local LAST_OUT=/tmp/tmp.66ZzKt2nc1
++ mktemp
+ local LAST_OUT=/tmp/tmp.7Z1FAxuoO9
++ mktemp
+ local LAST_ERR=/tmp/tmp.O4AGimsQJN
+ local exit_status=0
+ local timeout=4
+ local LAST_ERR=/tmp/tmp.9jcgDzirRE
+ local exit_status=0
+ local timeout=4
++ seq 0 2
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl delete --grace-period=0 --force=true namespace psmdb-operator
+ for i in $(seq 0 2)
+ set +e
+ kubectl delete --grace-period=0 --force=true namespace multi-storage-10969