Log: /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2256/e2e-tests/logs/multi-storage.log
Warning: version difference between client (1.35) and server (1.32) exceeds the supported minor version skew of +/-1
Warning: version difference between client (1.35) and server (1.32) exceeds the supported minor version skew of +/-1
Warning: version difference between client (1.35) and server (1.32) exceeds the supported minor version skew of +/-1
+ cluster=some-name
+ create_infra multi-storage-27036
+ local ns=multi-storage-27036
+ [[ 1 == 1 ]]
+ delete_crd
+ desc 'get and delete old CRDs and RBAC'
+ set +o xtrace


-----------------------------------------------------------------------------------
get and delete old CRDs and RBAC
-----------------------------------------------------------------------------------

+ kubectl_bin delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2256/deploy/crd.yaml --ignore-not-found --wait=false
++ mktemp
+ local LAST_OUT=/tmp/tmp.Y5ZgDTs3Zn
++ mktemp
+ local LAST_ERR=/tmp/tmp.Ql1aJGnF5F
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2256/deploy/crd.yaml --ignore-not-found --wait=false
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.Y5ZgDTs3Zn
+ cat /tmp/tmp.Ql1aJGnF5F
+ rm /tmp/tmp.Y5ZgDTs3Zn /tmp/tmp.Ql1aJGnF5F
+ return 0
++ yq eval .metadata.name /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2256/deploy/crd.yaml
++ grep -v '\-\-\-'
grep: warning: stray \ before -
grep: warning: stray \ before -
+ for crd_name in $(yq eval '.metadata.name' "${src_dir}/deploy/crd.yaml" | grep -v '\-\-\-')
+ kubectl get perconaservermongodbbackups.psmdb.percona.com --all-namespaces -o wide
+ grep -v NAMESPACE
+ xargs -L 1 sh -xc 'kubectl patch perconaservermongodbbackups.psmdb.percona.com -n $0 $1 --type=merge -p "{\"metadata\":{\"finalizers\":[]}}"'
error: the server doesn't have a resource type "perconaservermongodbbackups"
+ kubectl patch perconaservermongodbbackups.psmdb.percona.com -n sh --type=merge -p '{"metadata":{"finalizers":[]}}'
error: the server doesn't have a resource type "perconaservermongodbbackups"
+ :
+ kubectl_bin wait --for=delete crd perconaservermongodbbackups.psmdb.percona.com
++ mktemp
+ local LAST_OUT=/tmp/tmp.gtmNRysfVn
++ mktemp
+ local LAST_ERR=/tmp/tmp.omeOkvWubE
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl wait --for=delete crd perconaservermongodbbackups.psmdb.percona.com
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.gtmNRysfVn
+ cat /tmp/tmp.omeOkvWubE
+ rm /tmp/tmp.gtmNRysfVn /tmp/tmp.omeOkvWubE
+ return 0
+ for crd_name in $(yq eval '.metadata.name' "${src_dir}/deploy/crd.yaml" | grep -v '\-\-\-')
+ kubectl get perconaservermongodbrestores.psmdb.percona.com --all-namespaces -o wide
+ grep -v NAMESPACE
+ xargs -L 1 sh -xc 'kubectl patch perconaservermongodbrestores.psmdb.percona.com -n $0 $1 --type=merge -p "{\"metadata\":{\"finalizers\":[]}}"'
error: the server doesn't have a resource type "perconaservermongodbrestores"
+ kubectl patch perconaservermongodbrestores.psmdb.percona.com -n sh --type=merge -p '{"metadata":{"finalizers":[]}}'
error: the server doesn't have a resource type "perconaservermongodbrestores"
+ :
+ kubectl_bin wait --for=delete crd perconaservermongodbrestores.psmdb.percona.com
++ mktemp
+ local LAST_OUT=/tmp/tmp.3awCUNJvnc
++ mktemp
+ local LAST_ERR=/tmp/tmp.tgGkfOXCNA
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl wait --for=delete crd perconaservermongodbrestores.psmdb.percona.com
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.3awCUNJvnc
+ cat /tmp/tmp.tgGkfOXCNA
+ rm /tmp/tmp.3awCUNJvnc /tmp/tmp.tgGkfOXCNA
+ return 0
+ for crd_name in $(yq eval '.metadata.name' "${src_dir}/deploy/crd.yaml" | grep -v '\-\-\-')
+ kubectl get perconaservermongodbs.psmdb.percona.com --all-namespaces -o wide
+ grep -v NAMESPACE
+ xargs -L 1 sh -xc 'kubectl patch perconaservermongodbs.psmdb.percona.com -n $0 $1 --type=merge -p "{\"metadata\":{\"finalizers\":[]}}"'
error: the server doesn't have a resource type "perconaservermongodbs"
+ kubectl patch perconaservermongodbs.psmdb.percona.com -n sh --type=merge -p '{"metadata":{"finalizers":[]}}'
error: the server doesn't have a resource type "perconaservermongodbs"
+ :
+ kubectl_bin wait --for=delete crd perconaservermongodbs.psmdb.percona.com
++ mktemp
+ local LAST_OUT=/tmp/tmp.wgkUo13FFu
++ mktemp
+ local LAST_ERR=/tmp/tmp.WnRaAi8d4V
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl wait --for=delete crd perconaservermongodbs.psmdb.percona.com
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.wgkUo13FFu
+ cat /tmp/tmp.WnRaAi8d4V
+ rm /tmp/tmp.wgkUo13FFu /tmp/tmp.WnRaAi8d4V
+ return 0
+ local rbac_yaml=rbac.yaml
+ '[' -n psmdb-operator ']'
+ rbac_yaml=cw-rbac.yaml
+ kubectl_bin delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2256/deploy/cw-rbac.yaml --ignore-not-found
++ mktemp
+ local LAST_OUT=/tmp/tmp.Dj8pXNT1J9
++ mktemp
+ local LAST_ERR=/tmp/tmp.9Nc1ex9ytk
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2256/deploy/cw-rbac.yaml --ignore-not-found
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.Dj8pXNT1J9
+ cat /tmp/tmp.9Nc1ex9ytk
+ rm /tmp/tmp.Dj8pXNT1J9 /tmp/tmp.9Nc1ex9ytk
+ return 0
+ check_crd_for_deletion PR-2256-98b7d85f
+ local git_tag=PR-2256-98b7d85f
++ curl -s https://raw.githubusercontent.com/percona/percona-server-mongodb-operator/PR-2256-98b7d85f/deploy/crd.yaml
++ yq eval .metadata.name
++ /usr/sbin/sed s/---//g
++ /usr/sbin/sed ':a;N;$!ba;s/\n/ /g'
+ for crd_name in $(curl -s https://raw.githubusercontent.com/percona/percona-server-mongodb-operator/${git_tag}/deploy/crd.yaml | yq eval '.metadata.name' | $sed 's/---//g' | $sed ':a;N;$!ba;s/\n/ /g')
++ kubectl_bin get crd/null -o 'jsonpath={.status.conditions[-1].type}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.LVIYmjhErz
+++ mktemp
++ local LAST_ERR=/tmp/tmp.8689ImbQvt
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get crd/null -o 'jsonpath={.status.conditions[-1].type}'
++ exit_status=1
++ set -e
++ '[' 1 '!=' 0 -a -n 1 ']'
++ cat /tmp/tmp.LVIYmjhErz
++ cat /tmp/tmp.8689ImbQvt
Error from server (NotFound): customresourcedefinitions.apiextensions.k8s.io "null" not found
++ sleep 0
++ for i in $(seq 0 2)
++ set +e
++ kubectl get crd/null -o 'jsonpath={.status.conditions[-1].type}'
++ exit_status=1
++ set -e
++ '[' 1 '!=' 0 -a -n 1 ']'
++ cat /tmp/tmp.LVIYmjhErz
++ cat /tmp/tmp.8689ImbQvt
Error from server (NotFound): customresourcedefinitions.apiextensions.k8s.io "null" not found
++ sleep 4
++ for i in $(seq 0 2)
++ set +e
++ kubectl get crd/null -o 'jsonpath={.status.conditions[-1].type}'
++ exit_status=1
++ set -e
++ '[' 1 '!=' 0 -a -n 1 ']'
++ cat /tmp/tmp.LVIYmjhErz
++ cat /tmp/tmp.8689ImbQvt
Error from server (NotFound): customresourcedefinitions.apiextensions.k8s.io "null" not found
++ sleep 8
++ cat /tmp/tmp.LVIYmjhErz
++ cat /tmp/tmp.8689ImbQvt
Error from server (NotFound): customresourcedefinitions.apiextensions.k8s.io "null" not found
++ rm /tmp/tmp.LVIYmjhErz /tmp/tmp.8689ImbQvt
++ return 1
+ [[ '' == Terminating ]]
+ '[' -n psmdb-operator ']'
+ create_namespace psmdb-operator
+ local namespace=psmdb-operator
+ local skip_clean_namespace=
+ [[ 1 == 1 ]]
+ [[ -z '' ]]
+ destroy_chaos_mesh
++ helm list --all-namespaces --filter chaos-mesh
++ tail -n1
++ awk '-F ' '{print $2}'
++ sed s/NAMESPACE//
+ local chaos_mesh_ns=
+ desc 'destroy chaos-mesh'
+ set +o xtrace


-----------------------------------------------------------------------------------
destroy chaos-mesh
-----------------------------------------------------------------------------------

+ '[' -n '' ']'
++ kubectl get MutatingWebhookConfiguration
++ grep chaos-mesh
++ awk '{print $1}'
+ timeout 30 kubectl delete MutatingWebhookConfiguration
error: resource(s) were provided, but no name was specified
+ :
++ kubectl get ValidatingWebhookConfiguration
++ grep chaos-mesh
++ awk '{print $1}'
+ timeout 30 kubectl delete ValidatingWebhookConfiguration
error: resource(s) were provided, but no name was specified
+ :
++ kubectl get ValidatingWebhookConfiguration
++ grep validate-auth
++ awk '{print $1}'
+ timeout 30 kubectl delete ValidatingWebhookConfiguration
error: resource(s) were provided, but no name was specified
+ :
++ kubectl api-resources
++ awk '{print $1}'
++ grep chaos-mesh
++ kubectl get crd
++ grep chaos-mesh.org
++ awk '{print $1}'
+ timeout 30 kubectl delete crd
error: resource(s) were provided, but no name was specified
+ :
++ kubectl get clusterrolebinding
++ grep chaos-mesh
++ awk '{print $1}'
+ timeout 30 kubectl delete clusterrolebinding
error: resource(s) were provided, but no name was specified
+ :
++ kubectl get clusterrole
++ grep chaos-mesh
++ awk '{print $1}'
+ timeout 30 kubectl delete clusterrole
error: resource(s) were provided, but no name was specified
+ :
+ desc 'cleaned up all old namespaces'
+ set +o xtrace


-----------------------------------------------------------------------------------
cleaned up all old namespaces
-----------------------------------------------------------------------------------

+ '[' -n '' ']'
+ kubectl_bin get ns
+ grep -E -v '^kube-|^default|Terminating|psmdb-operator|openshift|^gke-|^gmp-|^NAME'
++ mktemp
+ awk '{print$1}'
+ xargs kubectl delete ns
+ desc 'cleaned up old namespaces psmdb-operator'
+ set +o xtrace


-----------------------------------------------------------------------------------
cleaned up old namespaces psmdb-operator
-----------------------------------------------------------------------------------

+ kubectl_bin delete namespace psmdb-operator --ignore-not-found
+ local LAST_OUT=/tmp/tmp.2VhKjKSl4E
++ mktemp
++ mktemp
+ local LAST_ERR=/tmp/tmp.eXRmRfOrfj
+ local exit_status=0
+ local timeout=4
+ local LAST_OUT=/tmp/tmp.DVXTAA01M1
++ seq 0 2
++ mktemp
+ for i in $(seq 0 2)
+ set +e
+ kubectl get ns
+ local LAST_ERR=/tmp/tmp.iDUw8mBBkp
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl delete namespace psmdb-operator --ignore-not-found
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.2VhKjKSl4E
+ cat /tmp/tmp.eXRmRfOrfj
+ rm /tmp/tmp.2VhKjKSl4E /tmp/tmp.eXRmRfOrfj
+ return 0
namespace "multi-storage-27880" deleted
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.DVXTAA01M1
namespace "psmdb-operator" deleted
+ cat /tmp/tmp.iDUw8mBBkp
+ rm /tmp/tmp.DVXTAA01M1 /tmp/tmp.iDUw8mBBkp
+ return 0
+ kubectl_bin wait --for=delete namespace psmdb-operator
++ mktemp
+ local LAST_OUT=/tmp/tmp.p2DjqD3YtC
++ mktemp
+ local LAST_ERR=/tmp/tmp.JLqMyFsKIX
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl wait --for=delete namespace psmdb-operator
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.p2DjqD3YtC
+ cat /tmp/tmp.JLqMyFsKIX
+ rm /tmp/tmp.p2DjqD3YtC /tmp/tmp.JLqMyFsKIX
+ return 0
+ desc 'create namespace psmdb-operator'
+ set +o xtrace


-----------------------------------------------------------------------------------
create namespace psmdb-operator
-----------------------------------------------------------------------------------

+ kubectl_bin create namespace psmdb-operator
++ mktemp
+ local LAST_OUT=/tmp/tmp.I4EYLlNz02
++ mktemp
+ local LAST_ERR=/tmp/tmp.jQl7hrp5Q1
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl create namespace psmdb-operator
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.I4EYLlNz02
namespace/psmdb-operator created
+ cat /tmp/tmp.jQl7hrp5Q1
+ rm /tmp/tmp.I4EYLlNz02 /tmp/tmp.jQl7hrp5Q1
+ return 0
+ set_kube_ctx psmdb-operator
+ local namespace=psmdb-operator
++ kubectl_bin config current-context
+++ mktemp
++ local LAST_OUT=/tmp/tmp.NBKfg9QR0p
+++ mktemp
++ local LAST_ERR=/tmp/tmp.2SAy5S008r
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl config current-context
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.NBKfg9QR0p
++ cat /tmp/tmp.2SAy5S008r
++ rm /tmp/tmp.NBKfg9QR0p /tmp/tmp.2SAy5S008r
++ return 0
+ kubectl_bin config set-context gke_cloud-dev-112233_us-central1-a_jen-psmdb-2256-98b7d85f-7-cluster4 --namespace=psmdb-operator
++ mktemp
+ local LAST_OUT=/tmp/tmp.bDH1qPhjSc
++ mktemp
+ local LAST_ERR=/tmp/tmp.chZowj5NSq
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl config set-context gke_cloud-dev-112233_us-central1-a_jen-psmdb-2256-98b7d85f-7-cluster4 --namespace=psmdb-operator
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.bDH1qPhjSc
Context "gke_cloud-dev-112233_us-central1-a_jen-psmdb-2256-98b7d85f-7-cluster4" modified.
+ cat /tmp/tmp.chZowj5NSq
+ rm /tmp/tmp.bDH1qPhjSc /tmp/tmp.chZowj5NSq
+ return 0
+ deploy_operator
+ desc 'start PSMDB operator: docker.io/perconalab/percona-server-mongodb-operator:PR-2256-98b7d85f'
+ set +o xtrace


-----------------------------------------------------------------------------------
start PSMDB operator: docker.io/perconalab/percona-server-mongodb-operator:PR-2256-98b7d85f
-----------------------------------------------------------------------------------

+ local cr_file
+ '[' -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2256/e2e-tests/multi-storage/conf/crd.yaml ']'
+ cr_file=/mnt/jenkins/workspace/cloud-psmdb-operator_PR-2256/deploy/crd.yaml
+ kubectl_bin apply --server-side --force-conflicts -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2256/deploy/crd.yaml
++ mktemp
+ local LAST_OUT=/tmp/tmp.vXq1AhgAh6
++ mktemp
+ local LAST_ERR=/tmp/tmp.XzcgC7WPiU
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl apply --server-side --force-conflicts -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2256/deploy/crd.yaml
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.vXq1AhgAh6
customresourcedefinition.apiextensions.k8s.io/perconaservermongodbbackups.psmdb.percona.com serverside-applied
customresourcedefinition.apiextensions.k8s.io/perconaservermongodbrestores.psmdb.percona.com serverside-applied
customresourcedefinition.apiextensions.k8s.io/perconaservermongodbs.psmdb.percona.com serverside-applied
+ cat /tmp/tmp.XzcgC7WPiU
+ rm /tmp/tmp.vXq1AhgAh6 /tmp/tmp.XzcgC7WPiU
+ return 0
+ '[' -n psmdb-operator ']'
+ apply_rbac cw-rbac
+ local operator_namespace=psmdb-operator
+ local rbac=cw-rbac
+ cat /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2256/deploy/cw-rbac.yaml
+ sed -e 's^namespace: .*^namespace: psmdb-operator^'
+ kubectl_bin apply -n psmdb-operator -f -
++ mktemp
+ local LAST_OUT=/tmp/tmp.tuCHbqNFlb
++ mktemp
+ local LAST_ERR=/tmp/tmp.M1Yb9ysbfI
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl apply -n psmdb-operator -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.tuCHbqNFlb
clusterrole.rbac.authorization.k8s.io/percona-server-mongodb-operator created
serviceaccount/percona-server-mongodb-operator created
clusterrolebinding.rbac.authorization.k8s.io/service-account-percona-server-mongodb-operator created
+ cat /tmp/tmp.M1Yb9ysbfI
+ rm /tmp/tmp.tuCHbqNFlb /tmp/tmp.M1Yb9ysbfI
+ return 0
+ yq eval $'\n\t\t\t(.spec.template.spec.containers[].image = "docker.io/perconalab/percona-server-mongodb-operator:PR-2256-98b7d85f") |\n\t\t\t((.. | select(.[] == "DISABLE_TELEMETRY")) |= .value="true") |\n\t\t\t((.. | select(.[] == "LOG_LEVEL")) |= .value="DEBUG")' /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2256/deploy/cw-operator.yaml
+ kubectl_bin apply -n psmdb-operator -f -
++ mktemp
+ local LAST_OUT=/tmp/tmp.Ow3WvBC0ML
++ mktemp
+ local LAST_ERR=/tmp/tmp.N4PcrOgXWf
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl apply -n psmdb-operator -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.Ow3WvBC0ML
deployment.apps/percona-server-mongodb-operator created
+ cat /tmp/tmp.N4PcrOgXWf
+ rm /tmp/tmp.Ow3WvBC0ML /tmp/tmp.N4PcrOgXWf
+ return 0
+ sleep 20
++ get_operator_pod
++ kubectl_bin get pods --selector=name=percona-server-mongodb-operator -o 'jsonpath={.items[].metadata.name}' -n psmdb-operator
+++ mktemp
++ local LAST_OUT=/tmp/tmp.6r5P5S6efu
+++ mktemp
++ local LAST_ERR=/tmp/tmp.wZOIl2lGfs
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=percona-server-mongodb-operator -o 'jsonpath={.items[].metadata.name}' -n psmdb-operator
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.6r5P5S6efu
++ cat /tmp/tmp.wZOIl2lGfs
++ rm /tmp/tmp.6r5P5S6efu /tmp/tmp.wZOIl2lGfs
++ return 0
+ wait_operator_pod percona-server-mongodb-operator-7d9c7747d9-phwxv
+ local pod=percona-server-mongodb-operator-7d9c7747d9-phwxv
+ set +o xtrace
waiting for pod/percona-server-mongodb-operator-7d9c7747d9-phwxv to be ready.OK
+ echo 'Print operator info from log'
Print operator info from log
+ grep 'Manager starting up'
++ get_operator_pod
++ kubectl_bin get pods --selector=name=percona-server-mongodb-operator -o 'jsonpath={.items[].metadata.name}' -n psmdb-operator
+++ mktemp
++ local LAST_OUT=/tmp/tmp.7610lEwTVP
+++ mktemp
++ local LAST_ERR=/tmp/tmp.zB6OA7ejE6
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=percona-server-mongodb-operator -o 'jsonpath={.items[].metadata.name}' -n psmdb-operator
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.7610lEwTVP
++ cat /tmp/tmp.zB6OA7ejE6
++ rm /tmp/tmp.7610lEwTVP /tmp/tmp.zB6OA7ejE6
++ return 0
+ kubectl_bin logs -n psmdb-operator percona-server-mongodb-operator-7d9c7747d9-phwxv
++ mktemp
+ local LAST_OUT=/tmp/tmp.oLkSX1tIo5
++ mktemp
+ local LAST_ERR=/tmp/tmp.6oZ9AXnlfM
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl logs -n psmdb-operator percona-server-mongodb-operator-7d9c7747d9-phwxv
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.oLkSX1tIo5
+ cat /tmp/tmp.6oZ9AXnlfM
+ rm /tmp/tmp.oLkSX1tIo5 /tmp/tmp.6oZ9AXnlfM
+ return 0
2026-04-14T11:09:58.570Z	INFO	setup	Manager starting up	{"gitCommit": "98b7d85f6fc9316ce5207960e26f4bbf2b33bfd3", "gitBranch": "PR-2256-98b7d85f", "buildTime": "", "goVersion": "go1.25.9", "os": "linux", "arch": "amd64"}
+ create_namespace multi-storage-27036
+ local namespace=multi-storage-27036
+ local skip_clean_namespace=
+ [[ 1 == 1 ]]
+ [[ -z '' ]]
+ destroy_chaos_mesh
++ helm list --all-namespaces --filter chaos-mesh
++ tail -n1
++ awk '-F ' '{print $2}'
++ sed s/NAMESPACE//
+ local chaos_mesh_ns=
+ desc 'destroy chaos-mesh'
+ set +o xtrace


-----------------------------------------------------------------------------------
destroy chaos-mesh
-----------------------------------------------------------------------------------

+ '[' -n '' ']'
++ kubectl get MutatingWebhookConfiguration
++ grep chaos-mesh
++ awk '{print $1}'
+ timeout 30 kubectl delete MutatingWebhookConfiguration
error: resource(s) were provided, but no name was specified
+ :
++ kubectl get ValidatingWebhookConfiguration
++ grep chaos-mesh
++ awk '{print $1}'
+ timeout 30 kubectl delete ValidatingWebhookConfiguration
error: resource(s) were provided, but no name was specified
+ :
++ kubectl get ValidatingWebhookConfiguration
++ grep validate-auth
++ awk '{print $1}'
+ timeout 30 kubectl delete ValidatingWebhookConfiguration
error: resource(s) were provided, but no name was specified
+ :
++ kubectl api-resources
++ awk '{print $1}'
++ grep chaos-mesh
++ kubectl get crd
++ grep chaos-mesh.org
++ awk '{print $1}'
+ timeout 30 kubectl delete crd
error: resource(s) were provided, but no name was specified
+ :
++ kubectl get clusterrolebinding
++ grep chaos-mesh
++ awk '{print $1}'
+ timeout 30 kubectl delete clusterrolebinding
error: resource(s) were provided, but no name was specified
+ :
++ kubectl get clusterrole
++ grep chaos-mesh
++ awk '{print $1}'
+ timeout 30 kubectl delete clusterrole
error: resource(s) were provided, but no name was specified
+ :
+ desc 'cleaned up all old namespaces'
+ set +o xtrace


-----------------------------------------------------------------------------------
cleaned up all old namespaces
-----------------------------------------------------------------------------------

+ kubectl_bin get ns
+ grep -E -v '^kube-|^default|Terminating|psmdb-operator|openshift|^gke-|^gmp-|^NAME'
++ mktemp
+ awk '{print$1}'
+ xargs kubectl delete ns
+ '[' -n '' ']'
+ desc 'cleaned up old namespaces multi-storage-27036'
+ set +o xtrace


-----------------------------------------------------------------------------------
cleaned up old namespaces multi-storage-27036
-----------------------------------------------------------------------------------

+ kubectl_bin delete namespace multi-storage-27036 --ignore-not-found
+ local LAST_OUT=/tmp/tmp.WDfrWmLmT4
++ mktemp
++ mktemp
+ local LAST_OUT=/tmp/tmp.JZCPEiraWw
+ local LAST_ERR=/tmp/tmp.Knk0FOvMkp
+ local exit_status=0
+ local timeout=4
++ mktemp
++ seq 0 2
+ local LAST_ERR=/tmp/tmp.Srp5FSVNWu
+ local exit_status=0
+ local timeout=4
+ for i in $(seq 0 2)
+ set +e
+ kubectl get ns
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl delete namespace multi-storage-27036 --ignore-not-found
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.WDfrWmLmT4
+ cat /tmp/tmp.Knk0FOvMkp
+ rm /tmp/tmp.WDfrWmLmT4 /tmp/tmp.Knk0FOvMkp
+ return 0
error: resource(s) were provided, but no name was specified
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.JZCPEiraWw
+ cat /tmp/tmp.Srp5FSVNWu
+ rm /tmp/tmp.JZCPEiraWw /tmp/tmp.Srp5FSVNWu
+ return 0
+ kubectl_bin wait --for=delete namespace multi-storage-27036
++ mktemp
+ local LAST_OUT=/tmp/tmp.zHH5iqbOWO
++ mktemp
+ local LAST_ERR=/tmp/tmp.ACPp5Bc9Uk
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl wait --for=delete namespace multi-storage-27036
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.zHH5iqbOWO
+ cat /tmp/tmp.ACPp5Bc9Uk
+ rm /tmp/tmp.zHH5iqbOWO /tmp/tmp.ACPp5Bc9Uk
+ return 0
+ desc 'create namespace multi-storage-27036'
+ set +o xtrace


-----------------------------------------------------------------------------------
create namespace multi-storage-27036
-----------------------------------------------------------------------------------

+ kubectl_bin create namespace multi-storage-27036
++ mktemp
+ local LAST_OUT=/tmp/tmp.ssOZDzjAKP
++ mktemp
+ local LAST_ERR=/tmp/tmp.adaMpveucc
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl create namespace multi-storage-27036
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.ssOZDzjAKP
namespace/multi-storage-27036 created
+ cat /tmp/tmp.adaMpveucc
+ rm /tmp/tmp.ssOZDzjAKP /tmp/tmp.adaMpveucc
+ return 0
+ set_kube_ctx multi-storage-27036
+ local namespace=multi-storage-27036
++ kubectl_bin config current-context
+++ mktemp
++ local LAST_OUT=/tmp/tmp.vJ2izcV47I
+++ mktemp
++ local LAST_ERR=/tmp/tmp.oOVrwrhYEM
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl config current-context
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.vJ2izcV47I
++ cat /tmp/tmp.oOVrwrhYEM
++ rm /tmp/tmp.vJ2izcV47I /tmp/tmp.oOVrwrhYEM
++ return 0
+ kubectl_bin config set-context gke_cloud-dev-112233_us-central1-a_jen-psmdb-2256-98b7d85f-7-cluster4 --namespace=multi-storage-27036
++ mktemp
+ local LAST_OUT=/tmp/tmp.TirasAysqD
++ mktemp
+ local LAST_ERR=/tmp/tmp.AG9jGrk74Z
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl config set-context gke_cloud-dev-112233_us-central1-a_jen-psmdb-2256-98b7d85f-7-cluster4 --namespace=multi-storage-27036
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.TirasAysqD
Context "gke_cloud-dev-112233_us-central1-a_jen-psmdb-2256-98b7d85f-7-cluster4" modified.
+ cat /tmp/tmp.AG9jGrk74Z
+ rm /tmp/tmp.TirasAysqD /tmp/tmp.AG9jGrk74Z
+ return 0
+ deploy_minio
+ local cert_secret=
+ local service_name=minio-service
+ desc 'install MinIO: minio-service'
+ set +o xtrace


-----------------------------------------------------------------------------------
install MinIO: minio-service
-----------------------------------------------------------------------------------

+ helm uninstall minio-service
+ :
+ helm repo remove minio
"minio" has been removed from your repositories
+ helm repo add minio https://charts.min.io/
"minio" has been added to your repositories
+ local endpoint=http://minio-service:9000
+ minio_args=('--version' '5.4.0' '--set' 'replicas=1' '--set' 'mode=standalone' '--set' 'resources.requests.memory=256Mi' '--set' 'rootUser=rootuser' '--set' 'rootPassword=rootpass123' '--set' 'users[0].accessKey=some-access-key' '--set' 'users[0].secretKey=some-secret-key' '--set' 'users[0].policy=consoleAdmin' '--set' 'service.type=ClusterIP' '--set' 'configPathmc=/tmp/' '--set' 'securityContext.enabled=false' '--set' 'persistence.size=2G' '--set' 'fullnameOverride=minio-service' '--set' 'serviceAccount.create=true' '--set' 'serviceAccount.name=minio-service-sa')
+ local minio_args
+ [[ -n '' ]]
+ retry 10 60 helm install minio-service --version 5.4.0 --set replicas=1 --set mode=standalone --set resources.requests.memory=256Mi --set rootUser=rootuser --set rootPassword=rootpass123 --set 'users[0].accessKey=some-access-key' --set 'users[0].secretKey=some-secret-key' --set 'users[0].policy=consoleAdmin' --set service.type=ClusterIP --set configPathmc=/tmp/ --set securityContext.enabled=false --set persistence.size=2G --set fullnameOverride=minio-service --set serviceAccount.create=true --set serviceAccount.name=minio-service-sa minio/minio
+ local max=10
+ local delay=60
+ shift 2
+ local n=1
+ helm install minio-service --version 5.4.0 --set replicas=1 --set mode=standalone --set resources.requests.memory=256Mi --set rootUser=rootuser --set rootPassword=rootpass123 --set 'users[0].accessKey=some-access-key' --set 'users[0].secretKey=some-secret-key' --set 'users[0].policy=consoleAdmin' --set service.type=ClusterIP --set configPathmc=/tmp/ --set securityContext.enabled=false --set persistence.size=2G --set fullnameOverride=minio-service --set serviceAccount.create=true --set serviceAccount.name=minio-service-sa minio/minio
NAME: minio-service
LAST DEPLOYED: Tue Apr 14 11:10:30 2026
NAMESPACE: multi-storage-27036
STATUS: deployed
REVISION: 1
TEST SUITE: None
NOTES:
MinIO can be accessed via port 9000 on the following DNS name from within your cluster:
minio-service.multi-storage-27036.cluster.local

To access MinIO from localhost, run the below commands:

  1. export POD_NAME=$(kubectl get pods --namespace multi-storage-27036 -l "release=minio-service" -o jsonpath="{.items[0].metadata.name}")

  2. kubectl port-forward $POD_NAME 9000 --namespace multi-storage-27036

Read more about port forwarding here: http://kubernetes.io/docs/user-guide/kubectl/kubectl_port-forward/

You can now access MinIO server on http://localhost:9000. Follow the below steps to connect to MinIO server with mc client:

  1. Download the MinIO mc client - https://min.io/docs/minio/linux/reference/minio-mc.html#quickstart

  2. export MC_HOST_minio-service-local=http://$(kubectl get secret --namespace multi-storage-27036 minio-service -o jsonpath="{.data.rootUser}" | base64 --decode):$(kubectl get secret --namespace multi-storage-27036 minio-service -o jsonpath="{.data.rootPassword}" | base64 --decode)@localhost:9000

  3. mc ls minio-service-local
++ kubectl_bin get pods --selector=release=minio-service -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.9wXvWzJnN9
+++ mktemp
++ local LAST_ERR=/tmp/tmp.YYByhICHsP
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=release=minio-service -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.9wXvWzJnN9
++ cat /tmp/tmp.YYByhICHsP
++ rm /tmp/tmp.9wXvWzJnN9 /tmp/tmp.YYByhICHsP
++ return 0
+ local MINIO_POD=minio-service-6d5f646cdc-k7qx8
+ wait_pod minio-service-6d5f646cdc-k7qx8
+ local pod=minio-service-6d5f646cdc-k7qx8
+ set +o xtrace
waiting for pod/minio-service-6d5f646cdc-k7qx8 to be ready.OK
+ '[' -n psmdb-operator ']'
+ kubectl_bin create svc -n psmdb-operator externalname minio-service --external-name=minio-service.multi-storage-27036.svc.cluster.local --tcp=9000
service/minio-service created
+ create_minio_bucket operator-testing http://minio-service:9000
+ local bucket=operator-testing
+ local endpoint=http://minio-service:9000
+ kubectl_bin run -i --rm aws-cli --image=docker.io/perconalab/awscli --restart=Never -- bash -c $'AWS_ACCESS_KEY_ID=some-access-key \t\tAWS_SECRET_ACCESS_KEY=some-secret-key \t\tAWS_DEFAULT_REGION=us-east-1         /usr/bin/aws --no-verify-ssl --endpoint-url http://minio-service:9000 s3 mb s3://operator-testing'
++ mktemp
+ local LAST_OUT=/tmp/tmp.QUaVDQK8CU
++ mktemp
+ local LAST_ERR=/tmp/tmp.gXVB7z8ljU
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl run -i --rm aws-cli --image=docker.io/perconalab/awscli --restart=Never -- bash -c $'AWS_ACCESS_KEY_ID=some-access-key \t\tAWS_SECRET_ACCESS_KEY=some-secret-key \t\tAWS_DEFAULT_REGION=us-east-1         /usr/bin/aws --no-verify-ssl --endpoint-url http://minio-service:9000 s3 mb s3://operator-testing'
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.QUaVDQK8CU
make_bucket: operator-testing
pod "aws-cli" deleted from multi-storage-27036 namespace
+ cat /tmp/tmp.gXVB7z8ljU
+ rm /tmp/tmp.QUaVDQK8CU /tmp/tmp.gXVB7z8ljU
+ return 0
+ endpoint=http://minio-service:9000
+ create_minio_bucket operator-testing-2 http://minio-service:9000
+ local bucket=operator-testing-2
+ local endpoint=http://minio-service:9000
+ kubectl_bin run -i --rm aws-cli --image=docker.io/perconalab/awscli --restart=Never -- bash -c $'AWS_ACCESS_KEY_ID=some-access-key \t\tAWS_SECRET_ACCESS_KEY=some-secret-key \t\tAWS_DEFAULT_REGION=us-east-1         /usr/bin/aws --no-verify-ssl --endpoint-url http://minio-service:9000 s3 mb s3://operator-testing-2'
++ mktemp
+ local LAST_OUT=/tmp/tmp.oHwwoj9OaU
++ mktemp
+ local LAST_ERR=/tmp/tmp.2eKyiQOjpi
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl run -i --rm aws-cli --image=docker.io/perconalab/awscli --restart=Never -- bash -c $'AWS_ACCESS_KEY_ID=some-access-key \t\tAWS_SECRET_ACCESS_KEY=some-secret-key \t\tAWS_DEFAULT_REGION=us-east-1         /usr/bin/aws --no-verify-ssl --endpoint-url http://minio-service:9000 s3 mb s3://operator-testing-2'
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.oHwwoj9OaU
make_bucket: operator-testing-2
pod "aws-cli" deleted from multi-storage-27036 namespace
+ cat /tmp/tmp.2eKyiQOjpi
All commands and output from this session will be recorded in container logs, including credentials and sensitive information passed through the command prompt.
If you don't see a command prompt, try pressing enter.
warning: couldn't attach to pod/aws-cli, falling back to streaming logs: Internal error occurred: unable to upgrade connection: container aws-cli not found in pod aws-cli_multi-storage-27036
+ rm /tmp/tmp.oHwwoj9OaU /tmp/tmp.2eKyiQOjpi
+ return 0
+ create_minio_bucket operator-testing-3 http://minio-service:9000
+ local bucket=operator-testing-3
+ local endpoint=http://minio-service:9000
+ kubectl_bin run -i --rm aws-cli --image=docker.io/perconalab/awscli --restart=Never -- bash -c $'AWS_ACCESS_KEY_ID=some-access-key \t\tAWS_SECRET_ACCESS_KEY=some-secret-key \t\tAWS_DEFAULT_REGION=us-east-1         /usr/bin/aws --no-verify-ssl --endpoint-url http://minio-service:9000 s3 mb s3://operator-testing-3'
++ mktemp
+ local LAST_OUT=/tmp/tmp.h2haW2qYbK
++ mktemp
+ local LAST_ERR=/tmp/tmp.Y4Qw1kMVcU
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl run -i --rm aws-cli --image=docker.io/perconalab/awscli --restart=Never -- bash -c $'AWS_ACCESS_KEY_ID=some-access-key \t\tAWS_SECRET_ACCESS_KEY=some-secret-key \t\tAWS_DEFAULT_REGION=us-east-1         /usr/bin/aws --no-verify-ssl --endpoint-url http://minio-service:9000 s3 mb s3://operator-testing-3'
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.h2haW2qYbK
make_bucket: operator-testing-3
pod "aws-cli" deleted from multi-storage-27036 namespace
+ cat /tmp/tmp.Y4Qw1kMVcU
+ rm /tmp/tmp.h2haW2qYbK /tmp/tmp.Y4Qw1kMVcU
+ return 0
+ log 'create secrets and start client'
+ set +o xtrace
[2026-04-14T11:11:15+0000] create secrets and start client
+ kubectl_bin apply -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2256/e2e-tests/conf/secrets.yml -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2256/e2e-tests/conf/client.yml -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2256/e2e-tests/conf/minio-secret.yml
++ mktemp
+ local LAST_OUT=/tmp/tmp.NF9iBHlE1q
++ mktemp
+ local LAST_ERR=/tmp/tmp.XFNGEPwq5x
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl apply -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2256/e2e-tests/conf/secrets.yml -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2256/e2e-tests/conf/client.yml -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2256/e2e-tests/conf/minio-secret.yml
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.NF9iBHlE1q
secret/some-users created
deployment.apps/psmdb-client created
secret/minio-secret created
+ cat /tmp/tmp.XFNGEPwq5x
+ rm /tmp/tmp.NF9iBHlE1q /tmp/tmp.XFNGEPwq5x
+ return 0
+ log 'create PSMDB cluster: some-name'
+ set +o xtrace
[2026-04-14T11:11:17+0000] create PSMDB cluster: some-name
+ apply_cluster /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2256/e2e-tests/multi-storage/conf/some-name.yml
+ '[' -z '' ']'
+ cat_config /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2256/e2e-tests/multi-storage/conf/some-name.yml
+ kubectl_bin apply -f -
+ cat /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2256/e2e-tests/multi-storage/conf/some-name.yml
+ yq eval '(.spec | select(has("pmm"))).pmm.image = "docker.io/percona/pmm-client:2.44.1-1"'
+ yq eval '(.spec | select(.image == null)).image = "docker.io/perconalab/percona-server-mongodb-operator:main-mongod8.0"'
+ yq eval '(.spec | select(has("initImage"))).initImage = "docker.io/perconalab/percona-server-mongodb-operator:PR-2256-98b7d85f"'
+ yq eval '(.spec | select(has("backup"))).backup.image = "docker.io/perconalab/percona-server-mongodb-operator:main-backup"'
++ mktemp
+ yq eval '.spec.upgradeOptions.apply="Never"'
+ /usr/sbin/sed -e s/NAME_SPACE/multi-storage-27036/g
+ local LAST_OUT=/tmp/tmp.3f7yXMnkVH
++ mktemp
+ local LAST_ERR=/tmp/tmp.hVs1B3q9Hb
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl apply -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.3f7yXMnkVH
perconaservermongodb.psmdb.percona.com/some-name created
+ cat /tmp/tmp.hVs1B3q9Hb
+ rm /tmp/tmp.3f7yXMnkVH /tmp/tmp.hVs1B3q9Hb
+ return 0
+ log 'check if all 3 Pods started'
+ set +o xtrace
[2026-04-14T11:11:20+0000] check if all 3 Pods started
+ wait_for_running some-name-rs0 3
+ local name=some-name-rs0
+ let last_pod=2
+ local check_cluster_readyness=true
+ set_debug
+ [[ 1 == 1 ]]
+ set -o xtrace
+ local rs_name=rs0
+ local cluster_name=some-name
++ seq 0 2
+ for i in $(seq 0 $last_pod)
+ [[ 0 -eq 2 ]]
+ wait_pod some-name-rs0-0
+ local pod=some-name-rs0-0
+ set +o xtrace
waiting for pod/some-name-rs0-0 to be ready.....OK
+ for i in $(seq 0 $last_pod)
+ [[ 1 -eq 2 ]]
+ wait_pod some-name-rs0-1
+ local pod=some-name-rs0-1
+ set +o xtrace
waiting for pod/some-name-rs0-1 to be ready.....OK
+ for i in $(seq 0 $last_pod)
+ [[ 2 -eq 2 ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].arbiter.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.19m7ymmVCH
+++ mktemp
++ local LAST_ERR=/tmp/tmp.P2T5nBk8n8
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].arbiter.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.19m7ymmVCH
++ cat /tmp/tmp.P2T5nBk8n8
++ rm /tmp/tmp.19m7ymmVCH /tmp/tmp.P2T5nBk8n8
++ return 0
+ [[ '' == true ]]
+ wait_pod some-name-rs0-2
+ local pod=some-name-rs0-2
+ set +o xtrace
waiting for pod/some-name-rs0-2 to be ready......OK
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].nonvoting.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.GSwLo1n65s
+++ mktemp
++ local LAST_ERR=/tmp/tmp.w82PNKI0Ph
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].nonvoting.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.GSwLo1n65s
++ cat /tmp/tmp.w82PNKI0Ph
++ rm /tmp/tmp.GSwLo1n65s /tmp/tmp.w82PNKI0Ph
++ return 0
+ [[ '' == true ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].hidden.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.Zby3Yr8CpF
+++ mktemp
++ local LAST_ERR=/tmp/tmp.jNSo3seILy
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].hidden.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.Zby3Yr8CpF
++ cat /tmp/tmp.jNSo3seILy
++ rm /tmp/tmp.Zby3Yr8CpF /tmp/tmp.jNSo3seILy
++ return 0
+ [[ '' == true ]]
+ sleep 10
+ [[ true == true ]]
+ set +x
Waiting for cluster readyness............................
+ wait_backup_agent some-name-rs0-0
+ local agent_pod=some-name-rs0-0
+ set +o xtrace
waiting for pbm-agent to be ready in some-name-rs0-0...2026-04-14T11:12:56.000+0000 I listening for the commands

+ wait_backup_agent some-name-rs0-1
+ local agent_pod=some-name-rs0-1
+ set +o xtrace
waiting for pbm-agent to be ready in some-name-rs0-1...2026-04-14T11:13:31.000+0000 I listening for the commands

+ wait_backup_agent some-name-rs0-2
+ local agent_pod=some-name-rs0-2
+ set +o xtrace
waiting for pbm-agent to be ready in some-name-rs0-2...2026-04-14T11:13:33.000+0000 I listening for the commands

+ create_user
+ log 'create user myApp'
+ set +o xtrace
[2026-04-14T11:13:40+0000] create user myApp
+ run_mongo 'db.createUser({user:"myApp",pwd:"myPass",roles:[{db:"myApp",role:"readWrite"}]})' userAdmin:userAdmin123456@some-name-rs0.multi-storage-27036 mongodb
+ local 'command=db.createUser({user:"myApp",pwd:"myPass",roles:[{db:"myApp",role:"readWrite"}]})'
+ local uri=userAdmin:userAdmin123456@some-name-rs0.multi-storage-27036
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local replica_set=rs0
+ [[ userAdmin:userAdmin123456@some-name-rs0.multi-storage-27036 == *cfg* ]]
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.8VzNkQIhyV
+++ mktemp
++ local LAST_ERR=/tmp/tmp.8xmTG2JvNN
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.8VzNkQIhyV
++ cat /tmp/tmp.8xmTG2JvNN
++ rm /tmp/tmp.8VzNkQIhyV /tmp/tmp.8xmTG2JvNN
++ return 0
+ local client_container=psmdb-client-bb8b97679-cgccl
+ kubectl_bin exec psmdb-client-bb8b97679-cgccl -- bash -c 'printf '\''db.createUser({user:"myApp",pwd:"myPass",roles:[{db:"myApp",role:"readWrite"}]})\n'\'' | mongo mongodb://userAdmin:userAdmin123456@some-name-rs0.multi-storage-27036.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
++ mktemp
+ local LAST_OUT=/tmp/tmp.H6tizUsSij
++ mktemp
+ local LAST_ERR=/tmp/tmp.f3sw3BNXsO
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-bb8b97679-cgccl -- bash -c 'printf '\''db.createUser({user:"myApp",pwd:"myPass",roles:[{db:"myApp",role:"readWrite"}]})\n'\'' | mongo mongodb://userAdmin:userAdmin123456@some-name-rs0.multi-storage-27036.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.H6tizUsSij
Percona Server for MongoDB shell version v4.4.29-28
connecting to: mongodb://some-name-rs0.multi-storage-27036.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb&replicaSet=rs0&ssl=false
Implicit session: session { "id" : UUID("6d9b440d-d53a-458f-b7e7-257c4ec6f1b8") }
Percona Server for MongoDB server version: v8.0.20-8
WARNING: shell and server versions do not match
Successfully added user: {
	"user" : "myApp",
	"roles" : [
		{
			"db" : "myApp",
			"role" : "readWrite"
		}
	]
}
bye
+ cat /tmp/tmp.f3sw3BNXsO
+ rm /tmp/tmp.H6tizUsSij /tmp/tmp.f3sw3BNXsO
+ return 0
+ write_document 100500
+ local x=100500
+ local cmp_postfix=
+ log 'write document: 100500'
+ set +o xtrace
[2026-04-14T11:13:42+0000] write document: 100500
+ run_mongo 'use myApp\n db.test.insert({ x: 100500 })' myApp:myPass@some-name-rs0.multi-storage-27036 mongodb
+ local 'command=use myApp\n db.test.insert({ x: 100500 })'
+ local uri=myApp:myPass@some-name-rs0.multi-storage-27036
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local replica_set=rs0
+ [[ myApp:myPass@some-name-rs0.multi-storage-27036 == *cfg* ]]
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.U8GFbgXXSx
+++ mktemp
++ local LAST_ERR=/tmp/tmp.jn8yrmtYqe
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.U8GFbgXXSx
++ cat /tmp/tmp.jn8yrmtYqe
++ rm /tmp/tmp.U8GFbgXXSx /tmp/tmp.jn8yrmtYqe
++ return 0
+ local client_container=psmdb-client-bb8b97679-cgccl
+ kubectl_bin exec psmdb-client-bb8b97679-cgccl -- bash -c 'printf '\''use myApp\n db.test.insert({ x: 100500 })\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0.multi-storage-27036.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
++ mktemp
+ local LAST_OUT=/tmp/tmp.EBkNPzu74G
++ mktemp
+ local LAST_ERR=/tmp/tmp.orEU2N6nTG
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-bb8b97679-cgccl -- bash -c 'printf '\''use myApp\n db.test.insert({ x: 100500 })\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0.multi-storage-27036.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.EBkNPzu74G
Percona Server for MongoDB shell version v4.4.29-28
connecting to: mongodb://some-name-rs0.multi-storage-27036.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb&replicaSet=rs0&ssl=false
Implicit session: session { "id" : UUID("4f53e8b9-188a-4c71-af10-e989feecaf1c") }
Percona Server for MongoDB server version: v8.0.20-8
WARNING: shell and server versions do not match
switched to db myApp
WriteResult({ "nInserted" : 1 })
bye
+ cat /tmp/tmp.orEU2N6nTG
+ rm /tmp/tmp.EBkNPzu74G /tmp/tmp.orEU2N6nTG
+ return 0
+ run_backup minio-1 backup-minio-1 logical
+ local storage=minio-1
+ local name=backup-minio-1
+ local type=logical
+ log 'creating backup: backup-minio-1 (logical)'
+ set +o xtrace
[2026-04-14T11:13:45+0000] creating backup: backup-minio-1 (logical)
+ yq eval $'.metadata.name = "backup-minio-1"\n\t\t\t| .spec.storageName = "minio-1"\n\t\t\t| .spec.type = "logical"' /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2256/e2e-tests/multi-storage/conf/backup.yml
+ kubectl_bin apply -f -
++ mktemp
+ local LAST_OUT=/tmp/tmp.CptU1yQpHM
++ mktemp
+ local LAST_ERR=/tmp/tmp.GisdQf1LQM
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl apply -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.CptU1yQpHM
perconaservermongodbbackup.psmdb.percona.com/backup-minio-1 created
+ cat /tmp/tmp.GisdQf1LQM
+ rm /tmp/tmp.CptU1yQpHM /tmp/tmp.GisdQf1LQM
+ return 0
+ wait_backup backup-minio-1
+ local backup_name=backup-minio-1
+ local target_state=ready
+ set +o xtrace
waiting for backup-minio-1 to reach ready state.......OK
+ run_backup minio-2 backup-minio-2 logical
+ local storage=minio-2
+ local name=backup-minio-2
+ local type=logical
+ log 'creating backup: backup-minio-2 (logical)'
+ set +o xtrace
[2026-04-14T11:13:59+0000] creating backup: backup-minio-2 (logical)
+ yq eval $'.metadata.name = "backup-minio-2"\n\t\t\t| .spec.storageName = "minio-2"\n\t\t\t| .spec.type = "logical"' /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2256/e2e-tests/multi-storage/conf/backup.yml
+ kubectl_bin apply -f -
++ mktemp
+ local LAST_OUT=/tmp/tmp.4VDyrLaDIm
++ mktemp
+ local LAST_ERR=/tmp/tmp.PZf5hfwfuT
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl apply -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.4VDyrLaDIm
perconaservermongodbbackup.psmdb.percona.com/backup-minio-2 created
+ cat /tmp/tmp.PZf5hfwfuT
+ rm /tmp/tmp.4VDyrLaDIm /tmp/tmp.PZf5hfwfuT
+ return 0
+ wait_backup backup-minio-2
+ local backup_name=backup-minio-2
+ local target_state=ready
+ set +o xtrace
waiting for backup-minio-2 to reach ready state.......OK
+ run_backup minio-3 backup-minio-3 physical
+ local storage=minio-3
+ local name=backup-minio-3
+ local type=physical
+ log 'creating backup: backup-minio-3 (physical)'
+ set +o xtrace
[2026-04-14T11:14:13+0000] creating backup: backup-minio-3 (physical)
+ yq eval $'.metadata.name = "backup-minio-3"\n\t\t\t| .spec.storageName = "minio-3"\n\t\t\t| .spec.type = "physical"' /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2256/e2e-tests/multi-storage/conf/backup.yml
+ kubectl_bin apply -f -
++ mktemp
+ local LAST_OUT=/tmp/tmp.Azd8CQmyos
++ mktemp
+ local LAST_ERR=/tmp/tmp.bsoaVE8CvK
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl apply -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.Azd8CQmyos
perconaservermongodbbackup.psmdb.percona.com/backup-minio-3 created
+ cat /tmp/tmp.bsoaVE8CvK
+ rm /tmp/tmp.Azd8CQmyos /tmp/tmp.bsoaVE8CvK
+ return 0
+ wait_backup backup-minio-3
+ local backup_name=backup-minio-3
+ local target_state=ready
+ set +o xtrace
waiting for backup-minio-3 to reach ready state.......OK
+ write_document 100501
+ local x=100501
+ local cmp_postfix=
+ log 'write document: 100501'
+ set +o xtrace
[2026-04-14T11:14:28+0000] write document: 100501
+ run_mongo 'use myApp\n db.test.insert({ x: 100501 })' myApp:myPass@some-name-rs0.multi-storage-27036 mongodb
+ local 'command=use myApp\n db.test.insert({ x: 100501 })'
+ local uri=myApp:myPass@some-name-rs0.multi-storage-27036
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local replica_set=rs0
+ [[ myApp:myPass@some-name-rs0.multi-storage-27036 == *cfg* ]]
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.X6SEkHPnuY
+++ mktemp
++ local LAST_ERR=/tmp/tmp.DRfFWac7mx
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.X6SEkHPnuY
++ cat /tmp/tmp.DRfFWac7mx
++ rm /tmp/tmp.X6SEkHPnuY /tmp/tmp.DRfFWac7mx
++ return 0
+ local client_container=psmdb-client-bb8b97679-cgccl
+ kubectl_bin exec psmdb-client-bb8b97679-cgccl -- bash -c 'printf '\''use myApp\n db.test.insert({ x: 100501 })\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0.multi-storage-27036.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
++ mktemp
+ local LAST_OUT=/tmp/tmp.RKfF84nlmF
++ mktemp
+ local LAST_ERR=/tmp/tmp.bbcHo6aSaH
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-bb8b97679-cgccl -- bash -c 'printf '\''use myApp\n db.test.insert({ x: 100501 })\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0.multi-storage-27036.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.RKfF84nlmF
Percona Server for MongoDB shell version v4.4.29-28
connecting to: mongodb://some-name-rs0.multi-storage-27036.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb&replicaSet=rs0&ssl=false
Implicit session: session { "id" : UUID("016b9cbf-5139-432d-8004-02772fbfc10a") }
Percona Server for MongoDB server version: v8.0.20-8
WARNING: shell and server versions do not match
switched to db myApp
WriteResult({ "nInserted" : 1 })
bye
+ cat /tmp/tmp.bbcHo6aSaH
+ rm /tmp/tmp.RKfF84nlmF /tmp/tmp.bbcHo6aSaH
+ return 0
+ write_document 100502
+ local x=100502
+ local cmp_postfix=
+ log 'write document: 100502'
+ set +o xtrace
[2026-04-14T11:14:31+0000] write document: 100502
+ run_mongo 'use myApp\n db.test.insert({ x: 100502 })' myApp:myPass@some-name-rs0.multi-storage-27036 mongodb
+ local 'command=use myApp\n db.test.insert({ x: 100502 })'
+ local uri=myApp:myPass@some-name-rs0.multi-storage-27036
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local replica_set=rs0
+ [[ myApp:myPass@some-name-rs0.multi-storage-27036 == *cfg* ]]
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.x5qNf66ywj
+++ mktemp
++ local LAST_ERR=/tmp/tmp.X05vquwRrV
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.x5qNf66ywj
++ cat /tmp/tmp.X05vquwRrV
++ rm /tmp/tmp.x5qNf66ywj /tmp/tmp.X05vquwRrV
++ return 0
+ local client_container=psmdb-client-bb8b97679-cgccl
+ kubectl_bin exec psmdb-client-bb8b97679-cgccl -- bash -c 'printf '\''use myApp\n db.test.insert({ x: 100502 })\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0.multi-storage-27036.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
++ mktemp
+ local LAST_OUT=/tmp/tmp.WAkeAxuAdo
++ mktemp
+ local LAST_ERR=/tmp/tmp.Ca8WMomDFi
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-bb8b97679-cgccl -- bash -c 'printf '\''use myApp\n db.test.insert({ x: 100502 })\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0.multi-storage-27036.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.WAkeAxuAdo
Percona Server for MongoDB shell version v4.4.29-28
connecting to: mongodb://some-name-rs0.multi-storage-27036.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb&replicaSet=rs0&ssl=false
Implicit session: session { "id" : UUID("1ff26f87-0a67-4b43-a3a9-f663d33b5c58") }
Percona Server for MongoDB server version: v8.0.20-8
WARNING: shell and server versions do not match
switched to db myApp
WriteResult({ "nInserted" : 1 })
bye
+ cat /tmp/tmp.Ca8WMomDFi
+ rm /tmp/tmp.WAkeAxuAdo /tmp/tmp.Ca8WMomDFi
+ return 0
+ write_document 100503
+ local x=100503
+ local cmp_postfix=
+ log 'write document: 100503'
+ set +o xtrace
[2026-04-14T11:14:33+0000] write document: 100503
+ run_mongo 'use myApp\n db.test.insert({ x: 100503 })' myApp:myPass@some-name-rs0.multi-storage-27036 mongodb
+ local 'command=use myApp\n db.test.insert({ x: 100503 })'
+ local uri=myApp:myPass@some-name-rs0.multi-storage-27036
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local replica_set=rs0
+ [[ myApp:myPass@some-name-rs0.multi-storage-27036 == *cfg* ]]
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.E928JsBcNd
+++ mktemp
++ local LAST_ERR=/tmp/tmp.3STjJvJbUi
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.E928JsBcNd
++ cat /tmp/tmp.3STjJvJbUi
++ rm /tmp/tmp.E928JsBcNd /tmp/tmp.3STjJvJbUi
++ return 0
+ local client_container=psmdb-client-bb8b97679-cgccl
+ kubectl_bin exec psmdb-client-bb8b97679-cgccl -- bash -c 'printf '\''use myApp\n db.test.insert({ x: 100503 })\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0.multi-storage-27036.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
++ mktemp
+ local LAST_OUT=/tmp/tmp.sfYrUdmKuQ
++ mktemp
+ local LAST_ERR=/tmp/tmp.e9GwvUsndZ
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-bb8b97679-cgccl -- bash -c 'printf '\''use myApp\n db.test.insert({ x: 100503 })\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0.multi-storage-27036.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.sfYrUdmKuQ
Percona Server for MongoDB shell version v4.4.29-28
connecting to: mongodb://some-name-rs0.multi-storage-27036.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb&replicaSet=rs0&ssl=false
Implicit session: session { "id" : UUID("e24836c9-beae-4451-8527-84c406795f85") }
Percona Server for MongoDB server version: v8.0.20-8
WARNING: shell and server versions do not match
switched to db myApp
WriteResult({ "nInserted" : 1 })
bye
+ cat /tmp/tmp.e9GwvUsndZ
+ rm /tmp/tmp.sfYrUdmKuQ /tmp/tmp.e9GwvUsndZ
+ return 0
+ write_document 100504
+ local x=100504
+ local cmp_postfix=
+ log 'write document: 100504'
+ set +o xtrace
[2026-04-14T11:14:35+0000] write document: 100504
+ run_mongo 'use myApp\n db.test.insert({ x: 100504 })' myApp:myPass@some-name-rs0.multi-storage-27036 mongodb
+ local 'command=use myApp\n db.test.insert({ x: 100504 })'
+ local uri=myApp:myPass@some-name-rs0.multi-storage-27036
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local replica_set=rs0
+ [[ myApp:myPass@some-name-rs0.multi-storage-27036 == *cfg* ]]
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.TaRFX5XoBU
+++ mktemp
++ local LAST_ERR=/tmp/tmp.UTONOPx1xe
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.TaRFX5XoBU
++ cat /tmp/tmp.UTONOPx1xe
++ rm /tmp/tmp.TaRFX5XoBU /tmp/tmp.UTONOPx1xe
++ return 0
+ local client_container=psmdb-client-bb8b97679-cgccl
+ kubectl_bin exec psmdb-client-bb8b97679-cgccl -- bash -c 'printf '\''use myApp\n db.test.insert({ x: 100504 })\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0.multi-storage-27036.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
++ mktemp
+ local LAST_OUT=/tmp/tmp.W6rt0GZaez
++ mktemp
+ local LAST_ERR=/tmp/tmp.3Eizb51Nt1
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-bb8b97679-cgccl -- bash -c 'printf '\''use myApp\n db.test.insert({ x: 100504 })\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0.multi-storage-27036.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.W6rt0GZaez
Percona Server for MongoDB shell version v4.4.29-28
connecting to: mongodb://some-name-rs0.multi-storage-27036.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb&replicaSet=rs0&ssl=false
Implicit session: session { "id" : UUID("5fd780ac-1886-4ff3-b65d-d48ed5457c91") }
Percona Server for MongoDB server version: v8.0.20-8
WARNING: shell and server versions do not match
switched to db myApp
WriteResult({ "nInserted" : 1 })
bye
+ cat /tmp/tmp.3Eizb51Nt1
+ rm /tmp/tmp.W6rt0GZaez /tmp/tmp.3Eizb51Nt1
+ return 0
+ write_document 100505
+ local x=100505
+ local cmp_postfix=
+ log 'write document: 100505'
+ set +o xtrace
[2026-04-14T11:14:38+0000] write document: 100505
+ run_mongo 'use myApp\n db.test.insert({ x: 100505 })' myApp:myPass@some-name-rs0.multi-storage-27036 mongodb
+ local 'command=use myApp\n db.test.insert({ x: 100505 })'
+ local uri=myApp:myPass@some-name-rs0.multi-storage-27036
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local replica_set=rs0
+ [[ myApp:myPass@some-name-rs0.multi-storage-27036 == *cfg* ]]
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.aMnwPjuFxH
+++ mktemp
++ local LAST_ERR=/tmp/tmp.u5DBIjneGe
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.aMnwPjuFxH
++ cat /tmp/tmp.u5DBIjneGe
++ rm /tmp/tmp.aMnwPjuFxH /tmp/tmp.u5DBIjneGe
++ return 0
+ local client_container=psmdb-client-bb8b97679-cgccl
+ kubectl_bin exec psmdb-client-bb8b97679-cgccl -- bash -c 'printf '\''use myApp\n db.test.insert({ x: 100505 })\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0.multi-storage-27036.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
++ mktemp
+ local LAST_OUT=/tmp/tmp.JIdTxQmYM9
++ mktemp
+ local LAST_ERR=/tmp/tmp.g3T0BvSrCm
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-bb8b97679-cgccl -- bash -c 'printf '\''use myApp\n db.test.insert({ x: 100505 })\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0.multi-storage-27036.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.JIdTxQmYM9
Percona Server for MongoDB shell version v4.4.29-28
connecting to: mongodb://some-name-rs0.multi-storage-27036.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb&replicaSet=rs0&ssl=false
Implicit session: session { "id" : UUID("2d3a0dc1-0f64-4600-9464-6aac59d560e7") }
Percona Server for MongoDB server version: v8.0.20-8
WARNING: shell and server versions do not match
switched to db myApp
WriteResult({ "nInserted" : 1 })
bye
+ cat /tmp/tmp.g3T0BvSrCm
+ rm /tmp/tmp.JIdTxQmYM9 /tmp/tmp.g3T0BvSrCm
+ return 0
+ compare_mongo_cmd find myApp:myPass@some-name-rs0.multi-storage-27036
+ local command=find
+ local uri=myApp:myPass@some-name-rs0.multi-storage-27036
+ local postfix=
+ local suffix=
+ local database=myApp
+ local collection=test
+ local sort=
+ local tls=false
+ local replicaset=
+ local 'full_command=db.test.find()'
+ [[ -n '' ]]
+ log 'running db.test.find() in myApp'
+ set +o xtrace
[2026-04-14T11:14:41+0000] running db.test.find() in myApp
+ [[ false == true ]]
+ mongo_command=run_mongo
+ run_mongo 'use myApp\n db.test.find()' myApp:myPass@some-name-rs0.multi-storage-27036 mongodb '' ''
+ local 'command=use myApp\n db.test.find()'
+ /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
+ local uri=myApp:myPass@some-name-rs0.multi-storage-27036
+ local driver=mongodb
+ grep -E -v 'I NETWORK|W NETWORK|F NETWORK|"c":"NETWORK"|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|Started a new thread for the timer service'
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local replica_set=rs0
+ [[ myApp:myPass@some-name-rs0.multi-storage-27036 == *cfg* ]]
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.JvyLvxZ6QJ
+++ mktemp
++ local LAST_ERR=/tmp/tmp.QPJ4NJ6ifO
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.JvyLvxZ6QJ
++ cat /tmp/tmp.QPJ4NJ6ifO
++ rm /tmp/tmp.JvyLvxZ6QJ /tmp/tmp.QPJ4NJ6ifO
++ return 0
+ local client_container=psmdb-client-bb8b97679-cgccl
+ kubectl_bin exec psmdb-client-bb8b97679-cgccl -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0.multi-storage-27036.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
++ mktemp
+ local LAST_OUT=/tmp/tmp.nqRovZJcqO
++ mktemp
+ local LAST_ERR=/tmp/tmp.qKF6sQ30D8
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-bb8b97679-cgccl -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0.multi-storage-27036.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.nqRovZJcqO
+ cat /tmp/tmp.qKF6sQ30D8
+ rm /tmp/tmp.nqRovZJcqO /tmp/tmp.qKF6sQ30D8
+ return 0
+ [[ 0 -eq 0 ]]
+ diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2256/e2e-tests/multi-storage/compare/find.json /tmp/tmp.H2WE2QZul5/find
+ sleep_with_log 70 'wait for oplog chunks to be uploaded'
+ local d=70
+ local 'msg=wait for oplog chunks to be uploaded'
+ log 'wait for oplog chunks to be uploaded. sleeping for 70 seconds'
+ set +o xtrace
[2026-04-14T11:14:43+0000] wait for oplog chunks to be uploaded. sleeping for 70 seconds
+ sleep 70
+ wait_for_restorable_time backup-minio-3
+ local backup_name=backup-minio-3
++ get_latest_restorable_time backup-minio-3
++ local backup_name=backup-minio-3
+++ kubectl_bin get psmdb-backup backup-minio-3 -o yaml
+++ yq .status.latestRestorableTime
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.Rn9UTZPeyi
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.bOwCkM8WMl
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in $(seq 0 2)
+++ set +e
+++ kubectl get psmdb-backup backup-minio-3 -o yaml
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.Rn9UTZPeyi
+++ cat /tmp/tmp.bOwCkM8WMl
+++ rm /tmp/tmp.Rn9UTZPeyi /tmp/tmp.bOwCkM8WMl
+++ return 0
++ local time=2026-04-14T11:15:37Z
++ [[ 2026-04-14T11:15:37Z == null ]]
++ echo -n 2026-04-14T11:15:37Z
+ local latest_restorable_time=2026-04-14T11:15:37Z
+ local retries=0
+ echo -n 'waiting for psmdb-backup/backup-minio-3'\''s latest restorable time'
waiting for psmdb-backup/backup-minio-3's latest restorable time+ [[ 2026-04-14T11:15:37Z != null ]]
+ echo '.OK latestRestorableTime: 2026-04-14T11:15:37Z'
.OK latestRestorableTime: 2026-04-14T11:15:37Z
+ wait_for_advanced_restorable_time backup-minio-3
+ local backup_name=backup-minio-3
+++ get_latest_restorable_time backup-minio-3
+++ local backup_name=backup-minio-3
++++ yq .status.latestRestorableTime
++++ kubectl_bin get psmdb-backup backup-minio-3 -o yaml
+++++ mktemp
++++ local LAST_OUT=/tmp/tmp.bwVNcW4X9N
+++++ mktemp
++++ local LAST_ERR=/tmp/tmp.I37KI0ZBE4
++++ local exit_status=0
++++ local timeout=4
+++++ seq 0 2
++++ for i in $(seq 0 2)
++++ set +e
++++ kubectl get psmdb-backup backup-minio-3 -o yaml
++++ exit_status=0
++++ set -e
++++ '[' 0 '!=' 0 -a -n 1 ']'
++++ break
++++ cat /tmp/tmp.bwVNcW4X9N
++++ cat /tmp/tmp.I37KI0ZBE4
++++ rm /tmp/tmp.bwVNcW4X9N /tmp/tmp.I37KI0ZBE4
++++ return 0
+++ local time=2026-04-14T11:15:37Z
+++ [[ 2026-04-14T11:15:37Z == null ]]
+++ echo -n 2026-04-14T11:15:37Z
++ datetime_to_timestamp 2026-04-14T11:15:37Z
++ local datetime=2026-04-14T11:15:37Z
++ TZ=UTC
++ /usr/sbin/date -d2026-04-14T11:15:37Z +%s
+ local latest_restorable_time=1776165337
++ get_backup_last_write backup-minio-3
++ local backup_name=backup-minio-3
+++ kubectl_bin get psmdb-backup backup-minio-3 -o yaml
+++ yq .status.pbmName
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.d0YqbBxB9h
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.09k21Z5MYZ
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in $(seq 0 2)
+++ set +e
+++ kubectl get psmdb-backup backup-minio-3 -o yaml
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.d0YqbBxB9h
+++ cat /tmp/tmp.09k21Z5MYZ
+++ rm /tmp/tmp.d0YqbBxB9h /tmp/tmp.09k21Z5MYZ
+++ return 0
++ local pbm_name=2026-04-14T11:14:16Z
++ kubectl_bin exec some-name-rs0-0 -c backup-agent -- pbm list -o json
++ jq --arg v 2026-04-14T11:14:16Z '.snapshots[] | select(.name==$v) | .restoreTo'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.otu1ll2SSk
+++ mktemp
++ local LAST_ERR=/tmp/tmp.iKOcwpmbW4
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl exec some-name-rs0-0 -c backup-agent -- pbm list -o json
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.otu1ll2SSk
++ cat /tmp/tmp.iKOcwpmbW4
++ rm /tmp/tmp.otu1ll2SSk /tmp/tmp.iKOcwpmbW4
++ return 0
+ local latest_write=1776165258
+ local retries=0
++ timestamp_to_datetime 1776165258
++ local ts=1776165258
++ TZ=UTC
++ /usr/sbin/date -d@1776165258 +%Y-%m-%dT%H:%M:%S%Z
+ echo -n 'waiting for psmdb-backup/backup-minio-3'\''s latest restorable time to advance its latest write (2026-04-14T11:14:18UTC)'
waiting for psmdb-backup/backup-minio-3's latest restorable time to advance its latest write (2026-04-14T11:14:18UTC)+ [[ 1776165337 -gt 1776165258 ]]
++ timestamp_to_datetime 1776165337
++ local ts=1776165337
++ TZ=UTC
++ /usr/sbin/date -d@1776165337 +%Y-%m-%dT%H:%M:%S%Z
+ echo '.OK latestRestorableTime: 2026-04-14T11:15:37UTC'
.OK latestRestorableTime: 2026-04-14T11:15:37UTC
++ get_latest_restorable_time backup-minio-3
++ local backup_name=backup-minio-3
+++ kubectl_bin get psmdb-backup backup-minio-3 -o yaml
+++ yq .status.latestRestorableTime
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.IHMH7BJehQ
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.a0CF7t0oi3
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in $(seq 0 2)
+++ set +e
+++ kubectl get psmdb-backup backup-minio-3 -o yaml
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.IHMH7BJehQ
+++ cat /tmp/tmp.a0CF7t0oi3
+++ rm /tmp/tmp.IHMH7BJehQ /tmp/tmp.a0CF7t0oi3
+++ return 0
++ local time=2026-04-14T11:15:37Z
++ [[ 2026-04-14T11:15:37Z == null ]]
++ echo -n 2026-04-14T11:15:37Z
+ restore_time=2026-04-14T11:15:37Z
++ format_pitr_target 2026-04-14T11:15:37Z
++ local target=2026-04-14T11:15:37Z
++ echo 2026-04-14T11:15:37Z
++ sed s/Z//
++ sed 's/T/ /'
+ check_recovery backup-minio-3 '2026-04-14 11:15:37'
+ local backup_name=backup-minio-3
+ local 'restore_time=2026-04-14 11:15:37'
+ local cmp_postfix=
+ drop_collection myApp test myApp myPass
+ local db=myApp
+ local collection=test
+ local user=myApp
+ local pass=myPass
+ log 'dropping collection: myApp.test'
+ set +o xtrace
[2026-04-14T11:15:59+0000] dropping collection: myApp.test
+ run_mongo 'use myApp\n db.test.drop()' myApp:myPass@some-name-rs0.multi-storage-27036 mongodb
+ local 'command=use myApp\n db.test.drop()'
+ local uri=myApp:myPass@some-name-rs0.multi-storage-27036
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local replica_set=rs0
+ [[ myApp:myPass@some-name-rs0.multi-storage-27036 == *cfg* ]]
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.fY06q6OX5I
+++ mktemp
++ local LAST_ERR=/tmp/tmp.G8v5mMKbv6
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.fY06q6OX5I
++ cat /tmp/tmp.G8v5mMKbv6
++ rm /tmp/tmp.fY06q6OX5I /tmp/tmp.G8v5mMKbv6
++ return 0
+ local client_container=psmdb-client-bb8b97679-cgccl
+ kubectl_bin exec psmdb-client-bb8b97679-cgccl -- bash -c 'printf '\''use myApp\n db.test.drop()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0.multi-storage-27036.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
++ mktemp
+ local LAST_OUT=/tmp/tmp.XYDOxh3E1G
++ mktemp
+ local LAST_ERR=/tmp/tmp.snmtrxvii6
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-bb8b97679-cgccl -- bash -c 'printf '\''use myApp\n db.test.drop()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0.multi-storage-27036.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.XYDOxh3E1G
Percona Server for MongoDB shell version v4.4.29-28
connecting to: mongodb://some-name-rs0.multi-storage-27036.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb&replicaSet=rs0&ssl=false
Implicit session: session { "id" : UUID("58106d2f-d5f7-455e-979b-2294cdfd4e02") }
Percona Server for MongoDB server version: v8.0.20-8
WARNING: shell and server versions do not match
switched to db myApp
true
bye
+ cat /tmp/tmp.snmtrxvii6
+ rm /tmp/tmp.XYDOxh3E1G /tmp/tmp.snmtrxvii6
+ return 0
+ log 'creating restore: restore-backup-minio-3 restore_to: 2026-04-14 11:15:37'
+ set +o xtrace
[2026-04-14T11:16:02+0000] creating restore: restore-backup-minio-3 restore_to: 2026-04-14 11:15:37
+ /usr/sbin/sed -e 's/name:/name: restore-backup-minio-3/'
+ cat /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2256/e2e-tests/multi-storage/conf/restore.yml
+ /usr/sbin/sed -e 's/backupName:/backupName: backup-minio-3/'
+ /usr/sbin/sed -e 's/date:/date: 2026-04-14 11:15:37/'
+ kubectl_bin apply -f -
++ mktemp
+ local LAST_OUT=/tmp/tmp.gN3O4QPIOn
++ mktemp
+ local LAST_ERR=/tmp/tmp.8WQPQChaQK
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl apply -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.gN3O4QPIOn
perconaservermongodbrestore.psmdb.percona.com/restore-backup-minio-3 created
+ cat /tmp/tmp.8WQPQChaQK
+ rm /tmp/tmp.gN3O4QPIOn /tmp/tmp.8WQPQChaQK
+ return 0
+ wait_restore backup-minio-3 some-name requested 0 900 1
+ local backup_name=backup-minio-3
+ local cluster_name=some-name
+ local target_state=requested
+ local wait_cluster_consistency=0
+ local wait_time=900
+ local ok_if_ready=1
+ set +o xtrace
Waiting for the psmdb-restore/restore-backup-minio-3 object to be created.OK
Waiting psmdb-restore/restore-backup-minio-3 to reach state "requested" ...OK after 2 minutes
+ [[ 0 -eq 1 ]]
+ wait_restore backup-minio-3 some-name ready 0 1600
+ local backup_name=backup-minio-3
+ local cluster_name=some-name
+ local target_state=ready
+ local wait_cluster_consistency=0
+ local wait_time=1600
+ local ok_if_ready=0
+ set +o xtrace
Waiting for the psmdb-restore/restore-backup-minio-3 object to be created.OK
Waiting psmdb-restore/restore-backup-minio-3 to reach state "ready" ..OK after 1 minutes
+ [[ 0 -eq 1 ]]
+ wait_for_running some-name-rs0 3
+ local name=some-name-rs0
+ let last_pod=2
+ local check_cluster_readyness=true
+ set_debug
+ [[ 1 == 1 ]]
+ set -o xtrace
+ local rs_name=rs0
+ local cluster_name=some-name
++ seq 0 2
+ for i in $(seq 0 $last_pod)
+ [[ 0 -eq 2 ]]
+ wait_pod some-name-rs0-0
+ local pod=some-name-rs0-0
+ set +o xtrace
waiting for pod/some-name-rs0-0 to be ready.OK
+ for i in $(seq 0 $last_pod)
+ [[ 1 -eq 2 ]]
+ wait_pod some-name-rs0-1
+ local pod=some-name-rs0-1
+ set +o xtrace
waiting for pod/some-name-rs0-1 to be ready........OK
+ for i in $(seq 0 $last_pod)
+ [[ 2 -eq 2 ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].arbiter.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.382dGb2OUz
+++ mktemp
++ local LAST_ERR=/tmp/tmp.ItigxlAPrT
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].arbiter.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.382dGb2OUz
++ cat /tmp/tmp.ItigxlAPrT
++ rm /tmp/tmp.382dGb2OUz /tmp/tmp.ItigxlAPrT
++ return 0
+ [[ '' == true ]]
+ wait_pod some-name-rs0-2
+ local pod=some-name-rs0-2
+ set +o xtrace
waiting for pod/some-name-rs0-2 to be ready...OK
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].nonvoting.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.LcY4ByDimM
+++ mktemp
++ local LAST_ERR=/tmp/tmp.l6TovBpyIO
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].nonvoting.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.LcY4ByDimM
++ cat /tmp/tmp.l6TovBpyIO
++ rm /tmp/tmp.LcY4ByDimM /tmp/tmp.l6TovBpyIO
++ return 0
+ [[ '' == true ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].hidden.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.WqOfYCgQ8p
+++ mktemp
++ local LAST_ERR=/tmp/tmp.TQvHuDK1qg
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].hidden.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.WqOfYCgQ8p
++ cat /tmp/tmp.TQvHuDK1qg
++ rm /tmp/tmp.WqOfYCgQ8p /tmp/tmp.TQvHuDK1qg
++ return 0
+ [[ '' == true ]]
+ sleep 10
+ [[ true == true ]]
+ set +x
Waiting for cluster readyness.
+ compare_mongo_cmd find myApp:myPass@some-name-rs0.multi-storage-27036 '' .svc.cluster.local myApp test 'sort( { x: 1 } )'
+ local command=find
+ local uri=myApp:myPass@some-name-rs0.multi-storage-27036
+ local postfix=
+ local suffix=.svc.cluster.local
+ local database=myApp
+ local collection=test
+ local 'sort=sort( { x: 1 } )'
+ local tls=false
+ local replicaset=
+ local 'full_command=db.test.find()'
+ [[ -n sort( { x: 1 } ) ]]
+ full_command='db.test.find().sort( { x: 1 } )'
+ log 'running db.test.find().sort( { x: 1 } ) in myApp'
+ set +o xtrace
[2026-04-14T11:22:27+0000] running db.test.find().sort( { x: 1 } ) in myApp
+ [[ false == true ]]
+ mongo_command=run_mongo
+ run_mongo 'use myApp\n db.test.find().sort( { x: 1 } )' myApp:myPass@some-name-rs0.multi-storage-27036 mongodb .svc.cluster.local ''
+ local 'command=use myApp\n db.test.find().sort( { x: 1 } )'
+ local uri=myApp:myPass@some-name-rs0.multi-storage-27036
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ grep -E -v 'I NETWORK|W NETWORK|F NETWORK|"c":"NETWORK"|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|Started a new thread for the timer service'
+ local replica_set=rs0
+ [[ myApp:myPass@some-name-rs0.multi-storage-27036 == *cfg* ]]
+ /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.V8vr2m8Qe1
+++ mktemp
++ local LAST_ERR=/tmp/tmp.VHGlEt7vd1
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.V8vr2m8Qe1
++ cat /tmp/tmp.VHGlEt7vd1
++ rm /tmp/tmp.V8vr2m8Qe1 /tmp/tmp.VHGlEt7vd1
++ return 0
+ local client_container=psmdb-client-bb8b97679-cgccl
+ kubectl_bin exec psmdb-client-bb8b97679-cgccl -- bash -c 'printf '\''use myApp\n db.test.find().sort( { x: 1 } )\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0.multi-storage-27036.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
++ mktemp
+ local LAST_OUT=/tmp/tmp.J7LdYblijH
++ mktemp
+ local LAST_ERR=/tmp/tmp.VRZ07dQAXB
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-bb8b97679-cgccl -- bash -c 'printf '\''use myApp\n db.test.find().sort( { x: 1 } )\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0.multi-storage-27036.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.J7LdYblijH
+ cat /tmp/tmp.VRZ07dQAXB
+ rm /tmp/tmp.J7LdYblijH /tmp/tmp.VRZ07dQAXB
+ return 0
+ [[ 0 -eq 0 ]]
+ diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2256/e2e-tests/multi-storage/compare/find.json /tmp/tmp.H2WE2QZul5/find
+ sleep_with_log 60 'wait resync to finish'
+ local d=60
+ local 'msg=wait resync to finish'
+ log 'wait resync to finish. sleeping for 60 seconds'
+ set +o xtrace
[2026-04-14T11:22:30+0000] wait resync to finish. sleeping for 60 seconds
+ sleep 60
+ log 'changing main storage from minio-1 to minio-2'
+ set +o xtrace
[2026-04-14T11:23:30+0000] changing main storage from minio-1 to minio-2
+ kubectl patch psmdb some-name --type=json $'-p=[\n  {"op": "remove", "path": "/spec/backup/storages/minio-1/main"},\n  {"op": "add", "path": "/spec/backup/storages/minio-2/main", "value": true}\n]'
perconaservermongodb.psmdb.percona.com/some-name patched
+ sleep_with_log 60 'wait resync to finish'
+ local d=60
+ local 'msg=wait resync to finish'
+ log 'wait resync to finish. sleeping for 60 seconds'
+ set +o xtrace
[2026-04-14T11:23:31+0000] wait resync to finish. sleeping for 60 seconds
+ sleep 60
+ run_backup minio-2 backup-minio-2-1 logical
+ local storage=minio-2
+ local name=backup-minio-2-1
+ local type=logical
+ log 'creating backup: backup-minio-2-1 (logical)'
+ set +o xtrace
[2026-04-14T11:24:31+0000] creating backup: backup-minio-2-1 (logical)
+ yq eval $'.metadata.name = "backup-minio-2-1"\n\t\t\t| .spec.storageName = "minio-2"\n\t\t\t| .spec.type = "logical"' /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2256/e2e-tests/multi-storage/conf/backup.yml
+ kubectl_bin apply -f -
++ mktemp
+ local LAST_OUT=/tmp/tmp.Yif38uTecy
++ mktemp
+ local LAST_ERR=/tmp/tmp.fH90kiQkdI
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl apply -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.Yif38uTecy
perconaservermongodbbackup.psmdb.percona.com/backup-minio-2-1 created
+ cat /tmp/tmp.fH90kiQkdI
+ rm /tmp/tmp.Yif38uTecy /tmp/tmp.fH90kiQkdI
+ return 0
+ wait_backup backup-minio-2-1
+ local backup_name=backup-minio-2-1
+ local target_state=ready
+ set +o xtrace
waiting for backup-minio-2-1 to reach ready state.....OK
+ write_document 100506
+ local x=100506
+ local cmp_postfix=
+ log 'write document: 100506'
+ set +o xtrace
[2026-04-14T11:24:41+0000] write document: 100506
+ run_mongo 'use myApp\n db.test.insert({ x: 100506 })' myApp:myPass@some-name-rs0.multi-storage-27036 mongodb
+ local 'command=use myApp\n db.test.insert({ x: 100506 })'
+ local uri=myApp:myPass@some-name-rs0.multi-storage-27036
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local replica_set=rs0
+ [[ myApp:myPass@some-name-rs0.multi-storage-27036 == *cfg* ]]
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.4hfqJrSFhr
+++ mktemp
++ local LAST_ERR=/tmp/tmp.yHJG9c7Y2Z
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.4hfqJrSFhr
++ cat /tmp/tmp.yHJG9c7Y2Z
++ rm /tmp/tmp.4hfqJrSFhr /tmp/tmp.yHJG9c7Y2Z
++ return 0
+ local client_container=psmdb-client-bb8b97679-cgccl
+ kubectl_bin exec psmdb-client-bb8b97679-cgccl -- bash -c 'printf '\''use myApp\n db.test.insert({ x: 100506 })\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0.multi-storage-27036.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
++ mktemp
+ local LAST_OUT=/tmp/tmp.CktBSlcvE1
++ mktemp
+ local LAST_ERR=/tmp/tmp.RM2mQP3FZA
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-bb8b97679-cgccl -- bash -c 'printf '\''use myApp\n db.test.insert({ x: 100506 })\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0.multi-storage-27036.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.CktBSlcvE1
Percona Server for MongoDB shell version v4.4.29-28
connecting to: mongodb://some-name-rs0.multi-storage-27036.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb&replicaSet=rs0&ssl=false
Implicit session: session { "id" : UUID("0e9e3357-44f3-4b5b-ab40-41e4786359f8") }
Percona Server for MongoDB server version: v8.0.20-8
WARNING: shell and server versions do not match
switched to db myApp
WriteResult({ "nInserted" : 1 })
bye
+ cat /tmp/tmp.RM2mQP3FZA
+ rm /tmp/tmp.CktBSlcvE1 /tmp/tmp.RM2mQP3FZA
+ return 0
+ write_document 100507
+ local x=100507
+ local cmp_postfix=
+ log 'write document: 100507'
+ set +o xtrace
[2026-04-14T11:24:43+0000] write document: 100507
+ run_mongo 'use myApp\n db.test.insert({ x: 100507 })' myApp:myPass@some-name-rs0.multi-storage-27036 mongodb
+ local 'command=use myApp\n db.test.insert({ x: 100507 })'
+ local uri=myApp:myPass@some-name-rs0.multi-storage-27036
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local replica_set=rs0
+ [[ myApp:myPass@some-name-rs0.multi-storage-27036 == *cfg* ]]
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.E12AZg15G9
+++ mktemp
++ local LAST_ERR=/tmp/tmp.nqgnJISRWY
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.E12AZg15G9
++ cat /tmp/tmp.nqgnJISRWY
++ rm /tmp/tmp.E12AZg15G9 /tmp/tmp.nqgnJISRWY
++ return 0
+ local client_container=psmdb-client-bb8b97679-cgccl
+ kubectl_bin exec psmdb-client-bb8b97679-cgccl -- bash -c 'printf '\''use myApp\n db.test.insert({ x: 100507 })\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0.multi-storage-27036.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
++ mktemp
+ local LAST_OUT=/tmp/tmp.lClaCxCnYK
++ mktemp
+ local LAST_ERR=/tmp/tmp.GkDaCwV9Bk
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-bb8b97679-cgccl -- bash -c 'printf '\''use myApp\n db.test.insert({ x: 100507 })\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0.multi-storage-27036.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.lClaCxCnYK
Percona Server for MongoDB shell version v4.4.29-28
connecting to: mongodb://some-name-rs0.multi-storage-27036.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb&replicaSet=rs0&ssl=false
Implicit session: session { "id" : UUID("c550ed87-e937-4b85-b0b9-e44d6a6a6e9a") }
Percona Server for MongoDB server version: v8.0.20-8
WARNING: shell and server versions do not match
switched to db myApp
WriteResult({ "nInserted" : 1 })
bye
+ cat /tmp/tmp.GkDaCwV9Bk
+ rm /tmp/tmp.lClaCxCnYK /tmp/tmp.GkDaCwV9Bk
+ return 0
+ write_document 100508
+ local x=100508
+ local cmp_postfix=
+ log 'write document: 100508'
+ set +o xtrace
[2026-04-14T11:24:46+0000] write document: 100508
+ run_mongo 'use myApp\n db.test.insert({ x: 100508 })' myApp:myPass@some-name-rs0.multi-storage-27036 mongodb
+ local 'command=use myApp\n db.test.insert({ x: 100508 })'
+ local uri=myApp:myPass@some-name-rs0.multi-storage-27036
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local replica_set=rs0
+ [[ myApp:myPass@some-name-rs0.multi-storage-27036 == *cfg* ]]
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.z52647WQDx
+++ mktemp
++ local LAST_ERR=/tmp/tmp.fdpm4Qe1qe
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.z52647WQDx
++ cat /tmp/tmp.fdpm4Qe1qe
++ rm /tmp/tmp.z52647WQDx /tmp/tmp.fdpm4Qe1qe
++ return 0
+ local client_container=psmdb-client-bb8b97679-cgccl
+ kubectl_bin exec psmdb-client-bb8b97679-cgccl -- bash -c 'printf '\''use myApp\n db.test.insert({ x: 100508 })\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0.multi-storage-27036.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
++ mktemp
+ local LAST_OUT=/tmp/tmp.8ktF5x4BqE
++ mktemp
+ local LAST_ERR=/tmp/tmp.a97O0aST6X
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-bb8b97679-cgccl -- bash -c 'printf '\''use myApp\n db.test.insert({ x: 100508 })\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0.multi-storage-27036.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.8ktF5x4BqE
Percona Server for MongoDB shell version v4.4.29-28
connecting to: mongodb://some-name-rs0.multi-storage-27036.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb&replicaSet=rs0&ssl=false
Implicit session: session { "id" : UUID("3a6779f0-2f2b-4891-8eb1-19e9885c59aa") }
Percona Server for MongoDB server version: v8.0.20-8
WARNING: shell and server versions do not match
switched to db myApp
WriteResult({ "nInserted" : 1 })
bye
+ cat /tmp/tmp.a97O0aST6X
+ rm /tmp/tmp.8ktF5x4BqE /tmp/tmp.a97O0aST6X
+ return 0
+ write_document 100509
+ local x=100509
+ local cmp_postfix=
+ log 'write document: 100509'
+ set +o xtrace
[2026-04-14T11:24:48+0000] write document: 100509
+ run_mongo 'use myApp\n db.test.insert({ x: 100509 })' myApp:myPass@some-name-rs0.multi-storage-27036 mongodb
+ local 'command=use myApp\n db.test.insert({ x: 100509 })'
+ local uri=myApp:myPass@some-name-rs0.multi-storage-27036
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local replica_set=rs0
+ [[ myApp:myPass@some-name-rs0.multi-storage-27036 == *cfg* ]]
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.MZBcIaWNp4
+++ mktemp
++ local LAST_ERR=/tmp/tmp.ibYU7VnX4Y
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.MZBcIaWNp4
++ cat /tmp/tmp.ibYU7VnX4Y
++ rm /tmp/tmp.MZBcIaWNp4 /tmp/tmp.ibYU7VnX4Y
++ return 0
+ local client_container=psmdb-client-bb8b97679-cgccl
+ kubectl_bin exec psmdb-client-bb8b97679-cgccl -- bash -c 'printf '\''use myApp\n db.test.insert({ x: 100509 })\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0.multi-storage-27036.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
++ mktemp
+ local LAST_OUT=/tmp/tmp.zIk2g6sB7I
++ mktemp
+ local LAST_ERR=/tmp/tmp.37hJYWGEzM
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-bb8b97679-cgccl -- bash -c 'printf '\''use myApp\n db.test.insert({ x: 100509 })\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0.multi-storage-27036.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.zIk2g6sB7I
Percona Server for MongoDB shell version v4.4.29-28
connecting to: mongodb://some-name-rs0.multi-storage-27036.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb&replicaSet=rs0&ssl=false
Implicit session: session { "id" : UUID("b6dde6fa-ad5e-4f95-8351-bb9ac3a1db27") }
Percona Server for MongoDB server version: v8.0.20-8
WARNING: shell and server versions do not match
switched to db myApp
WriteResult({ "nInserted" : 1 })
bye
+ cat /tmp/tmp.37hJYWGEzM
+ rm /tmp/tmp.zIk2g6sB7I /tmp/tmp.37hJYWGEzM
+ return 0
+ write_document 100510
+ local x=100510
+ local cmp_postfix=
+ log 'write document: 100510'
+ set +o xtrace
[2026-04-14T11:24:50+0000] write document: 100510
+ run_mongo 'use myApp\n db.test.insert({ x: 100510 })' myApp:myPass@some-name-rs0.multi-storage-27036 mongodb
+ local 'command=use myApp\n db.test.insert({ x: 100510 })'
+ local uri=myApp:myPass@some-name-rs0.multi-storage-27036
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local replica_set=rs0
+ [[ myApp:myPass@some-name-rs0.multi-storage-27036 == *cfg* ]]
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.jwYUfLf58n
+++ mktemp
++ local LAST_ERR=/tmp/tmp.vgkG2axnGi
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.jwYUfLf58n
++ cat /tmp/tmp.vgkG2axnGi
++ rm /tmp/tmp.jwYUfLf58n /tmp/tmp.vgkG2axnGi
++ return 0
+ local client_container=psmdb-client-bb8b97679-cgccl
+ kubectl_bin exec psmdb-client-bb8b97679-cgccl -- bash -c 'printf '\''use myApp\n db.test.insert({ x: 100510 })\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0.multi-storage-27036.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
++ mktemp
+ local LAST_OUT=/tmp/tmp.WSQDHuBfUi
++ mktemp
+ local LAST_ERR=/tmp/tmp.D19mx7JO4c
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-bb8b97679-cgccl -- bash -c 'printf '\''use myApp\n db.test.insert({ x: 100510 })\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0.multi-storage-27036.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.WSQDHuBfUi
Percona Server for MongoDB shell version v4.4.29-28
connecting to: mongodb://some-name-rs0.multi-storage-27036.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb&replicaSet=rs0&ssl=false
Implicit session: session { "id" : UUID("562cd734-249a-42b7-aee2-f6b9770a7127") }
Percona Server for MongoDB server version: v8.0.20-8
WARNING: shell and server versions do not match
switched to db myApp
WriteResult({ "nInserted" : 1 })
bye
+ cat /tmp/tmp.D19mx7JO4c
+ rm /tmp/tmp.WSQDHuBfUi /tmp/tmp.D19mx7JO4c
+ return 0
+ compare_mongo_cmd find myApp:myPass@some-name-rs0.multi-storage-27036 -2nd .svc.cluster.local myApp test 'sort( { x: 1 } )'
+ local command=find
+ local uri=myApp:myPass@some-name-rs0.multi-storage-27036
+ local postfix=-2nd
+ local suffix=.svc.cluster.local
+ local database=myApp
+ local collection=test
+ local 'sort=sort( { x: 1 } )'
+ local tls=false
+ local replicaset=
+ local 'full_command=db.test.find()'
+ [[ -n sort( { x: 1 } ) ]]
+ full_command='db.test.find().sort( { x: 1 } )'
+ log 'running db.test.find().sort( { x: 1 } ) in myApp'
+ set +o xtrace
[2026-04-14T11:24:52+0000] running db.test.find().sort( { x: 1 } ) in myApp
+ [[ false == true ]]
+ mongo_command=run_mongo
+ grep -E -v 'I NETWORK|W NETWORK|F NETWORK|"c":"NETWORK"|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|Started a new thread for the timer service'
+ run_mongo 'use myApp\n db.test.find().sort( { x: 1 } )' myApp:myPass@some-name-rs0.multi-storage-27036 mongodb .svc.cluster.local ''
+ /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
+ local 'command=use myApp\n db.test.find().sort( { x: 1 } )'
+ local uri=myApp:myPass@some-name-rs0.multi-storage-27036
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local replica_set=rs0
+ [[ myApp:myPass@some-name-rs0.multi-storage-27036 == *cfg* ]]
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.NjoB4qu6ej
+++ mktemp
++ local LAST_ERR=/tmp/tmp.CEIt13dtgB
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.NjoB4qu6ej
++ cat /tmp/tmp.CEIt13dtgB
++ rm /tmp/tmp.NjoB4qu6ej /tmp/tmp.CEIt13dtgB
++ return 0
+ local client_container=psmdb-client-bb8b97679-cgccl
+ kubectl_bin exec psmdb-client-bb8b97679-cgccl -- bash -c 'printf '\''use myApp\n db.test.find().sort( { x: 1 } )\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0.multi-storage-27036.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
++ mktemp
+ local LAST_OUT=/tmp/tmp.yaNx1sJRkn
++ mktemp
+ local LAST_ERR=/tmp/tmp.M4tszcUel9
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-bb8b97679-cgccl -- bash -c 'printf '\''use myApp\n db.test.find().sort( { x: 1 } )\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0.multi-storage-27036.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.yaNx1sJRkn
+ cat /tmp/tmp.M4tszcUel9
+ rm /tmp/tmp.yaNx1sJRkn /tmp/tmp.M4tszcUel9
+ return 0
+ [[ 0 -eq 0 ]]
+ diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2256/e2e-tests/multi-storage/compare/find-2nd.json /tmp/tmp.H2WE2QZul5/find-2nd
+ wait_for_restorable_time backup-minio-2-1
+ local backup_name=backup-minio-2-1
++ get_latest_restorable_time backup-minio-2-1
++ local backup_name=backup-minio-2-1
+++ kubectl_bin get psmdb-backup backup-minio-2-1 -o yaml
+++ yq .status.latestRestorableTime
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.kKhySvAK1w
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.KRCLyDvy8D
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in $(seq 0 2)
+++ set +e
+++ kubectl get psmdb-backup backup-minio-2-1 -o yaml
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.kKhySvAK1w
+++ cat /tmp/tmp.KRCLyDvy8D
+++ rm /tmp/tmp.kKhySvAK1w /tmp/tmp.KRCLyDvy8D
+++ return 0
++ local time=2026-04-14T11:24:34Z
++ [[ 2026-04-14T11:24:34Z == null ]]
++ echo -n 2026-04-14T11:24:34Z
+ local latest_restorable_time=2026-04-14T11:24:34Z
+ local retries=0
+ echo -n 'waiting for psmdb-backup/backup-minio-2-1'\''s latest restorable time'
waiting for psmdb-backup/backup-minio-2-1's latest restorable time+ [[ 2026-04-14T11:24:34Z != null ]]
+ echo '.OK latestRestorableTime: 2026-04-14T11:24:34Z'
.OK latestRestorableTime: 2026-04-14T11:24:34Z
+ wait_for_advanced_restorable_time backup-minio-2-1
+ local backup_name=backup-minio-2-1
+++ get_latest_restorable_time backup-minio-2-1
+++ local backup_name=backup-minio-2-1
++++ kubectl_bin get psmdb-backup backup-minio-2-1 -o yaml
++++ yq .status.latestRestorableTime
+++++ mktemp
++++ local LAST_OUT=/tmp/tmp.Nc5uKNLjyX
+++++ mktemp
++++ local LAST_ERR=/tmp/tmp.9judlm5KRy
++++ local exit_status=0
++++ local timeout=4
+++++ seq 0 2
++++ for i in $(seq 0 2)
++++ set +e
++++ kubectl get psmdb-backup backup-minio-2-1 -o yaml
++++ exit_status=0
++++ set -e
++++ '[' 0 '!=' 0 -a -n 1 ']'
++++ break
++++ cat /tmp/tmp.Nc5uKNLjyX
++++ cat /tmp/tmp.9judlm5KRy
++++ rm /tmp/tmp.Nc5uKNLjyX /tmp/tmp.9judlm5KRy
++++ return 0
+++ local time=2026-04-14T11:24:34Z
+++ [[ 2026-04-14T11:24:34Z == null ]]
+++ echo -n 2026-04-14T11:24:34Z
++ datetime_to_timestamp 2026-04-14T11:24:34Z
++ local datetime=2026-04-14T11:24:34Z
++ TZ=UTC
++ /usr/sbin/date -d2026-04-14T11:24:34Z +%s
+ local latest_restorable_time=1776165874
++ get_backup_last_write backup-minio-2-1
++ local backup_name=backup-minio-2-1
+++ kubectl_bin get psmdb-backup backup-minio-2-1 -o yaml
+++ yq .status.pbmName
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.TgGwzhJevK
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.74u6C0dDIw
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in $(seq 0 2)
+++ set +e
+++ kubectl get psmdb-backup backup-minio-2-1 -o yaml
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.TgGwzhJevK
+++ cat /tmp/tmp.74u6C0dDIw
+++ rm /tmp/tmp.TgGwzhJevK /tmp/tmp.74u6C0dDIw
+++ return 0
++ local pbm_name=2026-04-14T11:24:34Z
++ kubectl_bin exec some-name-rs0-0 -c backup-agent -- pbm list -o json
++ jq --arg v 2026-04-14T11:24:34Z '.snapshots[] | select(.name==$v) | .restoreTo'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.Tlpk2juVzM
+++ mktemp
++ local LAST_ERR=/tmp/tmp.BGWsNjZSvJ
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl exec some-name-rs0-0 -c backup-agent -- pbm list -o json
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.Tlpk2juVzM
++ cat /tmp/tmp.BGWsNjZSvJ
++ rm /tmp/tmp.Tlpk2juVzM /tmp/tmp.BGWsNjZSvJ
++ return 0
+ local latest_write=1776165878
+ local retries=0
++ timestamp_to_datetime 1776165878
++ local ts=1776165878
++ TZ=UTC
++ /usr/sbin/date -d@1776165878 +%Y-%m-%dT%H:%M:%S%Z
+ echo -n 'waiting for psmdb-backup/backup-minio-2-1'\''s latest restorable time to advance its latest write (2026-04-14T11:24:38UTC)'
waiting for psmdb-backup/backup-minio-2-1's latest restorable time to advance its latest write (2026-04-14T11:24:38UTC)+ [[ 1776165874 -gt 1776165878 ]]
+++ get_latest_restorable_time backup-minio-2-1
+++ local backup_name=backup-minio-2-1
++++ kubectl_bin get psmdb-backup backup-minio-2-1 -o yaml
++++ yq .status.latestRestorableTime
+++++ mktemp
++++ local LAST_OUT=/tmp/tmp.gVcJuvvhum
+++++ mktemp
++++ local LAST_ERR=/tmp/tmp.CjW1mxsa8F
++++ local exit_status=0
++++ local timeout=4
+++++ seq 0 2
++++ for i in $(seq 0 2)
++++ set +e
++++ kubectl get psmdb-backup backup-minio-2-1 -o yaml
++++ exit_status=0
++++ set -e
++++ '[' 0 '!=' 0 -a -n 1 ']'
++++ break
++++ cat /tmp/tmp.gVcJuvvhum
++++ cat /tmp/tmp.CjW1mxsa8F
++++ rm /tmp/tmp.gVcJuvvhum /tmp/tmp.CjW1mxsa8F
++++ return 0
+++ local time=2026-04-14T11:24:34Z
+++ [[ 2026-04-14T11:24:34Z == null ]]
+++ echo -n 2026-04-14T11:24:34Z
++ datetime_to_timestamp 2026-04-14T11:24:34Z
++ local datetime=2026-04-14T11:24:34Z
++ TZ=UTC
++ /usr/sbin/date -d2026-04-14T11:24:34Z +%s
+ latest_restorable_time=1776165874
+ [[ 0 -gt 5 ]]
+ let retries+=1
+ echo -n .
.+ sleep_with_log 70 'wait for oplog span min + 10 sec'
+ local d=70
+ local 'msg=wait for oplog span min + 10 sec'
+ log 'wait for oplog span min + 10 sec. sleeping for 70 seconds'
+ set +o xtrace
[2026-04-14T11:24:59+0000] wait for oplog span min + 10 sec. sleeping for 70 seconds
+ sleep 70
+ [[ 1776165874 -gt 1776165878 ]]
+++ get_latest_restorable_time backup-minio-2-1
+++ local backup_name=backup-minio-2-1
++++ yq .status.latestRestorableTime
++++ kubectl_bin get psmdb-backup backup-minio-2-1 -o yaml
+++++ mktemp
++++ local LAST_OUT=/tmp/tmp.xNxrlcR04d
+++++ mktemp
++++ local LAST_ERR=/tmp/tmp.8qnuAgdV76
++++ local exit_status=0
++++ local timeout=4
+++++ seq 0 2
++++ for i in $(seq 0 2)
++++ set +e
++++ kubectl get psmdb-backup backup-minio-2-1 -o yaml
++++ exit_status=0
++++ set -e
++++ '[' 0 '!=' 0 -a -n 1 ']'
++++ break
++++ cat /tmp/tmp.xNxrlcR04d
++++ cat /tmp/tmp.8qnuAgdV76
++++ rm /tmp/tmp.xNxrlcR04d /tmp/tmp.8qnuAgdV76
++++ return 0
+++ local time=2026-04-14T11:25:31Z
+++ [[ 2026-04-14T11:25:31Z == null ]]
+++ echo -n 2026-04-14T11:25:31Z
++ datetime_to_timestamp 2026-04-14T11:25:31Z
++ local datetime=2026-04-14T11:25:31Z
++ TZ=UTC
++ /usr/sbin/date -d2026-04-14T11:25:31Z +%s
+ latest_restorable_time=1776165931
+ [[ 1 -gt 5 ]]
+ let retries+=1
+ echo -n .
.+ sleep_with_log 70 'wait for oplog span min + 10 sec'
+ local d=70
+ local 'msg=wait for oplog span min + 10 sec'
+ log 'wait for oplog span min + 10 sec. sleeping for 70 seconds'
+ set +o xtrace
[2026-04-14T11:26:10+0000] wait for oplog span min + 10 sec. sleeping for 70 seconds
+ sleep 70
+ [[ 1776165931 -gt 1776165878 ]]
++ timestamp_to_datetime 1776165931
++ local ts=1776165931
++ TZ=UTC
++ /usr/sbin/date -d@1776165931 +%Y-%m-%dT%H:%M:%S%Z
+ echo '.OK latestRestorableTime: 2026-04-14T11:25:31UTC'
.OK latestRestorableTime: 2026-04-14T11:25:31UTC
++ get_latest_restorable_time backup-minio-2-1
++ local backup_name=backup-minio-2-1
+++ kubectl_bin get psmdb-backup backup-minio-2-1 -o yaml
+++ yq .status.latestRestorableTime
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.dHW8ZlGB4e
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.Imauo5eZGP
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in $(seq 0 2)
+++ set +e
+++ kubectl get psmdb-backup backup-minio-2-1 -o yaml
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.dHW8ZlGB4e
+++ cat /tmp/tmp.Imauo5eZGP
+++ rm /tmp/tmp.dHW8ZlGB4e /tmp/tmp.Imauo5eZGP
+++ return 0
++ local time=2026-04-14T11:26:50Z
++ [[ 2026-04-14T11:26:50Z == null ]]
++ echo -n 2026-04-14T11:26:50Z
+ restore_time=2026-04-14T11:26:50Z
++ format_pitr_target 2026-04-14T11:26:50Z
++ local target=2026-04-14T11:26:50Z
++ echo 2026-04-14T11:26:50Z
++ sed 's/T/ /'
++ sed s/Z//
+ check_recovery backup-minio-2-1 '2026-04-14 11:26:50' -2nd
+ local backup_name=backup-minio-2-1
+ local 'restore_time=2026-04-14 11:26:50'
+ local cmp_postfix=-2nd
+ drop_collection myApp test myApp myPass
+ local db=myApp
+ local collection=test
+ local user=myApp
+ local pass=myPass
+ log 'dropping collection: myApp.test'
+ set +o xtrace
[2026-04-14T11:27:21+0000] dropping collection: myApp.test
+ run_mongo 'use myApp\n db.test.drop()' myApp:myPass@some-name-rs0.multi-storage-27036 mongodb
+ local 'command=use myApp\n db.test.drop()'
+ local uri=myApp:myPass@some-name-rs0.multi-storage-27036
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local replica_set=rs0
+ [[ myApp:myPass@some-name-rs0.multi-storage-27036 == *cfg* ]]
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.6jxQ0RNgim
+++ mktemp
++ local LAST_ERR=/tmp/tmp.RyZ30XF0Un
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.6jxQ0RNgim
++ cat /tmp/tmp.RyZ30XF0Un
++ rm /tmp/tmp.6jxQ0RNgim /tmp/tmp.RyZ30XF0Un
++ return 0
+ local client_container=psmdb-client-bb8b97679-cgccl
+ kubectl_bin exec psmdb-client-bb8b97679-cgccl -- bash -c 'printf '\''use myApp\n db.test.drop()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0.multi-storage-27036.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
++ mktemp
+ local LAST_OUT=/tmp/tmp.eRxm4TBtVe
++ mktemp
+ local LAST_ERR=/tmp/tmp.aytIgA1COl
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-bb8b97679-cgccl -- bash -c 'printf '\''use myApp\n db.test.drop()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0.multi-storage-27036.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.eRxm4TBtVe
Percona Server for MongoDB shell version v4.4.29-28
connecting to: mongodb://some-name-rs0.multi-storage-27036.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb&replicaSet=rs0&ssl=false
Implicit session: session { "id" : UUID("f1a07875-8a51-4457-8b1c-49436bfc6cdb") }
Percona Server for MongoDB server version: v8.0.20-8
WARNING: shell and server versions do not match
switched to db myApp
true
bye
+ cat /tmp/tmp.aytIgA1COl
+ rm /tmp/tmp.eRxm4TBtVe /tmp/tmp.aytIgA1COl
+ return 0
+ log 'creating restore: restore-backup-minio-2-1 restore_to: 2026-04-14 11:26:50'
+ set +o xtrace
[2026-04-14T11:27:23+0000] creating restore: restore-backup-minio-2-1 restore_to: 2026-04-14 11:26:50
+ cat /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2256/e2e-tests/multi-storage/conf/restore.yml
+ /usr/sbin/sed -e 's/name:/name: restore-backup-minio-2-1/'
+ /usr/sbin/sed -e 's/date:/date: 2026-04-14 11:26:50/'
+ kubectl_bin apply -f -
+ /usr/sbin/sed -e 's/backupName:/backupName: backup-minio-2-1/'
++ mktemp
+ local LAST_OUT=/tmp/tmp.8SRgmgUnPP
++ mktemp
+ local LAST_ERR=/tmp/tmp.47YzMwHekP
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl apply -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.8SRgmgUnPP
perconaservermongodbrestore.psmdb.percona.com/restore-backup-minio-2-1 created
+ cat /tmp/tmp.47YzMwHekP
+ rm /tmp/tmp.8SRgmgUnPP /tmp/tmp.47YzMwHekP
+ return 0
+ wait_restore backup-minio-2-1 some-name requested 0 900 1
+ local backup_name=backup-minio-2-1
+ local cluster_name=some-name
+ local target_state=requested
+ local wait_cluster_consistency=0
+ local wait_time=900
+ local ok_if_ready=1
+ set +o xtrace
Waiting for the psmdb-restore/restore-backup-minio-2-1 object to be created.OK
Waiting psmdb-restore/restore-backup-minio-2-1 to reach state "requested" .OK after 0 minutes
+ [[ 0 -eq 1 ]]
+ wait_restore backup-minio-2-1 some-name ready 0 1600
+ local backup_name=backup-minio-2-1
+ local cluster_name=some-name
+ local target_state=ready
+ local wait_cluster_consistency=0
+ local wait_time=1600
+ local ok_if_ready=0
+ set +o xtrace
Waiting for the psmdb-restore/restore-backup-minio-2-1 object to be created.OK
Waiting psmdb-restore/restore-backup-minio-2-1 to reach state "ready" .OK after 0 minutes
+ [[ 0 -eq 1 ]]
+ wait_for_running some-name-rs0 3
+ local name=some-name-rs0
+ let last_pod=2
+ local check_cluster_readyness=true
+ set_debug
+ [[ 1 == 1 ]]
+ set -o xtrace
+ local rs_name=rs0
+ local cluster_name=some-name
++ seq 0 2
+ for i in $(seq 0 $last_pod)
+ [[ 0 -eq 2 ]]
+ wait_pod some-name-rs0-0
+ local pod=some-name-rs0-0
+ set +o xtrace
waiting for pod/some-name-rs0-0 to be ready.OK
+ for i in $(seq 0 $last_pod)
+ [[ 1 -eq 2 ]]
+ wait_pod some-name-rs0-1
+ local pod=some-name-rs0-1
+ set +o xtrace
waiting for pod/some-name-rs0-1 to be ready.OK
+ for i in $(seq 0 $last_pod)
+ [[ 2 -eq 2 ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].arbiter.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.SdZyFE9V1d
+++ mktemp
++ local LAST_ERR=/tmp/tmp.gmvHkwfNuF
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].arbiter.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.SdZyFE9V1d
++ cat /tmp/tmp.gmvHkwfNuF
++ rm /tmp/tmp.SdZyFE9V1d /tmp/tmp.gmvHkwfNuF
++ return 0
+ [[ '' == true ]]
+ wait_pod some-name-rs0-2
+ local pod=some-name-rs0-2
+ set +o xtrace
waiting for pod/some-name-rs0-2 to be ready.OK
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].nonvoting.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.q9i4Vsxekl
+++ mktemp
++ local LAST_ERR=/tmp/tmp.kj2gROXIJC
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].nonvoting.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.q9i4Vsxekl
++ cat /tmp/tmp.kj2gROXIJC
++ rm /tmp/tmp.q9i4Vsxekl /tmp/tmp.kj2gROXIJC
++ return 0
+ [[ '' == true ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].hidden.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.Km8CH4fKLz
+++ mktemp
++ local LAST_ERR=/tmp/tmp.y3AIV7aCQr
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].hidden.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.Km8CH4fKLz
++ cat /tmp/tmp.y3AIV7aCQr
++ rm /tmp/tmp.Km8CH4fKLz /tmp/tmp.y3AIV7aCQr
++ return 0
+ [[ '' == true ]]
+ sleep 10
+ [[ true == true ]]
+ set +x
Waiting for cluster readyness
+ compare_mongo_cmd find myApp:myPass@some-name-rs0.multi-storage-27036 -2nd .svc.cluster.local myApp test 'sort( { x: 1 } )'
+ local command=find
+ local uri=myApp:myPass@some-name-rs0.multi-storage-27036
+ local postfix=-2nd
+ local suffix=.svc.cluster.local
+ local database=myApp
+ local collection=test
+ local 'sort=sort( { x: 1 } )'
+ local tls=false
+ local replicaset=
+ local 'full_command=db.test.find()'
+ [[ -n sort( { x: 1 } ) ]]
+ full_command='db.test.find().sort( { x: 1 } )'
+ log 'running db.test.find().sort( { x: 1 } ) in myApp'
+ set +o xtrace
[2026-04-14T11:28:05+0000] running db.test.find().sort( { x: 1 } ) in myApp
+ [[ false == true ]]
+ mongo_command=run_mongo
+ run_mongo 'use myApp\n db.test.find().sort( { x: 1 } )' myApp:myPass@some-name-rs0.multi-storage-27036 mongodb .svc.cluster.local ''
+ local 'command=use myApp\n db.test.find().sort( { x: 1 } )'
+ local uri=myApp:myPass@some-name-rs0.multi-storage-27036
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local replica_set=rs0
+ [[ myApp:myPass@some-name-rs0.multi-storage-27036 == *cfg* ]]
+ grep -E -v 'I NETWORK|W NETWORK|F NETWORK|"c":"NETWORK"|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|Started a new thread for the timer service'
+ /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.HAEI7qLBaS
+++ mktemp
++ local LAST_ERR=/tmp/tmp.CccNdseONT
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.HAEI7qLBaS
++ cat /tmp/tmp.CccNdseONT
++ rm /tmp/tmp.HAEI7qLBaS /tmp/tmp.CccNdseONT
++ return 0
+ local client_container=psmdb-client-bb8b97679-cgccl
+ kubectl_bin exec psmdb-client-bb8b97679-cgccl -- bash -c 'printf '\''use myApp\n db.test.find().sort( { x: 1 } )\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0.multi-storage-27036.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
++ mktemp
+ local LAST_OUT=/tmp/tmp.yMQWiq7cwI
++ mktemp
+ local LAST_ERR=/tmp/tmp.IFtkYcisLC
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-bb8b97679-cgccl -- bash -c 'printf '\''use myApp\n db.test.find().sort( { x: 1 } )\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0.multi-storage-27036.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.yMQWiq7cwI
+ cat /tmp/tmp.IFtkYcisLC
+ rm /tmp/tmp.yMQWiq7cwI /tmp/tmp.IFtkYcisLC
+ return 0
+ [[ 0 -eq 0 ]]
+ diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2256/e2e-tests/multi-storage/compare/find-2nd.json /tmp/tmp.H2WE2QZul5/find-2nd
+ log 'dropping pbmBackups collection and starting resync'
+ set +o xtrace
[2026-04-14T11:28:08+0000] dropping pbmBackups collection and starting resync
+ drop_metadata_and_resync
++ kubectl get psmdb-backup --no-headers
++ wc -l
+ local psmdb_backup_count=4
+ log 'there are 4 psmdb-backups in multi-storage-27036'
+ set +o xtrace
[2026-04-14T11:28:09+0000] there are 4 psmdb-backups in multi-storage-27036
+ drop_collection admin pbmBackups backup backup123456
+ local db=admin
+ local collection=pbmBackups
+ local user=backup
+ local pass=backup123456
+ log 'dropping collection: admin.pbmBackups'
+ set +o xtrace
[2026-04-14T11:28:09+0000] dropping collection: admin.pbmBackups
+ run_mongo 'use admin\n db.pbmBackups.drop()' backup:backup123456@some-name-rs0.multi-storage-27036 mongodb
+ local 'command=use admin\n db.pbmBackups.drop()'
+ local uri=backup:backup123456@some-name-rs0.multi-storage-27036
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local replica_set=rs0
+ [[ backup:backup123456@some-name-rs0.multi-storage-27036 == *cfg* ]]
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.aLZP5v4X70
+++ mktemp
++ local LAST_ERR=/tmp/tmp.XV3gP5bC2y
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.aLZP5v4X70
++ cat /tmp/tmp.XV3gP5bC2y
++ rm /tmp/tmp.aLZP5v4X70 /tmp/tmp.XV3gP5bC2y
++ return 0
+ local client_container=psmdb-client-bb8b97679-cgccl
+ kubectl_bin exec psmdb-client-bb8b97679-cgccl -- bash -c 'printf '\''use admin\n db.pbmBackups.drop()\n'\'' | mongo mongodb://backup:backup123456@some-name-rs0.multi-storage-27036.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
++ mktemp
+ local LAST_OUT=/tmp/tmp.ipVmNt9S1Z
++ mktemp
+ local LAST_ERR=/tmp/tmp.PHRfYofSLi
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-bb8b97679-cgccl -- bash -c 'printf '\''use admin\n db.pbmBackups.drop()\n'\'' | mongo mongodb://backup:backup123456@some-name-rs0.multi-storage-27036.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.ipVmNt9S1Z
Percona Server for MongoDB shell version v4.4.29-28
connecting to: mongodb://some-name-rs0.multi-storage-27036.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb&replicaSet=rs0&ssl=false
Implicit session: session { "id" : UUID("bad790f8-8ed1-49f1-8074-3b40155398fc") }
Percona Server for MongoDB server version: v8.0.20-8
WARNING: shell and server versions do not match
switched to db admin
true
bye
+ cat /tmp/tmp.PHRfYofSLi
+ rm /tmp/tmp.ipVmNt9S1Z /tmp/tmp.PHRfYofSLi
+ return 0
+ kubectl_bin annotate psmdb some-name percona.com/resync-pbm=true
++ mktemp
+ local LAST_OUT=/tmp/tmp.ebjWrybejT
++ mktemp
+ local LAST_ERR=/tmp/tmp.fp28mMZBlD
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl annotate psmdb some-name percona.com/resync-pbm=true
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.ebjWrybejT
perconaservermongodb.psmdb.percona.com/some-name annotated
+ cat /tmp/tmp.fp28mMZBlD
+ rm /tmp/tmp.ebjWrybejT /tmp/tmp.fp28mMZBlD
+ return 0
+ sleep_with_log 10 'wait resync to start'
+ local d=10
+ local 'msg=wait resync to start'
+ log 'wait resync to start. sleeping for 10 seconds'
+ set +o xtrace
[2026-04-14T11:28:13+0000] wait resync to start. sleeping for 10 seconds
+ sleep 10
+ local retries=0
+ echo -n 'waiting for resync to finish'
waiting for resync to finish++ kubectl_bin get psmdb some-name -o yaml
++ yq '.metadata.annotations."percona.com/resync-in-progress"'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.UvJSmDt2oL
+++ mktemp
++ local LAST_ERR=/tmp/tmp.Q5sqgnclRl
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o yaml
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.UvJSmDt2oL
++ cat /tmp/tmp.Q5sqgnclRl
++ rm /tmp/tmp.UvJSmDt2oL /tmp/tmp.Q5sqgnclRl
++ return 0
+ local resync_running=null
+ [[ null == null ]]
+ echo .OK
.OK
++ kubectl exec some-name-rs0-0 -c backup-agent -- pbm list -o json
++ jq '.snapshots | length'
+ local backup_count=4
+ log 'there are 4 backups in pbmBackups collection after resync'
+ set +o xtrace
[2026-04-14T11:28:26+0000] there are 4 backups in pbmBackups collection after resync
++ echo 4
++ xargs
++ echo 4
++ xargs
+ [[ 4 != 4 ]]
+ log 'resync finished successfully'
+ set +o xtrace
[2026-04-14T11:28:26+0000] resync finished successfully
+ log 'checking backup deletion'
+ set +o xtrace
[2026-04-14T11:28:26+0000] checking backup deletion
+ check_backup_deletion
++ kubectl_bin exec some-name-rs0-0 -c backup-agent -- pbm status -o json
++ jq '.backups.pitrChunks.pitrChunks[0].range.start'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.1lEuMXQa7r
+++ mktemp
++ local LAST_ERR=/tmp/tmp.s3L2mSTwit
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl exec some-name-rs0-0 -c backup-agent -- pbm status -o json
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.1lEuMXQa7r
++ cat /tmp/tmp.s3L2mSTwit
++ rm /tmp/tmp.1lEuMXQa7r /tmp/tmp.s3L2mSTwit
++ return 0
+ local initial_chunk_start=1776165874
+ log 'deleting psmdb-backup/backup-minio-3'
+ set +o xtrace
[2026-04-14T11:28:28+0000] deleting psmdb-backup/backup-minio-3
+ kubectl_bin delete psmdb-backup backup-minio-3
++ mktemp
+ local LAST_OUT=/tmp/tmp.1RrH6DwA7z
++ mktemp
+ local LAST_ERR=/tmp/tmp.KYNWvLtPEI
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl delete psmdb-backup backup-minio-3
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.1RrH6DwA7z
perconaservermongodbbackup.psmdb.percona.com "backup-minio-3" deleted from multi-storage-27036 namespace
+ cat /tmp/tmp.KYNWvLtPEI
+ rm /tmp/tmp.1RrH6DwA7z /tmp/tmp.KYNWvLtPEI
+ return 0
+ log 'checking if chunks are deleted'
+ set +o xtrace
[2026-04-14T11:28:29+0000] checking if chunks are deleted
++ kubectl_bin exec some-name-rs0-0 -c backup-agent -- pbm status -o json
++ jq '.backups.pitrChunks.pitrChunks[0].range.start'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.bFYuqXZNFj
+++ mktemp
++ local LAST_ERR=/tmp/tmp.NuNgNJmbke
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl exec some-name-rs0-0 -c backup-agent -- pbm status -o json
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.bFYuqXZNFj
++ cat /tmp/tmp.NuNgNJmbke
++ rm /tmp/tmp.bFYuqXZNFj /tmp/tmp.NuNgNJmbke
++ return 0
+ local chunk_start=1776165874
+ [[ 1776165874 != 1776165874 ]]
+ log 'chunks are not deleted, OK.'
+ set +o xtrace
[2026-04-14T11:28:31+0000] chunks are not deleted, OK.
+ log 'deleting psmdb-backup/backup-minio-2-1'
+ set +o xtrace
[2026-04-14T11:28:31+0000] deleting psmdb-backup/backup-minio-2-1
+ kubectl_bin delete psmdb-backup backup-minio-2-1
++ mktemp
+ local LAST_OUT=/tmp/tmp.x0l0PeRHzk
++ mktemp
+ local LAST_ERR=/tmp/tmp.fxcYfe349o
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl delete psmdb-backup backup-minio-2-1
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.x0l0PeRHzk
perconaservermongodbbackup.psmdb.percona.com "backup-minio-2-1" deleted from multi-storage-27036 namespace
+ cat /tmp/tmp.fxcYfe349o
+ rm /tmp/tmp.x0l0PeRHzk /tmp/tmp.fxcYfe349o
+ return 0
+ log 'checking if chunks are deleted'
+ set +o xtrace
[2026-04-14T11:28:32+0000] checking if chunks are deleted
++ jq '.backups.pitrChunks.pitrChunks[0].range.start'
++ kubectl_bin exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ mktemp
++ local LAST_OUT=/tmp/tmp.53MTdPA7ux
+++ mktemp
++ local LAST_ERR=/tmp/tmp.xnArLtB3Wr
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl exec some-name-rs0-0 -c backup-agent -- pbm status -o json
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.53MTdPA7ux
++ cat /tmp/tmp.xnArLtB3Wr
++ rm /tmp/tmp.53MTdPA7ux /tmp/tmp.xnArLtB3Wr
++ return 0
+ chunk_start=1776165931
+ [[ 1776165874 == 1776165931 ]]
+ log 'chunks are deleted, OK.'
+ set +o xtrace
[2026-04-14T11:28:34+0000] chunks are deleted, OK.
+ destroy multi-storage-27036
+ local namespace=multi-storage-27036
+ local ignore_logs=true
+ [[ 0 == 1 ]]
+ desc 'destroy cluster/operator and all other resources'
+ set +o xtrace


-----------------------------------------------------------------------------------
destroy cluster/operator and all other resources
-----------------------------------------------------------------------------------

+ '[' true == false ']'
+ delete_backups
+ desc 'Delete psmdb-backup'
+ set +o xtrace


-----------------------------------------------------------------------------------
Delete psmdb-backup
-----------------------------------------------------------------------------------

++ kubectl_bin get psmdb-backup --no-headers
++ wc -l
+++ mktemp
++ local LAST_OUT=/tmp/tmp.6awbHrIpXh
+++ mktemp
++ local LAST_ERR=/tmp/tmp.7Nd3hxW7qL
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb-backup --no-headers
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.6awbHrIpXh
++ cat /tmp/tmp.7Nd3hxW7qL
++ rm /tmp/tmp.6awbHrIpXh /tmp/tmp.7Nd3hxW7qL
++ return 0
+ '[' 2 '!=' 0 ']'
+ kubectl_bin get psmdb-backup
++ mktemp
+ local LAST_OUT=/tmp/tmp.0tTPdOY2gJ
++ mktemp
+ local LAST_ERR=/tmp/tmp.ZMQ3JP7zeR
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl get psmdb-backup
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.0tTPdOY2gJ
NAME             CLUSTER     STORAGE   DESTINATION                                    TYPE      SIZE      STATUS   COMPLETED   AGE
backup-minio-1   some-name   minio-1   s3://operator-testing/2026-04-14T11:13:48Z     logical   53.58KB   ready    14m         14m
backup-minio-2   some-name   minio-2   s3://operator-testing-2/2026-04-14T11:14:02Z   logical   56.27KB   ready    14m         14m
+ cat /tmp/tmp.ZMQ3JP7zeR
+ rm /tmp/tmp.0tTPdOY2gJ /tmp/tmp.ZMQ3JP7zeR
+ return 0
+ kubectl_bin delete psmdb-backup --all
++ mktemp
+ local LAST_OUT=/tmp/tmp.iKhAbSsV4e
++ mktemp
+ local LAST_ERR=/tmp/tmp.N78zCp9dPq
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl delete psmdb-backup --all
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.iKhAbSsV4e
perconaservermongodbbackup.psmdb.percona.com "backup-minio-1" deleted from multi-storage-27036 namespace
perconaservermongodbbackup.psmdb.percona.com "backup-minio-2" deleted from multi-storage-27036 namespace
+ cat /tmp/tmp.N78zCp9dPq
+ rm /tmp/tmp.iKhAbSsV4e /tmp/tmp.N78zCp9dPq
+ return 0
+ delete_crd
+ desc 'get and delete old CRDs and RBAC'
+ set +o xtrace


-----------------------------------------------------------------------------------
get and delete old CRDs and RBAC
-----------------------------------------------------------------------------------

+ kubectl_bin delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2256/deploy/crd.yaml --ignore-not-found --wait=false
++ mktemp
+ local LAST_OUT=/tmp/tmp.A8ZCna1OCH
++ mktemp
+ local LAST_ERR=/tmp/tmp.ylmz1Dz3uA
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2256/deploy/crd.yaml --ignore-not-found --wait=false
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.A8ZCna1OCH
customresourcedefinition.apiextensions.k8s.io "perconaservermongodbbackups.psmdb.percona.com" deleted
customresourcedefinition.apiextensions.k8s.io "perconaservermongodbrestores.psmdb.percona.com" deleted
customresourcedefinition.apiextensions.k8s.io "perconaservermongodbs.psmdb.percona.com" deleted
+ cat /tmp/tmp.ylmz1Dz3uA
+ rm /tmp/tmp.A8ZCna1OCH /tmp/tmp.ylmz1Dz3uA
+ return 0
++ yq eval .metadata.name /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2256/deploy/crd.yaml
++ grep -v '\-\-\-'
grep: warning: stray \ before -
grep: warning: stray \ before -
+ for crd_name in $(yq eval '.metadata.name' "${src_dir}/deploy/crd.yaml" | grep -v '\-\-\-')
+ kubectl get perconaservermongodbbackups.psmdb.percona.com --all-namespaces -o wide
+ grep -v NAMESPACE
+ xargs -L 1 sh -xc 'kubectl patch perconaservermongodbbackups.psmdb.percona.com -n $0 $1 --type=merge -p "{\"metadata\":{\"finalizers\":[]}}"'
error: the server doesn't have a resource type "perconaservermongodbbackups"
+ kubectl patch perconaservermongodbbackups.psmdb.percona.com -n sh --type=merge -p '{"metadata":{"finalizers":[]}}'
error: the server doesn't have a resource type "perconaservermongodbbackups"
+ :
+ kubectl_bin wait --for=delete crd perconaservermongodbbackups.psmdb.percona.com
++ mktemp
+ local LAST_OUT=/tmp/tmp.EJkC5CN8am
++ mktemp
+ local LAST_ERR=/tmp/tmp.98mnPey8ah
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl wait --for=delete crd perconaservermongodbbackups.psmdb.percona.com
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.EJkC5CN8am
+ cat /tmp/tmp.98mnPey8ah
+ rm /tmp/tmp.EJkC5CN8am /tmp/tmp.98mnPey8ah
+ return 0
+ for crd_name in $(yq eval '.metadata.name' "${src_dir}/deploy/crd.yaml" | grep -v '\-\-\-')
+ kubectl get perconaservermongodbrestores.psmdb.percona.com --all-namespaces -o wide
+ grep -v NAMESPACE
+ xargs -L 1 sh -xc 'kubectl patch perconaservermongodbrestores.psmdb.percona.com -n $0 $1 --type=merge -p "{\"metadata\":{\"finalizers\":[]}}"'
error: the server doesn't have a resource type "perconaservermongodbrestores"
+ kubectl patch perconaservermongodbrestores.psmdb.percona.com -n sh --type=merge -p '{"metadata":{"finalizers":[]}}'
error: the server doesn't have a resource type "perconaservermongodbrestores"
+ :
+ kubectl_bin wait --for=delete crd perconaservermongodbrestores.psmdb.percona.com
++ mktemp
+ local LAST_OUT=/tmp/tmp.OhsV3DclQx
++ mktemp
+ local LAST_ERR=/tmp/tmp.USBCiD1AT0
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl wait --for=delete crd perconaservermongodbrestores.psmdb.percona.com
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.OhsV3DclQx
+ cat /tmp/tmp.USBCiD1AT0
+ rm /tmp/tmp.OhsV3DclQx /tmp/tmp.USBCiD1AT0
+ return 0
+ for crd_name in $(yq eval '.metadata.name' "${src_dir}/deploy/crd.yaml" | grep -v '\-\-\-')
+ kubectl get perconaservermongodbs.psmdb.percona.com --all-namespaces -o wide
+ grep -v NAMESPACE
+ xargs -L 1 sh -xc 'kubectl patch perconaservermongodbs.psmdb.percona.com -n $0 $1 --type=merge -p "{\"metadata\":{\"finalizers\":[]}}"'
error: the server doesn't have a resource type "perconaservermongodbs"
+ kubectl patch perconaservermongodbs.psmdb.percona.com -n sh --type=merge -p '{"metadata":{"finalizers":[]}}'
error: the server doesn't have a resource type "perconaservermongodbs"
+ :
+ kubectl_bin wait --for=delete crd perconaservermongodbs.psmdb.percona.com
++ mktemp
+ local LAST_OUT=/tmp/tmp.V1MbnQsEW0
++ mktemp
+ local LAST_ERR=/tmp/tmp.4nZfQaseoi
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl wait --for=delete crd perconaservermongodbs.psmdb.percona.com
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.V1MbnQsEW0
+ cat /tmp/tmp.4nZfQaseoi
+ rm /tmp/tmp.V1MbnQsEW0 /tmp/tmp.4nZfQaseoi
+ return 0
+ local rbac_yaml=rbac.yaml
+ '[' -n psmdb-operator ']'
+ rbac_yaml=cw-rbac.yaml
+ kubectl_bin delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2256/deploy/cw-rbac.yaml --ignore-not-found
++ mktemp
+ local LAST_OUT=/tmp/tmp.wKNLDcmlNO
++ mktemp
+ local LAST_ERR=/tmp/tmp.FkUJkJUNJo
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2256/deploy/cw-rbac.yaml --ignore-not-found
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.wKNLDcmlNO
clusterrole.rbac.authorization.k8s.io "percona-server-mongodb-operator" deleted
clusterrolebinding.rbac.authorization.k8s.io "service-account-percona-server-mongodb-operator" deleted
+ cat /tmp/tmp.FkUJkJUNJo
+ rm /tmp/tmp.wKNLDcmlNO /tmp/tmp.FkUJkJUNJo
+ return 0
+ destroy_cert_manager
+ kubectl_bin delete -f https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml
++ mktemp
+ local LAST_OUT=/tmp/tmp.iqRlctZuHI
++ mktemp
+ local LAST_ERR=/tmp/tmp.l1Xh4xEBh0
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl delete -f https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml
+ exit_status=1
+ set -e
+ '[' 1 '!=' 0 -a -n 1 ']'
+ cat /tmp/tmp.iqRlctZuHI
+ cat /tmp/tmp.l1Xh4xEBh0
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": namespaces "cert-manager" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "challenges.acme.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "orders.acme.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "certificaterequests.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "certificates.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "clusterissuers.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "issuers.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": serviceaccounts "cert-manager-cainjector" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": serviceaccounts "cert-manager" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": serviceaccounts "cert-manager-webhook" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-cainjector" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-issuers" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-clusterissuers" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-certificates" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-orders" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-challenges" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-ingress-shim" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-cluster-view" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-view" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-edit" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-approve:cert-manager-io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-certificatesigningrequests" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-webhook:subjectaccessreviews" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-cainjector" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-issuers" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-clusterissuers" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-certificates" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-orders" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-challenges" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-ingress-shim" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-approve:cert-manager-io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-certificatesigningrequests" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-webhook:subjectaccessreviews" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager-cainjector:leaderelection" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager:leaderelection" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager-tokenrequest" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager-webhook:dynamic-serving" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager-cainjector:leaderelection" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager:leaderelection" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager-tokenrequest" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager-webhook:dynamic-serving" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": services "cert-manager-cainjector" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": services "cert-manager" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": services "cert-manager-webhook" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": deployments.apps "cert-manager-cainjector" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": deployments.apps "cert-manager" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": deployments.apps "cert-manager-webhook" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": mutatingwebhookconfigurations.admissionregistration.k8s.io "cert-manager-webhook" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": validatingwebhookconfigurations.admissionregistration.k8s.io "cert-manager-webhook" not found
+ sleep 0
+ for i in $(seq 0 2)
+ set +e
+ kubectl delete -f https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml
+ exit_status=1
+ set -e
+ '[' 1 '!=' 0 -a -n 1 ']'
+ cat /tmp/tmp.iqRlctZuHI
+ cat /tmp/tmp.l1Xh4xEBh0
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": namespaces "cert-manager" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "challenges.acme.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "orders.acme.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "certificaterequests.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "certificates.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "clusterissuers.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "issuers.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": serviceaccounts "cert-manager-cainjector" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": serviceaccounts "cert-manager" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": serviceaccounts "cert-manager-webhook" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-cainjector" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-issuers" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-clusterissuers" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-certificates" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-orders" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-challenges" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-ingress-shim" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-cluster-view" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-view" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-edit" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-approve:cert-manager-io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-certificatesigningrequests" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-webhook:subjectaccessreviews" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-cainjector" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-issuers" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-clusterissuers" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-certificates" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-orders" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-challenges" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-ingress-shim" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-approve:cert-manager-io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-certificatesigningrequests" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-webhook:subjectaccessreviews" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager-cainjector:leaderelection" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager:leaderelection" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager-tokenrequest" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager-webhook:dynamic-serving" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager-cainjector:leaderelection" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager:leaderelection" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager-tokenrequest" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager-webhook:dynamic-serving" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": services "cert-manager-cainjector" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": services "cert-manager" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": services "cert-manager-webhook" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": deployments.apps "cert-manager-cainjector" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": deployments.apps "cert-manager" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": deployments.apps "cert-manager-webhook" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": mutatingwebhookconfigurations.admissionregistration.k8s.io "cert-manager-webhook" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": validatingwebhookconfigurations.admissionregistration.k8s.io "cert-manager-webhook" not found
+ sleep 4
+ for i in $(seq 0 2)
+ set +e
+ kubectl delete -f https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml
+ exit_status=1
+ set -e
+ '[' 1 '!=' 0 -a -n 1 ']'
+ cat /tmp/tmp.iqRlctZuHI
+ cat /tmp/tmp.l1Xh4xEBh0
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": namespaces "cert-manager" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "challenges.acme.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "orders.acme.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "certificaterequests.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "certificates.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "clusterissuers.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "issuers.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": serviceaccounts "cert-manager-cainjector" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": serviceaccounts "cert-manager" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": serviceaccounts "cert-manager-webhook" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-cainjector" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-issuers" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-clusterissuers" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-certificates" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-orders" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-challenges" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-ingress-shim" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-cluster-view" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-view" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-edit" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-approve:cert-manager-io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-certificatesigningrequests" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-webhook:subjectaccessreviews" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-cainjector" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-issuers" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-clusterissuers" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-certificates" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-orders" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-challenges" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-ingress-shim" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-approve:cert-manager-io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-certificatesigningrequests" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-webhook:subjectaccessreviews" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager-cainjector:leaderelection" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager:leaderelection" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager-tokenrequest" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager-webhook:dynamic-serving" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager-cainjector:leaderelection" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager:leaderelection" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager-tokenrequest" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager-webhook:dynamic-serving" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": services "cert-manager-cainjector" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": services "cert-manager" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": services "cert-manager-webhook" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": deployments.apps "cert-manager-cainjector" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": deployments.apps "cert-manager" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": deployments.apps "cert-manager-webhook" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": mutatingwebhookconfigurations.admissionregistration.k8s.io "cert-manager-webhook" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": validatingwebhookconfigurations.admissionregistration.k8s.io "cert-manager-webhook" not found
+ sleep 8
+ cat /tmp/tmp.iqRlctZuHI
+ cat /tmp/tmp.l1Xh4xEBh0
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": namespaces "cert-manager" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "challenges.acme.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "orders.acme.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "certificaterequests.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "certificates.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "clusterissuers.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "issuers.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": serviceaccounts "cert-manager-cainjector" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": serviceaccounts "cert-manager" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": serviceaccounts "cert-manager-webhook" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-cainjector" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-issuers" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-clusterissuers" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-certificates" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-orders" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-challenges" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-ingress-shim" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-cluster-view" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-view" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-edit" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-approve:cert-manager-io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-certificatesigningrequests" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-webhook:subjectaccessreviews" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-cainjector" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-issuers" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-clusterissuers" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-certificates" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-orders" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-challenges" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-ingress-shim" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-approve:cert-manager-io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-certificatesigningrequests" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-webhook:subjectaccessreviews" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager-cainjector:leaderelection" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager:leaderelection" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager-tokenrequest" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager-webhook:dynamic-serving" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager-cainjector:leaderelection" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager:leaderelection" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager-tokenrequest" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager-webhook:dynamic-serving" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": services "cert-manager-cainjector" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": services "cert-manager" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": services "cert-manager-webhook" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": deployments.apps "cert-manager-cainjector" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": deployments.apps "cert-manager" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": deployments.apps "cert-manager-webhook" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": mutatingwebhookconfigurations.admissionregistration.k8s.io "cert-manager-webhook" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": validatingwebhookconfigurations.admissionregistration.k8s.io "cert-manager-webhook" not found
+ rm /tmp/tmp.iqRlctZuHI /tmp/tmp.l1Xh4xEBh0
+ return 1
+ true
+ '[' -n '' ']'
+ '[' -n psmdb-operator ']'
+ kubectl_bin delete --grace-period=0 --force=true namespace multi-storage-27036
+ rm -rf /tmp/tmp.H2WE2QZul5
+ kubectl_bin delete --grace-period=0 --force=true namespace psmdb-operator
++ mktemp
++ mktemp
+ local LAST_OUT=/tmp/tmp.zKUFL34Gph
++ mktemp
+ desc 'test passed'
+ set +o xtrace


-----------------------------------------------------------------------------------
test passed
-----------------------------------------------------------------------------------

+ local LAST_ERR=/tmp/tmp.Ktbe5qWyIv
+ local exit_status=0
+ local timeout=4
+ local LAST_OUT=/tmp/tmp.c1TQOIsxUY
++ seq 0 2
++ mktemp
+ local LAST_ERR=/tmp/tmp.Y4vIhbD6m9
+ local exit_status=0
+ local timeout=4
+ for i in $(seq 0 2)
+ set +e
+ kubectl delete --grace-period=0 --force=true namespace multi-storage-27036
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl delete --grace-period=0 --force=true namespace psmdb-operator