Log: /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2009/e2e-tests/logs/multi-storage.log
grep: warning: stray \ before -
WARNING: version difference between client (1.33) and server (1.30) exceeds the supported minor version skew of +/-1
WARNING: version difference between client (1.33) and server (1.30) exceeds the supported minor version skew of +/-1
WARNING: version difference between client (1.33) and server (1.30) exceeds the supported minor version skew of +/-1
+ cluster=some-name
+ create_infra multi-storage-17828
+ local ns=multi-storage-17828
+ [[ 1 == 1 ]]
+ delete_crd
+ desc 'get and delete old CRDs and RBAC'
+ set +o xtrace


-----------------------------------------------------------------------------------
get and delete old CRDs and RBAC
-----------------------------------------------------------------------------------

+ kubectl_bin delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2009/deploy/crd.yaml --ignore-not-found --wait=false
++ mktemp
+ local LAST_OUT=/tmp/tmp.FQRpQf6019
++ mktemp
+ local LAST_ERR=/tmp/tmp.gZjFZSQqEw
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2009/deploy/crd.yaml --ignore-not-found --wait=false
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.FQRpQf6019
customresourcedefinition.apiextensions.k8s.io "perconaservermongodbbackups.psmdb.percona.com" deleted
customresourcedefinition.apiextensions.k8s.io "perconaservermongodbrestores.psmdb.percona.com" deleted
customresourcedefinition.apiextensions.k8s.io "perconaservermongodbs.psmdb.percona.com" deleted
+ cat /tmp/tmp.gZjFZSQqEw
+ rm /tmp/tmp.FQRpQf6019 /tmp/tmp.gZjFZSQqEw
+ return 0
++ yq eval .metadata.name /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2009/deploy/crd.yaml
++ grep -v '\-\-\-'
grep: warning: stray \ before -
grep: warning: stray \ before -
+ for crd_name in $(yq eval '.metadata.name' "${src_dir}/deploy/crd.yaml" | grep -v '\-\-\-')
+ kubectl get perconaservermongodbbackups.psmdb.percona.com --all-namespaces -o wide
+ grep -v NAMESPACE
+ xargs -L 1 sh -xc 'kubectl patch perconaservermongodbbackups.psmdb.percona.com -n $0 $1 --type=merge -p "{\"metadata\":{\"finalizers\":[]}}"'
No resources found
+ kubectl patch perconaservermongodbbackups.psmdb.percona.com -n sh --type=merge -p '{"metadata":{"finalizers":[]}}'
error: resource(s) were provided, but no name was specified
+ :
+ kubectl_bin wait --for=delete crd perconaservermongodbbackups.psmdb.percona.com
++ mktemp
+ local LAST_OUT=/tmp/tmp.5QOYOOq8El
++ mktemp
+ local LAST_ERR=/tmp/tmp.quc96DmJwG
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl wait --for=delete crd perconaservermongodbbackups.psmdb.percona.com
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.5QOYOOq8El
+ cat /tmp/tmp.quc96DmJwG
+ rm /tmp/tmp.5QOYOOq8El /tmp/tmp.quc96DmJwG
+ return 0
+ for crd_name in $(yq eval '.metadata.name' "${src_dir}/deploy/crd.yaml" | grep -v '\-\-\-')
+ kubectl get perconaservermongodbrestores.psmdb.percona.com --all-namespaces -o wide
+ grep -v NAMESPACE
+ xargs -L 1 sh -xc 'kubectl patch perconaservermongodbrestores.psmdb.percona.com -n $0 $1 --type=merge -p "{\"metadata\":{\"finalizers\":[]}}"'
error: the server doesn't have a resource type "perconaservermongodbrestores"
+ kubectl patch perconaservermongodbrestores.psmdb.percona.com -n sh --type=merge -p '{"metadata":{"finalizers":[]}}'
error: the server doesn't have a resource type "perconaservermongodbrestores"
+ :
+ kubectl_bin wait --for=delete crd perconaservermongodbrestores.psmdb.percona.com
++ mktemp
+ local LAST_OUT=/tmp/tmp.1O9AvpBqbt
++ mktemp
+ local LAST_ERR=/tmp/tmp.Dbqv9gYwqa
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl wait --for=delete crd perconaservermongodbrestores.psmdb.percona.com
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.1O9AvpBqbt
+ cat /tmp/tmp.Dbqv9gYwqa
+ rm /tmp/tmp.1O9AvpBqbt /tmp/tmp.Dbqv9gYwqa
+ return 0
+ for crd_name in $(yq eval '.metadata.name' "${src_dir}/deploy/crd.yaml" | grep -v '\-\-\-')
+ kubectl get perconaservermongodbs.psmdb.percona.com --all-namespaces -o wide
+ grep -v NAMESPACE
+ xargs -L 1 sh -xc 'kubectl patch perconaservermongodbs.psmdb.percona.com -n $0 $1 --type=merge -p "{\"metadata\":{\"finalizers\":[]}}"'
error: the server doesn't have a resource type "perconaservermongodbs"
+ kubectl patch perconaservermongodbs.psmdb.percona.com -n sh --type=merge -p '{"metadata":{"finalizers":[]}}'
error: the server doesn't have a resource type "perconaservermongodbs"
+ :
+ kubectl_bin wait --for=delete crd perconaservermongodbs.psmdb.percona.com
++ mktemp
+ local LAST_OUT=/tmp/tmp.Ab5UJJsweb
++ mktemp
+ local LAST_ERR=/tmp/tmp.MjxG3Fism0
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl wait --for=delete crd perconaservermongodbs.psmdb.percona.com
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.Ab5UJJsweb
+ cat /tmp/tmp.MjxG3Fism0
+ rm /tmp/tmp.Ab5UJJsweb /tmp/tmp.MjxG3Fism0
+ return 0
+ local rbac_yaml=rbac.yaml
+ '[' -n psmdb-operator ']'
+ rbac_yaml=cw-rbac.yaml
+ kubectl_bin delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2009/deploy/cw-rbac.yaml --ignore-not-found
++ mktemp
+ local LAST_OUT=/tmp/tmp.zF0cPJ06QB
++ mktemp
+ local LAST_ERR=/tmp/tmp.buB7VXIPm9
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2009/deploy/cw-rbac.yaml --ignore-not-found
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.zF0cPJ06QB
clusterrole.rbac.authorization.k8s.io "percona-server-mongodb-operator" deleted
clusterrolebinding.rbac.authorization.k8s.io "service-account-percona-server-mongodb-operator" deleted
+ cat /tmp/tmp.buB7VXIPm9
+ rm /tmp/tmp.zF0cPJ06QB /tmp/tmp.buB7VXIPm9
+ return 0
+ check_crd_for_deletion PR-2009-645e0543
+ local git_tag=PR-2009-645e0543
++ curl -s https://raw.githubusercontent.com/percona/percona-server-mongodb-operator/PR-2009-645e0543/deploy/crd.yaml
++ yq eval .metadata.name
++ /usr/sbin/sed s/---//g
++ /usr/sbin/sed ':a;N;$!ba;s/\n/ /g'
+ for crd_name in $(curl -s https://raw.githubusercontent.com/percona/percona-server-mongodb-operator/${git_tag}/deploy/crd.yaml | yq eval '.metadata.name' | $sed 's/---//g' | $sed ':a;N;$!ba;s/\n/ /g')
++ kubectl_bin get crd/null -o 'jsonpath={.status.conditions[-1].type}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.CwGTpv5DAt
+++ mktemp
++ local LAST_ERR=/tmp/tmp.7kmQX8XRe2
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get crd/null -o 'jsonpath={.status.conditions[-1].type}'
++ exit_status=1
++ set -e
++ '[' 1 '!=' 0 -a -n 1 ']'
++ cat /tmp/tmp.CwGTpv5DAt
++ cat /tmp/tmp.7kmQX8XRe2
Error from server (NotFound): customresourcedefinitions.apiextensions.k8s.io "null" not found
++ sleep 0
++ for i in $(seq 0 2)
++ set +e
++ kubectl get crd/null -o 'jsonpath={.status.conditions[-1].type}'
++ exit_status=1
++ set -e
++ '[' 1 '!=' 0 -a -n 1 ']'
++ cat /tmp/tmp.CwGTpv5DAt
++ cat /tmp/tmp.7kmQX8XRe2
Error from server (NotFound): customresourcedefinitions.apiextensions.k8s.io "null" not found
++ sleep 4
++ for i in $(seq 0 2)
++ set +e
++ kubectl get crd/null -o 'jsonpath={.status.conditions[-1].type}'
++ exit_status=1
++ set -e
++ '[' 1 '!=' 0 -a -n 1 ']'
++ cat /tmp/tmp.CwGTpv5DAt
++ cat /tmp/tmp.7kmQX8XRe2
Error from server (NotFound): customresourcedefinitions.apiextensions.k8s.io "null" not found
++ sleep 8
++ cat /tmp/tmp.CwGTpv5DAt
++ cat /tmp/tmp.7kmQX8XRe2
Error from server (NotFound): customresourcedefinitions.apiextensions.k8s.io "null" not found
++ rm /tmp/tmp.CwGTpv5DAt /tmp/tmp.7kmQX8XRe2
++ return 1
+ [[ '' == \T\e\r\m\i\n\a\t\i\n\g ]]
+ '[' -n psmdb-operator ']'
+ create_namespace psmdb-operator
+ local namespace=psmdb-operator
+ local skip_clean_namespace=
+ [[ 1 == 1 ]]
+ [[ -z '' ]]
+ destroy_chaos_mesh
++ helm list --all-namespaces --filter chaos-mesh
++ tail -n1
++ awk '-F ' '{print $2}'
++ sed s/NAMESPACE//
+ local chaos_mesh_ns=
+ desc 'destroy chaos-mesh'
+ set +o xtrace


-----------------------------------------------------------------------------------
destroy chaos-mesh
-----------------------------------------------------------------------------------

+ '[' -n '' ']'
++ kubectl get MutatingWebhookConfiguration
++ grep chaos-mesh
++ awk '{print $1}'
+ timeout 30 kubectl delete MutatingWebhookConfiguration
error: resource(s) were provided, but no name was specified
+ :
++ kubectl get ValidatingWebhookConfiguration
++ grep chaos-mesh
++ awk '{print $1}'
+ timeout 30 kubectl delete ValidatingWebhookConfiguration
error: resource(s) were provided, but no name was specified
+ :
++ kubectl get ValidatingWebhookConfiguration
++ grep validate-auth
++ awk '{print $1}'
+ timeout 30 kubectl delete ValidatingWebhookConfiguration
error: resource(s) were provided, but no name was specified
+ :
++ kubectl api-resources
++ grep chaos-mesh
++ awk '{print $1}'
++ kubectl get crd
++ grep chaos-mesh.org
++ awk '{print $1}'
+ timeout 30 kubectl delete crd
error: resource(s) were provided, but no name was specified
+ :
++ kubectl get clusterrolebinding
++ grep chaos-mesh
++ awk '{print $1}'
+ timeout 30 kubectl delete clusterrolebinding
error: resource(s) were provided, but no name was specified
+ :
++ kubectl get clusterrole
++ grep chaos-mesh
++ awk '{print $1}'
+ timeout 30 kubectl delete clusterrole
error: resource(s) were provided, but no name was specified
+ :
+ desc 'cleaned up all old namespaces'
+ set +o xtrace


-----------------------------------------------------------------------------------
cleaned up all old namespaces
-----------------------------------------------------------------------------------

+ kubectl_bin get ns
+ egrep -v '^kube-|^default|Terminating|psmdb-operator|openshift|^gke-|^gmp-|^NAME'
+ awk '{print$1}'
++ mktemp
+ '[' -n '' ']'
+ xargs kubectl delete ns
+ desc 'cleaned up old namespaces psmdb-operator'
+ set +o xtrace


-----------------------------------------------------------------------------------
cleaned up old namespaces psmdb-operator
-----------------------------------------------------------------------------------

+ kubectl_bin delete namespace psmdb-operator --ignore-not-found
++ mktemp
+ local LAST_OUT=/tmp/tmp.pc7KU0nzAd
egrep: warning: egrep is obsolescent; using grep -E
++ mktemp
+ local LAST_OUT=/tmp/tmp.Ojf9xbNEZb
++ mktemp
+ local LAST_ERR=/tmp/tmp.vYpBLHg75P
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ local LAST_ERR=/tmp/tmp.lEqkseX8bV
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl get ns
+ for i in $(seq 0 2)
+ set +e
+ kubectl delete namespace psmdb-operator --ignore-not-found
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.pc7KU0nzAd
+ cat /tmp/tmp.vYpBLHg75P
+ rm /tmp/tmp.pc7KU0nzAd /tmp/tmp.vYpBLHg75P
+ return 0
namespace "multi-storage-14729" deleted
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.Ojf9xbNEZb
namespace "psmdb-operator" deleted
+ cat /tmp/tmp.lEqkseX8bV
+ rm /tmp/tmp.Ojf9xbNEZb /tmp/tmp.lEqkseX8bV
+ return 0
+ kubectl_bin wait --for=delete namespace psmdb-operator
++ mktemp
+ local LAST_OUT=/tmp/tmp.Kp1YoGT3ob
++ mktemp
+ local LAST_ERR=/tmp/tmp.voV8KvlHDr
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl wait --for=delete namespace psmdb-operator
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.Kp1YoGT3ob
+ cat /tmp/tmp.voV8KvlHDr
+ rm /tmp/tmp.Kp1YoGT3ob /tmp/tmp.voV8KvlHDr
+ return 0
+ desc 'create namespace psmdb-operator'
+ set +o xtrace


-----------------------------------------------------------------------------------
create namespace psmdb-operator
-----------------------------------------------------------------------------------

+ kubectl_bin create namespace psmdb-operator
++ mktemp
+ local LAST_OUT=/tmp/tmp.aWtdDvhRFf
++ mktemp
+ local LAST_ERR=/tmp/tmp.0qmRRBiCKO
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl create namespace psmdb-operator
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.aWtdDvhRFf
namespace/psmdb-operator created
+ cat /tmp/tmp.0qmRRBiCKO
+ rm /tmp/tmp.aWtdDvhRFf /tmp/tmp.0qmRRBiCKO
+ return 0
+ set_kube_ctx psmdb-operator
+ local namespace=psmdb-operator
++ kubectl_bin config current-context
+++ mktemp
++ local LAST_OUT=/tmp/tmp.gIwyTPBtCx
+++ mktemp
++ local LAST_ERR=/tmp/tmp.42qnKKBFIQ
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl config current-context
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.gIwyTPBtCx
++ cat /tmp/tmp.42qnKKBFIQ
++ rm /tmp/tmp.gIwyTPBtCx /tmp/tmp.42qnKKBFIQ
++ return 0
+ kubectl_bin config set-context gke_cloud-dev-112233_us-central1-a_jen-psmdb-2009-645e0543-2-cluster10 --namespace=psmdb-operator
++ mktemp
+ local LAST_OUT=/tmp/tmp.I5kCe7BdLG
++ mktemp
+ local LAST_ERR=/tmp/tmp.Rq2DQWIiK0
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl config set-context gke_cloud-dev-112233_us-central1-a_jen-psmdb-2009-645e0543-2-cluster10 --namespace=psmdb-operator
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.I5kCe7BdLG
Context "gke_cloud-dev-112233_us-central1-a_jen-psmdb-2009-645e0543-2-cluster10" modified.
+ cat /tmp/tmp.Rq2DQWIiK0
+ rm /tmp/tmp.I5kCe7BdLG /tmp/tmp.Rq2DQWIiK0
+ return 0
+ deploy_operator
+ desc 'start PSMDB operator'
+ set +o xtrace


-----------------------------------------------------------------------------------
start PSMDB operator
-----------------------------------------------------------------------------------

+ local cr_file
+ '[' -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2009/e2e-tests/multi-storage/conf/crd.yaml ']'
+ cr_file=/mnt/jenkins/workspace/cloud-psmdb-operator_PR-2009/deploy/crd.yaml
+ kubectl_bin apply --server-side --force-conflicts -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2009/deploy/crd.yaml
++ mktemp
+ local LAST_OUT=/tmp/tmp.bKo6jNpbnd
++ mktemp
+ local LAST_ERR=/tmp/tmp.fTrPjQ35lP
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl apply --server-side --force-conflicts -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2009/deploy/crd.yaml
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.bKo6jNpbnd
customresourcedefinition.apiextensions.k8s.io/perconaservermongodbbackups.psmdb.percona.com serverside-applied
customresourcedefinition.apiextensions.k8s.io/perconaservermongodbrestores.psmdb.percona.com serverside-applied
customresourcedefinition.apiextensions.k8s.io/perconaservermongodbs.psmdb.percona.com serverside-applied
+ cat /tmp/tmp.fTrPjQ35lP
+ rm /tmp/tmp.bKo6jNpbnd /tmp/tmp.fTrPjQ35lP
+ return 0
+ '[' -n psmdb-operator ']'
+ apply_rbac cw-rbac
+ local operator_namespace=psmdb-operator
+ local rbac=cw-rbac
+ cat /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2009/deploy/cw-rbac.yaml
+ sed -e 's^namespace: .*^namespace: psmdb-operator^'
+ kubectl_bin apply -n psmdb-operator -f -
++ mktemp
+ local LAST_OUT=/tmp/tmp.M4iCW2fFOM
++ mktemp
+ local LAST_ERR=/tmp/tmp.lbCHSRbhHI
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl apply -n psmdb-operator -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.M4iCW2fFOM
clusterrole.rbac.authorization.k8s.io/percona-server-mongodb-operator created
serviceaccount/percona-server-mongodb-operator created
clusterrolebinding.rbac.authorization.k8s.io/service-account-percona-server-mongodb-operator created
+ cat /tmp/tmp.lbCHSRbhHI
+ rm /tmp/tmp.M4iCW2fFOM /tmp/tmp.lbCHSRbhHI
+ return 0
+ yq eval '
			(.spec.template.spec.containers[].image = "perconalab/percona-server-mongodb-operator:PR-2009-645e0543") |
			((.. | select(.[] == "DISABLE_TELEMETRY")) |= .value="true") |
			((.. | select(.[] == "LOG_LEVEL")) |= .value="DEBUG")' /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2009/deploy/cw-operator.yaml
+ kubectl_bin apply -f -
++ mktemp
+ local LAST_OUT=/tmp/tmp.cTSXvKpIUb
++ mktemp
+ local LAST_ERR=/tmp/tmp.UuHPELi4iJ
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl apply -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.cTSXvKpIUb
deployment.apps/percona-server-mongodb-operator created
+ cat /tmp/tmp.UuHPELi4iJ
+ rm /tmp/tmp.cTSXvKpIUb /tmp/tmp.UuHPELi4iJ
+ return 0
+ sleep 2
++ get_operator_pod
++ kubectl_bin get pods --selector=name=percona-server-mongodb-operator -o 'jsonpath={.items[].metadata.name}' -n psmdb-operator
+++ mktemp
++ local LAST_OUT=/tmp/tmp.zUT3l6HqWs
+++ mktemp
++ local LAST_ERR=/tmp/tmp.k4XMjOP3iB
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=percona-server-mongodb-operator -o 'jsonpath={.items[].metadata.name}' -n psmdb-operator
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.zUT3l6HqWs
++ cat /tmp/tmp.k4XMjOP3iB
++ rm /tmp/tmp.zUT3l6HqWs /tmp/tmp.k4XMjOP3iB
++ return 0
+ wait_pod percona-server-mongodb-operator-677cf67bbc-49dmx
+ local pod=percona-server-mongodb-operator-677cf67bbc-49dmx
+ set +o xtrace
waiting for pod/percona-server-mongodb-operator-677cf67bbc-49dmx to be ready.OK
+ echo 'Print operator info from log'
Print operator info from log
+ grep 'Manager starting up'
++ get_operator_pod
++ kubectl_bin get pods --selector=name=percona-server-mongodb-operator -o 'jsonpath={.items[].metadata.name}' -n psmdb-operator
+++ mktemp
++ local LAST_OUT=/tmp/tmp.8vlZ9WdSev
+++ mktemp
++ local LAST_ERR=/tmp/tmp.4yUDQqTy6J
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=percona-server-mongodb-operator -o 'jsonpath={.items[].metadata.name}' -n psmdb-operator
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.8vlZ9WdSev
++ cat /tmp/tmp.4yUDQqTy6J
++ rm /tmp/tmp.8vlZ9WdSev /tmp/tmp.4yUDQqTy6J
++ return 0
+ kubectl_bin logs percona-server-mongodb-operator-677cf67bbc-49dmx
++ mktemp
+ local LAST_OUT=/tmp/tmp.32uQ88b1GD
++ mktemp
+ local LAST_ERR=/tmp/tmp.tDO6hpHMIZ
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl logs percona-server-mongodb-operator-677cf67bbc-49dmx
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.32uQ88b1GD
+ cat /tmp/tmp.tDO6hpHMIZ
+ rm /tmp/tmp.32uQ88b1GD /tmp/tmp.tDO6hpHMIZ
+ return 0
2025-08-07T09:01:25.265Z	INFO	setup	Manager starting up	{"gitCommit": "645e054370b1c94a71b245f25e4838f59389aa71", "gitBranch": "PR-2009-645e0543", "buildTime": "", "goVersion": "go1.24.6", "os": "linux", "arch": "amd64"}
+ create_namespace multi-storage-17828
+ local namespace=multi-storage-17828
+ local skip_clean_namespace=
+ [[ 1 == 1 ]]
+ [[ -z '' ]]
+ destroy_chaos_mesh
++ helm list --all-namespaces --filter chaos-mesh
++ tail -n1
++ awk '-F ' '{print $2}'
++ sed s/NAMESPACE//
+ local chaos_mesh_ns=
+ desc 'destroy chaos-mesh'
+ set +o xtrace


-----------------------------------------------------------------------------------
destroy chaos-mesh
-----------------------------------------------------------------------------------

+ '[' -n '' ']'
++ kubectl get MutatingWebhookConfiguration
++ grep chaos-mesh
++ awk '{print $1}'
+ timeout 30 kubectl delete MutatingWebhookConfiguration
error: resource(s) were provided, but no name was specified
+ :
++ kubectl get ValidatingWebhookConfiguration
++ grep chaos-mesh
++ awk '{print $1}'
+ timeout 30 kubectl delete ValidatingWebhookConfiguration
error: resource(s) were provided, but no name was specified
+ :
++ kubectl get ValidatingWebhookConfiguration
++ grep validate-auth
++ awk '{print $1}'
+ timeout 30 kubectl delete ValidatingWebhookConfiguration
error: resource(s) were provided, but no name was specified
+ :
++ kubectl api-resources
++ grep chaos-mesh
++ awk '{print $1}'
++ kubectl get crd
++ grep chaos-mesh.org
++ awk '{print $1}'
+ timeout 30 kubectl delete crd
error: resource(s) were provided, but no name was specified
+ :
++ kubectl get clusterrolebinding
++ grep chaos-mesh
++ awk '{print $1}'
+ timeout 30 kubectl delete clusterrolebinding
error: resource(s) were provided, but no name was specified
+ :
++ kubectl get clusterrole
++ grep chaos-mesh
++ awk '{print $1}'
+ timeout 30 kubectl delete clusterrole
error: resource(s) were provided, but no name was specified
+ :
+ desc 'cleaned up all old namespaces'
+ set +o xtrace


-----------------------------------------------------------------------------------
cleaned up all old namespaces
-----------------------------------------------------------------------------------

+ kubectl_bin get ns
+ egrep -v '^kube-|^default|Terminating|psmdb-operator|openshift|^gke-|^gmp-|^NAME'
+ awk '{print$1}'
++ mktemp
+ '[' -n '' ']'
+ desc 'cleaned up old namespaces multi-storage-17828'
+ set +o xtrace


-----------------------------------------------------------------------------------
cleaned up old namespaces multi-storage-17828
-----------------------------------------------------------------------------------

+ kubectl_bin delete namespace multi-storage-17828 --ignore-not-found
+ xargs kubectl delete ns
egrep: warning: egrep is obsolescent; using grep -E
++ mktemp
+ local LAST_OUT=/tmp/tmp.C0NjO3Fypi
+ local LAST_OUT=/tmp/tmp.DNLh0GJ6rV
++ mktemp
++ mktemp
+ local LAST_ERR=/tmp/tmp.e9zTDQ7pqh
+ local exit_status=0
+ local timeout=4
+ local LAST_ERR=/tmp/tmp.J49gPwkeOj
+ local exit_status=0
+ local timeout=4
++ seq 0 2
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl delete namespace multi-storage-17828 --ignore-not-found
+ for i in $(seq 0 2)
+ set +e
+ kubectl get ns
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.C0NjO3Fypi
+ cat /tmp/tmp.J49gPwkeOj
+ rm /tmp/tmp.C0NjO3Fypi /tmp/tmp.J49gPwkeOj
+ return 0
error: resource(s) were provided, but no name was specified
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.DNLh0GJ6rV
+ cat /tmp/tmp.e9zTDQ7pqh
+ rm /tmp/tmp.DNLh0GJ6rV /tmp/tmp.e9zTDQ7pqh
+ return 0
+ kubectl_bin wait --for=delete namespace multi-storage-17828
++ mktemp
+ local LAST_OUT=/tmp/tmp.54xDrhDYfA
++ mktemp
+ local LAST_ERR=/tmp/tmp.JnTTaQ2bAL
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl wait --for=delete namespace multi-storage-17828
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.54xDrhDYfA
+ cat /tmp/tmp.JnTTaQ2bAL
+ rm /tmp/tmp.54xDrhDYfA /tmp/tmp.JnTTaQ2bAL
+ return 0
+ desc 'create namespace multi-storage-17828'
+ set +o xtrace


-----------------------------------------------------------------------------------
create namespace multi-storage-17828
-----------------------------------------------------------------------------------

+ kubectl_bin create namespace multi-storage-17828
++ mktemp
+ local LAST_OUT=/tmp/tmp.XGEsnjqAlD
++ mktemp
+ local LAST_ERR=/tmp/tmp.dEzaYNb1GV
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl create namespace multi-storage-17828
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.XGEsnjqAlD
namespace/multi-storage-17828 created
+ cat /tmp/tmp.dEzaYNb1GV
+ rm /tmp/tmp.XGEsnjqAlD /tmp/tmp.dEzaYNb1GV
+ return 0
+ set_kube_ctx multi-storage-17828
+ local namespace=multi-storage-17828
++ kubectl_bin config current-context
+++ mktemp
++ local LAST_OUT=/tmp/tmp.7C26KVlytO
+++ mktemp
++ local LAST_ERR=/tmp/tmp.mzJinl9xed
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl config current-context
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.7C26KVlytO
++ cat /tmp/tmp.mzJinl9xed
++ rm /tmp/tmp.7C26KVlytO /tmp/tmp.mzJinl9xed
++ return 0
+ kubectl_bin config set-context gke_cloud-dev-112233_us-central1-a_jen-psmdb-2009-645e0543-2-cluster10 --namespace=multi-storage-17828
++ mktemp
+ local LAST_OUT=/tmp/tmp.rSNWKy38Rv
++ mktemp
+ local LAST_ERR=/tmp/tmp.LI9AUJyIYU
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl config set-context gke_cloud-dev-112233_us-central1-a_jen-psmdb-2009-645e0543-2-cluster10 --namespace=multi-storage-17828
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.rSNWKy38Rv
Context "gke_cloud-dev-112233_us-central1-a_jen-psmdb-2009-645e0543-2-cluster10" modified.
+ cat /tmp/tmp.LI9AUJyIYU
+ rm /tmp/tmp.rSNWKy38Rv /tmp/tmp.LI9AUJyIYU
+ return 0
+ deploy_minio
+ desc 'install Minio'
+ set +o xtrace


-----------------------------------------------------------------------------------
install Minio
-----------------------------------------------------------------------------------

+ helm uninstall minio-service
Error: uninstall: Release not loaded: minio-service: release: not found
+ :
+ helm repo remove minio
"minio" has been removed from your repositories
+ helm repo add minio https://charts.min.io/
"minio" has been added to your repositories
+ retry 10 60 helm install minio-service --version 5.4.0 --set replicas=1 --set mode=standalone --set resources.requests.memory=256Mi --set rootUser=rootuser --set rootPassword=rootpass123 --set 'users[0].accessKey=some-access-key' --set 'users[0].secretKey=some-secret-key' --set 'users[0].policy=consoleAdmin' --set service.type=ClusterIP --set configPathmc=/tmp/.minio/ --set persistence.size=2G --set securityContext.enabled=false minio/minio
+ local max=10
+ local delay=60
+ shift 2
+ local n=1
+ helm install minio-service --version 5.4.0 --set replicas=1 --set mode=standalone --set resources.requests.memory=256Mi --set rootUser=rootuser --set rootPassword=rootpass123 --set 'users[0].accessKey=some-access-key' --set 'users[0].secretKey=some-secret-key' --set 'users[0].policy=consoleAdmin' --set service.type=ClusterIP --set configPathmc=/tmp/.minio/ --set persistence.size=2G --set securityContext.enabled=false minio/minio
NAME: minio-service
LAST DEPLOYED: Thu Aug  7 09:01:41 2025
NAMESPACE: multi-storage-17828
STATUS: deployed
REVISION: 1
TEST SUITE: None
NOTES:
MinIO can be accessed via port 9000 on the following DNS name from within your cluster:
minio-service.multi-storage-17828.cluster.local

To access MinIO from localhost, run the below commands:

  1. export POD_NAME=$(kubectl get pods --namespace multi-storage-17828 -l "release=minio-service" -o jsonpath="{.items[0].metadata.name}")

  2. kubectl port-forward $POD_NAME 9000 --namespace multi-storage-17828

Read more about port forwarding here: http://kubernetes.io/docs/user-guide/kubectl/kubectl_port-forward/

You can now access MinIO server on http://localhost:9000. Follow the below steps to connect to MinIO server with mc client:

  1. Download the MinIO mc client - https://min.io/docs/minio/linux/reference/minio-mc.html#quickstart

  2. export MC_HOST_minio-service-local=http://$(kubectl get secret --namespace multi-storage-17828 minio-service -o jsonpath="{.data.rootUser}" | base64 --decode):$(kubectl get secret --namespace multi-storage-17828 minio-service -o jsonpath="{.data.rootPassword}" | base64 --decode)@localhost:9000

  3. mc ls minio-service-local
++ kubectl_bin get pods --selector=release=minio-service -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.rRV2JGPgBN
+++ mktemp
++ local LAST_ERR=/tmp/tmp.jybnqGInq5
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=release=minio-service -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.rRV2JGPgBN
++ cat /tmp/tmp.jybnqGInq5
++ rm /tmp/tmp.rRV2JGPgBN /tmp/tmp.jybnqGInq5
++ return 0
+ MINIO_POD=minio-service-86dfccd949-kvsk7
+ wait_pod minio-service-86dfccd949-kvsk7
+ local pod=minio-service-86dfccd949-kvsk7
+ set +o xtrace
waiting for pod/minio-service-86dfccd949-kvsk7 to be ready.OK
+ '[' -n psmdb-operator ']'
+ kubectl_bin create svc -n psmdb-operator externalname minio-service --external-name=minio-service.multi-storage-17828.svc.cluster.local --tcp=9000
++ mktemp
+ local LAST_OUT=/tmp/tmp.HAtqRMt9JR
++ mktemp
+ local LAST_ERR=/tmp/tmp.B6isA9Nswq
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl create svc -n psmdb-operator externalname minio-service --external-name=minio-service.multi-storage-17828.svc.cluster.local --tcp=9000
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.HAtqRMt9JR
service/minio-service created
+ cat /tmp/tmp.B6isA9Nswq
+ rm /tmp/tmp.HAtqRMt9JR /tmp/tmp.B6isA9Nswq
+ return 0
+ create_minio_bucket operator-testing
+ local bucket=operator-testing
+ kubectl_bin run -i --rm aws-cli --image=perconalab/awscli --restart=Never -- bash -c 'AWS_ACCESS_KEY_ID=some-access-key 		AWS_SECRET_ACCESS_KEY=some-secret-key 		AWS_DEFAULT_REGION=us-east-1         /usr/bin/aws --endpoint-url http://minio-service:9000 s3 mb s3://operator-testing'
++ mktemp
+ local LAST_OUT=/tmp/tmp.9sU15R9uvh
++ mktemp
+ local LAST_ERR=/tmp/tmp.NxW7IGVqbm
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl run -i --rm aws-cli --image=perconalab/awscli --restart=Never -- bash -c 'AWS_ACCESS_KEY_ID=some-access-key 		AWS_SECRET_ACCESS_KEY=some-secret-key 		AWS_DEFAULT_REGION=us-east-1         /usr/bin/aws --endpoint-url http://minio-service:9000 s3 mb s3://operator-testing'
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.9sU15R9uvh
make_bucket: operator-testing
pod "aws-cli" deleted
+ cat /tmp/tmp.NxW7IGVqbm
If you don't see a command prompt, try pressing enter.
warning: couldn't attach to pod/aws-cli, falling back to streaming logs: Internal error occurred: Internal error occurred: error attaching to container: container is in CONTAINER_EXITED state
+ rm /tmp/tmp.9sU15R9uvh /tmp/tmp.NxW7IGVqbm
+ return 0
+ create_minio_bucket operator-testing-2
+ local bucket=operator-testing-2
+ kubectl_bin run -i --rm aws-cli --image=perconalab/awscli --restart=Never -- bash -c 'AWS_ACCESS_KEY_ID=some-access-key 		AWS_SECRET_ACCESS_KEY=some-secret-key 		AWS_DEFAULT_REGION=us-east-1         /usr/bin/aws --endpoint-url http://minio-service:9000 s3 mb s3://operator-testing-2'
++ mktemp
+ local LAST_OUT=/tmp/tmp.gofqAjXOHL
++ mktemp
+ local LAST_ERR=/tmp/tmp.VTSQILb2Wn
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl run -i --rm aws-cli --image=perconalab/awscli --restart=Never -- bash -c 'AWS_ACCESS_KEY_ID=some-access-key 		AWS_SECRET_ACCESS_KEY=some-secret-key 		AWS_DEFAULT_REGION=us-east-1         /usr/bin/aws --endpoint-url http://minio-service:9000 s3 mb s3://operator-testing-2'
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.gofqAjXOHL
make_bucket: operator-testing-2
pod "aws-cli" deleted
+ cat /tmp/tmp.VTSQILb2Wn
+ rm /tmp/tmp.gofqAjXOHL /tmp/tmp.VTSQILb2Wn
+ return 0
+ create_minio_bucket operator-testing-3
+ local bucket=operator-testing-3
+ kubectl_bin run -i --rm aws-cli --image=perconalab/awscli --restart=Never -- bash -c 'AWS_ACCESS_KEY_ID=some-access-key 		AWS_SECRET_ACCESS_KEY=some-secret-key 		AWS_DEFAULT_REGION=us-east-1         /usr/bin/aws --endpoint-url http://minio-service:9000 s3 mb s3://operator-testing-3'
++ mktemp
+ local LAST_OUT=/tmp/tmp.vX87Y8JA7n
++ mktemp
+ local LAST_ERR=/tmp/tmp.i4KeOc2XXe
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl run -i --rm aws-cli --image=perconalab/awscli --restart=Never -- bash -c 'AWS_ACCESS_KEY_ID=some-access-key 		AWS_SECRET_ACCESS_KEY=some-secret-key 		AWS_DEFAULT_REGION=us-east-1         /usr/bin/aws --endpoint-url http://minio-service:9000 s3 mb s3://operator-testing-3'
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.vX87Y8JA7n
make_bucket: operator-testing-3
pod "aws-cli" deleted
+ cat /tmp/tmp.i4KeOc2XXe
If you don't see a command prompt, try pressing enter.
warning: couldn't attach to pod/aws-cli, falling back to streaming logs: Internal error occurred: unable to upgrade connection: container aws-cli not found in pod aws-cli_multi-storage-17828
+ rm /tmp/tmp.vX87Y8JA7n /tmp/tmp.i4KeOc2XXe
+ return 0
+ log 'create secrets and start client'
+ set +o xtrace
[2025-08-07T09:02:31+0000] create secrets and start client
+ kubectl_bin apply -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2009/e2e-tests/conf/secrets.yml -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2009/e2e-tests/conf/client.yml -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2009/e2e-tests/conf/minio-secret.yml
++ mktemp
+ local LAST_OUT=/tmp/tmp.0m39pm2580
++ mktemp
+ local LAST_ERR=/tmp/tmp.bL1khJpubl
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl apply -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2009/e2e-tests/conf/secrets.yml -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2009/e2e-tests/conf/client.yml -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2009/e2e-tests/conf/minio-secret.yml
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.0m39pm2580
secret/some-users created
deployment.apps/psmdb-client created
secret/minio-secret created
+ cat /tmp/tmp.bL1khJpubl
+ rm /tmp/tmp.0m39pm2580 /tmp/tmp.bL1khJpubl
+ return 0
+ log 'create PSMDB cluster: some-name'
+ set +o xtrace
[2025-08-07T09:02:34+0000] create PSMDB cluster: some-name
+ apply_cluster /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2009/e2e-tests/multi-storage/conf/some-name.yml
+ '[' -z '' ']'
+ cat_config /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2009/e2e-tests/multi-storage/conf/some-name.yml
+ kubectl_bin apply -f -
+ cat /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2009/e2e-tests/multi-storage/conf/some-name.yml
+ yq eval '(.spec | select(.image == null)).image = "perconalab/percona-server-mongodb-operator:main-mongod7.0"'
+ yq eval '(.spec | select(has("pmm"))).pmm.image = "perconalab/pmm-client:dev-latest"'
++ mktemp
+ yq eval '(.spec | select(has("initImage"))).initImage = "perconalab/percona-server-mongodb-operator:PR-2009-645e0543"'
+ yq eval '(.spec | select(has("backup"))).backup.image = "perconalab/percona-server-mongodb-operator:main-backup"'
+ yq eval '.spec.upgradeOptions.apply="Never"'
+ local LAST_OUT=/tmp/tmp.2GabdQCDhB
++ mktemp
+ local LAST_ERR=/tmp/tmp.Cw1grOq4nE
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl apply -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.2GabdQCDhB
perconaservermongodb.psmdb.percona.com/some-name created
+ cat /tmp/tmp.Cw1grOq4nE
+ rm /tmp/tmp.2GabdQCDhB /tmp/tmp.Cw1grOq4nE
+ return 0
+ log 'check if all 3 Pods started'
+ set +o xtrace
[2025-08-07T09:02:36+0000] check if all 3 Pods started
+ wait_for_running some-name-rs0 3
+ local name=some-name-rs0
+ let last_pod=2
+ local check_cluster_readyness=true
+ set_debug
+ [[ 1 == 1 ]]
+ set -o xtrace
+ local rs_name=rs0
+ local cluster_name=some-name
++ seq 0 2
+ for i in $(seq 0 $last_pod)
+ [[ 0 -eq 2 ]]
+ wait_pod some-name-rs0-0
+ local pod=some-name-rs0-0
+ set +o xtrace
waiting for pod/some-name-rs0-0 to be ready..........OK
+ for i in $(seq 0 $last_pod)
+ [[ 1 -eq 2 ]]
+ wait_pod some-name-rs0-1
+ local pod=some-name-rs0-1
+ set +o xtrace
waiting for pod/some-name-rs0-1 to be ready.............OK
+ for i in $(seq 0 $last_pod)
+ [[ 2 -eq 2 ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].arbiter.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.Jz4MfpQxDj
+++ mktemp
++ local LAST_ERR=/tmp/tmp.pPETR5GUHi
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].arbiter.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.Jz4MfpQxDj
++ cat /tmp/tmp.pPETR5GUHi
++ rm /tmp/tmp.Jz4MfpQxDj /tmp/tmp.pPETR5GUHi
++ return 0
+ [[ '' == \t\r\u\e ]]
+ wait_pod some-name-rs0-2
+ local pod=some-name-rs0-2
+ set +o xtrace
waiting for pod/some-name-rs0-2 to be ready.........OK
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].non_voting.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.rhdte0ldMu
+++ mktemp
++ local LAST_ERR=/tmp/tmp.SQlvy3DPD5
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].non_voting.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.rhdte0ldMu
++ cat /tmp/tmp.SQlvy3DPD5
++ rm /tmp/tmp.rhdte0ldMu /tmp/tmp.SQlvy3DPD5
++ return 0
+ [[ '' == \t\r\u\e ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].hidden.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.UVf5jsL6rd
+++ mktemp
++ local LAST_ERR=/tmp/tmp.3lt8kGdmwj
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].hidden.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.UVf5jsL6rd
++ cat /tmp/tmp.3lt8kGdmwj
++ rm /tmp/tmp.UVf5jsL6rd /tmp/tmp.3lt8kGdmwj
++ return 0
+ [[ '' == \t\r\u\e ]]
+ sleep 10
+ [[ true == \t\r\u\e ]]
+ set +x
Waiting for cluster readyness
+ wait_backup_agent some-name-rs0-0
+ local agent_pod=some-name-rs0-0
+ set +o xtrace
waiting for pbm-agent to be ready in some-name-rs0-0...2025-08-07T09:04:17.000+0000 I listening for the commands

+ wait_backup_agent some-name-rs0-1
+ local agent_pod=some-name-rs0-1
+ set +o xtrace
waiting for pbm-agent to be ready in some-name-rs0-1...2025-08-07T09:04:20.000+0000 I listening for the commands

+ wait_backup_agent some-name-rs0-2
+ local agent_pod=some-name-rs0-2
+ set +o xtrace
waiting for pbm-agent to be ready in some-name-rs0-2...2025-08-07T09:04:26.000+0000 I listening for the commands

+ create_user
+ log 'create user myApp'
+ set +o xtrace
[2025-08-07T09:04:29+0000] create user myApp
+ run_mongo 'db.createUser({user:"myApp",pwd:"myPass",roles:[{db:"myApp",role:"readWrite"}]})' userAdmin:userAdmin123456@some-name-rs0.multi-storage-17828 mongodb
+ local 'command=db.createUser({user:"myApp",pwd:"myPass",roles:[{db:"myApp",role:"readWrite"}]})'
+ local uri=userAdmin:userAdmin123456@some-name-rs0.multi-storage-17828
+ local driver=mongodb
+ local suffix=.svc.cluster.local
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.J28ljHW5k6
+++ mktemp
++ local LAST_ERR=/tmp/tmp.EoKSB1OP57
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.J28ljHW5k6
++ cat /tmp/tmp.EoKSB1OP57
++ rm /tmp/tmp.J28ljHW5k6 /tmp/tmp.EoKSB1OP57
++ return 0
+ local client_container=psmdb-client-66f577db5f-cfk8q
+ local mongo_flag=
+ [[ userAdmin:userAdmin123456@some-name-rs0.multi-storage-17828 == *cfg* ]]
+ replica_set=rs0
+ kubectl_bin exec psmdb-client-66f577db5f-cfk8q -- bash -c 'printf '\''db.createUser({user:"myApp",pwd:"myPass",roles:[{db:"myApp",role:"readWrite"}]})\n'\'' | mongo mongodb://userAdmin:userAdmin123456@some-name-rs0.multi-storage-17828.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
++ mktemp
+ local LAST_OUT=/tmp/tmp.rawLSz1byT
++ mktemp
+ local LAST_ERR=/tmp/tmp.qrlc2wFo1W
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-66f577db5f-cfk8q -- bash -c 'printf '\''db.createUser({user:"myApp",pwd:"myPass",roles:[{db:"myApp",role:"readWrite"}]})\n'\'' | mongo mongodb://userAdmin:userAdmin123456@some-name-rs0.multi-storage-17828.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.rawLSz1byT
Percona Server for MongoDB shell version v4.4.29-28
connecting to: mongodb://some-name-rs0.multi-storage-17828.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb&replicaSet=rs0&ssl=false
Implicit session: session { "id" : UUID("5c667fa1-6ada-4325-b453-46b6192311ae") }
Percona Server for MongoDB server version: v7.0.22-12
WARNING: shell and server versions do not match
Successfully added user: {
	"user" : "myApp",
	"roles" : [
		{
			"db" : "myApp",
			"role" : "readWrite"
		}
	]
}
bye
+ cat /tmp/tmp.qrlc2wFo1W
+ rm /tmp/tmp.rawLSz1byT /tmp/tmp.qrlc2wFo1W
+ return 0
+ write_document 100500
+ local x=100500
+ local cmp_postfix=
+ log 'write document: 100500'
+ set +o xtrace
[2025-08-07T09:04:32+0000] write document: 100500
+ run_mongo 'use myApp\n db.test.insert({ x: 100500 })' myApp:myPass@some-name-rs0.multi-storage-17828 mongodb
+ local 'command=use myApp\n db.test.insert({ x: 100500 })'
+ local uri=myApp:myPass@some-name-rs0.multi-storage-17828
+ local driver=mongodb
+ local suffix=.svc.cluster.local
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.Rdj3JmCYtF
+++ mktemp
++ local LAST_ERR=/tmp/tmp.oDg1AdJ6pM
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.Rdj3JmCYtF
++ cat /tmp/tmp.oDg1AdJ6pM
++ rm /tmp/tmp.Rdj3JmCYtF /tmp/tmp.oDg1AdJ6pM
++ return 0
+ local client_container=psmdb-client-66f577db5f-cfk8q
+ local mongo_flag=
+ [[ myApp:myPass@some-name-rs0.multi-storage-17828 == *cfg* ]]
+ replica_set=rs0
+ kubectl_bin exec psmdb-client-66f577db5f-cfk8q -- bash -c 'printf '\''use myApp\n db.test.insert({ x: 100500 })\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0.multi-storage-17828.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
++ mktemp
+ local LAST_OUT=/tmp/tmp.qVYzPGhtzS
++ mktemp
+ local LAST_ERR=/tmp/tmp.4DpLLUnePT
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-66f577db5f-cfk8q -- bash -c 'printf '\''use myApp\n db.test.insert({ x: 100500 })\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0.multi-storage-17828.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.qVYzPGhtzS
Percona Server for MongoDB shell version v4.4.29-28
connecting to: mongodb://some-name-rs0.multi-storage-17828.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb&replicaSet=rs0&ssl=false
Implicit session: session { "id" : UUID("495b6f26-f198-4a1c-8d0d-e2c88ef45c26") }
Percona Server for MongoDB server version: v7.0.22-12
WARNING: shell and server versions do not match
switched to db myApp
WriteResult({ "nInserted" : 1 })
bye
+ cat /tmp/tmp.4DpLLUnePT
+ rm /tmp/tmp.qVYzPGhtzS /tmp/tmp.4DpLLUnePT
+ return 0
+ run_backup minio-1 backup-minio-1 logical
+ local storage=minio-1
+ local name=backup-minio-1
+ local type=logical
+ log 'creating backup: backup-minio-1 (logical)'
+ set +o xtrace
[2025-08-07T09:04:34+0000] creating backup: backup-minio-1 (logical)
+ yq eval '.metadata.name = "backup-minio-1"
			| .spec.storageName = "minio-1"
			| .spec.type = "logical"' /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2009/e2e-tests/multi-storage/conf/backup.yml
+ kubectl_bin apply -f -
++ mktemp
+ local LAST_OUT=/tmp/tmp.xYAcsbKUvc
++ mktemp
+ local LAST_ERR=/tmp/tmp.dsDuZfWnok
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl apply -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.xYAcsbKUvc
perconaservermongodbbackup.psmdb.percona.com/backup-minio-1 created
+ cat /tmp/tmp.dsDuZfWnok
+ rm /tmp/tmp.xYAcsbKUvc /tmp/tmp.dsDuZfWnok
+ return 0
+ wait_backup backup-minio-1
+ local backup_name=backup-minio-1
+ local target_state=ready
+ set +o xtrace
waiting for backup-minio-1 to reach ready state......
+ run_backup minio-2 backup-minio-2 logical
+ local storage=minio-2
+ local name=backup-minio-2
+ local type=logical
+ log 'creating backup: backup-minio-2 (logical)'
+ set +o xtrace
[2025-08-07T09:04:49+0000] creating backup: backup-minio-2 (logical)
+ yq eval '.metadata.name = "backup-minio-2"
			| .spec.storageName = "minio-2"
			| .spec.type = "logical"' /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2009/e2e-tests/multi-storage/conf/backup.yml
+ kubectl_bin apply -f -
++ mktemp
+ local LAST_OUT=/tmp/tmp.2uEWDrwgtD
++ mktemp
+ local LAST_ERR=/tmp/tmp.N7q02sdi2Q
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl apply -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.2uEWDrwgtD
perconaservermongodbbackup.psmdb.percona.com/backup-minio-2 created
+ cat /tmp/tmp.N7q02sdi2Q
+ rm /tmp/tmp.2uEWDrwgtD /tmp/tmp.N7q02sdi2Q
+ return 0
+ wait_backup backup-minio-2
+ local backup_name=backup-minio-2
+ local target_state=ready
+ set +o xtrace
waiting for backup-minio-2 to reach ready state.....
+ run_backup minio-3 backup-minio-3 physical
+ local storage=minio-3
+ local name=backup-minio-3
+ local type=physical
+ log 'creating backup: backup-minio-3 (physical)'
+ set +o xtrace
[2025-08-07T09:05:02+0000] creating backup: backup-minio-3 (physical)
+ yq eval '.metadata.name = "backup-minio-3"
			| .spec.storageName = "minio-3"
			| .spec.type = "physical"' /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2009/e2e-tests/multi-storage/conf/backup.yml
+ kubectl_bin apply -f -
++ mktemp
+ local LAST_OUT=/tmp/tmp.oAcqcmDYOf
++ mktemp
+ local LAST_ERR=/tmp/tmp.sIHxWoQhAf
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl apply -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.oAcqcmDYOf
perconaservermongodbbackup.psmdb.percona.com/backup-minio-3 created
+ cat /tmp/tmp.sIHxWoQhAf
+ rm /tmp/tmp.oAcqcmDYOf /tmp/tmp.sIHxWoQhAf
+ return 0
+ wait_backup backup-minio-3
+ local backup_name=backup-minio-3
+ local target_state=ready
+ set +o xtrace
waiting for backup-minio-3 to reach ready state.....
+ write_document 100501
+ local x=100501
+ local cmp_postfix=
+ log 'write document: 100501'
+ set +o xtrace
[2025-08-07T09:05:15+0000] write document: 100501
+ run_mongo 'use myApp\n db.test.insert({ x: 100501 })' myApp:myPass@some-name-rs0.multi-storage-17828 mongodb
+ local 'command=use myApp\n db.test.insert({ x: 100501 })'
+ local uri=myApp:myPass@some-name-rs0.multi-storage-17828
+ local driver=mongodb
+ local suffix=.svc.cluster.local
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.yc9olSqvKe
+++ mktemp
++ local LAST_ERR=/tmp/tmp.2pZu2MIw3q
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.yc9olSqvKe
++ cat /tmp/tmp.2pZu2MIw3q
++ rm /tmp/tmp.yc9olSqvKe /tmp/tmp.2pZu2MIw3q
++ return 0
+ local client_container=psmdb-client-66f577db5f-cfk8q
+ local mongo_flag=
+ [[ myApp:myPass@some-name-rs0.multi-storage-17828 == *cfg* ]]
+ replica_set=rs0
+ kubectl_bin exec psmdb-client-66f577db5f-cfk8q -- bash -c 'printf '\''use myApp\n db.test.insert({ x: 100501 })\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0.multi-storage-17828.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
++ mktemp
+ local LAST_OUT=/tmp/tmp.aGKFZJBkg2
++ mktemp
+ local LAST_ERR=/tmp/tmp.Hnq1LZ98iO
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-66f577db5f-cfk8q -- bash -c 'printf '\''use myApp\n db.test.insert({ x: 100501 })\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0.multi-storage-17828.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.aGKFZJBkg2
Percona Server for MongoDB shell version v4.4.29-28
connecting to: mongodb://some-name-rs0.multi-storage-17828.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb&replicaSet=rs0&ssl=false
Implicit session: session { "id" : UUID("1919ea2e-2d45-4b5e-a3d4-4359bf137451") }
Percona Server for MongoDB server version: v7.0.22-12
WARNING: shell and server versions do not match
switched to db myApp
WriteResult({ "nInserted" : 1 })
bye
+ cat /tmp/tmp.Hnq1LZ98iO
+ rm /tmp/tmp.aGKFZJBkg2 /tmp/tmp.Hnq1LZ98iO
+ return 0
+ write_document 100502
+ local x=100502
+ local cmp_postfix=
+ log 'write document: 100502'
+ set +o xtrace
[2025-08-07T09:05:18+0000] write document: 100502
+ run_mongo 'use myApp\n db.test.insert({ x: 100502 })' myApp:myPass@some-name-rs0.multi-storage-17828 mongodb
+ local 'command=use myApp\n db.test.insert({ x: 100502 })'
+ local uri=myApp:myPass@some-name-rs0.multi-storage-17828
+ local driver=mongodb
+ local suffix=.svc.cluster.local
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.MXNKwwHCWJ
+++ mktemp
++ local LAST_ERR=/tmp/tmp.ezXXjefTU0
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.MXNKwwHCWJ
++ cat /tmp/tmp.ezXXjefTU0
++ rm /tmp/tmp.MXNKwwHCWJ /tmp/tmp.ezXXjefTU0
++ return 0
+ local client_container=psmdb-client-66f577db5f-cfk8q
+ local mongo_flag=
+ [[ myApp:myPass@some-name-rs0.multi-storage-17828 == *cfg* ]]
+ replica_set=rs0
+ kubectl_bin exec psmdb-client-66f577db5f-cfk8q -- bash -c 'printf '\''use myApp\n db.test.insert({ x: 100502 })\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0.multi-storage-17828.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
++ mktemp
+ local LAST_OUT=/tmp/tmp.91tvb2ctyI
++ mktemp
+ local LAST_ERR=/tmp/tmp.rsf5sDmKmf
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-66f577db5f-cfk8q -- bash -c 'printf '\''use myApp\n db.test.insert({ x: 100502 })\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0.multi-storage-17828.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.91tvb2ctyI
Percona Server for MongoDB shell version v4.4.29-28
connecting to: mongodb://some-name-rs0.multi-storage-17828.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb&replicaSet=rs0&ssl=false
Implicit session: session { "id" : UUID("1f0c306c-5b62-422d-9044-e1190dc55abb") }
Percona Server for MongoDB server version: v7.0.22-12
WARNING: shell and server versions do not match
switched to db myApp
WriteResult({ "nInserted" : 1 })
bye
+ cat /tmp/tmp.rsf5sDmKmf
+ rm /tmp/tmp.91tvb2ctyI /tmp/tmp.rsf5sDmKmf
+ return 0
+ write_document 100503
+ local x=100503
+ local cmp_postfix=
+ log 'write document: 100503'
+ set +o xtrace
[2025-08-07T09:05:20+0000] write document: 100503
+ run_mongo 'use myApp\n db.test.insert({ x: 100503 })' myApp:myPass@some-name-rs0.multi-storage-17828 mongodb
+ local 'command=use myApp\n db.test.insert({ x: 100503 })'
+ local uri=myApp:myPass@some-name-rs0.multi-storage-17828
+ local driver=mongodb
+ local suffix=.svc.cluster.local
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.nZduMrs9y3
+++ mktemp
++ local LAST_ERR=/tmp/tmp.cKmrD0yycV
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.nZduMrs9y3
++ cat /tmp/tmp.cKmrD0yycV
++ rm /tmp/tmp.nZduMrs9y3 /tmp/tmp.cKmrD0yycV
++ return 0
+ local client_container=psmdb-client-66f577db5f-cfk8q
+ local mongo_flag=
+ [[ myApp:myPass@some-name-rs0.multi-storage-17828 == *cfg* ]]
+ replica_set=rs0
+ kubectl_bin exec psmdb-client-66f577db5f-cfk8q -- bash -c 'printf '\''use myApp\n db.test.insert({ x: 100503 })\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0.multi-storage-17828.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
++ mktemp
+ local LAST_OUT=/tmp/tmp.BNFqOzqBKo
++ mktemp
+ local LAST_ERR=/tmp/tmp.3Z5OIUUQI9
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-66f577db5f-cfk8q -- bash -c 'printf '\''use myApp\n db.test.insert({ x: 100503 })\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0.multi-storage-17828.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.BNFqOzqBKo
Percona Server for MongoDB shell version v4.4.29-28
connecting to: mongodb://some-name-rs0.multi-storage-17828.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb&replicaSet=rs0&ssl=false
Implicit session: session { "id" : UUID("be72ba42-891f-488c-af96-138ca634483f") }
Percona Server for MongoDB server version: v7.0.22-12
WARNING: shell and server versions do not match
switched to db myApp
WriteResult({ "nInserted" : 1 })
bye
+ cat /tmp/tmp.3Z5OIUUQI9
+ rm /tmp/tmp.BNFqOzqBKo /tmp/tmp.3Z5OIUUQI9
+ return 0
+ write_document 100504
+ local x=100504
+ local cmp_postfix=
+ log 'write document: 100504'
+ set +o xtrace
[2025-08-07T09:05:22+0000] write document: 100504
+ run_mongo 'use myApp\n db.test.insert({ x: 100504 })' myApp:myPass@some-name-rs0.multi-storage-17828 mongodb
+ local 'command=use myApp\n db.test.insert({ x: 100504 })'
+ local uri=myApp:myPass@some-name-rs0.multi-storage-17828
+ local driver=mongodb
+ local suffix=.svc.cluster.local
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.KoXiOZJ57d
+++ mktemp
++ local LAST_ERR=/tmp/tmp.y3su0jhBQ5
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.KoXiOZJ57d
++ cat /tmp/tmp.y3su0jhBQ5
++ rm /tmp/tmp.KoXiOZJ57d /tmp/tmp.y3su0jhBQ5
++ return 0
+ local client_container=psmdb-client-66f577db5f-cfk8q
+ local mongo_flag=
+ [[ myApp:myPass@some-name-rs0.multi-storage-17828 == *cfg* ]]
+ replica_set=rs0
+ kubectl_bin exec psmdb-client-66f577db5f-cfk8q -- bash -c 'printf '\''use myApp\n db.test.insert({ x: 100504 })\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0.multi-storage-17828.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
++ mktemp
+ local LAST_OUT=/tmp/tmp.UGHm6azIrw
++ mktemp
+ local LAST_ERR=/tmp/tmp.g2LL9omGfi
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-66f577db5f-cfk8q -- bash -c 'printf '\''use myApp\n db.test.insert({ x: 100504 })\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0.multi-storage-17828.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.UGHm6azIrw
Percona Server for MongoDB shell version v4.4.29-28
connecting to: mongodb://some-name-rs0.multi-storage-17828.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb&replicaSet=rs0&ssl=false
Implicit session: session { "id" : UUID("5fb428a1-a8c5-484c-8c00-338637cb002e") }
Percona Server for MongoDB server version: v7.0.22-12
WARNING: shell and server versions do not match
switched to db myApp
WriteResult({ "nInserted" : 1 })
bye
+ cat /tmp/tmp.g2LL9omGfi
+ rm /tmp/tmp.UGHm6azIrw /tmp/tmp.g2LL9omGfi
+ return 0
+ write_document 100505
+ local x=100505
+ local cmp_postfix=
+ log 'write document: 100505'
+ set +o xtrace
[2025-08-07T09:05:24+0000] write document: 100505
+ run_mongo 'use myApp\n db.test.insert({ x: 100505 })' myApp:myPass@some-name-rs0.multi-storage-17828 mongodb
+ local 'command=use myApp\n db.test.insert({ x: 100505 })'
+ local uri=myApp:myPass@some-name-rs0.multi-storage-17828
+ local driver=mongodb
+ local suffix=.svc.cluster.local
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.CgGkPg6K9E
+++ mktemp
++ local LAST_ERR=/tmp/tmp.BCWUmay4x9
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.CgGkPg6K9E
++ cat /tmp/tmp.BCWUmay4x9
++ rm /tmp/tmp.CgGkPg6K9E /tmp/tmp.BCWUmay4x9
++ return 0
+ local client_container=psmdb-client-66f577db5f-cfk8q
+ local mongo_flag=
+ [[ myApp:myPass@some-name-rs0.multi-storage-17828 == *cfg* ]]
+ replica_set=rs0
+ kubectl_bin exec psmdb-client-66f577db5f-cfk8q -- bash -c 'printf '\''use myApp\n db.test.insert({ x: 100505 })\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0.multi-storage-17828.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
++ mktemp
+ local LAST_OUT=/tmp/tmp.btaVcAvVqL
++ mktemp
+ local LAST_ERR=/tmp/tmp.MZwVAFIp8o
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-66f577db5f-cfk8q -- bash -c 'printf '\''use myApp\n db.test.insert({ x: 100505 })\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0.multi-storage-17828.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.btaVcAvVqL
Percona Server for MongoDB shell version v4.4.29-28
connecting to: mongodb://some-name-rs0.multi-storage-17828.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb&replicaSet=rs0&ssl=false
Implicit session: session { "id" : UUID("3238d7eb-9887-422d-83bd-0524220d154e") }
Percona Server for MongoDB server version: v7.0.22-12
WARNING: shell and server versions do not match
switched to db myApp
WriteResult({ "nInserted" : 1 })
bye
+ cat /tmp/tmp.MZwVAFIp8o
+ rm /tmp/tmp.btaVcAvVqL /tmp/tmp.MZwVAFIp8o
+ return 0
+ compare_mongo_cmd find myApp:myPass@some-name-rs0.multi-storage-17828
+ local command=find
+ local uri=myApp:myPass@some-name-rs0.multi-storage-17828
+ local postfix=
+ local suffix=
+ local database=myApp
+ local collection=test
+ local sort=
+ local 'full_command=db.test.find()'
+ [[ ! -z '' ]]
+ log 'running db.test.find() in myApp'
+ set +o xtrace
[2025-08-07T09:05:27+0000] running db.test.find() in myApp
+ run_mongo 'use myApp\n db.test.find()' myApp:myPass@some-name-rs0.multi-storage-17828 mongodb ''
+ local 'command=use myApp\n db.test.find()'
+ local uri=myApp:myPass@some-name-rs0.multi-storage-17828
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ egrep -v 'I NETWORK|W NETWORK|F NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:'
+ /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
egrep: warning: egrep is obsolescent; using grep -E
++ local LAST_OUT=/tmp/tmp.6bdKogDP1k
+++ mktemp
++ local LAST_ERR=/tmp/tmp.xdzB54VMTO
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.6bdKogDP1k
++ cat /tmp/tmp.xdzB54VMTO
++ rm /tmp/tmp.6bdKogDP1k /tmp/tmp.xdzB54VMTO
++ return 0
+ local client_container=psmdb-client-66f577db5f-cfk8q
+ local mongo_flag=
+ [[ myApp:myPass@some-name-rs0.multi-storage-17828 == *cfg* ]]
+ replica_set=rs0
+ kubectl_bin exec psmdb-client-66f577db5f-cfk8q -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0.multi-storage-17828.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
++ mktemp
+ local LAST_OUT=/tmp/tmp.7z5qAb8iha
++ mktemp
+ local LAST_ERR=/tmp/tmp.XnDhoNBrWz
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-66f577db5f-cfk8q -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0.multi-storage-17828.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.7z5qAb8iha
+ cat /tmp/tmp.XnDhoNBrWz
+ rm /tmp/tmp.7z5qAb8iha /tmp/tmp.XnDhoNBrWz
+ return 0
+ [[ 0 -eq 0 ]]
+ diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2009/e2e-tests/multi-storage/compare/find.json /tmp/tmp.i5FaNrGCBi/find
+ sleep_with_log 70 'wait for oplog chunks to be uploaded'
+ local d=70
+ local 'msg=wait for oplog chunks to be uploaded'
+ log 'wait for oplog chunks to be uploaded. sleeping for 70 seconds'
+ set +o xtrace
[2025-08-07T09:05:29+0000] wait for oplog chunks to be uploaded. sleeping for 70 seconds
+ sleep 70
+ wait_for_restorable_time backup-minio-3
+ local backup_name=backup-minio-3
++ get_latest_restorable_time backup-minio-3
++ local backup_name=backup-minio-3
+++ kubectl_bin get psmdb-backup backup-minio-3 -o yaml
+++ yq .status.latestRestorableTime
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.vqiGVEsru6
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.OMuAXGzSya
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in $(seq 0 2)
+++ set +e
+++ kubectl get psmdb-backup backup-minio-3 -o yaml
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.vqiGVEsru6
+++ cat /tmp/tmp.OMuAXGzSya
+++ rm /tmp/tmp.vqiGVEsru6 /tmp/tmp.OMuAXGzSya
+++ return 0
++ local time=2025-08-07T09:06:00Z
++ [[ 2025-08-07T09:06:00Z == \n\u\l\l ]]
++ echo -n 2025-08-07T09:06:00Z
+ local latest_restorable_time=2025-08-07T09:06:00Z
+ local retries=0
+ echo -n 'waiting for psmdb-backup/backup-minio-3'\''s latest restorable time'
waiting for psmdb-backup/backup-minio-3's latest restorable time+ [[ 2025-08-07T09:06:00Z != \n\u\l\l ]]
+ echo '.OK latestRestorableTime: 2025-08-07T09:06:00Z'
.OK latestRestorableTime: 2025-08-07T09:06:00Z
+ wait_for_advanced_restorable_time backup-minio-3
+ local backup_name=backup-minio-3
+++ get_latest_restorable_time backup-minio-3
+++ local backup_name=backup-minio-3
++++ kubectl_bin get psmdb-backup backup-minio-3 -o yaml
++++ yq .status.latestRestorableTime
+++++ mktemp
++++ local LAST_OUT=/tmp/tmp.sbY2ell2OS
+++++ mktemp
++++ local LAST_ERR=/tmp/tmp.8pmnUWoyWO
++++ local exit_status=0
++++ local timeout=4
+++++ seq 0 2
++++ for i in $(seq 0 2)
++++ set +e
++++ kubectl get psmdb-backup backup-minio-3 -o yaml
++++ exit_status=0
++++ set -e
++++ '[' 0 '!=' 0 -a -n 1 ']'
++++ break
++++ cat /tmp/tmp.sbY2ell2OS
++++ cat /tmp/tmp.8pmnUWoyWO
++++ rm /tmp/tmp.sbY2ell2OS /tmp/tmp.8pmnUWoyWO
++++ return 0
+++ local time=2025-08-07T09:06:00Z
+++ [[ 2025-08-07T09:06:00Z == \n\u\l\l ]]
+++ echo -n 2025-08-07T09:06:00Z
++ datetime_to_timestamp 2025-08-07T09:06:00Z
++ local datetime=2025-08-07T09:06:00Z
++ TZ=UTC
++ /usr/sbin/date -d2025-08-07T09:06:00Z +%s
+ local latest_restorable_time=1754557560
++ get_backup_last_write backup-minio-3
++ local backup_name=backup-minio-3
+++ kubectl_bin get psmdb-backup backup-minio-3 -o yaml
+++ yq .status.pbmName
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.qrI2eSp1Gk
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.aBR0LTRCl9
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in $(seq 0 2)
+++ set +e
+++ kubectl get psmdb-backup backup-minio-3 -o yaml
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.qrI2eSp1Gk
+++ cat /tmp/tmp.aBR0LTRCl9
+++ rm /tmp/tmp.qrI2eSp1Gk /tmp/tmp.aBR0LTRCl9
+++ return 0
++ local pbm_name=2025-08-07T09:05:05Z
++ kubectl_bin exec some-name-rs0-0 -c backup-agent -- pbm list -o json
++ jq --arg v 2025-08-07T09:05:05Z '.snapshots[] | select(.name==$v) | .restoreTo'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.16QIeSFLgg
+++ mktemp
++ local LAST_ERR=/tmp/tmp.caqIp5ardI
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl exec some-name-rs0-0 -c backup-agent -- pbm list -o json
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.16QIeSFLgg
++ cat /tmp/tmp.caqIp5ardI
++ rm /tmp/tmp.16QIeSFLgg /tmp/tmp.caqIp5ardI
++ return 0
+ local latest_write=1754557507
+ local retries=0
++ timestamp_to_datetime 1754557507
++ local ts=1754557507
++ TZ=UTC
++ /usr/sbin/date -d@1754557507 +%Y-%m-%dT%H:%M:%S%Z
+ echo -n 'waiting for psmdb-backup/backup-minio-3'\''s latest restorable time to advance its latest write (2025-08-07T09:05:07UTC)'
waiting for psmdb-backup/backup-minio-3's latest restorable time to advance its latest write (2025-08-07T09:05:07UTC)+ [[ 1754557560 -gt 1754557507 ]]
++ timestamp_to_datetime 1754557560
++ local ts=1754557560
++ TZ=UTC
++ /usr/sbin/date -d@1754557560 +%Y-%m-%dT%H:%M:%S%Z
+ echo '.OK latestRestorableTime: 2025-08-07T09:06:00UTC'
.OK latestRestorableTime: 2025-08-07T09:06:00UTC
++ get_latest_restorable_time backup-minio-3
++ local backup_name=backup-minio-3
+++ kubectl_bin get psmdb-backup backup-minio-3 -o yaml
+++ yq .status.latestRestorableTime
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.Vlowx4UEZk
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.nqRs4Rh99n
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in $(seq 0 2)
+++ set +e
+++ kubectl get psmdb-backup backup-minio-3 -o yaml
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.Vlowx4UEZk
+++ cat /tmp/tmp.nqRs4Rh99n
+++ rm /tmp/tmp.Vlowx4UEZk /tmp/tmp.nqRs4Rh99n
+++ return 0
++ local time=2025-08-07T09:06:00Z
++ [[ 2025-08-07T09:06:00Z == \n\u\l\l ]]
++ echo -n 2025-08-07T09:06:00Z
+ restore_time=2025-08-07T09:06:00Z
++ format_pitr_target 2025-08-07T09:06:00Z
++ local target=2025-08-07T09:06:00Z
++ echo 2025-08-07T09:06:00Z
++ sed 's/T/ /'
++ sed s/Z//
+ check_recovery backup-minio-3 '2025-08-07 09:06:00'
+ local backup_name=backup-minio-3
+ local 'restore_time=2025-08-07 09:06:00'
+ local cmp_postfix=
+ drop_collection myApp test myApp myPass
+ local db=myApp
+ local collection=test
+ local user=myApp
+ local pass=myPass
+ log 'dropping collection: myApp.test'
+ set +o xtrace
[2025-08-07T09:06:44+0000] dropping collection: myApp.test
+ run_mongo 'use myApp\n db.test.drop()' myApp:myPass@some-name-rs0.multi-storage-17828 mongodb
+ local 'command=use myApp\n db.test.drop()'
+ local uri=myApp:myPass@some-name-rs0.multi-storage-17828
+ local driver=mongodb
+ local suffix=.svc.cluster.local
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.STees56ARz
+++ mktemp
++ local LAST_ERR=/tmp/tmp.O91ve4aQzK
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.STees56ARz
++ cat /tmp/tmp.O91ve4aQzK
++ rm /tmp/tmp.STees56ARz /tmp/tmp.O91ve4aQzK
++ return 0
+ local client_container=psmdb-client-66f577db5f-cfk8q
+ local mongo_flag=
+ [[ myApp:myPass@some-name-rs0.multi-storage-17828 == *cfg* ]]
+ replica_set=rs0
+ kubectl_bin exec psmdb-client-66f577db5f-cfk8q -- bash -c 'printf '\''use myApp\n db.test.drop()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0.multi-storage-17828.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
++ mktemp
+ local LAST_OUT=/tmp/tmp.4LMYD7owXw
++ mktemp
+ local LAST_ERR=/tmp/tmp.vOK3rHrmXg
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-66f577db5f-cfk8q -- bash -c 'printf '\''use myApp\n db.test.drop()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0.multi-storage-17828.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.4LMYD7owXw
Percona Server for MongoDB shell version v4.4.29-28
connecting to: mongodb://some-name-rs0.multi-storage-17828.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb&replicaSet=rs0&ssl=false
Implicit session: session { "id" : UUID("6b7347eb-9f8c-4ad5-9211-15845f2aef87") }
Percona Server for MongoDB server version: v7.0.22-12
WARNING: shell and server versions do not match
switched to db myApp
true
bye
+ cat /tmp/tmp.vOK3rHrmXg
+ rm /tmp/tmp.4LMYD7owXw /tmp/tmp.vOK3rHrmXg
+ return 0
+ log 'creating restore: restore-backup-minio-3 restore_to: 2025-08-07 09:06:00'
+ set +o xtrace
[2025-08-07T09:06:45+0000] creating restore: restore-backup-minio-3 restore_to: 2025-08-07 09:06:00
+ cat /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2009/e2e-tests/multi-storage/conf/restore.yml
+ /usr/sbin/sed -e 's/name:/name: restore-backup-minio-3/'
+ /usr/sbin/sed -e 's/backupName:/backupName: backup-minio-3/'
+ /usr/sbin/sed -e 's/date:/date: 2025-08-07 09:06:00/'
+ kubectl_bin apply -f -
++ mktemp
+ local LAST_OUT=/tmp/tmp.eQv79dsmIy
++ mktemp
+ local LAST_ERR=/tmp/tmp.4GaMcokW3l
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl apply -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.eQv79dsmIy
perconaservermongodbrestore.psmdb.percona.com/restore-backup-minio-3 created
+ cat /tmp/tmp.4GaMcokW3l
+ rm /tmp/tmp.eQv79dsmIy /tmp/tmp.4GaMcokW3l
+ return 0
+ wait_restore backup-minio-3 some-name requested 0 900 1
+ local backup_name=backup-minio-3
+ local cluster_name=some-name
+ local target_state=requested
+ local wait_cluster_consistency=0
+ local wait_time=900
+ local ok_if_ready=1
+ set +o xtrace
Waiting for the psmdb-restore/restore-backup-minio-3 object to be createdOK
Waiting psmdb-restore/restore-backup-minio-3 to reach state "requested" ..OK after 2 minutes
+ [[ 0 -eq 1 ]]
+ wait_restore backup-minio-3 some-name ready 0 1600
+ local backup_name=backup-minio-3
+ local cluster_name=some-name
+ local target_state=ready
+ local wait_cluster_consistency=0
+ local wait_time=1600
+ local ok_if_ready=0
+ set +o xtrace
Waiting for the psmdb-restore/restore-backup-minio-3 object to be createdOK
Waiting psmdb-restore/restore-backup-minio-3 to reach state "ready" .OK after 1 minutes
+ [[ 0 -eq 1 ]]
+ wait_for_running some-name-rs0 3
+ local name=some-name-rs0
+ let last_pod=2
+ local check_cluster_readyness=true
+ set_debug
+ [[ 1 == 1 ]]
+ set -o xtrace
+ local rs_name=rs0
+ local cluster_name=some-name
++ seq 0 2
+ for i in $(seq 0 $last_pod)
+ [[ 0 -eq 2 ]]
+ wait_pod some-name-rs0-0
+ local pod=some-name-rs0-0
+ set +o xtrace
waiting for pod/some-name-rs0-0 to be ready.OK
+ for i in $(seq 0 $last_pod)
+ [[ 1 -eq 2 ]]
+ wait_pod some-name-rs0-1
+ local pod=some-name-rs0-1
+ set +o xtrace
waiting for pod/some-name-rs0-1 to be ready.OK
+ for i in $(seq 0 $last_pod)
+ [[ 2 -eq 2 ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].arbiter.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.2PxolW7XtB
+++ mktemp
++ local LAST_ERR=/tmp/tmp.7jborV3N5s
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].arbiter.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.2PxolW7XtB
++ cat /tmp/tmp.7jborV3N5s
++ rm /tmp/tmp.2PxolW7XtB /tmp/tmp.7jborV3N5s
++ return 0
+ [[ '' == \t\r\u\e ]]
+ wait_pod some-name-rs0-2
+ local pod=some-name-rs0-2
+ set +o xtrace
waiting for pod/some-name-rs0-2 to be ready.OK
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].non_voting.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.g1glDw6I5C
+++ mktemp
++ local LAST_ERR=/tmp/tmp.G07cUrH38g
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].non_voting.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.g1glDw6I5C
++ cat /tmp/tmp.G07cUrH38g
++ rm /tmp/tmp.g1glDw6I5C /tmp/tmp.G07cUrH38g
++ return 0
+ [[ '' == \t\r\u\e ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].hidden.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.mhgOAGATq6
+++ mktemp
++ local LAST_ERR=/tmp/tmp.sPSm04iBmz
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].hidden.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.mhgOAGATq6
++ cat /tmp/tmp.sPSm04iBmz
++ rm /tmp/tmp.mhgOAGATq6 /tmp/tmp.sPSm04iBmz
++ return 0
+ [[ '' == \t\r\u\e ]]
+ sleep 10
+ [[ true == \t\r\u\e ]]
+ set +x
Waiting for cluster readyness..............................................................
+ compare_mongo_cmd find myApp:myPass@some-name-rs0.multi-storage-17828 '' .svc.cluster.local myApp test 'sort( { x: 1 } )'
+ local command=find
+ local uri=myApp:myPass@some-name-rs0.multi-storage-17828
+ local postfix=
+ local suffix=.svc.cluster.local
+ local database=myApp
+ local collection=test
+ local 'sort=sort( { x: 1 } )'
+ local 'full_command=db.test.find()'
+ [[ ! -z sort( { x: 1 } ) ]]
+ full_command='db.test.find().sort( { x: 1 } )'
+ log 'running db.test.find().sort( { x: 1 } ) in myApp'
+ set +o xtrace
[2025-08-07T09:13:18+0000] running db.test.find().sort( { x: 1 } ) in myApp
+ run_mongo 'use myApp\n db.test.find().sort( { x: 1 } )' myApp:myPass@some-name-rs0.multi-storage-17828 mongodb .svc.cluster.local
+ local 'command=use myApp\n db.test.find().sort( { x: 1 } )'
+ local uri=myApp:myPass@some-name-rs0.multi-storage-17828
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ egrep -v 'I NETWORK|W NETWORK|F NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:'
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+ /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
+++ mktemp
egrep: warning: egrep is obsolescent; using grep -E
++ local LAST_OUT=/tmp/tmp.Aq6jWYnfwU
+++ mktemp
++ local LAST_ERR=/tmp/tmp.jySNpAtusd
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.Aq6jWYnfwU
++ cat /tmp/tmp.jySNpAtusd
++ rm /tmp/tmp.Aq6jWYnfwU /tmp/tmp.jySNpAtusd
++ return 0
+ local client_container=psmdb-client-66f577db5f-cfk8q
+ local mongo_flag=
+ [[ myApp:myPass@some-name-rs0.multi-storage-17828 == *cfg* ]]
+ replica_set=rs0
+ kubectl_bin exec psmdb-client-66f577db5f-cfk8q -- bash -c 'printf '\''use myApp\n db.test.find().sort( { x: 1 } )\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0.multi-storage-17828.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
++ mktemp
+ local LAST_OUT=/tmp/tmp.t2vQ9oTOWV
++ mktemp
+ local LAST_ERR=/tmp/tmp.PkYNt1h1tX
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-66f577db5f-cfk8q -- bash -c 'printf '\''use myApp\n db.test.find().sort( { x: 1 } )\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0.multi-storage-17828.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.t2vQ9oTOWV
+ cat /tmp/tmp.PkYNt1h1tX
+ rm /tmp/tmp.t2vQ9oTOWV /tmp/tmp.PkYNt1h1tX
+ return 0
+ [[ 0 -eq 0 ]]
+ diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2009/e2e-tests/multi-storage/compare/find.json /tmp/tmp.i5FaNrGCBi/find
+ sleep_with_log 60 'wait resync to finish'
+ local d=60
+ local 'msg=wait resync to finish'
+ log 'wait resync to finish. sleeping for 60 seconds'
+ set +o xtrace
[2025-08-07T09:13:21+0000] wait resync to finish. sleeping for 60 seconds
+ sleep 60
+ log 'changing main storage from minio-1 to minio-2'
+ set +o xtrace
[2025-08-07T09:14:21+0000] changing main storage from minio-1 to minio-2
+ kubectl patch psmdb some-name --type=json '-p=[
  {"op": "remove", "path": "/spec/backup/storages/minio-1/main"},
  {"op": "add", "path": "/spec/backup/storages/minio-2/main", "value": true}
]'
perconaservermongodb.psmdb.percona.com/some-name patched
+ sleep_with_log 60 'wait resync to finish'
+ local d=60
+ local 'msg=wait resync to finish'
+ log 'wait resync to finish. sleeping for 60 seconds'
+ set +o xtrace
[2025-08-07T09:14:22+0000] wait resync to finish. sleeping for 60 seconds
+ sleep 60
+ run_backup minio-2 backup-minio-2-1 logical
+ local storage=minio-2
+ local name=backup-minio-2-1
+ local type=logical
+ log 'creating backup: backup-minio-2-1 (logical)'
+ set +o xtrace
[2025-08-07T09:15:22+0000] creating backup: backup-minio-2-1 (logical)
+ yq eval '.metadata.name = "backup-minio-2-1"
			| .spec.storageName = "minio-2"
			| .spec.type = "logical"' /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2009/e2e-tests/multi-storage/conf/backup.yml
+ kubectl_bin apply -f -
++ mktemp
+ local LAST_OUT=/tmp/tmp.G9eRBy6Ysg
++ mktemp
+ local LAST_ERR=/tmp/tmp.2MZdfijNF4
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl apply -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.G9eRBy6Ysg
perconaservermongodbbackup.psmdb.percona.com/backup-minio-2-1 created
+ cat /tmp/tmp.2MZdfijNF4
+ rm /tmp/tmp.G9eRBy6Ysg /tmp/tmp.2MZdfijNF4
+ return 0
+ wait_backup backup-minio-2-1
+ local backup_name=backup-minio-2-1
+ local target_state=ready
+ set +o xtrace
waiting for backup-minio-2-1 to reach ready state.....
+ write_document 100506
+ local x=100506
+ local cmp_postfix=
+ log 'write document: 100506'
+ set +o xtrace
[2025-08-07T09:15:35+0000] write document: 100506
+ run_mongo 'use myApp\n db.test.insert({ x: 100506 })' myApp:myPass@some-name-rs0.multi-storage-17828 mongodb
+ local 'command=use myApp\n db.test.insert({ x: 100506 })'
+ local uri=myApp:myPass@some-name-rs0.multi-storage-17828
+ local driver=mongodb
+ local suffix=.svc.cluster.local
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.5riV4WyvIJ
+++ mktemp
++ local LAST_ERR=/tmp/tmp.b1meKQDmdD
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.5riV4WyvIJ
++ cat /tmp/tmp.b1meKQDmdD
++ rm /tmp/tmp.5riV4WyvIJ /tmp/tmp.b1meKQDmdD
++ return 0
+ local client_container=psmdb-client-66f577db5f-cfk8q
+ local mongo_flag=
+ [[ myApp:myPass@some-name-rs0.multi-storage-17828 == *cfg* ]]
+ replica_set=rs0
+ kubectl_bin exec psmdb-client-66f577db5f-cfk8q -- bash -c 'printf '\''use myApp\n db.test.insert({ x: 100506 })\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0.multi-storage-17828.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
++ mktemp
+ local LAST_OUT=/tmp/tmp.kJfK6ggQRP
++ mktemp
+ local LAST_ERR=/tmp/tmp.aXtsHrocBo
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-66f577db5f-cfk8q -- bash -c 'printf '\''use myApp\n db.test.insert({ x: 100506 })\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0.multi-storage-17828.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.kJfK6ggQRP
Percona Server for MongoDB shell version v4.4.29-28
connecting to: mongodb://some-name-rs0.multi-storage-17828.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb&replicaSet=rs0&ssl=false
Implicit session: session { "id" : UUID("65b59276-0281-4bad-b4c7-35bc5737a3b1") }
Percona Server for MongoDB server version: v7.0.22-12
WARNING: shell and server versions do not match
switched to db myApp
WriteResult({ "nInserted" : 1 })
bye
+ cat /tmp/tmp.aXtsHrocBo
+ rm /tmp/tmp.kJfK6ggQRP /tmp/tmp.aXtsHrocBo
+ return 0
+ write_document 100507
+ local x=100507
+ local cmp_postfix=
+ log 'write document: 100507'
+ set +o xtrace
[2025-08-07T09:15:38+0000] write document: 100507
+ run_mongo 'use myApp\n db.test.insert({ x: 100507 })' myApp:myPass@some-name-rs0.multi-storage-17828 mongodb
+ local 'command=use myApp\n db.test.insert({ x: 100507 })'
+ local uri=myApp:myPass@some-name-rs0.multi-storage-17828
+ local driver=mongodb
+ local suffix=.svc.cluster.local
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.ajyH83BcIN
+++ mktemp
++ local LAST_ERR=/tmp/tmp.hN9DTFwUOb
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.ajyH83BcIN
++ cat /tmp/tmp.hN9DTFwUOb
++ rm /tmp/tmp.ajyH83BcIN /tmp/tmp.hN9DTFwUOb
++ return 0
+ local client_container=psmdb-client-66f577db5f-cfk8q
+ local mongo_flag=
+ [[ myApp:myPass@some-name-rs0.multi-storage-17828 == *cfg* ]]
+ replica_set=rs0
+ kubectl_bin exec psmdb-client-66f577db5f-cfk8q -- bash -c 'printf '\''use myApp\n db.test.insert({ x: 100507 })\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0.multi-storage-17828.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
++ mktemp
+ local LAST_OUT=/tmp/tmp.CpxVgZryEp
++ mktemp
+ local LAST_ERR=/tmp/tmp.evjwWpwcSO
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-66f577db5f-cfk8q -- bash -c 'printf '\''use myApp\n db.test.insert({ x: 100507 })\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0.multi-storage-17828.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.CpxVgZryEp
Percona Server for MongoDB shell version v4.4.29-28
connecting to: mongodb://some-name-rs0.multi-storage-17828.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb&replicaSet=rs0&ssl=false
Implicit session: session { "id" : UUID("75c4baa9-906a-42a8-befe-5d31614e8136") }
Percona Server for MongoDB server version: v7.0.22-12
WARNING: shell and server versions do not match
switched to db myApp
WriteResult({ "nInserted" : 1 })
bye
+ cat /tmp/tmp.evjwWpwcSO
+ rm /tmp/tmp.CpxVgZryEp /tmp/tmp.evjwWpwcSO
+ return 0
+ write_document 100508
+ local x=100508
+ local cmp_postfix=
+ log 'write document: 100508'
+ set +o xtrace
[2025-08-07T09:15:40+0000] write document: 100508
+ run_mongo 'use myApp\n db.test.insert({ x: 100508 })' myApp:myPass@some-name-rs0.multi-storage-17828 mongodb
+ local 'command=use myApp\n db.test.insert({ x: 100508 })'
+ local uri=myApp:myPass@some-name-rs0.multi-storage-17828
+ local driver=mongodb
+ local suffix=.svc.cluster.local
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.FjUS0G23Ja
+++ mktemp
++ local LAST_ERR=/tmp/tmp.L2ABJ9hjfb
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.FjUS0G23Ja
++ cat /tmp/tmp.L2ABJ9hjfb
++ rm /tmp/tmp.FjUS0G23Ja /tmp/tmp.L2ABJ9hjfb
++ return 0
+ local client_container=psmdb-client-66f577db5f-cfk8q
+ local mongo_flag=
+ [[ myApp:myPass@some-name-rs0.multi-storage-17828 == *cfg* ]]
+ replica_set=rs0
+ kubectl_bin exec psmdb-client-66f577db5f-cfk8q -- bash -c 'printf '\''use myApp\n db.test.insert({ x: 100508 })\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0.multi-storage-17828.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
++ mktemp
+ local LAST_OUT=/tmp/tmp.jfmktdNPFn
++ mktemp
+ local LAST_ERR=/tmp/tmp.9QHcp4784j
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-66f577db5f-cfk8q -- bash -c 'printf '\''use myApp\n db.test.insert({ x: 100508 })\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0.multi-storage-17828.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.jfmktdNPFn
Percona Server for MongoDB shell version v4.4.29-28
connecting to: mongodb://some-name-rs0.multi-storage-17828.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb&replicaSet=rs0&ssl=false
Implicit session: session { "id" : UUID("27481a8f-ccf6-447a-ad4b-d04b4dd29137") }
Percona Server for MongoDB server version: v7.0.22-12
WARNING: shell and server versions do not match
switched to db myApp
WriteResult({ "nInserted" : 1 })
bye
+ cat /tmp/tmp.9QHcp4784j
+ rm /tmp/tmp.jfmktdNPFn /tmp/tmp.9QHcp4784j
+ return 0
+ write_document 100509
+ local x=100509
+ local cmp_postfix=
+ log 'write document: 100509'
+ set +o xtrace
[2025-08-07T09:15:43+0000] write document: 100509
+ run_mongo 'use myApp\n db.test.insert({ x: 100509 })' myApp:myPass@some-name-rs0.multi-storage-17828 mongodb
+ local 'command=use myApp\n db.test.insert({ x: 100509 })'
+ local uri=myApp:myPass@some-name-rs0.multi-storage-17828
+ local driver=mongodb
+ local suffix=.svc.cluster.local
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.VrWI75H0fX
+++ mktemp
++ local LAST_ERR=/tmp/tmp.fJ5bWmrLUx
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.VrWI75H0fX
++ cat /tmp/tmp.fJ5bWmrLUx
++ rm /tmp/tmp.VrWI75H0fX /tmp/tmp.fJ5bWmrLUx
++ return 0
+ local client_container=psmdb-client-66f577db5f-cfk8q
+ local mongo_flag=
+ [[ myApp:myPass@some-name-rs0.multi-storage-17828 == *cfg* ]]
+ replica_set=rs0
+ kubectl_bin exec psmdb-client-66f577db5f-cfk8q -- bash -c 'printf '\''use myApp\n db.test.insert({ x: 100509 })\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0.multi-storage-17828.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
++ mktemp
+ local LAST_OUT=/tmp/tmp.FEedDa1nG8
++ mktemp
+ local LAST_ERR=/tmp/tmp.IYmvh6bbHL
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-66f577db5f-cfk8q -- bash -c 'printf '\''use myApp\n db.test.insert({ x: 100509 })\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0.multi-storage-17828.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.FEedDa1nG8
Percona Server for MongoDB shell version v4.4.29-28
connecting to: mongodb://some-name-rs0.multi-storage-17828.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb&replicaSet=rs0&ssl=false
Implicit session: session { "id" : UUID("ca0416cd-4cfd-4540-99df-489b0bd5641b") }
Percona Server for MongoDB server version: v7.0.22-12
WARNING: shell and server versions do not match
switched to db myApp
WriteResult({ "nInserted" : 1 })
bye
+ cat /tmp/tmp.IYmvh6bbHL
+ rm /tmp/tmp.FEedDa1nG8 /tmp/tmp.IYmvh6bbHL
+ return 0
+ write_document 100510
+ local x=100510
+ local cmp_postfix=
+ log 'write document: 100510'
+ set +o xtrace
[2025-08-07T09:15:46+0000] write document: 100510
+ run_mongo 'use myApp\n db.test.insert({ x: 100510 })' myApp:myPass@some-name-rs0.multi-storage-17828 mongodb
+ local 'command=use myApp\n db.test.insert({ x: 100510 })'
+ local uri=myApp:myPass@some-name-rs0.multi-storage-17828
+ local driver=mongodb
+ local suffix=.svc.cluster.local
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.OwE0cVEijA
+++ mktemp
++ local LAST_ERR=/tmp/tmp.ZMebwHzIuF
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.OwE0cVEijA
++ cat /tmp/tmp.ZMebwHzIuF
++ rm /tmp/tmp.OwE0cVEijA /tmp/tmp.ZMebwHzIuF
++ return 0
+ local client_container=psmdb-client-66f577db5f-cfk8q
+ local mongo_flag=
+ [[ myApp:myPass@some-name-rs0.multi-storage-17828 == *cfg* ]]
+ replica_set=rs0
+ kubectl_bin exec psmdb-client-66f577db5f-cfk8q -- bash -c 'printf '\''use myApp\n db.test.insert({ x: 100510 })\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0.multi-storage-17828.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
++ mktemp
+ local LAST_OUT=/tmp/tmp.LwyelvPl4P
++ mktemp
+ local LAST_ERR=/tmp/tmp.AWtcItfOZ4
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-66f577db5f-cfk8q -- bash -c 'printf '\''use myApp\n db.test.insert({ x: 100510 })\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0.multi-storage-17828.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.LwyelvPl4P
Percona Server for MongoDB shell version v4.4.29-28
connecting to: mongodb://some-name-rs0.multi-storage-17828.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb&replicaSet=rs0&ssl=false
Implicit session: session { "id" : UUID("1935b89d-9777-42d4-815a-953b4ac76e96") }
Percona Server for MongoDB server version: v7.0.22-12
WARNING: shell and server versions do not match
switched to db myApp
WriteResult({ "nInserted" : 1 })
bye
+ cat /tmp/tmp.AWtcItfOZ4
+ rm /tmp/tmp.LwyelvPl4P /tmp/tmp.AWtcItfOZ4
+ return 0
+ compare_mongo_cmd find myApp:myPass@some-name-rs0.multi-storage-17828 -2nd .svc.cluster.local myApp test 'sort( { x: 1 } )'
+ local command=find
+ local uri=myApp:myPass@some-name-rs0.multi-storage-17828
+ local postfix=-2nd
+ local suffix=.svc.cluster.local
+ local database=myApp
+ local collection=test
+ local 'sort=sort( { x: 1 } )'
+ local 'full_command=db.test.find()'
+ [[ ! -z sort( { x: 1 } ) ]]
+ full_command='db.test.find().sort( { x: 1 } )'
+ log 'running db.test.find().sort( { x: 1 } ) in myApp'
+ set +o xtrace
[2025-08-07T09:15:48+0000] running db.test.find().sort( { x: 1 } ) in myApp
+ run_mongo 'use myApp\n db.test.find().sort( { x: 1 } )' myApp:myPass@some-name-rs0.multi-storage-17828 mongodb .svc.cluster.local
+ egrep -v 'I NETWORK|W NETWORK|F NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:'
+ local 'command=use myApp\n db.test.find().sort( { x: 1 } )'
+ local uri=myApp:myPass@some-name-rs0.multi-storage-17828
+ local driver=mongodb
+ /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
+ local suffix=.svc.cluster.local
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
egrep: warning: egrep is obsolescent; using grep -E
++ local LAST_OUT=/tmp/tmp.pmBe9LP3xR
+++ mktemp
++ local LAST_ERR=/tmp/tmp.6tfLwtC80a
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.pmBe9LP3xR
++ cat /tmp/tmp.6tfLwtC80a
++ rm /tmp/tmp.pmBe9LP3xR /tmp/tmp.6tfLwtC80a
++ return 0
+ local client_container=psmdb-client-66f577db5f-cfk8q
+ local mongo_flag=
+ [[ myApp:myPass@some-name-rs0.multi-storage-17828 == *cfg* ]]
+ replica_set=rs0
+ kubectl_bin exec psmdb-client-66f577db5f-cfk8q -- bash -c 'printf '\''use myApp\n db.test.find().sort( { x: 1 } )\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0.multi-storage-17828.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
++ mktemp
+ local LAST_OUT=/tmp/tmp.eHkMhCZoLR
++ mktemp
+ local LAST_ERR=/tmp/tmp.o01pmMTQTr
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-66f577db5f-cfk8q -- bash -c 'printf '\''use myApp\n db.test.find().sort( { x: 1 } )\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0.multi-storage-17828.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.eHkMhCZoLR
+ cat /tmp/tmp.o01pmMTQTr
+ rm /tmp/tmp.eHkMhCZoLR /tmp/tmp.o01pmMTQTr
+ return 0
+ [[ 0 -eq 0 ]]
+ diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2009/e2e-tests/multi-storage/compare/find-2nd.json /tmp/tmp.i5FaNrGCBi/find-2nd
+ wait_for_restorable_time backup-minio-2-1
+ local backup_name=backup-minio-2-1
++ get_latest_restorable_time backup-minio-2-1
++ local backup_name=backup-minio-2-1
+++ kubectl_bin get psmdb-backup backup-minio-2-1 -o yaml
+++ yq .status.latestRestorableTime
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.E4Kc39PDqs
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.lSVWYX9UHH
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in $(seq 0 2)
+++ set +e
+++ kubectl get psmdb-backup backup-minio-2-1 -o yaml
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.E4Kc39PDqs
+++ cat /tmp/tmp.lSVWYX9UHH
+++ rm /tmp/tmp.E4Kc39PDqs /tmp/tmp.lSVWYX9UHH
+++ return 0
++ local time=2025-08-07T09:15:26Z
++ [[ 2025-08-07T09:15:26Z == \n\u\l\l ]]
++ echo -n 2025-08-07T09:15:26Z
+ local latest_restorable_time=2025-08-07T09:15:26Z
+ local retries=0
+ echo -n 'waiting for psmdb-backup/backup-minio-2-1'\''s latest restorable time'
waiting for psmdb-backup/backup-minio-2-1's latest restorable time+ [[ 2025-08-07T09:15:26Z != \n\u\l\l ]]
+ echo '.OK latestRestorableTime: 2025-08-07T09:15:26Z'
.OK latestRestorableTime: 2025-08-07T09:15:26Z
+ wait_for_advanced_restorable_time backup-minio-2-1
+ local backup_name=backup-minio-2-1
+++ get_latest_restorable_time backup-minio-2-1
+++ local backup_name=backup-minio-2-1
++++ kubectl_bin get psmdb-backup backup-minio-2-1 -o yaml
++++ yq .status.latestRestorableTime
+++++ mktemp
++++ local LAST_OUT=/tmp/tmp.Jn7RxdS9jI
+++++ mktemp
++++ local LAST_ERR=/tmp/tmp.Cc3X3cNu5W
++++ local exit_status=0
++++ local timeout=4
+++++ seq 0 2
++++ for i in $(seq 0 2)
++++ set +e
++++ kubectl get psmdb-backup backup-minio-2-1 -o yaml
++++ exit_status=0
++++ set -e
++++ '[' 0 '!=' 0 -a -n 1 ']'
++++ break
++++ cat /tmp/tmp.Jn7RxdS9jI
++++ cat /tmp/tmp.Cc3X3cNu5W
++++ rm /tmp/tmp.Jn7RxdS9jI /tmp/tmp.Cc3X3cNu5W
++++ return 0
+++ local time=2025-08-07T09:15:26Z
+++ [[ 2025-08-07T09:15:26Z == \n\u\l\l ]]
+++ echo -n 2025-08-07T09:15:26Z
++ datetime_to_timestamp 2025-08-07T09:15:26Z
++ local datetime=2025-08-07T09:15:26Z
++ TZ=UTC
++ /usr/sbin/date -d2025-08-07T09:15:26Z +%s
+ local latest_restorable_time=1754558126
++ get_backup_last_write backup-minio-2-1
++ local backup_name=backup-minio-2-1
+++ kubectl_bin get psmdb-backup backup-minio-2-1 -o yaml
+++ yq .status.pbmName
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.s4onqNq5G4
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.qRmQUn1PU2
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in $(seq 0 2)
+++ set +e
+++ kubectl get psmdb-backup backup-minio-2-1 -o yaml
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.s4onqNq5G4
+++ cat /tmp/tmp.qRmQUn1PU2
+++ rm /tmp/tmp.s4onqNq5G4 /tmp/tmp.qRmQUn1PU2
+++ return 0
++ local pbm_name=2025-08-07T09:15:25Z
++ kubectl_bin exec some-name-rs0-0 -c backup-agent -- pbm list -o json
++ jq --arg v 2025-08-07T09:15:25Z '.snapshots[] | select(.name==$v) | .restoreTo'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.6tBQfeQDgA
+++ mktemp
++ local LAST_ERR=/tmp/tmp.GbTYo9lekc
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl exec some-name-rs0-0 -c backup-agent -- pbm list -o json
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.6tBQfeQDgA
++ cat /tmp/tmp.GbTYo9lekc
++ rm /tmp/tmp.6tBQfeQDgA /tmp/tmp.GbTYo9lekc
++ return 0
+ local latest_write=1754558131
+ local retries=0
++ timestamp_to_datetime 1754558131
++ local ts=1754558131
++ TZ=UTC
++ /usr/sbin/date -d@1754558131 +%Y-%m-%dT%H:%M:%S%Z
+ echo -n 'waiting for psmdb-backup/backup-minio-2-1'\''s latest restorable time to advance its latest write (2025-08-07T09:15:31UTC)'
waiting for psmdb-backup/backup-minio-2-1's latest restorable time to advance its latest write (2025-08-07T09:15:31UTC)+ [[ 1754558126 -gt 1754558131 ]]
+++ get_latest_restorable_time backup-minio-2-1
+++ local backup_name=backup-minio-2-1
++++ kubectl_bin get psmdb-backup backup-minio-2-1 -o yaml
++++ yq .status.latestRestorableTime
+++++ mktemp
++++ local LAST_OUT=/tmp/tmp.4OVPR8loOH
+++++ mktemp
++++ local LAST_ERR=/tmp/tmp.Xp79QmgnVV
++++ local exit_status=0
++++ local timeout=4
+++++ seq 0 2
++++ for i in $(seq 0 2)
++++ set +e
++++ kubectl get psmdb-backup backup-minio-2-1 -o yaml
++++ exit_status=0
++++ set -e
++++ '[' 0 '!=' 0 -a -n 1 ']'
++++ break
++++ cat /tmp/tmp.4OVPR8loOH
++++ cat /tmp/tmp.Xp79QmgnVV
++++ rm /tmp/tmp.4OVPR8loOH /tmp/tmp.Xp79QmgnVV
++++ return 0
+++ local time=2025-08-07T09:15:26Z
+++ [[ 2025-08-07T09:15:26Z == \n\u\l\l ]]
+++ echo -n 2025-08-07T09:15:26Z
++ datetime_to_timestamp 2025-08-07T09:15:26Z
++ local datetime=2025-08-07T09:15:26Z
++ TZ=UTC
++ /usr/sbin/date -d2025-08-07T09:15:26Z +%s
+ latest_restorable_time=1754558126
+ [[ 0 -gt 5 ]]
+ let retries+=1
+ echo -n .
.+ sleep_with_log 70 'wait for oplog span min + 10 sec'
+ local d=70
+ local 'msg=wait for oplog span min + 10 sec'
+ log 'wait for oplog span min + 10 sec. sleeping for 70 seconds'
+ set +o xtrace
[2025-08-07T09:15:55+0000] wait for oplog span min + 10 sec. sleeping for 70 seconds
+ sleep 70
+ [[ 1754558126 -gt 1754558131 ]]
+++ get_latest_restorable_time backup-minio-2-1
+++ local backup_name=backup-minio-2-1
++++ kubectl_bin get psmdb-backup backup-minio-2-1 -o yaml
++++ yq .status.latestRestorableTime
+++++ mktemp
++++ local LAST_OUT=/tmp/tmp.iuj0I7lOlh
+++++ mktemp
++++ local LAST_ERR=/tmp/tmp.8WyssIXEcd
++++ local exit_status=0
++++ local timeout=4
+++++ seq 0 2
++++ for i in $(seq 0 2)
++++ set +e
++++ kubectl get psmdb-backup backup-minio-2-1 -o yaml
++++ exit_status=0
++++ set -e
++++ '[' 0 '!=' 0 -a -n 1 ']'
++++ break
++++ cat /tmp/tmp.iuj0I7lOlh
++++ cat /tmp/tmp.8WyssIXEcd
++++ rm /tmp/tmp.iuj0I7lOlh /tmp/tmp.8WyssIXEcd
++++ return 0
+++ local time=2025-08-07T09:16:59Z
+++ [[ 2025-08-07T09:16:59Z == \n\u\l\l ]]
+++ echo -n 2025-08-07T09:16:59Z
++ datetime_to_timestamp 2025-08-07T09:16:59Z
++ local datetime=2025-08-07T09:16:59Z
++ TZ=UTC
++ /usr/sbin/date -d2025-08-07T09:16:59Z +%s
+ latest_restorable_time=1754558219
+ [[ 1 -gt 5 ]]
+ let retries+=1
+ echo -n .
.+ sleep_with_log 70 'wait for oplog span min + 10 sec'
+ local d=70
+ local 'msg=wait for oplog span min + 10 sec'
+ log 'wait for oplog span min + 10 sec. sleeping for 70 seconds'
+ set +o xtrace
[2025-08-07T09:17:06+0000] wait for oplog span min + 10 sec. sleeping for 70 seconds
+ sleep 70
+ [[ 1754558219 -gt 1754558131 ]]
++ timestamp_to_datetime 1754558219
++ local ts=1754558219
++ TZ=UTC
++ /usr/sbin/date -d@1754558219 +%Y-%m-%dT%H:%M:%S%Z
+ echo '.OK latestRestorableTime: 2025-08-07T09:16:59UTC'
.OK latestRestorableTime: 2025-08-07T09:16:59UTC
++ get_latest_restorable_time backup-minio-2-1
++ local backup_name=backup-minio-2-1
+++ kubectl_bin get psmdb-backup backup-minio-2-1 -o yaml
+++ yq .status.latestRestorableTime
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.h7Htjhac6u
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.zum14XdnSG
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in $(seq 0 2)
+++ set +e
+++ kubectl get psmdb-backup backup-minio-2-1 -o yaml
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.h7Htjhac6u
+++ cat /tmp/tmp.zum14XdnSG
+++ rm /tmp/tmp.h7Htjhac6u /tmp/tmp.zum14XdnSG
+++ return 0
++ local time=2025-08-07T09:17:59Z
++ [[ 2025-08-07T09:17:59Z == \n\u\l\l ]]
++ echo -n 2025-08-07T09:17:59Z
+ restore_time=2025-08-07T09:17:59Z
++ format_pitr_target 2025-08-07T09:17:59Z
++ local target=2025-08-07T09:17:59Z
++ echo 2025-08-07T09:17:59Z
++ sed 's/T/ /'
++ sed s/Z//
+ check_recovery backup-minio-2-1 '2025-08-07 09:17:59' -2nd
+ local backup_name=backup-minio-2-1
+ local 'restore_time=2025-08-07 09:17:59'
+ local cmp_postfix=-2nd
+ drop_collection myApp test myApp myPass
+ local db=myApp
+ local collection=test
+ local user=myApp
+ local pass=myPass
+ log 'dropping collection: myApp.test'
+ set +o xtrace
[2025-08-07T09:18:18+0000] dropping collection: myApp.test
+ run_mongo 'use myApp\n db.test.drop()' myApp:myPass@some-name-rs0.multi-storage-17828 mongodb
+ local 'command=use myApp\n db.test.drop()'
+ local uri=myApp:myPass@some-name-rs0.multi-storage-17828
+ local driver=mongodb
+ local suffix=.svc.cluster.local
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.cWq7WfUWcf
+++ mktemp
++ local LAST_ERR=/tmp/tmp.usx4xnST3r
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.cWq7WfUWcf
++ cat /tmp/tmp.usx4xnST3r
++ rm /tmp/tmp.cWq7WfUWcf /tmp/tmp.usx4xnST3r
++ return 0
+ local client_container=psmdb-client-66f577db5f-cfk8q
+ local mongo_flag=
+ [[ myApp:myPass@some-name-rs0.multi-storage-17828 == *cfg* ]]
+ replica_set=rs0
+ kubectl_bin exec psmdb-client-66f577db5f-cfk8q -- bash -c 'printf '\''use myApp\n db.test.drop()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0.multi-storage-17828.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
++ mktemp
+ local LAST_OUT=/tmp/tmp.Fbsxsnl20p
++ mktemp
+ local LAST_ERR=/tmp/tmp.bQxDS4y69y
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-66f577db5f-cfk8q -- bash -c 'printf '\''use myApp\n db.test.drop()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0.multi-storage-17828.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.Fbsxsnl20p
Percona Server for MongoDB shell version v4.4.29-28
connecting to: mongodb://some-name-rs0.multi-storage-17828.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb&replicaSet=rs0&ssl=false
Implicit session: session { "id" : UUID("e5b0378b-d045-483f-a972-b9286f16a227") }
Percona Server for MongoDB server version: v7.0.22-12
WARNING: shell and server versions do not match
switched to db myApp
true
bye
+ cat /tmp/tmp.bQxDS4y69y
+ rm /tmp/tmp.Fbsxsnl20p /tmp/tmp.bQxDS4y69y
+ return 0
+ log 'creating restore: restore-backup-minio-2-1 restore_to: 2025-08-07 09:17:59'
+ set +o xtrace
[2025-08-07T09:18:20+0000] creating restore: restore-backup-minio-2-1 restore_to: 2025-08-07 09:17:59
+ cat /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2009/e2e-tests/multi-storage/conf/restore.yml
+ /usr/sbin/sed -e 's/name:/name: restore-backup-minio-2-1/'
+ /usr/sbin/sed -e 's/backupName:/backupName: backup-minio-2-1/'
+ /usr/sbin/sed -e 's/date:/date: 2025-08-07 09:17:59/'
+ kubectl_bin apply -f -
++ mktemp
+ local LAST_OUT=/tmp/tmp.PUWLwPnpvj
++ mktemp
+ local LAST_ERR=/tmp/tmp.8WuvCJqjqm
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl apply -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.PUWLwPnpvj
perconaservermongodbrestore.psmdb.percona.com/restore-backup-minio-2-1 created
+ cat /tmp/tmp.8WuvCJqjqm
+ rm /tmp/tmp.PUWLwPnpvj /tmp/tmp.8WuvCJqjqm
+ return 0
+ wait_restore backup-minio-2-1 some-name requested 0 900 1
+ local backup_name=backup-minio-2-1
+ local cluster_name=some-name
+ local target_state=requested
+ local wait_cluster_consistency=0
+ local wait_time=900
+ local ok_if_ready=1
+ set +o xtrace
Waiting for the psmdb-restore/restore-backup-minio-2-1 object to be createdOK
Waiting psmdb-restore/restore-backup-minio-2-1 to reach state "requested" OK after 0 minutes
+ [[ 0 -eq 1 ]]
+ wait_restore backup-minio-2-1 some-name ready 0 1600
+ local backup_name=backup-minio-2-1
+ local cluster_name=some-name
+ local target_state=ready
+ local wait_cluster_consistency=0
+ local wait_time=1600
+ local ok_if_ready=0
+ set +o xtrace
Waiting for the psmdb-restore/restore-backup-minio-2-1 object to be createdOK
Waiting psmdb-restore/restore-backup-minio-2-1 to reach state "ready" OK after 0 minutes
+ [[ 0 -eq 1 ]]
+ wait_for_running some-name-rs0 3
+ local name=some-name-rs0
+ let last_pod=2
+ local check_cluster_readyness=true
+ set_debug
+ [[ 1 == 1 ]]
+ set -o xtrace
+ local rs_name=rs0
+ local cluster_name=some-name
++ seq 0 2
+ for i in $(seq 0 $last_pod)
+ [[ 0 -eq 2 ]]
+ wait_pod some-name-rs0-0
+ local pod=some-name-rs0-0
+ set +o xtrace
waiting for pod/some-name-rs0-0 to be ready.OK
+ for i in $(seq 0 $last_pod)
+ [[ 1 -eq 2 ]]
+ wait_pod some-name-rs0-1
+ local pod=some-name-rs0-1
+ set +o xtrace
waiting for pod/some-name-rs0-1 to be ready.OK
+ for i in $(seq 0 $last_pod)
+ [[ 2 -eq 2 ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].arbiter.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.UUKK14bnzv
+++ mktemp
++ local LAST_ERR=/tmp/tmp.wSuDdY0erv
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].arbiter.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.UUKK14bnzv
++ cat /tmp/tmp.wSuDdY0erv
++ rm /tmp/tmp.UUKK14bnzv /tmp/tmp.wSuDdY0erv
++ return 0
+ [[ '' == \t\r\u\e ]]
+ wait_pod some-name-rs0-2
+ local pod=some-name-rs0-2
+ set +o xtrace
waiting for pod/some-name-rs0-2 to be ready.OK
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].non_voting.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.W2fpYa3eTO
+++ mktemp
++ local LAST_ERR=/tmp/tmp.uBt1QqJeMd
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].non_voting.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.W2fpYa3eTO
++ cat /tmp/tmp.uBt1QqJeMd
++ rm /tmp/tmp.W2fpYa3eTO /tmp/tmp.uBt1QqJeMd
++ return 0
+ [[ '' == \t\r\u\e ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].hidden.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.oqnIxDXymB
+++ mktemp
++ local LAST_ERR=/tmp/tmp.uCsOLMZHgN
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].hidden.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.oqnIxDXymB
++ cat /tmp/tmp.uCsOLMZHgN
++ rm /tmp/tmp.oqnIxDXymB /tmp/tmp.uCsOLMZHgN
++ return 0
+ [[ '' == \t\r\u\e ]]
+ sleep 10
+ [[ true == \t\r\u\e ]]
+ set +x
Waiting for cluster readyness
+ compare_mongo_cmd find myApp:myPass@some-name-rs0.multi-storage-17828 -2nd .svc.cluster.local myApp test 'sort( { x: 1 } )'
+ local command=find
+ local uri=myApp:myPass@some-name-rs0.multi-storage-17828
+ local postfix=-2nd
+ local suffix=.svc.cluster.local
+ local database=myApp
+ local collection=test
+ local 'sort=sort( { x: 1 } )'
+ local 'full_command=db.test.find()'
+ [[ ! -z sort( { x: 1 } ) ]]
+ full_command='db.test.find().sort( { x: 1 } )'
+ log 'running db.test.find().sort( { x: 1 } ) in myApp'
+ set +o xtrace
[2025-08-07T09:19:01+0000] running db.test.find().sort( { x: 1 } ) in myApp
+ run_mongo 'use myApp\n db.test.find().sort( { x: 1 } )' myApp:myPass@some-name-rs0.multi-storage-17828 mongodb .svc.cluster.local
+ local 'command=use myApp\n db.test.find().sort( { x: 1 } )'
+ local uri=myApp:myPass@some-name-rs0.multi-storage-17828
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ egrep -v 'I NETWORK|W NETWORK|F NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:'
+ /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
egrep: warning: egrep is obsolescent; using grep -E
++ local LAST_OUT=/tmp/tmp.lPvCOsQhf2
+++ mktemp
++ local LAST_ERR=/tmp/tmp.7u1BzH34M2
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.lPvCOsQhf2
++ cat /tmp/tmp.7u1BzH34M2
++ rm /tmp/tmp.lPvCOsQhf2 /tmp/tmp.7u1BzH34M2
++ return 0
+ local client_container=psmdb-client-66f577db5f-cfk8q
+ local mongo_flag=
+ [[ myApp:myPass@some-name-rs0.multi-storage-17828 == *cfg* ]]
+ replica_set=rs0
+ kubectl_bin exec psmdb-client-66f577db5f-cfk8q -- bash -c 'printf '\''use myApp\n db.test.find().sort( { x: 1 } )\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0.multi-storage-17828.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
++ mktemp
+ local LAST_OUT=/tmp/tmp.wtYZrgS54i
++ mktemp
+ local LAST_ERR=/tmp/tmp.usCPwAWs1p
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-66f577db5f-cfk8q -- bash -c 'printf '\''use myApp\n db.test.find().sort( { x: 1 } )\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0.multi-storage-17828.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.wtYZrgS54i
+ cat /tmp/tmp.usCPwAWs1p
+ rm /tmp/tmp.wtYZrgS54i /tmp/tmp.usCPwAWs1p
+ return 0
+ [[ 0 -eq 0 ]]
+ diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2009/e2e-tests/multi-storage/compare/find-2nd.json /tmp/tmp.i5FaNrGCBi/find-2nd
+ log 'dropping pbmBackups collection and starting resync'
+ set +o xtrace
[2025-08-07T09:19:04+0000] dropping pbmBackups collection and starting resync
+ drop_metadata_and_resync
++ kubectl get psmdb-backup --no-headers
++ wc -l
+ local psmdb_backup_count=4
+ log 'there are 4 psmdb-backups in multi-storage-17828'
+ set +o xtrace
[2025-08-07T09:19:05+0000] there are 4 psmdb-backups in multi-storage-17828
+ drop_collection admin pbmBackups backup backup123456
+ local db=admin
+ local collection=pbmBackups
+ local user=backup
+ local pass=backup123456
+ log 'dropping collection: admin.pbmBackups'
+ set +o xtrace
[2025-08-07T09:19:05+0000] dropping collection: admin.pbmBackups
+ run_mongo 'use admin\n db.pbmBackups.drop()' backup:backup123456@some-name-rs0.multi-storage-17828 mongodb
+ local 'command=use admin\n db.pbmBackups.drop()'
+ local uri=backup:backup123456@some-name-rs0.multi-storage-17828
+ local driver=mongodb
+ local suffix=.svc.cluster.local
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.Jho2TVKlyA
+++ mktemp
++ local LAST_ERR=/tmp/tmp.2mPBxKFqbE
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.Jho2TVKlyA
++ cat /tmp/tmp.2mPBxKFqbE
++ rm /tmp/tmp.Jho2TVKlyA /tmp/tmp.2mPBxKFqbE
++ return 0
+ local client_container=psmdb-client-66f577db5f-cfk8q
+ local mongo_flag=
+ [[ backup:backup123456@some-name-rs0.multi-storage-17828 == *cfg* ]]
+ replica_set=rs0
+ kubectl_bin exec psmdb-client-66f577db5f-cfk8q -- bash -c 'printf '\''use admin\n db.pbmBackups.drop()\n'\'' | mongo mongodb://backup:backup123456@some-name-rs0.multi-storage-17828.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
++ mktemp
+ local LAST_OUT=/tmp/tmp.PFBIcYQAYF
++ mktemp
+ local LAST_ERR=/tmp/tmp.7soATmyWFM
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-66f577db5f-cfk8q -- bash -c 'printf '\''use admin\n db.pbmBackups.drop()\n'\'' | mongo mongodb://backup:backup123456@some-name-rs0.multi-storage-17828.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.PFBIcYQAYF
Percona Server for MongoDB shell version v4.4.29-28
connecting to: mongodb://some-name-rs0.multi-storage-17828.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb&replicaSet=rs0&ssl=false
Implicit session: session { "id" : UUID("9363f83a-6477-4547-9229-a05e5f5cbaab") }
Percona Server for MongoDB server version: v7.0.22-12
WARNING: shell and server versions do not match
switched to db admin
true
bye
+ cat /tmp/tmp.7soATmyWFM
+ rm /tmp/tmp.PFBIcYQAYF /tmp/tmp.7soATmyWFM
+ return 0
+ kubectl_bin annotate psmdb some-name percona.com/resync-pbm=true
++ mktemp
+ local LAST_OUT=/tmp/tmp.X5s2BxOpS1
++ mktemp
+ local LAST_ERR=/tmp/tmp.T439Zwv6Nb
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl annotate psmdb some-name percona.com/resync-pbm=true
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.X5s2BxOpS1
perconaservermongodb.psmdb.percona.com/some-name annotated
+ cat /tmp/tmp.T439Zwv6Nb
+ rm /tmp/tmp.X5s2BxOpS1 /tmp/tmp.T439Zwv6Nb
+ return 0
+ sleep_with_log 10 'wait resync to start'
+ local d=10
+ local 'msg=wait resync to start'
+ log 'wait resync to start. sleeping for 10 seconds'
+ set +o xtrace
[2025-08-07T09:19:08+0000] wait resync to start. sleeping for 10 seconds
+ sleep 10
+ local retries=0
+ echo -n 'waiting for resync to finish'
waiting for resync to finish++ kubectl_bin get psmdb some-name -o yaml
++ yq '.metadata.annotations."percona.com/resync-in-progress"'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.evb4qbJNwA
+++ mktemp
++ local LAST_ERR=/tmp/tmp.u3q8L1u4xq
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o yaml
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.evb4qbJNwA
++ cat /tmp/tmp.u3q8L1u4xq
++ rm /tmp/tmp.evb4qbJNwA /tmp/tmp.u3q8L1u4xq
++ return 0
+ local resync_running=null
+ [[ null == \n\u\l\l ]]
+ echo .OK
.OK
++ kubectl exec some-name-rs0-0 -c backup-agent -- pbm list -o json
++ jq '.snapshots | length'
+ local backup_count=4
+ log 'there are 4 backups in pbmBackups collection after resync'
+ set +o xtrace
[2025-08-07T09:19:21+0000] there are 4 backups in pbmBackups collection after resync
++ echo 4
++ xargs
++ echo 4
++ xargs
+ [[ 4 != 4 ]]
+ log 'resync finished successfully'
+ set +o xtrace
[2025-08-07T09:19:21+0000] resync finished successfully
+ log 'checking backup deletion'
+ set +o xtrace
[2025-08-07T09:19:21+0000] checking backup deletion
+ check_backup_deletion
++ kubectl_bin exec some-name-rs0-0 -c backup-agent -- pbm status -o json
++ jq '.backups.pitrChunks.pitrChunks[0].range.start'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.ZXE42CDIQG
+++ mktemp
++ local LAST_ERR=/tmp/tmp.mPUrPS8zy1
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl exec some-name-rs0-0 -c backup-agent -- pbm status -o json
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.ZXE42CDIQG
++ cat /tmp/tmp.mPUrPS8zy1
++ rm /tmp/tmp.ZXE42CDIQG /tmp/tmp.mPUrPS8zy1
++ return 0
+ local initial_chunk_start=1754558126
+ log 'deleting psmdb-backup/backup-minio-3'
+ set +o xtrace
[2025-08-07T09:19:23+0000] deleting psmdb-backup/backup-minio-3
+ kubectl_bin delete psmdb-backup backup-minio-3
++ mktemp
+ local LAST_OUT=/tmp/tmp.zjdOs2qDlD
++ mktemp
+ local LAST_ERR=/tmp/tmp.YBOqTFt79T
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl delete psmdb-backup backup-minio-3
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.zjdOs2qDlD
perconaservermongodbbackup.psmdb.percona.com "backup-minio-3" deleted
+ cat /tmp/tmp.YBOqTFt79T
+ rm /tmp/tmp.zjdOs2qDlD /tmp/tmp.YBOqTFt79T
+ return 0
+ log 'checking if chunks are deleted'
+ set +o xtrace
[2025-08-07T09:19:25+0000] checking if chunks are deleted
++ kubectl_bin exec some-name-rs0-0 -c backup-agent -- pbm status -o json
++ jq '.backups.pitrChunks.pitrChunks[0].range.start'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.sZ3weEBsyD
+++ mktemp
++ local LAST_ERR=/tmp/tmp.l0U7hXIWxT
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl exec some-name-rs0-0 -c backup-agent -- pbm status -o json
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.sZ3weEBsyD
++ cat /tmp/tmp.l0U7hXIWxT
++ rm /tmp/tmp.sZ3weEBsyD /tmp/tmp.l0U7hXIWxT
++ return 0
+ local chunk_start=1754558126
+ [[ 1754558126 != 1754558126 ]]
+ log 'chunks are not deleted, OK.'
+ set +o xtrace
[2025-08-07T09:19:26+0000] chunks are not deleted, OK.
+ log 'deleting psmdb-backup/backup-minio-2-1'
+ set +o xtrace
[2025-08-07T09:19:26+0000] deleting psmdb-backup/backup-minio-2-1
+ kubectl_bin delete psmdb-backup backup-minio-2-1
++ mktemp
+ local LAST_OUT=/tmp/tmp.FSiVgfKuJ4
++ mktemp
+ local LAST_ERR=/tmp/tmp.u9kK8vvzZJ
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl delete psmdb-backup backup-minio-2-1
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.FSiVgfKuJ4
perconaservermongodbbackup.psmdb.percona.com "backup-minio-2-1" deleted
+ cat /tmp/tmp.u9kK8vvzZJ
+ rm /tmp/tmp.FSiVgfKuJ4 /tmp/tmp.u9kK8vvzZJ
+ return 0
+ log 'checking if chunks are deleted'
+ set +o xtrace
[2025-08-07T09:19:28+0000] checking if chunks are deleted
++ kubectl_bin exec some-name-rs0-0 -c backup-agent -- pbm status -o json
++ jq '.backups.pitrChunks.pitrChunks[0].range.start'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.9Adza3cFoI
+++ mktemp
++ local LAST_ERR=/tmp/tmp.17dj2c9rNa
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl exec some-name-rs0-0 -c backup-agent -- pbm status -o json
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.9Adza3cFoI
++ cat /tmp/tmp.17dj2c9rNa
++ rm /tmp/tmp.9Adza3cFoI /tmp/tmp.17dj2c9rNa
++ return 0
+ chunk_start=1754558219
+ [[ 1754558126 == 1754558219 ]]
+ log 'chunks are deleted, OK.'
+ set +o xtrace
[2025-08-07T09:19:29+0000] chunks are deleted, OK.
+ destroy multi-storage-17828
+ local namespace=multi-storage-17828
+ local ignore_logs=true
+ [[ 0 == 1 ]]
+ desc 'destroy cluster/operator and all other resources'
+ set +o xtrace


-----------------------------------------------------------------------------------
destroy cluster/operator and all other resources
-----------------------------------------------------------------------------------

+ '[' true == false ']'
+ delete_backups
+ desc 'Delete psmdb-backup'
+ set +o xtrace


-----------------------------------------------------------------------------------
Delete psmdb-backup
-----------------------------------------------------------------------------------

++ kubectl_bin get psmdb-backup --no-headers
++ wc -l
+++ mktemp
++ local LAST_OUT=/tmp/tmp.l78oT4g9Ud
+++ mktemp
++ local LAST_ERR=/tmp/tmp.14BQpYTApS
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb-backup --no-headers
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.l78oT4g9Ud
++ cat /tmp/tmp.14BQpYTApS
++ rm /tmp/tmp.l78oT4g9Ud /tmp/tmp.14BQpYTApS
++ return 0
+ '[' 2 '!=' 0 ']'
+ kubectl_bin get psmdb-backup
++ mktemp
+ local LAST_OUT=/tmp/tmp.mCjqW8TFXn
++ mktemp
+ local LAST_ERR=/tmp/tmp.Lr8CudtW3e
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl get psmdb-backup
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.mCjqW8TFXn
NAME             CLUSTER     STORAGE   DESTINATION                                    TYPE      SIZE      STATUS   COMPLETED   AGE
backup-minio-1   some-name   minio-1   s3://operator-testing/2025-08-07T09:04:37Z     logical   48.32KB   ready    14m         14m
backup-minio-2   some-name   minio-2   s3://operator-testing-2/2025-08-07T09:04:51Z   logical   52.07KB   ready    14m         14m
+ cat /tmp/tmp.Lr8CudtW3e
+ rm /tmp/tmp.mCjqW8TFXn /tmp/tmp.Lr8CudtW3e
+ return 0
+ kubectl_bin delete psmdb-backup --all
++ mktemp
+ local LAST_OUT=/tmp/tmp.9P61pMTj7p
++ mktemp
+ local LAST_ERR=/tmp/tmp.R3VeVqsj2e
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl delete psmdb-backup --all
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.9P61pMTj7p
perconaservermongodbbackup.psmdb.percona.com "backup-minio-1" deleted
perconaservermongodbbackup.psmdb.percona.com "backup-minio-2" deleted
+ cat /tmp/tmp.R3VeVqsj2e
+ rm /tmp/tmp.9P61pMTj7p /tmp/tmp.R3VeVqsj2e
+ return 0
+ delete_crd
+ desc 'get and delete old CRDs and RBAC'
+ set +o xtrace


-----------------------------------------------------------------------------------
get and delete old CRDs and RBAC
-----------------------------------------------------------------------------------

+ kubectl_bin delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2009/deploy/crd.yaml --ignore-not-found --wait=false
++ mktemp
+ local LAST_OUT=/tmp/tmp.UhEY9GWsga
++ mktemp
+ local LAST_ERR=/tmp/tmp.FmRHtCN27x
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2009/deploy/crd.yaml --ignore-not-found --wait=false
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.UhEY9GWsga
customresourcedefinition.apiextensions.k8s.io "perconaservermongodbbackups.psmdb.percona.com" deleted
customresourcedefinition.apiextensions.k8s.io "perconaservermongodbrestores.psmdb.percona.com" deleted
customresourcedefinition.apiextensions.k8s.io "perconaservermongodbs.psmdb.percona.com" deleted
+ cat /tmp/tmp.FmRHtCN27x
+ rm /tmp/tmp.UhEY9GWsga /tmp/tmp.FmRHtCN27x
+ return 0
++ yq eval .metadata.name /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2009/deploy/crd.yaml
++ grep -v '\-\-\-'
grep: warning: stray \ before -
grep: warning: stray \ before -
+ for crd_name in $(yq eval '.metadata.name' "${src_dir}/deploy/crd.yaml" | grep -v '\-\-\-')
+ kubectl get perconaservermongodbbackups.psmdb.percona.com --all-namespaces -o wide
+ grep -v NAMESPACE
+ xargs -L 1 sh -xc 'kubectl patch perconaservermongodbbackups.psmdb.percona.com -n $0 $1 --type=merge -p "{\"metadata\":{\"finalizers\":[]}}"'
error: the server doesn't have a resource type "perconaservermongodbbackups"
+ kubectl patch perconaservermongodbbackups.psmdb.percona.com -n sh --type=merge -p '{"metadata":{"finalizers":[]}}'
error: the server doesn't have a resource type "perconaservermongodbbackups"
+ :
+ kubectl_bin wait --for=delete crd perconaservermongodbbackups.psmdb.percona.com
++ mktemp
+ local LAST_OUT=/tmp/tmp.8h8yhRrwYJ
++ mktemp
+ local LAST_ERR=/tmp/tmp.HoYSiWamjK
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl wait --for=delete crd perconaservermongodbbackups.psmdb.percona.com
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.8h8yhRrwYJ
+ cat /tmp/tmp.HoYSiWamjK
+ rm /tmp/tmp.8h8yhRrwYJ /tmp/tmp.HoYSiWamjK
+ return 0
+ for crd_name in $(yq eval '.metadata.name' "${src_dir}/deploy/crd.yaml" | grep -v '\-\-\-')
+ kubectl get perconaservermongodbrestores.psmdb.percona.com --all-namespaces -o wide
+ grep -v NAMESPACE
+ xargs -L 1 sh -xc 'kubectl patch perconaservermongodbrestores.psmdb.percona.com -n $0 $1 --type=merge -p "{\"metadata\":{\"finalizers\":[]}}"'
error: the server doesn't have a resource type "perconaservermongodbrestores"
+ kubectl patch perconaservermongodbrestores.psmdb.percona.com -n sh --type=merge -p '{"metadata":{"finalizers":[]}}'
error: the server doesn't have a resource type "perconaservermongodbrestores"
+ :
+ kubectl_bin wait --for=delete crd perconaservermongodbrestores.psmdb.percona.com
++ mktemp
+ local LAST_OUT=/tmp/tmp.wOzeZMhEru
++ mktemp
+ local LAST_ERR=/tmp/tmp.Id3vC8Ijau
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl wait --for=delete crd perconaservermongodbrestores.psmdb.percona.com
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.wOzeZMhEru
+ cat /tmp/tmp.Id3vC8Ijau
+ rm /tmp/tmp.wOzeZMhEru /tmp/tmp.Id3vC8Ijau
+ return 0
+ for crd_name in $(yq eval '.metadata.name' "${src_dir}/deploy/crd.yaml" | grep -v '\-\-\-')
+ kubectl get perconaservermongodbs.psmdb.percona.com --all-namespaces -o wide
+ grep -v NAMESPACE
+ xargs -L 1 sh -xc 'kubectl patch perconaservermongodbs.psmdb.percona.com -n $0 $1 --type=merge -p "{\"metadata\":{\"finalizers\":[]}}"'
error: the server doesn't have a resource type "perconaservermongodbs"
+ kubectl patch perconaservermongodbs.psmdb.percona.com -n sh --type=merge -p '{"metadata":{"finalizers":[]}}'
error: the server doesn't have a resource type "perconaservermongodbs"
+ :
+ kubectl_bin wait --for=delete crd perconaservermongodbs.psmdb.percona.com
++ mktemp
+ local LAST_OUT=/tmp/tmp.d4vcyGGO00
++ mktemp
+ local LAST_ERR=/tmp/tmp.btGJmZE6gg
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl wait --for=delete crd perconaservermongodbs.psmdb.percona.com
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.d4vcyGGO00
+ cat /tmp/tmp.btGJmZE6gg
+ rm /tmp/tmp.d4vcyGGO00 /tmp/tmp.btGJmZE6gg
+ return 0
+ local rbac_yaml=rbac.yaml
+ '[' -n psmdb-operator ']'
+ rbac_yaml=cw-rbac.yaml
+ kubectl_bin delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2009/deploy/cw-rbac.yaml --ignore-not-found
++ mktemp
+ local LAST_OUT=/tmp/tmp.P72gwGG5jp
++ mktemp
+ local LAST_ERR=/tmp/tmp.ordcdtuDbK
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2009/deploy/cw-rbac.yaml --ignore-not-found
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.P72gwGG5jp
clusterrole.rbac.authorization.k8s.io "percona-server-mongodb-operator" deleted
clusterrolebinding.rbac.authorization.k8s.io "service-account-percona-server-mongodb-operator" deleted
+ cat /tmp/tmp.ordcdtuDbK
+ rm /tmp/tmp.P72gwGG5jp /tmp/tmp.ordcdtuDbK
+ return 0
+ destroy_cert_manager
+ kubectl_bin delete -f https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml
++ mktemp
+ local LAST_OUT=/tmp/tmp.8jbGAUfHPG
++ mktemp
+ local LAST_ERR=/tmp/tmp.6X4dLPyI5a
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl delete -f https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml
+ exit_status=1
+ set -e
+ '[' 1 '!=' 0 -a -n 1 ']'
+ cat /tmp/tmp.8jbGAUfHPG
+ cat /tmp/tmp.6X4dLPyI5a
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": namespaces "cert-manager" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "certificaterequests.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "certificates.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "challenges.acme.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "clusterissuers.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "issuers.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "orders.acme.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": serviceaccounts "cert-manager-cainjector" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": serviceaccounts "cert-manager" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": serviceaccounts "cert-manager-webhook" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-cainjector" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-issuers" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-clusterissuers" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-certificates" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-orders" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-challenges" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-ingress-shim" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-cluster-view" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-view" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-edit" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-approve:cert-manager-io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-certificatesigningrequests" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-webhook:subjectaccessreviews" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-cainjector" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-issuers" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-clusterissuers" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-certificates" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-orders" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-challenges" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-ingress-shim" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-approve:cert-manager-io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-certificatesigningrequests" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-webhook:subjectaccessreviews" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager-cainjector:leaderelection" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager:leaderelection" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager-tokenrequest" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager-webhook:dynamic-serving" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager-cainjector:leaderelection" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager:leaderelection" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager-cert-manager-tokenrequest" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager-webhook:dynamic-serving" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": services "cert-manager-cainjector" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": services "cert-manager" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": services "cert-manager-webhook" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": deployments.apps "cert-manager-cainjector" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": deployments.apps "cert-manager" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": deployments.apps "cert-manager-webhook" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": mutatingwebhookconfigurations.admissionregistration.k8s.io "cert-manager-webhook" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": validatingwebhookconfigurations.admissionregistration.k8s.io "cert-manager-webhook" not found
+ sleep 0
+ for i in $(seq 0 2)
+ set +e
+ kubectl delete -f https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml
+ exit_status=1
+ set -e
+ '[' 1 '!=' 0 -a -n 1 ']'
+ cat /tmp/tmp.8jbGAUfHPG
+ cat /tmp/tmp.6X4dLPyI5a
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": namespaces "cert-manager" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "certificaterequests.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "certificates.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "challenges.acme.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "clusterissuers.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "issuers.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "orders.acme.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": serviceaccounts "cert-manager-cainjector" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": serviceaccounts "cert-manager" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": serviceaccounts "cert-manager-webhook" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-cainjector" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-issuers" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-clusterissuers" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-certificates" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-orders" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-challenges" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-ingress-shim" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-cluster-view" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-view" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-edit" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-approve:cert-manager-io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-certificatesigningrequests" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-webhook:subjectaccessreviews" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-cainjector" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-issuers" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-clusterissuers" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-certificates" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-orders" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-challenges" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-ingress-shim" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-approve:cert-manager-io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-certificatesigningrequests" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-webhook:subjectaccessreviews" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager-cainjector:leaderelection" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager:leaderelection" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager-tokenrequest" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager-webhook:dynamic-serving" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager-cainjector:leaderelection" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager:leaderelection" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager-cert-manager-tokenrequest" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager-webhook:dynamic-serving" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": services "cert-manager-cainjector" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": services "cert-manager" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": services "cert-manager-webhook" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": deployments.apps "cert-manager-cainjector" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": deployments.apps "cert-manager" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": deployments.apps "cert-manager-webhook" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": mutatingwebhookconfigurations.admissionregistration.k8s.io "cert-manager-webhook" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": validatingwebhookconfigurations.admissionregistration.k8s.io "cert-manager-webhook" not found
+ sleep 4
+ for i in $(seq 0 2)
+ set +e
+ kubectl delete -f https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml
+ exit_status=1
+ set -e
+ '[' 1 '!=' 0 -a -n 1 ']'
+ cat /tmp/tmp.8jbGAUfHPG
+ cat /tmp/tmp.6X4dLPyI5a
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": namespaces "cert-manager" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "certificaterequests.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "certificates.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "challenges.acme.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "clusterissuers.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "issuers.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "orders.acme.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": serviceaccounts "cert-manager-cainjector" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": serviceaccounts "cert-manager" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": serviceaccounts "cert-manager-webhook" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-cainjector" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-issuers" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-clusterissuers" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-certificates" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-orders" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-challenges" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-ingress-shim" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-cluster-view" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-view" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-edit" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-approve:cert-manager-io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-certificatesigningrequests" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-webhook:subjectaccessreviews" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-cainjector" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-issuers" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-clusterissuers" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-certificates" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-orders" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-challenges" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-ingress-shim" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-approve:cert-manager-io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-certificatesigningrequests" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-webhook:subjectaccessreviews" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager-cainjector:leaderelection" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager:leaderelection" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager-tokenrequest" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager-webhook:dynamic-serving" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager-cainjector:leaderelection" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager:leaderelection" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager-cert-manager-tokenrequest" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager-webhook:dynamic-serving" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": services "cert-manager-cainjector" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": services "cert-manager" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": services "cert-manager-webhook" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": deployments.apps "cert-manager-cainjector" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": deployments.apps "cert-manager" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": deployments.apps "cert-manager-webhook" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": mutatingwebhookconfigurations.admissionregistration.k8s.io "cert-manager-webhook" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": validatingwebhookconfigurations.admissionregistration.k8s.io "cert-manager-webhook" not found
+ sleep 8
+ cat /tmp/tmp.8jbGAUfHPG
+ cat /tmp/tmp.6X4dLPyI5a
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": namespaces "cert-manager" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "certificaterequests.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "certificates.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "challenges.acme.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "clusterissuers.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "issuers.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "orders.acme.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": serviceaccounts "cert-manager-cainjector" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": serviceaccounts "cert-manager" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": serviceaccounts "cert-manager-webhook" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-cainjector" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-issuers" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-clusterissuers" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-certificates" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-orders" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-challenges" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-ingress-shim" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-cluster-view" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-view" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-edit" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-approve:cert-manager-io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-certificatesigningrequests" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-webhook:subjectaccessreviews" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-cainjector" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-issuers" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-clusterissuers" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-certificates" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-orders" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-challenges" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-ingress-shim" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-approve:cert-manager-io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-certificatesigningrequests" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-webhook:subjectaccessreviews" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager-cainjector:leaderelection" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager:leaderelection" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager-tokenrequest" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager-webhook:dynamic-serving" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager-cainjector:leaderelection" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager:leaderelection" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager-cert-manager-tokenrequest" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager-webhook:dynamic-serving" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": services "cert-manager-cainjector" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": services "cert-manager" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": services "cert-manager-webhook" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": deployments.apps "cert-manager-cainjector" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": deployments.apps "cert-manager" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": deployments.apps "cert-manager-webhook" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": mutatingwebhookconfigurations.admissionregistration.k8s.io "cert-manager-webhook" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": validatingwebhookconfigurations.admissionregistration.k8s.io "cert-manager-webhook" not found
+ rm /tmp/tmp.8jbGAUfHPG /tmp/tmp.6X4dLPyI5a
+ return 1
+ true
+ '[' -n '' ']'
+ '[' -n psmdb-operator ']'
+ kubectl_bin delete --grace-period=0 --force=true namespace multi-storage-17828
+ rm -rf /tmp/tmp.i5FaNrGCBi
+ kubectl_bin delete --grace-period=0 --force=true namespace psmdb-operator
++ mktemp
++ mktemp
+ local LAST_OUT=/tmp/tmp.g72XBHiIQb
+ log 'test passed'
+ set +o xtrace
++ mktemp
+ local LAST_OUT=/tmp/tmp.OZ6UtnLmfM
++ mktemp
+ local LAST_ERR=/tmp/tmp.IL6FAymQFT
+ local exit_status=0
+ local timeout=4
[2025-08-07T09:20:22+0000] test passed
++ seq 0 2
+ local LAST_ERR=/tmp/tmp.PsCoViPe4o
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl delete --grace-period=0 --force=true namespace multi-storage-17828
+ for i in $(seq 0 2)
+ set +e
+ kubectl delete --grace-period=0 --force=true namespace psmdb-operator