Log: /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2256/e2e-tests/logs/demand-backup.log
Warning: version difference between client (1.35) and server (1.32) exceeds the supported minor version skew of +/-1
Warning: version difference between client (1.35) and server (1.32) exceeds the supported minor version skew of +/-1
Warning: version difference between client (1.35) and server (1.32) exceeds the supported minor version skew of +/-1
+ create_infra demand-backup-7562
+ local ns=demand-backup-7562
+ [[ 1 == 1 ]]
+ delete_crd
+ desc 'get and delete old CRDs and RBAC'
+ set +o xtrace


-----------------------------------------------------------------------------------
get and delete old CRDs and RBAC
-----------------------------------------------------------------------------------

+ kubectl_bin delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2256/deploy/crd.yaml --ignore-not-found --wait=false
++ mktemp
+ local LAST_OUT=/tmp/tmp.StpNJqCbKs
++ mktemp
+ local LAST_ERR=/tmp/tmp.iArdjH0Q4V
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2256/deploy/crd.yaml --ignore-not-found --wait=false
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.StpNJqCbKs
customresourcedefinition.apiextensions.k8s.io "perconaservermongodbbackups.psmdb.percona.com" deleted
customresourcedefinition.apiextensions.k8s.io "perconaservermongodbrestores.psmdb.percona.com" deleted
customresourcedefinition.apiextensions.k8s.io "perconaservermongodbs.psmdb.percona.com" deleted
+ cat /tmp/tmp.iArdjH0Q4V
+ rm /tmp/tmp.StpNJqCbKs /tmp/tmp.iArdjH0Q4V
+ return 0
++ yq eval .metadata.name /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2256/deploy/crd.yaml
++ grep -v '\-\-\-'
grep: warning: stray \ before -
grep: warning: stray \ before -
+ for crd_name in $(yq eval '.metadata.name' "${src_dir}/deploy/crd.yaml" | grep -v '\-\-\-')
+ grep -v NAMESPACE
+ kubectl get perconaservermongodbbackups.psmdb.percona.com --all-namespaces -o wide
+ xargs -L 1 sh -xc 'kubectl patch perconaservermongodbbackups.psmdb.percona.com -n $0 $1 --type=merge -p "{\"metadata\":{\"finalizers\":[]}}"'
No resources found
+ kubectl patch perconaservermongodbbackups.psmdb.percona.com -n sh --type=merge -p '{"metadata":{"finalizers":[]}}'
error: resource(s) were provided, but no name was specified
+ :
+ kubectl_bin wait --for=delete crd perconaservermongodbbackups.psmdb.percona.com
++ mktemp
+ local LAST_OUT=/tmp/tmp.3H66lVCt4K
++ mktemp
+ local LAST_ERR=/tmp/tmp.fplwHAtcct
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl wait --for=delete crd perconaservermongodbbackups.psmdb.percona.com
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.3H66lVCt4K
+ cat /tmp/tmp.fplwHAtcct
+ rm /tmp/tmp.3H66lVCt4K /tmp/tmp.fplwHAtcct
+ return 0
+ for crd_name in $(yq eval '.metadata.name' "${src_dir}/deploy/crd.yaml" | grep -v '\-\-\-')
+ kubectl get perconaservermongodbrestores.psmdb.percona.com --all-namespaces -o wide
+ grep -v NAMESPACE
+ xargs -L 1 sh -xc 'kubectl patch perconaservermongodbrestores.psmdb.percona.com -n $0 $1 --type=merge -p "{\"metadata\":{\"finalizers\":[]}}"'
error: the server doesn't have a resource type "perconaservermongodbrestores"
+ kubectl patch perconaservermongodbrestores.psmdb.percona.com -n sh --type=merge -p '{"metadata":{"finalizers":[]}}'
error: the server doesn't have a resource type "perconaservermongodbrestores"
+ :
+ kubectl_bin wait --for=delete crd perconaservermongodbrestores.psmdb.percona.com
++ mktemp
+ local LAST_OUT=/tmp/tmp.bannfGv4ST
++ mktemp
+ local LAST_ERR=/tmp/tmp.kAUL3MayxE
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl wait --for=delete crd perconaservermongodbrestores.psmdb.percona.com
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.bannfGv4ST
+ cat /tmp/tmp.kAUL3MayxE
+ rm /tmp/tmp.bannfGv4ST /tmp/tmp.kAUL3MayxE
+ return 0
+ for crd_name in $(yq eval '.metadata.name' "${src_dir}/deploy/crd.yaml" | grep -v '\-\-\-')
+ kubectl get perconaservermongodbs.psmdb.percona.com --all-namespaces -o wide
+ grep -v NAMESPACE
+ xargs -L 1 sh -xc 'kubectl patch perconaservermongodbs.psmdb.percona.com -n $0 $1 --type=merge -p "{\"metadata\":{\"finalizers\":[]}}"'
error: the server doesn't have a resource type "perconaservermongodbs"
+ kubectl patch perconaservermongodbs.psmdb.percona.com -n sh --type=merge -p '{"metadata":{"finalizers":[]}}'
error: the server doesn't have a resource type "perconaservermongodbs"
+ :
+ kubectl_bin wait --for=delete crd perconaservermongodbs.psmdb.percona.com
++ mktemp
+ local LAST_OUT=/tmp/tmp.HTdJtCGN29
++ mktemp
+ local LAST_ERR=/tmp/tmp.INR2zjBeTS
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl wait --for=delete crd perconaservermongodbs.psmdb.percona.com
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.HTdJtCGN29
+ cat /tmp/tmp.INR2zjBeTS
+ rm /tmp/tmp.HTdJtCGN29 /tmp/tmp.INR2zjBeTS
+ return 0
+ local rbac_yaml=rbac.yaml
+ '[' -n psmdb-operator ']'
+ rbac_yaml=cw-rbac.yaml
+ kubectl_bin delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2256/deploy/cw-rbac.yaml --ignore-not-found
++ mktemp
+ local LAST_OUT=/tmp/tmp.kGWLctmpQ9
++ mktemp
+ local LAST_ERR=/tmp/tmp.aizLrqdAr7
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2256/deploy/cw-rbac.yaml --ignore-not-found
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.kGWLctmpQ9
clusterrole.rbac.authorization.k8s.io "percona-server-mongodb-operator" deleted
clusterrolebinding.rbac.authorization.k8s.io "service-account-percona-server-mongodb-operator" deleted
+ cat /tmp/tmp.aizLrqdAr7
+ rm /tmp/tmp.kGWLctmpQ9 /tmp/tmp.aizLrqdAr7
+ return 0
+ check_crd_for_deletion PR-2256-98b7d85f
+ local git_tag=PR-2256-98b7d85f
++ curl -s https://raw.githubusercontent.com/percona/percona-server-mongodb-operator/PR-2256-98b7d85f/deploy/crd.yaml
++ yq eval .metadata.name
++ /usr/sbin/sed s/---//g
++ /usr/sbin/sed ':a;N;$!ba;s/\n/ /g'
+ for crd_name in $(curl -s https://raw.githubusercontent.com/percona/percona-server-mongodb-operator/${git_tag}/deploy/crd.yaml | yq eval '.metadata.name' | $sed 's/---//g' | $sed ':a;N;$!ba;s/\n/ /g')
++ kubectl_bin get crd/null -o 'jsonpath={.status.conditions[-1].type}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.0d8oDNWeqq
+++ mktemp
++ local LAST_ERR=/tmp/tmp.GkjGXU2Ksk
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get crd/null -o 'jsonpath={.status.conditions[-1].type}'
++ exit_status=1
++ set -e
++ '[' 1 '!=' 0 -a -n 1 ']'
++ cat /tmp/tmp.0d8oDNWeqq
++ cat /tmp/tmp.GkjGXU2Ksk
Error from server (NotFound): customresourcedefinitions.apiextensions.k8s.io "null" not found
++ sleep 0
++ for i in $(seq 0 2)
++ set +e
++ kubectl get crd/null -o 'jsonpath={.status.conditions[-1].type}'
++ exit_status=1
++ set -e
++ '[' 1 '!=' 0 -a -n 1 ']'
++ cat /tmp/tmp.0d8oDNWeqq
++ cat /tmp/tmp.GkjGXU2Ksk
Error from server (NotFound): customresourcedefinitions.apiextensions.k8s.io "null" not found
++ sleep 4
++ for i in $(seq 0 2)
++ set +e
++ kubectl get crd/null -o 'jsonpath={.status.conditions[-1].type}'
++ exit_status=1
++ set -e
++ '[' 1 '!=' 0 -a -n 1 ']'
++ cat /tmp/tmp.0d8oDNWeqq
++ cat /tmp/tmp.GkjGXU2Ksk
Error from server (NotFound): customresourcedefinitions.apiextensions.k8s.io "null" not found
++ sleep 8
++ cat /tmp/tmp.0d8oDNWeqq
++ cat /tmp/tmp.GkjGXU2Ksk
Error from server (NotFound): customresourcedefinitions.apiextensions.k8s.io "null" not found
++ rm /tmp/tmp.0d8oDNWeqq /tmp/tmp.GkjGXU2Ksk
++ return 1
+ [[ '' == Terminating ]]
+ '[' -n psmdb-operator ']'
+ create_namespace psmdb-operator
+ local namespace=psmdb-operator
+ local skip_clean_namespace=
+ [[ 1 == 1 ]]
+ [[ -z '' ]]
+ destroy_chaos_mesh
++ helm list --all-namespaces --filter chaos-mesh
++ tail -n1
++ awk '-F ' '{print $2}'
++ sed s/NAMESPACE//
+ local chaos_mesh_ns=
+ desc 'destroy chaos-mesh'
+ set +o xtrace


-----------------------------------------------------------------------------------
destroy chaos-mesh
-----------------------------------------------------------------------------------

+ '[' -n '' ']'
++ kubectl get MutatingWebhookConfiguration
++ grep chaos-mesh
++ awk '{print $1}'
+ timeout 30 kubectl delete MutatingWebhookConfiguration
error: resource(s) were provided, but no name was specified
+ :
++ kubectl get ValidatingWebhookConfiguration
++ grep chaos-mesh
++ awk '{print $1}'
+ timeout 30 kubectl delete ValidatingWebhookConfiguration
error: resource(s) were provided, but no name was specified
+ :
++ kubectl get ValidatingWebhookConfiguration
++ grep validate-auth
++ awk '{print $1}'
+ timeout 30 kubectl delete ValidatingWebhookConfiguration
error: resource(s) were provided, but no name was specified
+ :
++ kubectl api-resources
++ grep chaos-mesh
++ awk '{print $1}'
++ kubectl get crd
++ grep chaos-mesh.org
++ awk '{print $1}'
+ timeout 30 kubectl delete crd
error: resource(s) were provided, but no name was specified
+ :
++ kubectl get clusterrolebinding
++ grep chaos-mesh
++ awk '{print $1}'
+ timeout 30 kubectl delete clusterrolebinding
error: resource(s) were provided, but no name was specified
+ :
++ kubectl get clusterrole
++ grep chaos-mesh
++ awk '{print $1}'
+ timeout 30 kubectl delete clusterrole
error: resource(s) were provided, but no name was specified
+ :
+ desc 'cleaned up all old namespaces'
+ set +o xtrace


-----------------------------------------------------------------------------------
cleaned up all old namespaces
-----------------------------------------------------------------------------------

+ kubectl_bin get ns
+ grep -E -v '^kube-|^default|Terminating|psmdb-operator|openshift|^gke-|^gmp-|^NAME'
+ awk '{print$1}'
++ mktemp
+ '[' -n '' ']'
+ desc 'cleaned up old namespaces psmdb-operator'
+ xargs kubectl delete ns
+ set +o xtrace


-----------------------------------------------------------------------------------
cleaned up old namespaces psmdb-operator
-----------------------------------------------------------------------------------

+ kubectl_bin delete namespace psmdb-operator --ignore-not-found
++ mktemp
+ local LAST_OUT=/tmp/tmp.5nlhUym1Np
++ mktemp
+ local LAST_OUT=/tmp/tmp.yM6vbtVChu
++ mktemp
+ local LAST_ERR=/tmp/tmp.PMFkyifjub
+ local exit_status=0
+ local timeout=4
+ local LAST_ERR=/tmp/tmp.XzxmbbOPhU
+ local exit_status=0
+ local timeout=4
++ seq 0 2
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ for i in $(seq 0 2)
+ kubectl delete namespace psmdb-operator --ignore-not-found
+ set +e
+ kubectl get ns
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.5nlhUym1Np
+ cat /tmp/tmp.PMFkyifjub
+ rm /tmp/tmp.5nlhUym1Np /tmp/tmp.PMFkyifjub
+ return 0
namespace "demand-backup-25747" deleted
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.yM6vbtVChu
namespace "psmdb-operator" deleted
+ cat /tmp/tmp.XzxmbbOPhU
+ rm /tmp/tmp.yM6vbtVChu /tmp/tmp.XzxmbbOPhU
+ return 0
+ kubectl_bin wait --for=delete namespace psmdb-operator
++ mktemp
+ local LAST_OUT=/tmp/tmp.kswCufQT4j
++ mktemp
+ local LAST_ERR=/tmp/tmp.f5RXXqi2ZK
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl wait --for=delete namespace psmdb-operator
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.kswCufQT4j
+ cat /tmp/tmp.f5RXXqi2ZK
+ rm /tmp/tmp.kswCufQT4j /tmp/tmp.f5RXXqi2ZK
+ return 0
+ desc 'create namespace psmdb-operator'
+ set +o xtrace


-----------------------------------------------------------------------------------
create namespace psmdb-operator
-----------------------------------------------------------------------------------

+ kubectl_bin create namespace psmdb-operator
++ mktemp
+ local LAST_OUT=/tmp/tmp.V5njZNbn0q
++ mktemp
+ local LAST_ERR=/tmp/tmp.jdOFvOE8rD
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl create namespace psmdb-operator
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.V5njZNbn0q
namespace/psmdb-operator created
+ cat /tmp/tmp.jdOFvOE8rD
+ rm /tmp/tmp.V5njZNbn0q /tmp/tmp.jdOFvOE8rD
+ return 0
+ set_kube_ctx psmdb-operator
+ local namespace=psmdb-operator
++ kubectl_bin config current-context
+++ mktemp
++ local LAST_OUT=/tmp/tmp.UGvogSISEd
+++ mktemp
++ local LAST_ERR=/tmp/tmp.U0xrLLOUE5
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl config current-context
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.UGvogSISEd
++ cat /tmp/tmp.U0xrLLOUE5
++ rm /tmp/tmp.UGvogSISEd /tmp/tmp.U0xrLLOUE5
++ return 0
+ kubectl_bin config set-context gke_cloud-dev-112233_us-central1-a_jen-psmdb-2256-98b7d85f-7-cluster10 --namespace=psmdb-operator
++ mktemp
+ local LAST_OUT=/tmp/tmp.qaDHoO3BE3
++ mktemp
+ local LAST_ERR=/tmp/tmp.XKOWptZ9Ua
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl config set-context gke_cloud-dev-112233_us-central1-a_jen-psmdb-2256-98b7d85f-7-cluster10 --namespace=psmdb-operator
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.qaDHoO3BE3
Context "gke_cloud-dev-112233_us-central1-a_jen-psmdb-2256-98b7d85f-7-cluster10" modified.
+ cat /tmp/tmp.XKOWptZ9Ua
+ rm /tmp/tmp.qaDHoO3BE3 /tmp/tmp.XKOWptZ9Ua
+ return 0
+ deploy_operator
+ desc 'start PSMDB operator: docker.io/perconalab/percona-server-mongodb-operator:PR-2256-98b7d85f'
+ set +o xtrace


-----------------------------------------------------------------------------------
start PSMDB operator: docker.io/perconalab/percona-server-mongodb-operator:PR-2256-98b7d85f
-----------------------------------------------------------------------------------

+ local cr_file
+ '[' -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2256/e2e-tests/demand-backup/conf/crd.yaml ']'
+ cr_file=/mnt/jenkins/workspace/cloud-psmdb-operator_PR-2256/deploy/crd.yaml
+ kubectl_bin apply --server-side --force-conflicts -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2256/deploy/crd.yaml
++ mktemp
+ local LAST_OUT=/tmp/tmp.8VpOf8kFiM
++ mktemp
+ local LAST_ERR=/tmp/tmp.qjrk3iKp3r
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl apply --server-side --force-conflicts -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2256/deploy/crd.yaml
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.8VpOf8kFiM
customresourcedefinition.apiextensions.k8s.io/perconaservermongodbbackups.psmdb.percona.com serverside-applied
customresourcedefinition.apiextensions.k8s.io/perconaservermongodbrestores.psmdb.percona.com serverside-applied
customresourcedefinition.apiextensions.k8s.io/perconaservermongodbs.psmdb.percona.com serverside-applied
+ cat /tmp/tmp.qjrk3iKp3r
+ rm /tmp/tmp.8VpOf8kFiM /tmp/tmp.qjrk3iKp3r
+ return 0
+ '[' -n psmdb-operator ']'
+ apply_rbac cw-rbac
+ local operator_namespace=psmdb-operator
+ local rbac=cw-rbac
+ cat /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2256/deploy/cw-rbac.yaml
+ sed -e 's^namespace: .*^namespace: psmdb-operator^'
+ kubectl_bin apply -n psmdb-operator -f -
++ mktemp
+ local LAST_OUT=/tmp/tmp.BHoOYS4enf
++ mktemp
+ local LAST_ERR=/tmp/tmp.nS35I3fdZE
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl apply -n psmdb-operator -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.BHoOYS4enf
clusterrole.rbac.authorization.k8s.io/percona-server-mongodb-operator created
serviceaccount/percona-server-mongodb-operator created
clusterrolebinding.rbac.authorization.k8s.io/service-account-percona-server-mongodb-operator created
+ cat /tmp/tmp.nS35I3fdZE
+ rm /tmp/tmp.BHoOYS4enf /tmp/tmp.nS35I3fdZE
+ return 0
+ yq eval $'\n\t\t\t(.spec.template.spec.containers[].image = "docker.io/perconalab/percona-server-mongodb-operator:PR-2256-98b7d85f") |\n\t\t\t((.. | select(.[] == "DISABLE_TELEMETRY")) |= .value="true") |\n\t\t\t((.. | select(.[] == "LOG_LEVEL")) |= .value="DEBUG")' /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2256/deploy/cw-operator.yaml
+ kubectl_bin apply -n psmdb-operator -f -
++ mktemp
+ local LAST_OUT=/tmp/tmp.ep8xuazPqL
++ mktemp
+ local LAST_ERR=/tmp/tmp.qXpV4bMENk
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl apply -n psmdb-operator -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.ep8xuazPqL
deployment.apps/percona-server-mongodb-operator created
+ cat /tmp/tmp.qXpV4bMENk
+ rm /tmp/tmp.ep8xuazPqL /tmp/tmp.qXpV4bMENk
+ return 0
+ sleep 20
++ get_operator_pod
++ kubectl_bin get pods --selector=name=percona-server-mongodb-operator -o 'jsonpath={.items[].metadata.name}' -n psmdb-operator
+++ mktemp
++ local LAST_OUT=/tmp/tmp.WWAArKrR5x
+++ mktemp
++ local LAST_ERR=/tmp/tmp.nQe12uz4WL
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=percona-server-mongodb-operator -o 'jsonpath={.items[].metadata.name}' -n psmdb-operator
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.WWAArKrR5x
++ cat /tmp/tmp.nQe12uz4WL
++ rm /tmp/tmp.WWAArKrR5x /tmp/tmp.nQe12uz4WL
++ return 0
+ wait_operator_pod percona-server-mongodb-operator-7d9c7747d9-tvtpx
+ local pod=percona-server-mongodb-operator-7d9c7747d9-tvtpx
+ set +o xtrace
waiting for pod/percona-server-mongodb-operator-7d9c7747d9-tvtpx to be ready.OK
+ echo 'Print operator info from log'
Print operator info from log
+ grep 'Manager starting up'
++ get_operator_pod
++ kubectl_bin get pods --selector=name=percona-server-mongodb-operator -o 'jsonpath={.items[].metadata.name}' -n psmdb-operator
+++ mktemp
++ local LAST_OUT=/tmp/tmp.O1vQKzbImK
+++ mktemp
++ local LAST_ERR=/tmp/tmp.0CptygNxyJ
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=percona-server-mongodb-operator -o 'jsonpath={.items[].metadata.name}' -n psmdb-operator
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.O1vQKzbImK
++ cat /tmp/tmp.0CptygNxyJ
++ rm /tmp/tmp.O1vQKzbImK /tmp/tmp.0CptygNxyJ
++ return 0
+ kubectl_bin logs -n psmdb-operator percona-server-mongodb-operator-7d9c7747d9-tvtpx
++ mktemp
+ local LAST_OUT=/tmp/tmp.pDWXDFhxZg
++ mktemp
+ local LAST_ERR=/tmp/tmp.PFmJWvnliI
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl logs -n psmdb-operator percona-server-mongodb-operator-7d9c7747d9-tvtpx
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.pDWXDFhxZg
+ cat /tmp/tmp.PFmJWvnliI
+ rm /tmp/tmp.pDWXDFhxZg /tmp/tmp.PFmJWvnliI
+ return 0
2026-04-14T09:31:54.422Z	INFO	setup	Manager starting up	{"gitCommit": "98b7d85f6fc9316ce5207960e26f4bbf2b33bfd3", "gitBranch": "PR-2256-98b7d85f", "buildTime": "", "goVersion": "go1.25.9", "os": "linux", "arch": "amd64"}
+ create_namespace demand-backup-7562
+ local namespace=demand-backup-7562
+ local skip_clean_namespace=
+ [[ 1 == 1 ]]
+ [[ -z '' ]]
+ destroy_chaos_mesh
++ helm list --all-namespaces --filter chaos-mesh
++ tail -n1
++ awk '-F ' '{print $2}'
++ sed s/NAMESPACE//
+ local chaos_mesh_ns=
+ desc 'destroy chaos-mesh'
+ set +o xtrace


-----------------------------------------------------------------------------------
destroy chaos-mesh
-----------------------------------------------------------------------------------

+ '[' -n '' ']'
++ kubectl get MutatingWebhookConfiguration
++ grep chaos-mesh
++ awk '{print $1}'
+ timeout 30 kubectl delete MutatingWebhookConfiguration
error: resource(s) were provided, but no name was specified
+ :
++ grep chaos-mesh
++ kubectl get ValidatingWebhookConfiguration
++ awk '{print $1}'
+ timeout 30 kubectl delete ValidatingWebhookConfiguration
error: resource(s) were provided, but no name was specified
+ :
++ kubectl get ValidatingWebhookConfiguration
++ grep validate-auth
++ awk '{print $1}'
+ timeout 30 kubectl delete ValidatingWebhookConfiguration
error: resource(s) were provided, but no name was specified
+ :
++ kubectl api-resources
++ grep chaos-mesh
++ awk '{print $1}'
++ kubectl get crd
++ grep chaos-mesh.org
++ awk '{print $1}'
+ timeout 30 kubectl delete crd
error: resource(s) were provided, but no name was specified
+ :
++ kubectl get clusterrolebinding
++ grep chaos-mesh
++ awk '{print $1}'
+ timeout 30 kubectl delete clusterrolebinding
error: resource(s) were provided, but no name was specified
+ :
++ kubectl get clusterrole
++ grep chaos-mesh
++ awk '{print $1}'
+ timeout 30 kubectl delete clusterrole
error: resource(s) were provided, but no name was specified
+ :
+ desc 'cleaned up all old namespaces'
+ set +o xtrace


-----------------------------------------------------------------------------------
cleaned up all old namespaces
-----------------------------------------------------------------------------------

+ kubectl_bin get ns
+ awk '{print$1}'
+ grep -E -v '^kube-|^default|Terminating|psmdb-operator|openshift|^gke-|^gmp-|^NAME'
+ '[' -n '' ']'
+ xargs kubectl delete ns
+ desc 'cleaned up old namespaces demand-backup-7562'
+ set +o xtrace


-----------------------------------------------------------------------------------
cleaned up old namespaces demand-backup-7562
-----------------------------------------------------------------------------------

+ kubectl_bin delete namespace demand-backup-7562 --ignore-not-found
++ mktemp
++ mktemp
+ local LAST_OUT=/tmp/tmp.D2aHsEpWFm
++ mktemp
+ local LAST_ERR=/tmp/tmp.iCPNoyCUkW
+ local exit_status=0
+ local timeout=4
+ local LAST_OUT=/tmp/tmp.CB28I5Dl9n
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl get ns
++ mktemp
+ local LAST_ERR=/tmp/tmp.WBspNdBVBl
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl delete namespace demand-backup-7562 --ignore-not-found
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.D2aHsEpWFm
+ cat /tmp/tmp.iCPNoyCUkW
+ rm /tmp/tmp.D2aHsEpWFm /tmp/tmp.iCPNoyCUkW
+ return 0
error: resource(s) were provided, but no name was specified
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.CB28I5Dl9n
+ cat /tmp/tmp.WBspNdBVBl
+ rm /tmp/tmp.CB28I5Dl9n /tmp/tmp.WBspNdBVBl
+ return 0
+ kubectl_bin wait --for=delete namespace demand-backup-7562
++ mktemp
+ local LAST_OUT=/tmp/tmp.kB5uHv5FP4
++ mktemp
+ local LAST_ERR=/tmp/tmp.5qMpQXRSEe
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl wait --for=delete namespace demand-backup-7562
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.kB5uHv5FP4
+ cat /tmp/tmp.5qMpQXRSEe
+ rm /tmp/tmp.kB5uHv5FP4 /tmp/tmp.5qMpQXRSEe
+ return 0
+ desc 'create namespace demand-backup-7562'
+ set +o xtrace


-----------------------------------------------------------------------------------
create namespace demand-backup-7562
-----------------------------------------------------------------------------------

+ kubectl_bin create namespace demand-backup-7562
++ mktemp
+ local LAST_OUT=/tmp/tmp.BS5G3kcDqe
++ mktemp
+ local LAST_ERR=/tmp/tmp.Rld5kcwvKj
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl create namespace demand-backup-7562
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.BS5G3kcDqe
namespace/demand-backup-7562 created
+ cat /tmp/tmp.Rld5kcwvKj
+ rm /tmp/tmp.BS5G3kcDqe /tmp/tmp.Rld5kcwvKj
+ return 0
+ set_kube_ctx demand-backup-7562
+ local namespace=demand-backup-7562
++ kubectl_bin config current-context
+++ mktemp
++ local LAST_OUT=/tmp/tmp.KsktiOgDt6
+++ mktemp
++ local LAST_ERR=/tmp/tmp.sOdEFnFQDT
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl config current-context
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.KsktiOgDt6
++ cat /tmp/tmp.sOdEFnFQDT
++ rm /tmp/tmp.KsktiOgDt6 /tmp/tmp.sOdEFnFQDT
++ return 0
+ kubectl_bin config set-context gke_cloud-dev-112233_us-central1-a_jen-psmdb-2256-98b7d85f-7-cluster10 --namespace=demand-backup-7562
++ mktemp
+ local LAST_OUT=/tmp/tmp.MnFaADeSQU
++ mktemp
+ local LAST_ERR=/tmp/tmp.E8IeTdNrUY
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl config set-context gke_cloud-dev-112233_us-central1-a_jen-psmdb-2256-98b7d85f-7-cluster10 --namespace=demand-backup-7562
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.MnFaADeSQU
Context "gke_cloud-dev-112233_us-central1-a_jen-psmdb-2256-98b7d85f-7-cluster10" modified.
+ cat /tmp/tmp.E8IeTdNrUY
+ rm /tmp/tmp.MnFaADeSQU /tmp/tmp.E8IeTdNrUY
+ return 0
+ deploy_minio
+ local cert_secret=
+ local service_name=minio-service
+ desc 'install MinIO: minio-service'
+ set +o xtrace


-----------------------------------------------------------------------------------
install MinIO: minio-service
-----------------------------------------------------------------------------------

+ helm uninstall minio-service
+ :
+ helm repo remove minio
"minio" has been removed from your repositories
+ helm repo add minio https://charts.min.io/
"minio" has been added to your repositories
+ local endpoint=http://minio-service:9000
+ minio_args=('--version' '5.4.0' '--set' 'replicas=1' '--set' 'mode=standalone' '--set' 'resources.requests.memory=256Mi' '--set' 'rootUser=rootuser' '--set' 'rootPassword=rootpass123' '--set' 'users[0].accessKey=some-access-key' '--set' 'users[0].secretKey=some-secret-key' '--set' 'users[0].policy=consoleAdmin' '--set' 'service.type=ClusterIP' '--set' 'configPathmc=/tmp/' '--set' 'securityContext.enabled=false' '--set' 'persistence.size=2G' '--set' 'fullnameOverride=minio-service' '--set' 'serviceAccount.create=true' '--set' 'serviceAccount.name=minio-service-sa')
+ local minio_args
+ [[ -n '' ]]
+ retry 10 60 helm install minio-service --version 5.4.0 --set replicas=1 --set mode=standalone --set resources.requests.memory=256Mi --set rootUser=rootuser --set rootPassword=rootpass123 --set 'users[0].accessKey=some-access-key' --set 'users[0].secretKey=some-secret-key' --set 'users[0].policy=consoleAdmin' --set service.type=ClusterIP --set configPathmc=/tmp/ --set securityContext.enabled=false --set persistence.size=2G --set fullnameOverride=minio-service --set serviceAccount.create=true --set serviceAccount.name=minio-service-sa minio/minio
+ local max=10
+ local delay=60
+ shift 2
+ local n=1
+ helm install minio-service --version 5.4.0 --set replicas=1 --set mode=standalone --set resources.requests.memory=256Mi --set rootUser=rootuser --set rootPassword=rootpass123 --set 'users[0].accessKey=some-access-key' --set 'users[0].secretKey=some-secret-key' --set 'users[0].policy=consoleAdmin' --set service.type=ClusterIP --set configPathmc=/tmp/ --set securityContext.enabled=false --set persistence.size=2G --set fullnameOverride=minio-service --set serviceAccount.create=true --set serviceAccount.name=minio-service-sa minio/minio
NAME: minio-service
LAST DEPLOYED: Tue Apr 14 09:32:29 2026
NAMESPACE: demand-backup-7562
STATUS: deployed
REVISION: 1
TEST SUITE: None
NOTES:
MinIO can be accessed via port 9000 on the following DNS name from within your cluster:
minio-service.demand-backup-7562.cluster.local

To access MinIO from localhost, run the below commands:

  1. export POD_NAME=$(kubectl get pods --namespace demand-backup-7562 -l "release=minio-service" -o jsonpath="{.items[0].metadata.name}")

  2. kubectl port-forward $POD_NAME 9000 --namespace demand-backup-7562

Read more about port forwarding here: http://kubernetes.io/docs/user-guide/kubectl/kubectl_port-forward/

You can now access MinIO server on http://localhost:9000. Follow the below steps to connect to MinIO server with mc client:

  1. Download the MinIO mc client - https://min.io/docs/minio/linux/reference/minio-mc.html#quickstart

  2. export MC_HOST_minio-service-local=http://$(kubectl get secret --namespace demand-backup-7562 minio-service -o jsonpath="{.data.rootUser}" | base64 --decode):$(kubectl get secret --namespace demand-backup-7562 minio-service -o jsonpath="{.data.rootPassword}" | base64 --decode)@localhost:9000

  3. mc ls minio-service-local
++ kubectl_bin get pods --selector=release=minio-service -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.FiqY3RocvP
+++ mktemp
++ local LAST_ERR=/tmp/tmp.CmZ8oIfUQk
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=release=minio-service -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.FiqY3RocvP
++ cat /tmp/tmp.CmZ8oIfUQk
++ rm /tmp/tmp.FiqY3RocvP /tmp/tmp.CmZ8oIfUQk
++ return 0
+ local MINIO_POD=minio-service-6d5f646cdc-mltw2
+ wait_pod minio-service-6d5f646cdc-mltw2
+ local pod=minio-service-6d5f646cdc-mltw2
+ set +o xtrace
waiting for pod/minio-service-6d5f646cdc-mltw2 to be ready.OK
+ '[' -n psmdb-operator ']'
+ kubectl_bin create svc -n psmdb-operator externalname minio-service --external-name=minio-service.demand-backup-7562.svc.cluster.local --tcp=9000
service/minio-service created
+ create_minio_bucket operator-testing http://minio-service:9000
+ local bucket=operator-testing
+ local endpoint=http://minio-service:9000
+ kubectl_bin run -i --rm aws-cli --image=docker.io/perconalab/awscli --restart=Never -- bash -c $'AWS_ACCESS_KEY_ID=some-access-key \t\tAWS_SECRET_ACCESS_KEY=some-secret-key \t\tAWS_DEFAULT_REGION=us-east-1         /usr/bin/aws --no-verify-ssl --endpoint-url http://minio-service:9000 s3 mb s3://operator-testing'
++ mktemp
+ local LAST_OUT=/tmp/tmp.WM8D6GTw7O
++ mktemp
+ local LAST_ERR=/tmp/tmp.NjAwVbyUTF
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl run -i --rm aws-cli --image=docker.io/perconalab/awscli --restart=Never -- bash -c $'AWS_ACCESS_KEY_ID=some-access-key \t\tAWS_SECRET_ACCESS_KEY=some-secret-key \t\tAWS_DEFAULT_REGION=us-east-1         /usr/bin/aws --no-verify-ssl --endpoint-url http://minio-service:9000 s3 mb s3://operator-testing'
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.WM8D6GTw7O
pod "aws-cli" deleted from demand-backup-7562 namespace
+ cat /tmp/tmp.NjAwVbyUTF
All commands and output from this session will be recorded in container logs, including credentials and sensitive information passed through the command prompt.
If you don't see a command prompt, try pressing enter.
+ rm /tmp/tmp.WM8D6GTw7O /tmp/tmp.NjAwVbyUTF
+ return 0
+ desc 'create secrets and start client'
+ set +o xtrace


-----------------------------------------------------------------------------------
create secrets and start client
-----------------------------------------------------------------------------------

+ cluster=some-name-rs0
+ kubectl_bin apply -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2256/e2e-tests/demand-backup/conf/secrets.yml -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2256/e2e-tests/conf/client.yml
++ mktemp
+ local LAST_OUT=/tmp/tmp.K68ILrYRBH
++ mktemp
+ local LAST_ERR=/tmp/tmp.4NYjLxmIqT
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl apply -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2256/e2e-tests/demand-backup/conf/secrets.yml -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2256/e2e-tests/conf/client.yml
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.K68ILrYRBH
secret/some-users created
deployment.apps/psmdb-client created
+ cat /tmp/tmp.4NYjLxmIqT
+ rm /tmp/tmp.K68ILrYRBH /tmp/tmp.4NYjLxmIqT
+ return 0
+ apply_s3_storage_secrets
+ desc 'create secrets for cloud storages'
+ set +o xtrace


-----------------------------------------------------------------------------------
create secrets for cloud storages
-----------------------------------------------------------------------------------

+ '[' -z '' ']'
+ kubectl_bin apply -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2256/e2e-tests/conf/minio-secret.yml -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2256/e2e-tests/conf/cloud-secret.yml
++ mktemp
+ local LAST_OUT=/tmp/tmp.E4jqJhMte8
++ mktemp
+ local LAST_ERR=/tmp/tmp.9L1rsi4nMb
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl apply -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2256/e2e-tests/conf/minio-secret.yml -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2256/e2e-tests/conf/cloud-secret.yml
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.E4jqJhMte8
secret/minio-secret created
secret/aws-s3-secret created
secret/gcp-cs-secret created
secret/azure-secret created
secret/gcp-cs-sa-key-secret created
+ cat /tmp/tmp.9L1rsi4nMb
+ rm /tmp/tmp.E4jqJhMte8 /tmp/tmp.9L1rsi4nMb
+ return 0
+ desc 'create first PSMDB cluster some-name-rs0'
+ set +o xtrace


-----------------------------------------------------------------------------------
create first PSMDB cluster some-name-rs0
-----------------------------------------------------------------------------------

+ apply_cluster /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2256/e2e-tests/demand-backup/conf/some-name-rs0.yml
+ '[' -z '' ']'
+ cat_config /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2256/e2e-tests/demand-backup/conf/some-name-rs0.yml
+ kubectl_bin apply -f -
+ cat /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2256/e2e-tests/demand-backup/conf/some-name-rs0.yml
+ yq eval '(.spec | select(.image == null)).image = "docker.io/perconalab/percona-server-mongodb-operator:main-mongod8.0"'
+ yq eval '(.spec | select(has("pmm"))).pmm.image = "docker.io/percona/pmm-client:2.44.1-1"'
++ mktemp
+ yq eval '(.spec | select(has("initImage"))).initImage = "docker.io/perconalab/percona-server-mongodb-operator:PR-2256-98b7d85f"'
+ yq eval '(.spec | select(has("backup"))).backup.image = "docker.io/perconalab/percona-server-mongodb-operator:main-backup"'
+ /usr/sbin/sed -e s/NAME_SPACE/demand-backup-7562/g
+ local LAST_OUT=/tmp/tmp.zxiTKhIIA4
+ yq eval '.spec.upgradeOptions.apply="Never"'
++ mktemp
+ local LAST_ERR=/tmp/tmp.8vhvmNUc6j
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl apply -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.zxiTKhIIA4
perconaservermongodb.psmdb.percona.com/some-name created
+ cat /tmp/tmp.8vhvmNUc6j
+ rm /tmp/tmp.zxiTKhIIA4 /tmp/tmp.8vhvmNUc6j
+ return 0
+ desc 'check if all 3 Pods started'
+ set +o xtrace


-----------------------------------------------------------------------------------
check if all 3 Pods started
-----------------------------------------------------------------------------------

+ wait_for_running some-name-rs0 3
+ local name=some-name-rs0
+ let last_pod=2
+ local check_cluster_readyness=true
+ set_debug
+ [[ 1 == 1 ]]
+ set -o xtrace
+ local rs_name=rs0
+ local cluster_name=some-name
++ seq 0 2
+ for i in $(seq 0 $last_pod)
+ [[ 0 -eq 2 ]]
+ wait_pod some-name-rs0-0
+ local pod=some-name-rs0-0
+ set +o xtrace
waiting for pod/some-name-rs0-0 to be ready......OK
+ for i in $(seq 0 $last_pod)
+ [[ 1 -eq 2 ]]
+ wait_pod some-name-rs0-1
+ local pod=some-name-rs0-1
+ set +o xtrace
waiting for pod/some-name-rs0-1 to be ready.......OK
+ for i in $(seq 0 $last_pod)
+ [[ 2 -eq 2 ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].arbiter.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.Aix7FLl355
+++ mktemp
++ local LAST_ERR=/tmp/tmp.fevRnZbfcB
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].arbiter.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.Aix7FLl355
++ cat /tmp/tmp.fevRnZbfcB
++ rm /tmp/tmp.Aix7FLl355 /tmp/tmp.fevRnZbfcB
++ return 0
+ [[ '' == true ]]
+ wait_pod some-name-rs0-2
+ local pod=some-name-rs0-2
+ set +o xtrace
waiting for pod/some-name-rs0-2 to be ready..............OK
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].nonvoting.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.rfl3QO3PAb
+++ mktemp
++ local LAST_ERR=/tmp/tmp.v0HEFgL8GC
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].nonvoting.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.rfl3QO3PAb
++ cat /tmp/tmp.v0HEFgL8GC
++ rm /tmp/tmp.rfl3QO3PAb /tmp/tmp.v0HEFgL8GC
++ return 0
+ [[ '' == true ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].hidden.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.AL6tmEKlrh
+++ mktemp
++ local LAST_ERR=/tmp/tmp.PJqj90wEfT
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].hidden.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.AL6tmEKlrh
++ cat /tmp/tmp.PJqj90wEfT
++ rm /tmp/tmp.AL6tmEKlrh /tmp/tmp.PJqj90wEfT
++ return 0
+ [[ '' == true ]]
+ sleep 10
+ [[ true == true ]]
+ set +x
Waiting for cluster readyness
+ desc 'check if service and statefulset created with expected config'
+ set +o xtrace


-----------------------------------------------------------------------------------
check if service and statefulset created with expected config
-----------------------------------------------------------------------------------

+ compare_kubectl statefulset/some-name-rs0
+ local resource=statefulset/some-name-rs0
+ local postfix=
+ local skip_generation_check=
+ local expected_result=/mnt/jenkins/workspace/cloud-psmdb-operator_PR-2256/e2e-tests/demand-backup/compare/statefulset_some-name-rs0.yml
+ local new_result=/tmp/tmp.pzOYzJaY7Q/statefulset_some-name-rs0.yml
+ '[' -n '' -a -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2256/e2e-tests/demand-backup/compare/statefulset_some-name-rs0-oc.yml ']'
+ kubectl_bin get -o yaml statefulset/some-name-rs0
+ yq eval $'\n\t\t\tdel(.metadata.ownerReferences[].apiVersion) |\n\t\t\tdel(.metadata.managedFields) |\n\t\t\tdel(.. | select(has("creationTimestamp")).creationTimestamp) |\n\t\t\tdel(.. | select(has("namespace")).namespace) |\n\t\t\tdel(.. | select(has("uid")).uid) |\n\t\t\tdel(.metadata.resourceVersion) |\n\t\t\tdel(.spec.template.spec.containers[].env[] | select(.name == "NAMESPACE")) |\n\t\t\tdel(.metadata.selfLink) |\n\t\t\tdel(.metadata.annotations."cloud.google.com/neg") |\n\t\t\tdel(.metadata.annotations."kubectl.kubernetes.io/last-applied-configuration") |\n\t\t\tdel(.. | select(has("image")).image) |\n\t\t\tdel(.. | select(has("clusterIP")).clusterIP) |\n\t\t\tdel(.. | select(has("clusterIPs")).clusterIPs) |\n\t\t\tdel(.. | select(has("dataSource")).dataSource) |\n\t\t\tdel(.. | select(has("procMount")).procMount) |\n\t\t\tdel(.. | select(has("storageClassName")).storageClassName) |\n\t\t\tdel(.. | select(has("finalizers")).finalizers) |\n\t\t\tdel(.. | select(has("kubernetes.io/pvc-protection"))."kubernetes.io/pvc-protection") |\n\t\t\tdel(.. | select(has("volumeName")).volumeName) |\n\t\t\tdel(.. | select(has("volume.beta.kubernetes.io/storage-provisioner"))."volume.beta.kubernetes.io/storage-provisioner") |\n\t\t\tdel(.. | select(has("volume.kubernetes.io/storage-provisioner"))."volume.kubernetes.io/storage-provisioner") |\n\t\t\tdel(.spec.volumeMode) |\n\t\t\tdel(.. | select(has("volume.kubernetes.io/selected-node"))."volume.kubernetes.io/selected-node") |\n\t\t\tdel(.. | select(has("percona.com/last-config-hash"))."percona.com/last-config-hash") |\n\t\t\tdel(.. | select(has("percona.com/configuration-hash"))."percona.com/configuration-hash") |\n\t\t\tdel(.. | select(has("percona.com/ssl-hash"))."percona.com/ssl-hash") |\n\t\t\tdel(.. | select(has("percona.com/ssl-internal-hash"))."percona.com/ssl-internal-hash") |\n\t\t\tdel(.spec.volumeClaimTemplates[].spec.volumeMode | select(. == "Filesystem")) |\n\t\t\tdel(.. | select(has("healthCheckNodePort")).healthCheckNodePort) |\n\t\t\tdel(.. | select(has("nodePort")).nodePort) |\n\t\t\tdel(.status) |\n\t\t\t(.. | select(tag == "!!str")) |= sub("demand-backup-7562", "NAME_SPACE") |\n\t\t\tdel(.spec.volumeClaimTemplates[].apiVersion) |\n\t\t\tdel(.spec.volumeClaimTemplates[].kind) |\n\t\t\tdel(.spec.ipFamilies) |\n\t\t\tdel(.spec.ipFamilyPolicy) |\n\t\t\t(.. | select(. == "extensions/v1beta1")) = "apps/v1" |\n\t\t\t(.. | select(. == "batch/v1beta1")) = "batch/v1" ' -
++ mktemp
+ local LAST_OUT=/tmp/tmp.7ByQ57rg5F
++ mktemp
+ local LAST_ERR=/tmp/tmp.R3rSaQdap1
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl get -o yaml statefulset/some-name-rs0
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.7ByQ57rg5F
+ cat /tmp/tmp.R3rSaQdap1
+ rm /tmp/tmp.7ByQ57rg5F /tmp/tmp.R3rSaQdap1
+ return 0
+ yq -i eval 'del(.spec.persistentVolumeClaimRetentionPolicy)' /tmp/tmp.pzOYzJaY7Q/statefulset_some-name-rs0.yml
+ version_gt 1.22
++ echo '1.32 >= 1.22'
++ bc -l
+ '[' 1 -eq 1 ']'
+ return 0
+ yq -i eval 'del(.spec.internalTrafficPolicy)' /tmp/tmp.pzOYzJaY7Q/statefulset_some-name-rs0.yml
+ yq -i eval 'del(.spec.allocateLoadBalancerNodePorts)' /tmp/tmp.pzOYzJaY7Q/statefulset_some-name-rs0.yml
+ [[ /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2256/e2e-tests/demand-backup/compare/statefulset_some-name-rs0.yml == */cronjob* ]]
+ '[' -n '' ']'
+ [[ 0 -eq 0 ]]
+ diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2256/e2e-tests/demand-backup/compare/statefulset_some-name-rs0.yml /tmp/tmp.pzOYzJaY7Q/statefulset_some-name-rs0.yml
+ log 'compare_kubectl: statefulset/some-name-rs0 OK'
+ set +o xtrace
[2026-04-14T09:35:03+0000] compare_kubectl: statefulset/some-name-rs0 OK
+ desc 'create user'
+ set +o xtrace


-----------------------------------------------------------------------------------
create user
-----------------------------------------------------------------------------------

+ run_mongo 'db.createUser({user:"myApp",pwd:"myPass",roles:[{db:"myApp",role:"readWrite"}]})' userAdmin:userAdmin123456@some-name-rs0.demand-backup-7562
+ local 'command=db.createUser({user:"myApp",pwd:"myPass",roles:[{db:"myApp",role:"readWrite"}]})'
+ local uri=userAdmin:userAdmin123456@some-name-rs0.demand-backup-7562
+ local driver=mongodb+srv
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local replica_set=rs0
+ [[ userAdmin:userAdmin123456@some-name-rs0.demand-backup-7562 == *cfg* ]]
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.PbaUGrQzr2
+++ mktemp
++ local LAST_ERR=/tmp/tmp.shZNy0gZ6f
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.PbaUGrQzr2
++ cat /tmp/tmp.shZNy0gZ6f
++ rm /tmp/tmp.PbaUGrQzr2 /tmp/tmp.shZNy0gZ6f
++ return 0
+ local client_container=psmdb-client-bb8b97679-62vtp
+ kubectl_bin exec psmdb-client-bb8b97679-62vtp -- bash -c 'printf '\''db.createUser({user:"myApp",pwd:"myPass",roles:[{db:"myApp",role:"readWrite"}]})\n'\'' | mongo mongodb+srv://userAdmin:userAdmin123456@some-name-rs0.demand-backup-7562.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
++ mktemp
+ local LAST_OUT=/tmp/tmp.TJcJMQnYcH
++ mktemp
+ local LAST_ERR=/tmp/tmp.451Vqc3Jgu
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-bb8b97679-62vtp -- bash -c 'printf '\''db.createUser({user:"myApp",pwd:"myPass",roles:[{db:"myApp",role:"readWrite"}]})\n'\'' | mongo mongodb+srv://userAdmin:userAdmin123456@some-name-rs0.demand-backup-7562.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.TJcJMQnYcH
Percona Server for MongoDB shell version v4.4.29-28
connecting to: mongodb://some-name-rs0-1.some-name-rs0.demand-backup-7562.svc.cluster.local:27017,some-name-rs0-2.some-name-rs0.demand-backup-7562.svc.cluster.local:27017,some-name-rs0-0.some-name-rs0.demand-backup-7562.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb&replicaSet=rs0&ssl=false
Implicit session: session { "id" : UUID("a0a47e6b-4985-48f2-8692-d04de83ddc22") }
Percona Server for MongoDB server version: v8.0.20-8
WARNING: shell and server versions do not match
Successfully added user: {
	"user" : "myApp",
	"roles" : [
		{
			"db" : "myApp",
			"role" : "readWrite"
		}
	]
}
bye
+ cat /tmp/tmp.451Vqc3Jgu
+ rm /tmp/tmp.TJcJMQnYcH /tmp/tmp.451Vqc3Jgu
+ return 0
+ sleep 2
+ desc 'write data, read from all'
+ set +o xtrace


-----------------------------------------------------------------------------------
write data, read from all
-----------------------------------------------------------------------------------

+ run_mongo 'use myApp\n db.test.insert({ x: 100500 })' myApp:myPass@some-name-rs0.demand-backup-7562
+ local 'command=use myApp\n db.test.insert({ x: 100500 })'
+ local uri=myApp:myPass@some-name-rs0.demand-backup-7562
+ local driver=mongodb+srv
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local replica_set=rs0
+ [[ myApp:myPass@some-name-rs0.demand-backup-7562 == *cfg* ]]
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.1UNpY1shbc
+++ mktemp
++ local LAST_ERR=/tmp/tmp.pR5HToBiXn
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.1UNpY1shbc
++ cat /tmp/tmp.pR5HToBiXn
++ rm /tmp/tmp.1UNpY1shbc /tmp/tmp.pR5HToBiXn
++ return 0
+ local client_container=psmdb-client-bb8b97679-62vtp
+ kubectl_bin exec psmdb-client-bb8b97679-62vtp -- bash -c 'printf '\''use myApp\n db.test.insert({ x: 100500 })\n'\'' | mongo mongodb+srv://myApp:myPass@some-name-rs0.demand-backup-7562.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
++ mktemp
+ local LAST_OUT=/tmp/tmp.iZ8kp981Yl
++ mktemp
+ local LAST_ERR=/tmp/tmp.lr9VCep0GQ
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-bb8b97679-62vtp -- bash -c 'printf '\''use myApp\n db.test.insert({ x: 100500 })\n'\'' | mongo mongodb+srv://myApp:myPass@some-name-rs0.demand-backup-7562.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.iZ8kp981Yl
Percona Server for MongoDB shell version v4.4.29-28
connecting to: mongodb://some-name-rs0-0.some-name-rs0.demand-backup-7562.svc.cluster.local:27017,some-name-rs0-1.some-name-rs0.demand-backup-7562.svc.cluster.local:27017,some-name-rs0-2.some-name-rs0.demand-backup-7562.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb&replicaSet=rs0&ssl=false
Implicit session: session { "id" : UUID("2c310ead-db1e-4f41-b9d1-8afba2569626") }
Percona Server for MongoDB server version: v8.0.20-8
WARNING: shell and server versions do not match
switched to db myApp
WriteResult({ "nInserted" : 1 })
bye
+ cat /tmp/tmp.lr9VCep0GQ
+ rm /tmp/tmp.iZ8kp981Yl /tmp/tmp.lr9VCep0GQ
+ return 0
+ run_mongo 'use myApp\n db.test2.insert({ x: 100501 })' myApp:myPass@some-name-rs0.demand-backup-7562
+ local 'command=use myApp\n db.test2.insert({ x: 100501 })'
+ local uri=myApp:myPass@some-name-rs0.demand-backup-7562
+ local driver=mongodb+srv
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local replica_set=rs0
+ [[ myApp:myPass@some-name-rs0.demand-backup-7562 == *cfg* ]]
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.gx9ASydw7s
+++ mktemp
++ local LAST_ERR=/tmp/tmp.z6LEr40WkY
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.gx9ASydw7s
++ cat /tmp/tmp.z6LEr40WkY
++ rm /tmp/tmp.gx9ASydw7s /tmp/tmp.z6LEr40WkY
++ return 0
+ local client_container=psmdb-client-bb8b97679-62vtp
+ kubectl_bin exec psmdb-client-bb8b97679-62vtp -- bash -c 'printf '\''use myApp\n db.test2.insert({ x: 100501 })\n'\'' | mongo mongodb+srv://myApp:myPass@some-name-rs0.demand-backup-7562.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
++ mktemp
+ local LAST_OUT=/tmp/tmp.cyDkUtif1i
++ mktemp
+ local LAST_ERR=/tmp/tmp.qW6zThazNJ
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-bb8b97679-62vtp -- bash -c 'printf '\''use myApp\n db.test2.insert({ x: 100501 })\n'\'' | mongo mongodb+srv://myApp:myPass@some-name-rs0.demand-backup-7562.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.cyDkUtif1i
Percona Server for MongoDB shell version v4.4.29-28
connecting to: mongodb://some-name-rs0-1.some-name-rs0.demand-backup-7562.svc.cluster.local:27017,some-name-rs0-2.some-name-rs0.demand-backup-7562.svc.cluster.local:27017,some-name-rs0-0.some-name-rs0.demand-backup-7562.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb&replicaSet=rs0&ssl=false
Implicit session: session { "id" : UUID("a522af98-167b-48fd-a66a-9f98f9b647be") }
Percona Server for MongoDB server version: v8.0.20-8
WARNING: shell and server versions do not match
switched to db myApp
WriteResult({ "nInserted" : 1 })
bye
+ cat /tmp/tmp.qW6zThazNJ
+ rm /tmp/tmp.cyDkUtif1i /tmp/tmp.qW6zThazNJ
+ return 0
+ run_mongo 'use myApp\n db.test3.insert({ x: 100502 })' myApp:myPass@some-name-rs0.demand-backup-7562
+ local 'command=use myApp\n db.test3.insert({ x: 100502 })'
+ local uri=myApp:myPass@some-name-rs0.demand-backup-7562
+ local driver=mongodb+srv
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local replica_set=rs0
+ [[ myApp:myPass@some-name-rs0.demand-backup-7562 == *cfg* ]]
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.3QJQcIberN
+++ mktemp
++ local LAST_ERR=/tmp/tmp.Apmlpya214
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.3QJQcIberN
++ cat /tmp/tmp.Apmlpya214
++ rm /tmp/tmp.3QJQcIberN /tmp/tmp.Apmlpya214
++ return 0
+ local client_container=psmdb-client-bb8b97679-62vtp
+ kubectl_bin exec psmdb-client-bb8b97679-62vtp -- bash -c 'printf '\''use myApp\n db.test3.insert({ x: 100502 })\n'\'' | mongo mongodb+srv://myApp:myPass@some-name-rs0.demand-backup-7562.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
++ mktemp
+ local LAST_OUT=/tmp/tmp.XexqzCw4RD
++ mktemp
+ local LAST_ERR=/tmp/tmp.33YRHQwMUN
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-bb8b97679-62vtp -- bash -c 'printf '\''use myApp\n db.test3.insert({ x: 100502 })\n'\'' | mongo mongodb+srv://myApp:myPass@some-name-rs0.demand-backup-7562.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.XexqzCw4RD
Percona Server for MongoDB shell version v4.4.29-28
connecting to: mongodb://some-name-rs0-2.some-name-rs0.demand-backup-7562.svc.cluster.local:27017,some-name-rs0-0.some-name-rs0.demand-backup-7562.svc.cluster.local:27017,some-name-rs0-1.some-name-rs0.demand-backup-7562.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb&replicaSet=rs0&ssl=false
Implicit session: session { "id" : UUID("bef1fa3f-858a-4299-8a26-89e2d26c73fd") }
Percona Server for MongoDB server version: v8.0.20-8
WARNING: shell and server versions do not match
switched to db myApp
WriteResult({ "nInserted" : 1 })
bye
+ cat /tmp/tmp.33YRHQwMUN
+ rm /tmp/tmp.XexqzCw4RD /tmp/tmp.33YRHQwMUN
+ return 0
+ custom_user_name=test1user
+ custom_role_name=test1role
+ run_mongo 'use myApp\n db.createUser({user: "test1user",pwd:"test1pass",roles:[]})' userAdmin:userAdmin123456@some-name-rs0.demand-backup-7562
+ local 'command=use myApp\n db.createUser({user: "test1user",pwd:"test1pass",roles:[]})'
+ local uri=userAdmin:userAdmin123456@some-name-rs0.demand-backup-7562
+ local driver=mongodb+srv
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local replica_set=rs0
+ [[ userAdmin:userAdmin123456@some-name-rs0.demand-backup-7562 == *cfg* ]]
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.pnPvx5e0NM
+++ mktemp
++ local LAST_ERR=/tmp/tmp.kagisqtajd
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.pnPvx5e0NM
++ cat /tmp/tmp.kagisqtajd
++ rm /tmp/tmp.pnPvx5e0NM /tmp/tmp.kagisqtajd
++ return 0
+ local client_container=psmdb-client-bb8b97679-62vtp
+ kubectl_bin exec psmdb-client-bb8b97679-62vtp -- bash -c 'printf '\''use myApp\n db.createUser({user: "test1user",pwd:"test1pass",roles:[]})\n'\'' | mongo mongodb+srv://userAdmin:userAdmin123456@some-name-rs0.demand-backup-7562.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
++ mktemp
+ local LAST_OUT=/tmp/tmp.VfYYWYhL1p
++ mktemp
+ local LAST_ERR=/tmp/tmp.g28t7LsIUd
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-bb8b97679-62vtp -- bash -c 'printf '\''use myApp\n db.createUser({user: "test1user",pwd:"test1pass",roles:[]})\n'\'' | mongo mongodb+srv://userAdmin:userAdmin123456@some-name-rs0.demand-backup-7562.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.VfYYWYhL1p
Percona Server for MongoDB shell version v4.4.29-28
connecting to: mongodb://some-name-rs0-1.some-name-rs0.demand-backup-7562.svc.cluster.local:27017,some-name-rs0-2.some-name-rs0.demand-backup-7562.svc.cluster.local:27017,some-name-rs0-0.some-name-rs0.demand-backup-7562.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb&replicaSet=rs0&ssl=false
Implicit session: session { "id" : UUID("2c1bd62c-2311-4f9e-88f4-66898c3ebaef") }
Percona Server for MongoDB server version: v8.0.20-8
WARNING: shell and server versions do not match
switched to db myApp
Successfully added user: { "user" : "test1user", "roles" : [ ] }
bye
+ cat /tmp/tmp.g28t7LsIUd
+ rm /tmp/tmp.VfYYWYhL1p /tmp/tmp.g28t7LsIUd
+ return 0
+ run_mongo 'use myApp\n db.createRole({"role":"test1role", privileges:[],roles:[]})' userAdmin:userAdmin123456@some-name-rs0.demand-backup-7562
+ local 'command=use myApp\n db.createRole({"role":"test1role", privileges:[],roles:[]})'
+ local uri=userAdmin:userAdmin123456@some-name-rs0.demand-backup-7562
+ local driver=mongodb+srv
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local replica_set=rs0
+ [[ userAdmin:userAdmin123456@some-name-rs0.demand-backup-7562 == *cfg* ]]
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.ll6YaZYdQQ
+++ mktemp
++ local LAST_ERR=/tmp/tmp.tKOe8hCpoc
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.ll6YaZYdQQ
++ cat /tmp/tmp.tKOe8hCpoc
++ rm /tmp/tmp.ll6YaZYdQQ /tmp/tmp.tKOe8hCpoc
++ return 0
+ local client_container=psmdb-client-bb8b97679-62vtp
+ kubectl_bin exec psmdb-client-bb8b97679-62vtp -- bash -c 'printf '\''use myApp\n db.createRole({"role":"test1role", privileges:[],roles:[]})\n'\'' | mongo mongodb+srv://userAdmin:userAdmin123456@some-name-rs0.demand-backup-7562.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
++ mktemp
+ local LAST_OUT=/tmp/tmp.ws1mPSoTOT
++ mktemp
+ local LAST_ERR=/tmp/tmp.9mQKplwC2M
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-bb8b97679-62vtp -- bash -c 'printf '\''use myApp\n db.createRole({"role":"test1role", privileges:[],roles:[]})\n'\'' | mongo mongodb+srv://userAdmin:userAdmin123456@some-name-rs0.demand-backup-7562.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.ws1mPSoTOT
Percona Server for MongoDB shell version v4.4.29-28
connecting to: mongodb://some-name-rs0-2.some-name-rs0.demand-backup-7562.svc.cluster.local:27017,some-name-rs0-0.some-name-rs0.demand-backup-7562.svc.cluster.local:27017,some-name-rs0-1.some-name-rs0.demand-backup-7562.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb&replicaSet=rs0&ssl=false
Implicit session: session { "id" : UUID("536625e9-1ce0-48ff-894b-d5a80af57297") }
Percona Server for MongoDB server version: v8.0.20-8
WARNING: shell and server versions do not match
switched to db myApp
{ "role" : "test1role", "privileges" : [ ], "roles" : [ ] }
bye
+ cat /tmp/tmp.9mQKplwC2M
+ rm /tmp/tmp.ws1mPSoTOT /tmp/tmp.9mQKplwC2M
+ return 0
+ minikube_sleep
+ sleep_time=10
+ [[ '' == 1 ]]
+ compare_mongo_cmd find myApp:myPass@some-name-rs0-0.some-name-rs0.demand-backup-7562
+ local command=find
+ local uri=myApp:myPass@some-name-rs0-0.some-name-rs0.demand-backup-7562
+ local postfix=
+ local suffix=
+ local database=myApp
+ local collection=test
+ local sort=
+ local tls=false
+ local replicaset=
+ local 'full_command=db.test.find()'
+ [[ -n '' ]]
+ log 'running db.test.find() in myApp'
+ set +o xtrace
[2026-04-14T09:35:21+0000] running db.test.find() in myApp
+ [[ false == true ]]
+ mongo_command=run_mongo
+ run_mongo 'use myApp\n db.test.find()' myApp:myPass@some-name-rs0-0.some-name-rs0.demand-backup-7562 mongodb '' ''
+ local 'command=use myApp\n db.test.find()'
+ local uri=myApp:myPass@some-name-rs0-0.some-name-rs0.demand-backup-7562
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local replica_set=rs0
+ grep -E -v 'I NETWORK|W NETWORK|F NETWORK|"c":"NETWORK"|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|Started a new thread for the timer service'
+ [[ myApp:myPass@some-name-rs0-0.some-name-rs0.demand-backup-7562 == *cfg* ]]
+ /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.H6IZ1AUhDV
+++ mktemp
++ local LAST_ERR=/tmp/tmp.EbO8WAcDbQ
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.H6IZ1AUhDV
++ cat /tmp/tmp.EbO8WAcDbQ
++ rm /tmp/tmp.H6IZ1AUhDV /tmp/tmp.EbO8WAcDbQ
++ return 0
+ local client_container=psmdb-client-bb8b97679-62vtp
+ kubectl_bin exec psmdb-client-bb8b97679-62vtp -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-0.some-name-rs0.demand-backup-7562.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
++ mktemp
+ local LAST_OUT=/tmp/tmp.JdKNNMbYEM
++ mktemp
+ local LAST_ERR=/tmp/tmp.N5O0ehWXmn
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-bb8b97679-62vtp -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-0.some-name-rs0.demand-backup-7562.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.JdKNNMbYEM
+ cat /tmp/tmp.N5O0ehWXmn
+ rm /tmp/tmp.JdKNNMbYEM /tmp/tmp.N5O0ehWXmn
+ return 0
+ [[ 0 -eq 0 ]]
+ diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2256/e2e-tests/demand-backup/compare/find.json /tmp/tmp.pzOYzJaY7Q/find
+ compare_mongo_cmd find myApp:myPass@some-name-rs0-1.some-name-rs0.demand-backup-7562
+ local command=find
+ local uri=myApp:myPass@some-name-rs0-1.some-name-rs0.demand-backup-7562
+ local postfix=
+ local suffix=
+ local database=myApp
+ local collection=test
+ local sort=
+ local tls=false
+ local replicaset=
+ local 'full_command=db.test.find()'
+ [[ -n '' ]]
+ log 'running db.test.find() in myApp'
+ set +o xtrace
[2026-04-14T09:35:23+0000] running db.test.find() in myApp
+ [[ false == true ]]
+ mongo_command=run_mongo
+ run_mongo 'use myApp\n db.test.find()' myApp:myPass@some-name-rs0-1.some-name-rs0.demand-backup-7562 mongodb '' ''
+ local 'command=use myApp\n db.test.find()'
+ grep -E -v 'I NETWORK|W NETWORK|F NETWORK|"c":"NETWORK"|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|Started a new thread for the timer service'
+ local uri=myApp:myPass@some-name-rs0-1.some-name-rs0.demand-backup-7562
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local replica_set=rs0
+ [[ myApp:myPass@some-name-rs0-1.some-name-rs0.demand-backup-7562 == *cfg* ]]
+ /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.JcDrLyW6yS
+++ mktemp
++ local LAST_ERR=/tmp/tmp.9dQshxsiI4
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.JcDrLyW6yS
++ cat /tmp/tmp.9dQshxsiI4
++ rm /tmp/tmp.JcDrLyW6yS /tmp/tmp.9dQshxsiI4
++ return 0
+ local client_container=psmdb-client-bb8b97679-62vtp
+ kubectl_bin exec psmdb-client-bb8b97679-62vtp -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-1.some-name-rs0.demand-backup-7562.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
++ mktemp
+ local LAST_OUT=/tmp/tmp.Us6q3KTeOg
++ mktemp
+ local LAST_ERR=/tmp/tmp.AwP7lp27JA
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-bb8b97679-62vtp -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-1.some-name-rs0.demand-backup-7562.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.Us6q3KTeOg
+ cat /tmp/tmp.AwP7lp27JA
+ rm /tmp/tmp.Us6q3KTeOg /tmp/tmp.AwP7lp27JA
+ return 0
+ [[ 0 -eq 0 ]]
+ diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2256/e2e-tests/demand-backup/compare/find.json /tmp/tmp.pzOYzJaY7Q/find
+ compare_mongo_cmd find myApp:myPass@some-name-rs0-2.some-name-rs0.demand-backup-7562
+ local command=find
+ local uri=myApp:myPass@some-name-rs0-2.some-name-rs0.demand-backup-7562
+ local postfix=
+ local suffix=
+ local database=myApp
+ local collection=test
+ local sort=
+ local tls=false
+ local replicaset=
+ local 'full_command=db.test.find()'
+ [[ -n '' ]]
+ log 'running db.test.find() in myApp'
+ set +o xtrace
[2026-04-14T09:35:25+0000] running db.test.find() in myApp
+ [[ false == true ]]
+ mongo_command=run_mongo
+ run_mongo 'use myApp\n db.test.find()' myApp:myPass@some-name-rs0-2.some-name-rs0.demand-backup-7562 mongodb '' ''
+ local 'command=use myApp\n db.test.find()'
+ local uri=myApp:myPass@some-name-rs0-2.some-name-rs0.demand-backup-7562
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local replica_set=rs0
+ [[ myApp:myPass@some-name-rs0-2.some-name-rs0.demand-backup-7562 == *cfg* ]]
+ /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+ grep -E -v 'I NETWORK|W NETWORK|F NETWORK|"c":"NETWORK"|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|Started a new thread for the timer service'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.kEk6YcUswT
+++ mktemp
++ local LAST_ERR=/tmp/tmp.M3qW9jOyfO
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.kEk6YcUswT
++ cat /tmp/tmp.M3qW9jOyfO
++ rm /tmp/tmp.kEk6YcUswT /tmp/tmp.M3qW9jOyfO
++ return 0
+ local client_container=psmdb-client-bb8b97679-62vtp
+ kubectl_bin exec psmdb-client-bb8b97679-62vtp -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-2.some-name-rs0.demand-backup-7562.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
++ mktemp
+ local LAST_OUT=/tmp/tmp.hwuhYml04q
++ mktemp
+ local LAST_ERR=/tmp/tmp.xyMKLNTfBQ
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-bb8b97679-62vtp -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-2.some-name-rs0.demand-backup-7562.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.hwuhYml04q
+ cat /tmp/tmp.xyMKLNTfBQ
+ rm /tmp/tmp.hwuhYml04q /tmp/tmp.xyMKLNTfBQ
+ return 0
+ [[ 0 -eq 0 ]]
+ diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2256/e2e-tests/demand-backup/compare/find.json /tmp/tmp.pzOYzJaY7Q/find
+ wait_backup_agent some-name-rs0-0
+ local agent_pod=some-name-rs0-0
+ set +o xtrace
waiting for pbm-agent to be ready in some-name-rs0-0...2026-04-14T09:34:36.000+0000 I listening for the commands

+ wait_backup_agent some-name-rs0-1
+ local agent_pod=some-name-rs0-1
+ set +o xtrace
waiting for pbm-agent to be ready in some-name-rs0-1...2026-04-14T09:34:46.000+0000 I listening for the commands

+ wait_backup_agent some-name-rs0-2
+ local agent_pod=some-name-rs0-2
+ set +o xtrace
waiting for pbm-agent to be ready in some-name-rs0-2...2026-04-14T09:35:01.000+0000 I listening for the commands

+ '[' -z '' ']'
+ desc 'Check GCS profiles'
+ set +o xtrace


-----------------------------------------------------------------------------------
Check GCS profiles
-----------------------------------------------------------------------------------

+ compare_pbm_profile_setup some-name gcp-cs-s3
+ local cluster=some-name
+ local profile=gcp-cs-s3
+ local container=backup-agent
+ local pbm_binary=pbm
+ [[ backup-agent == mongod ]]
+ kubectl_bin exec some-name-rs0-0 -c backup-agent -- pbm profile show gcp-cs-s3
++ mktemp
+ local LAST_OUT=/tmp/tmp.Lzq04DsLgq
++ mktemp
+ local LAST_ERR=/tmp/tmp.uba45UiWdB
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec some-name-rs0-0 -c backup-agent -- pbm profile show gcp-cs-s3
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.Lzq04DsLgq
+ cat /tmp/tmp.uba45UiWdB
+ rm /tmp/tmp.Lzq04DsLgq /tmp/tmp.uba45UiWdB
+ return 0
+ diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2256/e2e-tests/demand-backup/compare/pbm_profile_gcp-cs-s3.yml /tmp/tmp.pzOYzJaY7Q/pbm_profile_gcp-cs-s3.yml
+ compare_pbm_profile_setup some-name gcp-cs-sa
+ local cluster=some-name
+ local profile=gcp-cs-sa
+ local container=backup-agent
+ local pbm_binary=pbm
+ [[ backup-agent == mongod ]]
+ kubectl_bin exec some-name-rs0-0 -c backup-agent -- pbm profile show gcp-cs-sa
++ mktemp
+ local LAST_OUT=/tmp/tmp.l3iy0MdreJ
++ mktemp
+ local LAST_ERR=/tmp/tmp.BRg4q4oHyg
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec some-name-rs0-0 -c backup-agent -- pbm profile show gcp-cs-sa
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.l3iy0MdreJ
+ cat /tmp/tmp.BRg4q4oHyg
+ rm /tmp/tmp.l3iy0MdreJ /tmp/tmp.BRg4q4oHyg
+ return 0
+ diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2256/e2e-tests/demand-backup/compare/pbm_profile_gcp-cs-sa.yml /tmp/tmp.pzOYzJaY7Q/pbm_profile_gcp-cs-sa.yml
+ backup_name_minio=backup-minio
+ '[' -z '' ']'
+ backup_name_aws=backup-aws-s3
+ backup_name_gcp_s3=backup-gcp-cs-s3
+ backup_name_gcp_sa=backup-gcp-cs-sa
+ backup_name_azure=backup-azure-blob
+ desc 'run backups'
+ set +o xtrace


-----------------------------------------------------------------------------------
run backups
-----------------------------------------------------------------------------------

+ '[' -z '' ']'
+ run_backup aws-s3
+ local storage=aws-s3
+ local backup_name=backup-aws-s3
+ local type=logical
+ log 'running backup backup-aws-s3'
+ set +o xtrace
[2026-04-14T09:35:34+0000] running backup backup-aws-s3
+ yq eval $'.metadata.name = "backup-aws-s3"\n\t\t\t| .spec.storageName = "aws-s3"\n\t\t\t| .spec.type = "logical"' /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2256/e2e-tests/demand-backup/conf/backup-aws-s3.yml
+ kubectl_bin apply -f -
++ mktemp
+ local LAST_OUT=/tmp/tmp.ULrKuIdbza
++ mktemp
+ local LAST_ERR=/tmp/tmp.zSObWzl2Vk
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl apply -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.ULrKuIdbza
perconaservermongodbbackup.psmdb.percona.com/backup-aws-s3 created
+ cat /tmp/tmp.zSObWzl2Vk
+ rm /tmp/tmp.ULrKuIdbza /tmp/tmp.zSObWzl2Vk
+ return 0
+ run_backup gcp-cs-s3
+ local storage=gcp-cs-s3
+ local backup_name=backup-gcp-cs-s3
+ local type=logical
+ log 'running backup backup-gcp-cs-s3'
+ set +o xtrace
[2026-04-14T09:35:36+0000] running backup backup-gcp-cs-s3
+ yq eval $'.metadata.name = "backup-gcp-cs-s3"\n\t\t\t| .spec.storageName = "gcp-cs-s3"\n\t\t\t| .spec.type = "logical"' /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2256/e2e-tests/demand-backup/conf/backup-gcp-cs-s3.yml
+ kubectl_bin apply -f -
++ mktemp
+ local LAST_OUT=/tmp/tmp.m3rhtxzQEa
++ mktemp
+ local LAST_ERR=/tmp/tmp.UUlto02mIz
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl apply -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.m3rhtxzQEa
perconaservermongodbbackup.psmdb.percona.com/backup-gcp-cs-s3 created
+ cat /tmp/tmp.UUlto02mIz
+ rm /tmp/tmp.m3rhtxzQEa /tmp/tmp.UUlto02mIz
+ return 0
+ run_backup gcp-cs-sa
+ local storage=gcp-cs-sa
+ local backup_name=backup-gcp-cs-sa
+ local type=logical
+ log 'running backup backup-gcp-cs-sa'
+ set +o xtrace
[2026-04-14T09:35:38+0000] running backup backup-gcp-cs-sa
+ yq eval $'.metadata.name = "backup-gcp-cs-sa"\n\t\t\t| .spec.storageName = "gcp-cs-sa"\n\t\t\t| .spec.type = "logical"' /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2256/e2e-tests/demand-backup/conf/backup-gcp-cs-sa.yml
+ kubectl_bin apply -f -
++ mktemp
+ local LAST_OUT=/tmp/tmp.pW2uArzLv6
++ mktemp
+ local LAST_ERR=/tmp/tmp.eX2egUhZpU
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl apply -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.pW2uArzLv6
perconaservermongodbbackup.psmdb.percona.com/backup-gcp-cs-sa created
+ cat /tmp/tmp.eX2egUhZpU
+ rm /tmp/tmp.pW2uArzLv6 /tmp/tmp.eX2egUhZpU
+ return 0
+ run_backup azure-blob
+ local storage=azure-blob
+ local backup_name=backup-azure-blob
+ local type=logical
+ log 'running backup backup-azure-blob'
+ set +o xtrace
[2026-04-14T09:35:40+0000] running backup backup-azure-blob
+ yq eval $'.metadata.name = "backup-azure-blob"\n\t\t\t| .spec.storageName = "azure-blob"\n\t\t\t| .spec.type = "logical"' /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2256/e2e-tests/demand-backup/conf/backup-azure-blob.yml
+ kubectl_bin apply -f -
++ mktemp
+ local LAST_OUT=/tmp/tmp.vBuGFQMtMA
++ mktemp
+ local LAST_ERR=/tmp/tmp.oW5t4mgT52
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl apply -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.vBuGFQMtMA
perconaservermongodbbackup.psmdb.percona.com/backup-azure-blob created
+ cat /tmp/tmp.oW5t4mgT52
+ rm /tmp/tmp.vBuGFQMtMA /tmp/tmp.oW5t4mgT52
+ return 0
+ run_backup minio
+ local storage=minio
+ local backup_name=backup-minio
+ local type=logical
+ log 'running backup backup-minio'
+ set +o xtrace
[2026-04-14T09:35:42+0000] running backup backup-minio
+ yq eval $'.metadata.name = "backup-minio"\n\t\t\t| .spec.storageName = "minio"\n\t\t\t| .spec.type = "logical"' /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2256/e2e-tests/demand-backup/conf/backup-minio.yml
+ kubectl_bin apply -f -
++ mktemp
+ local LAST_OUT=/tmp/tmp.Yu2i9Ag9bJ
++ mktemp
+ local LAST_ERR=/tmp/tmp.bacmYAnRJK
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl apply -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.Yu2i9Ag9bJ
perconaservermongodbbackup.psmdb.percona.com/backup-minio created
+ cat /tmp/tmp.bacmYAnRJK
+ rm /tmp/tmp.Yu2i9Ag9bJ /tmp/tmp.bacmYAnRJK
+ return 0
+ '[' -z '' ']'
+ wait_backup backup-aws-s3
+ local backup_name=backup-aws-s3
+ local target_state=ready
+ set +o xtrace
waiting for backup-aws-s3 to reach ready state...OK
+ wait_backup backup-gcp-cs-s3
+ local backup_name=backup-gcp-cs-s3
+ local target_state=ready
+ set +o xtrace
waiting for backup-gcp-cs-s3 to reach ready state............OK
+ wait_backup backup-gcp-cs-sa
+ local backup_name=backup-gcp-cs-sa
+ local target_state=ready
+ set +o xtrace
waiting for backup-gcp-cs-sa to reach ready state............OK
+ wait_backup backup-azure-blob
+ local backup_name=backup-azure-blob
+ local target_state=ready
+ set +o xtrace
waiting for backup-azure-blob to reach ready state..OK
+ wait_backup backup-minio
+ local backup_name=backup-minio
+ local target_state=ready
+ set +o xtrace
waiting for backup-minio to reach ready state..OK
+ sleep 5
+ '[' -z '' ']'
+ desc 'check backup and restore -- aws-s3'
+ set +o xtrace


-----------------------------------------------------------------------------------
check backup and restore -- aws-s3
-----------------------------------------------------------------------------------

++ get_backup_dest backup-aws-s3
++ local backup_name=backup-aws-s3
++ kubectl_bin get psmdb-backup backup-aws-s3 -o 'jsonpath={.status.destination}'
++ sed -e 's/.json$//'
++ sed 's|s3://||'
+++ mktemp
++ sed 's|azure://||'
++ sed 's|gs://||'
++ local LAST_OUT=/tmp/tmp.pgttlxHXw4
+++ mktemp
++ local LAST_ERR=/tmp/tmp.FpgNLZyQCx
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb-backup backup-aws-s3 -o 'jsonpath={.status.destination}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.pgttlxHXw4
++ cat /tmp/tmp.FpgNLZyQCx
++ rm /tmp/tmp.pgttlxHXw4 /tmp/tmp.FpgNLZyQCx
++ return 0
+ backup_dest_aws=operator-testing/psmdb-demand-backup/2026-04-14T09:35:36Z
+ setup_aws_credentials
+ local secret_name=aws-s3-secret
+ [[ -n AKIARXP3OARBMEQ6FSEW ]]
+ [[ -n gljUukL3/TuQnzrtJUgiURH6vqjt+SEo89jGS2Z/ ]]
+ echo 'AWS credentials already set in environment'
AWS credentials already set in environment
+ return 0
+ check_backup_existence_aws operator-testing/psmdb-demand-backup/2026-04-14T09:35:36Z /rs0/myApp.test.gz
++ echo operator-testing/psmdb-demand-backup/2026-04-14T09:35:36Z
++ cut -d/ -f1
+ bucket=operator-testing
++ echo operator-testing/psmdb-demand-backup/2026-04-14T09:35:36Z
++ cut -d/ -f2-
+ key_prefix=psmdb-demand-backup/2026-04-14T09:35:36Z
+ key=/rs0/myApp.test.gz
+ storage_name=aws-s3
+ retry=0
+ aws s3api head-object --bucket operator-testing --key psmdb-demand-backup/2026-04-14T09:35:36Z/rs0/myApp.test.gz
+ echo 'Backup operator-testing/psmdb-demand-backup/2026-04-14T09:35:36Z//rs0/myApp.test.gz found in aws-s3'
Backup operator-testing/psmdb-demand-backup/2026-04-14T09:35:36Z//rs0/myApp.test.gz found in aws-s3
+ run_recovery_check backup-aws-s3 some-name-rs0
+ local backup=backup-aws-s3
+ local cluster=some-name-rs0
+ run_mongo 'use myApp\n db.test.insert({ x: 100501 })' myApp:myPass@some-name-rs0.demand-backup-7562
+ local 'command=use myApp\n db.test.insert({ x: 100501 })'
+ local uri=myApp:myPass@some-name-rs0.demand-backup-7562
+ local driver=mongodb+srv
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local replica_set=rs0
+ [[ myApp:myPass@some-name-rs0.demand-backup-7562 == *cfg* ]]
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.Eg29INs0le
+++ mktemp
++ local LAST_ERR=/tmp/tmp.b0Mf7H3M9x
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.Eg29INs0le
++ cat /tmp/tmp.b0Mf7H3M9x
++ rm /tmp/tmp.Eg29INs0le /tmp/tmp.b0Mf7H3M9x
++ return 0
+ local client_container=psmdb-client-bb8b97679-62vtp
+ kubectl_bin exec psmdb-client-bb8b97679-62vtp -- bash -c 'printf '\''use myApp\n db.test.insert({ x: 100501 })\n'\'' | mongo mongodb+srv://myApp:myPass@some-name-rs0.demand-backup-7562.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
++ mktemp
+ local LAST_OUT=/tmp/tmp.kludTQG5SB
++ mktemp
+ local LAST_ERR=/tmp/tmp.D70kqFRN3O
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-bb8b97679-62vtp -- bash -c 'printf '\''use myApp\n db.test.insert({ x: 100501 })\n'\'' | mongo mongodb+srv://myApp:myPass@some-name-rs0.demand-backup-7562.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.kludTQG5SB
Percona Server for MongoDB shell version v4.4.29-28
connecting to: mongodb://some-name-rs0-2.some-name-rs0.demand-backup-7562.svc.cluster.local:27017,some-name-rs0-0.some-name-rs0.demand-backup-7562.svc.cluster.local:27017,some-name-rs0-1.some-name-rs0.demand-backup-7562.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb&replicaSet=rs0&ssl=false
Implicit session: session { "id" : UUID("73146078-28ae-4f3a-8227-3fd42a2fc3b5") }
Percona Server for MongoDB server version: v8.0.20-8
WARNING: shell and server versions do not match
switched to db myApp
WriteResult({ "nInserted" : 1 })
bye
+ cat /tmp/tmp.D70kqFRN3O
+ rm /tmp/tmp.kludTQG5SB /tmp/tmp.D70kqFRN3O
+ return 0
+ compare_mongo_cmd find myApp:myPass@some-name-rs0.demand-backup-7562 -2nd .svc.cluster.local myApp test
+ local command=find
+ local uri=myApp:myPass@some-name-rs0.demand-backup-7562
+ local postfix=-2nd
+ local suffix=.svc.cluster.local
+ local database=myApp
+ local collection=test
+ local sort=
+ local tls=false
+ local replicaset=
+ local 'full_command=db.test.find()'
+ [[ -n '' ]]
+ log 'running db.test.find() in myApp'
+ set +o xtrace
[2026-04-14T09:36:46+0000] running db.test.find() in myApp
+ [[ false == true ]]
+ mongo_command=run_mongo
+ run_mongo 'use myApp\n db.test.find()' myApp:myPass@some-name-rs0.demand-backup-7562 mongodb .svc.cluster.local ''
+ local 'command=use myApp\n db.test.find()'
+ local uri=myApp:myPass@some-name-rs0.demand-backup-7562
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local replica_set=rs0
+ [[ myApp:myPass@some-name-rs0.demand-backup-7562 == *cfg* ]]
+ grep -E -v 'I NETWORK|W NETWORK|F NETWORK|"c":"NETWORK"|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|Started a new thread for the timer service'
+ /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.CrzXqhqMxB
+++ mktemp
++ local LAST_ERR=/tmp/tmp.h3NpdYP2DL
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.CrzXqhqMxB
++ cat /tmp/tmp.h3NpdYP2DL
++ rm /tmp/tmp.CrzXqhqMxB /tmp/tmp.h3NpdYP2DL
++ return 0
+ local client_container=psmdb-client-bb8b97679-62vtp
+ kubectl_bin exec psmdb-client-bb8b97679-62vtp -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0.demand-backup-7562.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
++ mktemp
+ local LAST_OUT=/tmp/tmp.kOPL4snRkM
++ mktemp
+ local LAST_ERR=/tmp/tmp.YspELMmZ9U
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-bb8b97679-62vtp -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0.demand-backup-7562.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.kOPL4snRkM
+ cat /tmp/tmp.YspELMmZ9U
+ rm /tmp/tmp.kOPL4snRkM /tmp/tmp.YspELMmZ9U
+ return 0
+ [[ 0 -eq 0 ]]
+ diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2256/e2e-tests/demand-backup/compare/find-2nd.json /tmp/tmp.pzOYzJaY7Q/find-2nd
+ run_restore backup-aws-s3
+ local backup_name=backup-aws-s3
+ local restore_file=/mnt/jenkins/workspace/cloud-psmdb-operator_PR-2256/e2e-tests/demand-backup/conf/restore.yml
+ log 'running restore restore-backup-aws-s3'
+ set +o xtrace
[2026-04-14T09:36:48+0000] running restore restore-backup-aws-s3
+ cat /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2256/e2e-tests/demand-backup/conf/restore.yml
+ /usr/sbin/sed -e 's/name:/name: restore-backup-aws-s3/'
+ /usr/sbin/sed -e 's/backupName:/backupName: backup-aws-s3/'
+ kubectl_bin apply -f -
++ mktemp
+ local LAST_OUT=/tmp/tmp.HwvRdUwIaC
++ mktemp
+ local LAST_ERR=/tmp/tmp.D3URJkTOmS
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl apply -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.HwvRdUwIaC
perconaservermongodbrestore.psmdb.percona.com/restore-backup-aws-s3 created
+ cat /tmp/tmp.D3URJkTOmS
+ rm /tmp/tmp.HwvRdUwIaC /tmp/tmp.D3URJkTOmS
+ return 0
+ wait_restore backup-aws-s3 some-name
+ local backup_name=backup-aws-s3
+ local cluster_name=some-name
+ local target_state=ready
+ local wait_cluster_consistency=1
+ local wait_time=1780
+ local ok_if_ready=0
+ set +o xtrace
Waiting for the psmdb-restore/restore-backup-aws-s3 object to be created.OK
Waiting psmdb-restore/restore-backup-aws-s3 to reach state "ready" .OK after 0 minutes
+ [[ 1 -eq 1 ]]
+ wait_cluster_consistency some-name
+ local cluster_name=some-name
+ local wait_time=32
+ retry=0
+ sleep 7
+ echo -n 'waiting for cluster readyness'
waiting for cluster readyness++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.tWgvpoCCBR
+++ mktemp
++ local LAST_ERR=/tmp/tmp.il2Ihs1TPy
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.tWgvpoCCBR
++ cat /tmp/tmp.il2Ihs1TPy
++ rm /tmp/tmp.tWgvpoCCBR /tmp/tmp.il2Ihs1TPy
++ return 0
+ [[ ready == ready ]]
+ echo .OK
.OK
+ simple_data_check some-name-rs0 3 0
+ local cluster_name=some-name-rs0
+ let last_pod=3-1
+ local isSharded=0
+ local cluster_pfx=
+ '[' 0 -eq 1 ']'
++ seq 0 2
+ for i in $(seq 0 $last_pod)
+ compare_mongo_cmd find myApp:myPass@some-name-rs0-0.some-name-rs0.demand-backup-7562
+ local command=find
+ local uri=myApp:myPass@some-name-rs0-0.some-name-rs0.demand-backup-7562
+ local postfix=
+ local suffix=
+ local database=myApp
+ local collection=test
+ local sort=
+ local tls=false
+ local replicaset=
+ local 'full_command=db.test.find()'
+ [[ -n '' ]]
+ log 'running db.test.find() in myApp'
+ set +o xtrace
[2026-04-14T09:37:16+0000] running db.test.find() in myApp
+ [[ false == true ]]
+ mongo_command=run_mongo
+ run_mongo 'use myApp\n db.test.find()' myApp:myPass@some-name-rs0-0.some-name-rs0.demand-backup-7562 mongodb '' ''
+ local 'command=use myApp\n db.test.find()'
+ local uri=myApp:myPass@some-name-rs0-0.some-name-rs0.demand-backup-7562
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local replica_set=rs0
+ grep -E -v 'I NETWORK|W NETWORK|F NETWORK|"c":"NETWORK"|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|Started a new thread for the timer service'
+ [[ myApp:myPass@some-name-rs0-0.some-name-rs0.demand-backup-7562 == *cfg* ]]
+ /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.P5W1l2ZfFc
+++ mktemp
++ local LAST_ERR=/tmp/tmp.3Pvo5xo9v3
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.P5W1l2ZfFc
++ cat /tmp/tmp.3Pvo5xo9v3
++ rm /tmp/tmp.P5W1l2ZfFc /tmp/tmp.3Pvo5xo9v3
++ return 0
+ local client_container=psmdb-client-bb8b97679-62vtp
+ kubectl_bin exec psmdb-client-bb8b97679-62vtp -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-0.some-name-rs0.demand-backup-7562.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
++ mktemp
+ local LAST_OUT=/tmp/tmp.kwFdsaIIFn
++ mktemp
+ local LAST_ERR=/tmp/tmp.XtuJjYj2GM
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-bb8b97679-62vtp -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-0.some-name-rs0.demand-backup-7562.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.kwFdsaIIFn
+ cat /tmp/tmp.XtuJjYj2GM
+ rm /tmp/tmp.kwFdsaIIFn /tmp/tmp.XtuJjYj2GM
+ return 0
+ [[ 0 -eq 0 ]]
+ diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2256/e2e-tests/demand-backup/compare/find.json /tmp/tmp.pzOYzJaY7Q/find
+ for i in $(seq 0 $last_pod)
+ compare_mongo_cmd find myApp:myPass@some-name-rs0-1.some-name-rs0.demand-backup-7562
+ local command=find
+ local uri=myApp:myPass@some-name-rs0-1.some-name-rs0.demand-backup-7562
+ local postfix=
+ local suffix=
+ local database=myApp
+ local collection=test
+ local sort=
+ local tls=false
+ local replicaset=
+ local 'full_command=db.test.find()'
+ [[ -n '' ]]
+ log 'running db.test.find() in myApp'
+ set +o xtrace
[2026-04-14T09:37:18+0000] running db.test.find() in myApp
+ [[ false == true ]]
+ mongo_command=run_mongo
+ run_mongo 'use myApp\n db.test.find()' myApp:myPass@some-name-rs0-1.some-name-rs0.demand-backup-7562 mongodb '' ''
+ local 'command=use myApp\n db.test.find()'
+ local uri=myApp:myPass@some-name-rs0-1.some-name-rs0.demand-backup-7562
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ grep -E -v 'I NETWORK|W NETWORK|F NETWORK|"c":"NETWORK"|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|Started a new thread for the timer service'
+ local mongo_flag=
+ local replica_set=rs0
+ [[ myApp:myPass@some-name-rs0-1.some-name-rs0.demand-backup-7562 == *cfg* ]]
+ /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.IpcGj6XyQ7
+++ mktemp
++ local LAST_ERR=/tmp/tmp.fcjF0E1w5d
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.IpcGj6XyQ7
++ cat /tmp/tmp.fcjF0E1w5d
++ rm /tmp/tmp.IpcGj6XyQ7 /tmp/tmp.fcjF0E1w5d
++ return 0
+ local client_container=psmdb-client-bb8b97679-62vtp
+ kubectl_bin exec psmdb-client-bb8b97679-62vtp -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-1.some-name-rs0.demand-backup-7562.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
++ mktemp
+ local LAST_OUT=/tmp/tmp.IlpUSUl21I
++ mktemp
+ local LAST_ERR=/tmp/tmp.tZI6JKYc8v
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-bb8b97679-62vtp -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-1.some-name-rs0.demand-backup-7562.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.IlpUSUl21I
+ cat /tmp/tmp.tZI6JKYc8v
+ rm /tmp/tmp.IlpUSUl21I /tmp/tmp.tZI6JKYc8v
+ return 0
+ [[ 0 -eq 0 ]]
+ diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2256/e2e-tests/demand-backup/compare/find.json /tmp/tmp.pzOYzJaY7Q/find
+ for i in $(seq 0 $last_pod)
+ compare_mongo_cmd find myApp:myPass@some-name-rs0-2.some-name-rs0.demand-backup-7562
+ local command=find
+ local uri=myApp:myPass@some-name-rs0-2.some-name-rs0.demand-backup-7562
+ local postfix=
+ local suffix=
+ local database=myApp
+ local collection=test
+ local sort=
+ local tls=false
+ local replicaset=
+ local 'full_command=db.test.find()'
+ [[ -n '' ]]
+ log 'running db.test.find() in myApp'
+ set +o xtrace
[2026-04-14T09:37:21+0000] running db.test.find() in myApp
+ [[ false == true ]]
+ mongo_command=run_mongo
+ run_mongo 'use myApp\n db.test.find()' myApp:myPass@some-name-rs0-2.some-name-rs0.demand-backup-7562 mongodb '' ''
+ grep -E -v 'I NETWORK|W NETWORK|F NETWORK|"c":"NETWORK"|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|Started a new thread for the timer service'
+ local 'command=use myApp\n db.test.find()'
+ local uri=myApp:myPass@some-name-rs0-2.some-name-rs0.demand-backup-7562
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local replica_set=rs0
+ [[ myApp:myPass@some-name-rs0-2.some-name-rs0.demand-backup-7562 == *cfg* ]]
+ /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.UYfTd6tjRl
+++ mktemp
++ local LAST_ERR=/tmp/tmp.4ksEu7my5f
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.UYfTd6tjRl
++ cat /tmp/tmp.4ksEu7my5f
++ rm /tmp/tmp.UYfTd6tjRl /tmp/tmp.4ksEu7my5f
++ return 0
+ local client_container=psmdb-client-bb8b97679-62vtp
+ kubectl_bin exec psmdb-client-bb8b97679-62vtp -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-2.some-name-rs0.demand-backup-7562.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
++ mktemp
+ local LAST_OUT=/tmp/tmp.n6ZNLWQRj4
++ mktemp
+ local LAST_ERR=/tmp/tmp.IcoIht63uP
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-bb8b97679-62vtp -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-2.some-name-rs0.demand-backup-7562.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.n6ZNLWQRj4
+ cat /tmp/tmp.IcoIht63uP
+ rm /tmp/tmp.n6ZNLWQRj4 /tmp/tmp.IcoIht63uP
+ return 0
+ [[ 0 -eq 0 ]]
+ diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2256/e2e-tests/demand-backup/compare/find.json /tmp/tmp.pzOYzJaY7Q/find
+ desc 'check backup and restore -- gcp-cs-s3'
+ set +o xtrace


-----------------------------------------------------------------------------------
check backup and restore -- gcp-cs-s3
-----------------------------------------------------------------------------------

++ get_backup_dest backup-gcp-cs-s3
++ local backup_name=backup-gcp-cs-s3
++ kubectl_bin get psmdb-backup backup-gcp-cs-s3 -o 'jsonpath={.status.destination}'
++ sed -e 's/.json$//'
++ sed 's|s3://||'
++ sed 's|azure://||'
+++ mktemp
++ sed 's|gs://||'
++ local LAST_OUT=/tmp/tmp.Lse73ubnic
+++ mktemp
++ local LAST_ERR=/tmp/tmp.uaKbklyZXx
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb-backup backup-gcp-cs-s3 -o 'jsonpath={.status.destination}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.Lse73ubnic
++ cat /tmp/tmp.uaKbklyZXx
++ rm /tmp/tmp.Lse73ubnic /tmp/tmp.uaKbklyZXx
++ return 0
+ backup_dest_gcp_s3=operator-testing/psmdb-demand-backup/2026-04-14T09:35:59Z
+ setup_gcs_credentials
+ local secret_name=gcp-cs-secret
+ gsutil ls
+ echo 'Setting up GCS credentials from K8s secret: gcp-cs-secret'
Setting up GCS credentials from K8s secret: gcp-cs-secret
+ local trace_was_on=0
+ [[ ehxB == *x* ]]
+ trace_was_on=1
+ set +x
+ echo 'GCS credentials configured successfully'
GCS credentials configured successfully
+ check_backup_existence_gcs operator-testing/psmdb-demand-backup/2026-04-14T09:35:59Z /rs0/myApp.test.gz
+ backup_dest_gcp=operator-testing/psmdb-demand-backup/2026-04-14T09:35:59Z
+ obj=/rs0/myApp.test.gz
+ storage_name=gcp-cs
+ retry=0
+ gcs_path=gs://operator-testing/psmdb-demand-backup/2026-04-14T09:35:59Z/rs0/myApp.test.gz
+ gsutil ls gs://operator-testing/psmdb-demand-backup/2026-04-14T09:35:59Z/rs0/myApp.test.gz
+ echo 'Backup gs://operator-testing/psmdb-demand-backup/2026-04-14T09:35:59Z/rs0/myApp.test.gz found in gcp-cs'
Backup gs://operator-testing/psmdb-demand-backup/2026-04-14T09:35:59Z/rs0/myApp.test.gz found in gcp-cs
+ run_recovery_check backup-gcp-cs-s3 some-name-rs0
+ local backup=backup-gcp-cs-s3
+ local cluster=some-name-rs0
+ run_mongo 'use myApp\n db.test.insert({ x: 100501 })' myApp:myPass@some-name-rs0.demand-backup-7562
+ local 'command=use myApp\n db.test.insert({ x: 100501 })'
+ local uri=myApp:myPass@some-name-rs0.demand-backup-7562
+ local driver=mongodb+srv
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local replica_set=rs0
+ [[ myApp:myPass@some-name-rs0.demand-backup-7562 == *cfg* ]]
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.cVRtxzSDvN
+++ mktemp
++ local LAST_ERR=/tmp/tmp.lQAa4TYrJ2
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.cVRtxzSDvN
++ cat /tmp/tmp.lQAa4TYrJ2
++ rm /tmp/tmp.cVRtxzSDvN /tmp/tmp.lQAa4TYrJ2
++ return 0
+ local client_container=psmdb-client-bb8b97679-62vtp
+ kubectl_bin exec psmdb-client-bb8b97679-62vtp -- bash -c 'printf '\''use myApp\n db.test.insert({ x: 100501 })\n'\'' | mongo mongodb+srv://myApp:myPass@some-name-rs0.demand-backup-7562.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
++ mktemp
+ local LAST_OUT=/tmp/tmp.gbY6ivTqbt
++ mktemp
+ local LAST_ERR=/tmp/tmp.XStaKvAkYl
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-bb8b97679-62vtp -- bash -c 'printf '\''use myApp\n db.test.insert({ x: 100501 })\n'\'' | mongo mongodb+srv://myApp:myPass@some-name-rs0.demand-backup-7562.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.gbY6ivTqbt
Percona Server for MongoDB shell version v4.4.29-28
connecting to: mongodb://some-name-rs0-0.some-name-rs0.demand-backup-7562.svc.cluster.local:27017,some-name-rs0-1.some-name-rs0.demand-backup-7562.svc.cluster.local:27017,some-name-rs0-2.some-name-rs0.demand-backup-7562.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb&replicaSet=rs0&ssl=false
Implicit session: session { "id" : UUID("458b6bbf-27c1-458b-abf3-2783ca4fa8b6") }
Percona Server for MongoDB server version: v8.0.20-8
WARNING: shell and server versions do not match
switched to db myApp
WriteResult({ "nInserted" : 1 })
bye
+ cat /tmp/tmp.XStaKvAkYl
+ rm /tmp/tmp.gbY6ivTqbt /tmp/tmp.XStaKvAkYl
+ return 0
+ compare_mongo_cmd find myApp:myPass@some-name-rs0.demand-backup-7562 -2nd .svc.cluster.local myApp test
+ local command=find
+ local uri=myApp:myPass@some-name-rs0.demand-backup-7562
+ local postfix=-2nd
+ local suffix=.svc.cluster.local
+ local database=myApp
+ local collection=test
+ local sort=
+ local tls=false
+ local replicaset=
+ local 'full_command=db.test.find()'
+ [[ -n '' ]]
+ log 'running db.test.find() in myApp'
+ set +o xtrace
[2026-04-14T09:37:30+0000] running db.test.find() in myApp
+ [[ false == true ]]
+ mongo_command=run_mongo
+ run_mongo 'use myApp\n db.test.find()' myApp:myPass@some-name-rs0.demand-backup-7562 mongodb .svc.cluster.local ''
+ local 'command=use myApp\n db.test.find()'
+ local uri=myApp:myPass@some-name-rs0.demand-backup-7562
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local replica_set=rs0
+ grep -E -v 'I NETWORK|W NETWORK|F NETWORK|"c":"NETWORK"|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|Started a new thread for the timer service'
+ [[ myApp:myPass@some-name-rs0.demand-backup-7562 == *cfg* ]]
+ /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.RrRTzMULrK
+++ mktemp
++ local LAST_ERR=/tmp/tmp.JtpQQMOZIW
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.RrRTzMULrK
++ cat /tmp/tmp.JtpQQMOZIW
++ rm /tmp/tmp.RrRTzMULrK /tmp/tmp.JtpQQMOZIW
++ return 0
+ local client_container=psmdb-client-bb8b97679-62vtp
+ kubectl_bin exec psmdb-client-bb8b97679-62vtp -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0.demand-backup-7562.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
++ mktemp
+ local LAST_OUT=/tmp/tmp.F0jitfyV5v
++ mktemp
+ local LAST_ERR=/tmp/tmp.HUob02zkld
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-bb8b97679-62vtp -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0.demand-backup-7562.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.F0jitfyV5v
+ cat /tmp/tmp.HUob02zkld
+ rm /tmp/tmp.F0jitfyV5v /tmp/tmp.HUob02zkld
+ return 0
+ [[ 0 -eq 0 ]]
+ diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2256/e2e-tests/demand-backup/compare/find-2nd.json /tmp/tmp.pzOYzJaY7Q/find-2nd
+ run_restore backup-gcp-cs-s3
+ local backup_name=backup-gcp-cs-s3
+ local restore_file=/mnt/jenkins/workspace/cloud-psmdb-operator_PR-2256/e2e-tests/demand-backup/conf/restore.yml
+ log 'running restore restore-backup-gcp-cs-s3'
+ set +o xtrace
[2026-04-14T09:37:33+0000] running restore restore-backup-gcp-cs-s3
+ cat /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2256/e2e-tests/demand-backup/conf/restore.yml
+ /usr/sbin/sed -e 's/name:/name: restore-backup-gcp-cs-s3/'
+ /usr/sbin/sed -e 's/backupName:/backupName: backup-gcp-cs-s3/'
+ kubectl_bin apply -f -
++ mktemp
+ local LAST_OUT=/tmp/tmp.reAaV2xvS8
++ mktemp
+ local LAST_ERR=/tmp/tmp.iF4YGByvmH
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl apply -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.reAaV2xvS8
perconaservermongodbrestore.psmdb.percona.com/restore-backup-gcp-cs-s3 created
+ cat /tmp/tmp.iF4YGByvmH
+ rm /tmp/tmp.reAaV2xvS8 /tmp/tmp.iF4YGByvmH
+ return 0
+ wait_restore backup-gcp-cs-s3 some-name
+ local backup_name=backup-gcp-cs-s3
+ local cluster_name=some-name
+ local target_state=ready
+ local wait_cluster_consistency=1
+ local wait_time=1780
+ local ok_if_ready=0
+ set +o xtrace
Waiting for the psmdb-restore/restore-backup-gcp-cs-s3 object to be created.OK
Waiting psmdb-restore/restore-backup-gcp-cs-s3 to reach state "ready" .OK after 0 minutes
+ [[ 1 -eq 1 ]]
+ wait_cluster_consistency some-name
+ local cluster_name=some-name
+ local wait_time=32
+ retry=0
+ sleep 7
+ echo -n 'waiting for cluster readyness'
waiting for cluster readyness++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.LF0Crii8uC
+++ mktemp
++ local LAST_ERR=/tmp/tmp.VgTEOL1csq
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.LF0Crii8uC
++ cat /tmp/tmp.VgTEOL1csq
++ rm /tmp/tmp.LF0Crii8uC /tmp/tmp.VgTEOL1csq
++ return 0
+ [[ ready == ready ]]
+ echo .OK
.OK
+ simple_data_check some-name-rs0 3 0
+ local cluster_name=some-name-rs0
+ let last_pod=3-1
+ local isSharded=0
+ local cluster_pfx=
+ '[' 0 -eq 1 ']'
++ seq 0 2
+ for i in $(seq 0 $last_pod)
+ compare_mongo_cmd find myApp:myPass@some-name-rs0-0.some-name-rs0.demand-backup-7562
+ local command=find
+ local uri=myApp:myPass@some-name-rs0-0.some-name-rs0.demand-backup-7562
+ local postfix=
+ local suffix=
+ local database=myApp
+ local collection=test
+ local sort=
+ local tls=false
+ local replicaset=
+ local 'full_command=db.test.find()'
+ [[ -n '' ]]
+ log 'running db.test.find() in myApp'
+ set +o xtrace
[2026-04-14T09:38:00+0000] running db.test.find() in myApp
+ [[ false == true ]]
+ mongo_command=run_mongo
+ run_mongo 'use myApp\n db.test.find()' myApp:myPass@some-name-rs0-0.some-name-rs0.demand-backup-7562 mongodb '' ''
+ grep -E -v 'I NETWORK|W NETWORK|F NETWORK|"c":"NETWORK"|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|Started a new thread for the timer service'
+ local 'command=use myApp\n db.test.find()'
+ local uri=myApp:myPass@some-name-rs0-0.some-name-rs0.demand-backup-7562
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local replica_set=rs0
+ [[ myApp:myPass@some-name-rs0-0.some-name-rs0.demand-backup-7562 == *cfg* ]]
+ /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.kXf9HrXUC0
+++ mktemp
++ local LAST_ERR=/tmp/tmp.yxYIDHJCxV
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.kXf9HrXUC0
++ cat /tmp/tmp.yxYIDHJCxV
++ rm /tmp/tmp.kXf9HrXUC0 /tmp/tmp.yxYIDHJCxV
++ return 0
+ local client_container=psmdb-client-bb8b97679-62vtp
+ kubectl_bin exec psmdb-client-bb8b97679-62vtp -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-0.some-name-rs0.demand-backup-7562.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
++ mktemp
+ local LAST_OUT=/tmp/tmp.JTmjbubjUJ
++ mktemp
+ local LAST_ERR=/tmp/tmp.DlLttFvuKd
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-bb8b97679-62vtp -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-0.some-name-rs0.demand-backup-7562.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.JTmjbubjUJ
+ cat /tmp/tmp.DlLttFvuKd
+ rm /tmp/tmp.JTmjbubjUJ /tmp/tmp.DlLttFvuKd
+ return 0
+ [[ 0 -eq 0 ]]
+ diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2256/e2e-tests/demand-backup/compare/find.json /tmp/tmp.pzOYzJaY7Q/find
+ for i in $(seq 0 $last_pod)
+ compare_mongo_cmd find myApp:myPass@some-name-rs0-1.some-name-rs0.demand-backup-7562
+ local command=find
+ local uri=myApp:myPass@some-name-rs0-1.some-name-rs0.demand-backup-7562
+ local postfix=
+ local suffix=
+ local database=myApp
+ local collection=test
+ local sort=
+ local tls=false
+ local replicaset=
+ local 'full_command=db.test.find()'
+ [[ -n '' ]]
+ log 'running db.test.find() in myApp'
+ set +o xtrace
[2026-04-14T09:38:03+0000] running db.test.find() in myApp
+ [[ false == true ]]
+ mongo_command=run_mongo
+ run_mongo 'use myApp\n db.test.find()' myApp:myPass@some-name-rs0-1.some-name-rs0.demand-backup-7562 mongodb '' ''
+ local 'command=use myApp\n db.test.find()'
+ grep -E -v 'I NETWORK|W NETWORK|F NETWORK|"c":"NETWORK"|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|Started a new thread for the timer service'
+ local uri=myApp:myPass@some-name-rs0-1.some-name-rs0.demand-backup-7562
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local replica_set=rs0
+ [[ myApp:myPass@some-name-rs0-1.some-name-rs0.demand-backup-7562 == *cfg* ]]
+ /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.BBX9Sa1cb0
+++ mktemp
++ local LAST_ERR=/tmp/tmp.oFT9tp51wp
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.BBX9Sa1cb0
++ cat /tmp/tmp.oFT9tp51wp
++ rm /tmp/tmp.BBX9Sa1cb0 /tmp/tmp.oFT9tp51wp
++ return 0
+ local client_container=psmdb-client-bb8b97679-62vtp
+ kubectl_bin exec psmdb-client-bb8b97679-62vtp -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-1.some-name-rs0.demand-backup-7562.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
++ mktemp
+ local LAST_OUT=/tmp/tmp.I5o3XYnj5g
++ mktemp
+ local LAST_ERR=/tmp/tmp.X6BpqUMMFR
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-bb8b97679-62vtp -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-1.some-name-rs0.demand-backup-7562.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.I5o3XYnj5g
+ cat /tmp/tmp.X6BpqUMMFR
+ rm /tmp/tmp.I5o3XYnj5g /tmp/tmp.X6BpqUMMFR
+ return 0
+ [[ 0 -eq 0 ]]
+ diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2256/e2e-tests/demand-backup/compare/find.json /tmp/tmp.pzOYzJaY7Q/find
+ for i in $(seq 0 $last_pod)
+ compare_mongo_cmd find myApp:myPass@some-name-rs0-2.some-name-rs0.demand-backup-7562
+ local command=find
+ local uri=myApp:myPass@some-name-rs0-2.some-name-rs0.demand-backup-7562
+ local postfix=
+ local suffix=
+ local database=myApp
+ local collection=test
+ local sort=
+ local tls=false
+ local replicaset=
+ local 'full_command=db.test.find()'
+ [[ -n '' ]]
+ log 'running db.test.find() in myApp'
+ set +o xtrace
[2026-04-14T09:38:05+0000] running db.test.find() in myApp
+ [[ false == true ]]
+ mongo_command=run_mongo
+ run_mongo 'use myApp\n db.test.find()' myApp:myPass@some-name-rs0-2.some-name-rs0.demand-backup-7562 mongodb '' ''
+ local 'command=use myApp\n db.test.find()'
+ local uri=myApp:myPass@some-name-rs0-2.some-name-rs0.demand-backup-7562
+ local driver=mongodb
+ grep -E -v 'I NETWORK|W NETWORK|F NETWORK|"c":"NETWORK"|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|Started a new thread for the timer service'
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local replica_set=rs0
+ [[ myApp:myPass@some-name-rs0-2.some-name-rs0.demand-backup-7562 == *cfg* ]]
+ /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.cwOvRmpTgc
+++ mktemp
++ local LAST_ERR=/tmp/tmp.8ykQfA82qy
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.cwOvRmpTgc
++ cat /tmp/tmp.8ykQfA82qy
++ rm /tmp/tmp.cwOvRmpTgc /tmp/tmp.8ykQfA82qy
++ return 0
+ local client_container=psmdb-client-bb8b97679-62vtp
+ kubectl_bin exec psmdb-client-bb8b97679-62vtp -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-2.some-name-rs0.demand-backup-7562.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
++ mktemp
+ local LAST_OUT=/tmp/tmp.5oUZTo68wx
++ mktemp
+ local LAST_ERR=/tmp/tmp.A3lvAED8my
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-bb8b97679-62vtp -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-2.some-name-rs0.demand-backup-7562.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.5oUZTo68wx
+ cat /tmp/tmp.A3lvAED8my
+ rm /tmp/tmp.5oUZTo68wx /tmp/tmp.A3lvAED8my
+ return 0
+ [[ 0 -eq 0 ]]
+ diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2256/e2e-tests/demand-backup/compare/find.json /tmp/tmp.pzOYzJaY7Q/find
+ desc 'check backup and restore -- gcp-cs-sa'
+ set +o xtrace


-----------------------------------------------------------------------------------
check backup and restore -- gcp-cs-sa
-----------------------------------------------------------------------------------

++ get_backup_dest backup-gcp-cs-sa
++ local backup_name=backup-gcp-cs-sa
++ kubectl_bin get psmdb-backup backup-gcp-cs-sa -o 'jsonpath={.status.destination}'
++ sed -e 's/.json$//'
++ sed 's|s3://||'
++ sed 's|azure://||'
++ sed 's|gs://||'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.5mCAxZGBUi
+++ mktemp
++ local LAST_ERR=/tmp/tmp.hQLOr4Ew57
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb-backup backup-gcp-cs-sa -o 'jsonpath={.status.destination}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.5mCAxZGBUi
++ cat /tmp/tmp.hQLOr4Ew57
++ rm /tmp/tmp.5mCAxZGBUi /tmp/tmp.hQLOr4Ew57
++ return 0
+ backup_dest_gcp_sa=operator-testing/psmdb-demand-backup/2026-04-14T09:36:21Z
+ check_backup_existence_gcs operator-testing/psmdb-demand-backup/2026-04-14T09:36:21Z /rs0/myApp.test.gz
+ backup_dest_gcp=operator-testing/psmdb-demand-backup/2026-04-14T09:36:21Z
+ obj=/rs0/myApp.test.gz
+ storage_name=gcp-cs
+ retry=0
+ gcs_path=gs://operator-testing/psmdb-demand-backup/2026-04-14T09:36:21Z/rs0/myApp.test.gz
+ gsutil ls gs://operator-testing/psmdb-demand-backup/2026-04-14T09:36:21Z/rs0/myApp.test.gz
+ echo 'Backup gs://operator-testing/psmdb-demand-backup/2026-04-14T09:36:21Z/rs0/myApp.test.gz found in gcp-cs'
Backup gs://operator-testing/psmdb-demand-backup/2026-04-14T09:36:21Z/rs0/myApp.test.gz found in gcp-cs
+ run_recovery_check backup-gcp-cs-sa some-name-rs0
+ local backup=backup-gcp-cs-sa
+ local cluster=some-name-rs0
+ run_mongo 'use myApp\n db.test.insert({ x: 100501 })' myApp:myPass@some-name-rs0.demand-backup-7562
+ local 'command=use myApp\n db.test.insert({ x: 100501 })'
+ local uri=myApp:myPass@some-name-rs0.demand-backup-7562
+ local driver=mongodb+srv
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local replica_set=rs0
+ [[ myApp:myPass@some-name-rs0.demand-backup-7562 == *cfg* ]]
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.iq8hE2d2w0
+++ mktemp
++ local LAST_ERR=/tmp/tmp.jGaOt8j0xk
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.iq8hE2d2w0
++ cat /tmp/tmp.jGaOt8j0xk
++ rm /tmp/tmp.iq8hE2d2w0 /tmp/tmp.jGaOt8j0xk
++ return 0
+ local client_container=psmdb-client-bb8b97679-62vtp
+ kubectl_bin exec psmdb-client-bb8b97679-62vtp -- bash -c 'printf '\''use myApp\n db.test.insert({ x: 100501 })\n'\'' | mongo mongodb+srv://myApp:myPass@some-name-rs0.demand-backup-7562.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
++ mktemp
+ local LAST_OUT=/tmp/tmp.RmBtl1YSed
++ mktemp
+ local LAST_ERR=/tmp/tmp.QmW5vflKEP
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-bb8b97679-62vtp -- bash -c 'printf '\''use myApp\n db.test.insert({ x: 100501 })\n'\'' | mongo mongodb+srv://myApp:myPass@some-name-rs0.demand-backup-7562.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.RmBtl1YSed
Percona Server for MongoDB shell version v4.4.29-28
connecting to: mongodb://some-name-rs0-0.some-name-rs0.demand-backup-7562.svc.cluster.local:27017,some-name-rs0-1.some-name-rs0.demand-backup-7562.svc.cluster.local:27017,some-name-rs0-2.some-name-rs0.demand-backup-7562.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb&replicaSet=rs0&ssl=false
Implicit session: session { "id" : UUID("29945896-3c31-4172-9f63-e2d22b6ed2b2") }
Percona Server for MongoDB server version: v8.0.20-8
WARNING: shell and server versions do not match
switched to db myApp
WriteResult({ "nInserted" : 1 })
bye
+ cat /tmp/tmp.QmW5vflKEP
+ rm /tmp/tmp.RmBtl1YSed /tmp/tmp.QmW5vflKEP
+ return 0
+ compare_mongo_cmd find myApp:myPass@some-name-rs0.demand-backup-7562 -2nd .svc.cluster.local myApp test
+ local command=find
+ local uri=myApp:myPass@some-name-rs0.demand-backup-7562
+ local postfix=-2nd
+ local suffix=.svc.cluster.local
+ local database=myApp
+ local collection=test
+ local sort=
+ local tls=false
+ local replicaset=
+ local 'full_command=db.test.find()'
+ [[ -n '' ]]
+ log 'running db.test.find() in myApp'
+ set +o xtrace
[2026-04-14T09:38:13+0000] running db.test.find() in myApp
+ [[ false == true ]]
+ mongo_command=run_mongo
+ run_mongo 'use myApp\n db.test.find()' myApp:myPass@some-name-rs0.demand-backup-7562 mongodb .svc.cluster.local ''
+ local 'command=use myApp\n db.test.find()'
+ local uri=myApp:myPass@some-name-rs0.demand-backup-7562
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ grep -E -v 'I NETWORK|W NETWORK|F NETWORK|"c":"NETWORK"|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|Started a new thread for the timer service'
+ local replica_set=rs0
+ [[ myApp:myPass@some-name-rs0.demand-backup-7562 == *cfg* ]]
+ /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.aodLTloICv
+++ mktemp
++ local LAST_ERR=/tmp/tmp.nw1zTULbZo
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.aodLTloICv
++ cat /tmp/tmp.nw1zTULbZo
++ rm /tmp/tmp.aodLTloICv /tmp/tmp.nw1zTULbZo
++ return 0
+ local client_container=psmdb-client-bb8b97679-62vtp
+ kubectl_bin exec psmdb-client-bb8b97679-62vtp -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0.demand-backup-7562.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
++ mktemp
+ local LAST_OUT=/tmp/tmp.eKz0kefbRJ
++ mktemp
+ local LAST_ERR=/tmp/tmp.T1knZhiG4P
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-bb8b97679-62vtp -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0.demand-backup-7562.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.eKz0kefbRJ
+ cat /tmp/tmp.T1knZhiG4P
+ rm /tmp/tmp.eKz0kefbRJ /tmp/tmp.T1knZhiG4P
+ return 0
+ [[ 0 -eq 0 ]]
+ diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2256/e2e-tests/demand-backup/compare/find-2nd.json /tmp/tmp.pzOYzJaY7Q/find-2nd
+ run_restore backup-gcp-cs-sa
+ local backup_name=backup-gcp-cs-sa
+ local restore_file=/mnt/jenkins/workspace/cloud-psmdb-operator_PR-2256/e2e-tests/demand-backup/conf/restore.yml
+ log 'running restore restore-backup-gcp-cs-sa'
+ set +o xtrace
[2026-04-14T09:38:16+0000] running restore restore-backup-gcp-cs-sa
+ cat /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2256/e2e-tests/demand-backup/conf/restore.yml
+ /usr/sbin/sed -e 's/name:/name: restore-backup-gcp-cs-sa/'
+ /usr/sbin/sed -e 's/backupName:/backupName: backup-gcp-cs-sa/'
+ kubectl_bin apply -f -
++ mktemp
+ local LAST_OUT=/tmp/tmp.xzDiNvchXw
++ mktemp
+ local LAST_ERR=/tmp/tmp.vnWo8yuO9H
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl apply -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.xzDiNvchXw
perconaservermongodbrestore.psmdb.percona.com/restore-backup-gcp-cs-sa created
+ cat /tmp/tmp.vnWo8yuO9H
+ rm /tmp/tmp.xzDiNvchXw /tmp/tmp.vnWo8yuO9H
+ return 0
+ wait_restore backup-gcp-cs-sa some-name
+ local backup_name=backup-gcp-cs-sa
+ local cluster_name=some-name
+ local target_state=ready
+ local wait_cluster_consistency=1
+ local wait_time=1780
+ local ok_if_ready=0
+ set +o xtrace
Waiting for the psmdb-restore/restore-backup-gcp-cs-sa object to be created.OK
Waiting psmdb-restore/restore-backup-gcp-cs-sa to reach state "ready" .OK after 0 minutes
+ [[ 1 -eq 1 ]]
+ wait_cluster_consistency some-name
+ local cluster_name=some-name
+ local wait_time=32
+ retry=0
+ sleep 7
+ echo -n 'waiting for cluster readyness'
waiting for cluster readyness++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.R9ba7U1YUE
+++ mktemp
++ local LAST_ERR=/tmp/tmp.2x5RQ7bqCV
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.R9ba7U1YUE
++ cat /tmp/tmp.2x5RQ7bqCV
++ rm /tmp/tmp.R9ba7U1YUE /tmp/tmp.2x5RQ7bqCV
++ return 0
+ [[ ready == ready ]]
+ echo .OK
.OK
+ simple_data_check some-name-rs0 3 0
+ local cluster_name=some-name-rs0
+ let last_pod=3-1
+ local isSharded=0
+ local cluster_pfx=
+ '[' 0 -eq 1 ']'
++ seq 0 2
+ for i in $(seq 0 $last_pod)
+ compare_mongo_cmd find myApp:myPass@some-name-rs0-0.some-name-rs0.demand-backup-7562
+ local command=find
+ local uri=myApp:myPass@some-name-rs0-0.some-name-rs0.demand-backup-7562
+ local postfix=
+ local suffix=
+ local database=myApp
+ local collection=test
+ local sort=
+ local tls=false
+ local replicaset=
+ local 'full_command=db.test.find()'
+ [[ -n '' ]]
+ log 'running db.test.find() in myApp'
+ set +o xtrace
[2026-04-14T09:38:39+0000] running db.test.find() in myApp
+ [[ false == true ]]
+ mongo_command=run_mongo
+ run_mongo 'use myApp\n db.test.find()' myApp:myPass@some-name-rs0-0.some-name-rs0.demand-backup-7562 mongodb '' ''
+ local 'command=use myApp\n db.test.find()'
+ local uri=myApp:myPass@some-name-rs0-0.some-name-rs0.demand-backup-7562
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local replica_set=rs0
+ [[ myApp:myPass@some-name-rs0-0.some-name-rs0.demand-backup-7562 == *cfg* ]]
+ /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
+ grep -E -v 'I NETWORK|W NETWORK|F NETWORK|"c":"NETWORK"|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|Started a new thread for the timer service'
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.e0cUmXbyoz
+++ mktemp
++ local LAST_ERR=/tmp/tmp.nM3x0tCCvh
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.e0cUmXbyoz
++ cat /tmp/tmp.nM3x0tCCvh
++ rm /tmp/tmp.e0cUmXbyoz /tmp/tmp.nM3x0tCCvh
++ return 0
+ local client_container=psmdb-client-bb8b97679-62vtp
+ kubectl_bin exec psmdb-client-bb8b97679-62vtp -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-0.some-name-rs0.demand-backup-7562.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
++ mktemp
+ local LAST_OUT=/tmp/tmp.3pQ7mTipgM
++ mktemp
+ local LAST_ERR=/tmp/tmp.09ecq0Qkz4
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-bb8b97679-62vtp -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-0.some-name-rs0.demand-backup-7562.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.3pQ7mTipgM
+ cat /tmp/tmp.09ecq0Qkz4
+ rm /tmp/tmp.3pQ7mTipgM /tmp/tmp.09ecq0Qkz4
+ return 0
+ [[ 0 -eq 0 ]]
+ diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2256/e2e-tests/demand-backup/compare/find.json /tmp/tmp.pzOYzJaY7Q/find
+ for i in $(seq 0 $last_pod)
+ compare_mongo_cmd find myApp:myPass@some-name-rs0-1.some-name-rs0.demand-backup-7562
+ local command=find
+ local uri=myApp:myPass@some-name-rs0-1.some-name-rs0.demand-backup-7562
+ local postfix=
+ local suffix=
+ local database=myApp
+ local collection=test
+ local sort=
+ local tls=false
+ local replicaset=
+ local 'full_command=db.test.find()'
+ [[ -n '' ]]
+ log 'running db.test.find() in myApp'
+ set +o xtrace
[2026-04-14T09:38:42+0000] running db.test.find() in myApp
+ [[ false == true ]]
+ mongo_command=run_mongo
+ run_mongo 'use myApp\n db.test.find()' myApp:myPass@some-name-rs0-1.some-name-rs0.demand-backup-7562 mongodb '' ''
+ local 'command=use myApp\n db.test.find()'
+ grep -E -v 'I NETWORK|W NETWORK|F NETWORK|"c":"NETWORK"|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|Started a new thread for the timer service'
+ local uri=myApp:myPass@some-name-rs0-1.some-name-rs0.demand-backup-7562
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local replica_set=rs0
+ [[ myApp:myPass@some-name-rs0-1.some-name-rs0.demand-backup-7562 == *cfg* ]]
+ /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.opLXNvSVpU
+++ mktemp
++ local LAST_ERR=/tmp/tmp.9M4lpOBhWa
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.opLXNvSVpU
++ cat /tmp/tmp.9M4lpOBhWa
++ rm /tmp/tmp.opLXNvSVpU /tmp/tmp.9M4lpOBhWa
++ return 0
+ local client_container=psmdb-client-bb8b97679-62vtp
+ kubectl_bin exec psmdb-client-bb8b97679-62vtp -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-1.some-name-rs0.demand-backup-7562.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
++ mktemp
+ local LAST_OUT=/tmp/tmp.p23HAw5fyE
++ mktemp
+ local LAST_ERR=/tmp/tmp.9uexialafZ
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-bb8b97679-62vtp -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-1.some-name-rs0.demand-backup-7562.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.p23HAw5fyE
+ cat /tmp/tmp.9uexialafZ
+ rm /tmp/tmp.p23HAw5fyE /tmp/tmp.9uexialafZ
+ return 0
+ [[ 0 -eq 0 ]]
+ diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2256/e2e-tests/demand-backup/compare/find.json /tmp/tmp.pzOYzJaY7Q/find
+ for i in $(seq 0 $last_pod)
+ compare_mongo_cmd find myApp:myPass@some-name-rs0-2.some-name-rs0.demand-backup-7562
+ local command=find
+ local uri=myApp:myPass@some-name-rs0-2.some-name-rs0.demand-backup-7562
+ local postfix=
+ local suffix=
+ local database=myApp
+ local collection=test
+ local sort=
+ local tls=false
+ local replicaset=
+ local 'full_command=db.test.find()'
+ [[ -n '' ]]
+ log 'running db.test.find() in myApp'
+ set +o xtrace
[2026-04-14T09:38:45+0000] running db.test.find() in myApp
+ [[ false == true ]]
+ mongo_command=run_mongo
+ run_mongo 'use myApp\n db.test.find()' myApp:myPass@some-name-rs0-2.some-name-rs0.demand-backup-7562 mongodb '' ''
+ local 'command=use myApp\n db.test.find()'
+ local uri=myApp:myPass@some-name-rs0-2.some-name-rs0.demand-backup-7562
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local replica_set=rs0
+ [[ myApp:myPass@some-name-rs0-2.some-name-rs0.demand-backup-7562 == *cfg* ]]
+ /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
+ grep -E -v 'I NETWORK|W NETWORK|F NETWORK|"c":"NETWORK"|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|Started a new thread for the timer service'
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.NRuAPX45NF
+++ mktemp
++ local LAST_ERR=/tmp/tmp.0IJMAzduqQ
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.NRuAPX45NF
++ cat /tmp/tmp.0IJMAzduqQ
++ rm /tmp/tmp.NRuAPX45NF /tmp/tmp.0IJMAzduqQ
++ return 0
+ local client_container=psmdb-client-bb8b97679-62vtp
+ kubectl_bin exec psmdb-client-bb8b97679-62vtp -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-2.some-name-rs0.demand-backup-7562.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
++ mktemp
+ local LAST_OUT=/tmp/tmp.Jaakj5So27
++ mktemp
+ local LAST_ERR=/tmp/tmp.obFzCmgkyN
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-bb8b97679-62vtp -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-2.some-name-rs0.demand-backup-7562.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.Jaakj5So27
+ cat /tmp/tmp.obFzCmgkyN
+ rm /tmp/tmp.Jaakj5So27 /tmp/tmp.obFzCmgkyN
+ return 0
+ [[ 0 -eq 0 ]]
+ diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2256/e2e-tests/demand-backup/compare/find.json /tmp/tmp.pzOYzJaY7Q/find
+ desc 'check backup and restore -- azure-blob'
+ set +o xtrace


-----------------------------------------------------------------------------------
check backup and restore -- azure-blob
-----------------------------------------------------------------------------------

++ get_backup_dest backup-azure-blob
++ local backup_name=backup-azure-blob
++ kubectl_bin get psmdb-backup backup-azure-blob -o 'jsonpath={.status.destination}'
++ sed -e 's/.json$//'
++ sed 's|s3://||'
+++ mktemp
++ sed 's|azure://||'
++ sed 's|gs://||'
++ local LAST_OUT=/tmp/tmp.5Koiwt1I9J
+++ mktemp
++ local LAST_ERR=/tmp/tmp.dZhcJEdrJT
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb-backup backup-azure-blob -o 'jsonpath={.status.destination}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.5Koiwt1I9J
++ cat /tmp/tmp.dZhcJEdrJT
++ rm /tmp/tmp.5Koiwt1I9J /tmp/tmp.dZhcJEdrJT
++ return 0
+ backup_dest_azure=https://engk8soperators.blob.core.windows.net/operator-testing/psmdb-demand-backup/2026-04-14T09:35:47Z
+ setup_azure_credentials
+ local secret_name=azure-secret
+ echo 'Setting up Azure credentials from K8s secret: azure-secret'
Setting up Azure credentials from K8s secret: azure-secret
+ local trace_was_on=0
+ [[ ehxB == *x* ]]
+ trace_was_on=1
+ set +x
+ echo 'Azure credentials configured successfully'
Azure credentials configured successfully
+ check_backup_existence_azure https://engk8soperators.blob.core.windows.net/operator-testing/psmdb-demand-backup/2026-04-14T09:35:47Z /rs0/myApp.test.gz
++ echo https://engk8soperators.blob.core.windows.net/operator-testing/psmdb-demand-backup/2026-04-14T09:35:47Z
++ sed 's|https://[^/]*\.blob\.core\.windows\.net/||'
+ url_path=operator-testing/psmdb-demand-backup/2026-04-14T09:35:47Z
++ echo operator-testing/psmdb-demand-backup/2026-04-14T09:35:47Z
++ cut -d/ -f1
+ container=operator-testing
++ echo operator-testing/psmdb-demand-backup/2026-04-14T09:35:47Z
++ cut -d/ -f2-
+ blob_prefix=psmdb-demand-backup/2026-04-14T09:35:47Z
+ blob=/rs0/myApp.test.gz
+ storage_name=azure-blob
+ retry=0
+ blob_path=psmdb-demand-backup/2026-04-14T09:35:47Z/rs0/myApp.test.gz
+ az storage blob show --container-name operator-testing --name psmdb-demand-backup/2026-04-14T09:35:47Z/rs0/myApp.test.gz
+ echo 'Backup psmdb-demand-backup/2026-04-14T09:35:47Z/rs0/myApp.test.gz found in azure-blob'
Backup psmdb-demand-backup/2026-04-14T09:35:47Z/rs0/myApp.test.gz found in azure-blob
+ run_recovery_check backup-azure-blob some-name-rs0
+ local backup=backup-azure-blob
+ local cluster=some-name-rs0
+ run_mongo 'use myApp\n db.test.insert({ x: 100501 })' myApp:myPass@some-name-rs0.demand-backup-7562
+ local 'command=use myApp\n db.test.insert({ x: 100501 })'
+ local uri=myApp:myPass@some-name-rs0.demand-backup-7562
+ local driver=mongodb+srv
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local replica_set=rs0
+ [[ myApp:myPass@some-name-rs0.demand-backup-7562 == *cfg* ]]
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.3YPR0tf4OX
+++ mktemp
++ local LAST_ERR=/tmp/tmp.BWRBBddhkJ
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.3YPR0tf4OX
++ cat /tmp/tmp.BWRBBddhkJ
++ rm /tmp/tmp.3YPR0tf4OX /tmp/tmp.BWRBBddhkJ
++ return 0
+ local client_container=psmdb-client-bb8b97679-62vtp
+ kubectl_bin exec psmdb-client-bb8b97679-62vtp -- bash -c 'printf '\''use myApp\n db.test.insert({ x: 100501 })\n'\'' | mongo mongodb+srv://myApp:myPass@some-name-rs0.demand-backup-7562.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
++ mktemp
+ local LAST_OUT=/tmp/tmp.hXKAtX1vVF
++ mktemp
+ local LAST_ERR=/tmp/tmp.hiFmBLHAY9
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-bb8b97679-62vtp -- bash -c 'printf '\''use myApp\n db.test.insert({ x: 100501 })\n'\'' | mongo mongodb+srv://myApp:myPass@some-name-rs0.demand-backup-7562.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.hXKAtX1vVF
Percona Server for MongoDB shell version v4.4.29-28
connecting to: mongodb://some-name-rs0-0.some-name-rs0.demand-backup-7562.svc.cluster.local:27017,some-name-rs0-1.some-name-rs0.demand-backup-7562.svc.cluster.local:27017,some-name-rs0-2.some-name-rs0.demand-backup-7562.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb&replicaSet=rs0&ssl=false
Implicit session: session { "id" : UUID("b2d51e34-4927-41d8-9613-6089ea0bab9a") }
Percona Server for MongoDB server version: v8.0.20-8
WARNING: shell and server versions do not match
switched to db myApp
WriteResult({ "nInserted" : 1 })
bye
+ cat /tmp/tmp.hiFmBLHAY9
+ rm /tmp/tmp.hXKAtX1vVF /tmp/tmp.hiFmBLHAY9
+ return 0
+ compare_mongo_cmd find myApp:myPass@some-name-rs0.demand-backup-7562 -2nd .svc.cluster.local myApp test
+ local command=find
+ local uri=myApp:myPass@some-name-rs0.demand-backup-7562
+ local postfix=-2nd
+ local suffix=.svc.cluster.local
+ local database=myApp
+ local collection=test
+ local sort=
+ local tls=false
+ local replicaset=
+ local 'full_command=db.test.find()'
+ [[ -n '' ]]
+ log 'running db.test.find() in myApp'
+ set +o xtrace
[2026-04-14T09:38:57+0000] running db.test.find() in myApp
+ [[ false == true ]]
+ mongo_command=run_mongo
+ run_mongo 'use myApp\n db.test.find()' myApp:myPass@some-name-rs0.demand-backup-7562 mongodb .svc.cluster.local ''
+ local 'command=use myApp\n db.test.find()'
+ local uri=myApp:myPass@some-name-rs0.demand-backup-7562
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local replica_set=rs0
+ [[ myApp:myPass@some-name-rs0.demand-backup-7562 == *cfg* ]]
+ grep -E -v 'I NETWORK|W NETWORK|F NETWORK|"c":"NETWORK"|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|Started a new thread for the timer service'
+ /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.2mbD95nm6u
+++ mktemp
++ local LAST_ERR=/tmp/tmp.bv8Ltnp9KK
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.2mbD95nm6u
++ cat /tmp/tmp.bv8Ltnp9KK
++ rm /tmp/tmp.2mbD95nm6u /tmp/tmp.bv8Ltnp9KK
++ return 0
+ local client_container=psmdb-client-bb8b97679-62vtp
+ kubectl_bin exec psmdb-client-bb8b97679-62vtp -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0.demand-backup-7562.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
++ mktemp
+ local LAST_OUT=/tmp/tmp.7vqu4xz18z
++ mktemp
+ local LAST_ERR=/tmp/tmp.jQOZUna5Vr
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-bb8b97679-62vtp -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0.demand-backup-7562.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.7vqu4xz18z
+ cat /tmp/tmp.jQOZUna5Vr
+ rm /tmp/tmp.7vqu4xz18z /tmp/tmp.jQOZUna5Vr
+ return 0
+ [[ 0 -eq 0 ]]
+ diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2256/e2e-tests/demand-backup/compare/find-2nd.json /tmp/tmp.pzOYzJaY7Q/find-2nd
+ run_restore backup-azure-blob
+ local backup_name=backup-azure-blob
+ local restore_file=/mnt/jenkins/workspace/cloud-psmdb-operator_PR-2256/e2e-tests/demand-backup/conf/restore.yml
+ log 'running restore restore-backup-azure-blob'
+ set +o xtrace
[2026-04-14T09:39:00+0000] running restore restore-backup-azure-blob
+ cat /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2256/e2e-tests/demand-backup/conf/restore.yml
+ /usr/sbin/sed -e 's/name:/name: restore-backup-azure-blob/'
+ /usr/sbin/sed -e 's/backupName:/backupName: backup-azure-blob/'
+ kubectl_bin apply -f -
++ mktemp
+ local LAST_OUT=/tmp/tmp.KbwlDv7w3T
++ mktemp
+ local LAST_ERR=/tmp/tmp.MWdJEN4lWN
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl apply -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.KbwlDv7w3T
perconaservermongodbrestore.psmdb.percona.com/restore-backup-azure-blob created
+ cat /tmp/tmp.MWdJEN4lWN
+ rm /tmp/tmp.KbwlDv7w3T /tmp/tmp.MWdJEN4lWN
+ return 0
+ wait_restore backup-azure-blob some-name
+ local backup_name=backup-azure-blob
+ local cluster_name=some-name
+ local target_state=ready
+ local wait_cluster_consistency=1
+ local wait_time=1780
+ local ok_if_ready=0
+ set +o xtrace
Waiting for the psmdb-restore/restore-backup-azure-blob object to be created.OK
Waiting psmdb-restore/restore-backup-azure-blob to reach state "ready" .OK after 0 minutes
+ [[ 1 -eq 1 ]]
+ wait_cluster_consistency some-name
+ local cluster_name=some-name
+ local wait_time=32
+ retry=0
+ sleep 7
+ echo -n 'waiting for cluster readyness'
waiting for cluster readyness++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.Tch98RO2Nu
+++ mktemp
++ local LAST_ERR=/tmp/tmp.xSt9ORkDL4
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.Tch98RO2Nu
++ cat /tmp/tmp.xSt9ORkDL4
++ rm /tmp/tmp.Tch98RO2Nu /tmp/tmp.xSt9ORkDL4
++ return 0
+ [[ ready == ready ]]
+ echo .OK
.OK
+ simple_data_check some-name-rs0 3 0
+ local cluster_name=some-name-rs0
+ let last_pod=3-1
+ local isSharded=0
+ local cluster_pfx=
+ '[' 0 -eq 1 ']'
++ seq 0 2
+ for i in $(seq 0 $last_pod)
+ compare_mongo_cmd find myApp:myPass@some-name-rs0-0.some-name-rs0.demand-backup-7562
+ local command=find
+ local uri=myApp:myPass@some-name-rs0-0.some-name-rs0.demand-backup-7562
+ local postfix=
+ local suffix=
+ local database=myApp
+ local collection=test
+ local sort=
+ local tls=false
+ local replicaset=
+ local 'full_command=db.test.find()'
+ [[ -n '' ]]
+ log 'running db.test.find() in myApp'
+ set +o xtrace
[2026-04-14T09:39:23+0000] running db.test.find() in myApp
+ [[ false == true ]]
+ mongo_command=run_mongo
+ run_mongo 'use myApp\n db.test.find()' myApp:myPass@some-name-rs0-0.some-name-rs0.demand-backup-7562 mongodb '' ''
+ local 'command=use myApp\n db.test.find()'
+ local uri=myApp:myPass@some-name-rs0-0.some-name-rs0.demand-backup-7562
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local replica_set=rs0
+ [[ myApp:myPass@some-name-rs0-0.some-name-rs0.demand-backup-7562 == *cfg* ]]
+ grep -E -v 'I NETWORK|W NETWORK|F NETWORK|"c":"NETWORK"|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|Started a new thread for the timer service'
+ /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.4mdNnpwfpb
+++ mktemp
++ local LAST_ERR=/tmp/tmp.CHfSIuvkA9
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.4mdNnpwfpb
++ cat /tmp/tmp.CHfSIuvkA9
++ rm /tmp/tmp.4mdNnpwfpb /tmp/tmp.CHfSIuvkA9
++ return 0
+ local client_container=psmdb-client-bb8b97679-62vtp
+ kubectl_bin exec psmdb-client-bb8b97679-62vtp -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-0.some-name-rs0.demand-backup-7562.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
++ mktemp
+ local LAST_OUT=/tmp/tmp.8tdrqUARxC
++ mktemp
+ local LAST_ERR=/tmp/tmp.DJznjfISXo
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-bb8b97679-62vtp -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-0.some-name-rs0.demand-backup-7562.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.8tdrqUARxC
+ cat /tmp/tmp.DJznjfISXo
+ rm /tmp/tmp.8tdrqUARxC /tmp/tmp.DJznjfISXo
+ return 0
+ [[ 0 -eq 0 ]]
+ diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2256/e2e-tests/demand-backup/compare/find.json /tmp/tmp.pzOYzJaY7Q/find
+ for i in $(seq 0 $last_pod)
+ compare_mongo_cmd find myApp:myPass@some-name-rs0-1.some-name-rs0.demand-backup-7562
+ local command=find
+ local uri=myApp:myPass@some-name-rs0-1.some-name-rs0.demand-backup-7562
+ local postfix=
+ local suffix=
+ local database=myApp
+ local collection=test
+ local sort=
+ local tls=false
+ local replicaset=
+ local 'full_command=db.test.find()'
+ [[ -n '' ]]
+ log 'running db.test.find() in myApp'
+ set +o xtrace
[2026-04-14T09:39:26+0000] running db.test.find() in myApp
+ [[ false == true ]]
+ mongo_command=run_mongo
+ run_mongo 'use myApp\n db.test.find()' myApp:myPass@some-name-rs0-1.some-name-rs0.demand-backup-7562 mongodb '' ''
+ local 'command=use myApp\n db.test.find()'
+ local uri=myApp:myPass@some-name-rs0-1.some-name-rs0.demand-backup-7562
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local replica_set=rs0
+ [[ myApp:myPass@some-name-rs0-1.some-name-rs0.demand-backup-7562 == *cfg* ]]
+ grep -E -v 'I NETWORK|W NETWORK|F NETWORK|"c":"NETWORK"|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|Started a new thread for the timer service'
+ /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.WxkY0dBiL2
+++ mktemp
++ local LAST_ERR=/tmp/tmp.wnHYd92Wev
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.WxkY0dBiL2
++ cat /tmp/tmp.wnHYd92Wev
++ rm /tmp/tmp.WxkY0dBiL2 /tmp/tmp.wnHYd92Wev
++ return 0
+ local client_container=psmdb-client-bb8b97679-62vtp
+ kubectl_bin exec psmdb-client-bb8b97679-62vtp -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-1.some-name-rs0.demand-backup-7562.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
++ mktemp
+ local LAST_OUT=/tmp/tmp.YYSxJz8WUV
++ mktemp
+ local LAST_ERR=/tmp/tmp.V7I3gj8VTk
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-bb8b97679-62vtp -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-1.some-name-rs0.demand-backup-7562.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.YYSxJz8WUV
+ cat /tmp/tmp.V7I3gj8VTk
+ rm /tmp/tmp.YYSxJz8WUV /tmp/tmp.V7I3gj8VTk
+ return 0
+ [[ 0 -eq 0 ]]
+ diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2256/e2e-tests/demand-backup/compare/find.json /tmp/tmp.pzOYzJaY7Q/find
+ for i in $(seq 0 $last_pod)
+ compare_mongo_cmd find myApp:myPass@some-name-rs0-2.some-name-rs0.demand-backup-7562
+ local command=find
+ local uri=myApp:myPass@some-name-rs0-2.some-name-rs0.demand-backup-7562
+ local postfix=
+ local suffix=
+ local database=myApp
+ local collection=test
+ local sort=
+ local tls=false
+ local replicaset=
+ local 'full_command=db.test.find()'
+ [[ -n '' ]]
+ log 'running db.test.find() in myApp'
+ set +o xtrace
[2026-04-14T09:39:28+0000] running db.test.find() in myApp
+ [[ false == true ]]
+ mongo_command=run_mongo
+ run_mongo 'use myApp\n db.test.find()' myApp:myPass@some-name-rs0-2.some-name-rs0.demand-backup-7562 mongodb '' ''
+ local 'command=use myApp\n db.test.find()'
+ local uri=myApp:myPass@some-name-rs0-2.some-name-rs0.demand-backup-7562
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local replica_set=rs0
+ [[ myApp:myPass@some-name-rs0-2.some-name-rs0.demand-backup-7562 == *cfg* ]]
+ grep -E -v 'I NETWORK|W NETWORK|F NETWORK|"c":"NETWORK"|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|Started a new thread for the timer service'
+ /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.hAR0Khd7UY
+++ mktemp
++ local LAST_ERR=/tmp/tmp.QsnJvxwnrV
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.hAR0Khd7UY
++ cat /tmp/tmp.QsnJvxwnrV
++ rm /tmp/tmp.hAR0Khd7UY /tmp/tmp.QsnJvxwnrV
++ return 0
+ local client_container=psmdb-client-bb8b97679-62vtp
+ kubectl_bin exec psmdb-client-bb8b97679-62vtp -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-2.some-name-rs0.demand-backup-7562.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
++ mktemp
+ local LAST_OUT=/tmp/tmp.LrzqUvvowi
++ mktemp
+ local LAST_ERR=/tmp/tmp.eCWGri1Wxw
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-bb8b97679-62vtp -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-2.some-name-rs0.demand-backup-7562.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.LrzqUvvowi
+ cat /tmp/tmp.eCWGri1Wxw
+ rm /tmp/tmp.LrzqUvvowi /tmp/tmp.eCWGri1Wxw
+ return 0
+ [[ 0 -eq 0 ]]
+ diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2256/e2e-tests/demand-backup/compare/find.json /tmp/tmp.pzOYzJaY7Q/find
+ desc 'check backup and restore -- minio'
+ set +o xtrace


-----------------------------------------------------------------------------------
check backup and restore -- minio
-----------------------------------------------------------------------------------

+ check_backup_in_storage backup-minio minio rs0 myApp.test.gz
+ local backup=backup-minio
+ local storage_type=minio
+ local replset=rs0
+ local file=myApp.test.gz
+ local protocol=http
+ local endpoint
++ get_backup_dest backup-minio
++ local backup_name=backup-minio
++ /usr/sbin/sed 's|https://engk8soperators.blob.core.windows.net/||'
++ kubectl_bin get psmdb-backup backup-minio -o 'jsonpath={.status.destination}'
++ sed -e 's/.json$//'
+++ mktemp
++ sed 's|s3://||'
++ sed 's|gs://||'
++ sed 's|azure://||'
++ local LAST_OUT=/tmp/tmp.QkJDWsntch
+++ mktemp
++ local LAST_ERR=/tmp/tmp.UQMMWbK0rm
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb-backup backup-minio -o 'jsonpath={.status.destination}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.QkJDWsntch
++ cat /tmp/tmp.UQMMWbK0rm
++ rm /tmp/tmp.QkJDWsntch /tmp/tmp.UQMMWbK0rm
++ return 0
+ backup_dest=operator-testing/2026-04-14T09:36:10Z
+ case ${storage_type} in
+ endpoint=minio-service
+ kubectl_bin run -i --rm aws-cli --image=docker.io/perconalab/awscli --restart=Never -- /usr/bin/env AWS_ACCESS_KEY_ID=some-access-key AWS_SECRET_ACCESS_KEY=some-secret-key AWS_DEFAULT_REGION=us-east-1 /usr/bin/aws --no-verify-ssl --endpoint-url http://minio-service:9000 s3 ls s3://operator-testing/2026-04-14T09:36:10Z/rs0/myApp.test.gz
+ grep myApp.test.gz
++ mktemp
+ local LAST_OUT=/tmp/tmp.0n3KAUQYIT
++ mktemp
+ local LAST_ERR=/tmp/tmp.itqfvhPpQ8
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl run -i --rm aws-cli --image=docker.io/perconalab/awscli --restart=Never -- /usr/bin/env AWS_ACCESS_KEY_ID=some-access-key AWS_SECRET_ACCESS_KEY=some-secret-key AWS_DEFAULT_REGION=us-east-1 /usr/bin/aws --no-verify-ssl --endpoint-url http://minio-service:9000 s3 ls s3://operator-testing/2026-04-14T09:36:10Z/rs0/myApp.test.gz
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.0n3KAUQYIT
+ cat /tmp/tmp.itqfvhPpQ8
+ rm /tmp/tmp.0n3KAUQYIT /tmp/tmp.itqfvhPpQ8
+ return 0
2026-04-14 09:36:13         55 myApp.test.gz
+ run_recovery_check backup-minio some-name-rs0
+ local backup=backup-minio
+ local cluster=some-name-rs0
+ run_mongo 'use myApp\n db.test.insert({ x: 100501 })' myApp:myPass@some-name-rs0.demand-backup-7562
+ local 'command=use myApp\n db.test.insert({ x: 100501 })'
+ local uri=myApp:myPass@some-name-rs0.demand-backup-7562
+ local driver=mongodb+srv
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local replica_set=rs0
+ [[ myApp:myPass@some-name-rs0.demand-backup-7562 == *cfg* ]]
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.byo3SyHZur
+++ mktemp
++ local LAST_ERR=/tmp/tmp.icvkREVmGg
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.byo3SyHZur
++ cat /tmp/tmp.icvkREVmGg
++ rm /tmp/tmp.byo3SyHZur /tmp/tmp.icvkREVmGg
++ return 0
+ local client_container=psmdb-client-bb8b97679-62vtp
+ kubectl_bin exec psmdb-client-bb8b97679-62vtp -- bash -c 'printf '\''use myApp\n db.test.insert({ x: 100501 })\n'\'' | mongo mongodb+srv://myApp:myPass@some-name-rs0.demand-backup-7562.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
++ mktemp
+ local LAST_OUT=/tmp/tmp.VATaQDe6hh
++ mktemp
+ local LAST_ERR=/tmp/tmp.zoYeJYlrnh
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-bb8b97679-62vtp -- bash -c 'printf '\''use myApp\n db.test.insert({ x: 100501 })\n'\'' | mongo mongodb+srv://myApp:myPass@some-name-rs0.demand-backup-7562.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.VATaQDe6hh
Percona Server for MongoDB shell version v4.4.29-28
connecting to: mongodb://some-name-rs0-0.some-name-rs0.demand-backup-7562.svc.cluster.local:27017,some-name-rs0-1.some-name-rs0.demand-backup-7562.svc.cluster.local:27017,some-name-rs0-2.some-name-rs0.demand-backup-7562.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb&replicaSet=rs0&ssl=false
Implicit session: session { "id" : UUID("314c6fb3-e7df-4338-9e05-60c5b37f6a05") }
Percona Server for MongoDB server version: v8.0.20-8
WARNING: shell and server versions do not match
switched to db myApp
WriteResult({ "nInserted" : 1 })
bye
+ cat /tmp/tmp.zoYeJYlrnh
+ rm /tmp/tmp.VATaQDe6hh /tmp/tmp.zoYeJYlrnh
+ return 0
+ compare_mongo_cmd find myApp:myPass@some-name-rs0.demand-backup-7562 -2nd .svc.cluster.local myApp test
+ local command=find
+ local uri=myApp:myPass@some-name-rs0.demand-backup-7562
+ local postfix=-2nd
+ local suffix=.svc.cluster.local
+ local database=myApp
+ local collection=test
+ local sort=
+ local tls=false
+ local replicaset=
+ local 'full_command=db.test.find()'
+ [[ -n '' ]]
+ log 'running db.test.find() in myApp'
+ set +o xtrace
[2026-04-14T09:39:40+0000] running db.test.find() in myApp
+ [[ false == true ]]
+ mongo_command=run_mongo
+ run_mongo 'use myApp\n db.test.find()' myApp:myPass@some-name-rs0.demand-backup-7562 mongodb .svc.cluster.local ''
+ local 'command=use myApp\n db.test.find()'
+ local uri=myApp:myPass@some-name-rs0.demand-backup-7562
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local replica_set=rs0
+ [[ myApp:myPass@some-name-rs0.demand-backup-7562 == *cfg* ]]
+ grep -E -v 'I NETWORK|W NETWORK|F NETWORK|"c":"NETWORK"|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|Started a new thread for the timer service'
+ /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.cXwK1vM9Wy
+++ mktemp
++ local LAST_ERR=/tmp/tmp.44TwGRat7y
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.cXwK1vM9Wy
++ cat /tmp/tmp.44TwGRat7y
++ rm /tmp/tmp.cXwK1vM9Wy /tmp/tmp.44TwGRat7y
++ return 0
+ local client_container=psmdb-client-bb8b97679-62vtp
+ kubectl_bin exec psmdb-client-bb8b97679-62vtp -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0.demand-backup-7562.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
++ mktemp
+ local LAST_OUT=/tmp/tmp.CuchBfPFbT
++ mktemp
+ local LAST_ERR=/tmp/tmp.cZa5crnzep
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-bb8b97679-62vtp -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0.demand-backup-7562.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.CuchBfPFbT
+ cat /tmp/tmp.cZa5crnzep
+ rm /tmp/tmp.CuchBfPFbT /tmp/tmp.cZa5crnzep
+ return 0
+ [[ 0 -eq 0 ]]
+ diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2256/e2e-tests/demand-backup/compare/find-2nd.json /tmp/tmp.pzOYzJaY7Q/find-2nd
+ run_restore backup-minio
+ local backup_name=backup-minio
+ local restore_file=/mnt/jenkins/workspace/cloud-psmdb-operator_PR-2256/e2e-tests/demand-backup/conf/restore.yml
+ log 'running restore restore-backup-minio'
+ set +o xtrace
[2026-04-14T09:39:42+0000] running restore restore-backup-minio
+ cat /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2256/e2e-tests/demand-backup/conf/restore.yml
+ /usr/sbin/sed -e 's/name:/name: restore-backup-minio/'
+ /usr/sbin/sed -e 's/backupName:/backupName: backup-minio/'
+ kubectl_bin apply -f -
++ mktemp
+ local LAST_OUT=/tmp/tmp.IU3ANLsY8I
++ mktemp
+ local LAST_ERR=/tmp/tmp.UCvh0tA2R2
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl apply -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.IU3ANLsY8I
perconaservermongodbrestore.psmdb.percona.com/restore-backup-minio created
+ cat /tmp/tmp.UCvh0tA2R2
+ rm /tmp/tmp.IU3ANLsY8I /tmp/tmp.UCvh0tA2R2
+ return 0
+ wait_restore backup-minio some-name
+ local backup_name=backup-minio
+ local cluster_name=some-name
+ local target_state=ready
+ local wait_cluster_consistency=1
+ local wait_time=1780
+ local ok_if_ready=0
+ set +o xtrace
Waiting for the psmdb-restore/restore-backup-minio object to be created.OK
Waiting psmdb-restore/restore-backup-minio to reach state "ready" .OK after 0 minutes
+ [[ 1 -eq 1 ]]
+ wait_cluster_consistency some-name
+ local cluster_name=some-name
+ local wait_time=32
+ retry=0
+ sleep 7
+ echo -n 'waiting for cluster readyness'
waiting for cluster readyness++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.wDQwHPWwzq
+++ mktemp
++ local LAST_ERR=/tmp/tmp.LdLITqn6Ls
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.wDQwHPWwzq
++ cat /tmp/tmp.LdLITqn6Ls
++ rm /tmp/tmp.wDQwHPWwzq /tmp/tmp.LdLITqn6Ls
++ return 0
+ [[ ready == ready ]]
+ echo .OK
.OK
+ simple_data_check some-name-rs0 3 0
+ local cluster_name=some-name-rs0
+ let last_pod=3-1
+ local isSharded=0
+ local cluster_pfx=
+ '[' 0 -eq 1 ']'
++ seq 0 2
+ for i in $(seq 0 $last_pod)
+ compare_mongo_cmd find myApp:myPass@some-name-rs0-0.some-name-rs0.demand-backup-7562
+ local command=find
+ local uri=myApp:myPass@some-name-rs0-0.some-name-rs0.demand-backup-7562
+ local postfix=
+ local suffix=
+ local database=myApp
+ local collection=test
+ local sort=
+ local tls=false
+ local replicaset=
+ local 'full_command=db.test.find()'
+ [[ -n '' ]]
+ log 'running db.test.find() in myApp'
+ set +o xtrace
[2026-04-14T09:40:05+0000] running db.test.find() in myApp
+ [[ false == true ]]
+ mongo_command=run_mongo
+ grep -E -v 'I NETWORK|W NETWORK|F NETWORK|"c":"NETWORK"|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|Started a new thread for the timer service'
+ run_mongo 'use myApp\n db.test.find()' myApp:myPass@some-name-rs0-0.some-name-rs0.demand-backup-7562 mongodb '' ''
+ local 'command=use myApp\n db.test.find()'
+ local uri=myApp:myPass@some-name-rs0-0.some-name-rs0.demand-backup-7562
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local replica_set=rs0
+ [[ myApp:myPass@some-name-rs0-0.some-name-rs0.demand-backup-7562 == *cfg* ]]
+ /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.aGwUseUU3Q
+++ mktemp
++ local LAST_ERR=/tmp/tmp.uktnAbFzth
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.aGwUseUU3Q
++ cat /tmp/tmp.uktnAbFzth
++ rm /tmp/tmp.aGwUseUU3Q /tmp/tmp.uktnAbFzth
++ return 0
+ local client_container=psmdb-client-bb8b97679-62vtp
+ kubectl_bin exec psmdb-client-bb8b97679-62vtp -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-0.some-name-rs0.demand-backup-7562.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
++ mktemp
+ local LAST_OUT=/tmp/tmp.4167sm6csG
++ mktemp
+ local LAST_ERR=/tmp/tmp.Ot3x1ZV5g1
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-bb8b97679-62vtp -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-0.some-name-rs0.demand-backup-7562.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.4167sm6csG
+ cat /tmp/tmp.Ot3x1ZV5g1
+ rm /tmp/tmp.4167sm6csG /tmp/tmp.Ot3x1ZV5g1
+ return 0
+ [[ 0 -eq 0 ]]
+ diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2256/e2e-tests/demand-backup/compare/find.json /tmp/tmp.pzOYzJaY7Q/find
+ for i in $(seq 0 $last_pod)
+ compare_mongo_cmd find myApp:myPass@some-name-rs0-1.some-name-rs0.demand-backup-7562
+ local command=find
+ local uri=myApp:myPass@some-name-rs0-1.some-name-rs0.demand-backup-7562
+ local postfix=
+ local suffix=
+ local database=myApp
+ local collection=test
+ local sort=
+ local tls=false
+ local replicaset=
+ local 'full_command=db.test.find()'
+ [[ -n '' ]]
+ log 'running db.test.find() in myApp'
+ set +o xtrace
[2026-04-14T09:40:08+0000] running db.test.find() in myApp
+ [[ false == true ]]
+ mongo_command=run_mongo
+ run_mongo 'use myApp\n db.test.find()' myApp:myPass@some-name-rs0-1.some-name-rs0.demand-backup-7562 mongodb '' ''
+ local 'command=use myApp\n db.test.find()'
+ local uri=myApp:myPass@some-name-rs0-1.some-name-rs0.demand-backup-7562
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local replica_set=rs0
+ [[ myApp:myPass@some-name-rs0-1.some-name-rs0.demand-backup-7562 == *cfg* ]]
+ grep -E -v 'I NETWORK|W NETWORK|F NETWORK|"c":"NETWORK"|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|Started a new thread for the timer service'
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
+ /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
++ local LAST_OUT=/tmp/tmp.J6eKkgUQCk
+++ mktemp
++ local LAST_ERR=/tmp/tmp.CHe3HuOgTi
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.J6eKkgUQCk
++ cat /tmp/tmp.CHe3HuOgTi
++ rm /tmp/tmp.J6eKkgUQCk /tmp/tmp.CHe3HuOgTi
++ return 0
+ local client_container=psmdb-client-bb8b97679-62vtp
+ kubectl_bin exec psmdb-client-bb8b97679-62vtp -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-1.some-name-rs0.demand-backup-7562.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
++ mktemp
+ local LAST_OUT=/tmp/tmp.xqHwHxl7rv
++ mktemp
+ local LAST_ERR=/tmp/tmp.r2nrMyaIug
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-bb8b97679-62vtp -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-1.some-name-rs0.demand-backup-7562.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.xqHwHxl7rv
+ cat /tmp/tmp.r2nrMyaIug
+ rm /tmp/tmp.xqHwHxl7rv /tmp/tmp.r2nrMyaIug
+ return 0
+ [[ 0 -eq 0 ]]
+ diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2256/e2e-tests/demand-backup/compare/find.json /tmp/tmp.pzOYzJaY7Q/find
+ for i in $(seq 0 $last_pod)
+ compare_mongo_cmd find myApp:myPass@some-name-rs0-2.some-name-rs0.demand-backup-7562
+ local command=find
+ local uri=myApp:myPass@some-name-rs0-2.some-name-rs0.demand-backup-7562
+ local postfix=
+ local suffix=
+ local database=myApp
+ local collection=test
+ local sort=
+ local tls=false
+ local replicaset=
+ local 'full_command=db.test.find()'
+ [[ -n '' ]]
+ log 'running db.test.find() in myApp'
+ set +o xtrace
[2026-04-14T09:40:10+0000] running db.test.find() in myApp
+ [[ false == true ]]
+ mongo_command=run_mongo
+ run_mongo 'use myApp\n db.test.find()' myApp:myPass@some-name-rs0-2.some-name-rs0.demand-backup-7562 mongodb '' ''
+ local 'command=use myApp\n db.test.find()'
+ local uri=myApp:myPass@some-name-rs0-2.some-name-rs0.demand-backup-7562
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local replica_set=rs0
+ [[ myApp:myPass@some-name-rs0-2.some-name-rs0.demand-backup-7562 == *cfg* ]]
+ /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
+ grep -E -v 'I NETWORK|W NETWORK|F NETWORK|"c":"NETWORK"|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|Started a new thread for the timer service'
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.kBOZgt1D7i
+++ mktemp
++ local LAST_ERR=/tmp/tmp.FHm3epu1bA
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.kBOZgt1D7i
++ cat /tmp/tmp.FHm3epu1bA
++ rm /tmp/tmp.kBOZgt1D7i /tmp/tmp.FHm3epu1bA
++ return 0
+ local client_container=psmdb-client-bb8b97679-62vtp
+ kubectl_bin exec psmdb-client-bb8b97679-62vtp -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-2.some-name-rs0.demand-backup-7562.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
++ mktemp
+ local LAST_OUT=/tmp/tmp.hXH0pEfwqY
++ mktemp
+ local LAST_ERR=/tmp/tmp.LhnJnvWVo4
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-bb8b97679-62vtp -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-2.some-name-rs0.demand-backup-7562.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.hXH0pEfwqY
+ cat /tmp/tmp.LhnJnvWVo4
+ rm /tmp/tmp.hXH0pEfwqY /tmp/tmp.LhnJnvWVo4
+ return 0
+ [[ 0 -eq 0 ]]
+ diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2256/e2e-tests/demand-backup/compare/find.json /tmp/tmp.pzOYzJaY7Q/find
+ run_mongo 'use myApp\n db.dropUser("test1user")' userAdmin:userAdmin123456@some-name-rs0.demand-backup-7562
+ local 'command=use myApp\n db.dropUser("test1user")'
+ local uri=userAdmin:userAdmin123456@some-name-rs0.demand-backup-7562
+ local driver=mongodb+srv
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local replica_set=rs0
+ [[ userAdmin:userAdmin123456@some-name-rs0.demand-backup-7562 == *cfg* ]]
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.gOUwiGdFxc
+++ mktemp
++ local LAST_ERR=/tmp/tmp.weRgylnydO
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.gOUwiGdFxc
++ cat /tmp/tmp.weRgylnydO
++ rm /tmp/tmp.gOUwiGdFxc /tmp/tmp.weRgylnydO
++ return 0
+ local client_container=psmdb-client-bb8b97679-62vtp
+ kubectl_bin exec psmdb-client-bb8b97679-62vtp -- bash -c 'printf '\''use myApp\n db.dropUser("test1user")\n'\'' | mongo mongodb+srv://userAdmin:userAdmin123456@some-name-rs0.demand-backup-7562.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
++ mktemp
+ local LAST_OUT=/tmp/tmp.JAQSKmrVFF
++ mktemp
+ local LAST_ERR=/tmp/tmp.Kq40qmvrj2
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-bb8b97679-62vtp -- bash -c 'printf '\''use myApp\n db.dropUser("test1user")\n'\'' | mongo mongodb+srv://userAdmin:userAdmin123456@some-name-rs0.demand-backup-7562.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.JAQSKmrVFF
Percona Server for MongoDB shell version v4.4.29-28
connecting to: mongodb://some-name-rs0-0.some-name-rs0.demand-backup-7562.svc.cluster.local:27017,some-name-rs0-1.some-name-rs0.demand-backup-7562.svc.cluster.local:27017,some-name-rs0-2.some-name-rs0.demand-backup-7562.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb&replicaSet=rs0&ssl=false
Implicit session: session { "id" : UUID("43059d08-213b-48f8-a9b4-f238835ec0a5") }
Percona Server for MongoDB server version: v8.0.20-8
WARNING: shell and server versions do not match
switched to db myApp
true
bye
+ cat /tmp/tmp.Kq40qmvrj2
+ rm /tmp/tmp.JAQSKmrVFF /tmp/tmp.Kq40qmvrj2
+ return 0
+ run_mongo 'use myApp\n db.dropRole("test1role")' userAdmin:userAdmin123456@some-name-rs0.demand-backup-7562
+ local 'command=use myApp\n db.dropRole("test1role")'
+ local uri=userAdmin:userAdmin123456@some-name-rs0.demand-backup-7562
+ local driver=mongodb+srv
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local replica_set=rs0
+ [[ userAdmin:userAdmin123456@some-name-rs0.demand-backup-7562 == *cfg* ]]
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.5aNgze6S7F
+++ mktemp
++ local LAST_ERR=/tmp/tmp.UslfA0b7SC
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.5aNgze6S7F
++ cat /tmp/tmp.UslfA0b7SC
++ rm /tmp/tmp.5aNgze6S7F /tmp/tmp.UslfA0b7SC
++ return 0
+ local client_container=psmdb-client-bb8b97679-62vtp
+ kubectl_bin exec psmdb-client-bb8b97679-62vtp -- bash -c 'printf '\''use myApp\n db.dropRole("test1role")\n'\'' | mongo mongodb+srv://userAdmin:userAdmin123456@some-name-rs0.demand-backup-7562.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
++ mktemp
+ local LAST_OUT=/tmp/tmp.MIaBXnEOI5
++ mktemp
+ local LAST_ERR=/tmp/tmp.f99X8EnZqa
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-bb8b97679-62vtp -- bash -c 'printf '\''use myApp\n db.dropRole("test1role")\n'\'' | mongo mongodb+srv://userAdmin:userAdmin123456@some-name-rs0.demand-backup-7562.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.MIaBXnEOI5
Percona Server for MongoDB shell version v4.4.29-28
connecting to: mongodb://some-name-rs0-2.some-name-rs0.demand-backup-7562.svc.cluster.local:27017,some-name-rs0-0.some-name-rs0.demand-backup-7562.svc.cluster.local:27017,some-name-rs0-1.some-name-rs0.demand-backup-7562.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb&replicaSet=rs0&ssl=false
Implicit session: session { "id" : UUID("f3fa9cae-ab99-4679-9073-9e634b628746") }
Percona Server for MongoDB server version: v8.0.20-8
WARNING: shell and server versions do not match
switched to db myApp
true
bye
+ cat /tmp/tmp.f99X8EnZqa
+ rm /tmp/tmp.MIaBXnEOI5 /tmp/tmp.f99X8EnZqa
+ return 0
+ desc 'selective restore -- minio'
+ set +o xtrace


-----------------------------------------------------------------------------------
selective restore -- minio
-----------------------------------------------------------------------------------

+ run_recovery_check_selective backup-minio some-name-rs0 myApp test2 false test1user test1role
+ local backup=backup-minio
+ local cluster=some-name-rs0
+ local database=myApp
+ local collection=test2
+ local with_users_and_roles=false
+ local username=test1user
+ local role=test1role
+ restore_name=restore-backup-minio-selective
+ run_mongo 'use myApp\n db.test2.drop()' myApp:myPass@some-name-rs0.demand-backup-7562
+ local 'command=use myApp\n db.test2.drop()'
+ local uri=myApp:myPass@some-name-rs0.demand-backup-7562
+ local driver=mongodb+srv
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local replica_set=rs0
+ [[ myApp:myPass@some-name-rs0.demand-backup-7562 == *cfg* ]]
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.VWWJzEOzwb
+++ mktemp
++ local LAST_ERR=/tmp/tmp.0sW25x8Voj
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.VWWJzEOzwb
++ cat /tmp/tmp.0sW25x8Voj
++ rm /tmp/tmp.VWWJzEOzwb /tmp/tmp.0sW25x8Voj
++ return 0
+ local client_container=psmdb-client-bb8b97679-62vtp
+ kubectl_bin exec psmdb-client-bb8b97679-62vtp -- bash -c 'printf '\''use myApp\n db.test2.drop()\n'\'' | mongo mongodb+srv://myApp:myPass@some-name-rs0.demand-backup-7562.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
++ mktemp
+ local LAST_OUT=/tmp/tmp.A1iCmwMorQ
++ mktemp
+ local LAST_ERR=/tmp/tmp.7yoVFNbbYm
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-bb8b97679-62vtp -- bash -c 'printf '\''use myApp\n db.test2.drop()\n'\'' | mongo mongodb+srv://myApp:myPass@some-name-rs0.demand-backup-7562.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.A1iCmwMorQ
Percona Server for MongoDB shell version v4.4.29-28
connecting to: mongodb://some-name-rs0-0.some-name-rs0.demand-backup-7562.svc.cluster.local:27017,some-name-rs0-1.some-name-rs0.demand-backup-7562.svc.cluster.local:27017,some-name-rs0-2.some-name-rs0.demand-backup-7562.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb&replicaSet=rs0&ssl=false
Implicit session: session { "id" : UUID("f4c47e62-7ad4-464d-b9bd-5206932ab96a") }
Percona Server for MongoDB server version: v8.0.20-8
WARNING: shell and server versions do not match
switched to db myApp
true
bye
+ cat /tmp/tmp.7yoVFNbbYm
+ rm /tmp/tmp.A1iCmwMorQ /tmp/tmp.7yoVFNbbYm
+ return 0
++ collection_exists test2
++ local collection=test2
++ run_mongo 'use myApp\n JSON.stringify(db.getCollectionNames())' myApp:myPass@some-name-rs0.demand-backup-7562 '' '' --quiet
++ local 'command=use myApp\n JSON.stringify(db.getCollectionNames())'
++ local uri=myApp:myPass@some-name-rs0.demand-backup-7562
++ local driver=mongodb+srv
++ local suffix=.svc.cluster.local
++ local mongo_flag=--quiet
++ local replica_set=rs0
++ [[ myApp:myPass@some-name-rs0.demand-backup-7562 == *cfg* ]]
++ grep -v 'switched to'
++ jq 'index("test2") != null'
+++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.Oa5TMu1Egl
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.zjAORgSfki
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in $(seq 0 2)
+++ set +e
+++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.Oa5TMu1Egl
+++ cat /tmp/tmp.zjAORgSfki
+++ rm /tmp/tmp.Oa5TMu1Egl /tmp/tmp.zjAORgSfki
+++ return 0
++ local client_container=psmdb-client-bb8b97679-62vtp
++ kubectl_bin exec psmdb-client-bb8b97679-62vtp -- bash -c 'printf '\''use myApp\n JSON.stringify(db.getCollectionNames())\n'\'' | mongo mongodb+srv://myApp:myPass@some-name-rs0.demand-backup-7562.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 --quiet'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.BBLhabAdmi
+++ mktemp
++ local LAST_ERR=/tmp/tmp.nGZosTtwNh
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl exec psmdb-client-bb8b97679-62vtp -- bash -c 'printf '\''use myApp\n JSON.stringify(db.getCollectionNames())\n'\'' | mongo mongodb+srv://myApp:myPass@some-name-rs0.demand-backup-7562.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 --quiet'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.BBLhabAdmi
++ cat /tmp/tmp.nGZosTtwNh
++ rm /tmp/tmp.BBLhabAdmi /tmp/tmp.nGZosTtwNh
++ return 0
+ [[ false == true ]]
+ yq /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2256/e2e-tests/demand-backup/conf/restore.yml
+ yq '.metadata.name="restore-backup-minio-selective"'
+ yq '.spec.backupName="backup-minio"'
+ yq '.spec.selective.namespaces[0]="myApp.test"'
+ yq .spec.selective.withUsersAndRoles=false
+ kubectl_bin apply -f -
++ mktemp
+ local LAST_OUT=/tmp/tmp.NUCQ3slM9K
++ mktemp
+ local LAST_ERR=/tmp/tmp.UsaHOZD0R1
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl apply -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.NUCQ3slM9K
perconaservermongodbrestore.psmdb.percona.com/restore-backup-minio-selective created
+ cat /tmp/tmp.UsaHOZD0R1
+ rm /tmp/tmp.NUCQ3slM9K /tmp/tmp.UsaHOZD0R1
+ return 0
+ wait_restore backup-minio-selective some-name
+ local backup_name=backup-minio-selective
+ local cluster_name=some-name
+ local target_state=ready
+ local wait_cluster_consistency=1
+ local wait_time=1780
+ local ok_if_ready=0
+ set +o xtrace
Waiting for the psmdb-restore/restore-backup-minio-selective object to be created.OK
Waiting psmdb-restore/restore-backup-minio-selective to reach state "ready" .OK after 0 minutes
+ [[ 1 -eq 1 ]]
+ wait_cluster_consistency some-name
+ local cluster_name=some-name
+ local wait_time=32
+ retry=0
+ sleep 7
+ echo -n 'waiting for cluster readyness'
waiting for cluster readyness++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.0JKxHnDRzv
+++ mktemp
++ local LAST_ERR=/tmp/tmp.kNJpYYzjo8
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.0JKxHnDRzv
++ cat /tmp/tmp.kNJpYYzjo8
++ rm /tmp/tmp.0JKxHnDRzv /tmp/tmp.kNJpYYzjo8
++ return 0
+ [[ ready == ready ]]
+ echo .OK
.OK
++ collection_exists test2
++ local collection=test2
++ run_mongo 'use myApp\n JSON.stringify(db.getCollectionNames())' myApp:myPass@some-name-rs0.demand-backup-7562 '' '' --quiet
++ local 'command=use myApp\n JSON.stringify(db.getCollectionNames())'
++ local uri=myApp:myPass@some-name-rs0.demand-backup-7562
++ local driver=mongodb+srv
++ local suffix=.svc.cluster.local
++ local mongo_flag=--quiet
++ local replica_set=rs0
++ [[ myApp:myPass@some-name-rs0.demand-backup-7562 == *cfg* ]]
++ grep -v 'switched to'
++ jq 'index("test2") != null'
+++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.VHZFbjsSZl
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.STC6X8io7q
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in $(seq 0 2)
+++ set +e
+++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.VHZFbjsSZl
+++ cat /tmp/tmp.STC6X8io7q
+++ rm /tmp/tmp.VHZFbjsSZl /tmp/tmp.STC6X8io7q
+++ return 0
++ local client_container=psmdb-client-bb8b97679-62vtp
++ kubectl_bin exec psmdb-client-bb8b97679-62vtp -- bash -c 'printf '\''use myApp\n JSON.stringify(db.getCollectionNames())\n'\'' | mongo mongodb+srv://myApp:myPass@some-name-rs0.demand-backup-7562.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 --quiet'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.yXEErc8t2x
+++ mktemp
++ local LAST_ERR=/tmp/tmp.DkW5Fj5DZF
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl exec psmdb-client-bb8b97679-62vtp -- bash -c 'printf '\''use myApp\n JSON.stringify(db.getCollectionNames())\n'\'' | mongo mongodb+srv://myApp:myPass@some-name-rs0.demand-backup-7562.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 --quiet'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.yXEErc8t2x
++ cat /tmp/tmp.DkW5Fj5DZF
++ rm /tmp/tmp.yXEErc8t2x /tmp/tmp.DkW5Fj5DZF
++ return 0
+ [[ false == true ]]
+ [[ false == true ]]
+ [[ false == false ]]
++ user_exists test1user
++ local username=test1user
++ run_mongo 'use myApp\n JSON.stringify(db.getUsers())' userAdmin:userAdmin123456@some-name-rs0.demand-backup-7562 '' '' --quiet
++ local 'command=use myApp\n JSON.stringify(db.getUsers())'
++ local uri=userAdmin:userAdmin123456@some-name-rs0.demand-backup-7562
++ local driver=mongodb+srv
++ local suffix=.svc.cluster.local
++ local mongo_flag=--quiet
++ local replica_set=rs0
++ [[ userAdmin:userAdmin123456@some-name-rs0.demand-backup-7562 == *cfg* ]]
++ grep -v 'switched to'
++ jq 'any(.[]; ._id==myApp.test1user)'
+++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.8ZA53aJNnl
jq: error: myApp/0 is not defined at <top-level>, line 1:
any(.[]; ._id==myApp.test1user)               
jq: 1 compile error
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.clRi75C4Th
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in $(seq 0 2)
+++ set +e
+++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.8ZA53aJNnl
+++ cat /tmp/tmp.clRi75C4Th
+++ rm /tmp/tmp.8ZA53aJNnl /tmp/tmp.clRi75C4Th
+++ return 0
++ local client_container=psmdb-client-bb8b97679-62vtp
++ kubectl_bin exec psmdb-client-bb8b97679-62vtp -- bash -c 'printf '\''use myApp\n JSON.stringify(db.getUsers())\n'\'' | mongo mongodb+srv://userAdmin:userAdmin123456@some-name-rs0.demand-backup-7562.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 --quiet'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.3hwLXtS0hR
+++ mktemp
++ local LAST_ERR=/tmp/tmp.17sqEQxNSl
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl exec psmdb-client-bb8b97679-62vtp -- bash -c 'printf '\''use myApp\n JSON.stringify(db.getUsers())\n'\'' | mongo mongodb+srv://userAdmin:userAdmin123456@some-name-rs0.demand-backup-7562.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 --quiet'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.3hwLXtS0hR
++ cat /tmp/tmp.17sqEQxNSl
++ rm /tmp/tmp.3hwLXtS0hR /tmp/tmp.17sqEQxNSl
++ return 0
grep: write error: Broken pipe
+ [[ '' == true ]]
++ role_exists test1role
++ local role=test1role
++ run_mongo 'use myApp\n JSON.stringify(db.getRoles())' userAdmin:userAdmin123456@some-name-rs0.demand-backup-7562 '' '' --quiet
++ local 'command=use myApp\n JSON.stringify(db.getRoles())'
++ local uri=userAdmin:userAdmin123456@some-name-rs0.demand-backup-7562
++ local driver=mongodb+srv
++ local suffix=.svc.cluster.local
++ local mongo_flag=--quiet
++ local replica_set=rs0
++ [[ userAdmin:userAdmin123456@some-name-rs0.demand-backup-7562 == *cfg* ]]
++ jq 'any(.[]; ._id==myApp.test1role)'
++ grep -v 'switched to'
+++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++++ mktemp
jq: error: myApp/0 is not defined at <top-level>, line 1:
any(.[]; ._id==myApp.test1role)               
jq: 1 compile error
+++ local LAST_OUT=/tmp/tmp.AdEDSsanLc
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.MtVYGioySh
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in $(seq 0 2)
+++ set +e
+++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.AdEDSsanLc
+++ cat /tmp/tmp.MtVYGioySh
+++ rm /tmp/tmp.AdEDSsanLc /tmp/tmp.MtVYGioySh
+++ return 0
++ local client_container=psmdb-client-bb8b97679-62vtp
++ kubectl_bin exec psmdb-client-bb8b97679-62vtp -- bash -c 'printf '\''use myApp\n JSON.stringify(db.getRoles())\n'\'' | mongo mongodb+srv://userAdmin:userAdmin123456@some-name-rs0.demand-backup-7562.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 --quiet'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.JrlCFUtLle
+++ mktemp
++ local LAST_ERR=/tmp/tmp.78aM1dhn4g
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl exec psmdb-client-bb8b97679-62vtp -- bash -c 'printf '\''use myApp\n JSON.stringify(db.getRoles())\n'\'' | mongo mongodb+srv://userAdmin:userAdmin123456@some-name-rs0.demand-backup-7562.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 --quiet'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.JrlCFUtLle
++ cat /tmp/tmp.78aM1dhn4g
++ rm /tmp/tmp.JrlCFUtLle /tmp/tmp.78aM1dhn4g
++ return 0
grep: write error: Broken pipe
+ [[ '' == true ]]
+ kubectl_bin delete psmdb-restore restore-backup-minio-selective
++ mktemp
+ local LAST_OUT=/tmp/tmp.DlsVZGr4OW
++ mktemp
+ local LAST_ERR=/tmp/tmp.mlLJ7JHfml
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl delete psmdb-restore restore-backup-minio-selective
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.DlsVZGr4OW
perconaservermongodbrestore.psmdb.percona.com "restore-backup-minio-selective" deleted from demand-backup-7562 namespace
+ cat /tmp/tmp.mlLJ7JHfml
+ rm /tmp/tmp.DlsVZGr4OW /tmp/tmp.mlLJ7JHfml
+ return 0
+ run_mongo 'use myApp\n db.test2.insert({ x: 100501 })' myApp:myPass@some-name-rs0.demand-backup-7562
+ local 'command=use myApp\n db.test2.insert({ x: 100501 })'
+ local uri=myApp:myPass@some-name-rs0.demand-backup-7562
+ local driver=mongodb+srv
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local replica_set=rs0
+ [[ myApp:myPass@some-name-rs0.demand-backup-7562 == *cfg* ]]
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.vyL9VE0YWR
+++ mktemp
++ local LAST_ERR=/tmp/tmp.4CEhl5p88u
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.vyL9VE0YWR
++ cat /tmp/tmp.4CEhl5p88u
++ rm /tmp/tmp.vyL9VE0YWR /tmp/tmp.4CEhl5p88u
++ return 0
+ local client_container=psmdb-client-bb8b97679-62vtp
+ kubectl_bin exec psmdb-client-bb8b97679-62vtp -- bash -c 'printf '\''use myApp\n db.test2.insert({ x: 100501 })\n'\'' | mongo mongodb+srv://myApp:myPass@some-name-rs0.demand-backup-7562.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
++ mktemp
+ local LAST_OUT=/tmp/tmp.ZDpVimQdye
++ mktemp
+ local LAST_ERR=/tmp/tmp.FgkjYo2saS
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-bb8b97679-62vtp -- bash -c 'printf '\''use myApp\n db.test2.insert({ x: 100501 })\n'\'' | mongo mongodb+srv://myApp:myPass@some-name-rs0.demand-backup-7562.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.ZDpVimQdye
Percona Server for MongoDB shell version v4.4.29-28
connecting to: mongodb://some-name-rs0-2.some-name-rs0.demand-backup-7562.svc.cluster.local:27017,some-name-rs0-0.some-name-rs0.demand-backup-7562.svc.cluster.local:27017,some-name-rs0-1.some-name-rs0.demand-backup-7562.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb&replicaSet=rs0&ssl=false
Implicit session: session { "id" : UUID("acfd3034-7cb9-4c61-9245-e03d7ffd07ca") }
Percona Server for MongoDB server version: v8.0.20-8
WARNING: shell and server versions do not match
switched to db myApp
WriteResult({ "nInserted" : 1 })
bye
+ cat /tmp/tmp.FgkjYo2saS
+ rm /tmp/tmp.ZDpVimQdye /tmp/tmp.FgkjYo2saS
+ return 0
+ desc 'selective restore with users and roles -- minio'
+ set +o xtrace


-----------------------------------------------------------------------------------
selective restore with users and roles -- minio
-----------------------------------------------------------------------------------

+ run_recovery_check_selective backup-minio some-name-rs0 myApp test2 true test1user test1role
+ local backup=backup-minio
+ local cluster=some-name-rs0
+ local database=myApp
+ local collection=test2
+ local with_users_and_roles=true
+ local username=test1user
+ local role=test1role
+ restore_name=restore-backup-minio-selective
+ run_mongo 'use myApp\n db.test2.drop()' myApp:myPass@some-name-rs0.demand-backup-7562
+ local 'command=use myApp\n db.test2.drop()'
+ local uri=myApp:myPass@some-name-rs0.demand-backup-7562
+ local driver=mongodb+srv
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local replica_set=rs0
+ [[ myApp:myPass@some-name-rs0.demand-backup-7562 == *cfg* ]]
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.nJ9xiu50Xq
+++ mktemp
++ local LAST_ERR=/tmp/tmp.ofXnwJaeIR
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.nJ9xiu50Xq
++ cat /tmp/tmp.ofXnwJaeIR
++ rm /tmp/tmp.nJ9xiu50Xq /tmp/tmp.ofXnwJaeIR
++ return 0
+ local client_container=psmdb-client-bb8b97679-62vtp
+ kubectl_bin exec psmdb-client-bb8b97679-62vtp -- bash -c 'printf '\''use myApp\n db.test2.drop()\n'\'' | mongo mongodb+srv://myApp:myPass@some-name-rs0.demand-backup-7562.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
++ mktemp
+ local LAST_OUT=/tmp/tmp.nqRqKpnxg7
++ mktemp
+ local LAST_ERR=/tmp/tmp.sa0EGyFpDi
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-bb8b97679-62vtp -- bash -c 'printf '\''use myApp\n db.test2.drop()\n'\'' | mongo mongodb+srv://myApp:myPass@some-name-rs0.demand-backup-7562.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.nqRqKpnxg7
Percona Server for MongoDB shell version v4.4.29-28
connecting to: mongodb://some-name-rs0-1.some-name-rs0.demand-backup-7562.svc.cluster.local:27017,some-name-rs0-2.some-name-rs0.demand-backup-7562.svc.cluster.local:27017,some-name-rs0-0.some-name-rs0.demand-backup-7562.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb&replicaSet=rs0&ssl=false
Implicit session: session { "id" : UUID("4d35b500-a2c2-4a53-89c7-ea46a177b566") }
Percona Server for MongoDB server version: v8.0.20-8
WARNING: shell and server versions do not match
switched to db myApp
true
bye
+ cat /tmp/tmp.sa0EGyFpDi
+ rm /tmp/tmp.nqRqKpnxg7 /tmp/tmp.sa0EGyFpDi
+ return 0
++ collection_exists test2
++ local collection=test2
++ run_mongo 'use myApp\n JSON.stringify(db.getCollectionNames())' myApp:myPass@some-name-rs0.demand-backup-7562 '' '' --quiet
++ local 'command=use myApp\n JSON.stringify(db.getCollectionNames())'
++ grep -v 'switched to'
++ local uri=myApp:myPass@some-name-rs0.demand-backup-7562
++ local driver=mongodb+srv
++ local suffix=.svc.cluster.local
++ local mongo_flag=--quiet
++ local replica_set=rs0
++ [[ myApp:myPass@some-name-rs0.demand-backup-7562 == *cfg* ]]
++ jq 'index("test2") != null'
+++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.jzKZCqwODS
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.PkWa0qSJOo
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in $(seq 0 2)
+++ set +e
+++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.jzKZCqwODS
+++ cat /tmp/tmp.PkWa0qSJOo
+++ rm /tmp/tmp.jzKZCqwODS /tmp/tmp.PkWa0qSJOo
+++ return 0
++ local client_container=psmdb-client-bb8b97679-62vtp
++ kubectl_bin exec psmdb-client-bb8b97679-62vtp -- bash -c 'printf '\''use myApp\n JSON.stringify(db.getCollectionNames())\n'\'' | mongo mongodb+srv://myApp:myPass@some-name-rs0.demand-backup-7562.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 --quiet'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.vmkmWtUrxo
+++ mktemp
++ local LAST_ERR=/tmp/tmp.DWuCKwsMYl
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl exec psmdb-client-bb8b97679-62vtp -- bash -c 'printf '\''use myApp\n JSON.stringify(db.getCollectionNames())\n'\'' | mongo mongodb+srv://myApp:myPass@some-name-rs0.demand-backup-7562.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 --quiet'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.vmkmWtUrxo
++ cat /tmp/tmp.DWuCKwsMYl
++ rm /tmp/tmp.vmkmWtUrxo /tmp/tmp.DWuCKwsMYl
++ return 0
+ [[ false == true ]]
+ yq /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2256/e2e-tests/demand-backup/conf/restore.yml
+ yq '.metadata.name="restore-backup-minio-selective"'
+ yq '.spec.backupName="backup-minio"'
+ yq '.spec.selective.namespaces[0]="myApp.test"'
+ yq .spec.selective.withUsersAndRoles=true
+ kubectl_bin apply -f -
++ mktemp
+ local LAST_OUT=/tmp/tmp.a1epkKQXOu
++ mktemp
+ local LAST_ERR=/tmp/tmp.fFG0vKOCAP
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl apply -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.a1epkKQXOu
perconaservermongodbrestore.psmdb.percona.com/restore-backup-minio-selective created
+ cat /tmp/tmp.fFG0vKOCAP
+ rm /tmp/tmp.a1epkKQXOu /tmp/tmp.fFG0vKOCAP
+ return 0
+ wait_restore backup-minio-selective some-name
+ local backup_name=backup-minio-selective
+ local cluster_name=some-name
+ local target_state=ready
+ local wait_cluster_consistency=1
+ local wait_time=1780
+ local ok_if_ready=0
+ set +o xtrace
Waiting for the psmdb-restore/restore-backup-minio-selective object to be created.OK
Waiting psmdb-restore/restore-backup-minio-selective to reach state "ready" .OK after 0 minutes
+ [[ 1 -eq 1 ]]
+ wait_cluster_consistency some-name
+ local cluster_name=some-name
+ local wait_time=32
+ retry=0
+ sleep 7
+ echo -n 'waiting for cluster readyness'
waiting for cluster readyness++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.1iyvJtHgA3
+++ mktemp
++ local LAST_ERR=/tmp/tmp.8bJStG9wLG
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.1iyvJtHgA3
++ cat /tmp/tmp.8bJStG9wLG
++ rm /tmp/tmp.1iyvJtHgA3 /tmp/tmp.8bJStG9wLG
++ return 0
+ [[ ready == ready ]]
+ echo .OK
.OK
++ collection_exists test2
++ local collection=test2
++ run_mongo 'use myApp\n JSON.stringify(db.getCollectionNames())' myApp:myPass@some-name-rs0.demand-backup-7562 '' '' --quiet
++ local 'command=use myApp\n JSON.stringify(db.getCollectionNames())'
++ local uri=myApp:myPass@some-name-rs0.demand-backup-7562
++ local driver=mongodb+srv
++ grep -v 'switched to'
++ local suffix=.svc.cluster.local
++ local mongo_flag=--quiet
++ local replica_set=rs0
++ [[ myApp:myPass@some-name-rs0.demand-backup-7562 == *cfg* ]]
++ jq 'index("test2") != null'
+++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.gi6NO82HUE
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.XgPEwm2j1z
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in $(seq 0 2)
+++ set +e
+++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.gi6NO82HUE
+++ cat /tmp/tmp.XgPEwm2j1z
+++ rm /tmp/tmp.gi6NO82HUE /tmp/tmp.XgPEwm2j1z
+++ return 0
++ local client_container=psmdb-client-bb8b97679-62vtp
++ kubectl_bin exec psmdb-client-bb8b97679-62vtp -- bash -c 'printf '\''use myApp\n JSON.stringify(db.getCollectionNames())\n'\'' | mongo mongodb+srv://myApp:myPass@some-name-rs0.demand-backup-7562.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 --quiet'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.vOB4E47kfx
+++ mktemp
++ local LAST_ERR=/tmp/tmp.ZIK6bXRQjH
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl exec psmdb-client-bb8b97679-62vtp -- bash -c 'printf '\''use myApp\n JSON.stringify(db.getCollectionNames())\n'\'' | mongo mongodb+srv://myApp:myPass@some-name-rs0.demand-backup-7562.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 --quiet'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.vOB4E47kfx
++ cat /tmp/tmp.ZIK6bXRQjH
++ rm /tmp/tmp.vOB4E47kfx /tmp/tmp.ZIK6bXRQjH
++ return 0
+ [[ false == true ]]
+ [[ true == true ]]
++ user_exists test1user
++ local username=test1user
++ run_mongo 'use myApp\n JSON.stringify(db.getUsers())' userAdmin:userAdmin123456@some-name-rs0.demand-backup-7562 '' '' --quiet
++ grep -v 'switched to'
++ local 'command=use myApp\n JSON.stringify(db.getUsers())'
++ local uri=userAdmin:userAdmin123456@some-name-rs0.demand-backup-7562
++ local driver=mongodb+srv
++ local suffix=.svc.cluster.local
++ local mongo_flag=--quiet
++ local replica_set=rs0
++ [[ userAdmin:userAdmin123456@some-name-rs0.demand-backup-7562 == *cfg* ]]
++ jq 'any(.[]; ._id==myApp.test1user)'
+++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++++ mktemp
jq: error: myApp/0 is not defined at <top-level>, line 1:
any(.[]; ._id==myApp.test1user)               
jq: 1 compile error
+++ local LAST_OUT=/tmp/tmp.eSjErdLHii
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.t0iQaI1Dzx
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in $(seq 0 2)
+++ set +e
+++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.eSjErdLHii
+++ cat /tmp/tmp.t0iQaI1Dzx
+++ rm /tmp/tmp.eSjErdLHii /tmp/tmp.t0iQaI1Dzx
+++ return 0
++ local client_container=psmdb-client-bb8b97679-62vtp
++ kubectl_bin exec psmdb-client-bb8b97679-62vtp -- bash -c 'printf '\''use myApp\n JSON.stringify(db.getUsers())\n'\'' | mongo mongodb+srv://userAdmin:userAdmin123456@some-name-rs0.demand-backup-7562.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 --quiet'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.W5uRyPgtmf
+++ mktemp
++ local LAST_ERR=/tmp/tmp.wzkfXc6JYU
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl exec psmdb-client-bb8b97679-62vtp -- bash -c 'printf '\''use myApp\n JSON.stringify(db.getUsers())\n'\'' | mongo mongodb+srv://userAdmin:userAdmin123456@some-name-rs0.demand-backup-7562.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 --quiet'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.W5uRyPgtmf
++ cat /tmp/tmp.wzkfXc6JYU
++ rm /tmp/tmp.W5uRyPgtmf /tmp/tmp.wzkfXc6JYU
++ return 0
grep: write error: Broken pipe
+ [[ '' == false ]]
++ role_exists test1role
++ local role=test1role
++ run_mongo 'use myApp\n JSON.stringify(db.getRoles())' userAdmin:userAdmin123456@some-name-rs0.demand-backup-7562 '' '' --quiet
++ local 'command=use myApp\n JSON.stringify(db.getRoles())'
++ local uri=userAdmin:userAdmin123456@some-name-rs0.demand-backup-7562
++ local driver=mongodb+srv
++ local suffix=.svc.cluster.local
++ local mongo_flag=--quiet
++ local replica_set=rs0
++ [[ userAdmin:userAdmin123456@some-name-rs0.demand-backup-7562 == *cfg* ]]
++ grep -v 'switched to'
++ jq 'any(.[]; ._id==myApp.test1role)'
+++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.EPBDILgH9m
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.PMrSdwMTwR
+++ local exit_status=0
+++ local timeout=4
jq: error: myApp/0 is not defined at <top-level>, line 1:
any(.[]; ._id==myApp.test1role)               
jq: 1 compile error
++++ seq 0 2
+++ for i in $(seq 0 2)
+++ set +e
+++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.EPBDILgH9m
+++ cat /tmp/tmp.PMrSdwMTwR
+++ rm /tmp/tmp.EPBDILgH9m /tmp/tmp.PMrSdwMTwR
+++ return 0
++ local client_container=psmdb-client-bb8b97679-62vtp
++ kubectl_bin exec psmdb-client-bb8b97679-62vtp -- bash -c 'printf '\''use myApp\n JSON.stringify(db.getRoles())\n'\'' | mongo mongodb+srv://userAdmin:userAdmin123456@some-name-rs0.demand-backup-7562.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 --quiet'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.03Nroc8yUW
+++ mktemp
++ local LAST_ERR=/tmp/tmp.Ky4Qsjtiyf
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl exec psmdb-client-bb8b97679-62vtp -- bash -c 'printf '\''use myApp\n JSON.stringify(db.getRoles())\n'\'' | mongo mongodb+srv://userAdmin:userAdmin123456@some-name-rs0.demand-backup-7562.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 --quiet'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.03Nroc8yUW
++ cat /tmp/tmp.Ky4Qsjtiyf
++ rm /tmp/tmp.03Nroc8yUW /tmp/tmp.Ky4Qsjtiyf
++ return 0
grep: write error: Broken pipe
+ [[ '' == false ]]
+ desc 'restore from backup source, with storageName -- minio'
+ set +o xtrace


-----------------------------------------------------------------------------------
restore from backup source, with storageName -- minio
-----------------------------------------------------------------------------------

++ get_backup_dest backup-minio
++ local backup_name=backup-minio
++ kubectl_bin get psmdb-backup backup-minio -o 'jsonpath={.status.destination}'
++ sed -e 's/.json$//'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.oIupWOvpZN
+++ mktemp
++ sed 's|s3://||'
++ sed 's|azure://||'
++ sed 's|gs://||'
++ local LAST_ERR=/tmp/tmp.U7NdHX2obC
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb-backup backup-minio -o 'jsonpath={.status.destination}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.oIupWOvpZN
++ cat /tmp/tmp.U7NdHX2obC
++ rm /tmp/tmp.oIupWOvpZN /tmp/tmp.U7NdHX2obC
++ return 0
+ backup_dest_minio=operator-testing/2026-04-14T09:36:10Z
+ run_recovery_check_bkp_source backup-minio operator-testing/2026-04-14T09:36:10Z some-name-rs0 backup-minio-source-0
+ local backup=backup-minio
+ local backup_dest=operator-testing/2026-04-14T09:36:10Z
+ local cluster=some-name-rs0
+ local source=backup-minio-source-0
+ run_mongo 'use myApp\n db.test.insert({ x: 100501 })' myApp:myPass@some-name-rs0.demand-backup-7562
+ local 'command=use myApp\n db.test.insert({ x: 100501 })'
+ local uri=myApp:myPass@some-name-rs0.demand-backup-7562
+ local driver=mongodb+srv
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local replica_set=rs0
+ [[ myApp:myPass@some-name-rs0.demand-backup-7562 == *cfg* ]]
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.kIJzZ1LBmi
+++ mktemp
++ local LAST_ERR=/tmp/tmp.lRAxhNGRzR
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.kIJzZ1LBmi
++ cat /tmp/tmp.lRAxhNGRzR
++ rm /tmp/tmp.kIJzZ1LBmi /tmp/tmp.lRAxhNGRzR
++ return 0
+ local client_container=psmdb-client-bb8b97679-62vtp
+ kubectl_bin exec psmdb-client-bb8b97679-62vtp -- bash -c 'printf '\''use myApp\n db.test.insert({ x: 100501 })\n'\'' | mongo mongodb+srv://myApp:myPass@some-name-rs0.demand-backup-7562.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
++ mktemp
+ local LAST_OUT=/tmp/tmp.fhrjS3tTPJ
++ mktemp
+ local LAST_ERR=/tmp/tmp.o6uznUDvjf
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-bb8b97679-62vtp -- bash -c 'printf '\''use myApp\n db.test.insert({ x: 100501 })\n'\'' | mongo mongodb+srv://myApp:myPass@some-name-rs0.demand-backup-7562.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.fhrjS3tTPJ
Percona Server for MongoDB shell version v4.4.29-28
connecting to: mongodb://some-name-rs0-2.some-name-rs0.demand-backup-7562.svc.cluster.local:27017,some-name-rs0-0.some-name-rs0.demand-backup-7562.svc.cluster.local:27017,some-name-rs0-1.some-name-rs0.demand-backup-7562.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb&replicaSet=rs0&ssl=false
Implicit session: session { "id" : UUID("09378131-c8be-43ed-a887-504a257cdc4d") }
Percona Server for MongoDB server version: v8.0.20-8
WARNING: shell and server versions do not match
switched to db myApp
WriteResult({ "nInserted" : 1 })
bye
+ cat /tmp/tmp.o6uznUDvjf
+ rm /tmp/tmp.fhrjS3tTPJ /tmp/tmp.o6uznUDvjf
+ return 0
+ compare_mongo_cmd find myApp:myPass@some-name-rs0.demand-backup-7562 -2nd .svc.cluster.local myApp test
+ local command=find
+ local uri=myApp:myPass@some-name-rs0.demand-backup-7562
+ local postfix=-2nd
+ local suffix=.svc.cluster.local
+ local database=myApp
+ local collection=test
+ local sort=
+ local tls=false
+ local replicaset=
+ local 'full_command=db.test.find()'
+ [[ -n '' ]]
+ log 'running db.test.find() in myApp'
+ set +o xtrace
[2026-04-14T09:41:36+0000] running db.test.find() in myApp
+ [[ false == true ]]
+ mongo_command=run_mongo
+ run_mongo 'use myApp\n db.test.find()' myApp:myPass@some-name-rs0.demand-backup-7562 mongodb .svc.cluster.local ''
+ local 'command=use myApp\n db.test.find()'
+ local uri=myApp:myPass@some-name-rs0.demand-backup-7562
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local replica_set=rs0
+ [[ myApp:myPass@some-name-rs0.demand-backup-7562 == *cfg* ]]
+ grep -E -v 'I NETWORK|W NETWORK|F NETWORK|"c":"NETWORK"|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|Started a new thread for the timer service'
+ /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.pzQ9EY6ddu
+++ mktemp
++ local LAST_ERR=/tmp/tmp.lvCAXD5W7h
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.pzQ9EY6ddu
++ cat /tmp/tmp.lvCAXD5W7h
++ rm /tmp/tmp.pzQ9EY6ddu /tmp/tmp.lvCAXD5W7h
++ return 0
+ local client_container=psmdb-client-bb8b97679-62vtp
+ kubectl_bin exec psmdb-client-bb8b97679-62vtp -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0.demand-backup-7562.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
++ mktemp
+ local LAST_OUT=/tmp/tmp.4geV1vUk58
++ mktemp
+ local LAST_ERR=/tmp/tmp.xEAhe8QdwC
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-bb8b97679-62vtp -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0.demand-backup-7562.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.4geV1vUk58
+ cat /tmp/tmp.xEAhe8QdwC
+ rm /tmp/tmp.4geV1vUk58 /tmp/tmp.xEAhe8QdwC
+ return 0
+ [[ 0 -eq 0 ]]
+ diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2256/e2e-tests/demand-backup/compare/find-2nd.json /tmp/tmp.pzOYzJaY7Q/find-2nd
+ run_restore_backupsource backup-minio-source-0 operator-testing/2026-04-14T09:36:10Z
+ local backupName=backup-minio-source-0
+ local backupDest=operator-testing/2026-04-14T09:36:10Z
+ local storageName=
+ desc 'run restore restore-backup-minio-source-0 from backup backup-minio-source-0 destination is operator-testing/2026-04-14T09:36:10Z'
+ set +o xtrace


-----------------------------------------------------------------------------------
run restore restore-backup-minio-source-0 from backup backup-minio-source-0 destination is operator-testing/2026-04-14T09:36:10Z
-----------------------------------------------------------------------------------

+ '[' -z '' ']'
+ cat /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2256/e2e-tests/demand-backup/conf/restore-backupsource.yml
+ /usr/sbin/sed -e 's/name:/name: restore-backup-minio-source-0/'
+ /usr/sbin/sed -e 's|BACKUP-NAME|operator-testing/2026-04-14T09:36:10Z|'
+ /usr/sbin/sed -e /storageName/d
+ kubectl_bin apply -f -
++ mktemp
+ local LAST_OUT=/tmp/tmp.uJtMDyrW1H
++ mktemp
+ local LAST_ERR=/tmp/tmp.dbAJT62zgD
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl apply -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.uJtMDyrW1H
perconaservermongodbrestore.psmdb.percona.com/restore-backup-minio-source-0 created
+ cat /tmp/tmp.dbAJT62zgD
+ rm /tmp/tmp.uJtMDyrW1H /tmp/tmp.dbAJT62zgD
+ return 0
+ return
+ wait_restore backup-minio-source-0 some-name
+ local backup_name=backup-minio-source-0
+ local cluster_name=some-name
+ local target_state=ready
+ local wait_cluster_consistency=1
+ local wait_time=1780
+ local ok_if_ready=0
+ set +o xtrace
Waiting for the psmdb-restore/restore-backup-minio-source-0 object to be created.OK
Waiting psmdb-restore/restore-backup-minio-source-0 to reach state "ready" .OK after 0 minutes
+ [[ 1 -eq 1 ]]
+ wait_cluster_consistency some-name
+ local cluster_name=some-name
+ local wait_time=32
+ retry=0
+ sleep 7
+ echo -n 'waiting for cluster readyness'
waiting for cluster readyness++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.1nZF4TZwEa
+++ mktemp
++ local LAST_ERR=/tmp/tmp.vFCqs34uoT
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.1nZF4TZwEa
++ cat /tmp/tmp.vFCqs34uoT
++ rm /tmp/tmp.1nZF4TZwEa /tmp/tmp.vFCqs34uoT
++ return 0
+ [[ ready == ready ]]
+ echo .OK
.OK
+ simple_data_check some-name-rs0 3 0
+ local cluster_name=some-name-rs0
+ let last_pod=3-1
+ local isSharded=0
+ local cluster_pfx=
+ '[' 0 -eq 1 ']'
++ seq 0 2
+ for i in $(seq 0 $last_pod)
+ compare_mongo_cmd find myApp:myPass@some-name-rs0-0.some-name-rs0.demand-backup-7562
+ local command=find
+ local uri=myApp:myPass@some-name-rs0-0.some-name-rs0.demand-backup-7562
+ local postfix=
+ local suffix=
+ local database=myApp
+ local collection=test
+ local sort=
+ local tls=false
+ local replicaset=
+ local 'full_command=db.test.find()'
+ [[ -n '' ]]
+ log 'running db.test.find() in myApp'
+ set +o xtrace
[2026-04-14T09:42:01+0000] running db.test.find() in myApp
+ [[ false == true ]]
+ mongo_command=run_mongo
+ run_mongo 'use myApp\n db.test.find()' myApp:myPass@some-name-rs0-0.some-name-rs0.demand-backup-7562 mongodb '' ''
+ local 'command=use myApp\n db.test.find()'
+ local uri=myApp:myPass@some-name-rs0-0.some-name-rs0.demand-backup-7562
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local replica_set=rs0
+ [[ myApp:myPass@some-name-rs0-0.some-name-rs0.demand-backup-7562 == *cfg* ]]
+ grep -E -v 'I NETWORK|W NETWORK|F NETWORK|"c":"NETWORK"|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|Started a new thread for the timer service'
+ /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.WiVXXV5xh9
+++ mktemp
++ local LAST_ERR=/tmp/tmp.MhQrYSBYGS
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.WiVXXV5xh9
++ cat /tmp/tmp.MhQrYSBYGS
++ rm /tmp/tmp.WiVXXV5xh9 /tmp/tmp.MhQrYSBYGS
++ return 0
+ local client_container=psmdb-client-bb8b97679-62vtp
+ kubectl_bin exec psmdb-client-bb8b97679-62vtp -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-0.some-name-rs0.demand-backup-7562.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
++ mktemp
+ local LAST_OUT=/tmp/tmp.ZddjDmsTZb
++ mktemp
+ local LAST_ERR=/tmp/tmp.vD8YLjtcr0
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-bb8b97679-62vtp -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-0.some-name-rs0.demand-backup-7562.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.ZddjDmsTZb
+ cat /tmp/tmp.vD8YLjtcr0
+ rm /tmp/tmp.ZddjDmsTZb /tmp/tmp.vD8YLjtcr0
+ return 0
+ [[ 0 -eq 0 ]]
+ diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2256/e2e-tests/demand-backup/compare/find.json /tmp/tmp.pzOYzJaY7Q/find
+ for i in $(seq 0 $last_pod)
+ compare_mongo_cmd find myApp:myPass@some-name-rs0-1.some-name-rs0.demand-backup-7562
+ local command=find
+ local uri=myApp:myPass@some-name-rs0-1.some-name-rs0.demand-backup-7562
+ local postfix=
+ local suffix=
+ local database=myApp
+ local collection=test
+ local sort=
+ local tls=false
+ local replicaset=
+ local 'full_command=db.test.find()'
+ [[ -n '' ]]
+ log 'running db.test.find() in myApp'
+ set +o xtrace
[2026-04-14T09:42:04+0000] running db.test.find() in myApp
+ [[ false == true ]]
+ mongo_command=run_mongo
+ run_mongo 'use myApp\n db.test.find()' myApp:myPass@some-name-rs0-1.some-name-rs0.demand-backup-7562 mongodb '' ''
+ local 'command=use myApp\n db.test.find()'
+ local uri=myApp:myPass@some-name-rs0-1.some-name-rs0.demand-backup-7562
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local replica_set=rs0
+ [[ myApp:myPass@some-name-rs0-1.some-name-rs0.demand-backup-7562 == *cfg* ]]
+ grep -E -v 'I NETWORK|W NETWORK|F NETWORK|"c":"NETWORK"|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|Started a new thread for the timer service'
+ /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.yq2uoK4bXM
+++ mktemp
++ local LAST_ERR=/tmp/tmp.V6ulzcVoGo
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.yq2uoK4bXM
++ cat /tmp/tmp.V6ulzcVoGo
++ rm /tmp/tmp.yq2uoK4bXM /tmp/tmp.V6ulzcVoGo
++ return 0
+ local client_container=psmdb-client-bb8b97679-62vtp
+ kubectl_bin exec psmdb-client-bb8b97679-62vtp -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-1.some-name-rs0.demand-backup-7562.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
++ mktemp
+ local LAST_OUT=/tmp/tmp.HRnzrAkHga
++ mktemp
+ local LAST_ERR=/tmp/tmp.lIGc8JCiar
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-bb8b97679-62vtp -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-1.some-name-rs0.demand-backup-7562.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.HRnzrAkHga
+ cat /tmp/tmp.lIGc8JCiar
+ rm /tmp/tmp.HRnzrAkHga /tmp/tmp.lIGc8JCiar
+ return 0
+ [[ 0 -eq 0 ]]
+ diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2256/e2e-tests/demand-backup/compare/find.json /tmp/tmp.pzOYzJaY7Q/find
+ for i in $(seq 0 $last_pod)
+ compare_mongo_cmd find myApp:myPass@some-name-rs0-2.some-name-rs0.demand-backup-7562
+ local command=find
+ local uri=myApp:myPass@some-name-rs0-2.some-name-rs0.demand-backup-7562
+ local postfix=
+ local suffix=
+ local database=myApp
+ local collection=test
+ local sort=
+ local tls=false
+ local replicaset=
+ local 'full_command=db.test.find()'
+ [[ -n '' ]]
+ log 'running db.test.find() in myApp'
+ set +o xtrace
[2026-04-14T09:42:06+0000] running db.test.find() in myApp
+ [[ false == true ]]
+ mongo_command=run_mongo
+ run_mongo 'use myApp\n db.test.find()' myApp:myPass@some-name-rs0-2.some-name-rs0.demand-backup-7562 mongodb '' ''
+ local 'command=use myApp\n db.test.find()'
+ /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
+ local uri=myApp:myPass@some-name-rs0-2.some-name-rs0.demand-backup-7562
+ grep -E -v 'I NETWORK|W NETWORK|F NETWORK|"c":"NETWORK"|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|Started a new thread for the timer service'
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local replica_set=rs0
+ [[ myApp:myPass@some-name-rs0-2.some-name-rs0.demand-backup-7562 == *cfg* ]]
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.SkF8cd8kIa
+++ mktemp
++ local LAST_ERR=/tmp/tmp.mmWgycMMea
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.SkF8cd8kIa
++ cat /tmp/tmp.mmWgycMMea
++ rm /tmp/tmp.SkF8cd8kIa /tmp/tmp.mmWgycMMea
++ return 0
+ local client_container=psmdb-client-bb8b97679-62vtp
+ kubectl_bin exec psmdb-client-bb8b97679-62vtp -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-2.some-name-rs0.demand-backup-7562.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
++ mktemp
+ local LAST_OUT=/tmp/tmp.nYnbLTT5Wa
++ mktemp
+ local LAST_ERR=/tmp/tmp.TjSjjxvXaf
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-bb8b97679-62vtp -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-2.some-name-rs0.demand-backup-7562.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.nYnbLTT5Wa
+ cat /tmp/tmp.TjSjjxvXaf
+ rm /tmp/tmp.nYnbLTT5Wa /tmp/tmp.TjSjjxvXaf
+ return 0
+ [[ 0 -eq 0 ]]
+ diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2256/e2e-tests/demand-backup/compare/find.json /tmp/tmp.pzOYzJaY7Q/find
+ desc 'restore from backup source, no storageName -- minio'
+ set +o xtrace


-----------------------------------------------------------------------------------
restore from backup source, no storageName -- minio
-----------------------------------------------------------------------------------

++ get_backup_dest backup-minio
++ local backup_name=backup-minio
++ kubectl_bin get psmdb-backup backup-minio -o 'jsonpath={.status.destination}'
++ sed -e 's/.json$//'
+++ mktemp
++ sed 's|s3://||'
++ sed 's|azure://||'
++ sed 's|gs://||'
++ local LAST_OUT=/tmp/tmp.j7vj2ilgWM
+++ mktemp
++ local LAST_ERR=/tmp/tmp.Mb8fdzrBOd
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb-backup backup-minio -o 'jsonpath={.status.destination}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.j7vj2ilgWM
++ cat /tmp/tmp.Mb8fdzrBOd
++ rm /tmp/tmp.j7vj2ilgWM /tmp/tmp.Mb8fdzrBOd
++ return 0
+ backup_dest_minio=operator-testing/2026-04-14T09:36:10Z
+ run_recovery_check_bkp_source backup-minio operator-testing/2026-04-14T09:36:10Z some-name-rs0 backup-minio-source-1
+ local backup=backup-minio
+ local backup_dest=operator-testing/2026-04-14T09:36:10Z
+ local cluster=some-name-rs0
+ local source=backup-minio-source-1
+ run_mongo 'use myApp\n db.test.insert({ x: 100501 })' myApp:myPass@some-name-rs0.demand-backup-7562
+ local 'command=use myApp\n db.test.insert({ x: 100501 })'
+ local uri=myApp:myPass@some-name-rs0.demand-backup-7562
+ local driver=mongodb+srv
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local replica_set=rs0
+ [[ myApp:myPass@some-name-rs0.demand-backup-7562 == *cfg* ]]
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.TX5BwYRMyr
+++ mktemp
++ local LAST_ERR=/tmp/tmp.5Pd9NKbyMm
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.TX5BwYRMyr
++ cat /tmp/tmp.5Pd9NKbyMm
++ rm /tmp/tmp.TX5BwYRMyr /tmp/tmp.5Pd9NKbyMm
++ return 0
+ local client_container=psmdb-client-bb8b97679-62vtp
+ kubectl_bin exec psmdb-client-bb8b97679-62vtp -- bash -c 'printf '\''use myApp\n db.test.insert({ x: 100501 })\n'\'' | mongo mongodb+srv://myApp:myPass@some-name-rs0.demand-backup-7562.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
++ mktemp
+ local LAST_OUT=/tmp/tmp.1vLcvGsDol
++ mktemp
+ local LAST_ERR=/tmp/tmp.3aqqHEiEEu
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-bb8b97679-62vtp -- bash -c 'printf '\''use myApp\n db.test.insert({ x: 100501 })\n'\'' | mongo mongodb+srv://myApp:myPass@some-name-rs0.demand-backup-7562.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.1vLcvGsDol
Percona Server for MongoDB shell version v4.4.29-28
connecting to: mongodb://some-name-rs0-2.some-name-rs0.demand-backup-7562.svc.cluster.local:27017,some-name-rs0-0.some-name-rs0.demand-backup-7562.svc.cluster.local:27017,some-name-rs0-1.some-name-rs0.demand-backup-7562.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb&replicaSet=rs0&ssl=false
Implicit session: session { "id" : UUID("c3e28a86-67d4-4bf6-9977-f11ef9da37a1") }
Percona Server for MongoDB server version: v8.0.20-8
WARNING: shell and server versions do not match
switched to db myApp
WriteResult({ "nInserted" : 1 })
bye
+ cat /tmp/tmp.3aqqHEiEEu
+ rm /tmp/tmp.1vLcvGsDol /tmp/tmp.3aqqHEiEEu
+ return 0
+ compare_mongo_cmd find myApp:myPass@some-name-rs0.demand-backup-7562 -2nd .svc.cluster.local myApp test
+ local command=find
+ local uri=myApp:myPass@some-name-rs0.demand-backup-7562
+ local postfix=-2nd
+ local suffix=.svc.cluster.local
+ local database=myApp
+ local collection=test
+ local sort=
+ local tls=false
+ local replicaset=
+ local 'full_command=db.test.find()'
+ [[ -n '' ]]
+ log 'running db.test.find() in myApp'
+ set +o xtrace
[2026-04-14T09:42:13+0000] running db.test.find() in myApp
+ [[ false == true ]]
+ mongo_command=run_mongo
+ run_mongo 'use myApp\n db.test.find()' myApp:myPass@some-name-rs0.demand-backup-7562 mongodb .svc.cluster.local ''
+ grep -E -v 'I NETWORK|W NETWORK|F NETWORK|"c":"NETWORK"|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|Started a new thread for the timer service'
+ local 'command=use myApp\n db.test.find()'
+ local uri=myApp:myPass@some-name-rs0.demand-backup-7562
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local replica_set=rs0
+ [[ myApp:myPass@some-name-rs0.demand-backup-7562 == *cfg* ]]
+ /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.6goHjhAT7H
+++ mktemp
++ local LAST_ERR=/tmp/tmp.Ldt9Aa2vUJ
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.6goHjhAT7H
++ cat /tmp/tmp.Ldt9Aa2vUJ
++ rm /tmp/tmp.6goHjhAT7H /tmp/tmp.Ldt9Aa2vUJ
++ return 0
+ local client_container=psmdb-client-bb8b97679-62vtp
+ kubectl_bin exec psmdb-client-bb8b97679-62vtp -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0.demand-backup-7562.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
++ mktemp
+ local LAST_OUT=/tmp/tmp.Unh1cT5W1R
++ mktemp
+ local LAST_ERR=/tmp/tmp.MIWDBb4B0K
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-bb8b97679-62vtp -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0.demand-backup-7562.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.Unh1cT5W1R
+ cat /tmp/tmp.MIWDBb4B0K
+ rm /tmp/tmp.Unh1cT5W1R /tmp/tmp.MIWDBb4B0K
+ return 0
+ [[ 0 -eq 0 ]]
+ diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2256/e2e-tests/demand-backup/compare/find-2nd.json /tmp/tmp.pzOYzJaY7Q/find-2nd
+ run_restore_backupsource backup-minio-source-1 operator-testing/2026-04-14T09:36:10Z
+ local backupName=backup-minio-source-1
+ local backupDest=operator-testing/2026-04-14T09:36:10Z
+ local storageName=
+ desc 'run restore restore-backup-minio-source-1 from backup backup-minio-source-1 destination is operator-testing/2026-04-14T09:36:10Z'
+ set +o xtrace


-----------------------------------------------------------------------------------
run restore restore-backup-minio-source-1 from backup backup-minio-source-1 destination is operator-testing/2026-04-14T09:36:10Z
-----------------------------------------------------------------------------------

+ '[' -z '' ']'
+ cat /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2256/e2e-tests/demand-backup/conf/restore-backupsource.yml
+ /usr/sbin/sed -e 's|BACKUP-NAME|operator-testing/2026-04-14T09:36:10Z|'
+ /usr/sbin/sed -e 's/name:/name: restore-backup-minio-source-1/'
+ /usr/sbin/sed -e /storageName/d
+ kubectl_bin apply -f -
++ mktemp
+ local LAST_OUT=/tmp/tmp.dmlZWUBT2T
++ mktemp
+ local LAST_ERR=/tmp/tmp.78MNlla26x
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl apply -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.dmlZWUBT2T
perconaservermongodbrestore.psmdb.percona.com/restore-backup-minio-source-1 created
+ cat /tmp/tmp.78MNlla26x
+ rm /tmp/tmp.dmlZWUBT2T /tmp/tmp.78MNlla26x
+ return 0
+ return
+ wait_restore backup-minio-source-1 some-name
+ local backup_name=backup-minio-source-1
+ local cluster_name=some-name
+ local target_state=ready
+ local wait_cluster_consistency=1
+ local wait_time=1780
+ local ok_if_ready=0
+ set +o xtrace
Waiting for the psmdb-restore/restore-backup-minio-source-1 object to be created.OK
Waiting psmdb-restore/restore-backup-minio-source-1 to reach state "ready" .OK after 0 minutes
+ [[ 1 -eq 1 ]]
+ wait_cluster_consistency some-name
+ local cluster_name=some-name
+ local wait_time=32
+ retry=0
+ sleep 7
+ echo -n 'waiting for cluster readyness'
waiting for cluster readyness++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.ykohQAxTT7
+++ mktemp
++ local LAST_ERR=/tmp/tmp.FVynl5udGR
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.ykohQAxTT7
++ cat /tmp/tmp.FVynl5udGR
++ rm /tmp/tmp.ykohQAxTT7 /tmp/tmp.FVynl5udGR
++ return 0
+ [[ ready == ready ]]
+ echo .OK
.OK
+ simple_data_check some-name-rs0 3 0
+ local cluster_name=some-name-rs0
+ let last_pod=3-1
+ local isSharded=0
+ local cluster_pfx=
+ '[' 0 -eq 1 ']'
++ seq 0 2
+ for i in $(seq 0 $last_pod)
+ compare_mongo_cmd find myApp:myPass@some-name-rs0-0.some-name-rs0.demand-backup-7562
+ local command=find
+ local uri=myApp:myPass@some-name-rs0-0.some-name-rs0.demand-backup-7562
+ local postfix=
+ local suffix=
+ local database=myApp
+ local collection=test
+ local sort=
+ local tls=false
+ local replicaset=
+ local 'full_command=db.test.find()'
+ [[ -n '' ]]
+ log 'running db.test.find() in myApp'
+ set +o xtrace
[2026-04-14T09:42:38+0000] running db.test.find() in myApp
+ [[ false == true ]]
+ mongo_command=run_mongo
+ run_mongo 'use myApp\n db.test.find()' myApp:myPass@some-name-rs0-0.some-name-rs0.demand-backup-7562 mongodb '' ''
+ local 'command=use myApp\n db.test.find()'
+ local uri=myApp:myPass@some-name-rs0-0.some-name-rs0.demand-backup-7562
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local replica_set=rs0
+ [[ myApp:myPass@some-name-rs0-0.some-name-rs0.demand-backup-7562 == *cfg* ]]
+ grep -E -v 'I NETWORK|W NETWORK|F NETWORK|"c":"NETWORK"|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|Started a new thread for the timer service'
+ /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.6f9wOMDf5v
+++ mktemp
++ local LAST_ERR=/tmp/tmp.VJsNCp6rul
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.6f9wOMDf5v
++ cat /tmp/tmp.VJsNCp6rul
++ rm /tmp/tmp.6f9wOMDf5v /tmp/tmp.VJsNCp6rul
++ return 0
+ local client_container=psmdb-client-bb8b97679-62vtp
+ kubectl_bin exec psmdb-client-bb8b97679-62vtp -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-0.some-name-rs0.demand-backup-7562.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
++ mktemp
+ local LAST_OUT=/tmp/tmp.juKg2P9tQP
++ mktemp
+ local LAST_ERR=/tmp/tmp.PxyvaIIwWf
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-bb8b97679-62vtp -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-0.some-name-rs0.demand-backup-7562.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.juKg2P9tQP
+ cat /tmp/tmp.PxyvaIIwWf
+ rm /tmp/tmp.juKg2P9tQP /tmp/tmp.PxyvaIIwWf
+ return 0
+ [[ 0 -eq 0 ]]
+ diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2256/e2e-tests/demand-backup/compare/find.json /tmp/tmp.pzOYzJaY7Q/find
+ for i in $(seq 0 $last_pod)
+ compare_mongo_cmd find myApp:myPass@some-name-rs0-1.some-name-rs0.demand-backup-7562
+ local command=find
+ local uri=myApp:myPass@some-name-rs0-1.some-name-rs0.demand-backup-7562
+ local postfix=
+ local suffix=
+ local database=myApp
+ local collection=test
+ local sort=
+ local tls=false
+ local replicaset=
+ local 'full_command=db.test.find()'
+ [[ -n '' ]]
+ log 'running db.test.find() in myApp'
+ set +o xtrace
[2026-04-14T09:42:40+0000] running db.test.find() in myApp
+ [[ false == true ]]
+ mongo_command=run_mongo
+ run_mongo 'use myApp\n db.test.find()' myApp:myPass@some-name-rs0-1.some-name-rs0.demand-backup-7562 mongodb '' ''
+ local 'command=use myApp\n db.test.find()'
+ local uri=myApp:myPass@some-name-rs0-1.some-name-rs0.demand-backup-7562
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local replica_set=rs0
+ [[ myApp:myPass@some-name-rs0-1.some-name-rs0.demand-backup-7562 == *cfg* ]]
+ grep -E -v 'I NETWORK|W NETWORK|F NETWORK|"c":"NETWORK"|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|Started a new thread for the timer service'
+ /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.18XvzLHduA
+++ mktemp
++ local LAST_ERR=/tmp/tmp.T0bh4ar0Vd
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.18XvzLHduA
++ cat /tmp/tmp.T0bh4ar0Vd
++ rm /tmp/tmp.18XvzLHduA /tmp/tmp.T0bh4ar0Vd
++ return 0
+ local client_container=psmdb-client-bb8b97679-62vtp
+ kubectl_bin exec psmdb-client-bb8b97679-62vtp -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-1.some-name-rs0.demand-backup-7562.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
++ mktemp
+ local LAST_OUT=/tmp/tmp.XPvAmdMlhl
++ mktemp
+ local LAST_ERR=/tmp/tmp.K6lWBuUmQU
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-bb8b97679-62vtp -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-1.some-name-rs0.demand-backup-7562.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.XPvAmdMlhl
+ cat /tmp/tmp.K6lWBuUmQU
+ rm /tmp/tmp.XPvAmdMlhl /tmp/tmp.K6lWBuUmQU
+ return 0
+ [[ 0 -eq 0 ]]
+ diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2256/e2e-tests/demand-backup/compare/find.json /tmp/tmp.pzOYzJaY7Q/find
+ for i in $(seq 0 $last_pod)
+ compare_mongo_cmd find myApp:myPass@some-name-rs0-2.some-name-rs0.demand-backup-7562
+ local command=find
+ local uri=myApp:myPass@some-name-rs0-2.some-name-rs0.demand-backup-7562
+ local postfix=
+ local suffix=
+ local database=myApp
+ local collection=test
+ local sort=
+ local tls=false
+ local replicaset=
+ local 'full_command=db.test.find()'
+ [[ -n '' ]]
+ log 'running db.test.find() in myApp'
+ set +o xtrace
[2026-04-14T09:42:43+0000] running db.test.find() in myApp
+ [[ false == true ]]
+ mongo_command=run_mongo
+ run_mongo 'use myApp\n db.test.find()' myApp:myPass@some-name-rs0-2.some-name-rs0.demand-backup-7562 mongodb '' ''
+ local 'command=use myApp\n db.test.find()'
+ local uri=myApp:myPass@some-name-rs0-2.some-name-rs0.demand-backup-7562
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local replica_set=rs0
+ [[ myApp:myPass@some-name-rs0-2.some-name-rs0.demand-backup-7562 == *cfg* ]]
+ /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
+ grep -E -v 'I NETWORK|W NETWORK|F NETWORK|"c":"NETWORK"|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|Started a new thread for the timer service'
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.YlP4QwRL3k
+++ mktemp
++ local LAST_ERR=/tmp/tmp.y0mdY5kny6
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.YlP4QwRL3k
++ cat /tmp/tmp.y0mdY5kny6
++ rm /tmp/tmp.YlP4QwRL3k /tmp/tmp.y0mdY5kny6
++ return 0
+ local client_container=psmdb-client-bb8b97679-62vtp
+ kubectl_bin exec psmdb-client-bb8b97679-62vtp -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-2.some-name-rs0.demand-backup-7562.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
++ mktemp
+ local LAST_OUT=/tmp/tmp.4WUMXUTE4F
++ mktemp
+ local LAST_ERR=/tmp/tmp.SViSVzdRDE
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-bb8b97679-62vtp -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-2.some-name-rs0.demand-backup-7562.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.4WUMXUTE4F
+ cat /tmp/tmp.SViSVzdRDE
+ rm /tmp/tmp.4WUMXUTE4F /tmp/tmp.SViSVzdRDE
+ return 0
+ [[ 0 -eq 0 ]]
+ diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2256/e2e-tests/demand-backup/compare/find.json /tmp/tmp.pzOYzJaY7Q/find
+ desc 'delete backup and check if it is removed from bucket -- minio'
+ set +o xtrace


-----------------------------------------------------------------------------------
delete backup and check if it is removed from bucket -- minio
-----------------------------------------------------------------------------------

+ kubectl_bin delete psmdb-backup --all
++ mktemp
+ local LAST_OUT=/tmp/tmp.K5I4MpDNN0
++ mktemp
+ local LAST_ERR=/tmp/tmp.kcCqEeMtb2
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl delete psmdb-backup --all
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.K5I4MpDNN0
perconaservermongodbbackup.psmdb.percona.com "backup-aws-s3" deleted from demand-backup-7562 namespace
perconaservermongodbbackup.psmdb.percona.com "backup-azure-blob" deleted from demand-backup-7562 namespace
perconaservermongodbbackup.psmdb.percona.com "backup-gcp-cs-s3" deleted from demand-backup-7562 namespace
perconaservermongodbbackup.psmdb.percona.com "backup-gcp-cs-sa" deleted from demand-backup-7562 namespace
perconaservermongodbbackup.psmdb.percona.com "backup-minio" deleted from demand-backup-7562 namespace
+ cat /tmp/tmp.kcCqEeMtb2
+ rm /tmp/tmp.K5I4MpDNN0 /tmp/tmp.kcCqEeMtb2
+ return 0
++ kubectl_bin run -i --rm aws-cli --image=docker.io/perconalab/awscli --restart=Never -- /usr/bin/env AWS_ACCESS_KEY_ID=some-access-key AWS_SECRET_ACCESS_KEY=some-secret-key AWS_DEFAULT_REGION=us-east-1 /usr/bin/aws --endpoint-url http://minio-service:9000 s3 ls s3://operator-testing/
++ grep -c operator-testing/2026-04-14T09:36:10Z
++ cat
+++ mktemp
++ local LAST_OUT=/tmp/tmp.rqbLWT8qBt
+++ mktemp
++ local LAST_ERR=/tmp/tmp.eqhcjOjos5
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl run -i --rm aws-cli --image=docker.io/perconalab/awscli --restart=Never -- /usr/bin/env AWS_ACCESS_KEY_ID=some-access-key AWS_SECRET_ACCESS_KEY=some-secret-key AWS_DEFAULT_REGION=us-east-1 /usr/bin/aws --endpoint-url http://minio-service:9000 s3 ls s3://operator-testing/
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.rqbLWT8qBt
++ cat /tmp/tmp.eqhcjOjos5
++ rm /tmp/tmp.rqbLWT8qBt /tmp/tmp.eqhcjOjos5
++ return 0
+ backup_exists=0
+ [[ 0 -eq 1 ]]
+ '[' -z '' ']'
+ check_backup_deletion_aws operator-testing/psmdb-demand-backup/2026-04-14T09:35:36Z /rs0/myApp.test.gz
++ echo operator-testing/psmdb-demand-backup/2026-04-14T09:35:36Z
++ cut -d/ -f1
+ bucket=operator-testing
++ echo operator-testing/psmdb-demand-backup/2026-04-14T09:35:36Z
++ cut -d/ -f2-
+ key_prefix=psmdb-demand-backup/2026-04-14T09:35:36Z
+ key=/rs0/myApp.test.gz
+ storage_name=aws-s3
+ retry=0
+ aws s3api head-object --bucket operator-testing --key psmdb-demand-backup/2026-04-14T09:35:36Z/rs0/myApp.test.gz
+ echo 'Backup psmdb-demand-backup/2026-04-14T09:35:36Z/rs0/myApp.test.gz in bucket operator-testing not found in aws-s3'
Backup psmdb-demand-backup/2026-04-14T09:35:36Z/rs0/myApp.test.gz in bucket operator-testing not found in aws-s3
+ check_backup_deletion_gcs operator-testing/psmdb-demand-backup/2026-04-14T09:35:59Z /rs0/myApp.test.gz
+ backup_dest_gcp=operator-testing/psmdb-demand-backup/2026-04-14T09:35:59Z
+ obj=/rs0/myApp.test.gz
+ storage_name=gcp-cs
+ retry=0
+ gcs_path=gs://operator-testing/psmdb-demand-backup/2026-04-14T09:35:59Z/rs0/myApp.test.gz
+ gsutil ls gs://operator-testing/psmdb-demand-backup/2026-04-14T09:35:59Z/rs0/myApp.test.gz
+ echo 'Backup gs://operator-testing/psmdb-demand-backup/2026-04-14T09:35:59Z/rs0/myApp.test.gz not found in gcp-cs'
Backup gs://operator-testing/psmdb-demand-backup/2026-04-14T09:35:59Z/rs0/myApp.test.gz not found in gcp-cs
+ check_backup_deletion_gcs operator-testing/psmdb-demand-backup/2026-04-14T09:36:21Z /rs0/myApp.test.gz
+ backup_dest_gcp=operator-testing/psmdb-demand-backup/2026-04-14T09:36:21Z
+ obj=/rs0/myApp.test.gz
+ storage_name=gcp-cs
+ retry=0
+ gcs_path=gs://operator-testing/psmdb-demand-backup/2026-04-14T09:36:21Z/rs0/myApp.test.gz
+ gsutil ls gs://operator-testing/psmdb-demand-backup/2026-04-14T09:36:21Z/rs0/myApp.test.gz
+ echo 'Backup gs://operator-testing/psmdb-demand-backup/2026-04-14T09:36:21Z/rs0/myApp.test.gz not found in gcp-cs'
Backup gs://operator-testing/psmdb-demand-backup/2026-04-14T09:36:21Z/rs0/myApp.test.gz not found in gcp-cs
+ check_backup_deletion_azure https://engk8soperators.blob.core.windows.net/operator-testing/psmdb-demand-backup/2026-04-14T09:35:47Z /rs0/myApp.test.gz
++ echo https://engk8soperators.blob.core.windows.net/operator-testing/psmdb-demand-backup/2026-04-14T09:35:47Z
++ sed 's|https://[^/]*\.blob\.core\.windows\.net/||'
+ url_path=operator-testing/psmdb-demand-backup/2026-04-14T09:35:47Z
++ echo operator-testing/psmdb-demand-backup/2026-04-14T09:35:47Z
++ cut -d/ -f1
+ container=operator-testing
++ echo operator-testing/psmdb-demand-backup/2026-04-14T09:35:47Z
++ cut -d/ -f2-
+ blob_prefix=psmdb-demand-backup/2026-04-14T09:35:47Z
+ blob=/rs0/myApp.test.gz
+ storage_name=azure-blob
+ retry=0
+ blob_path=psmdb-demand-backup/2026-04-14T09:35:47Z/rs0/myApp.test.gz
+ az storage blob show --container-name operator-testing --name psmdb-demand-backup/2026-04-14T09:35:47Z/rs0/myApp.test.gz
+ echo 'Backup psmdb-demand-backup/2026-04-14T09:35:47Z/rs0/myApp.test.gz in container operator-testing not found in azure-blob'
Backup psmdb-demand-backup/2026-04-14T09:35:47Z/rs0/myApp.test.gz in container operator-testing not found in azure-blob
+ desc 'checking backup deletion without cr'
+ set +o xtrace


-----------------------------------------------------------------------------------
checking backup deletion without cr
-----------------------------------------------------------------------------------

+ run_backup minio
+ local storage=minio
+ local backup_name=backup-minio
+ local type=logical
+ log 'running backup backup-minio'
+ set +o xtrace
[2026-04-14T09:43:01+0000] running backup backup-minio
+ yq eval $'.metadata.name = "backup-minio"\n\t\t\t| .spec.storageName = "minio"\n\t\t\t| .spec.type = "logical"' /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2256/e2e-tests/demand-backup/conf/backup-minio.yml
+ kubectl_bin apply -f -
++ mktemp
+ local LAST_OUT=/tmp/tmp.9UV6Q6oalw
++ mktemp
+ local LAST_ERR=/tmp/tmp.nitEyLLMRn
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl apply -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.9UV6Q6oalw
perconaservermongodbbackup.psmdb.percona.com/backup-minio created
+ cat /tmp/tmp.nitEyLLMRn
+ rm /tmp/tmp.9UV6Q6oalw /tmp/tmp.nitEyLLMRn
+ return 0
+ '[' -z '' ']'
+ run_backup aws-s3
+ local storage=aws-s3
+ local backup_name=backup-aws-s3
+ local type=logical
+ log 'running backup backup-aws-s3'
+ set +o xtrace
[2026-04-14T09:43:03+0000] running backup backup-aws-s3
+ yq eval $'.metadata.name = "backup-aws-s3"\n\t\t\t| .spec.storageName = "aws-s3"\n\t\t\t| .spec.type = "logical"' /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2256/e2e-tests/demand-backup/conf/backup-aws-s3.yml
+ kubectl_bin apply -f -
++ mktemp
+ local LAST_OUT=/tmp/tmp.hl0kVeTfQG
++ mktemp
+ local LAST_ERR=/tmp/tmp.yBOoOtgteK
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl apply -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.hl0kVeTfQG
perconaservermongodbbackup.psmdb.percona.com/backup-aws-s3 created
+ cat /tmp/tmp.yBOoOtgteK
+ rm /tmp/tmp.hl0kVeTfQG /tmp/tmp.yBOoOtgteK
+ return 0
+ run_backup gcp-cs-s3
+ local storage=gcp-cs-s3
+ local backup_name=backup-gcp-cs-s3
+ local type=logical
+ log 'running backup backup-gcp-cs-s3'
+ set +o xtrace
[2026-04-14T09:43:05+0000] running backup backup-gcp-cs-s3
+ yq eval $'.metadata.name = "backup-gcp-cs-s3"\n\t\t\t| .spec.storageName = "gcp-cs-s3"\n\t\t\t| .spec.type = "logical"' /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2256/e2e-tests/demand-backup/conf/backup-gcp-cs-s3.yml
+ kubectl_bin apply -f -
++ mktemp
+ local LAST_OUT=/tmp/tmp.Bsd4RHFE0R
++ mktemp
+ local LAST_ERR=/tmp/tmp.iMcCoCTSmV
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl apply -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.Bsd4RHFE0R
perconaservermongodbbackup.psmdb.percona.com/backup-gcp-cs-s3 created
+ cat /tmp/tmp.iMcCoCTSmV
+ rm /tmp/tmp.Bsd4RHFE0R /tmp/tmp.iMcCoCTSmV
+ return 0
+ run_backup gcp-cs-sa
+ local storage=gcp-cs-sa
+ local backup_name=backup-gcp-cs-sa
+ local type=logical
+ log 'running backup backup-gcp-cs-sa'
+ set +o xtrace
[2026-04-14T09:43:07+0000] running backup backup-gcp-cs-sa
+ yq eval $'.metadata.name = "backup-gcp-cs-sa"\n\t\t\t| .spec.storageName = "gcp-cs-sa"\n\t\t\t| .spec.type = "logical"' /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2256/e2e-tests/demand-backup/conf/backup-gcp-cs-sa.yml
+ kubectl_bin apply -f -
++ mktemp
+ local LAST_OUT=/tmp/tmp.KG5AJANYmV
++ mktemp
+ local LAST_ERR=/tmp/tmp.rYahZB19UX
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl apply -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.KG5AJANYmV
perconaservermongodbbackup.psmdb.percona.com/backup-gcp-cs-sa created
+ cat /tmp/tmp.rYahZB19UX
+ rm /tmp/tmp.KG5AJANYmV /tmp/tmp.rYahZB19UX
+ return 0
+ run_backup azure-blob
+ local storage=azure-blob
+ local backup_name=backup-azure-blob
+ local type=logical
+ log 'running backup backup-azure-blob'
+ set +o xtrace
[2026-04-14T09:43:10+0000] running backup backup-azure-blob
+ yq eval $'.metadata.name = "backup-azure-blob"\n\t\t\t| .spec.storageName = "azure-blob"\n\t\t\t| .spec.type = "logical"' /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2256/e2e-tests/demand-backup/conf/backup-azure-blob.yml
+ kubectl_bin apply -f -
++ mktemp
+ local LAST_OUT=/tmp/tmp.dAW54YMr1c
++ mktemp
+ local LAST_ERR=/tmp/tmp.DewgoOO5OR
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl apply -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.dAW54YMr1c
perconaservermongodbbackup.psmdb.percona.com/backup-azure-blob created
+ cat /tmp/tmp.DewgoOO5OR
+ rm /tmp/tmp.dAW54YMr1c /tmp/tmp.DewgoOO5OR
+ return 0
+ wait_backup backup-minio
+ local backup_name=backup-minio
+ local target_state=ready
+ set +o xtrace
waiting for backup-minio to reach ready state..OK
+ '[' -z '' ']'
+ wait_backup backup-aws-s3
+ local backup_name=backup-aws-s3
+ local target_state=ready
+ set +o xtrace
waiting for backup-aws-s3 to reach ready state.............OK
+ wait_backup backup-gcp-cs-s3
+ local backup_name=backup-gcp-cs-s3
+ local target_state=ready
+ set +o xtrace
waiting for backup-gcp-cs-s3 to reach ready state..............OK
+ wait_backup backup-gcp-cs-sa
+ local backup_name=backup-gcp-cs-sa
+ local target_state=ready
+ set +o xtrace
waiting for backup-gcp-cs-sa to reach ready state..OK
+ wait_backup backup-azure-blob
+ local backup_name=backup-azure-blob
+ local target_state=ready
+ set +o xtrace
waiting for backup-azure-blob to reach ready state..OK
++ get_backup_dest backup-aws-s3
++ local backup_name=backup-aws-s3
++ kubectl_bin get psmdb-backup backup-aws-s3 -o 'jsonpath={.status.destination}'
++ sed -e 's/.json$//'
++ sed 's|s3://||'
++ sed 's|azure://||'
++ sed 's|gs://||'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.QivxYhM5hJ
+++ mktemp
++ local LAST_ERR=/tmp/tmp.wVRbFERO1s
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb-backup backup-aws-s3 -o 'jsonpath={.status.destination}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.QivxYhM5hJ
++ cat /tmp/tmp.wVRbFERO1s
++ rm /tmp/tmp.QivxYhM5hJ /tmp/tmp.wVRbFERO1s
++ return 0
+ backup_dest_aws=operator-testing/psmdb-demand-backup/2026-04-14T09:43:26Z
++ get_backup_dest backup-gcp-cs-s3
++ local backup_name=backup-gcp-cs-s3
++ kubectl_bin get psmdb-backup backup-gcp-cs-s3 -o 'jsonpath={.status.destination}'
++ sed -e 's/.json$//'
++ sed 's|s3://||'
+++ mktemp
++ sed 's|azure://||'
++ sed 's|gs://||'
++ local LAST_OUT=/tmp/tmp.BPnK6gKHkR
+++ mktemp
++ local LAST_ERR=/tmp/tmp.fdpWNA7FC0
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb-backup backup-gcp-cs-s3 -o 'jsonpath={.status.destination}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.BPnK6gKHkR
++ cat /tmp/tmp.fdpWNA7FC0
++ rm /tmp/tmp.BPnK6gKHkR /tmp/tmp.fdpWNA7FC0
++ return 0
+ backup_dest_gcp_s3=operator-testing/psmdb-demand-backup/2026-04-14T09:43:53Z
++ get_backup_dest backup-gcp-cs-sa
++ local backup_name=backup-gcp-cs-sa
++ kubectl_bin get psmdb-backup backup-gcp-cs-sa -o 'jsonpath={.status.destination}'
++ sed -e 's/.json$//'
++ sed 's|s3://||'
+++ mktemp
++ sed 's|azure://||'
++ sed 's|gs://||'
++ local LAST_OUT=/tmp/tmp.APK4HNh8IE
+++ mktemp
++ local LAST_ERR=/tmp/tmp.FpOQdYbynT
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb-backup backup-gcp-cs-sa -o 'jsonpath={.status.destination}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.APK4HNh8IE
++ cat /tmp/tmp.FpOQdYbynT
++ rm /tmp/tmp.APK4HNh8IE /tmp/tmp.FpOQdYbynT
++ return 0
+ backup_dest_gcp_sa=operator-testing/psmdb-demand-backup/2026-04-14T09:43:15Z
++ get_backup_dest backup-azure-blob
++ local backup_name=backup-azure-blob
++ kubectl_bin get psmdb-backup backup-azure-blob -o 'jsonpath={.status.destination}'
++ sed -e 's/.json$//'
++ sed 's|s3://||'
++ sed 's|azure://||'
+++ mktemp
++ sed 's|gs://||'
++ local LAST_OUT=/tmp/tmp.PU1DIVa53K
+++ mktemp
++ local LAST_ERR=/tmp/tmp.NWiIVR6uvw
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb-backup backup-azure-blob -o 'jsonpath={.status.destination}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.PU1DIVa53K
++ cat /tmp/tmp.NWiIVR6uvw
++ rm /tmp/tmp.PU1DIVa53K /tmp/tmp.NWiIVR6uvw
++ return 0
+ backup_dest_azure=https://engk8soperators.blob.core.windows.net/operator-testing/psmdb-demand-backup/2026-04-14T09:43:38Z
+ kubectl_bin delete psmdb --all
++ mktemp
+ local LAST_OUT=/tmp/tmp.uMlz9OO0Ga
++ mktemp
+ local LAST_ERR=/tmp/tmp.OdWf68q1HO
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl delete psmdb --all
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.uMlz9OO0Ga
perconaservermongodb.psmdb.percona.com "some-name" deleted from demand-backup-7562 namespace
+ cat /tmp/tmp.OdWf68q1HO
+ rm /tmp/tmp.uMlz9OO0Ga /tmp/tmp.OdWf68q1HO
+ return 0
+ sleep 60
+ desc 'delete backup and check if it is removed from bucket -- minio'
+ set +o xtrace


-----------------------------------------------------------------------------------
delete backup and check if it is removed from bucket -- minio
-----------------------------------------------------------------------------------

+ kubectl_bin delete psmdb-backup --all
++ mktemp
+ local LAST_OUT=/tmp/tmp.fth5gTZGCE
++ mktemp
+ local LAST_ERR=/tmp/tmp.rUzXJNsEW0
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl delete psmdb-backup --all
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.fth5gTZGCE
perconaservermongodbbackup.psmdb.percona.com "backup-aws-s3" deleted from demand-backup-7562 namespace
perconaservermongodbbackup.psmdb.percona.com "backup-azure-blob" deleted from demand-backup-7562 namespace
perconaservermongodbbackup.psmdb.percona.com "backup-gcp-cs-s3" deleted from demand-backup-7562 namespace
perconaservermongodbbackup.psmdb.percona.com "backup-gcp-cs-sa" deleted from demand-backup-7562 namespace
perconaservermongodbbackup.psmdb.percona.com "backup-minio" deleted from demand-backup-7562 namespace
+ cat /tmp/tmp.rUzXJNsEW0
+ rm /tmp/tmp.fth5gTZGCE /tmp/tmp.rUzXJNsEW0
+ return 0
++ kubectl_bin run -i --rm aws-cli --image=docker.io/perconalab/awscli --restart=Never -- /usr/bin/env AWS_ACCESS_KEY_ID=some-access-key AWS_SECRET_ACCESS_KEY=some-secret-key AWS_DEFAULT_REGION=us-east-1 /usr/bin/aws --endpoint-url http://minio-service:9000 s3 ls s3://operator-testing/
++ grep -c operator-testing/2026-04-14T09:36:10Z
++ cat
+++ mktemp
++ local LAST_OUT=/tmp/tmp.vl6S5UsAie
+++ mktemp
++ local LAST_ERR=/tmp/tmp.Rv9s2gqLmo
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl run -i --rm aws-cli --image=docker.io/perconalab/awscli --restart=Never -- /usr/bin/env AWS_ACCESS_KEY_ID=some-access-key AWS_SECRET_ACCESS_KEY=some-secret-key AWS_DEFAULT_REGION=us-east-1 /usr/bin/aws --endpoint-url http://minio-service:9000 s3 ls s3://operator-testing/
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.vl6S5UsAie
++ cat /tmp/tmp.Rv9s2gqLmo
++ rm /tmp/tmp.vl6S5UsAie /tmp/tmp.Rv9s2gqLmo
++ return 0
+ backup_exists=0
+ [[ 0 -eq 1 ]]
+ '[' -z '' ']'
+ check_backup_deletion_aws operator-testing/psmdb-demand-backup/2026-04-14T09:43:26Z /rs0/myApp.test.gz
++ echo operator-testing/psmdb-demand-backup/2026-04-14T09:43:26Z
++ cut -d/ -f1
+ bucket=operator-testing
++ echo operator-testing/psmdb-demand-backup/2026-04-14T09:43:26Z
++ cut -d/ -f2-
+ key_prefix=psmdb-demand-backup/2026-04-14T09:43:26Z
+ key=/rs0/myApp.test.gz
+ storage_name=aws-s3
+ retry=0
+ aws s3api head-object --bucket operator-testing --key psmdb-demand-backup/2026-04-14T09:43:26Z/rs0/myApp.test.gz
+ echo 'Backup psmdb-demand-backup/2026-04-14T09:43:26Z/rs0/myApp.test.gz in bucket operator-testing not found in aws-s3'
Backup psmdb-demand-backup/2026-04-14T09:43:26Z/rs0/myApp.test.gz in bucket operator-testing not found in aws-s3
+ check_backup_deletion_gcs operator-testing/psmdb-demand-backup/2026-04-14T09:43:53Z /rs0/myApp.test.gz
+ backup_dest_gcp=operator-testing/psmdb-demand-backup/2026-04-14T09:43:53Z
+ obj=/rs0/myApp.test.gz
+ storage_name=gcp-cs
+ retry=0
+ gcs_path=gs://operator-testing/psmdb-demand-backup/2026-04-14T09:43:53Z/rs0/myApp.test.gz
+ gsutil ls gs://operator-testing/psmdb-demand-backup/2026-04-14T09:43:53Z/rs0/myApp.test.gz
+ echo 'Backup gs://operator-testing/psmdb-demand-backup/2026-04-14T09:43:53Z/rs0/myApp.test.gz not found in gcp-cs'
Backup gs://operator-testing/psmdb-demand-backup/2026-04-14T09:43:53Z/rs0/myApp.test.gz not found in gcp-cs
+ check_backup_deletion_gcs operator-testing/psmdb-demand-backup/2026-04-14T09:43:15Z /rs0/myApp.test.gz
+ backup_dest_gcp=operator-testing/psmdb-demand-backup/2026-04-14T09:43:15Z
+ obj=/rs0/myApp.test.gz
+ storage_name=gcp-cs
+ retry=0
+ gcs_path=gs://operator-testing/psmdb-demand-backup/2026-04-14T09:43:15Z/rs0/myApp.test.gz
+ gsutil ls gs://operator-testing/psmdb-demand-backup/2026-04-14T09:43:15Z/rs0/myApp.test.gz
+ echo 'Backup gs://operator-testing/psmdb-demand-backup/2026-04-14T09:43:15Z/rs0/myApp.test.gz not found in gcp-cs'
Backup gs://operator-testing/psmdb-demand-backup/2026-04-14T09:43:15Z/rs0/myApp.test.gz not found in gcp-cs
+ check_backup_deletion_azure https://engk8soperators.blob.core.windows.net/operator-testing/psmdb-demand-backup/2026-04-14T09:43:38Z /rs0/myApp.test.gz
++ echo https://engk8soperators.blob.core.windows.net/operator-testing/psmdb-demand-backup/2026-04-14T09:43:38Z
++ sed 's|https://[^/]*\.blob\.core\.windows\.net/||'
+ url_path=operator-testing/psmdb-demand-backup/2026-04-14T09:43:38Z
++ echo operator-testing/psmdb-demand-backup/2026-04-14T09:43:38Z
++ cut -d/ -f1
+ container=operator-testing
++ echo operator-testing/psmdb-demand-backup/2026-04-14T09:43:38Z
++ cut -d/ -f2-
+ blob_prefix=psmdb-demand-backup/2026-04-14T09:43:38Z
+ blob=/rs0/myApp.test.gz
+ storage_name=azure-blob
+ retry=0
+ blob_path=psmdb-demand-backup/2026-04-14T09:43:38Z/rs0/myApp.test.gz
+ az storage blob show --container-name operator-testing --name psmdb-demand-backup/2026-04-14T09:43:38Z/rs0/myApp.test.gz
+ echo 'Backup psmdb-demand-backup/2026-04-14T09:43:38Z/rs0/myApp.test.gz in container operator-testing not found in azure-blob'
Backup psmdb-demand-backup/2026-04-14T09:43:38Z/rs0/myApp.test.gz in container operator-testing not found in azure-blob
+ destroy demand-backup-7562
+ local namespace=demand-backup-7562
+ local ignore_logs=true
+ [[ 0 == 1 ]]
+ desc 'destroy cluster/operator and all other resources'
+ set +o xtrace


-----------------------------------------------------------------------------------
destroy cluster/operator and all other resources
-----------------------------------------------------------------------------------

+ '[' true == false ']'
+ delete_backups
+ desc 'Delete psmdb-backup'
+ set +o xtrace


-----------------------------------------------------------------------------------
Delete psmdb-backup
-----------------------------------------------------------------------------------

++ kubectl_bin get psmdb-backup --no-headers
++ wc -l
+++ mktemp
++ local LAST_OUT=/tmp/tmp.DNdnMnX7gT
+++ mktemp
++ local LAST_ERR=/tmp/tmp.k0B1HDJ93z
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb-backup --no-headers
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.DNdnMnX7gT
++ cat /tmp/tmp.k0B1HDJ93z
No resources found in demand-backup-7562 namespace.
++ rm /tmp/tmp.DNdnMnX7gT /tmp/tmp.k0B1HDJ93z
++ return 0
+ '[' 0 '!=' 0 ']'
+ delete_crd
+ desc 'get and delete old CRDs and RBAC'
+ set +o xtrace


-----------------------------------------------------------------------------------
get and delete old CRDs and RBAC
-----------------------------------------------------------------------------------

+ kubectl_bin delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2256/deploy/crd.yaml --ignore-not-found --wait=false
++ mktemp
+ local LAST_OUT=/tmp/tmp.mEJAGvRtDi
++ mktemp
+ local LAST_ERR=/tmp/tmp.hStQc0zuJi
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2256/deploy/crd.yaml --ignore-not-found --wait=false
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.mEJAGvRtDi
customresourcedefinition.apiextensions.k8s.io "perconaservermongodbbackups.psmdb.percona.com" deleted
customresourcedefinition.apiextensions.k8s.io "perconaservermongodbrestores.psmdb.percona.com" deleted
customresourcedefinition.apiextensions.k8s.io "perconaservermongodbs.psmdb.percona.com" deleted
+ cat /tmp/tmp.hStQc0zuJi
+ rm /tmp/tmp.mEJAGvRtDi /tmp/tmp.hStQc0zuJi
+ return 0
++ yq eval .metadata.name /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2256/deploy/crd.yaml
++ grep -v '\-\-\-'
grep: warning: stray \ before -
grep: warning: stray \ before -
+ for crd_name in $(yq eval '.metadata.name' "${src_dir}/deploy/crd.yaml" | grep -v '\-\-\-')
+ kubectl get perconaservermongodbbackups.psmdb.percona.com --all-namespaces -o wide
+ xargs -L 1 sh -xc 'kubectl patch perconaservermongodbbackups.psmdb.percona.com -n $0 $1 --type=merge -p "{\"metadata\":{\"finalizers\":[]}}"'
+ grep -v NAMESPACE
error: the server doesn't have a resource type "perconaservermongodbbackups"
+ kubectl patch perconaservermongodbbackups.psmdb.percona.com -n sh --type=merge -p '{"metadata":{"finalizers":[]}}'
error: the server doesn't have a resource type "perconaservermongodbbackups"
+ :
+ kubectl_bin wait --for=delete crd perconaservermongodbbackups.psmdb.percona.com
++ mktemp
+ local LAST_OUT=/tmp/tmp.Ad5lEUwiPi
++ mktemp
+ local LAST_ERR=/tmp/tmp.yK62l5Z2Eo
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl wait --for=delete crd perconaservermongodbbackups.psmdb.percona.com
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.Ad5lEUwiPi
+ cat /tmp/tmp.yK62l5Z2Eo
+ rm /tmp/tmp.Ad5lEUwiPi /tmp/tmp.yK62l5Z2Eo
+ return 0
+ for crd_name in $(yq eval '.metadata.name' "${src_dir}/deploy/crd.yaml" | grep -v '\-\-\-')
+ kubectl get perconaservermongodbrestores.psmdb.percona.com --all-namespaces -o wide
+ grep -v NAMESPACE
+ xargs -L 1 sh -xc 'kubectl patch perconaservermongodbrestores.psmdb.percona.com -n $0 $1 --type=merge -p "{\"metadata\":{\"finalizers\":[]}}"'
error: the server doesn't have a resource type "perconaservermongodbrestores"
+ kubectl patch perconaservermongodbrestores.psmdb.percona.com -n sh --type=merge -p '{"metadata":{"finalizers":[]}}'
error: the server doesn't have a resource type "perconaservermongodbrestores"
+ :
+ kubectl_bin wait --for=delete crd perconaservermongodbrestores.psmdb.percona.com
++ mktemp
+ local LAST_OUT=/tmp/tmp.8LhjqEBtJJ
++ mktemp
+ local LAST_ERR=/tmp/tmp.8BOabS4JZM
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl wait --for=delete crd perconaservermongodbrestores.psmdb.percona.com
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.8LhjqEBtJJ
+ cat /tmp/tmp.8BOabS4JZM
+ rm /tmp/tmp.8LhjqEBtJJ /tmp/tmp.8BOabS4JZM
+ return 0
+ for crd_name in $(yq eval '.metadata.name' "${src_dir}/deploy/crd.yaml" | grep -v '\-\-\-')
+ kubectl get perconaservermongodbs.psmdb.percona.com --all-namespaces -o wide
+ grep -v NAMESPACE
+ xargs -L 1 sh -xc 'kubectl patch perconaservermongodbs.psmdb.percona.com -n $0 $1 --type=merge -p "{\"metadata\":{\"finalizers\":[]}}"'
error: the server doesn't have a resource type "perconaservermongodbs"
+ kubectl patch perconaservermongodbs.psmdb.percona.com -n sh --type=merge -p '{"metadata":{"finalizers":[]}}'
error: the server doesn't have a resource type "perconaservermongodbs"
+ :
+ kubectl_bin wait --for=delete crd perconaservermongodbs.psmdb.percona.com
++ mktemp
+ local LAST_OUT=/tmp/tmp.3uVBDDacPR
++ mktemp
+ local LAST_ERR=/tmp/tmp.wQMQfnbUKE
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl wait --for=delete crd perconaservermongodbs.psmdb.percona.com
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.3uVBDDacPR
+ cat /tmp/tmp.wQMQfnbUKE
+ rm /tmp/tmp.3uVBDDacPR /tmp/tmp.wQMQfnbUKE
+ return 0
+ local rbac_yaml=rbac.yaml
+ '[' -n psmdb-operator ']'
+ rbac_yaml=cw-rbac.yaml
+ kubectl_bin delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2256/deploy/cw-rbac.yaml --ignore-not-found
++ mktemp
+ local LAST_OUT=/tmp/tmp.RjbMkGQZ3j
++ mktemp
+ local LAST_ERR=/tmp/tmp.ZOKecwD0MU
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2256/deploy/cw-rbac.yaml --ignore-not-found
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.RjbMkGQZ3j
clusterrole.rbac.authorization.k8s.io "percona-server-mongodb-operator" deleted
clusterrolebinding.rbac.authorization.k8s.io "service-account-percona-server-mongodb-operator" deleted
+ cat /tmp/tmp.ZOKecwD0MU
+ rm /tmp/tmp.RjbMkGQZ3j /tmp/tmp.ZOKecwD0MU
+ return 0
+ destroy_cert_manager
+ kubectl_bin delete -f https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml
++ mktemp
+ local LAST_OUT=/tmp/tmp.7AjqOman5Z
++ mktemp
+ local LAST_ERR=/tmp/tmp.8wIAGyjUH5
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl delete -f https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml
+ exit_status=1
+ set -e
+ '[' 1 '!=' 0 -a -n 1 ']'
+ cat /tmp/tmp.7AjqOman5Z
+ cat /tmp/tmp.8wIAGyjUH5
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": namespaces "cert-manager" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "challenges.acme.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "orders.acme.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "certificaterequests.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "certificates.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "clusterissuers.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "issuers.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": serviceaccounts "cert-manager-cainjector" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": serviceaccounts "cert-manager" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": serviceaccounts "cert-manager-webhook" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-cainjector" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-issuers" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-clusterissuers" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-certificates" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-orders" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-challenges" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-ingress-shim" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-cluster-view" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-view" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-edit" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-approve:cert-manager-io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-certificatesigningrequests" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-webhook:subjectaccessreviews" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-cainjector" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-issuers" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-clusterissuers" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-certificates" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-orders" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-challenges" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-ingress-shim" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-approve:cert-manager-io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-certificatesigningrequests" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-webhook:subjectaccessreviews" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager-cainjector:leaderelection" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager:leaderelection" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager-tokenrequest" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager-webhook:dynamic-serving" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager-cainjector:leaderelection" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager:leaderelection" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager-tokenrequest" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager-webhook:dynamic-serving" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": services "cert-manager-cainjector" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": services "cert-manager" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": services "cert-manager-webhook" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": deployments.apps "cert-manager-cainjector" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": deployments.apps "cert-manager" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": deployments.apps "cert-manager-webhook" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": mutatingwebhookconfigurations.admissionregistration.k8s.io "cert-manager-webhook" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": validatingwebhookconfigurations.admissionregistration.k8s.io "cert-manager-webhook" not found
+ sleep 0
+ for i in $(seq 0 2)
+ set +e
+ kubectl delete -f https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml
+ exit_status=1
+ set -e
+ '[' 1 '!=' 0 -a -n 1 ']'
+ cat /tmp/tmp.7AjqOman5Z
+ cat /tmp/tmp.8wIAGyjUH5
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": namespaces "cert-manager" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "challenges.acme.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "orders.acme.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "certificaterequests.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "certificates.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "clusterissuers.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "issuers.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": serviceaccounts "cert-manager-cainjector" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": serviceaccounts "cert-manager" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": serviceaccounts "cert-manager-webhook" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-cainjector" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-issuers" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-clusterissuers" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-certificates" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-orders" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-challenges" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-ingress-shim" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-cluster-view" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-view" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-edit" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-approve:cert-manager-io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-certificatesigningrequests" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-webhook:subjectaccessreviews" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-cainjector" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-issuers" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-clusterissuers" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-certificates" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-orders" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-challenges" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-ingress-shim" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-approve:cert-manager-io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-certificatesigningrequests" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-webhook:subjectaccessreviews" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager-cainjector:leaderelection" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager:leaderelection" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager-tokenrequest" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager-webhook:dynamic-serving" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager-cainjector:leaderelection" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager:leaderelection" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager-tokenrequest" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager-webhook:dynamic-serving" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": services "cert-manager-cainjector" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": services "cert-manager" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": services "cert-manager-webhook" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": deployments.apps "cert-manager-cainjector" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": deployments.apps "cert-manager" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": deployments.apps "cert-manager-webhook" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": mutatingwebhookconfigurations.admissionregistration.k8s.io "cert-manager-webhook" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": validatingwebhookconfigurations.admissionregistration.k8s.io "cert-manager-webhook" not found
+ sleep 4
+ for i in $(seq 0 2)
+ set +e
+ kubectl delete -f https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml
+ exit_status=1
+ set -e
+ '[' 1 '!=' 0 -a -n 1 ']'
+ cat /tmp/tmp.7AjqOman5Z
+ cat /tmp/tmp.8wIAGyjUH5
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": namespaces "cert-manager" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "challenges.acme.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "orders.acme.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "certificaterequests.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "certificates.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "clusterissuers.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "issuers.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": serviceaccounts "cert-manager-cainjector" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": serviceaccounts "cert-manager" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": serviceaccounts "cert-manager-webhook" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-cainjector" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-issuers" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-clusterissuers" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-certificates" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-orders" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-challenges" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-ingress-shim" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-cluster-view" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-view" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-edit" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-approve:cert-manager-io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-certificatesigningrequests" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-webhook:subjectaccessreviews" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-cainjector" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-issuers" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-clusterissuers" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-certificates" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-orders" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-challenges" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-ingress-shim" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-approve:cert-manager-io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-certificatesigningrequests" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-webhook:subjectaccessreviews" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager-cainjector:leaderelection" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager:leaderelection" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager-tokenrequest" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager-webhook:dynamic-serving" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager-cainjector:leaderelection" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager:leaderelection" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager-tokenrequest" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager-webhook:dynamic-serving" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": services "cert-manager-cainjector" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": services "cert-manager" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": services "cert-manager-webhook" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": deployments.apps "cert-manager-cainjector" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": deployments.apps "cert-manager" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": deployments.apps "cert-manager-webhook" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": mutatingwebhookconfigurations.admissionregistration.k8s.io "cert-manager-webhook" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": validatingwebhookconfigurations.admissionregistration.k8s.io "cert-manager-webhook" not found
+ sleep 8
+ cat /tmp/tmp.7AjqOman5Z
+ cat /tmp/tmp.8wIAGyjUH5
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": namespaces "cert-manager" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "challenges.acme.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "orders.acme.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "certificaterequests.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "certificates.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "clusterissuers.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "issuers.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": serviceaccounts "cert-manager-cainjector" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": serviceaccounts "cert-manager" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": serviceaccounts "cert-manager-webhook" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-cainjector" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-issuers" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-clusterissuers" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-certificates" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-orders" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-challenges" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-ingress-shim" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-cluster-view" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-view" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-edit" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-approve:cert-manager-io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-certificatesigningrequests" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-webhook:subjectaccessreviews" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-cainjector" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-issuers" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-clusterissuers" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-certificates" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-orders" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-challenges" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-ingress-shim" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-approve:cert-manager-io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-certificatesigningrequests" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-webhook:subjectaccessreviews" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager-cainjector:leaderelection" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager:leaderelection" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager-tokenrequest" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager-webhook:dynamic-serving" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager-cainjector:leaderelection" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager:leaderelection" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager-tokenrequest" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager-webhook:dynamic-serving" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": services "cert-manager-cainjector" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": services "cert-manager" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": services "cert-manager-webhook" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": deployments.apps "cert-manager-cainjector" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": deployments.apps "cert-manager" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": deployments.apps "cert-manager-webhook" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": mutatingwebhookconfigurations.admissionregistration.k8s.io "cert-manager-webhook" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": validatingwebhookconfigurations.admissionregistration.k8s.io "cert-manager-webhook" not found
+ rm /tmp/tmp.7AjqOman5Z /tmp/tmp.8wIAGyjUH5
+ return 1
+ true
+ '[' -n '' ']'
+ '[' -n psmdb-operator ']'
+ kubectl_bin delete --grace-period=0 --force=true namespace demand-backup-7562
+ rm -rf /tmp/tmp.pzOYzJaY7Q
+ kubectl_bin delete --grace-period=0 --force=true namespace psmdb-operator
++ mktemp
++ mktemp
+ local LAST_OUT=/tmp/tmp.WnpRyfH4tU
++ mktemp
+ desc 'test passed'
+ set +o xtrace


-----------------------------------------------------------------------------------
test passed
-----------------------------------------------------------------------------------

+ local LAST_OUT=/tmp/tmp.XsRczL9xdR
+ local LAST_ERR=/tmp/tmp.SLw7xcSmEq
+ local exit_status=0
+ local timeout=4
++ mktemp
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl delete --grace-period=0 --force=true namespace psmdb-operator
+ local LAST_ERR=/tmp/tmp.F4XMxkoKhC
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl delete --grace-period=0 --force=true namespace demand-backup-7562