Log: /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2271/e2e-tests/logs/demand-backup.log
Warning: version difference between client (1.35) and server (1.32) exceeds the supported minor version skew of +/-1
Warning: version difference between client (1.35) and server (1.32) exceeds the supported minor version skew of +/-1
Warning: version difference between client (1.35) and server (1.32) exceeds the supported minor version skew of +/-1
+ create_infra demand-backup-2582
+ local ns=demand-backup-2582
+ [[ 1 == 1 ]]
+ delete_crd
+ desc 'get and delete old CRDs and RBAC'
+ set +o xtrace


-----------------------------------------------------------------------------------
get and delete old CRDs and RBAC
-----------------------------------------------------------------------------------

+ kubectl_bin delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2271/deploy/crd.yaml --ignore-not-found --wait=false
++ mktemp
+ local LAST_OUT=/tmp/tmp.RZPxQOwEKr
++ mktemp
+ local LAST_ERR=/tmp/tmp.Dl4603oPDj
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2271/deploy/crd.yaml --ignore-not-found --wait=false
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.RZPxQOwEKr
customresourcedefinition.apiextensions.k8s.io "perconaservermongodbbackups.psmdb.percona.com" deleted
customresourcedefinition.apiextensions.k8s.io "perconaservermongodbrestores.psmdb.percona.com" deleted
customresourcedefinition.apiextensions.k8s.io "perconaservermongodbs.psmdb.percona.com" deleted
+ cat /tmp/tmp.Dl4603oPDj
+ rm /tmp/tmp.RZPxQOwEKr /tmp/tmp.Dl4603oPDj
+ return 0
++ yq eval .metadata.name /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2271/deploy/crd.yaml
++ grep -v '\-\-\-'
grep: warning: stray \ before -
grep: warning: stray \ before -
+ for crd_name in $(yq eval '.metadata.name' "${src_dir}/deploy/crd.yaml" | grep -v '\-\-\-')
+ kubectl get perconaservermongodbbackups.psmdb.percona.com --all-namespaces -o wide
+ grep -v NAMESPACE
+ xargs -L 1 sh -xc 'kubectl patch perconaservermongodbbackups.psmdb.percona.com -n $0 $1 --type=merge -p "{\"metadata\":{\"finalizers\":[]}}"'
error: the server doesn't have a resource type "perconaservermongodbbackups"
+ kubectl patch perconaservermongodbbackups.psmdb.percona.com -n sh --type=merge -p '{"metadata":{"finalizers":[]}}'
error: the server doesn't have a resource type "perconaservermongodbbackups"
+ :
+ kubectl_bin wait --for=delete crd perconaservermongodbbackups.psmdb.percona.com
++ mktemp
+ local LAST_OUT=/tmp/tmp.EiT4xZspAK
++ mktemp
+ local LAST_ERR=/tmp/tmp.bPCf0O2thc
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl wait --for=delete crd perconaservermongodbbackups.psmdb.percona.com
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.EiT4xZspAK
+ cat /tmp/tmp.bPCf0O2thc
+ rm /tmp/tmp.EiT4xZspAK /tmp/tmp.bPCf0O2thc
+ return 0
+ for crd_name in $(yq eval '.metadata.name' "${src_dir}/deploy/crd.yaml" | grep -v '\-\-\-')
+ kubectl get perconaservermongodbrestores.psmdb.percona.com --all-namespaces -o wide
+ grep -v NAMESPACE
+ xargs -L 1 sh -xc 'kubectl patch perconaservermongodbrestores.psmdb.percona.com -n $0 $1 --type=merge -p "{\"metadata\":{\"finalizers\":[]}}"'
error: the server doesn't have a resource type "perconaservermongodbrestores"
+ kubectl patch perconaservermongodbrestores.psmdb.percona.com -n sh --type=merge -p '{"metadata":{"finalizers":[]}}'
error: the server doesn't have a resource type "perconaservermongodbrestores"
+ :
+ kubectl_bin wait --for=delete crd perconaservermongodbrestores.psmdb.percona.com
++ mktemp
+ local LAST_OUT=/tmp/tmp.4ItufXCziv
++ mktemp
+ local LAST_ERR=/tmp/tmp.BhxdyMalUH
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl wait --for=delete crd perconaservermongodbrestores.psmdb.percona.com
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.4ItufXCziv
+ cat /tmp/tmp.BhxdyMalUH
+ rm /tmp/tmp.4ItufXCziv /tmp/tmp.BhxdyMalUH
+ return 0
+ for crd_name in $(yq eval '.metadata.name' "${src_dir}/deploy/crd.yaml" | grep -v '\-\-\-')
+ kubectl get perconaservermongodbs.psmdb.percona.com --all-namespaces -o wide
+ grep -v NAMESPACE
+ xargs -L 1 sh -xc 'kubectl patch perconaservermongodbs.psmdb.percona.com -n $0 $1 --type=merge -p "{\"metadata\":{\"finalizers\":[]}}"'
error: the server doesn't have a resource type "perconaservermongodbs"
+ kubectl patch perconaservermongodbs.psmdb.percona.com -n sh --type=merge -p '{"metadata":{"finalizers":[]}}'
error: the server doesn't have a resource type "perconaservermongodbs"
+ :
+ kubectl_bin wait --for=delete crd perconaservermongodbs.psmdb.percona.com
++ mktemp
+ local LAST_OUT=/tmp/tmp.e6kfe7qrAW
++ mktemp
+ local LAST_ERR=/tmp/tmp.oh2RIMtUh3
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl wait --for=delete crd perconaservermongodbs.psmdb.percona.com
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.e6kfe7qrAW
+ cat /tmp/tmp.oh2RIMtUh3
+ rm /tmp/tmp.e6kfe7qrAW /tmp/tmp.oh2RIMtUh3
+ return 0
+ local rbac_yaml=rbac.yaml
+ '[' -n psmdb-operator ']'
+ rbac_yaml=cw-rbac.yaml
+ kubectl_bin delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2271/deploy/cw-rbac.yaml --ignore-not-found
++ mktemp
+ local LAST_OUT=/tmp/tmp.SmD4GluHqL
++ mktemp
+ local LAST_ERR=/tmp/tmp.Vnl0WrdAwY
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2271/deploy/cw-rbac.yaml --ignore-not-found
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.SmD4GluHqL
clusterrole.rbac.authorization.k8s.io "percona-server-mongodb-operator" deleted
clusterrolebinding.rbac.authorization.k8s.io "service-account-percona-server-mongodb-operator" deleted
+ cat /tmp/tmp.Vnl0WrdAwY
+ rm /tmp/tmp.SmD4GluHqL /tmp/tmp.Vnl0WrdAwY
+ return 0
+ check_crd_for_deletion PR-2271-c31e8c20
+ local git_tag=PR-2271-c31e8c20
++ curl -s https://raw.githubusercontent.com/percona/percona-server-mongodb-operator/PR-2271-c31e8c20/deploy/crd.yaml
++ /usr/sbin/sed s/---//g
++ /usr/sbin/sed ':a;N;$!ba;s/\n/ /g'
++ yq eval .metadata.name
+ for crd_name in $(curl -s https://raw.githubusercontent.com/percona/percona-server-mongodb-operator/${git_tag}/deploy/crd.yaml | yq eval '.metadata.name' | $sed 's/---//g' | $sed ':a;N;$!ba;s/\n/ /g')
++ kubectl_bin get crd/null -o 'jsonpath={.status.conditions[-1].type}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.R1w0Yo7wVL
+++ mktemp
++ local LAST_ERR=/tmp/tmp.Trtf9PyeSB
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get crd/null -o 'jsonpath={.status.conditions[-1].type}'
++ exit_status=1
++ set -e
++ '[' 1 '!=' 0 -a -n 1 ']'
++ cat /tmp/tmp.R1w0Yo7wVL
++ cat /tmp/tmp.Trtf9PyeSB
Error from server (NotFound): customresourcedefinitions.apiextensions.k8s.io "null" not found
++ sleep 0
++ for i in $(seq 0 2)
++ set +e
++ kubectl get crd/null -o 'jsonpath={.status.conditions[-1].type}'
++ exit_status=1
++ set -e
++ '[' 1 '!=' 0 -a -n 1 ']'
++ cat /tmp/tmp.R1w0Yo7wVL
++ cat /tmp/tmp.Trtf9PyeSB
Error from server (NotFound): customresourcedefinitions.apiextensions.k8s.io "null" not found
++ sleep 4
++ for i in $(seq 0 2)
++ set +e
++ kubectl get crd/null -o 'jsonpath={.status.conditions[-1].type}'
++ exit_status=1
++ set -e
++ '[' 1 '!=' 0 -a -n 1 ']'
++ cat /tmp/tmp.R1w0Yo7wVL
++ cat /tmp/tmp.Trtf9PyeSB
Error from server (NotFound): customresourcedefinitions.apiextensions.k8s.io "null" not found
++ sleep 8
++ cat /tmp/tmp.R1w0Yo7wVL
++ cat /tmp/tmp.Trtf9PyeSB
Error from server (NotFound): customresourcedefinitions.apiextensions.k8s.io "null" not found
++ rm /tmp/tmp.R1w0Yo7wVL /tmp/tmp.Trtf9PyeSB
++ return 1
+ [[ '' == \T\e\r\m\i\n\a\t\i\n\g ]]
+ '[' -n psmdb-operator ']'
+ create_namespace psmdb-operator
+ local namespace=psmdb-operator
+ local skip_clean_namespace=
+ [[ 1 == 1 ]]
+ [[ -z '' ]]
+ destroy_chaos_mesh
++ helm list --all-namespaces --filter chaos-mesh
++ tail -n1
++ awk '-F ' '{print $2}'
++ sed s/NAMESPACE//
+ local chaos_mesh_ns=
+ desc 'destroy chaos-mesh'
+ set +o xtrace


-----------------------------------------------------------------------------------
destroy chaos-mesh
-----------------------------------------------------------------------------------

+ '[' -n '' ']'
++ kubectl get MutatingWebhookConfiguration
++ grep chaos-mesh
++ awk '{print $1}'
+ timeout 30 kubectl delete MutatingWebhookConfiguration
error: resource(s) were provided, but no name was specified
+ :
++ kubectl get ValidatingWebhookConfiguration
++ grep chaos-mesh
++ awk '{print $1}'
+ timeout 30 kubectl delete ValidatingWebhookConfiguration
error: resource(s) were provided, but no name was specified
+ :
++ kubectl get ValidatingWebhookConfiguration
++ grep validate-auth
++ awk '{print $1}'
+ timeout 30 kubectl delete ValidatingWebhookConfiguration
error: resource(s) were provided, but no name was specified
+ :
++ kubectl api-resources
++ grep chaos-mesh
++ awk '{print $1}'
++ kubectl get crd
++ grep chaos-mesh.org
++ awk '{print $1}'
+ timeout 30 kubectl delete crd
error: resource(s) were provided, but no name was specified
+ :
++ kubectl get clusterrolebinding
++ grep chaos-mesh
++ awk '{print $1}'
+ timeout 30 kubectl delete clusterrolebinding
error: resource(s) were provided, but no name was specified
+ :
++ kubectl get clusterrole
++ grep chaos-mesh
++ awk '{print $1}'
+ timeout 30 kubectl delete clusterrole
error: resource(s) were provided, but no name was specified
+ :
+ desc 'cleaned up all old namespaces'
+ set +o xtrace


-----------------------------------------------------------------------------------
cleaned up all old namespaces
-----------------------------------------------------------------------------------

+ kubectl_bin get ns
+ grep -E -v '^kube-|^default|Terminating|psmdb-operator|openshift|^gke-|^gmp-|^NAME'
+ awk '{print$1}'
+ '[' -n '' ']'
+ desc 'cleaned up old namespaces psmdb-operator'
+ set +o xtrace
+ xargs kubectl delete ns


-----------------------------------------------------------------------------------
cleaned up old namespaces psmdb-operator
-----------------------------------------------------------------------------------

+ kubectl_bin delete namespace psmdb-operator --ignore-not-found
++ mktemp
++ mktemp
+ local LAST_OUT=/tmp/tmp.AMxXCCxiB4
+ local LAST_OUT=/tmp/tmp.fFY0Yd3lJ7
++ mktemp
++ mktemp
+ local LAST_ERR=/tmp/tmp.jX9zIhRG8z
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ local LAST_ERR=/tmp/tmp.Ju5tBwZzsO
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl get ns
+ for i in $(seq 0 2)
+ set +e
+ kubectl delete namespace psmdb-operator --ignore-not-found
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.AMxXCCxiB4
+ cat /tmp/tmp.jX9zIhRG8z
+ rm /tmp/tmp.AMxXCCxiB4 /tmp/tmp.jX9zIhRG8z
+ return 0
namespace "demand-backup-4537" deleted
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.fFY0Yd3lJ7
namespace "psmdb-operator" deleted
+ cat /tmp/tmp.Ju5tBwZzsO
+ rm /tmp/tmp.fFY0Yd3lJ7 /tmp/tmp.Ju5tBwZzsO
+ return 0
+ kubectl_bin wait --for=delete namespace psmdb-operator
++ mktemp
+ local LAST_OUT=/tmp/tmp.rZZBdJzCdm
++ mktemp
+ local LAST_ERR=/tmp/tmp.x4SzCIVJwd
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl wait --for=delete namespace psmdb-operator
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.rZZBdJzCdm
+ cat /tmp/tmp.x4SzCIVJwd
+ rm /tmp/tmp.rZZBdJzCdm /tmp/tmp.x4SzCIVJwd
+ return 0
+ desc 'create namespace psmdb-operator'
+ set +o xtrace


-----------------------------------------------------------------------------------
create namespace psmdb-operator
-----------------------------------------------------------------------------------

+ kubectl_bin create namespace psmdb-operator
++ mktemp
+ local LAST_OUT=/tmp/tmp.b122raxhT5
++ mktemp
+ local LAST_ERR=/tmp/tmp.YZ9Qq2wtBI
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl create namespace psmdb-operator
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.b122raxhT5
namespace/psmdb-operator created
+ cat /tmp/tmp.YZ9Qq2wtBI
+ rm /tmp/tmp.b122raxhT5 /tmp/tmp.YZ9Qq2wtBI
+ return 0
+ set_kube_ctx psmdb-operator
+ local namespace=psmdb-operator
++ kubectl_bin config current-context
+++ mktemp
++ local LAST_OUT=/tmp/tmp.Nl8akpWQ4I
+++ mktemp
++ local LAST_ERR=/tmp/tmp.FsVIDV8cbq
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl config current-context
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.Nl8akpWQ4I
++ cat /tmp/tmp.FsVIDV8cbq
++ rm /tmp/tmp.Nl8akpWQ4I /tmp/tmp.FsVIDV8cbq
++ return 0
+ kubectl_bin config set-context gke_cloud-dev-112233_us-central1-a_jen-psmdb-2271-c31e8c20-4-cluster10 --namespace=psmdb-operator
++ mktemp
+ local LAST_OUT=/tmp/tmp.YCMeaaTDQp
++ mktemp
+ local LAST_ERR=/tmp/tmp.n189pllIGj
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl config set-context gke_cloud-dev-112233_us-central1-a_jen-psmdb-2271-c31e8c20-4-cluster10 --namespace=psmdb-operator
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.YCMeaaTDQp
Context "gke_cloud-dev-112233_us-central1-a_jen-psmdb-2271-c31e8c20-4-cluster10" modified.
+ cat /tmp/tmp.n189pllIGj
+ rm /tmp/tmp.YCMeaaTDQp /tmp/tmp.n189pllIGj
+ return 0
+ deploy_operator
+ desc 'start PSMDB operator: docker.io/perconalab/percona-server-mongodb-operator:PR-2271-c31e8c20'
+ set +o xtrace


-----------------------------------------------------------------------------------
start PSMDB operator: docker.io/perconalab/percona-server-mongodb-operator:PR-2271-c31e8c20
-----------------------------------------------------------------------------------

+ local cr_file
+ '[' -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2271/e2e-tests/demand-backup/conf/crd.yaml ']'
+ cr_file=/mnt/jenkins/workspace/cloud-psmdb-operator_PR-2271/deploy/crd.yaml
+ kubectl_bin apply --server-side --force-conflicts -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2271/deploy/crd.yaml
++ mktemp
+ local LAST_OUT=/tmp/tmp.iUDaBaUPek
++ mktemp
+ local LAST_ERR=/tmp/tmp.e9MQMwD5kI
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl apply --server-side --force-conflicts -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2271/deploy/crd.yaml
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.iUDaBaUPek
customresourcedefinition.apiextensions.k8s.io/perconaservermongodbbackups.psmdb.percona.com serverside-applied
customresourcedefinition.apiextensions.k8s.io/perconaservermongodbrestores.psmdb.percona.com serverside-applied
customresourcedefinition.apiextensions.k8s.io/perconaservermongodbs.psmdb.percona.com serverside-applied
+ cat /tmp/tmp.e9MQMwD5kI
+ rm /tmp/tmp.iUDaBaUPek /tmp/tmp.e9MQMwD5kI
+ return 0
+ '[' -n psmdb-operator ']'
+ apply_rbac cw-rbac
+ local operator_namespace=psmdb-operator
+ local rbac=cw-rbac
+ cat /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2271/deploy/cw-rbac.yaml
+ sed -e 's^namespace: .*^namespace: psmdb-operator^'
+ kubectl_bin apply -n psmdb-operator -f -
++ mktemp
+ local LAST_OUT=/tmp/tmp.VN7uhpIpEM
++ mktemp
+ local LAST_ERR=/tmp/tmp.zQT9j94wkM
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl apply -n psmdb-operator -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.VN7uhpIpEM
clusterrole.rbac.authorization.k8s.io/percona-server-mongodb-operator created
serviceaccount/percona-server-mongodb-operator created
clusterrolebinding.rbac.authorization.k8s.io/service-account-percona-server-mongodb-operator created
+ cat /tmp/tmp.zQT9j94wkM
+ rm /tmp/tmp.VN7uhpIpEM /tmp/tmp.zQT9j94wkM
+ return 0
+ yq eval '
			(.spec.template.spec.containers[].image = "docker.io/perconalab/percona-server-mongodb-operator:PR-2271-c31e8c20") |
			((.. | select(.[] == "DISABLE_TELEMETRY")) |= .value="true") |
			((.. | select(.[] == "LOG_LEVEL")) |= .value="DEBUG")' /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2271/deploy/cw-operator.yaml
+ kubectl_bin apply -n psmdb-operator -f -
++ mktemp
+ local LAST_OUT=/tmp/tmp.TselfCFaMd
++ mktemp
+ local LAST_ERR=/tmp/tmp.sE3mpX4XaG
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl apply -n psmdb-operator -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.TselfCFaMd
deployment.apps/percona-server-mongodb-operator created
+ cat /tmp/tmp.sE3mpX4XaG
+ rm /tmp/tmp.TselfCFaMd /tmp/tmp.sE3mpX4XaG
+ return 0
+ sleep 20
++ get_operator_pod
++ kubectl_bin get pods --selector=name=percona-server-mongodb-operator -o 'jsonpath={.items[].metadata.name}' -n psmdb-operator
+++ mktemp
++ local LAST_OUT=/tmp/tmp.qRuBKg9Qsw
+++ mktemp
++ local LAST_ERR=/tmp/tmp.ZrOkbrT3Yw
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=percona-server-mongodb-operator -o 'jsonpath={.items[].metadata.name}' -n psmdb-operator
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.qRuBKg9Qsw
++ cat /tmp/tmp.ZrOkbrT3Yw
++ rm /tmp/tmp.qRuBKg9Qsw /tmp/tmp.ZrOkbrT3Yw
++ return 0
+ wait_operator_pod percona-server-mongodb-operator-59f5f7f8b4-l8ds5
+ local pod=percona-server-mongodb-operator-59f5f7f8b4-l8ds5
+ set +o xtrace
waiting for pod/percona-server-mongodb-operator-59f5f7f8b4-l8ds5 to be ready.OK
+ echo 'Print operator info from log'
Print operator info from log
+ grep 'Manager starting up'
++ get_operator_pod
++ kubectl_bin get pods --selector=name=percona-server-mongodb-operator -o 'jsonpath={.items[].metadata.name}' -n psmdb-operator
+++ mktemp
++ local LAST_OUT=/tmp/tmp.PKziYsBFC0
+++ mktemp
++ local LAST_ERR=/tmp/tmp.7F77DaHxl2
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=percona-server-mongodb-operator -o 'jsonpath={.items[].metadata.name}' -n psmdb-operator
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.PKziYsBFC0
++ cat /tmp/tmp.7F77DaHxl2
++ rm /tmp/tmp.PKziYsBFC0 /tmp/tmp.7F77DaHxl2
++ return 0
+ kubectl_bin logs -n psmdb-operator percona-server-mongodb-operator-59f5f7f8b4-l8ds5
++ mktemp
+ local LAST_OUT=/tmp/tmp.x7M1ptpUsx
++ mktemp
+ local LAST_ERR=/tmp/tmp.FpXiZprYUW
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl logs -n psmdb-operator percona-server-mongodb-operator-59f5f7f8b4-l8ds5
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.x7M1ptpUsx
+ cat /tmp/tmp.FpXiZprYUW
+ rm /tmp/tmp.x7M1ptpUsx /tmp/tmp.FpXiZprYUW
+ return 0
2026-03-26T08:10:59.975Z	INFO	setup	Manager starting up	{"gitCommit": "c31e8c20f2a920146c4a2410dc38a83148f57a2b", "gitBranch": "PR-2271-c31e8c20", "buildTime": "", "goVersion": "go1.25.8", "os": "linux", "arch": "amd64"}
+ create_namespace demand-backup-2582
+ local namespace=demand-backup-2582
+ local skip_clean_namespace=
+ [[ 1 == 1 ]]
+ [[ -z '' ]]
+ destroy_chaos_mesh
++ helm list --all-namespaces --filter chaos-mesh
++ tail -n1
++ awk '-F ' '{print $2}'
++ sed s/NAMESPACE//
+ local chaos_mesh_ns=
+ desc 'destroy chaos-mesh'
+ set +o xtrace


-----------------------------------------------------------------------------------
destroy chaos-mesh
-----------------------------------------------------------------------------------

+ '[' -n '' ']'
++ kubectl get MutatingWebhookConfiguration
++ grep chaos-mesh
++ awk '{print $1}'
+ timeout 30 kubectl delete MutatingWebhookConfiguration
error: resource(s) were provided, but no name was specified
+ :
++ kubectl get ValidatingWebhookConfiguration
++ grep chaos-mesh
++ awk '{print $1}'
+ timeout 30 kubectl delete ValidatingWebhookConfiguration
error: resource(s) were provided, but no name was specified
+ :
++ kubectl get ValidatingWebhookConfiguration
++ awk '{print $1}'
++ grep validate-auth
+ timeout 30 kubectl delete ValidatingWebhookConfiguration
error: resource(s) were provided, but no name was specified
+ :
++ kubectl api-resources
++ grep chaos-mesh
++ awk '{print $1}'
++ kubectl get crd
++ grep chaos-mesh.org
++ awk '{print $1}'
+ timeout 30 kubectl delete crd
error: resource(s) were provided, but no name was specified
+ :
++ kubectl get clusterrolebinding
++ grep chaos-mesh
++ awk '{print $1}'
+ timeout 30 kubectl delete clusterrolebinding
error: resource(s) were provided, but no name was specified
+ :
++ kubectl get clusterrole
++ grep chaos-mesh
++ awk '{print $1}'
+ timeout 30 kubectl delete clusterrole
error: resource(s) were provided, but no name was specified
+ :
+ desc 'cleaned up all old namespaces'
+ set +o xtrace


-----------------------------------------------------------------------------------
cleaned up all old namespaces
-----------------------------------------------------------------------------------

+ kubectl_bin get ns
+ grep -E -v '^kube-|^default|Terminating|psmdb-operator|openshift|^gke-|^gmp-|^NAME'
+ xargs kubectl delete ns
+ awk '{print$1}'
+ '[' -n '' ']'
+ desc 'cleaned up old namespaces demand-backup-2582'
+ set +o xtrace


-----------------------------------------------------------------------------------
cleaned up old namespaces demand-backup-2582
-----------------------------------------------------------------------------------

+ kubectl_bin delete namespace demand-backup-2582 --ignore-not-found
++ mktemp
++ mktemp
+ local LAST_OUT=/tmp/tmp.d5P7ZrK3KL
+ local LAST_OUT=/tmp/tmp.x6NPWFO7zG
++ mktemp
++ mktemp
+ local LAST_ERR=/tmp/tmp.MWmWIDP1fh
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ local LAST_ERR=/tmp/tmp.nqKLoDTwJn
+ local exit_status=0
+ local timeout=4
+ for i in $(seq 0 2)
+ set +e
+ kubectl get ns
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl delete namespace demand-backup-2582 --ignore-not-found
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.x6NPWFO7zG
+ cat /tmp/tmp.MWmWIDP1fh
+ rm /tmp/tmp.x6NPWFO7zG /tmp/tmp.MWmWIDP1fh
+ return 0
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.d5P7ZrK3KL
+ cat /tmp/tmp.nqKLoDTwJn
+ rm /tmp/tmp.d5P7ZrK3KL /tmp/tmp.nqKLoDTwJn
+ return 0
error: resource(s) were provided, but no name was specified
+ kubectl_bin wait --for=delete namespace demand-backup-2582
++ mktemp
+ local LAST_OUT=/tmp/tmp.2wTEJO7xhi
++ mktemp
+ local LAST_ERR=/tmp/tmp.EXjpZlz7eE
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl wait --for=delete namespace demand-backup-2582
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.2wTEJO7xhi
+ cat /tmp/tmp.EXjpZlz7eE
+ rm /tmp/tmp.2wTEJO7xhi /tmp/tmp.EXjpZlz7eE
+ return 0
+ desc 'create namespace demand-backup-2582'
+ set +o xtrace


-----------------------------------------------------------------------------------
create namespace demand-backup-2582
-----------------------------------------------------------------------------------

+ kubectl_bin create namespace demand-backup-2582
++ mktemp
+ local LAST_OUT=/tmp/tmp.GJrlzcDShs
++ mktemp
+ local LAST_ERR=/tmp/tmp.wBPeXflXYg
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl create namespace demand-backup-2582
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.GJrlzcDShs
namespace/demand-backup-2582 created
+ cat /tmp/tmp.wBPeXflXYg
+ rm /tmp/tmp.GJrlzcDShs /tmp/tmp.wBPeXflXYg
+ return 0
+ set_kube_ctx demand-backup-2582
+ local namespace=demand-backup-2582
++ kubectl_bin config current-context
+++ mktemp
++ local LAST_OUT=/tmp/tmp.sEBOxEJomz
+++ mktemp
++ local LAST_ERR=/tmp/tmp.2yorpoO333
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl config current-context
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.sEBOxEJomz
++ cat /tmp/tmp.2yorpoO333
++ rm /tmp/tmp.sEBOxEJomz /tmp/tmp.2yorpoO333
++ return 0
+ kubectl_bin config set-context gke_cloud-dev-112233_us-central1-a_jen-psmdb-2271-c31e8c20-4-cluster10 --namespace=demand-backup-2582
++ mktemp
+ local LAST_OUT=/tmp/tmp.JH8KqYQ2A4
++ mktemp
+ local LAST_ERR=/tmp/tmp.0wkBHVHQGU
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl config set-context gke_cloud-dev-112233_us-central1-a_jen-psmdb-2271-c31e8c20-4-cluster10 --namespace=demand-backup-2582
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.JH8KqYQ2A4
Context "gke_cloud-dev-112233_us-central1-a_jen-psmdb-2271-c31e8c20-4-cluster10" modified.
+ cat /tmp/tmp.0wkBHVHQGU
+ rm /tmp/tmp.JH8KqYQ2A4 /tmp/tmp.0wkBHVHQGU
+ return 0
+ deploy_minio
+ local cert_secret=
+ local service_name=minio-service
+ desc 'install MinIO: minio-service'
+ set +o xtrace


-----------------------------------------------------------------------------------
install MinIO: minio-service
-----------------------------------------------------------------------------------

+ helm uninstall minio-service
+ :
+ helm repo remove minio
"minio" has been removed from your repositories
+ helm repo add minio https://charts.min.io/
"minio" has been added to your repositories
+ local endpoint=http://minio-service:9000
+ minio_args=('--version' '5.4.0' '--set' 'replicas=1' '--set' 'mode=standalone' '--set' 'resources.requests.memory=256Mi' '--set' 'rootUser=rootuser' '--set' 'rootPassword=rootpass123' '--set' 'users[0].accessKey=some-access-key' '--set' 'users[0].secretKey=some-secret-key' '--set' 'users[0].policy=consoleAdmin' '--set' 'service.type=ClusterIP' '--set' 'configPathmc=/tmp/' '--set' 'securityContext.enabled=false' '--set' 'persistence.size=2G' '--set' 'fullnameOverride=minio-service' '--set' 'serviceAccount.create=true' '--set' 'serviceAccount.name=minio-service-sa')
+ local minio_args
+ [[ -n '' ]]
+ retry 10 60 helm install minio-service --version 5.4.0 --set replicas=1 --set mode=standalone --set resources.requests.memory=256Mi --set rootUser=rootuser --set rootPassword=rootpass123 --set 'users[0].accessKey=some-access-key' --set 'users[0].secretKey=some-secret-key' --set 'users[0].policy=consoleAdmin' --set service.type=ClusterIP --set configPathmc=/tmp/ --set securityContext.enabled=false --set persistence.size=2G --set fullnameOverride=minio-service --set serviceAccount.create=true --set serviceAccount.name=minio-service-sa minio/minio
+ local max=10
+ local delay=60
+ shift 2
+ local n=1
+ helm install minio-service --version 5.4.0 --set replicas=1 --set mode=standalone --set resources.requests.memory=256Mi --set rootUser=rootuser --set rootPassword=rootpass123 --set 'users[0].accessKey=some-access-key' --set 'users[0].secretKey=some-secret-key' --set 'users[0].policy=consoleAdmin' --set service.type=ClusterIP --set configPathmc=/tmp/ --set securityContext.enabled=false --set persistence.size=2G --set fullnameOverride=minio-service --set serviceAccount.create=true --set serviceAccount.name=minio-service-sa minio/minio
NAME: minio-service
LAST DEPLOYED: Thu Mar 26 08:11:35 2026
NAMESPACE: demand-backup-2582
STATUS: deployed
REVISION: 1
TEST SUITE: None
NOTES:
MinIO can be accessed via port 9000 on the following DNS name from within your cluster:
minio-service.demand-backup-2582.cluster.local

To access MinIO from localhost, run the below commands:

  1. export POD_NAME=$(kubectl get pods --namespace demand-backup-2582 -l "release=minio-service" -o jsonpath="{.items[0].metadata.name}")

  2. kubectl port-forward $POD_NAME 9000 --namespace demand-backup-2582

Read more about port forwarding here: http://kubernetes.io/docs/user-guide/kubectl/kubectl_port-forward/

You can now access MinIO server on http://localhost:9000. Follow the below steps to connect to MinIO server with mc client:

  1. Download the MinIO mc client - https://min.io/docs/minio/linux/reference/minio-mc.html#quickstart

  2. export MC_HOST_minio-service-local=http://$(kubectl get secret --namespace demand-backup-2582 minio-service -o jsonpath="{.data.rootUser}" | base64 --decode):$(kubectl get secret --namespace demand-backup-2582 minio-service -o jsonpath="{.data.rootPassword}" | base64 --decode)@localhost:9000

  3. mc ls minio-service-local
++ kubectl_bin get pods --selector=release=minio-service -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.dxbQzOcQZP
+++ mktemp
++ local LAST_ERR=/tmp/tmp.qBl7sjhT3l
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=release=minio-service -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.dxbQzOcQZP
++ cat /tmp/tmp.qBl7sjhT3l
++ rm /tmp/tmp.dxbQzOcQZP /tmp/tmp.qBl7sjhT3l
++ return 0
+ local MINIO_POD=minio-service-6d5f646cdc-p6jqf
+ wait_pod minio-service-6d5f646cdc-p6jqf
+ local pod=minio-service-6d5f646cdc-p6jqf
+ set +o xtrace
waiting for pod/minio-service-6d5f646cdc-p6jqf to be ready.OK
+ '[' -n psmdb-operator ']'
+ kubectl_bin create svc -n psmdb-operator externalname minio-service --external-name=minio-service.demand-backup-2582.svc.cluster.local --tcp=9000
service/minio-service created
+ create_minio_bucket operator-testing http://minio-service:9000
+ local bucket=operator-testing
+ local endpoint=http://minio-service:9000
+ kubectl_bin run -i --rm aws-cli --image=docker.io/perconalab/awscli --restart=Never -- bash -c 'AWS_ACCESS_KEY_ID=some-access-key 		AWS_SECRET_ACCESS_KEY=some-secret-key 		AWS_DEFAULT_REGION=us-east-1         /usr/bin/aws --no-verify-ssl --endpoint-url http://minio-service:9000 s3 mb s3://operator-testing'
++ mktemp
+ local LAST_OUT=/tmp/tmp.VHaXF7nBfz
++ mktemp
+ local LAST_ERR=/tmp/tmp.kSIA09BKDO
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl run -i --rm aws-cli --image=docker.io/perconalab/awscli --restart=Never -- bash -c 'AWS_ACCESS_KEY_ID=some-access-key 		AWS_SECRET_ACCESS_KEY=some-secret-key 		AWS_DEFAULT_REGION=us-east-1         /usr/bin/aws --no-verify-ssl --endpoint-url http://minio-service:9000 s3 mb s3://operator-testing'
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.VHaXF7nBfz
pod "aws-cli" deleted from demand-backup-2582 namespace
+ cat /tmp/tmp.kSIA09BKDO
All commands and output from this session will be recorded in container logs, including credentials and sensitive information passed through the command prompt.
If you don't see a command prompt, try pressing enter.
+ rm /tmp/tmp.VHaXF7nBfz /tmp/tmp.kSIA09BKDO
+ return 0
+ desc 'create secrets and start client'
+ set +o xtrace


-----------------------------------------------------------------------------------
create secrets and start client
-----------------------------------------------------------------------------------

+ cluster=some-name-rs0
+ kubectl_bin apply -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2271/e2e-tests/demand-backup/conf/secrets.yml -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2271/e2e-tests/conf/client.yml
++ mktemp
+ local LAST_OUT=/tmp/tmp.Mi75LlOXmL
++ mktemp
+ local LAST_ERR=/tmp/tmp.CEuFfp6qar
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl apply -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2271/e2e-tests/demand-backup/conf/secrets.yml -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2271/e2e-tests/conf/client.yml
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.Mi75LlOXmL
secret/some-users created
deployment.apps/psmdb-client created
+ cat /tmp/tmp.CEuFfp6qar
+ rm /tmp/tmp.Mi75LlOXmL /tmp/tmp.CEuFfp6qar
+ return 0
+ apply_s3_storage_secrets
+ desc 'create secrets for cloud storages'
+ set +o xtrace


-----------------------------------------------------------------------------------
create secrets for cloud storages
-----------------------------------------------------------------------------------

+ '[' -z '' ']'
+ kubectl_bin apply -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2271/e2e-tests/conf/minio-secret.yml -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2271/e2e-tests/conf/cloud-secret.yml
++ mktemp
+ local LAST_OUT=/tmp/tmp.DHHxsICfmc
++ mktemp
+ local LAST_ERR=/tmp/tmp.SoAaUa4t8g
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl apply -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2271/e2e-tests/conf/minio-secret.yml -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2271/e2e-tests/conf/cloud-secret.yml
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.DHHxsICfmc
secret/minio-secret created
secret/aws-s3-secret created
secret/gcp-cs-secret created
secret/azure-secret created
secret/gcp-cs-sa-key-secret created
+ cat /tmp/tmp.SoAaUa4t8g
+ rm /tmp/tmp.DHHxsICfmc /tmp/tmp.SoAaUa4t8g
+ return 0
+ desc 'create first PSMDB cluster some-name-rs0'
+ set +o xtrace


-----------------------------------------------------------------------------------
create first PSMDB cluster some-name-rs0
-----------------------------------------------------------------------------------

+ apply_cluster /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2271/e2e-tests/demand-backup/conf/some-name-rs0.yml
+ '[' -z '' ']'
+ kubectl_bin apply -f -
+ cat_config /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2271/e2e-tests/demand-backup/conf/some-name-rs0.yml
++ mktemp
+ cat /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2271/e2e-tests/demand-backup/conf/some-name-rs0.yml
+ yq eval '(.spec | select(.image == null)).image = "docker.io/perconalab/percona-server-mongodb-operator:main-mongod8.0"'
+ local LAST_OUT=/tmp/tmp.A3TiUf4N1O
+ yq eval '(.spec | select(has("pmm"))).pmm.image = "docker.io/percona/pmm-client:2.44.1-1"'
++ mktemp
+ local LAST_ERR=/tmp/tmp.BRgPkZmIZd
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ yq eval '(.spec | select(has("backup"))).backup.image = "docker.io/perconalab/percona-server-mongodb-operator:main-backup"'
+ for i in $(seq 0 2)
+ set +e
+ kubectl apply -f -
+ /usr/sbin/sed -e s/NAME_SPACE/demand-backup-2582/g
+ yq eval '.spec.upgradeOptions.apply="Never"'
+ yq eval '(.spec | select(has("initImage"))).initImage = "docker.io/perconalab/percona-server-mongodb-operator:PR-2271-c31e8c20"'
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.A3TiUf4N1O
perconaservermongodb.psmdb.percona.com/some-name created
+ cat /tmp/tmp.BRgPkZmIZd
+ rm /tmp/tmp.A3TiUf4N1O /tmp/tmp.BRgPkZmIZd
+ return 0
+ desc 'check if all 3 Pods started'
+ set +o xtrace


-----------------------------------------------------------------------------------
check if all 3 Pods started
-----------------------------------------------------------------------------------

+ wait_for_running some-name-rs0 3
+ local name=some-name-rs0
+ let last_pod=2
+ local check_cluster_readyness=true
+ set_debug
+ [[ 1 == 1 ]]
+ set -o xtrace
+ local rs_name=rs0
+ local cluster_name=some-name
++ seq 0 2
+ for i in $(seq 0 $last_pod)
+ [[ 0 -eq 2 ]]
+ wait_pod some-name-rs0-0
+ local pod=some-name-rs0-0
+ set +o xtrace
waiting for pod/some-name-rs0-0 to be ready.................OK
+ for i in $(seq 0 $last_pod)
+ [[ 1 -eq 2 ]]
+ wait_pod some-name-rs0-1
+ local pod=some-name-rs0-1
+ set +o xtrace
waiting for pod/some-name-rs0-1 to be ready.................OK
+ for i in $(seq 0 $last_pod)
+ [[ 2 -eq 2 ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].arbiter.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.wiAolHIS9C
+++ mktemp
++ local LAST_ERR=/tmp/tmp.wiRJkfiko7
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].arbiter.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.wiAolHIS9C
++ cat /tmp/tmp.wiRJkfiko7
++ rm /tmp/tmp.wiAolHIS9C /tmp/tmp.wiRJkfiko7
++ return 0
+ [[ '' == \t\r\u\e ]]
+ wait_pod some-name-rs0-2
+ local pod=some-name-rs0-2
+ set +o xtrace
waiting for pod/some-name-rs0-2 to be ready..................OK
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].nonvoting.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.psX8MT7AHF
+++ mktemp
++ local LAST_ERR=/tmp/tmp.yaMvjIUGEp
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].nonvoting.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.psX8MT7AHF
++ cat /tmp/tmp.yaMvjIUGEp
++ rm /tmp/tmp.psX8MT7AHF /tmp/tmp.yaMvjIUGEp
++ return 0
+ [[ '' == \t\r\u\e ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].hidden.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.dFnI6J4z0H
+++ mktemp
++ local LAST_ERR=/tmp/tmp.NqfPryhu7u
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].hidden.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.dFnI6J4z0H
++ cat /tmp/tmp.NqfPryhu7u
++ rm /tmp/tmp.dFnI6J4z0H /tmp/tmp.NqfPryhu7u
++ return 0
+ [[ '' == \t\r\u\e ]]
+ sleep 10
+ [[ true == \t\r\u\e ]]
+ set +x
Waiting for cluster readyness.........
+ desc 'check if service and statefulset created with expected config'
+ set +o xtrace


-----------------------------------------------------------------------------------
check if service and statefulset created with expected config
-----------------------------------------------------------------------------------

+ compare_kubectl statefulset/some-name-rs0
+ local resource=statefulset/some-name-rs0
+ local postfix=
+ local skip_generation_check=
+ local expected_result=/mnt/jenkins/workspace/cloud-psmdb-operator_PR-2271/e2e-tests/demand-backup/compare/statefulset_some-name-rs0.yml
+ local new_result=/tmp/tmp.Fq0pMPZsh8/statefulset_some-name-rs0.yml
+ '[' -n '' -a -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2271/e2e-tests/demand-backup/compare/statefulset_some-name-rs0-oc.yml ']'
+ kubectl_bin get -o yaml statefulset/some-name-rs0
+ yq eval '
			del(.metadata.ownerReferences[].apiVersion) |
			del(.metadata.managedFields) |
			del(.. | select(has("creationTimestamp")).creationTimestamp) |
			del(.. | select(has("namespace")).namespace) |
			del(.. | select(has("uid")).uid) |
			del(.metadata.resourceVersion) |
			del(.spec.template.spec.containers[].env[] | select(.name == "NAMESPACE")) |
			del(.metadata.selfLink) |
			del(.metadata.annotations."cloud.google.com/neg") |
			del(.metadata.annotations."kubectl.kubernetes.io/last-applied-configuration") |
			del(.. | select(has("image")).image) |
			del(.. | select(has("clusterIP")).clusterIP) |
			del(.. | select(has("clusterIPs")).clusterIPs) |
			del(.. | select(has("dataSource")).dataSource) |
			del(.. | select(has("procMount")).procMount) |
			del(.. | select(has("storageClassName")).storageClassName) |
			del(.. | select(has("finalizers")).finalizers) |
			del(.. | select(has("kubernetes.io/pvc-protection"))."kubernetes.io/pvc-protection") |
			del(.. | select(has("volumeName")).volumeName) |
			del(.. | select(has("volume.beta.kubernetes.io/storage-provisioner"))."volume.beta.kubernetes.io/storage-provisioner") |
			del(.. | select(has("volume.kubernetes.io/storage-provisioner"))."volume.kubernetes.io/storage-provisioner") |
			del(.spec.volumeMode) |
			del(.. | select(has("volume.kubernetes.io/selected-node"))."volume.kubernetes.io/selected-node") |
			del(.. | select(has("percona.com/last-config-hash"))."percona.com/last-config-hash") |
			del(.. | select(has("percona.com/configuration-hash"))."percona.com/configuration-hash") |
			del(.. | select(has("percona.com/ssl-hash"))."percona.com/ssl-hash") |
			del(.. | select(has("percona.com/ssl-internal-hash"))."percona.com/ssl-internal-hash") |
			del(.spec.volumeClaimTemplates[].spec.volumeMode | select(. == "Filesystem")) |
			del(.. | select(has("healthCheckNodePort")).healthCheckNodePort) |
			del(.. | select(has("nodePort")).nodePort) |
			del(.status) |
			(.. | select(tag == "!!str")) |= sub("demand-backup-2582", "NAME_SPACE") |
			del(.spec.volumeClaimTemplates[].apiVersion) |
			del(.spec.volumeClaimTemplates[].kind) |
			del(.spec.ipFamilies) |
			del(.spec.ipFamilyPolicy) |
			(.. | select(. == "extensions/v1beta1")) = "apps/v1" |
			(.. | select(. == "batch/v1beta1")) = "batch/v1" ' -
++ mktemp
+ local LAST_OUT=/tmp/tmp.Kth2LimC3e
++ mktemp
+ local LAST_ERR=/tmp/tmp.dLPLveYdxZ
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl get -o yaml statefulset/some-name-rs0
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.Kth2LimC3e
+ cat /tmp/tmp.dLPLveYdxZ
+ rm /tmp/tmp.Kth2LimC3e /tmp/tmp.dLPLveYdxZ
+ return 0
+ yq -i eval 'del(.spec.persistentVolumeClaimRetentionPolicy)' /tmp/tmp.Fq0pMPZsh8/statefulset_some-name-rs0.yml
+ version_gt 1.22
++ echo '1.32 >= 1.22'
++ bc -l
+ '[' 1 -eq 1 ']'
+ return 0
+ yq -i eval 'del(.spec.internalTrafficPolicy)' /tmp/tmp.Fq0pMPZsh8/statefulset_some-name-rs0.yml
+ yq -i eval 'del(.spec.allocateLoadBalancerNodePorts)' /tmp/tmp.Fq0pMPZsh8/statefulset_some-name-rs0.yml
+ [[ /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2271/e2e-tests/demand-backup/compare/statefulset_some-name-rs0.yml == */cronjob* ]]
+ '[' -n '' ']'
+ [[ 0 -eq 0 ]]
+ diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2271/e2e-tests/demand-backup/compare/statefulset_some-name-rs0.yml /tmp/tmp.Fq0pMPZsh8/statefulset_some-name-rs0.yml
+ log 'compare_kubectl: statefulset/some-name-rs0 OK'
+ set +o xtrace
[2026-03-26T08:15:05+0000] compare_kubectl: statefulset/some-name-rs0 OK
+ desc 'create user'
+ set +o xtrace


-----------------------------------------------------------------------------------
create user
-----------------------------------------------------------------------------------

+ run_mongo 'db.createUser({user:"myApp",pwd:"myPass",roles:[{db:"myApp",role:"readWrite"}]})' userAdmin:userAdmin123456@some-name-rs0.demand-backup-2582
+ local 'command=db.createUser({user:"myApp",pwd:"myPass",roles:[{db:"myApp",role:"readWrite"}]})'
+ local uri=userAdmin:userAdmin123456@some-name-rs0.demand-backup-2582
+ local driver=mongodb+srv
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local replica_set=rs0
+ [[ userAdmin:userAdmin123456@some-name-rs0.demand-backup-2582 == *cfg* ]]
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.vR9xfCFiVP
+++ mktemp
++ local LAST_ERR=/tmp/tmp.JznLKhhJ25
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.vR9xfCFiVP
++ cat /tmp/tmp.JznLKhhJ25
++ rm /tmp/tmp.vR9xfCFiVP /tmp/tmp.JznLKhhJ25
++ return 0
+ local client_container=psmdb-client-bb8b97679-wjlk6
+ kubectl_bin exec psmdb-client-bb8b97679-wjlk6 -- bash -c 'printf '\''db.createUser({user:"myApp",pwd:"myPass",roles:[{db:"myApp",role:"readWrite"}]})\n'\'' | mongo mongodb+srv://userAdmin:userAdmin123456@some-name-rs0.demand-backup-2582.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
++ mktemp
+ local LAST_OUT=/tmp/tmp.610ulKQmK9
++ mktemp
+ local LAST_ERR=/tmp/tmp.Dy4foSWfwb
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-bb8b97679-wjlk6 -- bash -c 'printf '\''db.createUser({user:"myApp",pwd:"myPass",roles:[{db:"myApp",role:"readWrite"}]})\n'\'' | mongo mongodb+srv://userAdmin:userAdmin123456@some-name-rs0.demand-backup-2582.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.610ulKQmK9
Percona Server for MongoDB shell version v4.4.29-28
connecting to: mongodb://some-name-rs0-2.some-name-rs0.demand-backup-2582.svc.cluster.local:27017,some-name-rs0-0.some-name-rs0.demand-backup-2582.svc.cluster.local:27017,some-name-rs0-1.some-name-rs0.demand-backup-2582.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb&replicaSet=rs0&ssl=false
Implicit session: session { "id" : UUID("aa8865fa-4a06-4968-b478-e463cb552bf6") }
Percona Server for MongoDB server version: v8.0.19-7
WARNING: shell and server versions do not match
Successfully added user: {
	"user" : "myApp",
	"roles" : [
		{
			"db" : "myApp",
			"role" : "readWrite"
		}
	]
}
bye
+ cat /tmp/tmp.Dy4foSWfwb
+ rm /tmp/tmp.610ulKQmK9 /tmp/tmp.Dy4foSWfwb
+ return 0
+ sleep 2
+ desc 'write data, read from all'
+ set +o xtrace


-----------------------------------------------------------------------------------
write data, read from all
-----------------------------------------------------------------------------------

+ run_mongo 'use myApp\n db.test.insert({ x: 100500 })' myApp:myPass@some-name-rs0.demand-backup-2582
+ local 'command=use myApp\n db.test.insert({ x: 100500 })'
+ local uri=myApp:myPass@some-name-rs0.demand-backup-2582
+ local driver=mongodb+srv
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local replica_set=rs0
+ [[ myApp:myPass@some-name-rs0.demand-backup-2582 == *cfg* ]]
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.hYWjWlfzIF
+++ mktemp
++ local LAST_ERR=/tmp/tmp.KHsZeewpkG
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.hYWjWlfzIF
++ cat /tmp/tmp.KHsZeewpkG
++ rm /tmp/tmp.hYWjWlfzIF /tmp/tmp.KHsZeewpkG
++ return 0
+ local client_container=psmdb-client-bb8b97679-wjlk6
+ kubectl_bin exec psmdb-client-bb8b97679-wjlk6 -- bash -c 'printf '\''use myApp\n db.test.insert({ x: 100500 })\n'\'' | mongo mongodb+srv://myApp:myPass@some-name-rs0.demand-backup-2582.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
++ mktemp
+ local LAST_OUT=/tmp/tmp.I3sxzsl60J
++ mktemp
+ local LAST_ERR=/tmp/tmp.MXBTgGINQU
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-bb8b97679-wjlk6 -- bash -c 'printf '\''use myApp\n db.test.insert({ x: 100500 })\n'\'' | mongo mongodb+srv://myApp:myPass@some-name-rs0.demand-backup-2582.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.I3sxzsl60J
Percona Server for MongoDB shell version v4.4.29-28
connecting to: mongodb://some-name-rs0-1.some-name-rs0.demand-backup-2582.svc.cluster.local:27017,some-name-rs0-2.some-name-rs0.demand-backup-2582.svc.cluster.local:27017,some-name-rs0-0.some-name-rs0.demand-backup-2582.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb&replicaSet=rs0&ssl=false
Implicit session: session { "id" : UUID("31cd9f25-c124-4303-abb0-e7c521754511") }
Percona Server for MongoDB server version: v8.0.19-7
WARNING: shell and server versions do not match
switched to db myApp
WriteResult({ "nInserted" : 1 })
bye
+ cat /tmp/tmp.MXBTgGINQU
+ rm /tmp/tmp.I3sxzsl60J /tmp/tmp.MXBTgGINQU
+ return 0
+ run_mongo 'use myApp\n db.test2.insert({ x: 100501 })' myApp:myPass@some-name-rs0.demand-backup-2582
+ local 'command=use myApp\n db.test2.insert({ x: 100501 })'
+ local uri=myApp:myPass@some-name-rs0.demand-backup-2582
+ local driver=mongodb+srv
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local replica_set=rs0
+ [[ myApp:myPass@some-name-rs0.demand-backup-2582 == *cfg* ]]
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.RPUEsN0ydg
+++ mktemp
++ local LAST_ERR=/tmp/tmp.jOVTsXv4mK
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.RPUEsN0ydg
++ cat /tmp/tmp.jOVTsXv4mK
++ rm /tmp/tmp.RPUEsN0ydg /tmp/tmp.jOVTsXv4mK
++ return 0
+ local client_container=psmdb-client-bb8b97679-wjlk6
+ kubectl_bin exec psmdb-client-bb8b97679-wjlk6 -- bash -c 'printf '\''use myApp\n db.test2.insert({ x: 100501 })\n'\'' | mongo mongodb+srv://myApp:myPass@some-name-rs0.demand-backup-2582.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
++ mktemp
+ local LAST_OUT=/tmp/tmp.4S9Kgb2W4G
++ mktemp
+ local LAST_ERR=/tmp/tmp.0onTLd6TPG
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-bb8b97679-wjlk6 -- bash -c 'printf '\''use myApp\n db.test2.insert({ x: 100501 })\n'\'' | mongo mongodb+srv://myApp:myPass@some-name-rs0.demand-backup-2582.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.4S9Kgb2W4G
Percona Server for MongoDB shell version v4.4.29-28
connecting to: mongodb://some-name-rs0-2.some-name-rs0.demand-backup-2582.svc.cluster.local:27017,some-name-rs0-0.some-name-rs0.demand-backup-2582.svc.cluster.local:27017,some-name-rs0-1.some-name-rs0.demand-backup-2582.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb&replicaSet=rs0&ssl=false
Implicit session: session { "id" : UUID("31d7b1bf-0ab5-45f8-a6e9-5059d525e0e7") }
Percona Server for MongoDB server version: v8.0.19-7
WARNING: shell and server versions do not match
switched to db myApp
WriteResult({ "nInserted" : 1 })
bye
+ cat /tmp/tmp.0onTLd6TPG
+ rm /tmp/tmp.4S9Kgb2W4G /tmp/tmp.0onTLd6TPG
+ return 0
+ run_mongo 'use myApp\n db.test3.insert({ x: 100502 })' myApp:myPass@some-name-rs0.demand-backup-2582
+ local 'command=use myApp\n db.test3.insert({ x: 100502 })'
+ local uri=myApp:myPass@some-name-rs0.demand-backup-2582
+ local driver=mongodb+srv
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local replica_set=rs0
+ [[ myApp:myPass@some-name-rs0.demand-backup-2582 == *cfg* ]]
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.lEJInCRXua
+++ mktemp
++ local LAST_ERR=/tmp/tmp.qCh8b0pXSa
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.lEJInCRXua
++ cat /tmp/tmp.qCh8b0pXSa
++ rm /tmp/tmp.lEJInCRXua /tmp/tmp.qCh8b0pXSa
++ return 0
+ local client_container=psmdb-client-bb8b97679-wjlk6
+ kubectl_bin exec psmdb-client-bb8b97679-wjlk6 -- bash -c 'printf '\''use myApp\n db.test3.insert({ x: 100502 })\n'\'' | mongo mongodb+srv://myApp:myPass@some-name-rs0.demand-backup-2582.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
++ mktemp
+ local LAST_OUT=/tmp/tmp.FPWrixkP4w
++ mktemp
+ local LAST_ERR=/tmp/tmp.qP2iDp1cO6
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-bb8b97679-wjlk6 -- bash -c 'printf '\''use myApp\n db.test3.insert({ x: 100502 })\n'\'' | mongo mongodb+srv://myApp:myPass@some-name-rs0.demand-backup-2582.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.FPWrixkP4w
Percona Server for MongoDB shell version v4.4.29-28
connecting to: mongodb://some-name-rs0-1.some-name-rs0.demand-backup-2582.svc.cluster.local:27017,some-name-rs0-2.some-name-rs0.demand-backup-2582.svc.cluster.local:27017,some-name-rs0-0.some-name-rs0.demand-backup-2582.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb&replicaSet=rs0&ssl=false
Implicit session: session { "id" : UUID("0c2e51f0-7af1-45b8-9b96-b686414c14ba") }
Percona Server for MongoDB server version: v8.0.19-7
WARNING: shell and server versions do not match
switched to db myApp
WriteResult({ "nInserted" : 1 })
bye
+ cat /tmp/tmp.qP2iDp1cO6
+ rm /tmp/tmp.FPWrixkP4w /tmp/tmp.qP2iDp1cO6
+ return 0
+ custom_user_name=test1user
+ custom_role_name=test1role
+ run_mongo 'use myApp\n db.createUser({user: "test1user",pwd:"test1pass",roles:[]})' userAdmin:userAdmin123456@some-name-rs0.demand-backup-2582
+ local 'command=use myApp\n db.createUser({user: "test1user",pwd:"test1pass",roles:[]})'
+ local uri=userAdmin:userAdmin123456@some-name-rs0.demand-backup-2582
+ local driver=mongodb+srv
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local replica_set=rs0
+ [[ userAdmin:userAdmin123456@some-name-rs0.demand-backup-2582 == *cfg* ]]
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.NcHSltda90
+++ mktemp
++ local LAST_ERR=/tmp/tmp.jdvoS7JSA3
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.NcHSltda90
++ cat /tmp/tmp.jdvoS7JSA3
++ rm /tmp/tmp.NcHSltda90 /tmp/tmp.jdvoS7JSA3
++ return 0
+ local client_container=psmdb-client-bb8b97679-wjlk6
+ kubectl_bin exec psmdb-client-bb8b97679-wjlk6 -- bash -c 'printf '\''use myApp\n db.createUser({user: "test1user",pwd:"test1pass",roles:[]})\n'\'' | mongo mongodb+srv://userAdmin:userAdmin123456@some-name-rs0.demand-backup-2582.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
++ mktemp
+ local LAST_OUT=/tmp/tmp.pOwSGWtBrh
++ mktemp
+ local LAST_ERR=/tmp/tmp.JCmJvaIzQT
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-bb8b97679-wjlk6 -- bash -c 'printf '\''use myApp\n db.createUser({user: "test1user",pwd:"test1pass",roles:[]})\n'\'' | mongo mongodb+srv://userAdmin:userAdmin123456@some-name-rs0.demand-backup-2582.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.pOwSGWtBrh
Percona Server for MongoDB shell version v4.4.29-28
connecting to: mongodb://some-name-rs0-0.some-name-rs0.demand-backup-2582.svc.cluster.local:27017,some-name-rs0-1.some-name-rs0.demand-backup-2582.svc.cluster.local:27017,some-name-rs0-2.some-name-rs0.demand-backup-2582.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb&replicaSet=rs0&ssl=false
Implicit session: session { "id" : UUID("285177cb-6429-4c98-8ffa-14451228ac08") }
Percona Server for MongoDB server version: v8.0.19-7
WARNING: shell and server versions do not match
switched to db myApp
Successfully added user: { "user" : "test1user", "roles" : [ ] }
bye
+ cat /tmp/tmp.JCmJvaIzQT
+ rm /tmp/tmp.pOwSGWtBrh /tmp/tmp.JCmJvaIzQT
+ return 0
+ run_mongo 'use myApp\n db.createRole({"role":"test1role", privileges:[],roles:[]})' userAdmin:userAdmin123456@some-name-rs0.demand-backup-2582
+ local 'command=use myApp\n db.createRole({"role":"test1role", privileges:[],roles:[]})'
+ local uri=userAdmin:userAdmin123456@some-name-rs0.demand-backup-2582
+ local driver=mongodb+srv
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local replica_set=rs0
+ [[ userAdmin:userAdmin123456@some-name-rs0.demand-backup-2582 == *cfg* ]]
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.EUii1MF009
+++ mktemp
++ local LAST_ERR=/tmp/tmp.165GPcS5A3
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.EUii1MF009
++ cat /tmp/tmp.165GPcS5A3
++ rm /tmp/tmp.EUii1MF009 /tmp/tmp.165GPcS5A3
++ return 0
+ local client_container=psmdb-client-bb8b97679-wjlk6
+ kubectl_bin exec psmdb-client-bb8b97679-wjlk6 -- bash -c 'printf '\''use myApp\n db.createRole({"role":"test1role", privileges:[],roles:[]})\n'\'' | mongo mongodb+srv://userAdmin:userAdmin123456@some-name-rs0.demand-backup-2582.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
++ mktemp
+ local LAST_OUT=/tmp/tmp.wwYEpNuUv6
++ mktemp
+ local LAST_ERR=/tmp/tmp.D78UUIeBSs
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-bb8b97679-wjlk6 -- bash -c 'printf '\''use myApp\n db.createRole({"role":"test1role", privileges:[],roles:[]})\n'\'' | mongo mongodb+srv://userAdmin:userAdmin123456@some-name-rs0.demand-backup-2582.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.wwYEpNuUv6
Percona Server for MongoDB shell version v4.4.29-28
connecting to: mongodb://some-name-rs0-0.some-name-rs0.demand-backup-2582.svc.cluster.local:27017,some-name-rs0-1.some-name-rs0.demand-backup-2582.svc.cluster.local:27017,some-name-rs0-2.some-name-rs0.demand-backup-2582.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb&replicaSet=rs0&ssl=false
Implicit session: session { "id" : UUID("099d916b-c5ff-4b5f-8c89-d91c4c4d32b7") }
Percona Server for MongoDB server version: v8.0.19-7
WARNING: shell and server versions do not match
switched to db myApp
{ "role" : "test1role", "privileges" : [ ], "roles" : [ ] }
bye
+ cat /tmp/tmp.D78UUIeBSs
+ rm /tmp/tmp.wwYEpNuUv6 /tmp/tmp.D78UUIeBSs
+ return 0
+ minikube_sleep
+ sleep_time=10
+ [[ '' == 1 ]]
+ compare_mongo_cmd find myApp:myPass@some-name-rs0-0.some-name-rs0.demand-backup-2582
+ local command=find
+ local uri=myApp:myPass@some-name-rs0-0.some-name-rs0.demand-backup-2582
+ local postfix=
+ local suffix=
+ local database=myApp
+ local collection=test
+ local sort=
+ local tls=false
+ local replicaset=
+ local 'full_command=db.test.find()'
+ [[ -n '' ]]
+ log 'running db.test.find() in myApp'
+ set +o xtrace
[2026-03-26T08:15:23+0000] running db.test.find() in myApp
+ [[ false == \t\r\u\e ]]
+ mongo_command=run_mongo
+ run_mongo 'use myApp\n db.test.find()' myApp:myPass@some-name-rs0-0.some-name-rs0.demand-backup-2582 mongodb '' ''
+ local 'command=use myApp\n db.test.find()'
+ local uri=myApp:myPass@some-name-rs0-0.some-name-rs0.demand-backup-2582
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local replica_set=rs0
+ [[ myApp:myPass@some-name-rs0-0.some-name-rs0.demand-backup-2582 == *cfg* ]]
+ grep -E -v 'I NETWORK|W NETWORK|F NETWORK|"c":"NETWORK"|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|Started a new thread for the timer service'
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+ /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.xHKkv3c9M5
+++ mktemp
++ local LAST_ERR=/tmp/tmp.9Tg7sfegSa
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.xHKkv3c9M5
++ cat /tmp/tmp.9Tg7sfegSa
++ rm /tmp/tmp.xHKkv3c9M5 /tmp/tmp.9Tg7sfegSa
++ return 0
+ local client_container=psmdb-client-bb8b97679-wjlk6
+ kubectl_bin exec psmdb-client-bb8b97679-wjlk6 -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-0.some-name-rs0.demand-backup-2582.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
++ mktemp
+ local LAST_OUT=/tmp/tmp.0eXj37Rjxx
++ mktemp
+ local LAST_ERR=/tmp/tmp.dt2sUnRyoS
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-bb8b97679-wjlk6 -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-0.some-name-rs0.demand-backup-2582.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.0eXj37Rjxx
+ cat /tmp/tmp.dt2sUnRyoS
+ rm /tmp/tmp.0eXj37Rjxx /tmp/tmp.dt2sUnRyoS
+ return 0
+ [[ 0 -eq 0 ]]
+ diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2271/e2e-tests/demand-backup/compare/find.json /tmp/tmp.Fq0pMPZsh8/find
+ compare_mongo_cmd find myApp:myPass@some-name-rs0-1.some-name-rs0.demand-backup-2582
+ local command=find
+ local uri=myApp:myPass@some-name-rs0-1.some-name-rs0.demand-backup-2582
+ local postfix=
+ local suffix=
+ local database=myApp
+ local collection=test
+ local sort=
+ local tls=false
+ local replicaset=
+ local 'full_command=db.test.find()'
+ [[ -n '' ]]
+ log 'running db.test.find() in myApp'
+ set +o xtrace
[2026-03-26T08:15:26+0000] running db.test.find() in myApp
+ [[ false == \t\r\u\e ]]
+ mongo_command=run_mongo
+ run_mongo 'use myApp\n db.test.find()' myApp:myPass@some-name-rs0-1.some-name-rs0.demand-backup-2582 mongodb '' ''
+ local 'command=use myApp\n db.test.find()'
+ local uri=myApp:myPass@some-name-rs0-1.some-name-rs0.demand-backup-2582
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local replica_set=rs0
+ [[ myApp:myPass@some-name-rs0-1.some-name-rs0.demand-backup-2582 == *cfg* ]]
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+ grep -E -v 'I NETWORK|W NETWORK|F NETWORK|"c":"NETWORK"|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|Started a new thread for the timer service'
+ /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.eIU5h01l4V
+++ mktemp
++ local LAST_ERR=/tmp/tmp.ql1QBA1lwl
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.eIU5h01l4V
++ cat /tmp/tmp.ql1QBA1lwl
++ rm /tmp/tmp.eIU5h01l4V /tmp/tmp.ql1QBA1lwl
++ return 0
+ local client_container=psmdb-client-bb8b97679-wjlk6
+ kubectl_bin exec psmdb-client-bb8b97679-wjlk6 -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-1.some-name-rs0.demand-backup-2582.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
++ mktemp
+ local LAST_OUT=/tmp/tmp.yLUZJyE3gg
++ mktemp
+ local LAST_ERR=/tmp/tmp.eIZ80cwi2L
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-bb8b97679-wjlk6 -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-1.some-name-rs0.demand-backup-2582.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.yLUZJyE3gg
+ cat /tmp/tmp.eIZ80cwi2L
+ rm /tmp/tmp.yLUZJyE3gg /tmp/tmp.eIZ80cwi2L
+ return 0
+ [[ 0 -eq 0 ]]
+ diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2271/e2e-tests/demand-backup/compare/find.json /tmp/tmp.Fq0pMPZsh8/find
+ compare_mongo_cmd find myApp:myPass@some-name-rs0-2.some-name-rs0.demand-backup-2582
+ local command=find
+ local uri=myApp:myPass@some-name-rs0-2.some-name-rs0.demand-backup-2582
+ local postfix=
+ local suffix=
+ local database=myApp
+ local collection=test
+ local sort=
+ local tls=false
+ local replicaset=
+ local 'full_command=db.test.find()'
+ [[ -n '' ]]
+ log 'running db.test.find() in myApp'
+ set +o xtrace
[2026-03-26T08:15:28+0000] running db.test.find() in myApp
+ [[ false == \t\r\u\e ]]
+ mongo_command=run_mongo
+ run_mongo 'use myApp\n db.test.find()' myApp:myPass@some-name-rs0-2.some-name-rs0.demand-backup-2582 mongodb '' ''
+ local 'command=use myApp\n db.test.find()'
+ local uri=myApp:myPass@some-name-rs0-2.some-name-rs0.demand-backup-2582
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ grep -E -v 'I NETWORK|W NETWORK|F NETWORK|"c":"NETWORK"|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|Started a new thread for the timer service'
+ local mongo_flag=
+ local replica_set=rs0
+ [[ myApp:myPass@some-name-rs0-2.some-name-rs0.demand-backup-2582 == *cfg* ]]
+ /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.uwfGPmsJxg
+++ mktemp
++ local LAST_ERR=/tmp/tmp.Zo92jMrPhL
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.uwfGPmsJxg
++ cat /tmp/tmp.Zo92jMrPhL
++ rm /tmp/tmp.uwfGPmsJxg /tmp/tmp.Zo92jMrPhL
++ return 0
+ local client_container=psmdb-client-bb8b97679-wjlk6
+ kubectl_bin exec psmdb-client-bb8b97679-wjlk6 -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-2.some-name-rs0.demand-backup-2582.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
++ mktemp
+ local LAST_OUT=/tmp/tmp.xvzMIZROrx
++ mktemp
+ local LAST_ERR=/tmp/tmp.eFxv5pRC7L
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-bb8b97679-wjlk6 -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-2.some-name-rs0.demand-backup-2582.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.xvzMIZROrx
+ cat /tmp/tmp.eFxv5pRC7L
+ rm /tmp/tmp.xvzMIZROrx /tmp/tmp.eFxv5pRC7L
+ return 0
+ [[ 0 -eq 0 ]]
+ diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2271/e2e-tests/demand-backup/compare/find.json /tmp/tmp.Fq0pMPZsh8/find
+ wait_backup_agent some-name-rs0-0
+ local agent_pod=some-name-rs0-0
+ set +o xtrace
waiting for pbm-agent to be ready in some-name-rs0-0...2026-03-26T08:14:19.000+0000 I listening for the commands

+ wait_backup_agent some-name-rs0-1
+ local agent_pod=some-name-rs0-1
+ set +o xtrace
waiting for pbm-agent to be ready in some-name-rs0-1...2026-03-26T08:14:52.000+0000 I listening for the commands

+ wait_backup_agent some-name-rs0-2
+ local agent_pod=some-name-rs0-2
+ set +o xtrace
waiting for pbm-agent to be ready in some-name-rs0-2...2026-03-26T08:15:03.000+0000 I listening for the commands

+ '[' -z '' ']'
+ desc 'Check GCS profiles'
+ set +o xtrace


-----------------------------------------------------------------------------------
Check GCS profiles
-----------------------------------------------------------------------------------

+ compare_pbm_profile_setup some-name gcp-cs-s3
+ local cluster=some-name
+ local profile=gcp-cs-s3
+ local container=backup-agent
+ local pbm_binary=pbm
+ [[ backup-agent == \m\o\n\g\o\d ]]
+ kubectl_bin exec some-name-rs0-0 -c backup-agent -- pbm profile show gcp-cs-s3
++ mktemp
+ local LAST_OUT=/tmp/tmp.esghQg0KCn
++ mktemp
+ local LAST_ERR=/tmp/tmp.RAzbhrX2AM
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec some-name-rs0-0 -c backup-agent -- pbm profile show gcp-cs-s3
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.esghQg0KCn
+ cat /tmp/tmp.RAzbhrX2AM
+ rm /tmp/tmp.esghQg0KCn /tmp/tmp.RAzbhrX2AM
+ return 0
+ diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2271/e2e-tests/demand-backup/compare/pbm_profile_gcp-cs-s3.yml /tmp/tmp.Fq0pMPZsh8/pbm_profile_gcp-cs-s3.yml
+ compare_pbm_profile_setup some-name gcp-cs-sa
+ local cluster=some-name
+ local profile=gcp-cs-sa
+ local container=backup-agent
+ local pbm_binary=pbm
+ [[ backup-agent == \m\o\n\g\o\d ]]
+ kubectl_bin exec some-name-rs0-0 -c backup-agent -- pbm profile show gcp-cs-sa
++ mktemp
+ local LAST_OUT=/tmp/tmp.WGoNzMgyyA
++ mktemp
+ local LAST_ERR=/tmp/tmp.Zu4DUUf5Fd
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec some-name-rs0-0 -c backup-agent -- pbm profile show gcp-cs-sa
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.WGoNzMgyyA
+ cat /tmp/tmp.Zu4DUUf5Fd
+ rm /tmp/tmp.WGoNzMgyyA /tmp/tmp.Zu4DUUf5Fd
+ return 0
+ diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2271/e2e-tests/demand-backup/compare/pbm_profile_gcp-cs-sa.yml /tmp/tmp.Fq0pMPZsh8/pbm_profile_gcp-cs-sa.yml
+ backup_name_minio=backup-minio
+ '[' -z '' ']'
+ backup_name_aws=backup-aws-s3
+ backup_name_gcp_s3=backup-gcp-cs-s3
+ backup_name_gcp_sa=backup-gcp-cs-sa
+ backup_name_azure=backup-azure-blob
+ desc 'run backups'
+ set +o xtrace


-----------------------------------------------------------------------------------
run backups
-----------------------------------------------------------------------------------

+ '[' -z '' ']'
+ run_backup aws-s3
+ local storage=aws-s3
+ local backup_name=backup-aws-s3
+ local type=logical
+ log 'running backup backup-aws-s3'
+ set +o xtrace
[2026-03-26T08:15:38+0000] running backup backup-aws-s3
+ yq eval '.metadata.name = "backup-aws-s3"
			| .spec.storageName = "aws-s3"
			| .spec.type = "logical"' /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2271/e2e-tests/demand-backup/conf/backup-aws-s3.yml
+ kubectl_bin apply -f -
++ mktemp
+ local LAST_OUT=/tmp/tmp.1ySMXRqybK
++ mktemp
+ local LAST_ERR=/tmp/tmp.2mMYqidibY
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl apply -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.1ySMXRqybK
perconaservermongodbbackup.psmdb.percona.com/backup-aws-s3 created
+ cat /tmp/tmp.2mMYqidibY
+ rm /tmp/tmp.1ySMXRqybK /tmp/tmp.2mMYqidibY
+ return 0
+ run_backup gcp-cs-s3
+ local storage=gcp-cs-s3
+ local backup_name=backup-gcp-cs-s3
+ local type=logical
+ log 'running backup backup-gcp-cs-s3'
+ set +o xtrace
[2026-03-26T08:15:40+0000] running backup backup-gcp-cs-s3
+ yq eval '.metadata.name = "backup-gcp-cs-s3"
			| .spec.storageName = "gcp-cs-s3"
			| .spec.type = "logical"' /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2271/e2e-tests/demand-backup/conf/backup-gcp-cs-s3.yml
+ kubectl_bin apply -f -
++ mktemp
+ local LAST_OUT=/tmp/tmp.t3vbYH2AkR
++ mktemp
+ local LAST_ERR=/tmp/tmp.4HiX6kOEdN
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl apply -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.t3vbYH2AkR
perconaservermongodbbackup.psmdb.percona.com/backup-gcp-cs-s3 created
+ cat /tmp/tmp.4HiX6kOEdN
+ rm /tmp/tmp.t3vbYH2AkR /tmp/tmp.4HiX6kOEdN
+ return 0
+ run_backup gcp-cs-sa
+ local storage=gcp-cs-sa
+ local backup_name=backup-gcp-cs-sa
+ local type=logical
+ log 'running backup backup-gcp-cs-sa'
+ set +o xtrace
[2026-03-26T08:15:42+0000] running backup backup-gcp-cs-sa
+ yq eval '.metadata.name = "backup-gcp-cs-sa"
			| .spec.storageName = "gcp-cs-sa"
			| .spec.type = "logical"' /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2271/e2e-tests/demand-backup/conf/backup-gcp-cs-sa.yml
+ kubectl_bin apply -f -
++ mktemp
+ local LAST_OUT=/tmp/tmp.VTxNguQgeH
++ mktemp
+ local LAST_ERR=/tmp/tmp.wOKgYW3YE3
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl apply -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.VTxNguQgeH
perconaservermongodbbackup.psmdb.percona.com/backup-gcp-cs-sa created
+ cat /tmp/tmp.wOKgYW3YE3
+ rm /tmp/tmp.VTxNguQgeH /tmp/tmp.wOKgYW3YE3
+ return 0
+ run_backup azure-blob
+ local storage=azure-blob
+ local backup_name=backup-azure-blob
+ local type=logical
+ log 'running backup backup-azure-blob'
+ set +o xtrace
[2026-03-26T08:15:44+0000] running backup backup-azure-blob
+ yq eval '.metadata.name = "backup-azure-blob"
			| .spec.storageName = "azure-blob"
			| .spec.type = "logical"' /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2271/e2e-tests/demand-backup/conf/backup-azure-blob.yml
+ kubectl_bin apply -f -
++ mktemp
+ local LAST_OUT=/tmp/tmp.4OkY7rIBeB
++ mktemp
+ local LAST_ERR=/tmp/tmp.vVvWXPW8r1
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl apply -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.4OkY7rIBeB
perconaservermongodbbackup.psmdb.percona.com/backup-azure-blob created
+ cat /tmp/tmp.vVvWXPW8r1
+ rm /tmp/tmp.4OkY7rIBeB /tmp/tmp.vVvWXPW8r1
+ return 0
+ run_backup minio
+ local storage=minio
+ local backup_name=backup-minio
+ local type=logical
+ log 'running backup backup-minio'
+ set +o xtrace
[2026-03-26T08:15:46+0000] running backup backup-minio
+ yq eval '.metadata.name = "backup-minio"
			| .spec.storageName = "minio"
			| .spec.type = "logical"' /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2271/e2e-tests/demand-backup/conf/backup-minio.yml
+ kubectl_bin apply -f -
++ mktemp
+ local LAST_OUT=/tmp/tmp.xnyjCS9pCb
++ mktemp
+ local LAST_ERR=/tmp/tmp.cqbOL7FYpD
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl apply -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.xnyjCS9pCb
perconaservermongodbbackup.psmdb.percona.com/backup-minio created
+ cat /tmp/tmp.cqbOL7FYpD
+ rm /tmp/tmp.xnyjCS9pCb /tmp/tmp.cqbOL7FYpD
+ return 0
+ '[' -z '' ']'
+ wait_backup backup-aws-s3
+ local backup_name=backup-aws-s3
+ local target_state=ready
+ set +o xtrace
waiting for backup-aws-s3 to reach ready state...OK
+ wait_backup backup-gcp-cs-s3
+ local backup_name=backup-gcp-cs-s3
+ local target_state=ready
+ set +o xtrace
waiting for backup-gcp-cs-s3 to reach ready state............OK
+ wait_backup backup-gcp-cs-sa
+ local backup_name=backup-gcp-cs-sa
+ local target_state=ready
+ set +o xtrace
waiting for backup-gcp-cs-sa to reach ready state............OK
+ wait_backup backup-azure-blob
+ local backup_name=backup-azure-blob
+ local target_state=ready
+ set +o xtrace
waiting for backup-azure-blob to reach ready state..OK
+ wait_backup backup-minio
+ local backup_name=backup-minio
+ local target_state=ready
+ set +o xtrace
waiting for backup-minio to reach ready state..OK
+ sleep 5
+ '[' -z '' ']'
+ desc 'check backup and restore -- aws-s3'
+ set +o xtrace


-----------------------------------------------------------------------------------
check backup and restore -- aws-s3
-----------------------------------------------------------------------------------

++ get_backup_dest backup-aws-s3
++ local backup_name=backup-aws-s3
++ kubectl_bin get psmdb-backup backup-aws-s3 -o 'jsonpath={.status.destination}'
++ sed -e 's/.json$//'
++ sed 's|s3://||'
++ sed 's|azure://||'
+++ mktemp
++ sed 's|gs://||'
++ local LAST_OUT=/tmp/tmp.dTVPEnFmBz
+++ mktemp
++ local LAST_ERR=/tmp/tmp.UmmYQS5FFE
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb-backup backup-aws-s3 -o 'jsonpath={.status.destination}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.dTVPEnFmBz
++ cat /tmp/tmp.UmmYQS5FFE
++ rm /tmp/tmp.dTVPEnFmBz /tmp/tmp.UmmYQS5FFE
++ return 0
+ backup_dest_aws=operator-testing/psmdb-demand-backup/2026-03-26T08:15:41Z
+ setup_aws_credentials
+ local secret_name=aws-s3-secret
+ [[ -n AKIARXP3OARBMEQ6FSEW ]]
+ [[ -n gljUukL3/TuQnzrtJUgiURH6vqjt+SEo89jGS2Z/ ]]
+ echo 'AWS credentials already set in environment'
AWS credentials already set in environment
+ return 0
+ check_backup_existence_aws operator-testing/psmdb-demand-backup/2026-03-26T08:15:41Z /rs0/myApp.test.gz
++ echo operator-testing/psmdb-demand-backup/2026-03-26T08:15:41Z
++ cut -d/ -f1
+ bucket=operator-testing
++ echo operator-testing/psmdb-demand-backup/2026-03-26T08:15:41Z
++ cut -d/ -f2-
+ key_prefix=psmdb-demand-backup/2026-03-26T08:15:41Z
+ key=/rs0/myApp.test.gz
+ storage_name=aws-s3
+ retry=0
+ aws s3api head-object --bucket operator-testing --key psmdb-demand-backup/2026-03-26T08:15:41Z/rs0/myApp.test.gz
+ echo 'Backup operator-testing/psmdb-demand-backup/2026-03-26T08:15:41Z//rs0/myApp.test.gz found in aws-s3'
Backup operator-testing/psmdb-demand-backup/2026-03-26T08:15:41Z//rs0/myApp.test.gz found in aws-s3
+ run_recovery_check backup-aws-s3 some-name-rs0
+ local backup=backup-aws-s3
+ local cluster=some-name-rs0
+ run_mongo 'use myApp\n db.test.insert({ x: 100501 })' myApp:myPass@some-name-rs0.demand-backup-2582
+ local 'command=use myApp\n db.test.insert({ x: 100501 })'
+ local uri=myApp:myPass@some-name-rs0.demand-backup-2582
+ local driver=mongodb+srv
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local replica_set=rs0
+ [[ myApp:myPass@some-name-rs0.demand-backup-2582 == *cfg* ]]
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.nITxZgia15
+++ mktemp
++ local LAST_ERR=/tmp/tmp.NUaLPTzgor
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.nITxZgia15
++ cat /tmp/tmp.NUaLPTzgor
++ rm /tmp/tmp.nITxZgia15 /tmp/tmp.NUaLPTzgor
++ return 0
+ local client_container=psmdb-client-bb8b97679-wjlk6
+ kubectl_bin exec psmdb-client-bb8b97679-wjlk6 -- bash -c 'printf '\''use myApp\n db.test.insert({ x: 100501 })\n'\'' | mongo mongodb+srv://myApp:myPass@some-name-rs0.demand-backup-2582.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
++ mktemp
+ local LAST_OUT=/tmp/tmp.N4mDW4KBYI
++ mktemp
+ local LAST_ERR=/tmp/tmp.HbR3jzgy05
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-bb8b97679-wjlk6 -- bash -c 'printf '\''use myApp\n db.test.insert({ x: 100501 })\n'\'' | mongo mongodb+srv://myApp:myPass@some-name-rs0.demand-backup-2582.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.N4mDW4KBYI
Percona Server for MongoDB shell version v4.4.29-28
connecting to: mongodb://some-name-rs0-0.some-name-rs0.demand-backup-2582.svc.cluster.local:27017,some-name-rs0-1.some-name-rs0.demand-backup-2582.svc.cluster.local:27017,some-name-rs0-2.some-name-rs0.demand-backup-2582.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb&replicaSet=rs0&ssl=false
Implicit session: session { "id" : UUID("4ae27c85-f455-417d-91f0-8bc4e963a1bb") }
Percona Server for MongoDB server version: v8.0.19-7
WARNING: shell and server versions do not match
switched to db myApp
WriteResult({ "nInserted" : 1 })
bye
+ cat /tmp/tmp.HbR3jzgy05
+ rm /tmp/tmp.N4mDW4KBYI /tmp/tmp.HbR3jzgy05
+ return 0
+ compare_mongo_cmd find myApp:myPass@some-name-rs0.demand-backup-2582 -2nd .svc.cluster.local myApp test
+ local command=find
+ local uri=myApp:myPass@some-name-rs0.demand-backup-2582
+ local postfix=-2nd
+ local suffix=.svc.cluster.local
+ local database=myApp
+ local collection=test
+ local sort=
+ local tls=false
+ local replicaset=
+ local 'full_command=db.test.find()'
+ [[ -n '' ]]
+ log 'running db.test.find() in myApp'
+ set +o xtrace
[2026-03-26T08:16:51+0000] running db.test.find() in myApp
+ [[ false == \t\r\u\e ]]
+ mongo_command=run_mongo
+ run_mongo 'use myApp\n db.test.find()' myApp:myPass@some-name-rs0.demand-backup-2582 mongodb .svc.cluster.local ''
+ local 'command=use myApp\n db.test.find()'
+ local uri=myApp:myPass@some-name-rs0.demand-backup-2582
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local replica_set=rs0
+ [[ myApp:myPass@some-name-rs0.demand-backup-2582 == *cfg* ]]
+ grep -E -v 'I NETWORK|W NETWORK|F NETWORK|"c":"NETWORK"|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|Started a new thread for the timer service'
+ /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.g9YtiIquBn
+++ mktemp
++ local LAST_ERR=/tmp/tmp.MpwlNQ4JHo
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.g9YtiIquBn
++ cat /tmp/tmp.MpwlNQ4JHo
++ rm /tmp/tmp.g9YtiIquBn /tmp/tmp.MpwlNQ4JHo
++ return 0
+ local client_container=psmdb-client-bb8b97679-wjlk6
+ kubectl_bin exec psmdb-client-bb8b97679-wjlk6 -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0.demand-backup-2582.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
++ mktemp
+ local LAST_OUT=/tmp/tmp.Pn7rYGn9aM
++ mktemp
+ local LAST_ERR=/tmp/tmp.KWgxh0e0dL
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-bb8b97679-wjlk6 -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0.demand-backup-2582.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.Pn7rYGn9aM
+ cat /tmp/tmp.KWgxh0e0dL
+ rm /tmp/tmp.Pn7rYGn9aM /tmp/tmp.KWgxh0e0dL
+ return 0
+ [[ 0 -eq 0 ]]
+ diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2271/e2e-tests/demand-backup/compare/find-2nd.json /tmp/tmp.Fq0pMPZsh8/find-2nd
+ run_restore backup-aws-s3
+ local backup_name=backup-aws-s3
+ local restore_file=/mnt/jenkins/workspace/cloud-psmdb-operator_PR-2271/e2e-tests/demand-backup/conf/restore.yml
+ log 'running restore restore-backup-aws-s3'
+ set +o xtrace
[2026-03-26T08:16:53+0000] running restore restore-backup-aws-s3
+ cat /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2271/e2e-tests/demand-backup/conf/restore.yml
+ /usr/sbin/sed -e 's/name:/name: restore-backup-aws-s3/'
+ /usr/sbin/sed -e 's/backupName:/backupName: backup-aws-s3/'
+ kubectl_bin apply -f -
++ mktemp
+ local LAST_OUT=/tmp/tmp.Ejn7qB3Z0u
++ mktemp
+ local LAST_ERR=/tmp/tmp.5HFDvCzwL8
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl apply -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.Ejn7qB3Z0u
perconaservermongodbrestore.psmdb.percona.com/restore-backup-aws-s3 created
+ cat /tmp/tmp.5HFDvCzwL8
+ rm /tmp/tmp.Ejn7qB3Z0u /tmp/tmp.5HFDvCzwL8
+ return 0
+ wait_restore backup-aws-s3 some-name
+ local backup_name=backup-aws-s3
+ local cluster_name=some-name
+ local target_state=ready
+ local wait_cluster_consistency=1
+ local wait_time=1780
+ local ok_if_ready=0
+ set +o xtrace
Waiting for the psmdb-restore/restore-backup-aws-s3 object to be created.OK
Waiting psmdb-restore/restore-backup-aws-s3 to reach state "ready" .OK after 0 minutes
+ [[ 1 -eq 1 ]]
+ wait_cluster_consistency some-name
+ local cluster_name=some-name
+ local wait_time=32
+ retry=0
+ sleep 7
+ echo -n 'waiting for cluster readyness'
waiting for cluster readyness++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.HvHoCUd3A2
+++ mktemp
++ local LAST_ERR=/tmp/tmp.iFFfKv2erx
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.HvHoCUd3A2
++ cat /tmp/tmp.iFFfKv2erx
++ rm /tmp/tmp.HvHoCUd3A2 /tmp/tmp.iFFfKv2erx
++ return 0
+ [[ ready == \r\e\a\d\y ]]
+ echo .OK
.OK
+ simple_data_check some-name-rs0 3 0
+ local cluster_name=some-name-rs0
+ let last_pod=3-1
+ local isSharded=0
+ local cluster_pfx=
+ '[' 0 -eq 1 ']'
++ seq 0 2
+ for i in $(seq 0 $last_pod)
+ compare_mongo_cmd find myApp:myPass@some-name-rs0-0.some-name-rs0.demand-backup-2582
+ local command=find
+ local uri=myApp:myPass@some-name-rs0-0.some-name-rs0.demand-backup-2582
+ local postfix=
+ local suffix=
+ local database=myApp
+ local collection=test
+ local sort=
+ local tls=false
+ local replicaset=
+ local 'full_command=db.test.find()'
+ [[ -n '' ]]
+ log 'running db.test.find() in myApp'
+ set +o xtrace
[2026-03-26T08:17:16+0000] running db.test.find() in myApp
+ [[ false == \t\r\u\e ]]
+ mongo_command=run_mongo
+ run_mongo 'use myApp\n db.test.find()' myApp:myPass@some-name-rs0-0.some-name-rs0.demand-backup-2582 mongodb '' ''
+ local 'command=use myApp\n db.test.find()'
+ local uri=myApp:myPass@some-name-rs0-0.some-name-rs0.demand-backup-2582
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local replica_set=rs0
+ [[ myApp:myPass@some-name-rs0-0.some-name-rs0.demand-backup-2582 == *cfg* ]]
+ grep -E -v 'I NETWORK|W NETWORK|F NETWORK|"c":"NETWORK"|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|Started a new thread for the timer service'
+ /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.08MwdJFF84
+++ mktemp
++ local LAST_ERR=/tmp/tmp.XOeEzkzk8m
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.08MwdJFF84
++ cat /tmp/tmp.XOeEzkzk8m
++ rm /tmp/tmp.08MwdJFF84 /tmp/tmp.XOeEzkzk8m
++ return 0
+ local client_container=psmdb-client-bb8b97679-wjlk6
+ kubectl_bin exec psmdb-client-bb8b97679-wjlk6 -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-0.some-name-rs0.demand-backup-2582.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
++ mktemp
+ local LAST_OUT=/tmp/tmp.bSn5pP0WKJ
++ mktemp
+ local LAST_ERR=/tmp/tmp.97p0QYre2R
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-bb8b97679-wjlk6 -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-0.some-name-rs0.demand-backup-2582.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.bSn5pP0WKJ
+ cat /tmp/tmp.97p0QYre2R
+ rm /tmp/tmp.bSn5pP0WKJ /tmp/tmp.97p0QYre2R
+ return 0
+ [[ 0 -eq 0 ]]
+ diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2271/e2e-tests/demand-backup/compare/find.json /tmp/tmp.Fq0pMPZsh8/find
+ for i in $(seq 0 $last_pod)
+ compare_mongo_cmd find myApp:myPass@some-name-rs0-1.some-name-rs0.demand-backup-2582
+ local command=find
+ local uri=myApp:myPass@some-name-rs0-1.some-name-rs0.demand-backup-2582
+ local postfix=
+ local suffix=
+ local database=myApp
+ local collection=test
+ local sort=
+ local tls=false
+ local replicaset=
+ local 'full_command=db.test.find()'
+ [[ -n '' ]]
+ log 'running db.test.find() in myApp'
+ set +o xtrace
[2026-03-26T08:17:19+0000] running db.test.find() in myApp
+ [[ false == \t\r\u\e ]]
+ mongo_command=run_mongo
+ run_mongo 'use myApp\n db.test.find()' myApp:myPass@some-name-rs0-1.some-name-rs0.demand-backup-2582 mongodb '' ''
+ local 'command=use myApp\n db.test.find()'
+ local uri=myApp:myPass@some-name-rs0-1.some-name-rs0.demand-backup-2582
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local replica_set=rs0
+ [[ myApp:myPass@some-name-rs0-1.some-name-rs0.demand-backup-2582 == *cfg* ]]
+ grep -E -v 'I NETWORK|W NETWORK|F NETWORK|"c":"NETWORK"|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|Started a new thread for the timer service'
+ /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.6preda6vts
+++ mktemp
++ local LAST_ERR=/tmp/tmp.3hpV18SMZI
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.6preda6vts
++ cat /tmp/tmp.3hpV18SMZI
++ rm /tmp/tmp.6preda6vts /tmp/tmp.3hpV18SMZI
++ return 0
+ local client_container=psmdb-client-bb8b97679-wjlk6
+ kubectl_bin exec psmdb-client-bb8b97679-wjlk6 -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-1.some-name-rs0.demand-backup-2582.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
++ mktemp
+ local LAST_OUT=/tmp/tmp.S5BYntCG8r
++ mktemp
+ local LAST_ERR=/tmp/tmp.xg50U26iPm
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-bb8b97679-wjlk6 -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-1.some-name-rs0.demand-backup-2582.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.S5BYntCG8r
+ cat /tmp/tmp.xg50U26iPm
+ rm /tmp/tmp.S5BYntCG8r /tmp/tmp.xg50U26iPm
+ return 0
+ [[ 0 -eq 0 ]]
+ diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2271/e2e-tests/demand-backup/compare/find.json /tmp/tmp.Fq0pMPZsh8/find
+ for i in $(seq 0 $last_pod)
+ compare_mongo_cmd find myApp:myPass@some-name-rs0-2.some-name-rs0.demand-backup-2582
+ local command=find
+ local uri=myApp:myPass@some-name-rs0-2.some-name-rs0.demand-backup-2582
+ local postfix=
+ local suffix=
+ local database=myApp
+ local collection=test
+ local sort=
+ local tls=false
+ local replicaset=
+ local 'full_command=db.test.find()'
+ [[ -n '' ]]
+ log 'running db.test.find() in myApp'
+ set +o xtrace
[2026-03-26T08:17:21+0000] running db.test.find() in myApp
+ [[ false == \t\r\u\e ]]
+ mongo_command=run_mongo
+ run_mongo 'use myApp\n db.test.find()' myApp:myPass@some-name-rs0-2.some-name-rs0.demand-backup-2582 mongodb '' ''
+ local 'command=use myApp\n db.test.find()'
+ local uri=myApp:myPass@some-name-rs0-2.some-name-rs0.demand-backup-2582
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local replica_set=rs0
+ [[ myApp:myPass@some-name-rs0-2.some-name-rs0.demand-backup-2582 == *cfg* ]]
+ grep -E -v 'I NETWORK|W NETWORK|F NETWORK|"c":"NETWORK"|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|Started a new thread for the timer service'
+ /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.1EafVKLqbV
+++ mktemp
++ local LAST_ERR=/tmp/tmp.eoTdNp9RL3
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.1EafVKLqbV
++ cat /tmp/tmp.eoTdNp9RL3
++ rm /tmp/tmp.1EafVKLqbV /tmp/tmp.eoTdNp9RL3
++ return 0
+ local client_container=psmdb-client-bb8b97679-wjlk6
+ kubectl_bin exec psmdb-client-bb8b97679-wjlk6 -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-2.some-name-rs0.demand-backup-2582.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
++ mktemp
+ local LAST_OUT=/tmp/tmp.a8rsSi58G8
++ mktemp
+ local LAST_ERR=/tmp/tmp.mvUB0A9ozP
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-bb8b97679-wjlk6 -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-2.some-name-rs0.demand-backup-2582.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.a8rsSi58G8
+ cat /tmp/tmp.mvUB0A9ozP
+ rm /tmp/tmp.a8rsSi58G8 /tmp/tmp.mvUB0A9ozP
+ return 0
+ [[ 0 -eq 0 ]]
+ diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2271/e2e-tests/demand-backup/compare/find.json /tmp/tmp.Fq0pMPZsh8/find
+ desc 'check backup and restore -- gcp-cs-s3'
+ set +o xtrace


-----------------------------------------------------------------------------------
check backup and restore -- gcp-cs-s3
-----------------------------------------------------------------------------------

++ get_backup_dest backup-gcp-cs-s3
++ local backup_name=backup-gcp-cs-s3
++ kubectl_bin get psmdb-backup backup-gcp-cs-s3 -o 'jsonpath={.status.destination}'
++ sed -e 's/.json$//'
++ sed 's|s3://||'
+++ mktemp
++ sed 's|azure://||'
++ sed 's|gs://||'
++ local LAST_OUT=/tmp/tmp.VZ7DT1avPE
+++ mktemp
++ local LAST_ERR=/tmp/tmp.sYzEcHNDP0
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb-backup backup-gcp-cs-s3 -o 'jsonpath={.status.destination}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.VZ7DT1avPE
++ cat /tmp/tmp.sYzEcHNDP0
++ rm /tmp/tmp.VZ7DT1avPE /tmp/tmp.sYzEcHNDP0
++ return 0
+ backup_dest_gcp_s3=operator-testing/psmdb-demand-backup/2026-03-26T08:16:03Z
+ setup_gcs_credentials
+ local secret_name=gcp-cs-secret
+ gsutil ls
+ echo 'Setting up GCS credentials from K8s secret: gcp-cs-secret'
Setting up GCS credentials from K8s secret: gcp-cs-secret
+ local trace_was_on=0
+ [[ ehxB == *x* ]]
+ trace_was_on=1
+ set +x
+ echo 'GCS credentials configured successfully'
GCS credentials configured successfully
+ check_backup_existence_gcs operator-testing/psmdb-demand-backup/2026-03-26T08:16:03Z /rs0/myApp.test.gz
+ backup_dest_gcp=operator-testing/psmdb-demand-backup/2026-03-26T08:16:03Z
+ obj=/rs0/myApp.test.gz
+ storage_name=gcp-cs
+ retry=0
+ gcs_path=gs://operator-testing/psmdb-demand-backup/2026-03-26T08:16:03Z/rs0/myApp.test.gz
+ gsutil ls gs://operator-testing/psmdb-demand-backup/2026-03-26T08:16:03Z/rs0/myApp.test.gz
+ echo 'Backup gs://operator-testing/psmdb-demand-backup/2026-03-26T08:16:03Z/rs0/myApp.test.gz found in gcp-cs'
Backup gs://operator-testing/psmdb-demand-backup/2026-03-26T08:16:03Z/rs0/myApp.test.gz found in gcp-cs
+ run_recovery_check backup-gcp-cs-s3 some-name-rs0
+ local backup=backup-gcp-cs-s3
+ local cluster=some-name-rs0
+ run_mongo 'use myApp\n db.test.insert({ x: 100501 })' myApp:myPass@some-name-rs0.demand-backup-2582
+ local 'command=use myApp\n db.test.insert({ x: 100501 })'
+ local uri=myApp:myPass@some-name-rs0.demand-backup-2582
+ local driver=mongodb+srv
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local replica_set=rs0
+ [[ myApp:myPass@some-name-rs0.demand-backup-2582 == *cfg* ]]
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.p8NjXRI3ES
+++ mktemp
++ local LAST_ERR=/tmp/tmp.qHQ0JiUhO3
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.p8NjXRI3ES
++ cat /tmp/tmp.qHQ0JiUhO3
++ rm /tmp/tmp.p8NjXRI3ES /tmp/tmp.qHQ0JiUhO3
++ return 0
+ local client_container=psmdb-client-bb8b97679-wjlk6
+ kubectl_bin exec psmdb-client-bb8b97679-wjlk6 -- bash -c 'printf '\''use myApp\n db.test.insert({ x: 100501 })\n'\'' | mongo mongodb+srv://myApp:myPass@some-name-rs0.demand-backup-2582.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
++ mktemp
+ local LAST_OUT=/tmp/tmp.8JRftpf8V5
++ mktemp
+ local LAST_ERR=/tmp/tmp.b5NO8qkVYJ
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-bb8b97679-wjlk6 -- bash -c 'printf '\''use myApp\n db.test.insert({ x: 100501 })\n'\'' | mongo mongodb+srv://myApp:myPass@some-name-rs0.demand-backup-2582.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.8JRftpf8V5
Percona Server for MongoDB shell version v4.4.29-28
connecting to: mongodb://some-name-rs0-2.some-name-rs0.demand-backup-2582.svc.cluster.local:27017,some-name-rs0-0.some-name-rs0.demand-backup-2582.svc.cluster.local:27017,some-name-rs0-1.some-name-rs0.demand-backup-2582.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb&replicaSet=rs0&ssl=false
Implicit session: session { "id" : UUID("fdf9e798-bb2b-4252-aec3-572a07922d3a") }
Percona Server for MongoDB server version: v8.0.19-7
WARNING: shell and server versions do not match
switched to db myApp
WriteResult({ "nInserted" : 1 })
bye
+ cat /tmp/tmp.b5NO8qkVYJ
+ rm /tmp/tmp.8JRftpf8V5 /tmp/tmp.b5NO8qkVYJ
+ return 0
+ compare_mongo_cmd find myApp:myPass@some-name-rs0.demand-backup-2582 -2nd .svc.cluster.local myApp test
+ local command=find
+ local uri=myApp:myPass@some-name-rs0.demand-backup-2582
+ local postfix=-2nd
+ local suffix=.svc.cluster.local
+ local database=myApp
+ local collection=test
+ local sort=
+ local tls=false
+ local replicaset=
+ local 'full_command=db.test.find()'
+ [[ -n '' ]]
+ log 'running db.test.find() in myApp'
+ set +o xtrace
[2026-03-26T08:17:32+0000] running db.test.find() in myApp
+ [[ false == \t\r\u\e ]]
+ mongo_command=run_mongo
+ grep -E -v 'I NETWORK|W NETWORK|F NETWORK|"c":"NETWORK"|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|Started a new thread for the timer service'
+ run_mongo 'use myApp\n db.test.find()' myApp:myPass@some-name-rs0.demand-backup-2582 mongodb .svc.cluster.local ''
+ /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
+ local 'command=use myApp\n db.test.find()'
+ local uri=myApp:myPass@some-name-rs0.demand-backup-2582
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local replica_set=rs0
+ [[ myApp:myPass@some-name-rs0.demand-backup-2582 == *cfg* ]]
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.crHCvqo9Oi
+++ mktemp
++ local LAST_ERR=/tmp/tmp.o6yzBIW1wT
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.crHCvqo9Oi
++ cat /tmp/tmp.o6yzBIW1wT
++ rm /tmp/tmp.crHCvqo9Oi /tmp/tmp.o6yzBIW1wT
++ return 0
+ local client_container=psmdb-client-bb8b97679-wjlk6
+ kubectl_bin exec psmdb-client-bb8b97679-wjlk6 -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0.demand-backup-2582.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
++ mktemp
+ local LAST_OUT=/tmp/tmp.YgEoiz9NXq
++ mktemp
+ local LAST_ERR=/tmp/tmp.OZVsnIFFCk
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-bb8b97679-wjlk6 -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0.demand-backup-2582.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.YgEoiz9NXq
+ cat /tmp/tmp.OZVsnIFFCk
+ rm /tmp/tmp.YgEoiz9NXq /tmp/tmp.OZVsnIFFCk
+ return 0
+ [[ 0 -eq 0 ]]
+ diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2271/e2e-tests/demand-backup/compare/find-2nd.json /tmp/tmp.Fq0pMPZsh8/find-2nd
+ run_restore backup-gcp-cs-s3
+ local backup_name=backup-gcp-cs-s3
+ local restore_file=/mnt/jenkins/workspace/cloud-psmdb-operator_PR-2271/e2e-tests/demand-backup/conf/restore.yml
+ log 'running restore restore-backup-gcp-cs-s3'
+ set +o xtrace
[2026-03-26T08:17:34+0000] running restore restore-backup-gcp-cs-s3
+ cat /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2271/e2e-tests/demand-backup/conf/restore.yml
+ /usr/sbin/sed -e 's/name:/name: restore-backup-gcp-cs-s3/'
+ /usr/sbin/sed -e 's/backupName:/backupName: backup-gcp-cs-s3/'
+ kubectl_bin apply -f -
++ mktemp
+ local LAST_OUT=/tmp/tmp.jaEOHYdNdj
++ mktemp
+ local LAST_ERR=/tmp/tmp.mbDdKy2kmE
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl apply -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.jaEOHYdNdj
perconaservermongodbrestore.psmdb.percona.com/restore-backup-gcp-cs-s3 created
+ cat /tmp/tmp.mbDdKy2kmE
+ rm /tmp/tmp.jaEOHYdNdj /tmp/tmp.mbDdKy2kmE
+ return 0
+ wait_restore backup-gcp-cs-s3 some-name
+ local backup_name=backup-gcp-cs-s3
+ local cluster_name=some-name
+ local target_state=ready
+ local wait_cluster_consistency=1
+ local wait_time=1780
+ local ok_if_ready=0
+ set +o xtrace
Waiting for the psmdb-restore/restore-backup-gcp-cs-s3 object to be created.OK
Waiting psmdb-restore/restore-backup-gcp-cs-s3 to reach state "ready" .OK after 0 minutes
+ [[ 1 -eq 1 ]]
+ wait_cluster_consistency some-name
+ local cluster_name=some-name
+ local wait_time=32
+ retry=0
+ sleep 7
+ echo -n 'waiting for cluster readyness'
waiting for cluster readyness++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.PogKeu9oBu
+++ mktemp
++ local LAST_ERR=/tmp/tmp.8KN5pDJC1g
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.PogKeu9oBu
++ cat /tmp/tmp.8KN5pDJC1g
++ rm /tmp/tmp.PogKeu9oBu /tmp/tmp.8KN5pDJC1g
++ return 0
+ [[ ready == \r\e\a\d\y ]]
+ echo .OK
.OK
+ simple_data_check some-name-rs0 3 0
+ local cluster_name=some-name-rs0
+ let last_pod=3-1
+ local isSharded=0
+ local cluster_pfx=
+ '[' 0 -eq 1 ']'
++ seq 0 2
+ for i in $(seq 0 $last_pod)
+ compare_mongo_cmd find myApp:myPass@some-name-rs0-0.some-name-rs0.demand-backup-2582
+ local command=find
+ local uri=myApp:myPass@some-name-rs0-0.some-name-rs0.demand-backup-2582
+ local postfix=
+ local suffix=
+ local database=myApp
+ local collection=test
+ local sort=
+ local tls=false
+ local replicaset=
+ local 'full_command=db.test.find()'
+ [[ -n '' ]]
+ log 'running db.test.find() in myApp'
+ set +o xtrace
[2026-03-26T08:17:57+0000] running db.test.find() in myApp
+ [[ false == \t\r\u\e ]]
+ mongo_command=run_mongo
+ run_mongo 'use myApp\n db.test.find()' myApp:myPass@some-name-rs0-0.some-name-rs0.demand-backup-2582 mongodb '' ''
+ grep -E -v 'I NETWORK|W NETWORK|F NETWORK|"c":"NETWORK"|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|Started a new thread for the timer service'
+ local 'command=use myApp\n db.test.find()'
+ local uri=myApp:myPass@some-name-rs0-0.some-name-rs0.demand-backup-2582
+ local driver=mongodb
+ /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local replica_set=rs0
+ [[ myApp:myPass@some-name-rs0-0.some-name-rs0.demand-backup-2582 == *cfg* ]]
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.lu4VRetkVq
+++ mktemp
++ local LAST_ERR=/tmp/tmp.ekrAMmDGtg
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.lu4VRetkVq
++ cat /tmp/tmp.ekrAMmDGtg
++ rm /tmp/tmp.lu4VRetkVq /tmp/tmp.ekrAMmDGtg
++ return 0
+ local client_container=psmdb-client-bb8b97679-wjlk6
+ kubectl_bin exec psmdb-client-bb8b97679-wjlk6 -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-0.some-name-rs0.demand-backup-2582.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
++ mktemp
+ local LAST_OUT=/tmp/tmp.m8peveP31w
++ mktemp
+ local LAST_ERR=/tmp/tmp.TrLqL02Ehi
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-bb8b97679-wjlk6 -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-0.some-name-rs0.demand-backup-2582.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.m8peveP31w
+ cat /tmp/tmp.TrLqL02Ehi
+ rm /tmp/tmp.m8peveP31w /tmp/tmp.TrLqL02Ehi
+ return 0
+ [[ 0 -eq 0 ]]
+ diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2271/e2e-tests/demand-backup/compare/find.json /tmp/tmp.Fq0pMPZsh8/find
+ for i in $(seq 0 $last_pod)
+ compare_mongo_cmd find myApp:myPass@some-name-rs0-1.some-name-rs0.demand-backup-2582
+ local command=find
+ local uri=myApp:myPass@some-name-rs0-1.some-name-rs0.demand-backup-2582
+ local postfix=
+ local suffix=
+ local database=myApp
+ local collection=test
+ local sort=
+ local tls=false
+ local replicaset=
+ local 'full_command=db.test.find()'
+ [[ -n '' ]]
+ log 'running db.test.find() in myApp'
+ set +o xtrace
[2026-03-26T08:17:59+0000] running db.test.find() in myApp
+ [[ false == \t\r\u\e ]]
+ mongo_command=run_mongo
+ run_mongo 'use myApp\n db.test.find()' myApp:myPass@some-name-rs0-1.some-name-rs0.demand-backup-2582 mongodb '' ''
+ local 'command=use myApp\n db.test.find()'
+ local uri=myApp:myPass@some-name-rs0-1.some-name-rs0.demand-backup-2582
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local replica_set=rs0
+ [[ myApp:myPass@some-name-rs0-1.some-name-rs0.demand-backup-2582 == *cfg* ]]
+ grep -E -v 'I NETWORK|W NETWORK|F NETWORK|"c":"NETWORK"|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|Started a new thread for the timer service'
+ /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.36CE18NPIE
+++ mktemp
++ local LAST_ERR=/tmp/tmp.5jc6Xig2qy
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.36CE18NPIE
++ cat /tmp/tmp.5jc6Xig2qy
++ rm /tmp/tmp.36CE18NPIE /tmp/tmp.5jc6Xig2qy
++ return 0
+ local client_container=psmdb-client-bb8b97679-wjlk6
+ kubectl_bin exec psmdb-client-bb8b97679-wjlk6 -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-1.some-name-rs0.demand-backup-2582.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
++ mktemp
+ local LAST_OUT=/tmp/tmp.DqedIB340e
++ mktemp
+ local LAST_ERR=/tmp/tmp.ls3tJ2MXTZ
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-bb8b97679-wjlk6 -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-1.some-name-rs0.demand-backup-2582.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.DqedIB340e
+ cat /tmp/tmp.ls3tJ2MXTZ
+ rm /tmp/tmp.DqedIB340e /tmp/tmp.ls3tJ2MXTZ
+ return 0
+ [[ 0 -eq 0 ]]
+ diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2271/e2e-tests/demand-backup/compare/find.json /tmp/tmp.Fq0pMPZsh8/find
+ for i in $(seq 0 $last_pod)
+ compare_mongo_cmd find myApp:myPass@some-name-rs0-2.some-name-rs0.demand-backup-2582
+ local command=find
+ local uri=myApp:myPass@some-name-rs0-2.some-name-rs0.demand-backup-2582
+ local postfix=
+ local suffix=
+ local database=myApp
+ local collection=test
+ local sort=
+ local tls=false
+ local replicaset=
+ local 'full_command=db.test.find()'
+ [[ -n '' ]]
+ log 'running db.test.find() in myApp'
+ set +o xtrace
[2026-03-26T08:18:02+0000] running db.test.find() in myApp
+ [[ false == \t\r\u\e ]]
+ mongo_command=run_mongo
+ run_mongo 'use myApp\n db.test.find()' myApp:myPass@some-name-rs0-2.some-name-rs0.demand-backup-2582 mongodb '' ''
+ local 'command=use myApp\n db.test.find()'
+ local uri=myApp:myPass@some-name-rs0-2.some-name-rs0.demand-backup-2582
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local replica_set=rs0
+ [[ myApp:myPass@some-name-rs0-2.some-name-rs0.demand-backup-2582 == *cfg* ]]
+ grep -E -v 'I NETWORK|W NETWORK|F NETWORK|"c":"NETWORK"|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|Started a new thread for the timer service'
+ /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.xPmqvUrNe5
+++ mktemp
++ local LAST_ERR=/tmp/tmp.u7wbzIH7DN
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.xPmqvUrNe5
++ cat /tmp/tmp.u7wbzIH7DN
++ rm /tmp/tmp.xPmqvUrNe5 /tmp/tmp.u7wbzIH7DN
++ return 0
+ local client_container=psmdb-client-bb8b97679-wjlk6
+ kubectl_bin exec psmdb-client-bb8b97679-wjlk6 -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-2.some-name-rs0.demand-backup-2582.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
++ mktemp
+ local LAST_OUT=/tmp/tmp.czsVDKKw6n
++ mktemp
+ local LAST_ERR=/tmp/tmp.vZFG6s3BHT
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-bb8b97679-wjlk6 -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-2.some-name-rs0.demand-backup-2582.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.czsVDKKw6n
+ cat /tmp/tmp.vZFG6s3BHT
+ rm /tmp/tmp.czsVDKKw6n /tmp/tmp.vZFG6s3BHT
+ return 0
+ [[ 0 -eq 0 ]]
+ diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2271/e2e-tests/demand-backup/compare/find.json /tmp/tmp.Fq0pMPZsh8/find
+ desc 'check backup and restore -- gcp-cs-sa'
+ set +o xtrace


-----------------------------------------------------------------------------------
check backup and restore -- gcp-cs-sa
-----------------------------------------------------------------------------------

++ get_backup_dest backup-gcp-cs-sa
++ local backup_name=backup-gcp-cs-sa
++ kubectl_bin get psmdb-backup backup-gcp-cs-sa -o 'jsonpath={.status.destination}'
++ sed -e 's/.json$//'
++ sed 's|s3://||'
++ sed 's|azure://||'
+++ mktemp
++ sed 's|gs://||'
++ local LAST_OUT=/tmp/tmp.MDqTEGSc45
+++ mktemp
++ local LAST_ERR=/tmp/tmp.yOkUz0A9qY
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb-backup backup-gcp-cs-sa -o 'jsonpath={.status.destination}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.MDqTEGSc45
++ cat /tmp/tmp.yOkUz0A9qY
++ rm /tmp/tmp.MDqTEGSc45 /tmp/tmp.yOkUz0A9qY
++ return 0
+ backup_dest_gcp_sa=operator-testing/psmdb-demand-backup/2026-03-26T08:16:25Z
+ check_backup_existence_gcs operator-testing/psmdb-demand-backup/2026-03-26T08:16:25Z /rs0/myApp.test.gz
+ backup_dest_gcp=operator-testing/psmdb-demand-backup/2026-03-26T08:16:25Z
+ obj=/rs0/myApp.test.gz
+ storage_name=gcp-cs
+ retry=0
+ gcs_path=gs://operator-testing/psmdb-demand-backup/2026-03-26T08:16:25Z/rs0/myApp.test.gz
+ gsutil ls gs://operator-testing/psmdb-demand-backup/2026-03-26T08:16:25Z/rs0/myApp.test.gz
+ echo 'Backup gs://operator-testing/psmdb-demand-backup/2026-03-26T08:16:25Z/rs0/myApp.test.gz found in gcp-cs'
Backup gs://operator-testing/psmdb-demand-backup/2026-03-26T08:16:25Z/rs0/myApp.test.gz found in gcp-cs
+ run_recovery_check backup-gcp-cs-sa some-name-rs0
+ local backup=backup-gcp-cs-sa
+ local cluster=some-name-rs0
+ run_mongo 'use myApp\n db.test.insert({ x: 100501 })' myApp:myPass@some-name-rs0.demand-backup-2582
+ local 'command=use myApp\n db.test.insert({ x: 100501 })'
+ local uri=myApp:myPass@some-name-rs0.demand-backup-2582
+ local driver=mongodb+srv
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local replica_set=rs0
+ [[ myApp:myPass@some-name-rs0.demand-backup-2582 == *cfg* ]]
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.VM0LmU2jCJ
+++ mktemp
++ local LAST_ERR=/tmp/tmp.OxboyAPcG6
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.VM0LmU2jCJ
++ cat /tmp/tmp.OxboyAPcG6
++ rm /tmp/tmp.VM0LmU2jCJ /tmp/tmp.OxboyAPcG6
++ return 0
+ local client_container=psmdb-client-bb8b97679-wjlk6
+ kubectl_bin exec psmdb-client-bb8b97679-wjlk6 -- bash -c 'printf '\''use myApp\n db.test.insert({ x: 100501 })\n'\'' | mongo mongodb+srv://myApp:myPass@some-name-rs0.demand-backup-2582.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
++ mktemp
+ local LAST_OUT=/tmp/tmp.5RS3cGC14C
++ mktemp
+ local LAST_ERR=/tmp/tmp.EftiCBWBQI
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-bb8b97679-wjlk6 -- bash -c 'printf '\''use myApp\n db.test.insert({ x: 100501 })\n'\'' | mongo mongodb+srv://myApp:myPass@some-name-rs0.demand-backup-2582.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.5RS3cGC14C
Percona Server for MongoDB shell version v4.4.29-28
connecting to: mongodb://some-name-rs0-2.some-name-rs0.demand-backup-2582.svc.cluster.local:27017,some-name-rs0-0.some-name-rs0.demand-backup-2582.svc.cluster.local:27017,some-name-rs0-1.some-name-rs0.demand-backup-2582.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb&replicaSet=rs0&ssl=false
Implicit session: session { "id" : UUID("cbc4d3b5-079c-473e-8bf5-f2140b756da2") }
Percona Server for MongoDB server version: v8.0.19-7
WARNING: shell and server versions do not match
switched to db myApp
WriteResult({ "nInserted" : 1 })
bye
+ cat /tmp/tmp.EftiCBWBQI
+ rm /tmp/tmp.5RS3cGC14C /tmp/tmp.EftiCBWBQI
+ return 0
+ compare_mongo_cmd find myApp:myPass@some-name-rs0.demand-backup-2582 -2nd .svc.cluster.local myApp test
+ local command=find
+ local uri=myApp:myPass@some-name-rs0.demand-backup-2582
+ local postfix=-2nd
+ local suffix=.svc.cluster.local
+ local database=myApp
+ local collection=test
+ local sort=
+ local tls=false
+ local replicaset=
+ local 'full_command=db.test.find()'
+ [[ -n '' ]]
+ log 'running db.test.find() in myApp'
+ set +o xtrace
[2026-03-26T08:18:10+0000] running db.test.find() in myApp
+ [[ false == \t\r\u\e ]]
+ mongo_command=run_mongo
+ run_mongo 'use myApp\n db.test.find()' myApp:myPass@some-name-rs0.demand-backup-2582 mongodb .svc.cluster.local ''
+ local 'command=use myApp\n db.test.find()'
+ local uri=myApp:myPass@some-name-rs0.demand-backup-2582
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local replica_set=rs0
+ [[ myApp:myPass@some-name-rs0.demand-backup-2582 == *cfg* ]]
+ grep -E -v 'I NETWORK|W NETWORK|F NETWORK|"c":"NETWORK"|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|Started a new thread for the timer service'
+ /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.jAMadDobjO
+++ mktemp
++ local LAST_ERR=/tmp/tmp.w4ZsyjLfIv
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.jAMadDobjO
++ cat /tmp/tmp.w4ZsyjLfIv
++ rm /tmp/tmp.jAMadDobjO /tmp/tmp.w4ZsyjLfIv
++ return 0
+ local client_container=psmdb-client-bb8b97679-wjlk6
+ kubectl_bin exec psmdb-client-bb8b97679-wjlk6 -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0.demand-backup-2582.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
++ mktemp
+ local LAST_OUT=/tmp/tmp.Z1D6vO98Td
++ mktemp
+ local LAST_ERR=/tmp/tmp.FQ0w8VpSRC
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-bb8b97679-wjlk6 -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0.demand-backup-2582.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.Z1D6vO98Td
+ cat /tmp/tmp.FQ0w8VpSRC
+ rm /tmp/tmp.Z1D6vO98Td /tmp/tmp.FQ0w8VpSRC
+ return 0
+ [[ 0 -eq 0 ]]
+ diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2271/e2e-tests/demand-backup/compare/find-2nd.json /tmp/tmp.Fq0pMPZsh8/find-2nd
+ run_restore backup-gcp-cs-sa
+ local backup_name=backup-gcp-cs-sa
+ local restore_file=/mnt/jenkins/workspace/cloud-psmdb-operator_PR-2271/e2e-tests/demand-backup/conf/restore.yml
+ log 'running restore restore-backup-gcp-cs-sa'
+ set +o xtrace
[2026-03-26T08:18:12+0000] running restore restore-backup-gcp-cs-sa
+ cat /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2271/e2e-tests/demand-backup/conf/restore.yml
+ /usr/sbin/sed -e 's/name:/name: restore-backup-gcp-cs-sa/'
+ /usr/sbin/sed -e 's/backupName:/backupName: backup-gcp-cs-sa/'
+ kubectl_bin apply -f -
++ mktemp
+ local LAST_OUT=/tmp/tmp.fFetkxiFHC
++ mktemp
+ local LAST_ERR=/tmp/tmp.cWt8AOlo8M
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl apply -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.fFetkxiFHC
perconaservermongodbrestore.psmdb.percona.com/restore-backup-gcp-cs-sa created
+ cat /tmp/tmp.cWt8AOlo8M
+ rm /tmp/tmp.fFetkxiFHC /tmp/tmp.cWt8AOlo8M
+ return 0
+ wait_restore backup-gcp-cs-sa some-name
+ local backup_name=backup-gcp-cs-sa
+ local cluster_name=some-name
+ local target_state=ready
+ local wait_cluster_consistency=1
+ local wait_time=1780
+ local ok_if_ready=0
+ set +o xtrace
Waiting for the psmdb-restore/restore-backup-gcp-cs-sa object to be created.OK
Waiting psmdb-restore/restore-backup-gcp-cs-sa to reach state "ready" .OK after 0 minutes
+ [[ 1 -eq 1 ]]
+ wait_cluster_consistency some-name
+ local cluster_name=some-name
+ local wait_time=32
+ retry=0
+ sleep 7
+ echo -n 'waiting for cluster readyness'
waiting for cluster readyness++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.KvDoW1Jt7f
+++ mktemp
++ local LAST_ERR=/tmp/tmp.JdZghGfd1b
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.KvDoW1Jt7f
++ cat /tmp/tmp.JdZghGfd1b
++ rm /tmp/tmp.KvDoW1Jt7f /tmp/tmp.JdZghGfd1b
++ return 0
+ [[ ready == \r\e\a\d\y ]]
+ echo .OK
.OK
+ simple_data_check some-name-rs0 3 0
+ local cluster_name=some-name-rs0
+ let last_pod=3-1
+ local isSharded=0
+ local cluster_pfx=
+ '[' 0 -eq 1 ']'
++ seq 0 2
+ for i in $(seq 0 $last_pod)
+ compare_mongo_cmd find myApp:myPass@some-name-rs0-0.some-name-rs0.demand-backup-2582
+ local command=find
+ local uri=myApp:myPass@some-name-rs0-0.some-name-rs0.demand-backup-2582
+ local postfix=
+ local suffix=
+ local database=myApp
+ local collection=test
+ local sort=
+ local tls=false
+ local replicaset=
+ local 'full_command=db.test.find()'
+ [[ -n '' ]]
+ log 'running db.test.find() in myApp'
+ set +o xtrace
[2026-03-26T08:18:40+0000] running db.test.find() in myApp
+ [[ false == \t\r\u\e ]]
+ mongo_command=run_mongo
+ run_mongo 'use myApp\n db.test.find()' myApp:myPass@some-name-rs0-0.some-name-rs0.demand-backup-2582 mongodb '' ''
+ local 'command=use myApp\n db.test.find()'
+ local uri=myApp:myPass@some-name-rs0-0.some-name-rs0.demand-backup-2582
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local replica_set=rs0
+ [[ myApp:myPass@some-name-rs0-0.some-name-rs0.demand-backup-2582 == *cfg* ]]
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+ grep -E -v 'I NETWORK|W NETWORK|F NETWORK|"c":"NETWORK"|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|Started a new thread for the timer service'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.3gCr0ItY6R
+ /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
+++ mktemp
++ local LAST_ERR=/tmp/tmp.g4cy7gVF12
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.3gCr0ItY6R
++ cat /tmp/tmp.g4cy7gVF12
++ rm /tmp/tmp.3gCr0ItY6R /tmp/tmp.g4cy7gVF12
++ return 0
+ local client_container=psmdb-client-bb8b97679-wjlk6
+ kubectl_bin exec psmdb-client-bb8b97679-wjlk6 -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-0.some-name-rs0.demand-backup-2582.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
++ mktemp
+ local LAST_OUT=/tmp/tmp.6nlT0iSJsU
++ mktemp
+ local LAST_ERR=/tmp/tmp.o0C8oa7jWX
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-bb8b97679-wjlk6 -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-0.some-name-rs0.demand-backup-2582.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.6nlT0iSJsU
+ cat /tmp/tmp.o0C8oa7jWX
+ rm /tmp/tmp.6nlT0iSJsU /tmp/tmp.o0C8oa7jWX
+ return 0
+ [[ 0 -eq 0 ]]
+ diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2271/e2e-tests/demand-backup/compare/find.json /tmp/tmp.Fq0pMPZsh8/find
+ for i in $(seq 0 $last_pod)
+ compare_mongo_cmd find myApp:myPass@some-name-rs0-1.some-name-rs0.demand-backup-2582
+ local command=find
+ local uri=myApp:myPass@some-name-rs0-1.some-name-rs0.demand-backup-2582
+ local postfix=
+ local suffix=
+ local database=myApp
+ local collection=test
+ local sort=
+ local tls=false
+ local replicaset=
+ local 'full_command=db.test.find()'
+ [[ -n '' ]]
+ log 'running db.test.find() in myApp'
+ set +o xtrace
[2026-03-26T08:18:42+0000] running db.test.find() in myApp
+ [[ false == \t\r\u\e ]]
+ mongo_command=run_mongo
+ run_mongo 'use myApp\n db.test.find()' myApp:myPass@some-name-rs0-1.some-name-rs0.demand-backup-2582 mongodb '' ''
+ local 'command=use myApp\n db.test.find()'
+ local uri=myApp:myPass@some-name-rs0-1.some-name-rs0.demand-backup-2582
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local replica_set=rs0
+ [[ myApp:myPass@some-name-rs0-1.some-name-rs0.demand-backup-2582 == *cfg* ]]
+ /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+ grep -E -v 'I NETWORK|W NETWORK|F NETWORK|"c":"NETWORK"|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|Started a new thread for the timer service'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.ikBCehltzv
+++ mktemp
++ local LAST_ERR=/tmp/tmp.gnwbfzEMiX
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.ikBCehltzv
++ cat /tmp/tmp.gnwbfzEMiX
++ rm /tmp/tmp.ikBCehltzv /tmp/tmp.gnwbfzEMiX
++ return 0
+ local client_container=psmdb-client-bb8b97679-wjlk6
+ kubectl_bin exec psmdb-client-bb8b97679-wjlk6 -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-1.some-name-rs0.demand-backup-2582.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
++ mktemp
+ local LAST_OUT=/tmp/tmp.39QzS7HzLj
++ mktemp
+ local LAST_ERR=/tmp/tmp.vQgoXm7bVM
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-bb8b97679-wjlk6 -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-1.some-name-rs0.demand-backup-2582.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.39QzS7HzLj
+ cat /tmp/tmp.vQgoXm7bVM
+ rm /tmp/tmp.39QzS7HzLj /tmp/tmp.vQgoXm7bVM
+ return 0
+ [[ 0 -eq 0 ]]
+ diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2271/e2e-tests/demand-backup/compare/find.json /tmp/tmp.Fq0pMPZsh8/find
+ for i in $(seq 0 $last_pod)
+ compare_mongo_cmd find myApp:myPass@some-name-rs0-2.some-name-rs0.demand-backup-2582
+ local command=find
+ local uri=myApp:myPass@some-name-rs0-2.some-name-rs0.demand-backup-2582
+ local postfix=
+ local suffix=
+ local database=myApp
+ local collection=test
+ local sort=
+ local tls=false
+ local replicaset=
+ local 'full_command=db.test.find()'
+ [[ -n '' ]]
+ log 'running db.test.find() in myApp'
+ set +o xtrace
[2026-03-26T08:18:45+0000] running db.test.find() in myApp
+ [[ false == \t\r\u\e ]]
+ mongo_command=run_mongo
+ grep -E -v 'I NETWORK|W NETWORK|F NETWORK|"c":"NETWORK"|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|Started a new thread for the timer service'
+ /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
+ run_mongo 'use myApp\n db.test.find()' myApp:myPass@some-name-rs0-2.some-name-rs0.demand-backup-2582 mongodb '' ''
+ local 'command=use myApp\n db.test.find()'
+ local uri=myApp:myPass@some-name-rs0-2.some-name-rs0.demand-backup-2582
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local replica_set=rs0
+ [[ myApp:myPass@some-name-rs0-2.some-name-rs0.demand-backup-2582 == *cfg* ]]
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.Go2OTodBYa
+++ mktemp
++ local LAST_ERR=/tmp/tmp.3a59VP8ztM
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.Go2OTodBYa
++ cat /tmp/tmp.3a59VP8ztM
++ rm /tmp/tmp.Go2OTodBYa /tmp/tmp.3a59VP8ztM
++ return 0
+ local client_container=psmdb-client-bb8b97679-wjlk6
+ kubectl_bin exec psmdb-client-bb8b97679-wjlk6 -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-2.some-name-rs0.demand-backup-2582.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
++ mktemp
+ local LAST_OUT=/tmp/tmp.S6InTV4APK
++ mktemp
+ local LAST_ERR=/tmp/tmp.IS2hgisuls
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-bb8b97679-wjlk6 -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-2.some-name-rs0.demand-backup-2582.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.S6InTV4APK
+ cat /tmp/tmp.IS2hgisuls
+ rm /tmp/tmp.S6InTV4APK /tmp/tmp.IS2hgisuls
+ return 0
+ [[ 0 -eq 0 ]]
+ diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2271/e2e-tests/demand-backup/compare/find.json /tmp/tmp.Fq0pMPZsh8/find
+ desc 'check backup and restore -- azure-blob'
+ set +o xtrace


-----------------------------------------------------------------------------------
check backup and restore -- azure-blob
-----------------------------------------------------------------------------------

++ get_backup_dest backup-azure-blob
++ local backup_name=backup-azure-blob
++ kubectl_bin get psmdb-backup backup-azure-blob -o 'jsonpath={.status.destination}'
++ sed -e 's/.json$//'
++ sed 's|s3://||'
+++ mktemp
++ sed 's|azure://||'
++ sed 's|gs://||'
++ local LAST_OUT=/tmp/tmp.hYe61KFrRI
+++ mktemp
++ local LAST_ERR=/tmp/tmp.5zTZibwfHy
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb-backup backup-azure-blob -o 'jsonpath={.status.destination}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.hYe61KFrRI
++ cat /tmp/tmp.5zTZibwfHy
++ rm /tmp/tmp.hYe61KFrRI /tmp/tmp.5zTZibwfHy
++ return 0
+ backup_dest_azure=https://engk8soperators.blob.core.windows.net/operator-testing/psmdb-demand-backup/2026-03-26T08:15:52Z
+ setup_azure_credentials
+ local secret_name=azure-secret
+ echo 'Setting up Azure credentials from K8s secret: azure-secret'
Setting up Azure credentials from K8s secret: azure-secret
+ local trace_was_on=0
+ [[ ehxB == *x* ]]
+ trace_was_on=1
+ set +x
+ echo 'Azure credentials configured successfully'
Azure credentials configured successfully
+ check_backup_existence_azure https://engk8soperators.blob.core.windows.net/operator-testing/psmdb-demand-backup/2026-03-26T08:15:52Z /rs0/myApp.test.gz
++ echo https://engk8soperators.blob.core.windows.net/operator-testing/psmdb-demand-backup/2026-03-26T08:15:52Z
++ sed 's|https://[^/]*\.blob\.core\.windows\.net/||'
+ url_path=operator-testing/psmdb-demand-backup/2026-03-26T08:15:52Z
++ echo operator-testing/psmdb-demand-backup/2026-03-26T08:15:52Z
++ cut -d/ -f1
+ container=operator-testing
++ echo operator-testing/psmdb-demand-backup/2026-03-26T08:15:52Z
++ cut -d/ -f2-
+ blob_prefix=psmdb-demand-backup/2026-03-26T08:15:52Z
+ blob=/rs0/myApp.test.gz
+ storage_name=azure-blob
+ retry=0
+ blob_path=psmdb-demand-backup/2026-03-26T08:15:52Z/rs0/myApp.test.gz
+ az storage blob show --container-name operator-testing --name psmdb-demand-backup/2026-03-26T08:15:52Z/rs0/myApp.test.gz
+ echo 'Backup psmdb-demand-backup/2026-03-26T08:15:52Z/rs0/myApp.test.gz found in azure-blob'
Backup psmdb-demand-backup/2026-03-26T08:15:52Z/rs0/myApp.test.gz found in azure-blob
+ run_recovery_check backup-azure-blob some-name-rs0
+ local backup=backup-azure-blob
+ local cluster=some-name-rs0
+ run_mongo 'use myApp\n db.test.insert({ x: 100501 })' myApp:myPass@some-name-rs0.demand-backup-2582
+ local 'command=use myApp\n db.test.insert({ x: 100501 })'
+ local uri=myApp:myPass@some-name-rs0.demand-backup-2582
+ local driver=mongodb+srv
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local replica_set=rs0
+ [[ myApp:myPass@some-name-rs0.demand-backup-2582 == *cfg* ]]
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.95wUU1H7vq
+++ mktemp
++ local LAST_ERR=/tmp/tmp.BRaXOOn540
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.95wUU1H7vq
++ cat /tmp/tmp.BRaXOOn540
++ rm /tmp/tmp.95wUU1H7vq /tmp/tmp.BRaXOOn540
++ return 0
+ local client_container=psmdb-client-bb8b97679-wjlk6
+ kubectl_bin exec psmdb-client-bb8b97679-wjlk6 -- bash -c 'printf '\''use myApp\n db.test.insert({ x: 100501 })\n'\'' | mongo mongodb+srv://myApp:myPass@some-name-rs0.demand-backup-2582.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
++ mktemp
+ local LAST_OUT=/tmp/tmp.EzSpMI8t7d
++ mktemp
+ local LAST_ERR=/tmp/tmp.mXajt9vZbL
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-bb8b97679-wjlk6 -- bash -c 'printf '\''use myApp\n db.test.insert({ x: 100501 })\n'\'' | mongo mongodb+srv://myApp:myPass@some-name-rs0.demand-backup-2582.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.EzSpMI8t7d
Percona Server for MongoDB shell version v4.4.29-28
connecting to: mongodb://some-name-rs0-2.some-name-rs0.demand-backup-2582.svc.cluster.local:27017,some-name-rs0-0.some-name-rs0.demand-backup-2582.svc.cluster.local:27017,some-name-rs0-1.some-name-rs0.demand-backup-2582.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb&replicaSet=rs0&ssl=false
Implicit session: session { "id" : UUID("b1b7a8cd-845f-4c07-a0f7-1afa2ea423bb") }
Percona Server for MongoDB server version: v8.0.19-7
WARNING: shell and server versions do not match
switched to db myApp
WriteResult({ "nInserted" : 1 })
bye
+ cat /tmp/tmp.mXajt9vZbL
+ rm /tmp/tmp.EzSpMI8t7d /tmp/tmp.mXajt9vZbL
+ return 0
+ compare_mongo_cmd find myApp:myPass@some-name-rs0.demand-backup-2582 -2nd .svc.cluster.local myApp test
+ local command=find
+ local uri=myApp:myPass@some-name-rs0.demand-backup-2582
+ local postfix=-2nd
+ local suffix=.svc.cluster.local
+ local database=myApp
+ local collection=test
+ local sort=
+ local tls=false
+ local replicaset=
+ local 'full_command=db.test.find()'
+ [[ -n '' ]]
+ log 'running db.test.find() in myApp'
+ set +o xtrace
[2026-03-26T08:18:54+0000] running db.test.find() in myApp
+ [[ false == \t\r\u\e ]]
+ mongo_command=run_mongo
+ run_mongo 'use myApp\n db.test.find()' myApp:myPass@some-name-rs0.demand-backup-2582 mongodb .svc.cluster.local ''
+ /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
+ grep -E -v 'I NETWORK|W NETWORK|F NETWORK|"c":"NETWORK"|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|Started a new thread for the timer service'
+ local 'command=use myApp\n db.test.find()'
+ local uri=myApp:myPass@some-name-rs0.demand-backup-2582
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local replica_set=rs0
+ [[ myApp:myPass@some-name-rs0.demand-backup-2582 == *cfg* ]]
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.E0L0eWNjnq
+++ mktemp
++ local LAST_ERR=/tmp/tmp.IG8zMaciHz
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.E0L0eWNjnq
++ cat /tmp/tmp.IG8zMaciHz
++ rm /tmp/tmp.E0L0eWNjnq /tmp/tmp.IG8zMaciHz
++ return 0
+ local client_container=psmdb-client-bb8b97679-wjlk6
+ kubectl_bin exec psmdb-client-bb8b97679-wjlk6 -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0.demand-backup-2582.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
++ mktemp
+ local LAST_OUT=/tmp/tmp.OKH8bPixID
++ mktemp
+ local LAST_ERR=/tmp/tmp.ItuA2Qapso
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-bb8b97679-wjlk6 -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0.demand-backup-2582.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.OKH8bPixID
+ cat /tmp/tmp.ItuA2Qapso
+ rm /tmp/tmp.OKH8bPixID /tmp/tmp.ItuA2Qapso
+ return 0
+ [[ 0 -eq 0 ]]
+ diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2271/e2e-tests/demand-backup/compare/find-2nd.json /tmp/tmp.Fq0pMPZsh8/find-2nd
+ run_restore backup-azure-blob
+ local backup_name=backup-azure-blob
+ local restore_file=/mnt/jenkins/workspace/cloud-psmdb-operator_PR-2271/e2e-tests/demand-backup/conf/restore.yml
+ log 'running restore restore-backup-azure-blob'
+ set +o xtrace
[2026-03-26T08:18:57+0000] running restore restore-backup-azure-blob
+ cat /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2271/e2e-tests/demand-backup/conf/restore.yml
+ /usr/sbin/sed -e 's/name:/name: restore-backup-azure-blob/'
+ /usr/sbin/sed -e 's/backupName:/backupName: backup-azure-blob/'
+ kubectl_bin apply -f -
++ mktemp
+ local LAST_OUT=/tmp/tmp.BbOtupRg4X
++ mktemp
+ local LAST_ERR=/tmp/tmp.NsnC4KKCqC
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl apply -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.BbOtupRg4X
perconaservermongodbrestore.psmdb.percona.com/restore-backup-azure-blob created
+ cat /tmp/tmp.NsnC4KKCqC
+ rm /tmp/tmp.BbOtupRg4X /tmp/tmp.NsnC4KKCqC
+ return 0
+ wait_restore backup-azure-blob some-name
+ local backup_name=backup-azure-blob
+ local cluster_name=some-name
+ local target_state=ready
+ local wait_cluster_consistency=1
+ local wait_time=1780
+ local ok_if_ready=0
+ set +o xtrace
Waiting for the psmdb-restore/restore-backup-azure-blob object to be created.OK
Waiting psmdb-restore/restore-backup-azure-blob to reach state "ready" .OK after 0 minutes
+ [[ 1 -eq 1 ]]
+ wait_cluster_consistency some-name
+ local cluster_name=some-name
+ local wait_time=32
+ retry=0
+ sleep 7
+ echo -n 'waiting for cluster readyness'
waiting for cluster readyness++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.wDLRm8ivFj
+++ mktemp
++ local LAST_ERR=/tmp/tmp.t3h7thFGI1
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.wDLRm8ivFj
++ cat /tmp/tmp.t3h7thFGI1
++ rm /tmp/tmp.wDLRm8ivFj /tmp/tmp.t3h7thFGI1
++ return 0
+ [[ ready == \r\e\a\d\y ]]
+ echo .OK
.OK
+ simple_data_check some-name-rs0 3 0
+ local cluster_name=some-name-rs0
+ let last_pod=3-1
+ local isSharded=0
+ local cluster_pfx=
+ '[' 0 -eq 1 ']'
++ seq 0 2
+ for i in $(seq 0 $last_pod)
+ compare_mongo_cmd find myApp:myPass@some-name-rs0-0.some-name-rs0.demand-backup-2582
+ local command=find
+ local uri=myApp:myPass@some-name-rs0-0.some-name-rs0.demand-backup-2582
+ local postfix=
+ local suffix=
+ local database=myApp
+ local collection=test
+ local sort=
+ local tls=false
+ local replicaset=
+ local 'full_command=db.test.find()'
+ [[ -n '' ]]
+ log 'running db.test.find() in myApp'
+ set +o xtrace
[2026-03-26T08:19:19+0000] running db.test.find() in myApp
+ [[ false == \t\r\u\e ]]
+ mongo_command=run_mongo
+ run_mongo 'use myApp\n db.test.find()' myApp:myPass@some-name-rs0-0.some-name-rs0.demand-backup-2582 mongodb '' ''
+ grep -E -v 'I NETWORK|W NETWORK|F NETWORK|"c":"NETWORK"|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|Started a new thread for the timer service'
+ local 'command=use myApp\n db.test.find()'
+ local uri=myApp:myPass@some-name-rs0-0.some-name-rs0.demand-backup-2582
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local replica_set=rs0
+ [[ myApp:myPass@some-name-rs0-0.some-name-rs0.demand-backup-2582 == *cfg* ]]
+ /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.NHj910vWB0
+++ mktemp
++ local LAST_ERR=/tmp/tmp.HoUq7909cF
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.NHj910vWB0
++ cat /tmp/tmp.HoUq7909cF
++ rm /tmp/tmp.NHj910vWB0 /tmp/tmp.HoUq7909cF
++ return 0
+ local client_container=psmdb-client-bb8b97679-wjlk6
+ kubectl_bin exec psmdb-client-bb8b97679-wjlk6 -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-0.some-name-rs0.demand-backup-2582.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
++ mktemp
+ local LAST_OUT=/tmp/tmp.8EmoNGrOJq
++ mktemp
+ local LAST_ERR=/tmp/tmp.0XcDwf4IgP
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-bb8b97679-wjlk6 -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-0.some-name-rs0.demand-backup-2582.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.8EmoNGrOJq
+ cat /tmp/tmp.0XcDwf4IgP
+ rm /tmp/tmp.8EmoNGrOJq /tmp/tmp.0XcDwf4IgP
+ return 0
+ [[ 0 -eq 0 ]]
+ diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2271/e2e-tests/demand-backup/compare/find.json /tmp/tmp.Fq0pMPZsh8/find
+ for i in $(seq 0 $last_pod)
+ compare_mongo_cmd find myApp:myPass@some-name-rs0-1.some-name-rs0.demand-backup-2582
+ local command=find
+ local uri=myApp:myPass@some-name-rs0-1.some-name-rs0.demand-backup-2582
+ local postfix=
+ local suffix=
+ local database=myApp
+ local collection=test
+ local sort=
+ local tls=false
+ local replicaset=
+ local 'full_command=db.test.find()'
+ [[ -n '' ]]
+ log 'running db.test.find() in myApp'
+ set +o xtrace
[2026-03-26T08:19:22+0000] running db.test.find() in myApp
+ [[ false == \t\r\u\e ]]
+ mongo_command=run_mongo
+ grep -E -v 'I NETWORK|W NETWORK|F NETWORK|"c":"NETWORK"|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|Started a new thread for the timer service'
+ run_mongo 'use myApp\n db.test.find()' myApp:myPass@some-name-rs0-1.some-name-rs0.demand-backup-2582 mongodb '' ''
+ local 'command=use myApp\n db.test.find()'
+ local uri=myApp:myPass@some-name-rs0-1.some-name-rs0.demand-backup-2582
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local replica_set=rs0
+ /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
+ [[ myApp:myPass@some-name-rs0-1.some-name-rs0.demand-backup-2582 == *cfg* ]]
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.jev8pTbzvx
+++ mktemp
++ local LAST_ERR=/tmp/tmp.bmMtRozBDd
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.jev8pTbzvx
++ cat /tmp/tmp.bmMtRozBDd
++ rm /tmp/tmp.jev8pTbzvx /tmp/tmp.bmMtRozBDd
++ return 0
+ local client_container=psmdb-client-bb8b97679-wjlk6
+ kubectl_bin exec psmdb-client-bb8b97679-wjlk6 -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-1.some-name-rs0.demand-backup-2582.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
++ mktemp
+ local LAST_OUT=/tmp/tmp.fKjQwcPplc
++ mktemp
+ local LAST_ERR=/tmp/tmp.lQnnfCAYBU
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-bb8b97679-wjlk6 -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-1.some-name-rs0.demand-backup-2582.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.fKjQwcPplc
+ cat /tmp/tmp.lQnnfCAYBU
+ rm /tmp/tmp.fKjQwcPplc /tmp/tmp.lQnnfCAYBU
+ return 0
+ [[ 0 -eq 0 ]]
+ diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2271/e2e-tests/demand-backup/compare/find.json /tmp/tmp.Fq0pMPZsh8/find
+ for i in $(seq 0 $last_pod)
+ compare_mongo_cmd find myApp:myPass@some-name-rs0-2.some-name-rs0.demand-backup-2582
+ local command=find
+ local uri=myApp:myPass@some-name-rs0-2.some-name-rs0.demand-backup-2582
+ local postfix=
+ local suffix=
+ local database=myApp
+ local collection=test
+ local sort=
+ local tls=false
+ local replicaset=
+ local 'full_command=db.test.find()'
+ [[ -n '' ]]
+ log 'running db.test.find() in myApp'
+ set +o xtrace
[2026-03-26T08:19:24+0000] running db.test.find() in myApp
+ [[ false == \t\r\u\e ]]
+ mongo_command=run_mongo
+ run_mongo 'use myApp\n db.test.find()' myApp:myPass@some-name-rs0-2.some-name-rs0.demand-backup-2582 mongodb '' ''
+ grep -E -v 'I NETWORK|W NETWORK|F NETWORK|"c":"NETWORK"|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|Started a new thread for the timer service'
+ local 'command=use myApp\n db.test.find()'
+ local uri=myApp:myPass@some-name-rs0-2.some-name-rs0.demand-backup-2582
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
+ local replica_set=rs0
+ [[ myApp:myPass@some-name-rs0-2.some-name-rs0.demand-backup-2582 == *cfg* ]]
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.qgEbyrOHIT
+++ mktemp
++ local LAST_ERR=/tmp/tmp.7HCysOGqhj
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.qgEbyrOHIT
++ cat /tmp/tmp.7HCysOGqhj
++ rm /tmp/tmp.qgEbyrOHIT /tmp/tmp.7HCysOGqhj
++ return 0
+ local client_container=psmdb-client-bb8b97679-wjlk6
+ kubectl_bin exec psmdb-client-bb8b97679-wjlk6 -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-2.some-name-rs0.demand-backup-2582.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
++ mktemp
+ local LAST_OUT=/tmp/tmp.OEv9DrdNFp
++ mktemp
+ local LAST_ERR=/tmp/tmp.HW83YOtzB5
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-bb8b97679-wjlk6 -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-2.some-name-rs0.demand-backup-2582.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.OEv9DrdNFp
+ cat /tmp/tmp.HW83YOtzB5
+ rm /tmp/tmp.OEv9DrdNFp /tmp/tmp.HW83YOtzB5
+ return 0
+ [[ 0 -eq 0 ]]
+ diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2271/e2e-tests/demand-backup/compare/find.json /tmp/tmp.Fq0pMPZsh8/find
+ desc 'check backup and restore -- minio'
+ set +o xtrace


-----------------------------------------------------------------------------------
check backup and restore -- minio
-----------------------------------------------------------------------------------

+ check_backup_in_storage backup-minio minio rs0 myApp.test.gz
+ local backup=backup-minio
+ local storage_type=minio
+ local replset=rs0
+ local file=myApp.test.gz
+ local protocol=http
+ local endpoint
++ get_backup_dest backup-minio
++ /usr/sbin/sed 's|https://engk8soperators.blob.core.windows.net/||'
++ local backup_name=backup-minio
++ kubectl_bin get psmdb-backup backup-minio -o 'jsonpath={.status.destination}'
++ sed -e 's/.json$//'
++ sed 's|s3://||'
+++ mktemp
++ sed 's|azure://||'
++ local LAST_OUT=/tmp/tmp.qEbeC4XqtN
++ sed 's|gs://||'
+++ mktemp
++ local LAST_ERR=/tmp/tmp.4PkXGAsyYA
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb-backup backup-minio -o 'jsonpath={.status.destination}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.qEbeC4XqtN
++ cat /tmp/tmp.4PkXGAsyYA
++ rm /tmp/tmp.qEbeC4XqtN /tmp/tmp.4PkXGAsyYA
++ return 0
+ backup_dest=operator-testing/2026-03-26T08:16:14Z
+ case ${storage_type} in
+ endpoint=minio-service
+ kubectl_bin run -i --rm aws-cli --image=docker.io/perconalab/awscli --restart=Never -- /usr/bin/env AWS_ACCESS_KEY_ID=some-access-key AWS_SECRET_ACCESS_KEY=some-secret-key AWS_DEFAULT_REGION=us-east-1 /usr/bin/aws --no-verify-ssl --endpoint-url http://minio-service:9000 s3 ls s3://operator-testing/2026-03-26T08:16:14Z/rs0/myApp.test.gz
+ grep myApp.test.gz
++ mktemp
+ local LAST_OUT=/tmp/tmp.Lv1LKmBNNN
++ mktemp
+ local LAST_ERR=/tmp/tmp.OmzY135bkF
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl run -i --rm aws-cli --image=docker.io/perconalab/awscli --restart=Never -- /usr/bin/env AWS_ACCESS_KEY_ID=some-access-key AWS_SECRET_ACCESS_KEY=some-secret-key AWS_DEFAULT_REGION=us-east-1 /usr/bin/aws --no-verify-ssl --endpoint-url http://minio-service:9000 s3 ls s3://operator-testing/2026-03-26T08:16:14Z/rs0/myApp.test.gz
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.Lv1LKmBNNN
+ cat /tmp/tmp.OmzY135bkF
+ rm /tmp/tmp.Lv1LKmBNNN /tmp/tmp.OmzY135bkF
+ return 0
2026-03-26 08:16:18         55 myApp.test.gz
+ run_recovery_check backup-minio some-name-rs0
+ local backup=backup-minio
+ local cluster=some-name-rs0
+ run_mongo 'use myApp\n db.test.insert({ x: 100501 })' myApp:myPass@some-name-rs0.demand-backup-2582
+ local 'command=use myApp\n db.test.insert({ x: 100501 })'
+ local uri=myApp:myPass@some-name-rs0.demand-backup-2582
+ local driver=mongodb+srv
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local replica_set=rs0
+ [[ myApp:myPass@some-name-rs0.demand-backup-2582 == *cfg* ]]
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.bH1EfuzjhO
+++ mktemp
++ local LAST_ERR=/tmp/tmp.5AN8D2v9HH
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.bH1EfuzjhO
++ cat /tmp/tmp.5AN8D2v9HH
++ rm /tmp/tmp.bH1EfuzjhO /tmp/tmp.5AN8D2v9HH
++ return 0
+ local client_container=psmdb-client-bb8b97679-wjlk6
+ kubectl_bin exec psmdb-client-bb8b97679-wjlk6 -- bash -c 'printf '\''use myApp\n db.test.insert({ x: 100501 })\n'\'' | mongo mongodb+srv://myApp:myPass@some-name-rs0.demand-backup-2582.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
++ mktemp
+ local LAST_OUT=/tmp/tmp.cLxxaSWlmp
++ mktemp
+ local LAST_ERR=/tmp/tmp.ulRZZI32bb
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-bb8b97679-wjlk6 -- bash -c 'printf '\''use myApp\n db.test.insert({ x: 100501 })\n'\'' | mongo mongodb+srv://myApp:myPass@some-name-rs0.demand-backup-2582.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.cLxxaSWlmp
Percona Server for MongoDB shell version v4.4.29-28
connecting to: mongodb://some-name-rs0-1.some-name-rs0.demand-backup-2582.svc.cluster.local:27017,some-name-rs0-2.some-name-rs0.demand-backup-2582.svc.cluster.local:27017,some-name-rs0-0.some-name-rs0.demand-backup-2582.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb&replicaSet=rs0&ssl=false
Implicit session: session { "id" : UUID("bd6ecb83-3a6a-4734-9902-239495cbca9d") }
Percona Server for MongoDB server version: v8.0.19-7
WARNING: shell and server versions do not match
switched to db myApp
WriteResult({ "nInserted" : 1 })
bye
+ cat /tmp/tmp.ulRZZI32bb
+ rm /tmp/tmp.cLxxaSWlmp /tmp/tmp.ulRZZI32bb
+ return 0
+ compare_mongo_cmd find myApp:myPass@some-name-rs0.demand-backup-2582 -2nd .svc.cluster.local myApp test
+ local command=find
+ local uri=myApp:myPass@some-name-rs0.demand-backup-2582
+ local postfix=-2nd
+ local suffix=.svc.cluster.local
+ local database=myApp
+ local collection=test
+ local sort=
+ local tls=false
+ local replicaset=
+ local 'full_command=db.test.find()'
+ [[ -n '' ]]
+ log 'running db.test.find() in myApp'
+ set +o xtrace
[2026-03-26T08:19:36+0000] running db.test.find() in myApp
+ [[ false == \t\r\u\e ]]
+ mongo_command=run_mongo
+ run_mongo 'use myApp\n db.test.find()' myApp:myPass@some-name-rs0.demand-backup-2582 mongodb .svc.cluster.local ''
+ local 'command=use myApp\n db.test.find()'
+ local uri=myApp:myPass@some-name-rs0.demand-backup-2582
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local replica_set=rs0
+ [[ myApp:myPass@some-name-rs0.demand-backup-2582 == *cfg* ]]
+ grep -E -v 'I NETWORK|W NETWORK|F NETWORK|"c":"NETWORK"|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|Started a new thread for the timer service'
+ /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.BEeiB8vIWH
+++ mktemp
++ local LAST_ERR=/tmp/tmp.UJFYwAfK6f
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.BEeiB8vIWH
++ cat /tmp/tmp.UJFYwAfK6f
++ rm /tmp/tmp.BEeiB8vIWH /tmp/tmp.UJFYwAfK6f
++ return 0
+ local client_container=psmdb-client-bb8b97679-wjlk6
+ kubectl_bin exec psmdb-client-bb8b97679-wjlk6 -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0.demand-backup-2582.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
++ mktemp
+ local LAST_OUT=/tmp/tmp.WVy7RuRVtd
++ mktemp
+ local LAST_ERR=/tmp/tmp.7dkZecTiD6
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-bb8b97679-wjlk6 -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0.demand-backup-2582.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.WVy7RuRVtd
+ cat /tmp/tmp.7dkZecTiD6
+ rm /tmp/tmp.WVy7RuRVtd /tmp/tmp.7dkZecTiD6
+ return 0
+ [[ 0 -eq 0 ]]
+ diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2271/e2e-tests/demand-backup/compare/find-2nd.json /tmp/tmp.Fq0pMPZsh8/find-2nd
+ run_restore backup-minio
+ local backup_name=backup-minio
+ local restore_file=/mnt/jenkins/workspace/cloud-psmdb-operator_PR-2271/e2e-tests/demand-backup/conf/restore.yml
+ log 'running restore restore-backup-minio'
+ set +o xtrace
[2026-03-26T08:19:38+0000] running restore restore-backup-minio
+ cat /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2271/e2e-tests/demand-backup/conf/restore.yml
+ /usr/sbin/sed -e 's/name:/name: restore-backup-minio/'
+ /usr/sbin/sed -e 's/backupName:/backupName: backup-minio/'
+ kubectl_bin apply -f -
++ mktemp
+ local LAST_OUT=/tmp/tmp.usskLdv9tt
++ mktemp
+ local LAST_ERR=/tmp/tmp.y8iIglw1xv
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl apply -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.usskLdv9tt
perconaservermongodbrestore.psmdb.percona.com/restore-backup-minio created
+ cat /tmp/tmp.y8iIglw1xv
+ rm /tmp/tmp.usskLdv9tt /tmp/tmp.y8iIglw1xv
+ return 0
+ wait_restore backup-minio some-name
+ local backup_name=backup-minio
+ local cluster_name=some-name
+ local target_state=ready
+ local wait_cluster_consistency=1
+ local wait_time=1780
+ local ok_if_ready=0
+ set +o xtrace
Waiting for the psmdb-restore/restore-backup-minio object to be created.OK
Waiting psmdb-restore/restore-backup-minio to reach state "ready" .OK after 0 minutes
+ [[ 1 -eq 1 ]]
+ wait_cluster_consistency some-name
+ local cluster_name=some-name
+ local wait_time=32
+ retry=0
+ sleep 7
+ echo -n 'waiting for cluster readyness'
waiting for cluster readyness++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.mZRKG3PTkp
+++ mktemp
++ local LAST_ERR=/tmp/tmp.O8jNB4CEpB
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.mZRKG3PTkp
++ cat /tmp/tmp.O8jNB4CEpB
++ rm /tmp/tmp.mZRKG3PTkp /tmp/tmp.O8jNB4CEpB
++ return 0
+ [[ ready == \r\e\a\d\y ]]
+ echo .OK
.OK
+ simple_data_check some-name-rs0 3 0
+ local cluster_name=some-name-rs0
+ let last_pod=3-1
+ local isSharded=0
+ local cluster_pfx=
+ '[' 0 -eq 1 ']'
++ seq 0 2
+ for i in $(seq 0 $last_pod)
+ compare_mongo_cmd find myApp:myPass@some-name-rs0-0.some-name-rs0.demand-backup-2582
+ local command=find
+ local uri=myApp:myPass@some-name-rs0-0.some-name-rs0.demand-backup-2582
+ local postfix=
+ local suffix=
+ local database=myApp
+ local collection=test
+ local sort=
+ local tls=false
+ local replicaset=
+ local 'full_command=db.test.find()'
+ [[ -n '' ]]
+ log 'running db.test.find() in myApp'
+ set +o xtrace
[2026-03-26T08:20:00+0000] running db.test.find() in myApp
+ [[ false == \t\r\u\e ]]
+ mongo_command=run_mongo
+ run_mongo 'use myApp\n db.test.find()' myApp:myPass@some-name-rs0-0.some-name-rs0.demand-backup-2582 mongodb '' ''
+ grep -E -v 'I NETWORK|W NETWORK|F NETWORK|"c":"NETWORK"|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|Started a new thread for the timer service'
+ local 'command=use myApp\n db.test.find()'
+ local uri=myApp:myPass@some-name-rs0-0.some-name-rs0.demand-backup-2582
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local replica_set=rs0
+ [[ myApp:myPass@some-name-rs0-0.some-name-rs0.demand-backup-2582 == *cfg* ]]
+ /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.0Qugp6f2Ov
+++ mktemp
++ local LAST_ERR=/tmp/tmp.wPFq90sZsG
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.0Qugp6f2Ov
++ cat /tmp/tmp.wPFq90sZsG
++ rm /tmp/tmp.0Qugp6f2Ov /tmp/tmp.wPFq90sZsG
++ return 0
+ local client_container=psmdb-client-bb8b97679-wjlk6
+ kubectl_bin exec psmdb-client-bb8b97679-wjlk6 -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-0.some-name-rs0.demand-backup-2582.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
++ mktemp
+ local LAST_OUT=/tmp/tmp.fXvovam2pY
++ mktemp
+ local LAST_ERR=/tmp/tmp.Z2K2TGAzcf
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-bb8b97679-wjlk6 -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-0.some-name-rs0.demand-backup-2582.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.fXvovam2pY
+ cat /tmp/tmp.Z2K2TGAzcf
+ rm /tmp/tmp.fXvovam2pY /tmp/tmp.Z2K2TGAzcf
+ return 0
+ [[ 0 -eq 0 ]]
+ diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2271/e2e-tests/demand-backup/compare/find.json /tmp/tmp.Fq0pMPZsh8/find
+ for i in $(seq 0 $last_pod)
+ compare_mongo_cmd find myApp:myPass@some-name-rs0-1.some-name-rs0.demand-backup-2582
+ local command=find
+ local uri=myApp:myPass@some-name-rs0-1.some-name-rs0.demand-backup-2582
+ local postfix=
+ local suffix=
+ local database=myApp
+ local collection=test
+ local sort=
+ local tls=false
+ local replicaset=
+ local 'full_command=db.test.find()'
+ [[ -n '' ]]
+ log 'running db.test.find() in myApp'
+ set +o xtrace
[2026-03-26T08:20:02+0000] running db.test.find() in myApp
+ [[ false == \t\r\u\e ]]
+ mongo_command=run_mongo
+ run_mongo 'use myApp\n db.test.find()' myApp:myPass@some-name-rs0-1.some-name-rs0.demand-backup-2582 mongodb '' ''
+ grep -E -v 'I NETWORK|W NETWORK|F NETWORK|"c":"NETWORK"|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|Started a new thread for the timer service'
+ local 'command=use myApp\n db.test.find()'
+ local uri=myApp:myPass@some-name-rs0-1.some-name-rs0.demand-backup-2582
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local replica_set=rs0
+ [[ myApp:myPass@some-name-rs0-1.some-name-rs0.demand-backup-2582 == *cfg* ]]
+ /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.53bhT5SD1A
+++ mktemp
++ local LAST_ERR=/tmp/tmp.tnQ8xDyW9H
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.53bhT5SD1A
++ cat /tmp/tmp.tnQ8xDyW9H
++ rm /tmp/tmp.53bhT5SD1A /tmp/tmp.tnQ8xDyW9H
++ return 0
+ local client_container=psmdb-client-bb8b97679-wjlk6
+ kubectl_bin exec psmdb-client-bb8b97679-wjlk6 -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-1.some-name-rs0.demand-backup-2582.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
++ mktemp
+ local LAST_OUT=/tmp/tmp.a6l0wpfedz
++ mktemp
+ local LAST_ERR=/tmp/tmp.7MtAbuWKZK
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-bb8b97679-wjlk6 -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-1.some-name-rs0.demand-backup-2582.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.a6l0wpfedz
+ cat /tmp/tmp.7MtAbuWKZK
+ rm /tmp/tmp.a6l0wpfedz /tmp/tmp.7MtAbuWKZK
+ return 0
+ [[ 0 -eq 0 ]]
+ diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2271/e2e-tests/demand-backup/compare/find.json /tmp/tmp.Fq0pMPZsh8/find
+ for i in $(seq 0 $last_pod)
+ compare_mongo_cmd find myApp:myPass@some-name-rs0-2.some-name-rs0.demand-backup-2582
+ local command=find
+ local uri=myApp:myPass@some-name-rs0-2.some-name-rs0.demand-backup-2582
+ local postfix=
+ local suffix=
+ local database=myApp
+ local collection=test
+ local sort=
+ local tls=false
+ local replicaset=
+ local 'full_command=db.test.find()'
+ [[ -n '' ]]
+ log 'running db.test.find() in myApp'
+ set +o xtrace
[2026-03-26T08:20:05+0000] running db.test.find() in myApp
+ [[ false == \t\r\u\e ]]
+ mongo_command=run_mongo
+ run_mongo 'use myApp\n db.test.find()' myApp:myPass@some-name-rs0-2.some-name-rs0.demand-backup-2582 mongodb '' ''
+ local 'command=use myApp\n db.test.find()'
+ local uri=myApp:myPass@some-name-rs0-2.some-name-rs0.demand-backup-2582
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ grep -E -v 'I NETWORK|W NETWORK|F NETWORK|"c":"NETWORK"|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|Started a new thread for the timer service'
+ local mongo_flag=
+ local replica_set=rs0
+ [[ myApp:myPass@some-name-rs0-2.some-name-rs0.demand-backup-2582 == *cfg* ]]
+ /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.C9Txzub7HC
+++ mktemp
++ local LAST_ERR=/tmp/tmp.NFOlTRuGyb
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.C9Txzub7HC
++ cat /tmp/tmp.NFOlTRuGyb
++ rm /tmp/tmp.C9Txzub7HC /tmp/tmp.NFOlTRuGyb
++ return 0
+ local client_container=psmdb-client-bb8b97679-wjlk6
+ kubectl_bin exec psmdb-client-bb8b97679-wjlk6 -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-2.some-name-rs0.demand-backup-2582.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
++ mktemp
+ local LAST_OUT=/tmp/tmp.jBFGZwDLSX
++ mktemp
+ local LAST_ERR=/tmp/tmp.LV4OT3Vt5n
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-bb8b97679-wjlk6 -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-2.some-name-rs0.demand-backup-2582.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.jBFGZwDLSX
+ cat /tmp/tmp.LV4OT3Vt5n
+ rm /tmp/tmp.jBFGZwDLSX /tmp/tmp.LV4OT3Vt5n
+ return 0
+ [[ 0 -eq 0 ]]
+ diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2271/e2e-tests/demand-backup/compare/find.json /tmp/tmp.Fq0pMPZsh8/find
+ run_mongo 'use myApp\n db.dropUser("test1user")' userAdmin:userAdmin123456@some-name-rs0.demand-backup-2582
+ local 'command=use myApp\n db.dropUser("test1user")'
+ local uri=userAdmin:userAdmin123456@some-name-rs0.demand-backup-2582
+ local driver=mongodb+srv
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local replica_set=rs0
+ [[ userAdmin:userAdmin123456@some-name-rs0.demand-backup-2582 == *cfg* ]]
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.aRNfMkH4Hr
+++ mktemp
++ local LAST_ERR=/tmp/tmp.xPphkPVxEL
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.aRNfMkH4Hr
++ cat /tmp/tmp.xPphkPVxEL
++ rm /tmp/tmp.aRNfMkH4Hr /tmp/tmp.xPphkPVxEL
++ return 0
+ local client_container=psmdb-client-bb8b97679-wjlk6
+ kubectl_bin exec psmdb-client-bb8b97679-wjlk6 -- bash -c 'printf '\''use myApp\n db.dropUser("test1user")\n'\'' | mongo mongodb+srv://userAdmin:userAdmin123456@some-name-rs0.demand-backup-2582.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
++ mktemp
+ local LAST_OUT=/tmp/tmp.IZZ9BW1YJq
++ mktemp
+ local LAST_ERR=/tmp/tmp.r2lTIOwsRs
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-bb8b97679-wjlk6 -- bash -c 'printf '\''use myApp\n db.dropUser("test1user")\n'\'' | mongo mongodb+srv://userAdmin:userAdmin123456@some-name-rs0.demand-backup-2582.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.IZZ9BW1YJq
Percona Server for MongoDB shell version v4.4.29-28
connecting to: mongodb://some-name-rs0-2.some-name-rs0.demand-backup-2582.svc.cluster.local:27017,some-name-rs0-0.some-name-rs0.demand-backup-2582.svc.cluster.local:27017,some-name-rs0-1.some-name-rs0.demand-backup-2582.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb&replicaSet=rs0&ssl=false
Implicit session: session { "id" : UUID("633a4b00-9067-4e3b-8347-93d2dacbf5b8") }
Percona Server for MongoDB server version: v8.0.19-7
WARNING: shell and server versions do not match
switched to db myApp
true
bye
+ cat /tmp/tmp.r2lTIOwsRs
+ rm /tmp/tmp.IZZ9BW1YJq /tmp/tmp.r2lTIOwsRs
+ return 0
+ run_mongo 'use myApp\n db.dropRole("test1role")' userAdmin:userAdmin123456@some-name-rs0.demand-backup-2582
+ local 'command=use myApp\n db.dropRole("test1role")'
+ local uri=userAdmin:userAdmin123456@some-name-rs0.demand-backup-2582
+ local driver=mongodb+srv
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local replica_set=rs0
+ [[ userAdmin:userAdmin123456@some-name-rs0.demand-backup-2582 == *cfg* ]]
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.gFAVVOq37b
+++ mktemp
++ local LAST_ERR=/tmp/tmp.kf8mleeBZs
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.gFAVVOq37b
++ cat /tmp/tmp.kf8mleeBZs
++ rm /tmp/tmp.gFAVVOq37b /tmp/tmp.kf8mleeBZs
++ return 0
+ local client_container=psmdb-client-bb8b97679-wjlk6
+ kubectl_bin exec psmdb-client-bb8b97679-wjlk6 -- bash -c 'printf '\''use myApp\n db.dropRole("test1role")\n'\'' | mongo mongodb+srv://userAdmin:userAdmin123456@some-name-rs0.demand-backup-2582.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
++ mktemp
+ local LAST_OUT=/tmp/tmp.tS4wVoRm20
++ mktemp
+ local LAST_ERR=/tmp/tmp.tpgsc86UvX
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-bb8b97679-wjlk6 -- bash -c 'printf '\''use myApp\n db.dropRole("test1role")\n'\'' | mongo mongodb+srv://userAdmin:userAdmin123456@some-name-rs0.demand-backup-2582.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.tS4wVoRm20
Percona Server for MongoDB shell version v4.4.29-28
connecting to: mongodb://some-name-rs0-1.some-name-rs0.demand-backup-2582.svc.cluster.local:27017,some-name-rs0-2.some-name-rs0.demand-backup-2582.svc.cluster.local:27017,some-name-rs0-0.some-name-rs0.demand-backup-2582.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb&replicaSet=rs0&ssl=false
Implicit session: session { "id" : UUID("30a26372-d065-4979-ade0-9d2170e624bd") }
Percona Server for MongoDB server version: v8.0.19-7
WARNING: shell and server versions do not match
switched to db myApp
true
bye
+ cat /tmp/tmp.tpgsc86UvX
+ rm /tmp/tmp.tS4wVoRm20 /tmp/tmp.tpgsc86UvX
+ return 0
+ desc 'selective restore -- minio'
+ set +o xtrace


-----------------------------------------------------------------------------------
selective restore -- minio
-----------------------------------------------------------------------------------

+ run_recovery_check_selective backup-minio some-name-rs0 myApp test2 false test1user test1role
+ local backup=backup-minio
+ local cluster=some-name-rs0
+ local database=myApp
+ local collection=test2
+ local with_users_and_roles=false
+ local username=test1user
+ local role=test1role
+ restore_name=restore-backup-minio-selective
+ run_mongo 'use myApp\n db.test2.drop()' myApp:myPass@some-name-rs0.demand-backup-2582
+ local 'command=use myApp\n db.test2.drop()'
+ local uri=myApp:myPass@some-name-rs0.demand-backup-2582
+ local driver=mongodb+srv
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local replica_set=rs0
+ [[ myApp:myPass@some-name-rs0.demand-backup-2582 == *cfg* ]]
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.zT2BzvtNfM
+++ mktemp
++ local LAST_ERR=/tmp/tmp.Ntw9FCGTno
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.zT2BzvtNfM
++ cat /tmp/tmp.Ntw9FCGTno
++ rm /tmp/tmp.zT2BzvtNfM /tmp/tmp.Ntw9FCGTno
++ return 0
+ local client_container=psmdb-client-bb8b97679-wjlk6
+ kubectl_bin exec psmdb-client-bb8b97679-wjlk6 -- bash -c 'printf '\''use myApp\n db.test2.drop()\n'\'' | mongo mongodb+srv://myApp:myPass@some-name-rs0.demand-backup-2582.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
++ mktemp
+ local LAST_OUT=/tmp/tmp.tM1YZYK33P
++ mktemp
+ local LAST_ERR=/tmp/tmp.7uYpaAaG3O
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-bb8b97679-wjlk6 -- bash -c 'printf '\''use myApp\n db.test2.drop()\n'\'' | mongo mongodb+srv://myApp:myPass@some-name-rs0.demand-backup-2582.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.tM1YZYK33P
Percona Server for MongoDB shell version v4.4.29-28
connecting to: mongodb://some-name-rs0-2.some-name-rs0.demand-backup-2582.svc.cluster.local:27017,some-name-rs0-0.some-name-rs0.demand-backup-2582.svc.cluster.local:27017,some-name-rs0-1.some-name-rs0.demand-backup-2582.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb&replicaSet=rs0&ssl=false
Implicit session: session { "id" : UUID("f08a3bff-86a8-4965-a6e3-db965adec8b2") }
Percona Server for MongoDB server version: v8.0.19-7
WARNING: shell and server versions do not match
switched to db myApp
true
bye
+ cat /tmp/tmp.7uYpaAaG3O
+ rm /tmp/tmp.tM1YZYK33P /tmp/tmp.7uYpaAaG3O
+ return 0
++ collection_exists test2
++ local collection=test2
++ run_mongo 'use myApp\n JSON.stringify(db.getCollectionNames())' myApp:myPass@some-name-rs0.demand-backup-2582 '' '' --quiet
++ local 'command=use myApp\n JSON.stringify(db.getCollectionNames())'
++ local uri=myApp:myPass@some-name-rs0.demand-backup-2582
++ local driver=mongodb+srv
++ local suffix=.svc.cluster.local
++ local mongo_flag=--quiet
++ local replica_set=rs0
++ [[ myApp:myPass@some-name-rs0.demand-backup-2582 == *cfg* ]]
++ grep -v 'switched to'
+++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ jq 'index("test2") != null'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.C23epNZvFj
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.9O81X6w4Qe
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in $(seq 0 2)
+++ set +e
+++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.C23epNZvFj
+++ cat /tmp/tmp.9O81X6w4Qe
+++ rm /tmp/tmp.C23epNZvFj /tmp/tmp.9O81X6w4Qe
+++ return 0
++ local client_container=psmdb-client-bb8b97679-wjlk6
++ kubectl_bin exec psmdb-client-bb8b97679-wjlk6 -- bash -c 'printf '\''use myApp\n JSON.stringify(db.getCollectionNames())\n'\'' | mongo mongodb+srv://myApp:myPass@some-name-rs0.demand-backup-2582.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 --quiet'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.NQkIBDHX88
+++ mktemp
++ local LAST_ERR=/tmp/tmp.84si0fgWJB
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl exec psmdb-client-bb8b97679-wjlk6 -- bash -c 'printf '\''use myApp\n JSON.stringify(db.getCollectionNames())\n'\'' | mongo mongodb+srv://myApp:myPass@some-name-rs0.demand-backup-2582.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 --quiet'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.NQkIBDHX88
++ cat /tmp/tmp.84si0fgWJB
++ rm /tmp/tmp.NQkIBDHX88 /tmp/tmp.84si0fgWJB
++ return 0
+ [[ false == \t\r\u\e ]]
+ yq '.metadata.name="restore-backup-minio-selective"'
+ yq /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2271/e2e-tests/demand-backup/conf/restore.yml
+ yq '.spec.backupName="backup-minio"'
+ yq '.spec.selective.namespaces[0]="myApp.test"'
+ kubectl_bin apply -f -
+ yq .spec.selective.withUsersAndRoles=false
++ mktemp
+ local LAST_OUT=/tmp/tmp.KGYUbL72po
++ mktemp
+ local LAST_ERR=/tmp/tmp.V4QWeZZynl
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl apply -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.KGYUbL72po
perconaservermongodbrestore.psmdb.percona.com/restore-backup-minio-selective created
+ cat /tmp/tmp.V4QWeZZynl
+ rm /tmp/tmp.KGYUbL72po /tmp/tmp.V4QWeZZynl
+ return 0
+ wait_restore backup-minio-selective some-name
+ local backup_name=backup-minio-selective
+ local cluster_name=some-name
+ local target_state=ready
+ local wait_cluster_consistency=1
+ local wait_time=1780
+ local ok_if_ready=0
+ set +o xtrace
Waiting for the psmdb-restore/restore-backup-minio-selective object to be created.OK
Waiting psmdb-restore/restore-backup-minio-selective to reach state "ready" .OK after 0 minutes
+ [[ 1 -eq 1 ]]
+ wait_cluster_consistency some-name
+ local cluster_name=some-name
+ local wait_time=32
+ retry=0
+ sleep 7
+ echo -n 'waiting for cluster readyness'
waiting for cluster readyness++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.mIhyIduANB
+++ mktemp
++ local LAST_ERR=/tmp/tmp.hDEJXRssV9
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.mIhyIduANB
++ cat /tmp/tmp.hDEJXRssV9
++ rm /tmp/tmp.mIhyIduANB /tmp/tmp.hDEJXRssV9
++ return 0
+ [[ ready == \r\e\a\d\y ]]
+ echo .OK
.OK
++ collection_exists test2
++ local collection=test2
++ run_mongo 'use myApp\n JSON.stringify(db.getCollectionNames())' myApp:myPass@some-name-rs0.demand-backup-2582 '' '' --quiet
++ local 'command=use myApp\n JSON.stringify(db.getCollectionNames())'
++ local uri=myApp:myPass@some-name-rs0.demand-backup-2582
++ local driver=mongodb+srv
++ local suffix=.svc.cluster.local
++ local mongo_flag=--quiet
++ local replica_set=rs0
++ [[ myApp:myPass@some-name-rs0.demand-backup-2582 == *cfg* ]]
++ grep -v 'switched to'
++ jq 'index("test2") != null'
+++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.QENQV7ek0G
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.9mJPkCpI3u
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in $(seq 0 2)
+++ set +e
+++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.QENQV7ek0G
+++ cat /tmp/tmp.9mJPkCpI3u
+++ rm /tmp/tmp.QENQV7ek0G /tmp/tmp.9mJPkCpI3u
+++ return 0
++ local client_container=psmdb-client-bb8b97679-wjlk6
++ kubectl_bin exec psmdb-client-bb8b97679-wjlk6 -- bash -c 'printf '\''use myApp\n JSON.stringify(db.getCollectionNames())\n'\'' | mongo mongodb+srv://myApp:myPass@some-name-rs0.demand-backup-2582.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 --quiet'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.8ux4KjRwz4
+++ mktemp
++ local LAST_ERR=/tmp/tmp.HjbRM2xU4R
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl exec psmdb-client-bb8b97679-wjlk6 -- bash -c 'printf '\''use myApp\n JSON.stringify(db.getCollectionNames())\n'\'' | mongo mongodb+srv://myApp:myPass@some-name-rs0.demand-backup-2582.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 --quiet'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.8ux4KjRwz4
++ cat /tmp/tmp.HjbRM2xU4R
++ rm /tmp/tmp.8ux4KjRwz4 /tmp/tmp.HjbRM2xU4R
++ return 0
+ [[ false == \t\r\u\e ]]
+ [[ false == \t\r\u\e ]]
+ [[ false == \f\a\l\s\e ]]
++ user_exists test1user
++ local username=test1user
++ grep -v 'switched to'
++ run_mongo 'use myApp\n JSON.stringify(db.getUsers())' userAdmin:userAdmin123456@some-name-rs0.demand-backup-2582 '' '' --quiet
++ jq 'any(.[]; ._id==myApp.test1user)'
++ local 'command=use myApp\n JSON.stringify(db.getUsers())'
++ local uri=userAdmin:userAdmin123456@some-name-rs0.demand-backup-2582
++ local driver=mongodb+srv
++ local suffix=.svc.cluster.local
++ local mongo_flag=--quiet
++ local replica_set=rs0
++ [[ userAdmin:userAdmin123456@some-name-rs0.demand-backup-2582 == *cfg* ]]
+++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
jq: error: myApp/0 is not defined at <top-level>, line 1:
any(.[]; ._id==myApp.test1user)               
jq: 1 compile error
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.Nt1Yb5I3OI
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.83NRRo3LXC
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in $(seq 0 2)
+++ set +e
+++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.Nt1Yb5I3OI
+++ cat /tmp/tmp.83NRRo3LXC
+++ rm /tmp/tmp.Nt1Yb5I3OI /tmp/tmp.83NRRo3LXC
+++ return 0
++ local client_container=psmdb-client-bb8b97679-wjlk6
++ kubectl_bin exec psmdb-client-bb8b97679-wjlk6 -- bash -c 'printf '\''use myApp\n JSON.stringify(db.getUsers())\n'\'' | mongo mongodb+srv://userAdmin:userAdmin123456@some-name-rs0.demand-backup-2582.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 --quiet'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.JmgjrEoUYU
+++ mktemp
++ local LAST_ERR=/tmp/tmp.JgqdZxvlX5
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl exec psmdb-client-bb8b97679-wjlk6 -- bash -c 'printf '\''use myApp\n JSON.stringify(db.getUsers())\n'\'' | mongo mongodb+srv://userAdmin:userAdmin123456@some-name-rs0.demand-backup-2582.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 --quiet'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.JmgjrEoUYU
++ cat /tmp/tmp.JgqdZxvlX5
++ rm /tmp/tmp.JmgjrEoUYU /tmp/tmp.JgqdZxvlX5
++ return 0
grep: write error: Broken pipe
+ [[ '' == \t\r\u\e ]]
++ role_exists test1role
++ local role=test1role
++ run_mongo 'use myApp\n JSON.stringify(db.getRoles())' userAdmin:userAdmin123456@some-name-rs0.demand-backup-2582 '' '' --quiet
++ local 'command=use myApp\n JSON.stringify(db.getRoles())'
++ local uri=userAdmin:userAdmin123456@some-name-rs0.demand-backup-2582
++ local driver=mongodb+srv
++ local suffix=.svc.cluster.local
++ local mongo_flag=--quiet
++ local replica_set=rs0
++ [[ userAdmin:userAdmin123456@some-name-rs0.demand-backup-2582 == *cfg* ]]
++ grep -v 'switched to'
++ jq 'any(.[]; ._id==myApp.test1role)'
+++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.M8KWFjRYf7
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.P3TBooGNUW
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in $(seq 0 2)
+++ set +e
+++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
jq: error: myApp/0 is not defined at <top-level>, line 1:
any(.[]; ._id==myApp.test1role)               
jq: 1 compile error
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.M8KWFjRYf7
+++ cat /tmp/tmp.P3TBooGNUW
+++ rm /tmp/tmp.M8KWFjRYf7 /tmp/tmp.P3TBooGNUW
+++ return 0
++ local client_container=psmdb-client-bb8b97679-wjlk6
++ kubectl_bin exec psmdb-client-bb8b97679-wjlk6 -- bash -c 'printf '\''use myApp\n JSON.stringify(db.getRoles())\n'\'' | mongo mongodb+srv://userAdmin:userAdmin123456@some-name-rs0.demand-backup-2582.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 --quiet'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.opkNYq6LyG
+++ mktemp
++ local LAST_ERR=/tmp/tmp.nDviMfPC9j
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl exec psmdb-client-bb8b97679-wjlk6 -- bash -c 'printf '\''use myApp\n JSON.stringify(db.getRoles())\n'\'' | mongo mongodb+srv://userAdmin:userAdmin123456@some-name-rs0.demand-backup-2582.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 --quiet'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.opkNYq6LyG
++ cat /tmp/tmp.nDviMfPC9j
++ rm /tmp/tmp.opkNYq6LyG /tmp/tmp.nDviMfPC9j
++ return 0
grep: write error: Broken pipe
+ [[ '' == \t\r\u\e ]]
+ kubectl_bin delete psmdb-restore restore-backup-minio-selective
++ mktemp
+ local LAST_OUT=/tmp/tmp.GHGoWnMnKC
++ mktemp
+ local LAST_ERR=/tmp/tmp.awNkuGJNOS
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl delete psmdb-restore restore-backup-minio-selective
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.GHGoWnMnKC
perconaservermongodbrestore.psmdb.percona.com "restore-backup-minio-selective" deleted from demand-backup-2582 namespace
+ cat /tmp/tmp.awNkuGJNOS
+ rm /tmp/tmp.GHGoWnMnKC /tmp/tmp.awNkuGJNOS
+ return 0
+ run_mongo 'use myApp\n db.test2.insert({ x: 100501 })' myApp:myPass@some-name-rs0.demand-backup-2582
+ local 'command=use myApp\n db.test2.insert({ x: 100501 })'
+ local uri=myApp:myPass@some-name-rs0.demand-backup-2582
+ local driver=mongodb+srv
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local replica_set=rs0
+ [[ myApp:myPass@some-name-rs0.demand-backup-2582 == *cfg* ]]
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.HeUcav16Iq
+++ mktemp
++ local LAST_ERR=/tmp/tmp.fM5CxrRrUl
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.HeUcav16Iq
++ cat /tmp/tmp.fM5CxrRrUl
++ rm /tmp/tmp.HeUcav16Iq /tmp/tmp.fM5CxrRrUl
++ return 0
+ local client_container=psmdb-client-bb8b97679-wjlk6
+ kubectl_bin exec psmdb-client-bb8b97679-wjlk6 -- bash -c 'printf '\''use myApp\n db.test2.insert({ x: 100501 })\n'\'' | mongo mongodb+srv://myApp:myPass@some-name-rs0.demand-backup-2582.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
++ mktemp
+ local LAST_OUT=/tmp/tmp.nWufB27gJN
++ mktemp
+ local LAST_ERR=/tmp/tmp.gvzu0kqzLy
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-bb8b97679-wjlk6 -- bash -c 'printf '\''use myApp\n db.test2.insert({ x: 100501 })\n'\'' | mongo mongodb+srv://myApp:myPass@some-name-rs0.demand-backup-2582.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.nWufB27gJN
Percona Server for MongoDB shell version v4.4.29-28
connecting to: mongodb://some-name-rs0-0.some-name-rs0.demand-backup-2582.svc.cluster.local:27017,some-name-rs0-1.some-name-rs0.demand-backup-2582.svc.cluster.local:27017,some-name-rs0-2.some-name-rs0.demand-backup-2582.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb&replicaSet=rs0&ssl=false
Implicit session: session { "id" : UUID("1ef5c5ea-9793-4a6e-b0a7-a6f81b77bf59") }
Percona Server for MongoDB server version: v8.0.19-7
WARNING: shell and server versions do not match
switched to db myApp
WriteResult({ "nInserted" : 1 })
bye
+ cat /tmp/tmp.gvzu0kqzLy
+ rm /tmp/tmp.nWufB27gJN /tmp/tmp.gvzu0kqzLy
+ return 0
+ desc 'selective restore with users and roles -- minio'
+ set +o xtrace


-----------------------------------------------------------------------------------
selective restore with users and roles -- minio
-----------------------------------------------------------------------------------

+ run_recovery_check_selective backup-minio some-name-rs0 myApp test2 true test1user test1role
+ local backup=backup-minio
+ local cluster=some-name-rs0
+ local database=myApp
+ local collection=test2
+ local with_users_and_roles=true
+ local username=test1user
+ local role=test1role
+ restore_name=restore-backup-minio-selective
+ run_mongo 'use myApp\n db.test2.drop()' myApp:myPass@some-name-rs0.demand-backup-2582
+ local 'command=use myApp\n db.test2.drop()'
+ local uri=myApp:myPass@some-name-rs0.demand-backup-2582
+ local driver=mongodb+srv
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local replica_set=rs0
+ [[ myApp:myPass@some-name-rs0.demand-backup-2582 == *cfg* ]]
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.ON9UG8QTNh
+++ mktemp
++ local LAST_ERR=/tmp/tmp.mS0m3kIgzs
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.ON9UG8QTNh
++ cat /tmp/tmp.mS0m3kIgzs
++ rm /tmp/tmp.ON9UG8QTNh /tmp/tmp.mS0m3kIgzs
++ return 0
+ local client_container=psmdb-client-bb8b97679-wjlk6
+ kubectl_bin exec psmdb-client-bb8b97679-wjlk6 -- bash -c 'printf '\''use myApp\n db.test2.drop()\n'\'' | mongo mongodb+srv://myApp:myPass@some-name-rs0.demand-backup-2582.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
++ mktemp
+ local LAST_OUT=/tmp/tmp.yQOWaAVEt9
++ mktemp
+ local LAST_ERR=/tmp/tmp.eL06EAcJyj
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-bb8b97679-wjlk6 -- bash -c 'printf '\''use myApp\n db.test2.drop()\n'\'' | mongo mongodb+srv://myApp:myPass@some-name-rs0.demand-backup-2582.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.yQOWaAVEt9
Percona Server for MongoDB shell version v4.4.29-28
connecting to: mongodb://some-name-rs0-2.some-name-rs0.demand-backup-2582.svc.cluster.local:27017,some-name-rs0-0.some-name-rs0.demand-backup-2582.svc.cluster.local:27017,some-name-rs0-1.some-name-rs0.demand-backup-2582.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb&replicaSet=rs0&ssl=false
Implicit session: session { "id" : UUID("23ad8c6c-c358-4ab3-93fa-e721b1675841") }
Percona Server for MongoDB server version: v8.0.19-7
WARNING: shell and server versions do not match
switched to db myApp
true
bye
+ cat /tmp/tmp.eL06EAcJyj
+ rm /tmp/tmp.yQOWaAVEt9 /tmp/tmp.eL06EAcJyj
+ return 0
++ collection_exists test2
++ local collection=test2
++ run_mongo 'use myApp\n JSON.stringify(db.getCollectionNames())' myApp:myPass@some-name-rs0.demand-backup-2582 '' '' --quiet
++ local 'command=use myApp\n JSON.stringify(db.getCollectionNames())'
++ local uri=myApp:myPass@some-name-rs0.demand-backup-2582
++ local driver=mongodb+srv
++ local suffix=.svc.cluster.local
++ local mongo_flag=--quiet
++ local replica_set=rs0
++ [[ myApp:myPass@some-name-rs0.demand-backup-2582 == *cfg* ]]
++ grep -v 'switched to'
+++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ jq 'index("test2") != null'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.D4RzL62nqn
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.1vPyXmDN99
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in $(seq 0 2)
+++ set +e
+++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.D4RzL62nqn
+++ cat /tmp/tmp.1vPyXmDN99
+++ rm /tmp/tmp.D4RzL62nqn /tmp/tmp.1vPyXmDN99
+++ return 0
++ local client_container=psmdb-client-bb8b97679-wjlk6
++ kubectl_bin exec psmdb-client-bb8b97679-wjlk6 -- bash -c 'printf '\''use myApp\n JSON.stringify(db.getCollectionNames())\n'\'' | mongo mongodb+srv://myApp:myPass@some-name-rs0.demand-backup-2582.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 --quiet'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.eBok3748EQ
+++ mktemp
++ local LAST_ERR=/tmp/tmp.s0MK1iOleC
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl exec psmdb-client-bb8b97679-wjlk6 -- bash -c 'printf '\''use myApp\n JSON.stringify(db.getCollectionNames())\n'\'' | mongo mongodb+srv://myApp:myPass@some-name-rs0.demand-backup-2582.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 --quiet'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.eBok3748EQ
++ cat /tmp/tmp.s0MK1iOleC
++ rm /tmp/tmp.eBok3748EQ /tmp/tmp.s0MK1iOleC
++ return 0
+ [[ false == \t\r\u\e ]]
+ yq /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2271/e2e-tests/demand-backup/conf/restore.yml
+ yq '.spec.backupName="backup-minio"'
+ yq '.metadata.name="restore-backup-minio-selective"'
+ yq '.spec.selective.namespaces[0]="myApp.test"'
+ yq .spec.selective.withUsersAndRoles=true
+ kubectl_bin apply -f -
++ mktemp
+ local LAST_OUT=/tmp/tmp.jGMF2y64GT
++ mktemp
+ local LAST_ERR=/tmp/tmp.K4x0dCnNd7
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl apply -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.jGMF2y64GT
perconaservermongodbrestore.psmdb.percona.com/restore-backup-minio-selective created
+ cat /tmp/tmp.K4x0dCnNd7
+ rm /tmp/tmp.jGMF2y64GT /tmp/tmp.K4x0dCnNd7
+ return 0
+ wait_restore backup-minio-selective some-name
+ local backup_name=backup-minio-selective
+ local cluster_name=some-name
+ local target_state=ready
+ local wait_cluster_consistency=1
+ local wait_time=1780
+ local ok_if_ready=0
+ set +o xtrace
Waiting for the psmdb-restore/restore-backup-minio-selective object to be created.OK
Waiting psmdb-restore/restore-backup-minio-selective to reach state "ready" .OK after 0 minutes
+ [[ 1 -eq 1 ]]
+ wait_cluster_consistency some-name
+ local cluster_name=some-name
+ local wait_time=32
+ retry=0
+ sleep 7
+ echo -n 'waiting for cluster readyness'
waiting for cluster readyness++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.iML3DnsRlJ
+++ mktemp
++ local LAST_ERR=/tmp/tmp.7iSEwSrnYm
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.iML3DnsRlJ
++ cat /tmp/tmp.7iSEwSrnYm
++ rm /tmp/tmp.iML3DnsRlJ /tmp/tmp.7iSEwSrnYm
++ return 0
+ [[ ready == \r\e\a\d\y ]]
+ echo .OK
.OK
++ collection_exists test2
++ local collection=test2
++ run_mongo 'use myApp\n JSON.stringify(db.getCollectionNames())' myApp:myPass@some-name-rs0.demand-backup-2582 '' '' --quiet
++ local 'command=use myApp\n JSON.stringify(db.getCollectionNames())'
++ local uri=myApp:myPass@some-name-rs0.demand-backup-2582
++ local driver=mongodb+srv
++ local suffix=.svc.cluster.local
++ grep -v 'switched to'
++ local mongo_flag=--quiet
++ local replica_set=rs0
++ [[ myApp:myPass@some-name-rs0.demand-backup-2582 == *cfg* ]]
++ jq 'index("test2") != null'
+++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.gidoPiZZen
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.CFxwzJDbKx
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in $(seq 0 2)
+++ set +e
+++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.gidoPiZZen
+++ cat /tmp/tmp.CFxwzJDbKx
+++ rm /tmp/tmp.gidoPiZZen /tmp/tmp.CFxwzJDbKx
+++ return 0
++ local client_container=psmdb-client-bb8b97679-wjlk6
++ kubectl_bin exec psmdb-client-bb8b97679-wjlk6 -- bash -c 'printf '\''use myApp\n JSON.stringify(db.getCollectionNames())\n'\'' | mongo mongodb+srv://myApp:myPass@some-name-rs0.demand-backup-2582.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 --quiet'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.RZ1sayivKK
+++ mktemp
++ local LAST_ERR=/tmp/tmp.ndu5IjfBGQ
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl exec psmdb-client-bb8b97679-wjlk6 -- bash -c 'printf '\''use myApp\n JSON.stringify(db.getCollectionNames())\n'\'' | mongo mongodb+srv://myApp:myPass@some-name-rs0.demand-backup-2582.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 --quiet'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.RZ1sayivKK
++ cat /tmp/tmp.ndu5IjfBGQ
++ rm /tmp/tmp.RZ1sayivKK /tmp/tmp.ndu5IjfBGQ
++ return 0
+ [[ false == \t\r\u\e ]]
+ [[ true == \t\r\u\e ]]
++ user_exists test1user
++ local username=test1user
++ run_mongo 'use myApp\n JSON.stringify(db.getUsers())' userAdmin:userAdmin123456@some-name-rs0.demand-backup-2582 '' '' --quiet
++ grep -v 'switched to'
++ local 'command=use myApp\n JSON.stringify(db.getUsers())'
++ local uri=userAdmin:userAdmin123456@some-name-rs0.demand-backup-2582
++ local driver=mongodb+srv
++ local suffix=.svc.cluster.local
++ local mongo_flag=--quiet
++ local replica_set=rs0
++ [[ userAdmin:userAdmin123456@some-name-rs0.demand-backup-2582 == *cfg* ]]
++ jq 'any(.[]; ._id==myApp.test1user)'
+++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++++ mktemp
jq: error: myApp/0 is not defined at <top-level>, line 1:
any(.[]; ._id==myApp.test1user)               
jq: 1 compile error
+++ local LAST_OUT=/tmp/tmp.BzkSGTLYlu
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.TobXqZCOIo
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in $(seq 0 2)
+++ set +e
+++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.BzkSGTLYlu
+++ cat /tmp/tmp.TobXqZCOIo
+++ rm /tmp/tmp.BzkSGTLYlu /tmp/tmp.TobXqZCOIo
+++ return 0
++ local client_container=psmdb-client-bb8b97679-wjlk6
++ kubectl_bin exec psmdb-client-bb8b97679-wjlk6 -- bash -c 'printf '\''use myApp\n JSON.stringify(db.getUsers())\n'\'' | mongo mongodb+srv://userAdmin:userAdmin123456@some-name-rs0.demand-backup-2582.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 --quiet'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.kdbrfiWfkE
+++ mktemp
++ local LAST_ERR=/tmp/tmp.Su5P3ZIeqt
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl exec psmdb-client-bb8b97679-wjlk6 -- bash -c 'printf '\''use myApp\n JSON.stringify(db.getUsers())\n'\'' | mongo mongodb+srv://userAdmin:userAdmin123456@some-name-rs0.demand-backup-2582.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 --quiet'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.kdbrfiWfkE
++ cat /tmp/tmp.Su5P3ZIeqt
++ rm /tmp/tmp.kdbrfiWfkE /tmp/tmp.Su5P3ZIeqt
++ return 0
grep: write error: Broken pipe
+ [[ '' == \f\a\l\s\e ]]
++ role_exists test1role
++ local role=test1role
++ grep -v 'switched to'
++ jq 'any(.[]; ._id==myApp.test1role)'
++ run_mongo 'use myApp\n JSON.stringify(db.getRoles())' userAdmin:userAdmin123456@some-name-rs0.demand-backup-2582 '' '' --quiet
++ local 'command=use myApp\n JSON.stringify(db.getRoles())'
++ local uri=userAdmin:userAdmin123456@some-name-rs0.demand-backup-2582
++ local driver=mongodb+srv
++ local suffix=.svc.cluster.local
++ local mongo_flag=--quiet
++ local replica_set=rs0
++ [[ userAdmin:userAdmin123456@some-name-rs0.demand-backup-2582 == *cfg* ]]
+++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
jq: error: myApp/0 is not defined at <top-level>, line 1:
any(.[]; ._id==myApp.test1role)               
jq: 1 compile error
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.QfcU3K5JQv
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.K2k4SK16Rh
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in $(seq 0 2)
+++ set +e
+++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.QfcU3K5JQv
+++ cat /tmp/tmp.K2k4SK16Rh
+++ rm /tmp/tmp.QfcU3K5JQv /tmp/tmp.K2k4SK16Rh
+++ return 0
++ local client_container=psmdb-client-bb8b97679-wjlk6
++ kubectl_bin exec psmdb-client-bb8b97679-wjlk6 -- bash -c 'printf '\''use myApp\n JSON.stringify(db.getRoles())\n'\'' | mongo mongodb+srv://userAdmin:userAdmin123456@some-name-rs0.demand-backup-2582.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 --quiet'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.Jmr8EK0ldD
+++ mktemp
++ local LAST_ERR=/tmp/tmp.7h1RGvDUth
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl exec psmdb-client-bb8b97679-wjlk6 -- bash -c 'printf '\''use myApp\n JSON.stringify(db.getRoles())\n'\'' | mongo mongodb+srv://userAdmin:userAdmin123456@some-name-rs0.demand-backup-2582.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 --quiet'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.Jmr8EK0ldD
++ cat /tmp/tmp.7h1RGvDUth
++ rm /tmp/tmp.Jmr8EK0ldD /tmp/tmp.7h1RGvDUth
++ return 0
grep: write error: Broken pipe
+ [[ '' == \f\a\l\s\e ]]
+ desc 'restore from backup source, with storageName -- minio'
+ set +o xtrace


-----------------------------------------------------------------------------------
restore from backup source, with storageName -- minio
-----------------------------------------------------------------------------------

++ get_backup_dest backup-minio
++ local backup_name=backup-minio
++ kubectl_bin get psmdb-backup backup-minio -o 'jsonpath={.status.destination}'
++ sed -e 's/.json$//'
++ sed 's|s3://||'
+++ mktemp
++ sed 's|azure://||'
++ sed 's|gs://||'
++ local LAST_OUT=/tmp/tmp.TY7SqAFE0X
+++ mktemp
++ local LAST_ERR=/tmp/tmp.pKMcbuC3mO
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb-backup backup-minio -o 'jsonpath={.status.destination}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.TY7SqAFE0X
++ cat /tmp/tmp.pKMcbuC3mO
++ rm /tmp/tmp.TY7SqAFE0X /tmp/tmp.pKMcbuC3mO
++ return 0
+ backup_dest_minio=operator-testing/2026-03-26T08:16:14Z
+ run_recovery_check_bkp_source backup-minio operator-testing/2026-03-26T08:16:14Z some-name-rs0 backup-minio-source-0
+ local backup=backup-minio
+ local backup_dest=operator-testing/2026-03-26T08:16:14Z
+ local cluster=some-name-rs0
+ local source=backup-minio-source-0
+ run_mongo 'use myApp\n db.test.insert({ x: 100501 })' myApp:myPass@some-name-rs0.demand-backup-2582
+ local 'command=use myApp\n db.test.insert({ x: 100501 })'
+ local uri=myApp:myPass@some-name-rs0.demand-backup-2582
+ local driver=mongodb+srv
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local replica_set=rs0
+ [[ myApp:myPass@some-name-rs0.demand-backup-2582 == *cfg* ]]
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.4k4O9x03gb
+++ mktemp
++ local LAST_ERR=/tmp/tmp.n8fE66ryHY
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.4k4O9x03gb
++ cat /tmp/tmp.n8fE66ryHY
++ rm /tmp/tmp.4k4O9x03gb /tmp/tmp.n8fE66ryHY
++ return 0
+ local client_container=psmdb-client-bb8b97679-wjlk6
+ kubectl_bin exec psmdb-client-bb8b97679-wjlk6 -- bash -c 'printf '\''use myApp\n db.test.insert({ x: 100501 })\n'\'' | mongo mongodb+srv://myApp:myPass@some-name-rs0.demand-backup-2582.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
++ mktemp
+ local LAST_OUT=/tmp/tmp.oI10rGuTq6
++ mktemp
+ local LAST_ERR=/tmp/tmp.GC9bPN3SQw
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-bb8b97679-wjlk6 -- bash -c 'printf '\''use myApp\n db.test.insert({ x: 100501 })\n'\'' | mongo mongodb+srv://myApp:myPass@some-name-rs0.demand-backup-2582.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.oI10rGuTq6
Percona Server for MongoDB shell version v4.4.29-28
connecting to: mongodb://some-name-rs0-1.some-name-rs0.demand-backup-2582.svc.cluster.local:27017,some-name-rs0-2.some-name-rs0.demand-backup-2582.svc.cluster.local:27017,some-name-rs0-0.some-name-rs0.demand-backup-2582.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb&replicaSet=rs0&ssl=false
Implicit session: session { "id" : UUID("00ab71b9-8f05-471a-b0f8-45c5e78bf9e8") }
Percona Server for MongoDB server version: v8.0.19-7
WARNING: shell and server versions do not match
switched to db myApp
WriteResult({ "nInserted" : 1 })
bye
+ cat /tmp/tmp.GC9bPN3SQw
+ rm /tmp/tmp.oI10rGuTq6 /tmp/tmp.GC9bPN3SQw
+ return 0
+ compare_mongo_cmd find myApp:myPass@some-name-rs0.demand-backup-2582 -2nd .svc.cluster.local myApp test
+ local command=find
+ local uri=myApp:myPass@some-name-rs0.demand-backup-2582
+ local postfix=-2nd
+ local suffix=.svc.cluster.local
+ local database=myApp
+ local collection=test
+ local sort=
+ local tls=false
+ local replicaset=
+ local 'full_command=db.test.find()'
+ [[ -n '' ]]
+ log 'running db.test.find() in myApp'
+ set +o xtrace
[2026-03-26T08:21:30+0000] running db.test.find() in myApp
+ [[ false == \t\r\u\e ]]
+ mongo_command=run_mongo
+ run_mongo 'use myApp\n db.test.find()' myApp:myPass@some-name-rs0.demand-backup-2582 mongodb .svc.cluster.local ''
+ grep -E -v 'I NETWORK|W NETWORK|F NETWORK|"c":"NETWORK"|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|Started a new thread for the timer service'
+ local 'command=use myApp\n db.test.find()'
+ local uri=myApp:myPass@some-name-rs0.demand-backup-2582
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local replica_set=rs0
+ [[ myApp:myPass@some-name-rs0.demand-backup-2582 == *cfg* ]]
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+ /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.vTwEs8lYm5
+++ mktemp
++ local LAST_ERR=/tmp/tmp.LPJEaVzUzD
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.vTwEs8lYm5
++ cat /tmp/tmp.LPJEaVzUzD
++ rm /tmp/tmp.vTwEs8lYm5 /tmp/tmp.LPJEaVzUzD
++ return 0
+ local client_container=psmdb-client-bb8b97679-wjlk6
+ kubectl_bin exec psmdb-client-bb8b97679-wjlk6 -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0.demand-backup-2582.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
++ mktemp
+ local LAST_OUT=/tmp/tmp.N1FgbvV2GI
++ mktemp
+ local LAST_ERR=/tmp/tmp.4kCzbtDkG2
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-bb8b97679-wjlk6 -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0.demand-backup-2582.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.N1FgbvV2GI
+ cat /tmp/tmp.4kCzbtDkG2
+ rm /tmp/tmp.N1FgbvV2GI /tmp/tmp.4kCzbtDkG2
+ return 0
+ [[ 0 -eq 0 ]]
+ diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2271/e2e-tests/demand-backup/compare/find-2nd.json /tmp/tmp.Fq0pMPZsh8/find-2nd
+ run_restore_backupsource backup-minio-source-0 operator-testing/2026-03-26T08:16:14Z
+ local backupName=backup-minio-source-0
+ local backupDest=operator-testing/2026-03-26T08:16:14Z
+ local storageName=
+ desc 'run restore restore-backup-minio-source-0 from backup backup-minio-source-0 destination is operator-testing/2026-03-26T08:16:14Z'
+ set +o xtrace


-----------------------------------------------------------------------------------
run restore restore-backup-minio-source-0 from backup backup-minio-source-0 destination is operator-testing/2026-03-26T08:16:14Z
-----------------------------------------------------------------------------------

+ '[' -z '' ']'
+ /usr/sbin/sed -e 's/name:/name: restore-backup-minio-source-0/'
+ cat /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2271/e2e-tests/demand-backup/conf/restore-backupsource.yml
+ /usr/sbin/sed -e /storageName/d
+ /usr/sbin/sed -e 's|BACKUP-NAME|operator-testing/2026-03-26T08:16:14Z|'
+ kubectl_bin apply -f -
++ mktemp
+ local LAST_OUT=/tmp/tmp.gsgCBB4JVS
++ mktemp
+ local LAST_ERR=/tmp/tmp.RsBXYpS0QV
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl apply -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.gsgCBB4JVS
perconaservermongodbrestore.psmdb.percona.com/restore-backup-minio-source-0 created
+ cat /tmp/tmp.RsBXYpS0QV
+ rm /tmp/tmp.gsgCBB4JVS /tmp/tmp.RsBXYpS0QV
+ return 0
+ return
+ wait_restore backup-minio-source-0 some-name
+ local backup_name=backup-minio-source-0
+ local cluster_name=some-name
+ local target_state=ready
+ local wait_cluster_consistency=1
+ local wait_time=1780
+ local ok_if_ready=0
+ set +o xtrace
Waiting for the psmdb-restore/restore-backup-minio-source-0 object to be created.OK
Waiting psmdb-restore/restore-backup-minio-source-0 to reach state "ready" .OK after 0 minutes
+ [[ 1 -eq 1 ]]
+ wait_cluster_consistency some-name
+ local cluster_name=some-name
+ local wait_time=32
+ retry=0
+ sleep 7
+ echo -n 'waiting for cluster readyness'
waiting for cluster readyness++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.PP4Ws7rMnP
+++ mktemp
++ local LAST_ERR=/tmp/tmp.2iVmIh183E
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.PP4Ws7rMnP
++ cat /tmp/tmp.2iVmIh183E
++ rm /tmp/tmp.PP4Ws7rMnP /tmp/tmp.2iVmIh183E
++ return 0
+ [[ ready == \r\e\a\d\y ]]
+ echo .OK
.OK
+ simple_data_check some-name-rs0 3 0
+ local cluster_name=some-name-rs0
+ let last_pod=3-1
+ local isSharded=0
+ local cluster_pfx=
+ '[' 0 -eq 1 ']'
++ seq 0 2
+ for i in $(seq 0 $last_pod)
+ compare_mongo_cmd find myApp:myPass@some-name-rs0-0.some-name-rs0.demand-backup-2582
+ local command=find
+ local uri=myApp:myPass@some-name-rs0-0.some-name-rs0.demand-backup-2582
+ local postfix=
+ local suffix=
+ local database=myApp
+ local collection=test
+ local sort=
+ local tls=false
+ local replicaset=
+ local 'full_command=db.test.find()'
+ [[ -n '' ]]
+ log 'running db.test.find() in myApp'
+ set +o xtrace
[2026-03-26T08:21:55+0000] running db.test.find() in myApp
+ [[ false == \t\r\u\e ]]
+ mongo_command=run_mongo
+ run_mongo 'use myApp\n db.test.find()' myApp:myPass@some-name-rs0-0.some-name-rs0.demand-backup-2582 mongodb '' ''
+ local 'command=use myApp\n db.test.find()'
+ local uri=myApp:myPass@some-name-rs0-0.some-name-rs0.demand-backup-2582
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local replica_set=rs0
+ [[ myApp:myPass@some-name-rs0-0.some-name-rs0.demand-backup-2582 == *cfg* ]]
+ grep -E -v 'I NETWORK|W NETWORK|F NETWORK|"c":"NETWORK"|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|Started a new thread for the timer service'
+ /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.4dLclUmVKC
+++ mktemp
++ local LAST_ERR=/tmp/tmp.joJCitudFU
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.4dLclUmVKC
++ cat /tmp/tmp.joJCitudFU
++ rm /tmp/tmp.4dLclUmVKC /tmp/tmp.joJCitudFU
++ return 0
+ local client_container=psmdb-client-bb8b97679-wjlk6
+ kubectl_bin exec psmdb-client-bb8b97679-wjlk6 -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-0.some-name-rs0.demand-backup-2582.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
++ mktemp
+ local LAST_OUT=/tmp/tmp.k1ES8NkMfe
++ mktemp
+ local LAST_ERR=/tmp/tmp.8S162sjrdv
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-bb8b97679-wjlk6 -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-0.some-name-rs0.demand-backup-2582.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.k1ES8NkMfe
+ cat /tmp/tmp.8S162sjrdv
+ rm /tmp/tmp.k1ES8NkMfe /tmp/tmp.8S162sjrdv
+ return 0
+ [[ 0 -eq 0 ]]
+ diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2271/e2e-tests/demand-backup/compare/find.json /tmp/tmp.Fq0pMPZsh8/find
+ for i in $(seq 0 $last_pod)
+ compare_mongo_cmd find myApp:myPass@some-name-rs0-1.some-name-rs0.demand-backup-2582
+ local command=find
+ local uri=myApp:myPass@some-name-rs0-1.some-name-rs0.demand-backup-2582
+ local postfix=
+ local suffix=
+ local database=myApp
+ local collection=test
+ local sort=
+ local tls=false
+ local replicaset=
+ local 'full_command=db.test.find()'
+ [[ -n '' ]]
+ log 'running db.test.find() in myApp'
+ set +o xtrace
[2026-03-26T08:21:57+0000] running db.test.find() in myApp
+ [[ false == \t\r\u\e ]]
+ mongo_command=run_mongo
+ run_mongo 'use myApp\n db.test.find()' myApp:myPass@some-name-rs0-1.some-name-rs0.demand-backup-2582 mongodb '' ''
+ local 'command=use myApp\n db.test.find()'
+ grep -E -v 'I NETWORK|W NETWORK|F NETWORK|"c":"NETWORK"|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|Started a new thread for the timer service'
+ local uri=myApp:myPass@some-name-rs0-1.some-name-rs0.demand-backup-2582
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local replica_set=rs0
+ [[ myApp:myPass@some-name-rs0-1.some-name-rs0.demand-backup-2582 == *cfg* ]]
+ /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.BzqMMlsiie
+++ mktemp
++ local LAST_ERR=/tmp/tmp.0uqUnWFxao
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.BzqMMlsiie
++ cat /tmp/tmp.0uqUnWFxao
++ rm /tmp/tmp.BzqMMlsiie /tmp/tmp.0uqUnWFxao
++ return 0
+ local client_container=psmdb-client-bb8b97679-wjlk6
+ kubectl_bin exec psmdb-client-bb8b97679-wjlk6 -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-1.some-name-rs0.demand-backup-2582.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
++ mktemp
+ local LAST_OUT=/tmp/tmp.P55SilNKKv
++ mktemp
+ local LAST_ERR=/tmp/tmp.tYpgRMOw0j
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-bb8b97679-wjlk6 -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-1.some-name-rs0.demand-backup-2582.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.P55SilNKKv
+ cat /tmp/tmp.tYpgRMOw0j
+ rm /tmp/tmp.P55SilNKKv /tmp/tmp.tYpgRMOw0j
+ return 0
+ [[ 0 -eq 0 ]]
+ diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2271/e2e-tests/demand-backup/compare/find.json /tmp/tmp.Fq0pMPZsh8/find
+ for i in $(seq 0 $last_pod)
+ compare_mongo_cmd find myApp:myPass@some-name-rs0-2.some-name-rs0.demand-backup-2582
+ local command=find
+ local uri=myApp:myPass@some-name-rs0-2.some-name-rs0.demand-backup-2582
+ local postfix=
+ local suffix=
+ local database=myApp
+ local collection=test
+ local sort=
+ local tls=false
+ local replicaset=
+ local 'full_command=db.test.find()'
+ [[ -n '' ]]
+ log 'running db.test.find() in myApp'
+ set +o xtrace
[2026-03-26T08:22:00+0000] running db.test.find() in myApp
+ [[ false == \t\r\u\e ]]
+ mongo_command=run_mongo
+ run_mongo 'use myApp\n db.test.find()' myApp:myPass@some-name-rs0-2.some-name-rs0.demand-backup-2582 mongodb '' ''
+ local 'command=use myApp\n db.test.find()'
+ local uri=myApp:myPass@some-name-rs0-2.some-name-rs0.demand-backup-2582
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local replica_set=rs0
+ [[ myApp:myPass@some-name-rs0-2.some-name-rs0.demand-backup-2582 == *cfg* ]]
+ grep -E -v 'I NETWORK|W NETWORK|F NETWORK|"c":"NETWORK"|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|Started a new thread for the timer service'
+ /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.9hPkqSDP98
+++ mktemp
++ local LAST_ERR=/tmp/tmp.8S7h47lftN
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.9hPkqSDP98
++ cat /tmp/tmp.8S7h47lftN
++ rm /tmp/tmp.9hPkqSDP98 /tmp/tmp.8S7h47lftN
++ return 0
+ local client_container=psmdb-client-bb8b97679-wjlk6
+ kubectl_bin exec psmdb-client-bb8b97679-wjlk6 -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-2.some-name-rs0.demand-backup-2582.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
++ mktemp
+ local LAST_OUT=/tmp/tmp.NLem9VMnx8
++ mktemp
+ local LAST_ERR=/tmp/tmp.bZvHbaRgkb
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-bb8b97679-wjlk6 -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-2.some-name-rs0.demand-backup-2582.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.NLem9VMnx8
+ cat /tmp/tmp.bZvHbaRgkb
+ rm /tmp/tmp.NLem9VMnx8 /tmp/tmp.bZvHbaRgkb
+ return 0
+ [[ 0 -eq 0 ]]
+ diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2271/e2e-tests/demand-backup/compare/find.json /tmp/tmp.Fq0pMPZsh8/find
+ desc 'restore from backup source, no storageName -- minio'
+ set +o xtrace


-----------------------------------------------------------------------------------
restore from backup source, no storageName -- minio
-----------------------------------------------------------------------------------

++ get_backup_dest backup-minio
++ local backup_name=backup-minio
++ kubectl_bin get psmdb-backup backup-minio -o 'jsonpath={.status.destination}'
++ sed -e 's/.json$//'
+++ mktemp
++ sed 's|s3://||'
++ sed 's|azure://||'
++ local LAST_OUT=/tmp/tmp.2L0rpfEHxV
++ sed 's|gs://||'
+++ mktemp
++ local LAST_ERR=/tmp/tmp.eHEBHAjL8j
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb-backup backup-minio -o 'jsonpath={.status.destination}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.2L0rpfEHxV
++ cat /tmp/tmp.eHEBHAjL8j
++ rm /tmp/tmp.2L0rpfEHxV /tmp/tmp.eHEBHAjL8j
++ return 0
+ backup_dest_minio=operator-testing/2026-03-26T08:16:14Z
+ run_recovery_check_bkp_source backup-minio operator-testing/2026-03-26T08:16:14Z some-name-rs0 backup-minio-source-1
+ local backup=backup-minio
+ local backup_dest=operator-testing/2026-03-26T08:16:14Z
+ local cluster=some-name-rs0
+ local source=backup-minio-source-1
+ run_mongo 'use myApp\n db.test.insert({ x: 100501 })' myApp:myPass@some-name-rs0.demand-backup-2582
+ local 'command=use myApp\n db.test.insert({ x: 100501 })'
+ local uri=myApp:myPass@some-name-rs0.demand-backup-2582
+ local driver=mongodb+srv
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local replica_set=rs0
+ [[ myApp:myPass@some-name-rs0.demand-backup-2582 == *cfg* ]]
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.g1DeC6gYEg
+++ mktemp
++ local LAST_ERR=/tmp/tmp.iSjJbgmhbQ
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.g1DeC6gYEg
++ cat /tmp/tmp.iSjJbgmhbQ
++ rm /tmp/tmp.g1DeC6gYEg /tmp/tmp.iSjJbgmhbQ
++ return 0
+ local client_container=psmdb-client-bb8b97679-wjlk6
+ kubectl_bin exec psmdb-client-bb8b97679-wjlk6 -- bash -c 'printf '\''use myApp\n db.test.insert({ x: 100501 })\n'\'' | mongo mongodb+srv://myApp:myPass@some-name-rs0.demand-backup-2582.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
++ mktemp
+ local LAST_OUT=/tmp/tmp.jcN4knhZ5C
++ mktemp
+ local LAST_ERR=/tmp/tmp.Z4ppDs5EYh
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-bb8b97679-wjlk6 -- bash -c 'printf '\''use myApp\n db.test.insert({ x: 100501 })\n'\'' | mongo mongodb+srv://myApp:myPass@some-name-rs0.demand-backup-2582.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.jcN4knhZ5C
Percona Server for MongoDB shell version v4.4.29-28
connecting to: mongodb://some-name-rs0-2.some-name-rs0.demand-backup-2582.svc.cluster.local:27017,some-name-rs0-0.some-name-rs0.demand-backup-2582.svc.cluster.local:27017,some-name-rs0-1.some-name-rs0.demand-backup-2582.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb&replicaSet=rs0&ssl=false
Implicit session: session { "id" : UUID("5deee9b4-94eb-407f-9c70-5d8b014a2b04") }
Percona Server for MongoDB server version: v8.0.19-7
WARNING: shell and server versions do not match
switched to db myApp
WriteResult({ "nInserted" : 1 })
bye
+ cat /tmp/tmp.Z4ppDs5EYh
+ rm /tmp/tmp.jcN4knhZ5C /tmp/tmp.Z4ppDs5EYh
+ return 0
+ compare_mongo_cmd find myApp:myPass@some-name-rs0.demand-backup-2582 -2nd .svc.cluster.local myApp test
+ local command=find
+ local uri=myApp:myPass@some-name-rs0.demand-backup-2582
+ local postfix=-2nd
+ local suffix=.svc.cluster.local
+ local database=myApp
+ local collection=test
+ local sort=
+ local tls=false
+ local replicaset=
+ local 'full_command=db.test.find()'
+ [[ -n '' ]]
+ log 'running db.test.find() in myApp'
+ set +o xtrace
[2026-03-26T08:22:06+0000] running db.test.find() in myApp
+ [[ false == \t\r\u\e ]]
+ mongo_command=run_mongo
+ run_mongo 'use myApp\n db.test.find()' myApp:myPass@some-name-rs0.demand-backup-2582 mongodb .svc.cluster.local ''
+ grep -E -v 'I NETWORK|W NETWORK|F NETWORK|"c":"NETWORK"|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|Started a new thread for the timer service'
+ local 'command=use myApp\n db.test.find()'
+ local uri=myApp:myPass@some-name-rs0.demand-backup-2582
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local replica_set=rs0
+ [[ myApp:myPass@some-name-rs0.demand-backup-2582 == *cfg* ]]
+ /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.ioLavBlhUe
+++ mktemp
++ local LAST_ERR=/tmp/tmp.cXzoDTm5TC
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.ioLavBlhUe
++ cat /tmp/tmp.cXzoDTm5TC
++ rm /tmp/tmp.ioLavBlhUe /tmp/tmp.cXzoDTm5TC
++ return 0
+ local client_container=psmdb-client-bb8b97679-wjlk6
+ kubectl_bin exec psmdb-client-bb8b97679-wjlk6 -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0.demand-backup-2582.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
++ mktemp
+ local LAST_OUT=/tmp/tmp.OXZZPgkAbH
++ mktemp
+ local LAST_ERR=/tmp/tmp.3AhoNKfyAM
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-bb8b97679-wjlk6 -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0.demand-backup-2582.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.OXZZPgkAbH
+ cat /tmp/tmp.3AhoNKfyAM
+ rm /tmp/tmp.OXZZPgkAbH /tmp/tmp.3AhoNKfyAM
+ return 0
+ [[ 0 -eq 0 ]]
+ diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2271/e2e-tests/demand-backup/compare/find-2nd.json /tmp/tmp.Fq0pMPZsh8/find-2nd
+ run_restore_backupsource backup-minio-source-1 operator-testing/2026-03-26T08:16:14Z
+ local backupName=backup-minio-source-1
+ local backupDest=operator-testing/2026-03-26T08:16:14Z
+ local storageName=
+ desc 'run restore restore-backup-minio-source-1 from backup backup-minio-source-1 destination is operator-testing/2026-03-26T08:16:14Z'
+ set +o xtrace


-----------------------------------------------------------------------------------
run restore restore-backup-minio-source-1 from backup backup-minio-source-1 destination is operator-testing/2026-03-26T08:16:14Z
-----------------------------------------------------------------------------------

+ '[' -z '' ']'
+ cat /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2271/e2e-tests/demand-backup/conf/restore-backupsource.yml
+ /usr/sbin/sed -e 's/name:/name: restore-backup-minio-source-1/'
+ /usr/sbin/sed -e 's|BACKUP-NAME|operator-testing/2026-03-26T08:16:14Z|'
+ /usr/sbin/sed -e /storageName/d
+ kubectl_bin apply -f -
++ mktemp
+ local LAST_OUT=/tmp/tmp.g3y5KUx8Th
++ mktemp
+ local LAST_ERR=/tmp/tmp.kZRTbJRJTE
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl apply -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.g3y5KUx8Th
perconaservermongodbrestore.psmdb.percona.com/restore-backup-minio-source-1 created
+ cat /tmp/tmp.kZRTbJRJTE
+ rm /tmp/tmp.g3y5KUx8Th /tmp/tmp.kZRTbJRJTE
+ return 0
+ return
+ wait_restore backup-minio-source-1 some-name
+ local backup_name=backup-minio-source-1
+ local cluster_name=some-name
+ local target_state=ready
+ local wait_cluster_consistency=1
+ local wait_time=1780
+ local ok_if_ready=0
+ set +o xtrace
Waiting for the psmdb-restore/restore-backup-minio-source-1 object to be created.OK
Waiting psmdb-restore/restore-backup-minio-source-1 to reach state "ready" .OK after 0 minutes
+ [[ 1 -eq 1 ]]
+ wait_cluster_consistency some-name
+ local cluster_name=some-name
+ local wait_time=32
+ retry=0
+ sleep 7
+ echo -n 'waiting for cluster readyness'
waiting for cluster readyness++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.ALQBe3qizW
+++ mktemp
++ local LAST_ERR=/tmp/tmp.umFZamNGe0
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.ALQBe3qizW
++ cat /tmp/tmp.umFZamNGe0
++ rm /tmp/tmp.ALQBe3qizW /tmp/tmp.umFZamNGe0
++ return 0
+ [[ ready == \r\e\a\d\y ]]
+ echo .OK
.OK
+ simple_data_check some-name-rs0 3 0
+ local cluster_name=some-name-rs0
+ let last_pod=3-1
+ local isSharded=0
+ local cluster_pfx=
+ '[' 0 -eq 1 ']'
++ seq 0 2
+ for i in $(seq 0 $last_pod)
+ compare_mongo_cmd find myApp:myPass@some-name-rs0-0.some-name-rs0.demand-backup-2582
+ local command=find
+ local uri=myApp:myPass@some-name-rs0-0.some-name-rs0.demand-backup-2582
+ local postfix=
+ local suffix=
+ local database=myApp
+ local collection=test
+ local sort=
+ local tls=false
+ local replicaset=
+ local 'full_command=db.test.find()'
+ [[ -n '' ]]
+ log 'running db.test.find() in myApp'
+ set +o xtrace
[2026-03-26T08:22:31+0000] running db.test.find() in myApp
+ [[ false == \t\r\u\e ]]
+ mongo_command=run_mongo
+ run_mongo 'use myApp\n db.test.find()' myApp:myPass@some-name-rs0-0.some-name-rs0.demand-backup-2582 mongodb '' ''
+ local 'command=use myApp\n db.test.find()'
+ local uri=myApp:myPass@some-name-rs0-0.some-name-rs0.demand-backup-2582
+ local driver=mongodb
+ grep -E -v 'I NETWORK|W NETWORK|F NETWORK|"c":"NETWORK"|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|Started a new thread for the timer service'
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local replica_set=rs0
+ [[ myApp:myPass@some-name-rs0-0.some-name-rs0.demand-backup-2582 == *cfg* ]]
+ /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.uNwXBrV9sk
+++ mktemp
++ local LAST_ERR=/tmp/tmp.TZf2hY6wBn
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.uNwXBrV9sk
++ cat /tmp/tmp.TZf2hY6wBn
++ rm /tmp/tmp.uNwXBrV9sk /tmp/tmp.TZf2hY6wBn
++ return 0
+ local client_container=psmdb-client-bb8b97679-wjlk6
+ kubectl_bin exec psmdb-client-bb8b97679-wjlk6 -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-0.some-name-rs0.demand-backup-2582.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
++ mktemp
+ local LAST_OUT=/tmp/tmp.ayCPQUq0fU
++ mktemp
+ local LAST_ERR=/tmp/tmp.Rwislwfl4n
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-bb8b97679-wjlk6 -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-0.some-name-rs0.demand-backup-2582.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.ayCPQUq0fU
+ cat /tmp/tmp.Rwislwfl4n
+ rm /tmp/tmp.ayCPQUq0fU /tmp/tmp.Rwislwfl4n
+ return 0
+ [[ 0 -eq 0 ]]
+ diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2271/e2e-tests/demand-backup/compare/find.json /tmp/tmp.Fq0pMPZsh8/find
+ for i in $(seq 0 $last_pod)
+ compare_mongo_cmd find myApp:myPass@some-name-rs0-1.some-name-rs0.demand-backup-2582
+ local command=find
+ local uri=myApp:myPass@some-name-rs0-1.some-name-rs0.demand-backup-2582
+ local postfix=
+ local suffix=
+ local database=myApp
+ local collection=test
+ local sort=
+ local tls=false
+ local replicaset=
+ local 'full_command=db.test.find()'
+ [[ -n '' ]]
+ log 'running db.test.find() in myApp'
+ set +o xtrace
[2026-03-26T08:22:34+0000] running db.test.find() in myApp
+ [[ false == \t\r\u\e ]]
+ mongo_command=run_mongo
+ run_mongo 'use myApp\n db.test.find()' myApp:myPass@some-name-rs0-1.some-name-rs0.demand-backup-2582 mongodb '' ''
+ grep -E -v 'I NETWORK|W NETWORK|F NETWORK|"c":"NETWORK"|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|Started a new thread for the timer service'
+ local 'command=use myApp\n db.test.find()'
+ local uri=myApp:myPass@some-name-rs0-1.some-name-rs0.demand-backup-2582
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local replica_set=rs0
+ [[ myApp:myPass@some-name-rs0-1.some-name-rs0.demand-backup-2582 == *cfg* ]]
+ /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.sAjILfQn7F
+++ mktemp
++ local LAST_ERR=/tmp/tmp.8o3lWn7wAy
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.sAjILfQn7F
++ cat /tmp/tmp.8o3lWn7wAy
++ rm /tmp/tmp.sAjILfQn7F /tmp/tmp.8o3lWn7wAy
++ return 0
+ local client_container=psmdb-client-bb8b97679-wjlk6
+ kubectl_bin exec psmdb-client-bb8b97679-wjlk6 -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-1.some-name-rs0.demand-backup-2582.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
++ mktemp
+ local LAST_OUT=/tmp/tmp.Z7jcN4165B
++ mktemp
+ local LAST_ERR=/tmp/tmp.pscPkCafyy
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-bb8b97679-wjlk6 -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-1.some-name-rs0.demand-backup-2582.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.Z7jcN4165B
+ cat /tmp/tmp.pscPkCafyy
+ rm /tmp/tmp.Z7jcN4165B /tmp/tmp.pscPkCafyy
+ return 0
+ [[ 0 -eq 0 ]]
+ diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2271/e2e-tests/demand-backup/compare/find.json /tmp/tmp.Fq0pMPZsh8/find
+ for i in $(seq 0 $last_pod)
+ compare_mongo_cmd find myApp:myPass@some-name-rs0-2.some-name-rs0.demand-backup-2582
+ local command=find
+ local uri=myApp:myPass@some-name-rs0-2.some-name-rs0.demand-backup-2582
+ local postfix=
+ local suffix=
+ local database=myApp
+ local collection=test
+ local sort=
+ local tls=false
+ local replicaset=
+ local 'full_command=db.test.find()'
+ [[ -n '' ]]
+ log 'running db.test.find() in myApp'
+ set +o xtrace
[2026-03-26T08:22:36+0000] running db.test.find() in myApp
+ [[ false == \t\r\u\e ]]
+ mongo_command=run_mongo
+ run_mongo 'use myApp\n db.test.find()' myApp:myPass@some-name-rs0-2.some-name-rs0.demand-backup-2582 mongodb '' ''
+ grep -E -v 'I NETWORK|W NETWORK|F NETWORK|"c":"NETWORK"|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|Started a new thread for the timer service'
+ /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
+ local 'command=use myApp\n db.test.find()'
+ local uri=myApp:myPass@some-name-rs0-2.some-name-rs0.demand-backup-2582
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local replica_set=rs0
+ [[ myApp:myPass@some-name-rs0-2.some-name-rs0.demand-backup-2582 == *cfg* ]]
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.qFmt94pprU
+++ mktemp
++ local LAST_ERR=/tmp/tmp.efwHtcIF0d
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.qFmt94pprU
++ cat /tmp/tmp.efwHtcIF0d
++ rm /tmp/tmp.qFmt94pprU /tmp/tmp.efwHtcIF0d
++ return 0
+ local client_container=psmdb-client-bb8b97679-wjlk6
+ kubectl_bin exec psmdb-client-bb8b97679-wjlk6 -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-2.some-name-rs0.demand-backup-2582.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
++ mktemp
+ local LAST_OUT=/tmp/tmp.bGRUmCXNiq
++ mktemp
+ local LAST_ERR=/tmp/tmp.Hy80NqpLv2
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-bb8b97679-wjlk6 -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-2.some-name-rs0.demand-backup-2582.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.bGRUmCXNiq
+ cat /tmp/tmp.Hy80NqpLv2
+ rm /tmp/tmp.bGRUmCXNiq /tmp/tmp.Hy80NqpLv2
+ return 0
+ [[ 0 -eq 0 ]]
+ diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2271/e2e-tests/demand-backup/compare/find.json /tmp/tmp.Fq0pMPZsh8/find
+ desc 'delete backup and check if it is removed from bucket -- minio'
+ set +o xtrace


-----------------------------------------------------------------------------------
delete backup and check if it is removed from bucket -- minio
-----------------------------------------------------------------------------------

+ kubectl_bin delete psmdb-backup --all
++ mktemp
+ local LAST_OUT=/tmp/tmp.TCtJusHmEL
++ mktemp
+ local LAST_ERR=/tmp/tmp.gtQoqq8m9a
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl delete psmdb-backup --all
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.TCtJusHmEL
perconaservermongodbbackup.psmdb.percona.com "backup-aws-s3" deleted from demand-backup-2582 namespace
perconaservermongodbbackup.psmdb.percona.com "backup-azure-blob" deleted from demand-backup-2582 namespace
perconaservermongodbbackup.psmdb.percona.com "backup-gcp-cs-s3" deleted from demand-backup-2582 namespace
perconaservermongodbbackup.psmdb.percona.com "backup-gcp-cs-sa" deleted from demand-backup-2582 namespace
perconaservermongodbbackup.psmdb.percona.com "backup-minio" deleted from demand-backup-2582 namespace
+ cat /tmp/tmp.gtQoqq8m9a
+ rm /tmp/tmp.TCtJusHmEL /tmp/tmp.gtQoqq8m9a
+ return 0
++ kubectl_bin run -i --rm aws-cli --image=docker.io/perconalab/awscli --restart=Never -- /usr/bin/env AWS_ACCESS_KEY_ID=some-access-key AWS_SECRET_ACCESS_KEY=some-secret-key AWS_DEFAULT_REGION=us-east-1 /usr/bin/aws --endpoint-url http://minio-service:9000 s3 ls s3://operator-testing/
++ grep -c operator-testing/2026-03-26T08:16:14Z
++ cat
+++ mktemp
++ local LAST_OUT=/tmp/tmp.F0FTdHaalO
+++ mktemp
++ local LAST_ERR=/tmp/tmp.fd2Dowu5fy
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl run -i --rm aws-cli --image=docker.io/perconalab/awscli --restart=Never -- /usr/bin/env AWS_ACCESS_KEY_ID=some-access-key AWS_SECRET_ACCESS_KEY=some-secret-key AWS_DEFAULT_REGION=us-east-1 /usr/bin/aws --endpoint-url http://minio-service:9000 s3 ls s3://operator-testing/
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.F0FTdHaalO
++ cat /tmp/tmp.fd2Dowu5fy
All commands and output from this session will be recorded in container logs, including credentials and sensitive information passed through the command prompt.
If you don't see a command prompt, try pressing enter.
warning: couldn't attach to pod/aws-cli, falling back to streaming logs: Internal error occurred: unable to upgrade connection: container aws-cli not found in pod aws-cli_demand-backup-2582
++ rm /tmp/tmp.F0FTdHaalO /tmp/tmp.fd2Dowu5fy
++ return 0
+ backup_exists=0
+ [[ 0 -eq 1 ]]
+ '[' -z '' ']'
+ check_backup_deletion_aws operator-testing/psmdb-demand-backup/2026-03-26T08:15:41Z /rs0/myApp.test.gz
++ echo operator-testing/psmdb-demand-backup/2026-03-26T08:15:41Z
++ cut -d/ -f1
+ bucket=operator-testing
++ echo operator-testing/psmdb-demand-backup/2026-03-26T08:15:41Z
++ cut -d/ -f2-
+ key_prefix=psmdb-demand-backup/2026-03-26T08:15:41Z
+ key=/rs0/myApp.test.gz
+ storage_name=aws-s3
+ retry=0
+ aws s3api head-object --bucket operator-testing --key psmdb-demand-backup/2026-03-26T08:15:41Z/rs0/myApp.test.gz
+ echo 'Backup psmdb-demand-backup/2026-03-26T08:15:41Z/rs0/myApp.test.gz in bucket operator-testing not found in aws-s3'
Backup psmdb-demand-backup/2026-03-26T08:15:41Z/rs0/myApp.test.gz in bucket operator-testing not found in aws-s3
+ check_backup_deletion_gcs operator-testing/psmdb-demand-backup/2026-03-26T08:16:03Z /rs0/myApp.test.gz
+ backup_dest_gcp=operator-testing/psmdb-demand-backup/2026-03-26T08:16:03Z
+ obj=/rs0/myApp.test.gz
+ storage_name=gcp-cs
+ retry=0
+ gcs_path=gs://operator-testing/psmdb-demand-backup/2026-03-26T08:16:03Z/rs0/myApp.test.gz
+ gsutil ls gs://operator-testing/psmdb-demand-backup/2026-03-26T08:16:03Z/rs0/myApp.test.gz
+ echo 'Backup gs://operator-testing/psmdb-demand-backup/2026-03-26T08:16:03Z/rs0/myApp.test.gz not found in gcp-cs'
Backup gs://operator-testing/psmdb-demand-backup/2026-03-26T08:16:03Z/rs0/myApp.test.gz not found in gcp-cs
+ check_backup_deletion_gcs operator-testing/psmdb-demand-backup/2026-03-26T08:16:25Z /rs0/myApp.test.gz
+ backup_dest_gcp=operator-testing/psmdb-demand-backup/2026-03-26T08:16:25Z
+ obj=/rs0/myApp.test.gz
+ storage_name=gcp-cs
+ retry=0
+ gcs_path=gs://operator-testing/psmdb-demand-backup/2026-03-26T08:16:25Z/rs0/myApp.test.gz
+ gsutil ls gs://operator-testing/psmdb-demand-backup/2026-03-26T08:16:25Z/rs0/myApp.test.gz
+ echo 'Backup gs://operator-testing/psmdb-demand-backup/2026-03-26T08:16:25Z/rs0/myApp.test.gz not found in gcp-cs'
Backup gs://operator-testing/psmdb-demand-backup/2026-03-26T08:16:25Z/rs0/myApp.test.gz not found in gcp-cs
+ check_backup_deletion_azure https://engk8soperators.blob.core.windows.net/operator-testing/psmdb-demand-backup/2026-03-26T08:15:52Z /rs0/myApp.test.gz
++ echo https://engk8soperators.blob.core.windows.net/operator-testing/psmdb-demand-backup/2026-03-26T08:15:52Z
++ sed 's|https://[^/]*\.blob\.core\.windows\.net/||'
+ url_path=operator-testing/psmdb-demand-backup/2026-03-26T08:15:52Z
++ cut -d/ -f1
++ echo operator-testing/psmdb-demand-backup/2026-03-26T08:15:52Z
+ container=operator-testing
++ echo operator-testing/psmdb-demand-backup/2026-03-26T08:15:52Z
++ cut -d/ -f2-
+ blob_prefix=psmdb-demand-backup/2026-03-26T08:15:52Z
+ blob=/rs0/myApp.test.gz
+ storage_name=azure-blob
+ retry=0
+ blob_path=psmdb-demand-backup/2026-03-26T08:15:52Z/rs0/myApp.test.gz
+ az storage blob show --container-name operator-testing --name psmdb-demand-backup/2026-03-26T08:15:52Z/rs0/myApp.test.gz
+ echo 'Backup psmdb-demand-backup/2026-03-26T08:15:52Z/rs0/myApp.test.gz in container operator-testing not found in azure-blob'
Backup psmdb-demand-backup/2026-03-26T08:15:52Z/rs0/myApp.test.gz in container operator-testing not found in azure-blob
+ desc 'checking backup deletion without cr'
+ set +o xtrace


-----------------------------------------------------------------------------------
checking backup deletion without cr
-----------------------------------------------------------------------------------

+ run_backup minio
+ local storage=minio
+ local backup_name=backup-minio
+ local type=logical
+ log 'running backup backup-minio'
+ set +o xtrace
[2026-03-26T08:22:55+0000] running backup backup-minio
+ yq eval '.metadata.name = "backup-minio"
			| .spec.storageName = "minio"
			| .spec.type = "logical"' /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2271/e2e-tests/demand-backup/conf/backup-minio.yml
+ kubectl_bin apply -f -
++ mktemp
+ local LAST_OUT=/tmp/tmp.cMgmS1zsIG
++ mktemp
+ local LAST_ERR=/tmp/tmp.EFPgO6nfWL
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl apply -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.cMgmS1zsIG
perconaservermongodbbackup.psmdb.percona.com/backup-minio created
+ cat /tmp/tmp.EFPgO6nfWL
+ rm /tmp/tmp.cMgmS1zsIG /tmp/tmp.EFPgO6nfWL
+ return 0
+ '[' -z '' ']'
+ run_backup aws-s3
+ local storage=aws-s3
+ local backup_name=backup-aws-s3
+ local type=logical
+ log 'running backup backup-aws-s3'
+ set +o xtrace
[2026-03-26T08:22:57+0000] running backup backup-aws-s3
+ yq eval '.metadata.name = "backup-aws-s3"
			| .spec.storageName = "aws-s3"
			| .spec.type = "logical"' /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2271/e2e-tests/demand-backup/conf/backup-aws-s3.yml
+ kubectl_bin apply -f -
++ mktemp
+ local LAST_OUT=/tmp/tmp.GCVcEYoCsV
++ mktemp
+ local LAST_ERR=/tmp/tmp.FCegw61x7W
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl apply -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.GCVcEYoCsV
perconaservermongodbbackup.psmdb.percona.com/backup-aws-s3 created
+ cat /tmp/tmp.FCegw61x7W
+ rm /tmp/tmp.GCVcEYoCsV /tmp/tmp.FCegw61x7W
+ return 0
+ run_backup gcp-cs-s3
+ local storage=gcp-cs-s3
+ local backup_name=backup-gcp-cs-s3
+ local type=logical
+ log 'running backup backup-gcp-cs-s3'
+ set +o xtrace
[2026-03-26T08:22:59+0000] running backup backup-gcp-cs-s3
+ yq eval '.metadata.name = "backup-gcp-cs-s3"
			| .spec.storageName = "gcp-cs-s3"
			| .spec.type = "logical"' /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2271/e2e-tests/demand-backup/conf/backup-gcp-cs-s3.yml
+ kubectl_bin apply -f -
++ mktemp
+ local LAST_OUT=/tmp/tmp.J42ZgeMc3x
++ mktemp
+ local LAST_ERR=/tmp/tmp.P6CdEVHlIO
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl apply -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.J42ZgeMc3x
perconaservermongodbbackup.psmdb.percona.com/backup-gcp-cs-s3 created
+ cat /tmp/tmp.P6CdEVHlIO
+ rm /tmp/tmp.J42ZgeMc3x /tmp/tmp.P6CdEVHlIO
+ return 0
+ run_backup gcp-cs-sa
+ local storage=gcp-cs-sa
+ local backup_name=backup-gcp-cs-sa
+ local type=logical
+ log 'running backup backup-gcp-cs-sa'
+ set +o xtrace
[2026-03-26T08:23:01+0000] running backup backup-gcp-cs-sa
+ yq eval '.metadata.name = "backup-gcp-cs-sa"
			| .spec.storageName = "gcp-cs-sa"
			| .spec.type = "logical"' /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2271/e2e-tests/demand-backup/conf/backup-gcp-cs-sa.yml
+ kubectl_bin apply -f -
++ mktemp
+ local LAST_OUT=/tmp/tmp.1JI7CZbOwi
++ mktemp
+ local LAST_ERR=/tmp/tmp.BWqajhq58c
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl apply -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.1JI7CZbOwi
perconaservermongodbbackup.psmdb.percona.com/backup-gcp-cs-sa created
+ cat /tmp/tmp.BWqajhq58c
+ rm /tmp/tmp.1JI7CZbOwi /tmp/tmp.BWqajhq58c
+ return 0
+ run_backup azure-blob
+ local storage=azure-blob
+ local backup_name=backup-azure-blob
+ local type=logical
+ log 'running backup backup-azure-blob'
+ set +o xtrace
[2026-03-26T08:23:03+0000] running backup backup-azure-blob
+ yq eval '.metadata.name = "backup-azure-blob"
			| .spec.storageName = "azure-blob"
			| .spec.type = "logical"' /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2271/e2e-tests/demand-backup/conf/backup-azure-blob.yml
+ kubectl_bin apply -f -
++ mktemp
+ local LAST_OUT=/tmp/tmp.oG3S00kSfg
++ mktemp
+ local LAST_ERR=/tmp/tmp.fKmttE8mXL
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl apply -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.oG3S00kSfg
perconaservermongodbbackup.psmdb.percona.com/backup-azure-blob created
+ cat /tmp/tmp.fKmttE8mXL
+ rm /tmp/tmp.oG3S00kSfg /tmp/tmp.fKmttE8mXL
+ return 0
+ wait_backup backup-minio
+ local backup_name=backup-minio
+ local target_state=ready
+ set +o xtrace
waiting for backup-minio to reach ready state..OK
+ '[' -z '' ']'
+ wait_backup backup-aws-s3
+ local backup_name=backup-aws-s3
+ local target_state=ready
+ set +o xtrace
waiting for backup-aws-s3 to reach ready state.............OK
+ wait_backup backup-gcp-cs-s3
+ local backup_name=backup-gcp-cs-s3
+ local target_state=ready
+ set +o xtrace
waiting for backup-gcp-cs-s3 to reach ready state............OK
+ wait_backup backup-gcp-cs-sa
+ local backup_name=backup-gcp-cs-sa
+ local target_state=ready
+ set +o xtrace
waiting for backup-gcp-cs-sa to reach ready state..OK
+ wait_backup backup-azure-blob
+ local backup_name=backup-azure-blob
+ local target_state=ready
+ set +o xtrace
waiting for backup-azure-blob to reach ready state..OK
++ get_backup_dest backup-aws-s3
++ local backup_name=backup-aws-s3
++ kubectl_bin get psmdb-backup backup-aws-s3 -o 'jsonpath={.status.destination}'
++ sed -e 's/.json$//'
++ sed 's|s3://||'
++ sed 's|azure://||'
+++ mktemp
++ sed 's|gs://||'
++ local LAST_OUT=/tmp/tmp.jJqNjK4ez2
+++ mktemp
++ local LAST_ERR=/tmp/tmp.L4rq0u6uEL
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb-backup backup-aws-s3 -o 'jsonpath={.status.destination}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.jJqNjK4ez2
++ cat /tmp/tmp.L4rq0u6uEL
++ rm /tmp/tmp.jJqNjK4ez2 /tmp/tmp.L4rq0u6uEL
++ return 0
+ backup_dest_aws=operator-testing/psmdb-demand-backup/2026-03-26T08:23:20Z
++ get_backup_dest backup-gcp-cs-s3
++ local backup_name=backup-gcp-cs-s3
++ sed 's|s3://||'
++ sed 's|gs://||'
++ sed 's|azure://||'
++ sed -e 's/.json$//'
++ kubectl_bin get psmdb-backup backup-gcp-cs-s3 -o 'jsonpath={.status.destination}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.uNcgxZ6qKQ
+++ mktemp
++ local LAST_ERR=/tmp/tmp.Epd35601WZ
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb-backup backup-gcp-cs-s3 -o 'jsonpath={.status.destination}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.uNcgxZ6qKQ
++ cat /tmp/tmp.Epd35601WZ
++ rm /tmp/tmp.uNcgxZ6qKQ /tmp/tmp.Epd35601WZ
++ return 0
+ backup_dest_gcp_s3=operator-testing/psmdb-demand-backup/2026-03-26T08:23:43Z
++ get_backup_dest backup-gcp-cs-sa
++ local backup_name=backup-gcp-cs-sa
++ kubectl_bin get psmdb-backup backup-gcp-cs-sa -o 'jsonpath={.status.destination}'
++ sed -e 's/.json$//'
++ sed 's|s3://||'
+++ mktemp
++ sed 's|azure://||'
++ sed 's|gs://||'
++ local LAST_OUT=/tmp/tmp.UuQryf5NWu
+++ mktemp
++ local LAST_ERR=/tmp/tmp.WA9sbVb5s0
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb-backup backup-gcp-cs-sa -o 'jsonpath={.status.destination}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.UuQryf5NWu
++ cat /tmp/tmp.WA9sbVb5s0
++ rm /tmp/tmp.UuQryf5NWu /tmp/tmp.WA9sbVb5s0
++ return 0
+ backup_dest_gcp_sa=operator-testing/psmdb-demand-backup/2026-03-26T08:23:08Z
++ get_backup_dest backup-azure-blob
++ local backup_name=backup-azure-blob
++ kubectl_bin get psmdb-backup backup-azure-blob -o 'jsonpath={.status.destination}'
++ sed -e 's/.json$//'
++ sed 's|s3://||'
++ sed 's|azure://||'
+++ mktemp
++ sed 's|gs://||'
++ local LAST_OUT=/tmp/tmp.LytjxutfWU
+++ mktemp
++ local LAST_ERR=/tmp/tmp.ZHSzZqYekL
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb-backup backup-azure-blob -o 'jsonpath={.status.destination}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.LytjxutfWU
++ cat /tmp/tmp.ZHSzZqYekL
++ rm /tmp/tmp.LytjxutfWU /tmp/tmp.ZHSzZqYekL
++ return 0
+ backup_dest_azure=https://engk8soperators.blob.core.windows.net/operator-testing/psmdb-demand-backup/2026-03-26T08:23:31Z
+ kubectl_bin delete psmdb --all
++ mktemp
+ local LAST_OUT=/tmp/tmp.HhQvvaV3e9
++ mktemp
+ local LAST_ERR=/tmp/tmp.fGcaB1nsaH
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl delete psmdb --all
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.HhQvvaV3e9
perconaservermongodb.psmdb.percona.com "some-name" deleted from demand-backup-2582 namespace
+ cat /tmp/tmp.fGcaB1nsaH
+ rm /tmp/tmp.HhQvvaV3e9 /tmp/tmp.fGcaB1nsaH
+ return 0
+ sleep 60
+ desc 'delete backup and check if it is removed from bucket -- minio'
+ set +o xtrace


-----------------------------------------------------------------------------------
delete backup and check if it is removed from bucket -- minio
-----------------------------------------------------------------------------------

+ kubectl_bin delete psmdb-backup --all
++ mktemp
+ local LAST_OUT=/tmp/tmp.PrrUcvbz0K
++ mktemp
+ local LAST_ERR=/tmp/tmp.iEHmJORKB5
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl delete psmdb-backup --all
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.PrrUcvbz0K
perconaservermongodbbackup.psmdb.percona.com "backup-aws-s3" deleted from demand-backup-2582 namespace
perconaservermongodbbackup.psmdb.percona.com "backup-azure-blob" deleted from demand-backup-2582 namespace
perconaservermongodbbackup.psmdb.percona.com "backup-gcp-cs-s3" deleted from demand-backup-2582 namespace
perconaservermongodbbackup.psmdb.percona.com "backup-gcp-cs-sa" deleted from demand-backup-2582 namespace
perconaservermongodbbackup.psmdb.percona.com "backup-minio" deleted from demand-backup-2582 namespace
+ cat /tmp/tmp.iEHmJORKB5
+ rm /tmp/tmp.PrrUcvbz0K /tmp/tmp.iEHmJORKB5
+ return 0
++ kubectl_bin run -i --rm aws-cli --image=docker.io/perconalab/awscli --restart=Never -- /usr/bin/env AWS_ACCESS_KEY_ID=some-access-key AWS_SECRET_ACCESS_KEY=some-secret-key AWS_DEFAULT_REGION=us-east-1 /usr/bin/aws --endpoint-url http://minio-service:9000 s3 ls s3://operator-testing/
++ grep -c operator-testing/2026-03-26T08:16:14Z
++ cat
+++ mktemp
++ local LAST_OUT=/tmp/tmp.XJt4t29ron
+++ mktemp
++ local LAST_ERR=/tmp/tmp.2KMIEUInOk
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl run -i --rm aws-cli --image=docker.io/perconalab/awscli --restart=Never -- /usr/bin/env AWS_ACCESS_KEY_ID=some-access-key AWS_SECRET_ACCESS_KEY=some-secret-key AWS_DEFAULT_REGION=us-east-1 /usr/bin/aws --endpoint-url http://minio-service:9000 s3 ls s3://operator-testing/
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.XJt4t29ron
++ cat /tmp/tmp.2KMIEUInOk
++ rm /tmp/tmp.XJt4t29ron /tmp/tmp.2KMIEUInOk
++ return 0
+ backup_exists=0
+ [[ 0 -eq 1 ]]
+ '[' -z '' ']'
+ check_backup_deletion_aws operator-testing/psmdb-demand-backup/2026-03-26T08:23:20Z /rs0/myApp.test.gz
++ echo operator-testing/psmdb-demand-backup/2026-03-26T08:23:20Z
++ cut -d/ -f1
+ bucket=operator-testing
++ echo operator-testing/psmdb-demand-backup/2026-03-26T08:23:20Z
++ cut -d/ -f2-
+ key_prefix=psmdb-demand-backup/2026-03-26T08:23:20Z
+ key=/rs0/myApp.test.gz
+ storage_name=aws-s3
+ retry=0
+ aws s3api head-object --bucket operator-testing --key psmdb-demand-backup/2026-03-26T08:23:20Z/rs0/myApp.test.gz
+ echo 'Backup psmdb-demand-backup/2026-03-26T08:23:20Z/rs0/myApp.test.gz in bucket operator-testing not found in aws-s3'
Backup psmdb-demand-backup/2026-03-26T08:23:20Z/rs0/myApp.test.gz in bucket operator-testing not found in aws-s3
+ check_backup_deletion_gcs operator-testing/psmdb-demand-backup/2026-03-26T08:23:43Z /rs0/myApp.test.gz
+ backup_dest_gcp=operator-testing/psmdb-demand-backup/2026-03-26T08:23:43Z
+ obj=/rs0/myApp.test.gz
+ storage_name=gcp-cs
+ retry=0
+ gcs_path=gs://operator-testing/psmdb-demand-backup/2026-03-26T08:23:43Z/rs0/myApp.test.gz
+ gsutil ls gs://operator-testing/psmdb-demand-backup/2026-03-26T08:23:43Z/rs0/myApp.test.gz
+ echo 'Backup gs://operator-testing/psmdb-demand-backup/2026-03-26T08:23:43Z/rs0/myApp.test.gz not found in gcp-cs'
Backup gs://operator-testing/psmdb-demand-backup/2026-03-26T08:23:43Z/rs0/myApp.test.gz not found in gcp-cs
+ check_backup_deletion_gcs operator-testing/psmdb-demand-backup/2026-03-26T08:23:08Z /rs0/myApp.test.gz
+ backup_dest_gcp=operator-testing/psmdb-demand-backup/2026-03-26T08:23:08Z
+ obj=/rs0/myApp.test.gz
+ storage_name=gcp-cs
+ retry=0
+ gcs_path=gs://operator-testing/psmdb-demand-backup/2026-03-26T08:23:08Z/rs0/myApp.test.gz
+ gsutil ls gs://operator-testing/psmdb-demand-backup/2026-03-26T08:23:08Z/rs0/myApp.test.gz
+ echo 'Backup gs://operator-testing/psmdb-demand-backup/2026-03-26T08:23:08Z/rs0/myApp.test.gz not found in gcp-cs'
Backup gs://operator-testing/psmdb-demand-backup/2026-03-26T08:23:08Z/rs0/myApp.test.gz not found in gcp-cs
+ check_backup_deletion_azure https://engk8soperators.blob.core.windows.net/operator-testing/psmdb-demand-backup/2026-03-26T08:23:31Z /rs0/myApp.test.gz
++ echo https://engk8soperators.blob.core.windows.net/operator-testing/psmdb-demand-backup/2026-03-26T08:23:31Z
++ sed 's|https://[^/]*\.blob\.core\.windows\.net/||'
+ url_path=operator-testing/psmdb-demand-backup/2026-03-26T08:23:31Z
++ echo operator-testing/psmdb-demand-backup/2026-03-26T08:23:31Z
++ cut -d/ -f1
+ container=operator-testing
++ cut -d/ -f2-
++ echo operator-testing/psmdb-demand-backup/2026-03-26T08:23:31Z
+ blob_prefix=psmdb-demand-backup/2026-03-26T08:23:31Z
+ blob=/rs0/myApp.test.gz
+ storage_name=azure-blob
+ retry=0
+ blob_path=psmdb-demand-backup/2026-03-26T08:23:31Z/rs0/myApp.test.gz
+ az storage blob show --container-name operator-testing --name psmdb-demand-backup/2026-03-26T08:23:31Z/rs0/myApp.test.gz
+ echo 'Backup psmdb-demand-backup/2026-03-26T08:23:31Z/rs0/myApp.test.gz in container operator-testing not found in azure-blob'
Backup psmdb-demand-backup/2026-03-26T08:23:31Z/rs0/myApp.test.gz in container operator-testing not found in azure-blob
+ destroy demand-backup-2582
+ local namespace=demand-backup-2582
+ local ignore_logs=true
+ [[ 0 == 1 ]]
+ desc 'destroy cluster/operator and all other resources'
+ set +o xtrace


-----------------------------------------------------------------------------------
destroy cluster/operator and all other resources
-----------------------------------------------------------------------------------

+ '[' true == false ']'
+ delete_backups
+ desc 'Delete psmdb-backup'
+ set +o xtrace


-----------------------------------------------------------------------------------
Delete psmdb-backup
-----------------------------------------------------------------------------------

++ kubectl_bin get psmdb-backup --no-headers
++ wc -l
+++ mktemp
++ local LAST_OUT=/tmp/tmp.ge8RY2k9lk
+++ mktemp
++ local LAST_ERR=/tmp/tmp.OYBY6429WH
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb-backup --no-headers
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.ge8RY2k9lk
++ cat /tmp/tmp.OYBY6429WH
No resources found in demand-backup-2582 namespace.
++ rm /tmp/tmp.ge8RY2k9lk /tmp/tmp.OYBY6429WH
++ return 0
+ '[' 0 '!=' 0 ']'
+ delete_crd
+ desc 'get and delete old CRDs and RBAC'
+ set +o xtrace


-----------------------------------------------------------------------------------
get and delete old CRDs and RBAC
-----------------------------------------------------------------------------------

+ kubectl_bin delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2271/deploy/crd.yaml --ignore-not-found --wait=false
++ mktemp
+ local LAST_OUT=/tmp/tmp.q3VTMLzUgb
++ mktemp
+ local LAST_ERR=/tmp/tmp.w3eJ9bBbZ3
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2271/deploy/crd.yaml --ignore-not-found --wait=false
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.q3VTMLzUgb
customresourcedefinition.apiextensions.k8s.io "perconaservermongodbbackups.psmdb.percona.com" deleted
customresourcedefinition.apiextensions.k8s.io "perconaservermongodbrestores.psmdb.percona.com" deleted
customresourcedefinition.apiextensions.k8s.io "perconaservermongodbs.psmdb.percona.com" deleted
+ cat /tmp/tmp.w3eJ9bBbZ3
+ rm /tmp/tmp.q3VTMLzUgb /tmp/tmp.w3eJ9bBbZ3
+ return 0
++ yq eval .metadata.name /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2271/deploy/crd.yaml
++ grep -v '\-\-\-'
grep: warning: stray \ before -
grep: warning: stray \ before -
+ for crd_name in $(yq eval '.metadata.name' "${src_dir}/deploy/crd.yaml" | grep -v '\-\-\-')
+ kubectl get perconaservermongodbbackups.psmdb.percona.com --all-namespaces -o wide
+ grep -v NAMESPACE
+ xargs -L 1 sh -xc 'kubectl patch perconaservermongodbbackups.psmdb.percona.com -n $0 $1 --type=merge -p "{\"metadata\":{\"finalizers\":[]}}"'
error: the server doesn't have a resource type "perconaservermongodbbackups"
+ kubectl patch perconaservermongodbbackups.psmdb.percona.com -n sh --type=merge -p '{"metadata":{"finalizers":[]}}'
error: the server doesn't have a resource type "perconaservermongodbbackups"
+ :
+ kubectl_bin wait --for=delete crd perconaservermongodbbackups.psmdb.percona.com
++ mktemp
+ local LAST_OUT=/tmp/tmp.TKze6YeBMd
++ mktemp
+ local LAST_ERR=/tmp/tmp.Jjsrrwxkun
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl wait --for=delete crd perconaservermongodbbackups.psmdb.percona.com
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.TKze6YeBMd
+ cat /tmp/tmp.Jjsrrwxkun
+ rm /tmp/tmp.TKze6YeBMd /tmp/tmp.Jjsrrwxkun
+ return 0
+ for crd_name in $(yq eval '.metadata.name' "${src_dir}/deploy/crd.yaml" | grep -v '\-\-\-')
+ kubectl get perconaservermongodbrestores.psmdb.percona.com --all-namespaces -o wide
+ grep -v NAMESPACE
+ xargs -L 1 sh -xc 'kubectl patch perconaservermongodbrestores.psmdb.percona.com -n $0 $1 --type=merge -p "{\"metadata\":{\"finalizers\":[]}}"'
error: the server doesn't have a resource type "perconaservermongodbrestores"
+ kubectl patch perconaservermongodbrestores.psmdb.percona.com -n sh --type=merge -p '{"metadata":{"finalizers":[]}}'
error: the server doesn't have a resource type "perconaservermongodbrestores"
+ :
+ kubectl_bin wait --for=delete crd perconaservermongodbrestores.psmdb.percona.com
++ mktemp
+ local LAST_OUT=/tmp/tmp.9b7yPKuXSW
++ mktemp
+ local LAST_ERR=/tmp/tmp.nL1ZP7f6kP
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl wait --for=delete crd perconaservermongodbrestores.psmdb.percona.com
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.9b7yPKuXSW
+ cat /tmp/tmp.nL1ZP7f6kP
+ rm /tmp/tmp.9b7yPKuXSW /tmp/tmp.nL1ZP7f6kP
+ return 0
+ for crd_name in $(yq eval '.metadata.name' "${src_dir}/deploy/crd.yaml" | grep -v '\-\-\-')
+ kubectl get perconaservermongodbs.psmdb.percona.com --all-namespaces -o wide
+ grep -v NAMESPACE
+ xargs -L 1 sh -xc 'kubectl patch perconaservermongodbs.psmdb.percona.com -n $0 $1 --type=merge -p "{\"metadata\":{\"finalizers\":[]}}"'
error: the server doesn't have a resource type "perconaservermongodbs"
+ kubectl patch perconaservermongodbs.psmdb.percona.com -n sh --type=merge -p '{"metadata":{"finalizers":[]}}'
error: the server doesn't have a resource type "perconaservermongodbs"
+ :
+ kubectl_bin wait --for=delete crd perconaservermongodbs.psmdb.percona.com
++ mktemp
+ local LAST_OUT=/tmp/tmp.8FcWq3ypxW
++ mktemp
+ local LAST_ERR=/tmp/tmp.W4ahrGpVz1
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl wait --for=delete crd perconaservermongodbs.psmdb.percona.com
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.8FcWq3ypxW
+ cat /tmp/tmp.W4ahrGpVz1
+ rm /tmp/tmp.8FcWq3ypxW /tmp/tmp.W4ahrGpVz1
+ return 0
+ local rbac_yaml=rbac.yaml
+ '[' -n psmdb-operator ']'
+ rbac_yaml=cw-rbac.yaml
+ kubectl_bin delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2271/deploy/cw-rbac.yaml --ignore-not-found
++ mktemp
+ local LAST_OUT=/tmp/tmp.qtWvK0Pq4t
++ mktemp
+ local LAST_ERR=/tmp/tmp.ti7XxwXZhK
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2271/deploy/cw-rbac.yaml --ignore-not-found
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.qtWvK0Pq4t
clusterrole.rbac.authorization.k8s.io "percona-server-mongodb-operator" deleted
clusterrolebinding.rbac.authorization.k8s.io "service-account-percona-server-mongodb-operator" deleted
+ cat /tmp/tmp.ti7XxwXZhK
+ rm /tmp/tmp.qtWvK0Pq4t /tmp/tmp.ti7XxwXZhK
+ return 0
+ destroy_cert_manager
+ kubectl_bin delete -f https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml
++ mktemp
+ local LAST_OUT=/tmp/tmp.UwblwQX7il
++ mktemp
+ local LAST_ERR=/tmp/tmp.UeyaxQOHUf
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl delete -f https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml
+ exit_status=1
+ set -e
+ '[' 1 '!=' 0 -a -n 1 ']'
+ cat /tmp/tmp.UwblwQX7il
+ cat /tmp/tmp.UeyaxQOHUf
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": namespaces "cert-manager" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "challenges.acme.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "orders.acme.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "certificaterequests.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "certificates.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "clusterissuers.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "issuers.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": serviceaccounts "cert-manager-cainjector" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": serviceaccounts "cert-manager" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": serviceaccounts "cert-manager-webhook" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-cainjector" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-issuers" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-clusterissuers" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-certificates" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-orders" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-challenges" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-ingress-shim" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-cluster-view" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-view" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-edit" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-approve:cert-manager-io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-certificatesigningrequests" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-webhook:subjectaccessreviews" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-cainjector" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-issuers" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-clusterissuers" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-certificates" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-orders" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-challenges" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-ingress-shim" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-approve:cert-manager-io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-certificatesigningrequests" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-webhook:subjectaccessreviews" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager-cainjector:leaderelection" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager:leaderelection" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager-tokenrequest" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager-webhook:dynamic-serving" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager-cainjector:leaderelection" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager:leaderelection" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager-tokenrequest" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager-webhook:dynamic-serving" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": services "cert-manager-cainjector" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": services "cert-manager" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": services "cert-manager-webhook" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": deployments.apps "cert-manager-cainjector" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": deployments.apps "cert-manager" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": deployments.apps "cert-manager-webhook" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": mutatingwebhookconfigurations.admissionregistration.k8s.io "cert-manager-webhook" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": validatingwebhookconfigurations.admissionregistration.k8s.io "cert-manager-webhook" not found
+ sleep 0
+ for i in $(seq 0 2)
+ set +e
+ kubectl delete -f https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml
+ exit_status=1
+ set -e
+ '[' 1 '!=' 0 -a -n 1 ']'
+ cat /tmp/tmp.UwblwQX7il
+ cat /tmp/tmp.UeyaxQOHUf
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": namespaces "cert-manager" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "challenges.acme.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "orders.acme.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "certificaterequests.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "certificates.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "clusterissuers.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "issuers.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": serviceaccounts "cert-manager-cainjector" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": serviceaccounts "cert-manager" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": serviceaccounts "cert-manager-webhook" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-cainjector" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-issuers" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-clusterissuers" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-certificates" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-orders" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-challenges" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-ingress-shim" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-cluster-view" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-view" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-edit" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-approve:cert-manager-io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-certificatesigningrequests" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-webhook:subjectaccessreviews" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-cainjector" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-issuers" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-clusterissuers" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-certificates" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-orders" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-challenges" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-ingress-shim" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-approve:cert-manager-io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-certificatesigningrequests" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-webhook:subjectaccessreviews" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager-cainjector:leaderelection" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager:leaderelection" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager-tokenrequest" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager-webhook:dynamic-serving" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager-cainjector:leaderelection" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager:leaderelection" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager-tokenrequest" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager-webhook:dynamic-serving" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": services "cert-manager-cainjector" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": services "cert-manager" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": services "cert-manager-webhook" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": deployments.apps "cert-manager-cainjector" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": deployments.apps "cert-manager" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": deployments.apps "cert-manager-webhook" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": mutatingwebhookconfigurations.admissionregistration.k8s.io "cert-manager-webhook" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": validatingwebhookconfigurations.admissionregistration.k8s.io "cert-manager-webhook" not found
+ sleep 4
+ for i in $(seq 0 2)
+ set +e
+ kubectl delete -f https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml
+ exit_status=1
+ set -e
+ '[' 1 '!=' 0 -a -n 1 ']'
+ cat /tmp/tmp.UwblwQX7il
+ cat /tmp/tmp.UeyaxQOHUf
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": namespaces "cert-manager" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "challenges.acme.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "orders.acme.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "certificaterequests.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "certificates.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "clusterissuers.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "issuers.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": serviceaccounts "cert-manager-cainjector" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": serviceaccounts "cert-manager" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": serviceaccounts "cert-manager-webhook" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-cainjector" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-issuers" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-clusterissuers" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-certificates" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-orders" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-challenges" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-ingress-shim" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-cluster-view" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-view" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-edit" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-approve:cert-manager-io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-certificatesigningrequests" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-webhook:subjectaccessreviews" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-cainjector" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-issuers" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-clusterissuers" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-certificates" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-orders" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-challenges" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-ingress-shim" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-approve:cert-manager-io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-certificatesigningrequests" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-webhook:subjectaccessreviews" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager-cainjector:leaderelection" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager:leaderelection" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager-tokenrequest" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager-webhook:dynamic-serving" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager-cainjector:leaderelection" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager:leaderelection" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager-tokenrequest" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager-webhook:dynamic-serving" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": services "cert-manager-cainjector" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": services "cert-manager" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": services "cert-manager-webhook" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": deployments.apps "cert-manager-cainjector" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": deployments.apps "cert-manager" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": deployments.apps "cert-manager-webhook" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": mutatingwebhookconfigurations.admissionregistration.k8s.io "cert-manager-webhook" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": validatingwebhookconfigurations.admissionregistration.k8s.io "cert-manager-webhook" not found
+ sleep 8
+ cat /tmp/tmp.UwblwQX7il
+ cat /tmp/tmp.UeyaxQOHUf
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": namespaces "cert-manager" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "challenges.acme.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "orders.acme.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "certificaterequests.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "certificates.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "clusterissuers.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "issuers.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": serviceaccounts "cert-manager-cainjector" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": serviceaccounts "cert-manager" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": serviceaccounts "cert-manager-webhook" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-cainjector" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-issuers" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-clusterissuers" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-certificates" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-orders" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-challenges" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-ingress-shim" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-cluster-view" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-view" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-edit" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-approve:cert-manager-io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-certificatesigningrequests" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-webhook:subjectaccessreviews" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-cainjector" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-issuers" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-clusterissuers" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-certificates" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-orders" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-challenges" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-ingress-shim" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-approve:cert-manager-io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-certificatesigningrequests" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-webhook:subjectaccessreviews" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager-cainjector:leaderelection" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager:leaderelection" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager-tokenrequest" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager-webhook:dynamic-serving" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager-cainjector:leaderelection" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager:leaderelection" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager-tokenrequest" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager-webhook:dynamic-serving" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": services "cert-manager-cainjector" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": services "cert-manager" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": services "cert-manager-webhook" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": deployments.apps "cert-manager-cainjector" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": deployments.apps "cert-manager" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": deployments.apps "cert-manager-webhook" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": mutatingwebhookconfigurations.admissionregistration.k8s.io "cert-manager-webhook" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": validatingwebhookconfigurations.admissionregistration.k8s.io "cert-manager-webhook" not found
+ rm /tmp/tmp.UwblwQX7il /tmp/tmp.UeyaxQOHUf
+ return 1
+ true
+ '[' -n '' ']'
+ '[' -n psmdb-operator ']'
+ kubectl_bin delete --grace-period=0 --force=true namespace demand-backup-2582
+ rm -rf /tmp/tmp.Fq0pMPZsh8
+ kubectl_bin delete --grace-period=0 --force=true namespace psmdb-operator
++ mktemp
++ mktemp
+ local LAST_OUT=/tmp/tmp.wk6KiTRgsT
+ local LAST_OUT=/tmp/tmp.nqaKcQVPUg
++ mktemp
+ desc 'test passed'
+ set +o xtrace
++ mktemp


-----------------------------------------------------------------------------------
test passed
-----------------------------------------------------------------------------------

+ local LAST_ERR=/tmp/tmp.W2rSaqvOyz
+ local exit_status=0
+ local timeout=4
+ local LAST_ERR=/tmp/tmp.zcmhgs1ZL9
+ local exit_status=0
+ local timeout=4
++ seq 0 2
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl delete --grace-period=0 --force=true namespace psmdb-operator
+ for i in $(seq 0 2)
+ set +e
+ kubectl delete --grace-period=0 --force=true namespace demand-backup-2582