Log: /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2283/e2e-tests/logs/demand-backup.log
Warning: version difference between client (1.35) and server (1.32) exceeds the supported minor version skew of +/-1
Warning: version difference between client (1.35) and server (1.32) exceeds the supported minor version skew of +/-1
Warning: version difference between client (1.35) and server (1.32) exceeds the supported minor version skew of +/-1
+ create_infra demand-backup-15110
+ local ns=demand-backup-15110
+ [[ 1 == 1 ]]
+ delete_crd
+ desc 'get and delete old CRDs and RBAC'
+ set +o xtrace


-----------------------------------------------------------------------------------
get and delete old CRDs and RBAC
-----------------------------------------------------------------------------------

+ kubectl_bin delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2283/deploy/crd.yaml --ignore-not-found --wait=false
++ mktemp
+ local LAST_OUT=/tmp/tmp.frVpBovGjP
++ mktemp
+ local LAST_ERR=/tmp/tmp.FjkftM6MEF
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2283/deploy/crd.yaml --ignore-not-found --wait=false
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.frVpBovGjP
customresourcedefinition.apiextensions.k8s.io "perconaservermongodbbackups.psmdb.percona.com" deleted
customresourcedefinition.apiextensions.k8s.io "perconaservermongodbrestores.psmdb.percona.com" deleted
customresourcedefinition.apiextensions.k8s.io "perconaservermongodbs.psmdb.percona.com" deleted
+ cat /tmp/tmp.FjkftM6MEF
+ rm /tmp/tmp.frVpBovGjP /tmp/tmp.FjkftM6MEF
+ return 0
++ yq eval .metadata.name /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2283/deploy/crd.yaml
++ grep -v '\-\-\-'
grep: warning: stray \ before -
grep: warning: stray \ before -
+ for crd_name in $(yq eval '.metadata.name' "${src_dir}/deploy/crd.yaml" | grep -v '\-\-\-')
+ kubectl get perconaservermongodbbackups.psmdb.percona.com --all-namespaces -o wide
+ grep -v NAMESPACE
+ xargs -L 1 sh -xc 'kubectl patch perconaservermongodbbackups.psmdb.percona.com -n $0 $1 --type=merge -p "{\"metadata\":{\"finalizers\":[]}}"'
+ kubectl patch perconaservermongodbbackups.psmdb.percona.com -n demand-backup-21473 backup-aws-s3 --type=merge -p '{"metadata":{"finalizers":[]}}'
perconaservermongodbbackup.psmdb.percona.com/backup-aws-s3 patched
+ kubectl patch perconaservermongodbbackups.psmdb.percona.com -n demand-backup-21473 backup-azure-blob --type=merge -p '{"metadata":{"finalizers":[]}}'
perconaservermongodbbackup.psmdb.percona.com/backup-azure-blob patched
+ kubectl patch perconaservermongodbbackups.psmdb.percona.com -n demand-backup-21473 backup-gcp-cs-s3 --type=merge -p '{"metadata":{"finalizers":[]}}'
perconaservermongodbbackup.psmdb.percona.com/backup-gcp-cs-s3 patched
+ kubectl patch perconaservermongodbbackups.psmdb.percona.com -n demand-backup-21473 backup-gcp-cs-sa --type=merge -p '{"metadata":{"finalizers":[]}}'
perconaservermongodbbackup.psmdb.percona.com/backup-gcp-cs-sa patched
+ kubectl patch perconaservermongodbbackups.psmdb.percona.com -n demand-backup-21473 backup-minio --type=merge -p '{"metadata":{"finalizers":[]}}'
perconaservermongodbbackup.psmdb.percona.com/backup-minio patched
+ kubectl_bin wait --for=delete crd perconaservermongodbbackups.psmdb.percona.com
++ mktemp
+ local LAST_OUT=/tmp/tmp.JaR2It8Umd
++ mktemp
+ local LAST_ERR=/tmp/tmp.3zEg7P3EUH
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl wait --for=delete crd perconaservermongodbbackups.psmdb.percona.com
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.JaR2It8Umd
customresourcedefinition.apiextensions.k8s.io/perconaservermongodbbackups.psmdb.percona.com condition met
+ cat /tmp/tmp.3zEg7P3EUH
+ rm /tmp/tmp.JaR2It8Umd /tmp/tmp.3zEg7P3EUH
+ return 0
+ for crd_name in $(yq eval '.metadata.name' "${src_dir}/deploy/crd.yaml" | grep -v '\-\-\-')
+ kubectl get perconaservermongodbrestores.psmdb.percona.com --all-namespaces -o wide
+ grep -v NAMESPACE
+ xargs -L 1 sh -xc 'kubectl patch perconaservermongodbrestores.psmdb.percona.com -n $0 $1 --type=merge -p "{\"metadata\":{\"finalizers\":[]}}"'
error: the server doesn't have a resource type "perconaservermongodbrestores"
+ kubectl patch perconaservermongodbrestores.psmdb.percona.com -n sh --type=merge -p '{"metadata":{"finalizers":[]}}'
error: the server doesn't have a resource type "perconaservermongodbrestores"
+ :
+ kubectl_bin wait --for=delete crd perconaservermongodbrestores.psmdb.percona.com
++ mktemp
+ local LAST_OUT=/tmp/tmp.e7gJ0O8Qcu
++ mktemp
+ local LAST_ERR=/tmp/tmp.eka5oPYJSJ
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl wait --for=delete crd perconaservermongodbrestores.psmdb.percona.com
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.e7gJ0O8Qcu
+ cat /tmp/tmp.eka5oPYJSJ
+ rm /tmp/tmp.e7gJ0O8Qcu /tmp/tmp.eka5oPYJSJ
+ return 0
+ for crd_name in $(yq eval '.metadata.name' "${src_dir}/deploy/crd.yaml" | grep -v '\-\-\-')
+ kubectl get perconaservermongodbs.psmdb.percona.com --all-namespaces -o wide
+ grep -v NAMESPACE
+ xargs -L 1 sh -xc 'kubectl patch perconaservermongodbs.psmdb.percona.com -n $0 $1 --type=merge -p "{\"metadata\":{\"finalizers\":[]}}"'
error: the server doesn't have a resource type "perconaservermongodbs"
+ kubectl patch perconaservermongodbs.psmdb.percona.com -n sh --type=merge -p '{"metadata":{"finalizers":[]}}'
error: the server doesn't have a resource type "perconaservermongodbs"
+ :
+ kubectl_bin wait --for=delete crd perconaservermongodbs.psmdb.percona.com
++ mktemp
+ local LAST_OUT=/tmp/tmp.XPZWouc820
++ mktemp
+ local LAST_ERR=/tmp/tmp.jND0tkRfTv
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl wait --for=delete crd perconaservermongodbs.psmdb.percona.com
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.XPZWouc820
+ cat /tmp/tmp.jND0tkRfTv
+ rm /tmp/tmp.XPZWouc820 /tmp/tmp.jND0tkRfTv
+ return 0
+ local rbac_yaml=rbac.yaml
+ '[' -n psmdb-operator ']'
+ rbac_yaml=cw-rbac.yaml
+ kubectl_bin delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2283/deploy/cw-rbac.yaml --ignore-not-found
++ mktemp
+ local LAST_OUT=/tmp/tmp.L65amhBqH1
++ mktemp
+ local LAST_ERR=/tmp/tmp.IXyTNUWtke
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2283/deploy/cw-rbac.yaml --ignore-not-found
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.L65amhBqH1
clusterrole.rbac.authorization.k8s.io "percona-server-mongodb-operator" deleted
clusterrolebinding.rbac.authorization.k8s.io "service-account-percona-server-mongodb-operator" deleted
+ cat /tmp/tmp.IXyTNUWtke
+ rm /tmp/tmp.L65amhBqH1 /tmp/tmp.IXyTNUWtke
+ return 0
+ check_crd_for_deletion PR-2283-e32dd8b3
+ local git_tag=PR-2283-e32dd8b3
++ curl -s https://raw.githubusercontent.com/percona/percona-server-mongodb-operator/PR-2283-e32dd8b3/deploy/crd.yaml
++ yq eval .metadata.name
++ /usr/sbin/sed ':a;N;$!ba;s/\n/ /g'
++ /usr/sbin/sed s/---//g
+ for crd_name in $(curl -s https://raw.githubusercontent.com/percona/percona-server-mongodb-operator/${git_tag}/deploy/crd.yaml | yq eval '.metadata.name' | $sed 's/---//g' | $sed ':a;N;$!ba;s/\n/ /g')
++ kubectl_bin get crd/null -o 'jsonpath={.status.conditions[-1].type}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.c5T08t3Uju
+++ mktemp
++ local LAST_ERR=/tmp/tmp.PfaArJvc9U
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get crd/null -o 'jsonpath={.status.conditions[-1].type}'
++ exit_status=1
++ set -e
++ '[' 1 '!=' 0 -a -n 1 ']'
++ cat /tmp/tmp.c5T08t3Uju
++ cat /tmp/tmp.PfaArJvc9U
Error from server (NotFound): customresourcedefinitions.apiextensions.k8s.io "null" not found
++ sleep 0
++ for i in $(seq 0 2)
++ set +e
++ kubectl get crd/null -o 'jsonpath={.status.conditions[-1].type}'
++ exit_status=1
++ set -e
++ '[' 1 '!=' 0 -a -n 1 ']'
++ cat /tmp/tmp.c5T08t3Uju
++ cat /tmp/tmp.PfaArJvc9U
Error from server (NotFound): customresourcedefinitions.apiextensions.k8s.io "null" not found
++ sleep 4
++ for i in $(seq 0 2)
++ set +e
++ kubectl get crd/null -o 'jsonpath={.status.conditions[-1].type}'
++ exit_status=1
++ set -e
++ '[' 1 '!=' 0 -a -n 1 ']'
++ cat /tmp/tmp.c5T08t3Uju
++ cat /tmp/tmp.PfaArJvc9U
Error from server (NotFound): customresourcedefinitions.apiextensions.k8s.io "null" not found
++ sleep 8
++ cat /tmp/tmp.c5T08t3Uju
++ cat /tmp/tmp.PfaArJvc9U
Error from server (NotFound): customresourcedefinitions.apiextensions.k8s.io "null" not found
++ rm /tmp/tmp.c5T08t3Uju /tmp/tmp.PfaArJvc9U
++ return 1
+ [[ '' == \T\e\r\m\i\n\a\t\i\n\g ]]
+ '[' -n psmdb-operator ']'
+ create_namespace psmdb-operator
+ local namespace=psmdb-operator
+ local skip_clean_namespace=
+ [[ 1 == 1 ]]
+ [[ -z '' ]]
+ destroy_chaos_mesh
++ helm list --all-namespaces --filter chaos-mesh
++ tail -n1
++ awk '-F ' '{print $2}'
++ sed s/NAMESPACE//
+ local chaos_mesh_ns=
+ desc 'destroy chaos-mesh'
+ set +o xtrace


-----------------------------------------------------------------------------------
destroy chaos-mesh
-----------------------------------------------------------------------------------

+ '[' -n '' ']'
++ kubectl get MutatingWebhookConfiguration
++ awk '{print $1}'
++ grep chaos-mesh
+ timeout 30 kubectl delete MutatingWebhookConfiguration
error: resource(s) were provided, but no name was specified
+ :
++ kubectl get ValidatingWebhookConfiguration
++ grep chaos-mesh
++ awk '{print $1}'
+ timeout 30 kubectl delete ValidatingWebhookConfiguration
error: resource(s) were provided, but no name was specified
+ :
++ kubectl get ValidatingWebhookConfiguration
++ grep validate-auth
++ awk '{print $1}'
+ timeout 30 kubectl delete ValidatingWebhookConfiguration
error: resource(s) were provided, but no name was specified
+ :
++ kubectl api-resources
++ grep chaos-mesh
++ awk '{print $1}'
++ kubectl get crd
++ grep chaos-mesh.org
++ awk '{print $1}'
+ timeout 30 kubectl delete crd
error: resource(s) were provided, but no name was specified
+ :
++ kubectl get clusterrolebinding
++ grep chaos-mesh
++ awk '{print $1}'
+ timeout 30 kubectl delete clusterrolebinding
error: resource(s) were provided, but no name was specified
+ :
++ grep chaos-mesh
++ kubectl get clusterrole
++ awk '{print $1}'
+ timeout 30 kubectl delete clusterrole
error: resource(s) were provided, but no name was specified
+ :
+ desc 'cleaned up all old namespaces'
+ set +o xtrace


-----------------------------------------------------------------------------------
cleaned up all old namespaces
-----------------------------------------------------------------------------------

+ kubectl_bin get ns
+ awk '{print$1}'
++ mktemp
+ '[' -n '' ']'
+ desc 'cleaned up old namespaces psmdb-operator'
+ set +o xtrace


-----------------------------------------------------------------------------------
cleaned up old namespaces psmdb-operator
-----------------------------------------------------------------------------------

+ kubectl_bin delete namespace psmdb-operator --ignore-not-found
++ mktemp
+ grep -E -v '^kube-|^default|Terminating|psmdb-operator|openshift|^gke-|^gmp-|^NAME'
+ local LAST_OUT=/tmp/tmp.uhED3oJaVg
++ mktemp
+ xargs kubectl delete ns
+ local LAST_OUT=/tmp/tmp.rtSrAg2dt5
++ mktemp
+ local LAST_ERR=/tmp/tmp.dhbDFsm8DN
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ local LAST_ERR=/tmp/tmp.uILAY8f4SJ
+ for i in $(seq 0 2)
+ set +e
+ local exit_status=0
+ kubectl get ns
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl delete namespace psmdb-operator --ignore-not-found
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.uhED3oJaVg
+ cat /tmp/tmp.dhbDFsm8DN
+ rm /tmp/tmp.uhED3oJaVg /tmp/tmp.dhbDFsm8DN
+ return 0
namespace "demand-backup-21473" deleted
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.rtSrAg2dt5
namespace "psmdb-operator" deleted
+ cat /tmp/tmp.uILAY8f4SJ
+ rm /tmp/tmp.rtSrAg2dt5 /tmp/tmp.uILAY8f4SJ
+ return 0
+ kubectl_bin wait --for=delete namespace psmdb-operator
++ mktemp
+ local LAST_OUT=/tmp/tmp.zOBm0CLuE0
++ mktemp
+ local LAST_ERR=/tmp/tmp.1QdgyYxdwj
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl wait --for=delete namespace psmdb-operator
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.zOBm0CLuE0
+ cat /tmp/tmp.1QdgyYxdwj
+ rm /tmp/tmp.zOBm0CLuE0 /tmp/tmp.1QdgyYxdwj
+ return 0
+ desc 'create namespace psmdb-operator'
+ set +o xtrace


-----------------------------------------------------------------------------------
create namespace psmdb-operator
-----------------------------------------------------------------------------------

+ kubectl_bin create namespace psmdb-operator
++ mktemp
+ local LAST_OUT=/tmp/tmp.kxVcb0XHFD
++ mktemp
+ local LAST_ERR=/tmp/tmp.MOCBNu7guH
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl create namespace psmdb-operator
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.kxVcb0XHFD
namespace/psmdb-operator created
+ cat /tmp/tmp.MOCBNu7guH
+ rm /tmp/tmp.kxVcb0XHFD /tmp/tmp.MOCBNu7guH
+ return 0
+ set_kube_ctx psmdb-operator
+ local namespace=psmdb-operator
++ kubectl_bin config current-context
+++ mktemp
++ local LAST_OUT=/tmp/tmp.Uml0O72bCj
+++ mktemp
++ local LAST_ERR=/tmp/tmp.1Eq8CXXIpx
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl config current-context
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.Uml0O72bCj
++ cat /tmp/tmp.1Eq8CXXIpx
++ rm /tmp/tmp.Uml0O72bCj /tmp/tmp.1Eq8CXXIpx
++ return 0
+ kubectl_bin config set-context gke_cloud-dev-112233_us-central1-a_jen-psmdb-2283-e32dd8b3-7-cluster10 --namespace=psmdb-operator
++ mktemp
+ local LAST_OUT=/tmp/tmp.ZIJUZBI2p7
++ mktemp
+ local LAST_ERR=/tmp/tmp.fM6WKHsKzG
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl config set-context gke_cloud-dev-112233_us-central1-a_jen-psmdb-2283-e32dd8b3-7-cluster10 --namespace=psmdb-operator
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.ZIJUZBI2p7
Context "gke_cloud-dev-112233_us-central1-a_jen-psmdb-2283-e32dd8b3-7-cluster10" modified.
+ cat /tmp/tmp.fM6WKHsKzG
+ rm /tmp/tmp.ZIJUZBI2p7 /tmp/tmp.fM6WKHsKzG
+ return 0
+ deploy_operator
+ desc 'start PSMDB operator: docker.io/perconalab/percona-server-mongodb-operator:PR-2283-e32dd8b3'
+ set +o xtrace


-----------------------------------------------------------------------------------
start PSMDB operator: docker.io/perconalab/percona-server-mongodb-operator:PR-2283-e32dd8b3
-----------------------------------------------------------------------------------

+ local cr_file
+ '[' -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2283/e2e-tests/demand-backup/conf/crd.yaml ']'
+ cr_file=/mnt/jenkins/workspace/cloud-psmdb-operator_PR-2283/deploy/crd.yaml
+ kubectl_bin apply --server-side --force-conflicts -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2283/deploy/crd.yaml
++ mktemp
+ local LAST_OUT=/tmp/tmp.Wak1LokTgJ
++ mktemp
+ local LAST_ERR=/tmp/tmp.ROE8f5dTwo
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl apply --server-side --force-conflicts -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2283/deploy/crd.yaml
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.Wak1LokTgJ
customresourcedefinition.apiextensions.k8s.io/perconaservermongodbbackups.psmdb.percona.com serverside-applied
customresourcedefinition.apiextensions.k8s.io/perconaservermongodbrestores.psmdb.percona.com serverside-applied
customresourcedefinition.apiextensions.k8s.io/perconaservermongodbs.psmdb.percona.com serverside-applied
+ cat /tmp/tmp.ROE8f5dTwo
+ rm /tmp/tmp.Wak1LokTgJ /tmp/tmp.ROE8f5dTwo
+ return 0
+ '[' -n psmdb-operator ']'
+ apply_rbac cw-rbac
+ local operator_namespace=psmdb-operator
+ local rbac=cw-rbac
+ cat /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2283/deploy/cw-rbac.yaml
+ sed -e 's^namespace: .*^namespace: psmdb-operator^'
+ kubectl_bin apply -n psmdb-operator -f -
++ mktemp
+ local LAST_OUT=/tmp/tmp.xU3Grd6tRc
++ mktemp
+ local LAST_ERR=/tmp/tmp.qrCjBMQxC1
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl apply -n psmdb-operator -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.xU3Grd6tRc
clusterrole.rbac.authorization.k8s.io/percona-server-mongodb-operator created
serviceaccount/percona-server-mongodb-operator created
clusterrolebinding.rbac.authorization.k8s.io/service-account-percona-server-mongodb-operator created
+ cat /tmp/tmp.qrCjBMQxC1
+ rm /tmp/tmp.xU3Grd6tRc /tmp/tmp.qrCjBMQxC1
+ return 0
+ yq eval '
			(.spec.template.spec.containers[].image = "docker.io/perconalab/percona-server-mongodb-operator:PR-2283-e32dd8b3") |
			((.. | select(.[] == "DISABLE_TELEMETRY")) |= .value="true") |
			((.. | select(.[] == "LOG_LEVEL")) |= .value="DEBUG")' /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2283/deploy/cw-operator.yaml
+ kubectl_bin apply -n psmdb-operator -f -
++ mktemp
+ local LAST_OUT=/tmp/tmp.8RPUhM77de
++ mktemp
+ local LAST_ERR=/tmp/tmp.QpEv5F6lk4
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl apply -n psmdb-operator -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.8RPUhM77de
deployment.apps/percona-server-mongodb-operator created
+ cat /tmp/tmp.QpEv5F6lk4
+ rm /tmp/tmp.8RPUhM77de /tmp/tmp.QpEv5F6lk4
+ return 0
+ sleep 20
++ get_operator_pod
++ kubectl_bin get pods --selector=name=percona-server-mongodb-operator -o 'jsonpath={.items[].metadata.name}' -n psmdb-operator
+++ mktemp
++ local LAST_OUT=/tmp/tmp.ckb0OEt5O1
+++ mktemp
++ local LAST_ERR=/tmp/tmp.qkOU4EooSD
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=percona-server-mongodb-operator -o 'jsonpath={.items[].metadata.name}' -n psmdb-operator
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.ckb0OEt5O1
++ cat /tmp/tmp.qkOU4EooSD
++ rm /tmp/tmp.ckb0OEt5O1 /tmp/tmp.qkOU4EooSD
++ return 0
+ wait_operator_pod percona-server-mongodb-operator-57ff5689b7-zn7ls
+ local pod=percona-server-mongodb-operator-57ff5689b7-zn7ls
+ set +o xtrace
waiting for pod/percona-server-mongodb-operator-57ff5689b7-zn7ls to be ready.OK
+ echo 'Print operator info from log'
Print operator info from log
+ grep 'Manager starting up'
++ get_operator_pod
++ kubectl_bin get pods --selector=name=percona-server-mongodb-operator -o 'jsonpath={.items[].metadata.name}' -n psmdb-operator
+++ mktemp
++ local LAST_OUT=/tmp/tmp.S4Vhuo7DJ4
+++ mktemp
++ local LAST_ERR=/tmp/tmp.LYe3oB8l5c
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=percona-server-mongodb-operator -o 'jsonpath={.items[].metadata.name}' -n psmdb-operator
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.S4Vhuo7DJ4
++ cat /tmp/tmp.LYe3oB8l5c
++ rm /tmp/tmp.S4Vhuo7DJ4 /tmp/tmp.LYe3oB8l5c
++ return 0
+ kubectl_bin logs -n psmdb-operator percona-server-mongodb-operator-57ff5689b7-zn7ls
++ mktemp
+ local LAST_OUT=/tmp/tmp.kODauoG7pU
++ mktemp
+ local LAST_ERR=/tmp/tmp.Bkj6KCmY8C
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl logs -n psmdb-operator percona-server-mongodb-operator-57ff5689b7-zn7ls
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.kODauoG7pU
+ cat /tmp/tmp.Bkj6KCmY8C
+ rm /tmp/tmp.kODauoG7pU /tmp/tmp.Bkj6KCmY8C
+ return 0
2026-03-27T11:05:50.312Z	INFO	setup	Manager starting up	{"gitCommit": "e32dd8b338bc8af11fb1e90bb80d577a1c65923f", "gitBranch": "PR-2283-e32dd8b3", "buildTime": "", "goVersion": "go1.25.8", "os": "linux", "arch": "amd64"}
+ create_namespace demand-backup-15110
+ local namespace=demand-backup-15110
+ local skip_clean_namespace=
+ [[ 1 == 1 ]]
+ [[ -z '' ]]
+ destroy_chaos_mesh
++ helm list --all-namespaces --filter chaos-mesh
++ tail -n1
++ awk '-F ' '{print $2}'
++ sed s/NAMESPACE//
+ local chaos_mesh_ns=
+ desc 'destroy chaos-mesh'
+ set +o xtrace


-----------------------------------------------------------------------------------
destroy chaos-mesh
-----------------------------------------------------------------------------------

+ '[' -n '' ']'
++ kubectl get MutatingWebhookConfiguration
++ grep chaos-mesh
++ awk '{print $1}'
+ timeout 30 kubectl delete MutatingWebhookConfiguration
error: resource(s) were provided, but no name was specified
+ :
++ kubectl get ValidatingWebhookConfiguration
++ grep chaos-mesh
++ awk '{print $1}'
+ timeout 30 kubectl delete ValidatingWebhookConfiguration
error: resource(s) were provided, but no name was specified
+ :
++ kubectl get ValidatingWebhookConfiguration
++ grep validate-auth
++ awk '{print $1}'
+ timeout 30 kubectl delete ValidatingWebhookConfiguration
error: resource(s) were provided, but no name was specified
+ :
++ kubectl api-resources
++ grep chaos-mesh
++ awk '{print $1}'
++ kubectl get crd
++ awk '{print $1}'
++ grep chaos-mesh.org
+ timeout 30 kubectl delete crd
error: resource(s) were provided, but no name was specified
+ :
++ kubectl get clusterrolebinding
++ grep chaos-mesh
++ awk '{print $1}'
+ timeout 30 kubectl delete clusterrolebinding
error: resource(s) were provided, but no name was specified
+ :
++ kubectl get clusterrole
++ grep chaos-mesh
++ awk '{print $1}'
+ timeout 30 kubectl delete clusterrole
error: resource(s) were provided, but no name was specified
+ :
+ desc 'cleaned up all old namespaces'
+ set +o xtrace


-----------------------------------------------------------------------------------
cleaned up all old namespaces
-----------------------------------------------------------------------------------

+ kubectl_bin get ns
+ grep -E -v '^kube-|^default|Terminating|psmdb-operator|openshift|^gke-|^gmp-|^NAME'
++ mktemp
+ xargs kubectl delete ns
+ '[' -n '' ']'
+ desc 'cleaned up old namespaces demand-backup-15110'
+ set +o xtrace


-----------------------------------------------------------------------------------
cleaned up old namespaces demand-backup-15110
-----------------------------------------------------------------------------------

+ kubectl_bin delete namespace demand-backup-15110 --ignore-not-found
+ local LAST_OUT=/tmp/tmp.EU3aKLgF1v
+ awk '{print$1}'
++ mktemp
+ local LAST_ERR=/tmp/tmp.j4ZfPGBYOR
+ local exit_status=0
+ local timeout=4
++ mktemp
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl get ns
+ local LAST_OUT=/tmp/tmp.IMTSmgXxhf
++ mktemp
+ local LAST_ERR=/tmp/tmp.Up4QJ9GTdJ
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl delete namespace demand-backup-15110 --ignore-not-found
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.EU3aKLgF1v
+ cat /tmp/tmp.j4ZfPGBYOR
+ rm /tmp/tmp.EU3aKLgF1v /tmp/tmp.j4ZfPGBYOR
+ return 0
error: resource(s) were provided, but no name was specified
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.IMTSmgXxhf
+ cat /tmp/tmp.Up4QJ9GTdJ
+ rm /tmp/tmp.IMTSmgXxhf /tmp/tmp.Up4QJ9GTdJ
+ return 0
+ kubectl_bin wait --for=delete namespace demand-backup-15110
++ mktemp
+ local LAST_OUT=/tmp/tmp.t4VfOQ0CIs
++ mktemp
+ local LAST_ERR=/tmp/tmp.Vwwec7ZrLi
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl wait --for=delete namespace demand-backup-15110
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.t4VfOQ0CIs
+ cat /tmp/tmp.Vwwec7ZrLi
+ rm /tmp/tmp.t4VfOQ0CIs /tmp/tmp.Vwwec7ZrLi
+ return 0
+ desc 'create namespace demand-backup-15110'
+ set +o xtrace


-----------------------------------------------------------------------------------
create namespace demand-backup-15110
-----------------------------------------------------------------------------------

+ kubectl_bin create namespace demand-backup-15110
++ mktemp
+ local LAST_OUT=/tmp/tmp.LTXj1RHhgk
++ mktemp
+ local LAST_ERR=/tmp/tmp.iw5ugZDvf4
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl create namespace demand-backup-15110
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.LTXj1RHhgk
namespace/demand-backup-15110 created
+ cat /tmp/tmp.iw5ugZDvf4
+ rm /tmp/tmp.LTXj1RHhgk /tmp/tmp.iw5ugZDvf4
+ return 0
+ set_kube_ctx demand-backup-15110
+ local namespace=demand-backup-15110
++ kubectl_bin config current-context
+++ mktemp
++ local LAST_OUT=/tmp/tmp.K1kWTf7FQf
+++ mktemp
++ local LAST_ERR=/tmp/tmp.UnGJJeH96d
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl config current-context
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.K1kWTf7FQf
++ cat /tmp/tmp.UnGJJeH96d
++ rm /tmp/tmp.K1kWTf7FQf /tmp/tmp.UnGJJeH96d
++ return 0
+ kubectl_bin config set-context gke_cloud-dev-112233_us-central1-a_jen-psmdb-2283-e32dd8b3-7-cluster10 --namespace=demand-backup-15110
++ mktemp
+ local LAST_OUT=/tmp/tmp.IY6mROYQkv
++ mktemp
+ local LAST_ERR=/tmp/tmp.KzbLdnEzTn
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl config set-context gke_cloud-dev-112233_us-central1-a_jen-psmdb-2283-e32dd8b3-7-cluster10 --namespace=demand-backup-15110
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.IY6mROYQkv
Context "gke_cloud-dev-112233_us-central1-a_jen-psmdb-2283-e32dd8b3-7-cluster10" modified.
+ cat /tmp/tmp.KzbLdnEzTn
+ rm /tmp/tmp.IY6mROYQkv /tmp/tmp.KzbLdnEzTn
+ return 0
+ deploy_minio
+ local cert_secret=
+ local service_name=minio-service
+ desc 'install MinIO: minio-service'
+ set +o xtrace


-----------------------------------------------------------------------------------
install MinIO: minio-service
-----------------------------------------------------------------------------------

+ helm uninstall minio-service
+ :
+ helm repo remove minio
"minio" has been removed from your repositories
+ helm repo add minio https://charts.min.io/
"minio" has been added to your repositories
+ local endpoint=http://minio-service:9000
+ minio_args=('--version' '5.4.0' '--set' 'replicas=1' '--set' 'mode=standalone' '--set' 'resources.requests.memory=256Mi' '--set' 'rootUser=rootuser' '--set' 'rootPassword=rootpass123' '--set' 'users[0].accessKey=some-access-key' '--set' 'users[0].secretKey=some-secret-key' '--set' 'users[0].policy=consoleAdmin' '--set' 'service.type=ClusterIP' '--set' 'configPathmc=/tmp/' '--set' 'securityContext.enabled=false' '--set' 'persistence.size=2G' '--set' 'fullnameOverride=minio-service' '--set' 'serviceAccount.create=true' '--set' 'serviceAccount.name=minio-service-sa')
+ local minio_args
+ [[ -n '' ]]
+ retry 10 60 helm install minio-service --version 5.4.0 --set replicas=1 --set mode=standalone --set resources.requests.memory=256Mi --set rootUser=rootuser --set rootPassword=rootpass123 --set 'users[0].accessKey=some-access-key' --set 'users[0].secretKey=some-secret-key' --set 'users[0].policy=consoleAdmin' --set service.type=ClusterIP --set configPathmc=/tmp/ --set securityContext.enabled=false --set persistence.size=2G --set fullnameOverride=minio-service --set serviceAccount.create=true --set serviceAccount.name=minio-service-sa minio/minio
+ local max=10
+ local delay=60
+ shift 2
+ local n=1
+ helm install minio-service --version 5.4.0 --set replicas=1 --set mode=standalone --set resources.requests.memory=256Mi --set rootUser=rootuser --set rootPassword=rootpass123 --set 'users[0].accessKey=some-access-key' --set 'users[0].secretKey=some-secret-key' --set 'users[0].policy=consoleAdmin' --set service.type=ClusterIP --set configPathmc=/tmp/ --set securityContext.enabled=false --set persistence.size=2G --set fullnameOverride=minio-service --set serviceAccount.create=true --set serviceAccount.name=minio-service-sa minio/minio
NAME: minio-service
LAST DEPLOYED: Fri Mar 27 11:06:25 2026
NAMESPACE: demand-backup-15110
STATUS: deployed
REVISION: 1
TEST SUITE: None
NOTES:
MinIO can be accessed via port 9000 on the following DNS name from within your cluster:
minio-service.demand-backup-15110.cluster.local

To access MinIO from localhost, run the below commands:

  1. export POD_NAME=$(kubectl get pods --namespace demand-backup-15110 -l "release=minio-service" -o jsonpath="{.items[0].metadata.name}")

  2. kubectl port-forward $POD_NAME 9000 --namespace demand-backup-15110

Read more about port forwarding here: http://kubernetes.io/docs/user-guide/kubectl/kubectl_port-forward/

You can now access MinIO server on http://localhost:9000. Follow the below steps to connect to MinIO server with mc client:

  1. Download the MinIO mc client - https://min.io/docs/minio/linux/reference/minio-mc.html#quickstart

  2. export MC_HOST_minio-service-local=http://$(kubectl get secret --namespace demand-backup-15110 minio-service -o jsonpath="{.data.rootUser}" | base64 --decode):$(kubectl get secret --namespace demand-backup-15110 minio-service -o jsonpath="{.data.rootPassword}" | base64 --decode)@localhost:9000

  3. mc ls minio-service-local
++ kubectl_bin get pods --selector=release=minio-service -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.3C8slyzQph
+++ mktemp
++ local LAST_ERR=/tmp/tmp.UJcyw8pJVU
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=release=minio-service -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.3C8slyzQph
++ cat /tmp/tmp.UJcyw8pJVU
++ rm /tmp/tmp.3C8slyzQph /tmp/tmp.UJcyw8pJVU
++ return 0
+ local MINIO_POD=minio-service-6d5f646cdc-pqpgz
+ wait_pod minio-service-6d5f646cdc-pqpgz
+ local pod=minio-service-6d5f646cdc-pqpgz
+ set +o xtrace
waiting for pod/minio-service-6d5f646cdc-pqpgz to be ready.OK
+ '[' -n psmdb-operator ']'
+ kubectl_bin create svc -n psmdb-operator externalname minio-service --external-name=minio-service.demand-backup-15110.svc.cluster.local --tcp=9000
service/minio-service created
+ create_minio_bucket operator-testing http://minio-service:9000
+ local bucket=operator-testing
+ local endpoint=http://minio-service:9000
+ kubectl_bin run -i --rm aws-cli --image=docker.io/perconalab/awscli --restart=Never -- bash -c 'AWS_ACCESS_KEY_ID=some-access-key 		AWS_SECRET_ACCESS_KEY=some-secret-key 		AWS_DEFAULT_REGION=us-east-1         /usr/bin/aws --no-verify-ssl --endpoint-url http://minio-service:9000 s3 mb s3://operator-testing'
++ mktemp
+ local LAST_OUT=/tmp/tmp.84xiWUzdTT
++ mktemp
+ local LAST_ERR=/tmp/tmp.hKICdpVtLM
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl run -i --rm aws-cli --image=docker.io/perconalab/awscli --restart=Never -- bash -c 'AWS_ACCESS_KEY_ID=some-access-key 		AWS_SECRET_ACCESS_KEY=some-secret-key 		AWS_DEFAULT_REGION=us-east-1         /usr/bin/aws --no-verify-ssl --endpoint-url http://minio-service:9000 s3 mb s3://operator-testing'
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.84xiWUzdTT
pod "aws-cli" deleted from demand-backup-15110 namespace
+ cat /tmp/tmp.hKICdpVtLM
All commands and output from this session will be recorded in container logs, including credentials and sensitive information passed through the command prompt.
If you don't see a command prompt, try pressing enter.
+ rm /tmp/tmp.84xiWUzdTT /tmp/tmp.hKICdpVtLM
+ return 0
+ desc 'create secrets and start client'
+ set +o xtrace


-----------------------------------------------------------------------------------
create secrets and start client
-----------------------------------------------------------------------------------

+ cluster=some-name-rs0
+ kubectl_bin apply -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2283/e2e-tests/demand-backup/conf/secrets.yml -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2283/e2e-tests/conf/client.yml
++ mktemp
+ local LAST_OUT=/tmp/tmp.A18AR8m8CM
++ mktemp
+ local LAST_ERR=/tmp/tmp.WBBQs66UcK
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl apply -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2283/e2e-tests/demand-backup/conf/secrets.yml -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2283/e2e-tests/conf/client.yml
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.A18AR8m8CM
secret/some-users created
deployment.apps/psmdb-client created
+ cat /tmp/tmp.WBBQs66UcK
+ rm /tmp/tmp.A18AR8m8CM /tmp/tmp.WBBQs66UcK
+ return 0
+ apply_s3_storage_secrets
+ desc 'create secrets for cloud storages'
+ set +o xtrace


-----------------------------------------------------------------------------------
create secrets for cloud storages
-----------------------------------------------------------------------------------

+ '[' -z '' ']'
+ kubectl_bin apply -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2283/e2e-tests/conf/minio-secret.yml -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2283/e2e-tests/conf/cloud-secret.yml
++ mktemp
+ local LAST_OUT=/tmp/tmp.9nrXax8v73
++ mktemp
+ local LAST_ERR=/tmp/tmp.JBRYTAVuCo
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl apply -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2283/e2e-tests/conf/minio-secret.yml -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2283/e2e-tests/conf/cloud-secret.yml
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.9nrXax8v73
secret/minio-secret created
secret/aws-s3-secret created
secret/gcp-cs-secret created
secret/azure-secret created
secret/gcp-cs-sa-key-secret created
+ cat /tmp/tmp.JBRYTAVuCo
+ rm /tmp/tmp.9nrXax8v73 /tmp/tmp.JBRYTAVuCo
+ return 0
+ desc 'create first PSMDB cluster some-name-rs0'
+ set +o xtrace


-----------------------------------------------------------------------------------
create first PSMDB cluster some-name-rs0
-----------------------------------------------------------------------------------

+ apply_cluster /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2283/e2e-tests/demand-backup/conf/some-name-rs0.yml
+ '[' -z '' ']'
+ cat_config /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2283/e2e-tests/demand-backup/conf/some-name-rs0.yml
+ kubectl_bin apply -f -
+ cat /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2283/e2e-tests/demand-backup/conf/some-name-rs0.yml
++ mktemp
+ yq eval '(.spec | select(.image == null)).image = "docker.io/perconalab/percona-server-mongodb-operator:main-mongod8.0"'
+ yq eval '(.spec | select(has("pmm"))).pmm.image = "docker.io/percona/pmm-client:2.44.1-1"'
+ yq eval '(.spec | select(has("initImage"))).initImage = "docker.io/perconalab/percona-server-mongodb-operator:PR-2283-e32dd8b3"'
+ local LAST_OUT=/tmp/tmp.CuUjihbhGl
+ yq eval '(.spec | select(has("backup"))).backup.image = "docker.io/perconalab/percona-server-mongodb-operator:main-backup"'
++ mktemp
+ /usr/sbin/sed -e s/NAME_SPACE/demand-backup-15110/g
+ yq eval '.spec.upgradeOptions.apply="Never"'
+ local LAST_ERR=/tmp/tmp.hAAIr55YY6
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl apply -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.CuUjihbhGl
perconaservermongodb.psmdb.percona.com/some-name created
+ cat /tmp/tmp.hAAIr55YY6
+ rm /tmp/tmp.CuUjihbhGl /tmp/tmp.hAAIr55YY6
+ return 0
+ desc 'check if all 3 Pods started'
+ set +o xtrace


-----------------------------------------------------------------------------------
check if all 3 Pods started
-----------------------------------------------------------------------------------

+ wait_for_running some-name-rs0 3
+ local name=some-name-rs0
+ let last_pod=2
+ local check_cluster_readyness=true
+ set_debug
+ [[ 1 == 1 ]]
+ set -o xtrace
+ local rs_name=rs0
+ local cluster_name=some-name
++ seq 0 2
+ for i in $(seq 0 $last_pod)
+ [[ 0 -eq 2 ]]
+ wait_pod some-name-rs0-0
+ local pod=some-name-rs0-0
+ set +o xtrace
waiting for pod/some-name-rs0-0 to be ready................OK
+ for i in $(seq 0 $last_pod)
+ [[ 1 -eq 2 ]]
+ wait_pod some-name-rs0-1
+ local pod=some-name-rs0-1
+ set +o xtrace
waiting for pod/some-name-rs0-1 to be ready..................OK
+ for i in $(seq 0 $last_pod)
+ [[ 2 -eq 2 ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].arbiter.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.2MsBw37FDE
+++ mktemp
++ local LAST_ERR=/tmp/tmp.QiyzObgVAS
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].arbiter.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.2MsBw37FDE
++ cat /tmp/tmp.QiyzObgVAS
++ rm /tmp/tmp.2MsBw37FDE /tmp/tmp.QiyzObgVAS
++ return 0
+ [[ '' == \t\r\u\e ]]
+ wait_pod some-name-rs0-2
+ local pod=some-name-rs0-2
+ set +o xtrace
waiting for pod/some-name-rs0-2 to be ready................OK
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].nonvoting.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.SDLdZiEPw1
+++ mktemp
++ local LAST_ERR=/tmp/tmp.exFCmHxsZe
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].nonvoting.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.SDLdZiEPw1
++ cat /tmp/tmp.exFCmHxsZe
++ rm /tmp/tmp.SDLdZiEPw1 /tmp/tmp.exFCmHxsZe
++ return 0
+ [[ '' == \t\r\u\e ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].hidden.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.wBbU55xT3q
+++ mktemp
++ local LAST_ERR=/tmp/tmp.Ev6Ri3BLti
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].hidden.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.wBbU55xT3q
++ cat /tmp/tmp.Ev6Ri3BLti
++ rm /tmp/tmp.wBbU55xT3q /tmp/tmp.Ev6Ri3BLti
++ return 0
+ [[ '' == \t\r\u\e ]]
+ sleep 10
+ [[ true == \t\r\u\e ]]
+ set +x
Waiting for cluster readyness........
+ desc 'check if service and statefulset created with expected config'
+ set +o xtrace


-----------------------------------------------------------------------------------
check if service and statefulset created with expected config
-----------------------------------------------------------------------------------

+ compare_kubectl statefulset/some-name-rs0
+ local resource=statefulset/some-name-rs0
+ local postfix=
+ local skip_generation_check=
+ local expected_result=/mnt/jenkins/workspace/cloud-psmdb-operator_PR-2283/e2e-tests/demand-backup/compare/statefulset_some-name-rs0.yml
+ local new_result=/tmp/tmp.SOjtfW5Qlc/statefulset_some-name-rs0.yml
+ '[' -n '' -a -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2283/e2e-tests/demand-backup/compare/statefulset_some-name-rs0-oc.yml ']'
+ kubectl_bin get -o yaml statefulset/some-name-rs0
+ yq eval '
			del(.metadata.ownerReferences[].apiVersion) |
			del(.metadata.managedFields) |
++ mktemp
			del(.. | select(has("creationTimestamp")).creationTimestamp) |
			del(.. | select(has("namespace")).namespace) |
			del(.. | select(has("uid")).uid) |
			del(.metadata.resourceVersion) |
			del(.spec.template.spec.containers[].env[] | select(.name == "NAMESPACE")) |
			del(.metadata.selfLink) |
			del(.metadata.annotations."cloud.google.com/neg") |
			del(.metadata.annotations."kubectl.kubernetes.io/last-applied-configuration") |
			del(.. | select(has("image")).image) |
			del(.. | select(has("clusterIP")).clusterIP) |
			del(.. | select(has("clusterIPs")).clusterIPs) |
			del(.. | select(has("dataSource")).dataSource) |
			del(.. | select(has("procMount")).procMount) |
			del(.. | select(has("storageClassName")).storageClassName) |
			del(.. | select(has("finalizers")).finalizers) |
			del(.. | select(has("kubernetes.io/pvc-protection"))."kubernetes.io/pvc-protection") |
			del(.. | select(has("volumeName")).volumeName) |
			del(.. | select(has("volume.beta.kubernetes.io/storage-provisioner"))."volume.beta.kubernetes.io/storage-provisioner") |
			del(.. | select(has("volume.kubernetes.io/storage-provisioner"))."volume.kubernetes.io/storage-provisioner") |
			del(.spec.volumeMode) |
			del(.. | select(has("volume.kubernetes.io/selected-node"))."volume.kubernetes.io/selected-node") |
			del(.. | select(has("percona.com/last-config-hash"))."percona.com/last-config-hash") |
			del(.. | select(has("percona.com/configuration-hash"))."percona.com/configuration-hash") |
			del(.. | select(has("percona.com/ssl-hash"))."percona.com/ssl-hash") |
			del(.. | select(has("percona.com/ssl-internal-hash"))."percona.com/ssl-internal-hash") |
			del(.spec.volumeClaimTemplates[].spec.volumeMode | select(. == "Filesystem")) |
			del(.. | select(has("healthCheckNodePort")).healthCheckNodePort) |
			del(.. | select(has("nodePort")).nodePort) |
			del(.status) |
			(.. | select(tag == "!!str")) |= sub("demand-backup-15110", "NAME_SPACE") |
			del(.spec.volumeClaimTemplates[].apiVersion) |
			del(.spec.volumeClaimTemplates[].kind) |
			del(.spec.ipFamilies) |
			del(.spec.ipFamilyPolicy) |
			(.. | select(. == "extensions/v1beta1")) = "apps/v1" |
			(.. | select(. == "batch/v1beta1")) = "batch/v1" ' -
+ local LAST_OUT=/tmp/tmp.k8y554qWy5
++ mktemp
+ local LAST_ERR=/tmp/tmp.96TQMrgsxi
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl get -o yaml statefulset/some-name-rs0
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.k8y554qWy5
+ cat /tmp/tmp.96TQMrgsxi
+ rm /tmp/tmp.k8y554qWy5 /tmp/tmp.96TQMrgsxi
+ return 0
+ yq -i eval 'del(.spec.persistentVolumeClaimRetentionPolicy)' /tmp/tmp.SOjtfW5Qlc/statefulset_some-name-rs0.yml
+ version_gt 1.22
++ echo '1.32 >= 1.22'
++ bc -l
+ '[' 1 -eq 1 ']'
+ return 0
+ yq -i eval 'del(.spec.internalTrafficPolicy)' /tmp/tmp.SOjtfW5Qlc/statefulset_some-name-rs0.yml
+ yq -i eval 'del(.spec.allocateLoadBalancerNodePorts)' /tmp/tmp.SOjtfW5Qlc/statefulset_some-name-rs0.yml
+ [[ /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2283/e2e-tests/demand-backup/compare/statefulset_some-name-rs0.yml == */cronjob* ]]
+ '[' -n '' ']'
+ [[ 0 -eq 0 ]]
+ diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2283/e2e-tests/demand-backup/compare/statefulset_some-name-rs0.yml /tmp/tmp.SOjtfW5Qlc/statefulset_some-name-rs0.yml
+ log 'compare_kubectl: statefulset/some-name-rs0 OK'
+ set +o xtrace
[2026-03-27T11:09:41+0000] compare_kubectl: statefulset/some-name-rs0 OK
+ desc 'create user'
+ set +o xtrace


-----------------------------------------------------------------------------------
create user
-----------------------------------------------------------------------------------

+ run_mongo 'db.createUser({user:"myApp",pwd:"myPass",roles:[{db:"myApp",role:"readWrite"}]})' userAdmin:userAdmin123456@some-name-rs0.demand-backup-15110
+ local 'command=db.createUser({user:"myApp",pwd:"myPass",roles:[{db:"myApp",role:"readWrite"}]})'
+ local uri=userAdmin:userAdmin123456@some-name-rs0.demand-backup-15110
+ local driver=mongodb+srv
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local replica_set=rs0
+ [[ userAdmin:userAdmin123456@some-name-rs0.demand-backup-15110 == *cfg* ]]
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.DpmpA2WXbz
+++ mktemp
++ local LAST_ERR=/tmp/tmp.cxV7DhyQld
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.DpmpA2WXbz
++ cat /tmp/tmp.cxV7DhyQld
++ rm /tmp/tmp.DpmpA2WXbz /tmp/tmp.cxV7DhyQld
++ return 0
+ local client_container=psmdb-client-bb8b97679-xfl2d
+ kubectl_bin exec psmdb-client-bb8b97679-xfl2d -- bash -c 'printf '\''db.createUser({user:"myApp",pwd:"myPass",roles:[{db:"myApp",role:"readWrite"}]})\n'\'' | mongo mongodb+srv://userAdmin:userAdmin123456@some-name-rs0.demand-backup-15110.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
++ mktemp
+ local LAST_OUT=/tmp/tmp.HXwbGKwCbP
++ mktemp
+ local LAST_ERR=/tmp/tmp.a7mgqxmoRC
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-bb8b97679-xfl2d -- bash -c 'printf '\''db.createUser({user:"myApp",pwd:"myPass",roles:[{db:"myApp",role:"readWrite"}]})\n'\'' | mongo mongodb+srv://userAdmin:userAdmin123456@some-name-rs0.demand-backup-15110.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.HXwbGKwCbP
Percona Server for MongoDB shell version v4.4.29-28
connecting to: mongodb://some-name-rs0-0.some-name-rs0.demand-backup-15110.svc.cluster.local:27017,some-name-rs0-2.some-name-rs0.demand-backup-15110.svc.cluster.local:27017,some-name-rs0-1.some-name-rs0.demand-backup-15110.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb&replicaSet=rs0&ssl=false
Implicit session: session { "id" : UUID("48480548-a556-4f70-980e-e97e2bc43047") }
Percona Server for MongoDB server version: v8.0.19-7
WARNING: shell and server versions do not match
Successfully added user: {
	"user" : "myApp",
	"roles" : [
		{
			"db" : "myApp",
			"role" : "readWrite"
		}
	]
}
bye
+ cat /tmp/tmp.a7mgqxmoRC
+ rm /tmp/tmp.HXwbGKwCbP /tmp/tmp.a7mgqxmoRC
+ return 0
+ sleep 2
+ desc 'write data, read from all'
+ set +o xtrace


-----------------------------------------------------------------------------------
write data, read from all
-----------------------------------------------------------------------------------

+ run_mongo 'use myApp\n db.test.insert({ x: 100500 })' myApp:myPass@some-name-rs0.demand-backup-15110
+ local 'command=use myApp\n db.test.insert({ x: 100500 })'
+ local uri=myApp:myPass@some-name-rs0.demand-backup-15110
+ local driver=mongodb+srv
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local replica_set=rs0
+ [[ myApp:myPass@some-name-rs0.demand-backup-15110 == *cfg* ]]
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.AsHPUI0LQT
+++ mktemp
++ local LAST_ERR=/tmp/tmp.BqPKb6WVNp
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.AsHPUI0LQT
++ cat /tmp/tmp.BqPKb6WVNp
++ rm /tmp/tmp.AsHPUI0LQT /tmp/tmp.BqPKb6WVNp
++ return 0
+ local client_container=psmdb-client-bb8b97679-xfl2d
+ kubectl_bin exec psmdb-client-bb8b97679-xfl2d -- bash -c 'printf '\''use myApp\n db.test.insert({ x: 100500 })\n'\'' | mongo mongodb+srv://myApp:myPass@some-name-rs0.demand-backup-15110.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
++ mktemp
+ local LAST_OUT=/tmp/tmp.s4vDVciWB8
++ mktemp
+ local LAST_ERR=/tmp/tmp.qfXGW8g1Gy
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-bb8b97679-xfl2d -- bash -c 'printf '\''use myApp\n db.test.insert({ x: 100500 })\n'\'' | mongo mongodb+srv://myApp:myPass@some-name-rs0.demand-backup-15110.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.s4vDVciWB8
Percona Server for MongoDB shell version v4.4.29-28
connecting to: mongodb://some-name-rs0-1.some-name-rs0.demand-backup-15110.svc.cluster.local:27017,some-name-rs0-0.some-name-rs0.demand-backup-15110.svc.cluster.local:27017,some-name-rs0-2.some-name-rs0.demand-backup-15110.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb&replicaSet=rs0&ssl=false
Implicit session: session { "id" : UUID("3a364326-b7b3-4db7-a376-30e0122ef06a") }
Percona Server for MongoDB server version: v8.0.19-7
WARNING: shell and server versions do not match
switched to db myApp
WriteResult({ "nInserted" : 1 })
bye
+ cat /tmp/tmp.qfXGW8g1Gy
+ rm /tmp/tmp.s4vDVciWB8 /tmp/tmp.qfXGW8g1Gy
+ return 0
+ run_mongo 'use myApp\n db.test2.insert({ x: 100501 })' myApp:myPass@some-name-rs0.demand-backup-15110
+ local 'command=use myApp\n db.test2.insert({ x: 100501 })'
+ local uri=myApp:myPass@some-name-rs0.demand-backup-15110
+ local driver=mongodb+srv
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local replica_set=rs0
+ [[ myApp:myPass@some-name-rs0.demand-backup-15110 == *cfg* ]]
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.7ZWW3k3TAL
+++ mktemp
++ local LAST_ERR=/tmp/tmp.KfBLuHB9ce
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.7ZWW3k3TAL
++ cat /tmp/tmp.KfBLuHB9ce
++ rm /tmp/tmp.7ZWW3k3TAL /tmp/tmp.KfBLuHB9ce
++ return 0
+ local client_container=psmdb-client-bb8b97679-xfl2d
+ kubectl_bin exec psmdb-client-bb8b97679-xfl2d -- bash -c 'printf '\''use myApp\n db.test2.insert({ x: 100501 })\n'\'' | mongo mongodb+srv://myApp:myPass@some-name-rs0.demand-backup-15110.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
++ mktemp
+ local LAST_OUT=/tmp/tmp.TKESHCN1Tv
++ mktemp
+ local LAST_ERR=/tmp/tmp.SUSJrjGfeR
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-bb8b97679-xfl2d -- bash -c 'printf '\''use myApp\n db.test2.insert({ x: 100501 })\n'\'' | mongo mongodb+srv://myApp:myPass@some-name-rs0.demand-backup-15110.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.TKESHCN1Tv
Percona Server for MongoDB shell version v4.4.29-28
connecting to: mongodb://some-name-rs0-0.some-name-rs0.demand-backup-15110.svc.cluster.local:27017,some-name-rs0-2.some-name-rs0.demand-backup-15110.svc.cluster.local:27017,some-name-rs0-1.some-name-rs0.demand-backup-15110.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb&replicaSet=rs0&ssl=false
Implicit session: session { "id" : UUID("32cd46dc-7434-40fe-91ee-9425d50dc774") }
Percona Server for MongoDB server version: v8.0.19-7
WARNING: shell and server versions do not match
switched to db myApp
WriteResult({ "nInserted" : 1 })
bye
+ cat /tmp/tmp.SUSJrjGfeR
+ rm /tmp/tmp.TKESHCN1Tv /tmp/tmp.SUSJrjGfeR
+ return 0
+ run_mongo 'use myApp\n db.test3.insert({ x: 100502 })' myApp:myPass@some-name-rs0.demand-backup-15110
+ local 'command=use myApp\n db.test3.insert({ x: 100502 })'
+ local uri=myApp:myPass@some-name-rs0.demand-backup-15110
+ local driver=mongodb+srv
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local replica_set=rs0
+ [[ myApp:myPass@some-name-rs0.demand-backup-15110 == *cfg* ]]
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.HF3t2HG9Lm
+++ mktemp
++ local LAST_ERR=/tmp/tmp.bjzc828a8o
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.HF3t2HG9Lm
++ cat /tmp/tmp.bjzc828a8o
++ rm /tmp/tmp.HF3t2HG9Lm /tmp/tmp.bjzc828a8o
++ return 0
+ local client_container=psmdb-client-bb8b97679-xfl2d
+ kubectl_bin exec psmdb-client-bb8b97679-xfl2d -- bash -c 'printf '\''use myApp\n db.test3.insert({ x: 100502 })\n'\'' | mongo mongodb+srv://myApp:myPass@some-name-rs0.demand-backup-15110.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
++ mktemp
+ local LAST_OUT=/tmp/tmp.z12eGRttQQ
++ mktemp
+ local LAST_ERR=/tmp/tmp.sOjn3JYZbl
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-bb8b97679-xfl2d -- bash -c 'printf '\''use myApp\n db.test3.insert({ x: 100502 })\n'\'' | mongo mongodb+srv://myApp:myPass@some-name-rs0.demand-backup-15110.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.z12eGRttQQ
Percona Server for MongoDB shell version v4.4.29-28
connecting to: mongodb://some-name-rs0-2.some-name-rs0.demand-backup-15110.svc.cluster.local:27017,some-name-rs0-1.some-name-rs0.demand-backup-15110.svc.cluster.local:27017,some-name-rs0-0.some-name-rs0.demand-backup-15110.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb&replicaSet=rs0&ssl=false
Implicit session: session { "id" : UUID("ff9740b7-9527-4405-b900-e89316786c02") }
Percona Server for MongoDB server version: v8.0.19-7
WARNING: shell and server versions do not match
switched to db myApp
WriteResult({ "nInserted" : 1 })
bye
+ cat /tmp/tmp.sOjn3JYZbl
+ rm /tmp/tmp.z12eGRttQQ /tmp/tmp.sOjn3JYZbl
+ return 0
+ custom_user_name=test1user
+ custom_role_name=test1role
+ run_mongo 'use myApp\n db.createUser({user: "test1user",pwd:"test1pass",roles:[]})' userAdmin:userAdmin123456@some-name-rs0.demand-backup-15110
+ local 'command=use myApp\n db.createUser({user: "test1user",pwd:"test1pass",roles:[]})'
+ local uri=userAdmin:userAdmin123456@some-name-rs0.demand-backup-15110
+ local driver=mongodb+srv
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local replica_set=rs0
+ [[ userAdmin:userAdmin123456@some-name-rs0.demand-backup-15110 == *cfg* ]]
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.qbEPsXVUyv
+++ mktemp
++ local LAST_ERR=/tmp/tmp.SQwO9AoPvT
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.qbEPsXVUyv
++ cat /tmp/tmp.SQwO9AoPvT
++ rm /tmp/tmp.qbEPsXVUyv /tmp/tmp.SQwO9AoPvT
++ return 0
+ local client_container=psmdb-client-bb8b97679-xfl2d
+ kubectl_bin exec psmdb-client-bb8b97679-xfl2d -- bash -c 'printf '\''use myApp\n db.createUser({user: "test1user",pwd:"test1pass",roles:[]})\n'\'' | mongo mongodb+srv://userAdmin:userAdmin123456@some-name-rs0.demand-backup-15110.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
++ mktemp
+ local LAST_OUT=/tmp/tmp.flPeUXw9dI
++ mktemp
+ local LAST_ERR=/tmp/tmp.8EmOXnZ83F
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-bb8b97679-xfl2d -- bash -c 'printf '\''use myApp\n db.createUser({user: "test1user",pwd:"test1pass",roles:[]})\n'\'' | mongo mongodb+srv://userAdmin:userAdmin123456@some-name-rs0.demand-backup-15110.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.flPeUXw9dI
Percona Server for MongoDB shell version v4.4.29-28
connecting to: mongodb://some-name-rs0-0.some-name-rs0.demand-backup-15110.svc.cluster.local:27017,some-name-rs0-2.some-name-rs0.demand-backup-15110.svc.cluster.local:27017,some-name-rs0-1.some-name-rs0.demand-backup-15110.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb&replicaSet=rs0&ssl=false
Implicit session: session { "id" : UUID("eea6372e-4b0c-495d-b67a-c7af0356d77d") }
Percona Server for MongoDB server version: v8.0.19-7
WARNING: shell and server versions do not match
switched to db myApp
Successfully added user: { "user" : "test1user", "roles" : [ ] }
bye
+ cat /tmp/tmp.8EmOXnZ83F
+ rm /tmp/tmp.flPeUXw9dI /tmp/tmp.8EmOXnZ83F
+ return 0
+ run_mongo 'use myApp\n db.createRole({"role":"test1role", privileges:[],roles:[]})' userAdmin:userAdmin123456@some-name-rs0.demand-backup-15110
+ local 'command=use myApp\n db.createRole({"role":"test1role", privileges:[],roles:[]})'
+ local uri=userAdmin:userAdmin123456@some-name-rs0.demand-backup-15110
+ local driver=mongodb+srv
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local replica_set=rs0
+ [[ userAdmin:userAdmin123456@some-name-rs0.demand-backup-15110 == *cfg* ]]
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.ZpIS209J9C
+++ mktemp
++ local LAST_ERR=/tmp/tmp.6AtGKmNi79
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.ZpIS209J9C
++ cat /tmp/tmp.6AtGKmNi79
++ rm /tmp/tmp.ZpIS209J9C /tmp/tmp.6AtGKmNi79
++ return 0
+ local client_container=psmdb-client-bb8b97679-xfl2d
+ kubectl_bin exec psmdb-client-bb8b97679-xfl2d -- bash -c 'printf '\''use myApp\n db.createRole({"role":"test1role", privileges:[],roles:[]})\n'\'' | mongo mongodb+srv://userAdmin:userAdmin123456@some-name-rs0.demand-backup-15110.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
++ mktemp
+ local LAST_OUT=/tmp/tmp.23aWOl3wP5
++ mktemp
+ local LAST_ERR=/tmp/tmp.GtelwDO6Wq
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-bb8b97679-xfl2d -- bash -c 'printf '\''use myApp\n db.createRole({"role":"test1role", privileges:[],roles:[]})\n'\'' | mongo mongodb+srv://userAdmin:userAdmin123456@some-name-rs0.demand-backup-15110.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.23aWOl3wP5
Percona Server for MongoDB shell version v4.4.29-28
connecting to: mongodb://some-name-rs0-2.some-name-rs0.demand-backup-15110.svc.cluster.local:27017,some-name-rs0-1.some-name-rs0.demand-backup-15110.svc.cluster.local:27017,some-name-rs0-0.some-name-rs0.demand-backup-15110.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb&replicaSet=rs0&ssl=false
Implicit session: session { "id" : UUID("f903bf1a-fe72-4b82-913a-ad900c170f4d") }
Percona Server for MongoDB server version: v8.0.19-7
WARNING: shell and server versions do not match
switched to db myApp
{ "role" : "test1role", "privileges" : [ ], "roles" : [ ] }
bye
+ cat /tmp/tmp.GtelwDO6Wq
+ rm /tmp/tmp.23aWOl3wP5 /tmp/tmp.GtelwDO6Wq
+ return 0
+ minikube_sleep
+ sleep_time=10
+ [[ '' == 1 ]]
+ compare_mongo_cmd find myApp:myPass@some-name-rs0-0.some-name-rs0.demand-backup-15110
+ local command=find
+ local uri=myApp:myPass@some-name-rs0-0.some-name-rs0.demand-backup-15110
+ local postfix=
+ local suffix=
+ local database=myApp
+ local collection=test
+ local sort=
+ local tls=false
+ local replicaset=
+ local 'full_command=db.test.find()'
+ [[ -n '' ]]
+ log 'running db.test.find() in myApp'
+ set +o xtrace
[2026-03-27T11:09:59+0000] running db.test.find() in myApp
+ [[ false == \t\r\u\e ]]
+ mongo_command=run_mongo
+ run_mongo 'use myApp\n db.test.find()' myApp:myPass@some-name-rs0-0.some-name-rs0.demand-backup-15110 mongodb '' ''
+ local 'command=use myApp\n db.test.find()'
+ local uri=myApp:myPass@some-name-rs0-0.some-name-rs0.demand-backup-15110
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local replica_set=rs0
+ [[ myApp:myPass@some-name-rs0-0.some-name-rs0.demand-backup-15110 == *cfg* ]]
+ grep -E -v 'I NETWORK|W NETWORK|F NETWORK|"c":"NETWORK"|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|Started a new thread for the timer service'
+ /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.Kmhq3bYwTK
+++ mktemp
++ local LAST_ERR=/tmp/tmp.o1UjADdvRN
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.Kmhq3bYwTK
++ cat /tmp/tmp.o1UjADdvRN
++ rm /tmp/tmp.Kmhq3bYwTK /tmp/tmp.o1UjADdvRN
++ return 0
+ local client_container=psmdb-client-bb8b97679-xfl2d
+ kubectl_bin exec psmdb-client-bb8b97679-xfl2d -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-0.some-name-rs0.demand-backup-15110.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
++ mktemp
+ local LAST_OUT=/tmp/tmp.lzFcy4bEYl
++ mktemp
+ local LAST_ERR=/tmp/tmp.bUiEdZTKCR
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-bb8b97679-xfl2d -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-0.some-name-rs0.demand-backup-15110.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.lzFcy4bEYl
+ cat /tmp/tmp.bUiEdZTKCR
+ rm /tmp/tmp.lzFcy4bEYl /tmp/tmp.bUiEdZTKCR
+ return 0
+ [[ 0 -eq 0 ]]
+ diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2283/e2e-tests/demand-backup/compare/find.json /tmp/tmp.SOjtfW5Qlc/find
+ compare_mongo_cmd find myApp:myPass@some-name-rs0-1.some-name-rs0.demand-backup-15110
+ local command=find
+ local uri=myApp:myPass@some-name-rs0-1.some-name-rs0.demand-backup-15110
+ local postfix=
+ local suffix=
+ local database=myApp
+ local collection=test
+ local sort=
+ local tls=false
+ local replicaset=
+ local 'full_command=db.test.find()'
+ [[ -n '' ]]
+ log 'running db.test.find() in myApp'
+ set +o xtrace
[2026-03-27T11:10:01+0000] running db.test.find() in myApp
+ [[ false == \t\r\u\e ]]
+ mongo_command=run_mongo
+ run_mongo 'use myApp\n db.test.find()' myApp:myPass@some-name-rs0-1.some-name-rs0.demand-backup-15110 mongodb '' ''
+ local 'command=use myApp\n db.test.find()'
+ local uri=myApp:myPass@some-name-rs0-1.some-name-rs0.demand-backup-15110
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local replica_set=rs0
+ [[ myApp:myPass@some-name-rs0-1.some-name-rs0.demand-backup-15110 == *cfg* ]]
+ grep -E -v 'I NETWORK|W NETWORK|F NETWORK|"c":"NETWORK"|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|Started a new thread for the timer service'
+ /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.eaWiOdGdV6
+++ mktemp
++ local LAST_ERR=/tmp/tmp.aLSl9xAXrC
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.eaWiOdGdV6
++ cat /tmp/tmp.aLSl9xAXrC
++ rm /tmp/tmp.eaWiOdGdV6 /tmp/tmp.aLSl9xAXrC
++ return 0
+ local client_container=psmdb-client-bb8b97679-xfl2d
+ kubectl_bin exec psmdb-client-bb8b97679-xfl2d -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-1.some-name-rs0.demand-backup-15110.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
++ mktemp
+ local LAST_OUT=/tmp/tmp.a28pjDH6dY
++ mktemp
+ local LAST_ERR=/tmp/tmp.9Vh8M0I6uN
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-bb8b97679-xfl2d -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-1.some-name-rs0.demand-backup-15110.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.a28pjDH6dY
+ cat /tmp/tmp.9Vh8M0I6uN
+ rm /tmp/tmp.a28pjDH6dY /tmp/tmp.9Vh8M0I6uN
+ return 0
+ [[ 0 -eq 0 ]]
+ diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2283/e2e-tests/demand-backup/compare/find.json /tmp/tmp.SOjtfW5Qlc/find
+ compare_mongo_cmd find myApp:myPass@some-name-rs0-2.some-name-rs0.demand-backup-15110
+ local command=find
+ local uri=myApp:myPass@some-name-rs0-2.some-name-rs0.demand-backup-15110
+ local postfix=
+ local suffix=
+ local database=myApp
+ local collection=test
+ local sort=
+ local tls=false
+ local replicaset=
+ local 'full_command=db.test.find()'
+ [[ -n '' ]]
+ log 'running db.test.find() in myApp'
+ set +o xtrace
[2026-03-27T11:10:04+0000] running db.test.find() in myApp
+ [[ false == \t\r\u\e ]]
+ mongo_command=run_mongo
+ grep -E -v 'I NETWORK|W NETWORK|F NETWORK|"c":"NETWORK"|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|Started a new thread for the timer service'
+ run_mongo 'use myApp\n db.test.find()' myApp:myPass@some-name-rs0-2.some-name-rs0.demand-backup-15110 mongodb '' ''
+ local 'command=use myApp\n db.test.find()'
+ /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
+ local uri=myApp:myPass@some-name-rs0-2.some-name-rs0.demand-backup-15110
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local replica_set=rs0
+ [[ myApp:myPass@some-name-rs0-2.some-name-rs0.demand-backup-15110 == *cfg* ]]
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.uGBGnoIvh3
+++ mktemp
++ local LAST_ERR=/tmp/tmp.ZCj78ICTGx
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.uGBGnoIvh3
++ cat /tmp/tmp.ZCj78ICTGx
++ rm /tmp/tmp.uGBGnoIvh3 /tmp/tmp.ZCj78ICTGx
++ return 0
+ local client_container=psmdb-client-bb8b97679-xfl2d
+ kubectl_bin exec psmdb-client-bb8b97679-xfl2d -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-2.some-name-rs0.demand-backup-15110.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
++ mktemp
+ local LAST_OUT=/tmp/tmp.Lkb0V9VS5v
++ mktemp
+ local LAST_ERR=/tmp/tmp.3PobU0jd0N
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-bb8b97679-xfl2d -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-2.some-name-rs0.demand-backup-15110.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.Lkb0V9VS5v
+ cat /tmp/tmp.3PobU0jd0N
+ rm /tmp/tmp.Lkb0V9VS5v /tmp/tmp.3PobU0jd0N
+ return 0
+ [[ 0 -eq 0 ]]
+ diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2283/e2e-tests/demand-backup/compare/find.json /tmp/tmp.SOjtfW5Qlc/find
+ wait_backup_agent some-name-rs0-0
+ local agent_pod=some-name-rs0-0
+ set +o xtrace
waiting for pbm-agent to be ready in some-name-rs0-0...2026-03-27T11:09:01.000+0000 I listening for the commands

+ wait_backup_agent some-name-rs0-1
+ local agent_pod=some-name-rs0-1
+ set +o xtrace
waiting for pbm-agent to be ready in some-name-rs0-1...2026-03-27T11:09:37.000+0000 I listening for the commands

+ wait_backup_agent some-name-rs0-2
+ local agent_pod=some-name-rs0-2
+ set +o xtrace
waiting for pbm-agent to be ready in some-name-rs0-2...2026-03-27T11:09:40.000+0000 I listening for the commands

+ '[' -z '' ']'
+ desc 'Check GCS profiles'
+ set +o xtrace


-----------------------------------------------------------------------------------
Check GCS profiles
-----------------------------------------------------------------------------------

+ compare_pbm_profile_setup some-name gcp-cs-s3
+ local cluster=some-name
+ local profile=gcp-cs-s3
+ local container=backup-agent
+ local pbm_binary=pbm
+ [[ backup-agent == \m\o\n\g\o\d ]]
+ kubectl_bin exec some-name-rs0-0 -c backup-agent -- pbm profile show gcp-cs-s3
++ mktemp
+ local LAST_OUT=/tmp/tmp.iyxn1OpgVF
++ mktemp
+ local LAST_ERR=/tmp/tmp.RNJGHJ45mx
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec some-name-rs0-0 -c backup-agent -- pbm profile show gcp-cs-s3
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.iyxn1OpgVF
+ cat /tmp/tmp.RNJGHJ45mx
+ rm /tmp/tmp.iyxn1OpgVF /tmp/tmp.RNJGHJ45mx
+ return 0
+ diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2283/e2e-tests/demand-backup/compare/pbm_profile_gcp-cs-s3.yml /tmp/tmp.SOjtfW5Qlc/pbm_profile_gcp-cs-s3.yml
+ compare_pbm_profile_setup some-name gcp-cs-sa
+ local cluster=some-name
+ local profile=gcp-cs-sa
+ local container=backup-agent
+ local pbm_binary=pbm
+ [[ backup-agent == \m\o\n\g\o\d ]]
+ kubectl_bin exec some-name-rs0-0 -c backup-agent -- pbm profile show gcp-cs-sa
++ mktemp
+ local LAST_OUT=/tmp/tmp.G09CR0DT93
++ mktemp
+ local LAST_ERR=/tmp/tmp.jgXcPFF1gB
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec some-name-rs0-0 -c backup-agent -- pbm profile show gcp-cs-sa
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.G09CR0DT93
+ cat /tmp/tmp.jgXcPFF1gB
+ rm /tmp/tmp.G09CR0DT93 /tmp/tmp.jgXcPFF1gB
+ return 0
+ diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2283/e2e-tests/demand-backup/compare/pbm_profile_gcp-cs-sa.yml /tmp/tmp.SOjtfW5Qlc/pbm_profile_gcp-cs-sa.yml
+ backup_name_minio=backup-minio
+ '[' -z '' ']'
+ backup_name_aws=backup-aws-s3
+ backup_name_gcp_s3=backup-gcp-cs-s3
+ backup_name_gcp_sa=backup-gcp-cs-sa
+ backup_name_azure=backup-azure-blob
+ desc 'run backups'
+ set +o xtrace


-----------------------------------------------------------------------------------
run backups
-----------------------------------------------------------------------------------

+ '[' -z '' ']'
+ run_backup aws-s3
+ local storage=aws-s3
+ local backup_name=backup-aws-s3
+ local type=logical
+ log 'running backup backup-aws-s3'
+ set +o xtrace
[2026-03-27T11:10:14+0000] running backup backup-aws-s3
+ yq eval '.metadata.name = "backup-aws-s3"
			| .spec.storageName = "aws-s3"
			| .spec.type = "logical"' /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2283/e2e-tests/demand-backup/conf/backup-aws-s3.yml
+ kubectl_bin apply -f -
++ mktemp
+ local LAST_OUT=/tmp/tmp.bP6suQ7f0r
++ mktemp
+ local LAST_ERR=/tmp/tmp.kQ7QWDQkkf
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl apply -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.bP6suQ7f0r
perconaservermongodbbackup.psmdb.percona.com/backup-aws-s3 created
+ cat /tmp/tmp.kQ7QWDQkkf
+ rm /tmp/tmp.bP6suQ7f0r /tmp/tmp.kQ7QWDQkkf
+ return 0
+ run_backup gcp-cs-s3
+ local storage=gcp-cs-s3
+ local backup_name=backup-gcp-cs-s3
+ local type=logical
+ log 'running backup backup-gcp-cs-s3'
+ set +o xtrace
[2026-03-27T11:10:16+0000] running backup backup-gcp-cs-s3
+ yq eval '.metadata.name = "backup-gcp-cs-s3"
			| .spec.storageName = "gcp-cs-s3"
			| .spec.type = "logical"' /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2283/e2e-tests/demand-backup/conf/backup-gcp-cs-s3.yml
+ kubectl_bin apply -f -
++ mktemp
+ local LAST_OUT=/tmp/tmp.4IxCKyKVbl
++ mktemp
+ local LAST_ERR=/tmp/tmp.nB71iBzu4l
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl apply -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.4IxCKyKVbl
perconaservermongodbbackup.psmdb.percona.com/backup-gcp-cs-s3 created
+ cat /tmp/tmp.nB71iBzu4l
+ rm /tmp/tmp.4IxCKyKVbl /tmp/tmp.nB71iBzu4l
+ return 0
+ run_backup gcp-cs-sa
+ local storage=gcp-cs-sa
+ local backup_name=backup-gcp-cs-sa
+ local type=logical
+ log 'running backup backup-gcp-cs-sa'
+ set +o xtrace
[2026-03-27T11:10:18+0000] running backup backup-gcp-cs-sa
+ yq eval '.metadata.name = "backup-gcp-cs-sa"
			| .spec.storageName = "gcp-cs-sa"
			| .spec.type = "logical"' /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2283/e2e-tests/demand-backup/conf/backup-gcp-cs-sa.yml
+ kubectl_bin apply -f -
++ mktemp
+ local LAST_OUT=/tmp/tmp.KX0labFXIh
++ mktemp
+ local LAST_ERR=/tmp/tmp.Gz3ifPUyWs
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl apply -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.KX0labFXIh
perconaservermongodbbackup.psmdb.percona.com/backup-gcp-cs-sa created
+ cat /tmp/tmp.Gz3ifPUyWs
+ rm /tmp/tmp.KX0labFXIh /tmp/tmp.Gz3ifPUyWs
+ return 0
+ run_backup azure-blob
+ local storage=azure-blob
+ local backup_name=backup-azure-blob
+ local type=logical
+ log 'running backup backup-azure-blob'
+ set +o xtrace
[2026-03-27T11:10:20+0000] running backup backup-azure-blob
+ yq eval '.metadata.name = "backup-azure-blob"
			| .spec.storageName = "azure-blob"
			| .spec.type = "logical"' /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2283/e2e-tests/demand-backup/conf/backup-azure-blob.yml
+ kubectl_bin apply -f -
++ mktemp
+ local LAST_OUT=/tmp/tmp.fadX1VV1TC
++ mktemp
+ local LAST_ERR=/tmp/tmp.kQKEcYVcrT
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl apply -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.fadX1VV1TC
perconaservermongodbbackup.psmdb.percona.com/backup-azure-blob created
+ cat /tmp/tmp.kQKEcYVcrT
+ rm /tmp/tmp.fadX1VV1TC /tmp/tmp.kQKEcYVcrT
+ return 0
+ run_backup minio
+ local storage=minio
+ local backup_name=backup-minio
+ local type=logical
+ log 'running backup backup-minio'
+ set +o xtrace
[2026-03-27T11:10:23+0000] running backup backup-minio
+ yq eval '.metadata.name = "backup-minio"
			| .spec.storageName = "minio"
			| .spec.type = "logical"' /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2283/e2e-tests/demand-backup/conf/backup-minio.yml
+ kubectl_bin apply -f -
++ mktemp
+ local LAST_OUT=/tmp/tmp.MFIDgvJhjr
++ mktemp
+ local LAST_ERR=/tmp/tmp.QgaOiLxEJf
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl apply -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.MFIDgvJhjr
perconaservermongodbbackup.psmdb.percona.com/backup-minio created
+ cat /tmp/tmp.QgaOiLxEJf
+ rm /tmp/tmp.MFIDgvJhjr /tmp/tmp.QgaOiLxEJf
+ return 0
+ '[' -z '' ']'
+ wait_backup backup-aws-s3
+ local backup_name=backup-aws-s3
+ local target_state=ready
+ set +o xtrace
waiting for backup-aws-s3 to reach ready state..OK
+ wait_backup backup-gcp-cs-s3
+ local backup_name=backup-gcp-cs-s3
+ local target_state=ready
+ set +o xtrace
waiting for backup-gcp-cs-s3 to reach ready state....................OK
+ wait_backup backup-gcp-cs-sa
+ local backup_name=backup-gcp-cs-sa
+ local target_state=ready
+ set +o xtrace
waiting for backup-gcp-cs-sa to reach ready state........OK
+ wait_backup backup-azure-blob
+ local backup_name=backup-azure-blob
+ local target_state=ready
+ set +o xtrace
waiting for backup-azure-blob to reach ready state..OK
+ wait_backup backup-minio
+ local backup_name=backup-minio
+ local target_state=ready
+ set +o xtrace
waiting for backup-minio to reach ready state..OK
+ sleep 5
+ '[' -z '' ']'
+ desc 'check backup and restore -- aws-s3'
+ set +o xtrace


-----------------------------------------------------------------------------------
check backup and restore -- aws-s3
-----------------------------------------------------------------------------------

++ get_backup_dest backup-aws-s3
++ local backup_name=backup-aws-s3
++ sed -e 's/.json$//'
++ kubectl_bin get psmdb-backup backup-aws-s3 -o 'jsonpath={.status.destination}'
++ sed 's|s3://||'
++ sed 's|gs://||'
++ sed 's|azure://||'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.jFAaAMtfsy
+++ mktemp
++ local LAST_ERR=/tmp/tmp.FDkRZo6Uc2
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb-backup backup-aws-s3 -o 'jsonpath={.status.destination}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.jFAaAMtfsy
++ cat /tmp/tmp.FDkRZo6Uc2
++ rm /tmp/tmp.jFAaAMtfsy /tmp/tmp.FDkRZo6Uc2
++ return 0
+ backup_dest_aws=operator-testing/psmdb-demand-backup/2026-03-27T11:10:16Z
+ setup_aws_credentials
+ local secret_name=aws-s3-secret
+ [[ -n AKIARXP3OARBMEQ6FSEW ]]
+ [[ -n gljUukL3/TuQnzrtJUgiURH6vqjt+SEo89jGS2Z/ ]]
+ echo 'AWS credentials already set in environment'
AWS credentials already set in environment
+ return 0
+ check_backup_existence_aws operator-testing/psmdb-demand-backup/2026-03-27T11:10:16Z /rs0/myApp.test.gz
++ echo operator-testing/psmdb-demand-backup/2026-03-27T11:10:16Z
++ cut -d/ -f1
+ bucket=operator-testing
++ echo operator-testing/psmdb-demand-backup/2026-03-27T11:10:16Z
++ cut -d/ -f2-
+ key_prefix=psmdb-demand-backup/2026-03-27T11:10:16Z
+ key=/rs0/myApp.test.gz
+ storage_name=aws-s3
+ retry=0
+ aws s3api head-object --bucket operator-testing --key psmdb-demand-backup/2026-03-27T11:10:16Z/rs0/myApp.test.gz
+ echo 'Backup operator-testing/psmdb-demand-backup/2026-03-27T11:10:16Z//rs0/myApp.test.gz found in aws-s3'
Backup operator-testing/psmdb-demand-backup/2026-03-27T11:10:16Z//rs0/myApp.test.gz found in aws-s3
+ run_recovery_check backup-aws-s3 some-name-rs0
+ local backup=backup-aws-s3
+ local cluster=some-name-rs0
+ run_mongo 'use myApp\n db.test.insert({ x: 100501 })' myApp:myPass@some-name-rs0.demand-backup-15110
+ local 'command=use myApp\n db.test.insert({ x: 100501 })'
+ local uri=myApp:myPass@some-name-rs0.demand-backup-15110
+ local driver=mongodb+srv
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local replica_set=rs0
+ [[ myApp:myPass@some-name-rs0.demand-backup-15110 == *cfg* ]]
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.gJyWlTjKb5
+++ mktemp
++ local LAST_ERR=/tmp/tmp.74mJc1nGJs
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.gJyWlTjKb5
++ cat /tmp/tmp.74mJc1nGJs
++ rm /tmp/tmp.gJyWlTjKb5 /tmp/tmp.74mJc1nGJs
++ return 0
+ local client_container=psmdb-client-bb8b97679-xfl2d
+ kubectl_bin exec psmdb-client-bb8b97679-xfl2d -- bash -c 'printf '\''use myApp\n db.test.insert({ x: 100501 })\n'\'' | mongo mongodb+srv://myApp:myPass@some-name-rs0.demand-backup-15110.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
++ mktemp
+ local LAST_OUT=/tmp/tmp.FnLY0NArum
++ mktemp
+ local LAST_ERR=/tmp/tmp.Khd8bBhdbR
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-bb8b97679-xfl2d -- bash -c 'printf '\''use myApp\n db.test.insert({ x: 100501 })\n'\'' | mongo mongodb+srv://myApp:myPass@some-name-rs0.demand-backup-15110.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.FnLY0NArum
Percona Server for MongoDB shell version v4.4.29-28
connecting to: mongodb://some-name-rs0-1.some-name-rs0.demand-backup-15110.svc.cluster.local:27017,some-name-rs0-0.some-name-rs0.demand-backup-15110.svc.cluster.local:27017,some-name-rs0-2.some-name-rs0.demand-backup-15110.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb&replicaSet=rs0&ssl=false
Implicit session: session { "id" : UUID("7ea93f46-66eb-4d1a-9956-33f08f8fd457") }
Percona Server for MongoDB server version: v8.0.19-7
WARNING: shell and server versions do not match
switched to db myApp
WriteResult({ "nInserted" : 1 })
bye
+ cat /tmp/tmp.Khd8bBhdbR
+ rm /tmp/tmp.FnLY0NArum /tmp/tmp.Khd8bBhdbR
+ return 0
+ compare_mongo_cmd find myApp:myPass@some-name-rs0.demand-backup-15110 -2nd .svc.cluster.local myApp test
+ local command=find
+ local uri=myApp:myPass@some-name-rs0.demand-backup-15110
+ local postfix=-2nd
+ local suffix=.svc.cluster.local
+ local database=myApp
+ local collection=test
+ local sort=
+ local tls=false
+ local replicaset=
+ local 'full_command=db.test.find()'
+ [[ -n '' ]]
+ log 'running db.test.find() in myApp'
+ set +o xtrace
[2026-03-27T11:11:33+0000] running db.test.find() in myApp
+ [[ false == \t\r\u\e ]]
+ mongo_command=run_mongo
+ run_mongo 'use myApp\n db.test.find()' myApp:myPass@some-name-rs0.demand-backup-15110 mongodb .svc.cluster.local ''
+ local 'command=use myApp\n db.test.find()'
+ local uri=myApp:myPass@some-name-rs0.demand-backup-15110
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local replica_set=rs0
+ [[ myApp:myPass@some-name-rs0.demand-backup-15110 == *cfg* ]]
+ grep -E -v 'I NETWORK|W NETWORK|F NETWORK|"c":"NETWORK"|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|Started a new thread for the timer service'
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+ /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.kYQElMwxdl
+++ mktemp
++ local LAST_ERR=/tmp/tmp.4F14RpUV8l
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.kYQElMwxdl
++ cat /tmp/tmp.4F14RpUV8l
++ rm /tmp/tmp.kYQElMwxdl /tmp/tmp.4F14RpUV8l
++ return 0
+ local client_container=psmdb-client-bb8b97679-xfl2d
+ kubectl_bin exec psmdb-client-bb8b97679-xfl2d -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0.demand-backup-15110.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
++ mktemp
+ local LAST_OUT=/tmp/tmp.5qnFmWodK5
++ mktemp
+ local LAST_ERR=/tmp/tmp.VbTlJn6IaQ
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-bb8b97679-xfl2d -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0.demand-backup-15110.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.5qnFmWodK5
+ cat /tmp/tmp.VbTlJn6IaQ
+ rm /tmp/tmp.5qnFmWodK5 /tmp/tmp.VbTlJn6IaQ
+ return 0
+ [[ 0 -eq 0 ]]
+ diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2283/e2e-tests/demand-backup/compare/find-2nd.json /tmp/tmp.SOjtfW5Qlc/find-2nd
+ run_restore backup-aws-s3
+ local backup_name=backup-aws-s3
+ local restore_file=/mnt/jenkins/workspace/cloud-psmdb-operator_PR-2283/e2e-tests/demand-backup/conf/restore.yml
+ log 'running restore restore-backup-aws-s3'
+ set +o xtrace
[2026-03-27T11:11:35+0000] running restore restore-backup-aws-s3
+ cat /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2283/e2e-tests/demand-backup/conf/restore.yml
+ /usr/sbin/sed -e 's/name:/name: restore-backup-aws-s3/'
+ /usr/sbin/sed -e 's/backupName:/backupName: backup-aws-s3/'
+ kubectl_bin apply -f -
++ mktemp
+ local LAST_OUT=/tmp/tmp.sFh6VRgtPO
++ mktemp
+ local LAST_ERR=/tmp/tmp.6P3o7O6p2f
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl apply -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.sFh6VRgtPO
perconaservermongodbrestore.psmdb.percona.com/restore-backup-aws-s3 created
+ cat /tmp/tmp.6P3o7O6p2f
+ rm /tmp/tmp.sFh6VRgtPO /tmp/tmp.6P3o7O6p2f
+ return 0
+ wait_restore backup-aws-s3 some-name
+ local backup_name=backup-aws-s3
+ local cluster_name=some-name
+ local target_state=ready
+ local wait_cluster_consistency=1
+ local wait_time=1780
+ local ok_if_ready=0
+ set +o xtrace
Waiting for the psmdb-restore/restore-backup-aws-s3 object to be created.OK
Waiting psmdb-restore/restore-backup-aws-s3 to reach state "ready" .OK after 0 minutes
+ [[ 1 -eq 1 ]]
+ wait_cluster_consistency some-name
+ local cluster_name=some-name
+ local wait_time=32
+ retry=0
+ sleep 7
+ echo -n 'waiting for cluster readyness'
waiting for cluster readyness++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.WL5DgCt4ZW
+++ mktemp
++ local LAST_ERR=/tmp/tmp.qyREDPwjy2
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.WL5DgCt4ZW
++ cat /tmp/tmp.qyREDPwjy2
++ rm /tmp/tmp.WL5DgCt4ZW /tmp/tmp.qyREDPwjy2
++ return 0
+ [[ ready == \r\e\a\d\y ]]
+ echo .OK
.OK
+ simple_data_check some-name-rs0 3 0
+ local cluster_name=some-name-rs0
+ let last_pod=3-1
+ local isSharded=0
+ local cluster_pfx=
+ '[' 0 -eq 1 ']'
++ seq 0 2
+ for i in $(seq 0 $last_pod)
+ compare_mongo_cmd find myApp:myPass@some-name-rs0-0.some-name-rs0.demand-backup-15110
+ local command=find
+ local uri=myApp:myPass@some-name-rs0-0.some-name-rs0.demand-backup-15110
+ local postfix=
+ local suffix=
+ local database=myApp
+ local collection=test
+ local sort=
+ local tls=false
+ local replicaset=
+ local 'full_command=db.test.find()'
+ [[ -n '' ]]
+ log 'running db.test.find() in myApp'
+ set +o xtrace
[2026-03-27T11:12:03+0000] running db.test.find() in myApp
+ [[ false == \t\r\u\e ]]
+ mongo_command=run_mongo
+ run_mongo 'use myApp\n db.test.find()' myApp:myPass@some-name-rs0-0.some-name-rs0.demand-backup-15110 mongodb '' ''
+ local 'command=use myApp\n db.test.find()'
+ local uri=myApp:myPass@some-name-rs0-0.some-name-rs0.demand-backup-15110
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local replica_set=rs0
+ [[ myApp:myPass@some-name-rs0-0.some-name-rs0.demand-backup-15110 == *cfg* ]]
+ grep -E -v 'I NETWORK|W NETWORK|F NETWORK|"c":"NETWORK"|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|Started a new thread for the timer service'
+ /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.9AKnBNJAhU
+++ mktemp
++ local LAST_ERR=/tmp/tmp.pzsST8XMta
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.9AKnBNJAhU
++ cat /tmp/tmp.pzsST8XMta
++ rm /tmp/tmp.9AKnBNJAhU /tmp/tmp.pzsST8XMta
++ return 0
+ local client_container=psmdb-client-bb8b97679-xfl2d
+ kubectl_bin exec psmdb-client-bb8b97679-xfl2d -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-0.some-name-rs0.demand-backup-15110.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
++ mktemp
+ local LAST_OUT=/tmp/tmp.FuhCF6M1a5
++ mktemp
+ local LAST_ERR=/tmp/tmp.yxSoUP2Ue2
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-bb8b97679-xfl2d -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-0.some-name-rs0.demand-backup-15110.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.FuhCF6M1a5
+ cat /tmp/tmp.yxSoUP2Ue2
+ rm /tmp/tmp.FuhCF6M1a5 /tmp/tmp.yxSoUP2Ue2
+ return 0
+ [[ 0 -eq 0 ]]
+ diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2283/e2e-tests/demand-backup/compare/find.json /tmp/tmp.SOjtfW5Qlc/find
+ for i in $(seq 0 $last_pod)
+ compare_mongo_cmd find myApp:myPass@some-name-rs0-1.some-name-rs0.demand-backup-15110
+ local command=find
+ local uri=myApp:myPass@some-name-rs0-1.some-name-rs0.demand-backup-15110
+ local postfix=
+ local suffix=
+ local database=myApp
+ local collection=test
+ local sort=
+ local tls=false
+ local replicaset=
+ local 'full_command=db.test.find()'
+ [[ -n '' ]]
+ log 'running db.test.find() in myApp'
+ set +o xtrace
[2026-03-27T11:12:06+0000] running db.test.find() in myApp
+ [[ false == \t\r\u\e ]]
+ mongo_command=run_mongo
+ run_mongo 'use myApp\n db.test.find()' myApp:myPass@some-name-rs0-1.some-name-rs0.demand-backup-15110 mongodb '' ''
+ local 'command=use myApp\n db.test.find()'
+ local uri=myApp:myPass@some-name-rs0-1.some-name-rs0.demand-backup-15110
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local replica_set=rs0
+ [[ myApp:myPass@some-name-rs0-1.some-name-rs0.demand-backup-15110 == *cfg* ]]
+ grep -E -v 'I NETWORK|W NETWORK|F NETWORK|"c":"NETWORK"|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|Started a new thread for the timer service'
+ /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.M6JTqgGiGB
+++ mktemp
++ local LAST_ERR=/tmp/tmp.zIpVNWPKCU
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.M6JTqgGiGB
++ cat /tmp/tmp.zIpVNWPKCU
++ rm /tmp/tmp.M6JTqgGiGB /tmp/tmp.zIpVNWPKCU
++ return 0
+ local client_container=psmdb-client-bb8b97679-xfl2d
+ kubectl_bin exec psmdb-client-bb8b97679-xfl2d -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-1.some-name-rs0.demand-backup-15110.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
++ mktemp
+ local LAST_OUT=/tmp/tmp.mipQWY002g
++ mktemp
+ local LAST_ERR=/tmp/tmp.g58TAOutVb
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-bb8b97679-xfl2d -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-1.some-name-rs0.demand-backup-15110.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.mipQWY002g
+ cat /tmp/tmp.g58TAOutVb
+ rm /tmp/tmp.mipQWY002g /tmp/tmp.g58TAOutVb
+ return 0
+ [[ 0 -eq 0 ]]
+ diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2283/e2e-tests/demand-backup/compare/find.json /tmp/tmp.SOjtfW5Qlc/find
+ for i in $(seq 0 $last_pod)
+ compare_mongo_cmd find myApp:myPass@some-name-rs0-2.some-name-rs0.demand-backup-15110
+ local command=find
+ local uri=myApp:myPass@some-name-rs0-2.some-name-rs0.demand-backup-15110
+ local postfix=
+ local suffix=
+ local database=myApp
+ local collection=test
+ local sort=
+ local tls=false
+ local replicaset=
+ local 'full_command=db.test.find()'
+ [[ -n '' ]]
+ log 'running db.test.find() in myApp'
+ set +o xtrace
[2026-03-27T11:12:08+0000] running db.test.find() in myApp
+ [[ false == \t\r\u\e ]]
+ mongo_command=run_mongo
+ grep -E -v 'I NETWORK|W NETWORK|F NETWORK|"c":"NETWORK"|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|Started a new thread for the timer service'
+ run_mongo 'use myApp\n db.test.find()' myApp:myPass@some-name-rs0-2.some-name-rs0.demand-backup-15110 mongodb '' ''
+ /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
+ local 'command=use myApp\n db.test.find()'
+ local uri=myApp:myPass@some-name-rs0-2.some-name-rs0.demand-backup-15110
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local replica_set=rs0
+ [[ myApp:myPass@some-name-rs0-2.some-name-rs0.demand-backup-15110 == *cfg* ]]
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.84qv8aE1Gl
+++ mktemp
++ local LAST_ERR=/tmp/tmp.5fiRBVsobk
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.84qv8aE1Gl
++ cat /tmp/tmp.5fiRBVsobk
++ rm /tmp/tmp.84qv8aE1Gl /tmp/tmp.5fiRBVsobk
++ return 0
+ local client_container=psmdb-client-bb8b97679-xfl2d
+ kubectl_bin exec psmdb-client-bb8b97679-xfl2d -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-2.some-name-rs0.demand-backup-15110.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
++ mktemp
+ local LAST_OUT=/tmp/tmp.Ls3UTwkIPF
++ mktemp
+ local LAST_ERR=/tmp/tmp.iDsvIDCk9d
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-bb8b97679-xfl2d -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-2.some-name-rs0.demand-backup-15110.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.Ls3UTwkIPF
+ cat /tmp/tmp.iDsvIDCk9d
+ rm /tmp/tmp.Ls3UTwkIPF /tmp/tmp.iDsvIDCk9d
+ return 0
+ [[ 0 -eq 0 ]]
+ diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2283/e2e-tests/demand-backup/compare/find.json /tmp/tmp.SOjtfW5Qlc/find
+ desc 'check backup and restore -- gcp-cs-s3'
+ set +o xtrace


-----------------------------------------------------------------------------------
check backup and restore -- gcp-cs-s3
-----------------------------------------------------------------------------------

++ get_backup_dest backup-gcp-cs-s3
++ local backup_name=backup-gcp-cs-s3
++ kubectl_bin get psmdb-backup backup-gcp-cs-s3 -o 'jsonpath={.status.destination}'
++ sed -e 's/.json$//'
++ sed 's|s3://||'
+++ mktemp
++ sed 's|azure://||'
++ sed 's|gs://||'
++ local LAST_OUT=/tmp/tmp.u83Lm2CcDo
+++ mktemp
++ local LAST_ERR=/tmp/tmp.ZUJNxCIRfR
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb-backup backup-gcp-cs-s3 -o 'jsonpath={.status.destination}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.u83Lm2CcDo
++ cat /tmp/tmp.ZUJNxCIRfR
++ rm /tmp/tmp.u83Lm2CcDo /tmp/tmp.ZUJNxCIRfR
++ return 0
+ backup_dest_gcp_s3=operator-testing/psmdb-demand-backup/2026-03-27T11:10:54Z
+ setup_gcs_credentials
+ local secret_name=gcp-cs-secret
+ gsutil ls
+ echo 'Setting up GCS credentials from K8s secret: gcp-cs-secret'
Setting up GCS credentials from K8s secret: gcp-cs-secret
+ local trace_was_on=0
+ [[ ehxB == *x* ]]
+ trace_was_on=1
+ set +x
+ echo 'GCS credentials configured successfully'
GCS credentials configured successfully
+ check_backup_existence_gcs operator-testing/psmdb-demand-backup/2026-03-27T11:10:54Z /rs0/myApp.test.gz
+ backup_dest_gcp=operator-testing/psmdb-demand-backup/2026-03-27T11:10:54Z
+ obj=/rs0/myApp.test.gz
+ storage_name=gcp-cs
+ retry=0
+ gcs_path=gs://operator-testing/psmdb-demand-backup/2026-03-27T11:10:54Z/rs0/myApp.test.gz
+ gsutil ls gs://operator-testing/psmdb-demand-backup/2026-03-27T11:10:54Z/rs0/myApp.test.gz
+ echo 'Backup gs://operator-testing/psmdb-demand-backup/2026-03-27T11:10:54Z/rs0/myApp.test.gz found in gcp-cs'
Backup gs://operator-testing/psmdb-demand-backup/2026-03-27T11:10:54Z/rs0/myApp.test.gz found in gcp-cs
+ run_recovery_check backup-gcp-cs-s3 some-name-rs0
+ local backup=backup-gcp-cs-s3
+ local cluster=some-name-rs0
+ run_mongo 'use myApp\n db.test.insert({ x: 100501 })' myApp:myPass@some-name-rs0.demand-backup-15110
+ local 'command=use myApp\n db.test.insert({ x: 100501 })'
+ local uri=myApp:myPass@some-name-rs0.demand-backup-15110
+ local driver=mongodb+srv
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local replica_set=rs0
+ [[ myApp:myPass@some-name-rs0.demand-backup-15110 == *cfg* ]]
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.F3mk9rHQ7l
+++ mktemp
++ local LAST_ERR=/tmp/tmp.w8qTVyc45s
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.F3mk9rHQ7l
++ cat /tmp/tmp.w8qTVyc45s
++ rm /tmp/tmp.F3mk9rHQ7l /tmp/tmp.w8qTVyc45s
++ return 0
+ local client_container=psmdb-client-bb8b97679-xfl2d
+ kubectl_bin exec psmdb-client-bb8b97679-xfl2d -- bash -c 'printf '\''use myApp\n db.test.insert({ x: 100501 })\n'\'' | mongo mongodb+srv://myApp:myPass@some-name-rs0.demand-backup-15110.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
++ mktemp
+ local LAST_OUT=/tmp/tmp.4vOrxWQRal
++ mktemp
+ local LAST_ERR=/tmp/tmp.YuS0jSUKGJ
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-bb8b97679-xfl2d -- bash -c 'printf '\''use myApp\n db.test.insert({ x: 100501 })\n'\'' | mongo mongodb+srv://myApp:myPass@some-name-rs0.demand-backup-15110.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.4vOrxWQRal
Percona Server for MongoDB shell version v4.4.29-28
connecting to: mongodb://some-name-rs0-0.some-name-rs0.demand-backup-15110.svc.cluster.local:27017,some-name-rs0-2.some-name-rs0.demand-backup-15110.svc.cluster.local:27017,some-name-rs0-1.some-name-rs0.demand-backup-15110.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb&replicaSet=rs0&ssl=false
Implicit session: session { "id" : UUID("ac879c20-c663-4664-9c16-0ebcf794d4e1") }
Percona Server for MongoDB server version: v8.0.19-7
WARNING: shell and server versions do not match
switched to db myApp
WriteResult({ "nInserted" : 1 })
bye
+ cat /tmp/tmp.YuS0jSUKGJ
+ rm /tmp/tmp.4vOrxWQRal /tmp/tmp.YuS0jSUKGJ
+ return 0
+ compare_mongo_cmd find myApp:myPass@some-name-rs0.demand-backup-15110 -2nd .svc.cluster.local myApp test
+ local command=find
+ local uri=myApp:myPass@some-name-rs0.demand-backup-15110
+ local postfix=-2nd
+ local suffix=.svc.cluster.local
+ local database=myApp
+ local collection=test
+ local sort=
+ local tls=false
+ local replicaset=
+ local 'full_command=db.test.find()'
+ [[ -n '' ]]
+ log 'running db.test.find() in myApp'
+ set +o xtrace
[2026-03-27T11:12:20+0000] running db.test.find() in myApp
+ [[ false == \t\r\u\e ]]
+ mongo_command=run_mongo
+ run_mongo 'use myApp\n db.test.find()' myApp:myPass@some-name-rs0.demand-backup-15110 mongodb .svc.cluster.local ''
+ local 'command=use myApp\n db.test.find()'
+ grep -E -v 'I NETWORK|W NETWORK|F NETWORK|"c":"NETWORK"|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|Started a new thread for the timer service'
+ local uri=myApp:myPass@some-name-rs0.demand-backup-15110
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local replica_set=rs0
+ [[ myApp:myPass@some-name-rs0.demand-backup-15110 == *cfg* ]]
+ /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.vsp2IjHqCX
+++ mktemp
++ local LAST_ERR=/tmp/tmp.WqnwZ5f5mR
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.vsp2IjHqCX
++ cat /tmp/tmp.WqnwZ5f5mR
++ rm /tmp/tmp.vsp2IjHqCX /tmp/tmp.WqnwZ5f5mR
++ return 0
+ local client_container=psmdb-client-bb8b97679-xfl2d
+ kubectl_bin exec psmdb-client-bb8b97679-xfl2d -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0.demand-backup-15110.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
++ mktemp
+ local LAST_OUT=/tmp/tmp.Sjc2O1YNQc
++ mktemp
+ local LAST_ERR=/tmp/tmp.I3CJlGKwNl
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-bb8b97679-xfl2d -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0.demand-backup-15110.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.Sjc2O1YNQc
+ cat /tmp/tmp.I3CJlGKwNl
+ rm /tmp/tmp.Sjc2O1YNQc /tmp/tmp.I3CJlGKwNl
+ return 0
+ [[ 0 -eq 0 ]]
+ diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2283/e2e-tests/demand-backup/compare/find-2nd.json /tmp/tmp.SOjtfW5Qlc/find-2nd
+ run_restore backup-gcp-cs-s3
+ local backup_name=backup-gcp-cs-s3
+ local restore_file=/mnt/jenkins/workspace/cloud-psmdb-operator_PR-2283/e2e-tests/demand-backup/conf/restore.yml
+ log 'running restore restore-backup-gcp-cs-s3'
+ set +o xtrace
[2026-03-27T11:12:22+0000] running restore restore-backup-gcp-cs-s3
+ cat /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2283/e2e-tests/demand-backup/conf/restore.yml
+ /usr/sbin/sed -e 's/name:/name: restore-backup-gcp-cs-s3/'
+ /usr/sbin/sed -e 's/backupName:/backupName: backup-gcp-cs-s3/'
+ kubectl_bin apply -f -
++ mktemp
+ local LAST_OUT=/tmp/tmp.a2PtZ5RkSr
++ mktemp
+ local LAST_ERR=/tmp/tmp.OqNXTR7BEh
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl apply -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.a2PtZ5RkSr
perconaservermongodbrestore.psmdb.percona.com/restore-backup-gcp-cs-s3 created
+ cat /tmp/tmp.OqNXTR7BEh
+ rm /tmp/tmp.a2PtZ5RkSr /tmp/tmp.OqNXTR7BEh
+ return 0
+ wait_restore backup-gcp-cs-s3 some-name
+ local backup_name=backup-gcp-cs-s3
+ local cluster_name=some-name
+ local target_state=ready
+ local wait_cluster_consistency=1
+ local wait_time=1780
+ local ok_if_ready=0
+ set +o xtrace
Waiting for the psmdb-restore/restore-backup-gcp-cs-s3 object to be created.OK
Waiting psmdb-restore/restore-backup-gcp-cs-s3 to reach state "ready" .....................OK after 20 minutes
+ [[ 1 -eq 1 ]]
+ wait_cluster_consistency some-name
+ local cluster_name=some-name
+ local wait_time=32
+ retry=0
+ sleep 7
+ echo -n 'waiting for cluster readyness'
waiting for cluster readyness++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.sNWcl6N5hd
+++ mktemp
++ local LAST_ERR=/tmp/tmp.QsWZnk0km3
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.sNWcl6N5hd
++ cat /tmp/tmp.QsWZnk0km3
++ rm /tmp/tmp.sNWcl6N5hd /tmp/tmp.QsWZnk0km3
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 1 -ge 32 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.zE1HUPzy0j
+++ mktemp
++ local LAST_ERR=/tmp/tmp.9UlC3vUHN6
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.zE1HUPzy0j
++ cat /tmp/tmp.9UlC3vUHN6
++ rm /tmp/tmp.zE1HUPzy0j /tmp/tmp.9UlC3vUHN6
++ return 0
+ [[ ready == \r\e\a\d\y ]]
+ echo .OK
.OK
+ simple_data_check some-name-rs0 3 0
+ local cluster_name=some-name-rs0
+ let last_pod=3-1
+ local isSharded=0
+ local cluster_pfx=
+ '[' 0 -eq 1 ']'
++ seq 0 2
+ for i in $(seq 0 $last_pod)
+ compare_mongo_cmd find myApp:myPass@some-name-rs0-0.some-name-rs0.demand-backup-15110
+ local command=find
+ local uri=myApp:myPass@some-name-rs0-0.some-name-rs0.demand-backup-15110
+ local postfix=
+ local suffix=
+ local database=myApp
+ local collection=test
+ local sort=
+ local tls=false
+ local replicaset=
+ local 'full_command=db.test.find()'
+ [[ -n '' ]]
+ log 'running db.test.find() in myApp'
+ set +o xtrace
[2026-03-27T11:33:10+0000] running db.test.find() in myApp
+ [[ false == \t\r\u\e ]]
+ mongo_command=run_mongo
+ run_mongo 'use myApp\n db.test.find()' myApp:myPass@some-name-rs0-0.some-name-rs0.demand-backup-15110 mongodb '' ''
+ local 'command=use myApp\n db.test.find()'
+ local uri=myApp:myPass@some-name-rs0-0.some-name-rs0.demand-backup-15110
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local replica_set=rs0
+ [[ myApp:myPass@some-name-rs0-0.some-name-rs0.demand-backup-15110 == *cfg* ]]
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+ grep -E -v 'I NETWORK|W NETWORK|F NETWORK|"c":"NETWORK"|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|Started a new thread for the timer service'
+ /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.qrVzI5da03
+++ mktemp
++ local LAST_ERR=/tmp/tmp.6AkDBy1vgF
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.qrVzI5da03
++ cat /tmp/tmp.6AkDBy1vgF
++ rm /tmp/tmp.qrVzI5da03 /tmp/tmp.6AkDBy1vgF
++ return 0
+ local client_container=psmdb-client-bb8b97679-rgs8m
+ kubectl_bin exec psmdb-client-bb8b97679-rgs8m -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-0.some-name-rs0.demand-backup-15110.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
++ mktemp
+ local LAST_OUT=/tmp/tmp.yKldqSoBuf
++ mktemp
+ local LAST_ERR=/tmp/tmp.ZwILAODUjC
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-bb8b97679-rgs8m -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-0.some-name-rs0.demand-backup-15110.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.yKldqSoBuf
+ cat /tmp/tmp.ZwILAODUjC
+ rm /tmp/tmp.yKldqSoBuf /tmp/tmp.ZwILAODUjC
+ return 0
+ [[ 0 -eq 0 ]]
+ diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2283/e2e-tests/demand-backup/compare/find.json /tmp/tmp.SOjtfW5Qlc/find
+ for i in $(seq 0 $last_pod)
+ compare_mongo_cmd find myApp:myPass@some-name-rs0-1.some-name-rs0.demand-backup-15110
+ local command=find
+ local uri=myApp:myPass@some-name-rs0-1.some-name-rs0.demand-backup-15110
+ local postfix=
+ local suffix=
+ local database=myApp
+ local collection=test
+ local sort=
+ local tls=false
+ local replicaset=
+ local 'full_command=db.test.find()'
+ [[ -n '' ]]
+ log 'running db.test.find() in myApp'
+ set +o xtrace
[2026-03-27T11:33:12+0000] running db.test.find() in myApp
+ [[ false == \t\r\u\e ]]
+ mongo_command=run_mongo
+ run_mongo 'use myApp\n db.test.find()' myApp:myPass@some-name-rs0-1.some-name-rs0.demand-backup-15110 mongodb '' ''
+ local 'command=use myApp\n db.test.find()'
+ local uri=myApp:myPass@some-name-rs0-1.some-name-rs0.demand-backup-15110
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local replica_set=rs0
+ [[ myApp:myPass@some-name-rs0-1.some-name-rs0.demand-backup-15110 == *cfg* ]]
+ /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
+ grep -E -v 'I NETWORK|W NETWORK|F NETWORK|"c":"NETWORK"|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|Started a new thread for the timer service'
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.4MgYl3dZP2
+++ mktemp
++ local LAST_ERR=/tmp/tmp.zLqv8bSD3k
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.4MgYl3dZP2
++ cat /tmp/tmp.zLqv8bSD3k
++ rm /tmp/tmp.4MgYl3dZP2 /tmp/tmp.zLqv8bSD3k
++ return 0
+ local client_container=psmdb-client-bb8b97679-rgs8m
+ kubectl_bin exec psmdb-client-bb8b97679-rgs8m -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-1.some-name-rs0.demand-backup-15110.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
++ mktemp
+ local LAST_OUT=/tmp/tmp.QJaezK1LPZ
++ mktemp
+ local LAST_ERR=/tmp/tmp.Y8mjYtTl4b
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-bb8b97679-rgs8m -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-1.some-name-rs0.demand-backup-15110.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.QJaezK1LPZ
+ cat /tmp/tmp.Y8mjYtTl4b
+ rm /tmp/tmp.QJaezK1LPZ /tmp/tmp.Y8mjYtTl4b
+ return 0
+ [[ 0 -eq 0 ]]
+ diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2283/e2e-tests/demand-backup/compare/find.json /tmp/tmp.SOjtfW5Qlc/find
+ for i in $(seq 0 $last_pod)
+ compare_mongo_cmd find myApp:myPass@some-name-rs0-2.some-name-rs0.demand-backup-15110
+ local command=find
+ local uri=myApp:myPass@some-name-rs0-2.some-name-rs0.demand-backup-15110
+ local postfix=
+ local suffix=
+ local database=myApp
+ local collection=test
+ local sort=
+ local tls=false
+ local replicaset=
+ local 'full_command=db.test.find()'
+ [[ -n '' ]]
+ log 'running db.test.find() in myApp'
+ set +o xtrace
[2026-03-27T11:33:14+0000] running db.test.find() in myApp
+ [[ false == \t\r\u\e ]]
+ mongo_command=run_mongo
+ run_mongo 'use myApp\n db.test.find()' myApp:myPass@some-name-rs0-2.some-name-rs0.demand-backup-15110 mongodb '' ''
+ local 'command=use myApp\n db.test.find()'
+ local uri=myApp:myPass@some-name-rs0-2.some-name-rs0.demand-backup-15110
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local replica_set=rs0
+ [[ myApp:myPass@some-name-rs0-2.some-name-rs0.demand-backup-15110 == *cfg* ]]
+ grep -E -v 'I NETWORK|W NETWORK|F NETWORK|"c":"NETWORK"|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|Started a new thread for the timer service'
+ /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.HK44yJDaYw
+++ mktemp
++ local LAST_ERR=/tmp/tmp.u8IyvEq3LW
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.HK44yJDaYw
++ cat /tmp/tmp.u8IyvEq3LW
++ rm /tmp/tmp.HK44yJDaYw /tmp/tmp.u8IyvEq3LW
++ return 0
+ local client_container=psmdb-client-bb8b97679-rgs8m
+ kubectl_bin exec psmdb-client-bb8b97679-rgs8m -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-2.some-name-rs0.demand-backup-15110.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
++ mktemp
+ local LAST_OUT=/tmp/tmp.IGR9mOeyag
++ mktemp
+ local LAST_ERR=/tmp/tmp.afpfLqys86
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-bb8b97679-rgs8m -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-2.some-name-rs0.demand-backup-15110.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.IGR9mOeyag
+ cat /tmp/tmp.afpfLqys86
+ rm /tmp/tmp.IGR9mOeyag /tmp/tmp.afpfLqys86
+ return 0
+ [[ 0 -eq 0 ]]
+ diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2283/e2e-tests/demand-backup/compare/find.json /tmp/tmp.SOjtfW5Qlc/find
+ desc 'check backup and restore -- gcp-cs-sa'
+ set +o xtrace


-----------------------------------------------------------------------------------
check backup and restore -- gcp-cs-sa
-----------------------------------------------------------------------------------

++ get_backup_dest backup-gcp-cs-sa
++ local backup_name=backup-gcp-cs-sa
++ kubectl_bin get psmdb-backup backup-gcp-cs-sa -o 'jsonpath={.status.destination}'
++ sed -e 's/.json$//'
+++ mktemp
++ sed 's|s3://||'
++ sed 's|azure://||'
++ sed 's|gs://||'
++ local LAST_OUT=/tmp/tmp.kUdt4ck58S
+++ mktemp
++ local LAST_ERR=/tmp/tmp.KrVv6Uy0xD
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb-backup backup-gcp-cs-sa -o 'jsonpath={.status.destination}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.kUdt4ck58S
++ cat /tmp/tmp.KrVv6Uy0xD
++ rm /tmp/tmp.kUdt4ck58S /tmp/tmp.KrVv6Uy0xD
++ return 0
+ backup_dest_gcp_sa=operator-testing/psmdb-demand-backup/2026-03-27T11:11:07Z
+ check_backup_existence_gcs operator-testing/psmdb-demand-backup/2026-03-27T11:11:07Z /rs0/myApp.test.gz
+ backup_dest_gcp=operator-testing/psmdb-demand-backup/2026-03-27T11:11:07Z
+ obj=/rs0/myApp.test.gz
+ storage_name=gcp-cs
+ retry=0
+ gcs_path=gs://operator-testing/psmdb-demand-backup/2026-03-27T11:11:07Z/rs0/myApp.test.gz
+ gsutil ls gs://operator-testing/psmdb-demand-backup/2026-03-27T11:11:07Z/rs0/myApp.test.gz
+ echo 'Backup gs://operator-testing/psmdb-demand-backup/2026-03-27T11:11:07Z/rs0/myApp.test.gz found in gcp-cs'
Backup gs://operator-testing/psmdb-demand-backup/2026-03-27T11:11:07Z/rs0/myApp.test.gz found in gcp-cs
+ run_recovery_check backup-gcp-cs-sa some-name-rs0
+ local backup=backup-gcp-cs-sa
+ local cluster=some-name-rs0
+ run_mongo 'use myApp\n db.test.insert({ x: 100501 })' myApp:myPass@some-name-rs0.demand-backup-15110
+ local 'command=use myApp\n db.test.insert({ x: 100501 })'
+ local uri=myApp:myPass@some-name-rs0.demand-backup-15110
+ local driver=mongodb+srv
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local replica_set=rs0
+ [[ myApp:myPass@some-name-rs0.demand-backup-15110 == *cfg* ]]
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.d0n5aMBLCM
+++ mktemp
++ local LAST_ERR=/tmp/tmp.driMQ2U1cl
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.d0n5aMBLCM
++ cat /tmp/tmp.driMQ2U1cl
++ rm /tmp/tmp.d0n5aMBLCM /tmp/tmp.driMQ2U1cl
++ return 0
+ local client_container=psmdb-client-bb8b97679-rgs8m
+ kubectl_bin exec psmdb-client-bb8b97679-rgs8m -- bash -c 'printf '\''use myApp\n db.test.insert({ x: 100501 })\n'\'' | mongo mongodb+srv://myApp:myPass@some-name-rs0.demand-backup-15110.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
++ mktemp
+ local LAST_OUT=/tmp/tmp.CMorztkha8
++ mktemp
+ local LAST_ERR=/tmp/tmp.na812TCqFU
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-bb8b97679-rgs8m -- bash -c 'printf '\''use myApp\n db.test.insert({ x: 100501 })\n'\'' | mongo mongodb+srv://myApp:myPass@some-name-rs0.demand-backup-15110.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.CMorztkha8
Percona Server for MongoDB shell version v4.4.29-28
connecting to: mongodb://some-name-rs0-0.some-name-rs0.demand-backup-15110.svc.cluster.local:27017,some-name-rs0-2.some-name-rs0.demand-backup-15110.svc.cluster.local:27017,some-name-rs0-1.some-name-rs0.demand-backup-15110.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb&replicaSet=rs0&ssl=false
Implicit session: session { "id" : UUID("30bd5dbb-edca-4c19-b24f-f4700cd1997f") }
Percona Server for MongoDB server version: v8.0.19-7
WARNING: shell and server versions do not match
switched to db myApp
WriteResult({ "nInserted" : 1 })
bye
+ cat /tmp/tmp.na812TCqFU
+ rm /tmp/tmp.CMorztkha8 /tmp/tmp.na812TCqFU
+ return 0
+ compare_mongo_cmd find myApp:myPass@some-name-rs0.demand-backup-15110 -2nd .svc.cluster.local myApp test
+ local command=find
+ local uri=myApp:myPass@some-name-rs0.demand-backup-15110
+ local postfix=-2nd
+ local suffix=.svc.cluster.local
+ local database=myApp
+ local collection=test
+ local sort=
+ local tls=false
+ local replicaset=
+ local 'full_command=db.test.find()'
+ [[ -n '' ]]
+ log 'running db.test.find() in myApp'
+ set +o xtrace
[2026-03-27T11:33:21+0000] running db.test.find() in myApp
+ [[ false == \t\r\u\e ]]
+ mongo_command=run_mongo
+ run_mongo 'use myApp\n db.test.find()' myApp:myPass@some-name-rs0.demand-backup-15110 mongodb .svc.cluster.local ''
+ local 'command=use myApp\n db.test.find()'
+ local uri=myApp:myPass@some-name-rs0.demand-backup-15110
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ grep -E -v 'I NETWORK|W NETWORK|F NETWORK|"c":"NETWORK"|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|Started a new thread for the timer service'
+ local replica_set=rs0
+ [[ myApp:myPass@some-name-rs0.demand-backup-15110 == *cfg* ]]
+ /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.xKCNFADEr2
+++ mktemp
++ local LAST_ERR=/tmp/tmp.AIgk4Ef7cm
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.xKCNFADEr2
++ cat /tmp/tmp.AIgk4Ef7cm
++ rm /tmp/tmp.xKCNFADEr2 /tmp/tmp.AIgk4Ef7cm
++ return 0
+ local client_container=psmdb-client-bb8b97679-rgs8m
+ kubectl_bin exec psmdb-client-bb8b97679-rgs8m -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0.demand-backup-15110.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
++ mktemp
+ local LAST_OUT=/tmp/tmp.v4nKoDsqQA
++ mktemp
+ local LAST_ERR=/tmp/tmp.hXDseddBlQ
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-bb8b97679-rgs8m -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0.demand-backup-15110.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.v4nKoDsqQA
+ cat /tmp/tmp.hXDseddBlQ
+ rm /tmp/tmp.v4nKoDsqQA /tmp/tmp.hXDseddBlQ
+ return 0
+ [[ 0 -eq 0 ]]
+ diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2283/e2e-tests/demand-backup/compare/find-2nd.json /tmp/tmp.SOjtfW5Qlc/find-2nd
+ run_restore backup-gcp-cs-sa
+ local backup_name=backup-gcp-cs-sa
+ local restore_file=/mnt/jenkins/workspace/cloud-psmdb-operator_PR-2283/e2e-tests/demand-backup/conf/restore.yml
+ log 'running restore restore-backup-gcp-cs-sa'
+ set +o xtrace
[2026-03-27T11:33:24+0000] running restore restore-backup-gcp-cs-sa
+ cat /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2283/e2e-tests/demand-backup/conf/restore.yml
+ /usr/sbin/sed -e 's/name:/name: restore-backup-gcp-cs-sa/'
+ /usr/sbin/sed -e 's/backupName:/backupName: backup-gcp-cs-sa/'
+ kubectl_bin apply -f -
++ mktemp
+ local LAST_OUT=/tmp/tmp.FjenWXfGbr
++ mktemp
+ local LAST_ERR=/tmp/tmp.Dv4ptB0CWE
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl apply -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.FjenWXfGbr
perconaservermongodbrestore.psmdb.percona.com/restore-backup-gcp-cs-sa created
+ cat /tmp/tmp.Dv4ptB0CWE
+ rm /tmp/tmp.FjenWXfGbr /tmp/tmp.Dv4ptB0CWE
+ return 0
+ wait_restore backup-gcp-cs-sa some-name
+ local backup_name=backup-gcp-cs-sa
+ local cluster_name=some-name
+ local target_state=ready
+ local wait_cluster_consistency=1
+ local wait_time=1780
+ local ok_if_ready=0
+ set +o xtrace
Waiting for the psmdb-restore/restore-backup-gcp-cs-sa object to be created.OK
Waiting psmdb-restore/restore-backup-gcp-cs-sa to reach state "ready" .OK after 0 minutes
+ [[ 1 -eq 1 ]]
+ wait_cluster_consistency some-name
+ local cluster_name=some-name
+ local wait_time=32
+ retry=0
+ sleep 7
+ echo -n 'waiting for cluster readyness'
waiting for cluster readyness++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.SnIaO30B2U
+++ mktemp
++ local LAST_ERR=/tmp/tmp.mAK85cGrO8
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.SnIaO30B2U
++ cat /tmp/tmp.mAK85cGrO8
++ rm /tmp/tmp.SnIaO30B2U /tmp/tmp.mAK85cGrO8
++ return 0
+ [[ ready == \r\e\a\d\y ]]
+ echo .OK
.OK
+ simple_data_check some-name-rs0 3 0
+ local cluster_name=some-name-rs0
+ let last_pod=3-1
+ local isSharded=0
+ local cluster_pfx=
+ '[' 0 -eq 1 ']'
++ seq 0 2
+ for i in $(seq 0 $last_pod)
+ compare_mongo_cmd find myApp:myPass@some-name-rs0-0.some-name-rs0.demand-backup-15110
+ local command=find
+ local uri=myApp:myPass@some-name-rs0-0.some-name-rs0.demand-backup-15110
+ local postfix=
+ local suffix=
+ local database=myApp
+ local collection=test
+ local sort=
+ local tls=false
+ local replicaset=
+ local 'full_command=db.test.find()'
+ [[ -n '' ]]
+ log 'running db.test.find() in myApp'
+ set +o xtrace
[2026-03-27T11:33:46+0000] running db.test.find() in myApp
+ [[ false == \t\r\u\e ]]
+ mongo_command=run_mongo
+ run_mongo 'use myApp\n db.test.find()' myApp:myPass@some-name-rs0-0.some-name-rs0.demand-backup-15110 mongodb '' ''
+ grep -E -v 'I NETWORK|W NETWORK|F NETWORK|"c":"NETWORK"|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|Started a new thread for the timer service'
+ local 'command=use myApp\n db.test.find()'
+ local uri=myApp:myPass@some-name-rs0-0.some-name-rs0.demand-backup-15110
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local replica_set=rs0
+ [[ myApp:myPass@some-name-rs0-0.some-name-rs0.demand-backup-15110 == *cfg* ]]
+ /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.I4d7wRaUzi
+++ mktemp
++ local LAST_ERR=/tmp/tmp.UbzQJ6XTZs
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.I4d7wRaUzi
++ cat /tmp/tmp.UbzQJ6XTZs
++ rm /tmp/tmp.I4d7wRaUzi /tmp/tmp.UbzQJ6XTZs
++ return 0
+ local client_container=psmdb-client-bb8b97679-rgs8m
+ kubectl_bin exec psmdb-client-bb8b97679-rgs8m -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-0.some-name-rs0.demand-backup-15110.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
++ mktemp
+ local LAST_OUT=/tmp/tmp.yFLoVWsB2o
++ mktemp
+ local LAST_ERR=/tmp/tmp.Ap6HtrTcVf
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-bb8b97679-rgs8m -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-0.some-name-rs0.demand-backup-15110.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.yFLoVWsB2o
+ cat /tmp/tmp.Ap6HtrTcVf
+ rm /tmp/tmp.yFLoVWsB2o /tmp/tmp.Ap6HtrTcVf
+ return 0
+ [[ 0 -eq 0 ]]
+ diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2283/e2e-tests/demand-backup/compare/find.json /tmp/tmp.SOjtfW5Qlc/find
+ for i in $(seq 0 $last_pod)
+ compare_mongo_cmd find myApp:myPass@some-name-rs0-1.some-name-rs0.demand-backup-15110
+ local command=find
+ local uri=myApp:myPass@some-name-rs0-1.some-name-rs0.demand-backup-15110
+ local postfix=
+ local suffix=
+ local database=myApp
+ local collection=test
+ local sort=
+ local tls=false
+ local replicaset=
+ local 'full_command=db.test.find()'
+ [[ -n '' ]]
+ log 'running db.test.find() in myApp'
+ set +o xtrace
[2026-03-27T11:33:49+0000] running db.test.find() in myApp
+ [[ false == \t\r\u\e ]]
+ mongo_command=run_mongo
+ run_mongo 'use myApp\n db.test.find()' myApp:myPass@some-name-rs0-1.some-name-rs0.demand-backup-15110 mongodb '' ''
+ local 'command=use myApp\n db.test.find()'
+ local uri=myApp:myPass@some-name-rs0-1.some-name-rs0.demand-backup-15110
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local replica_set=rs0
+ [[ myApp:myPass@some-name-rs0-1.some-name-rs0.demand-backup-15110 == *cfg* ]]
+ grep -E -v 'I NETWORK|W NETWORK|F NETWORK|"c":"NETWORK"|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|Started a new thread for the timer service'
+ /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.5qUrkdYy9d
+++ mktemp
++ local LAST_ERR=/tmp/tmp.K01M5pfFNL
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.5qUrkdYy9d
++ cat /tmp/tmp.K01M5pfFNL
++ rm /tmp/tmp.5qUrkdYy9d /tmp/tmp.K01M5pfFNL
++ return 0
+ local client_container=psmdb-client-bb8b97679-rgs8m
+ kubectl_bin exec psmdb-client-bb8b97679-rgs8m -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-1.some-name-rs0.demand-backup-15110.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
++ mktemp
+ local LAST_OUT=/tmp/tmp.rE5ICIb4az
++ mktemp
+ local LAST_ERR=/tmp/tmp.I6qKN2sjAl
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-bb8b97679-rgs8m -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-1.some-name-rs0.demand-backup-15110.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.rE5ICIb4az
+ cat /tmp/tmp.I6qKN2sjAl
+ rm /tmp/tmp.rE5ICIb4az /tmp/tmp.I6qKN2sjAl
+ return 0
+ [[ 0 -eq 0 ]]
+ diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2283/e2e-tests/demand-backup/compare/find.json /tmp/tmp.SOjtfW5Qlc/find
+ for i in $(seq 0 $last_pod)
+ compare_mongo_cmd find myApp:myPass@some-name-rs0-2.some-name-rs0.demand-backup-15110
+ local command=find
+ local uri=myApp:myPass@some-name-rs0-2.some-name-rs0.demand-backup-15110
+ local postfix=
+ local suffix=
+ local database=myApp
+ local collection=test
+ local sort=
+ local tls=false
+ local replicaset=
+ local 'full_command=db.test.find()'
+ [[ -n '' ]]
+ log 'running db.test.find() in myApp'
+ set +o xtrace
[2026-03-27T11:33:51+0000] running db.test.find() in myApp
+ [[ false == \t\r\u\e ]]
+ mongo_command=run_mongo
+ run_mongo 'use myApp\n db.test.find()' myApp:myPass@some-name-rs0-2.some-name-rs0.demand-backup-15110 mongodb '' ''
+ local 'command=use myApp\n db.test.find()'
+ local uri=myApp:myPass@some-name-rs0-2.some-name-rs0.demand-backup-15110
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local replica_set=rs0
+ [[ myApp:myPass@some-name-rs0-2.some-name-rs0.demand-backup-15110 == *cfg* ]]
+ grep -E -v 'I NETWORK|W NETWORK|F NETWORK|"c":"NETWORK"|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|Started a new thread for the timer service'
+ /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.PSvAkkelXy
+++ mktemp
++ local LAST_ERR=/tmp/tmp.52iXLC4W5K
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.PSvAkkelXy
++ cat /tmp/tmp.52iXLC4W5K
++ rm /tmp/tmp.PSvAkkelXy /tmp/tmp.52iXLC4W5K
++ return 0
+ local client_container=psmdb-client-bb8b97679-rgs8m
+ kubectl_bin exec psmdb-client-bb8b97679-rgs8m -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-2.some-name-rs0.demand-backup-15110.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
++ mktemp
+ local LAST_OUT=/tmp/tmp.m2zjOM5h0g
++ mktemp
+ local LAST_ERR=/tmp/tmp.NiG4CULIKe
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-bb8b97679-rgs8m -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-2.some-name-rs0.demand-backup-15110.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.m2zjOM5h0g
+ cat /tmp/tmp.NiG4CULIKe
+ rm /tmp/tmp.m2zjOM5h0g /tmp/tmp.NiG4CULIKe
+ return 0
+ [[ 0 -eq 0 ]]
+ diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2283/e2e-tests/demand-backup/compare/find.json /tmp/tmp.SOjtfW5Qlc/find
+ desc 'check backup and restore -- azure-blob'
+ set +o xtrace


-----------------------------------------------------------------------------------
check backup and restore -- azure-blob
-----------------------------------------------------------------------------------

++ get_backup_dest backup-azure-blob
++ local backup_name=backup-azure-blob
++ kubectl_bin get psmdb-backup backup-azure-blob -o 'jsonpath={.status.destination}'
++ sed -e 's/.json$//'
++ sed 's|s3://||'
+++ mktemp
++ sed 's|azure://||'
++ sed 's|gs://||'
++ local LAST_OUT=/tmp/tmp.YiiSDt2131
+++ mktemp
++ local LAST_ERR=/tmp/tmp.b8HUQ6lIrg
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb-backup backup-azure-blob -o 'jsonpath={.status.destination}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.YiiSDt2131
++ cat /tmp/tmp.b8HUQ6lIrg
++ rm /tmp/tmp.YiiSDt2131 /tmp/tmp.b8HUQ6lIrg
++ return 0
+ backup_dest_azure=https://engk8soperators.blob.core.windows.net/operator-testing/psmdb-demand-backup/2026-03-27T11:10:28Z
+ setup_azure_credentials
+ local secret_name=azure-secret
+ echo 'Setting up Azure credentials from K8s secret: azure-secret'
Setting up Azure credentials from K8s secret: azure-secret
+ local trace_was_on=0
+ [[ ehxB == *x* ]]
+ trace_was_on=1
+ set +x
+ echo 'Azure credentials configured successfully'
Azure credentials configured successfully
+ check_backup_existence_azure https://engk8soperators.blob.core.windows.net/operator-testing/psmdb-demand-backup/2026-03-27T11:10:28Z /rs0/myApp.test.gz
++ echo https://engk8soperators.blob.core.windows.net/operator-testing/psmdb-demand-backup/2026-03-27T11:10:28Z
++ sed 's|https://[^/]*\.blob\.core\.windows\.net/||'
+ url_path=operator-testing/psmdb-demand-backup/2026-03-27T11:10:28Z
++ echo operator-testing/psmdb-demand-backup/2026-03-27T11:10:28Z
++ cut -d/ -f1
+ container=operator-testing
++ echo operator-testing/psmdb-demand-backup/2026-03-27T11:10:28Z
++ cut -d/ -f2-
+ blob_prefix=psmdb-demand-backup/2026-03-27T11:10:28Z
+ blob=/rs0/myApp.test.gz
+ storage_name=azure-blob
+ retry=0
+ blob_path=psmdb-demand-backup/2026-03-27T11:10:28Z/rs0/myApp.test.gz
+ az storage blob show --container-name operator-testing --name psmdb-demand-backup/2026-03-27T11:10:28Z/rs0/myApp.test.gz
+ echo 'Backup psmdb-demand-backup/2026-03-27T11:10:28Z/rs0/myApp.test.gz found in azure-blob'
Backup psmdb-demand-backup/2026-03-27T11:10:28Z/rs0/myApp.test.gz found in azure-blob
+ run_recovery_check backup-azure-blob some-name-rs0
+ local backup=backup-azure-blob
+ local cluster=some-name-rs0
+ run_mongo 'use myApp\n db.test.insert({ x: 100501 })' myApp:myPass@some-name-rs0.demand-backup-15110
+ local 'command=use myApp\n db.test.insert({ x: 100501 })'
+ local uri=myApp:myPass@some-name-rs0.demand-backup-15110
+ local driver=mongodb+srv
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local replica_set=rs0
+ [[ myApp:myPass@some-name-rs0.demand-backup-15110 == *cfg* ]]
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.dlluSqwIK7
+++ mktemp
++ local LAST_ERR=/tmp/tmp.7faqOVPe4t
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.dlluSqwIK7
++ cat /tmp/tmp.7faqOVPe4t
++ rm /tmp/tmp.dlluSqwIK7 /tmp/tmp.7faqOVPe4t
++ return 0
+ local client_container=psmdb-client-bb8b97679-rgs8m
+ kubectl_bin exec psmdb-client-bb8b97679-rgs8m -- bash -c 'printf '\''use myApp\n db.test.insert({ x: 100501 })\n'\'' | mongo mongodb+srv://myApp:myPass@some-name-rs0.demand-backup-15110.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
++ mktemp
+ local LAST_OUT=/tmp/tmp.L9H8YqVAJb
++ mktemp
+ local LAST_ERR=/tmp/tmp.yWau579t02
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-bb8b97679-rgs8m -- bash -c 'printf '\''use myApp\n db.test.insert({ x: 100501 })\n'\'' | mongo mongodb+srv://myApp:myPass@some-name-rs0.demand-backup-15110.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.L9H8YqVAJb
Percona Server for MongoDB shell version v4.4.29-28
connecting to: mongodb://some-name-rs0-0.some-name-rs0.demand-backup-15110.svc.cluster.local:27017,some-name-rs0-2.some-name-rs0.demand-backup-15110.svc.cluster.local:27017,some-name-rs0-1.some-name-rs0.demand-backup-15110.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb&replicaSet=rs0&ssl=false
Implicit session: session { "id" : UUID("1ad9aae1-0843-4544-b5a4-039e542d7a27") }
Percona Server for MongoDB server version: v8.0.19-7
WARNING: shell and server versions do not match
switched to db myApp
WriteResult({ "nInserted" : 1 })
bye
+ cat /tmp/tmp.yWau579t02
+ rm /tmp/tmp.L9H8YqVAJb /tmp/tmp.yWau579t02
+ return 0
+ compare_mongo_cmd find myApp:myPass@some-name-rs0.demand-backup-15110 -2nd .svc.cluster.local myApp test
+ local command=find
+ local uri=myApp:myPass@some-name-rs0.demand-backup-15110
+ local postfix=-2nd
+ local suffix=.svc.cluster.local
+ local database=myApp
+ local collection=test
+ local sort=
+ local tls=false
+ local replicaset=
+ local 'full_command=db.test.find()'
+ [[ -n '' ]]
+ log 'running db.test.find() in myApp'
+ set +o xtrace
[2026-03-27T11:34:00+0000] running db.test.find() in myApp
+ [[ false == \t\r\u\e ]]
+ mongo_command=run_mongo
+ grep -E -v 'I NETWORK|W NETWORK|F NETWORK|"c":"NETWORK"|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|Started a new thread for the timer service'
+ run_mongo 'use myApp\n db.test.find()' myApp:myPass@some-name-rs0.demand-backup-15110 mongodb .svc.cluster.local ''
+ /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
+ local 'command=use myApp\n db.test.find()'
+ local uri=myApp:myPass@some-name-rs0.demand-backup-15110
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local replica_set=rs0
+ [[ myApp:myPass@some-name-rs0.demand-backup-15110 == *cfg* ]]
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.YXx4jEO7lB
+++ mktemp
++ local LAST_ERR=/tmp/tmp.KBR23FO2F4
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.YXx4jEO7lB
++ cat /tmp/tmp.KBR23FO2F4
++ rm /tmp/tmp.YXx4jEO7lB /tmp/tmp.KBR23FO2F4
++ return 0
+ local client_container=psmdb-client-bb8b97679-rgs8m
+ kubectl_bin exec psmdb-client-bb8b97679-rgs8m -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0.demand-backup-15110.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
++ mktemp
+ local LAST_OUT=/tmp/tmp.c0LB3uW7ZY
++ mktemp
+ local LAST_ERR=/tmp/tmp.YqsS8XBqjY
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-bb8b97679-rgs8m -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0.demand-backup-15110.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.c0LB3uW7ZY
+ cat /tmp/tmp.YqsS8XBqjY
+ rm /tmp/tmp.c0LB3uW7ZY /tmp/tmp.YqsS8XBqjY
+ return 0
+ [[ 0 -eq 0 ]]
+ diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2283/e2e-tests/demand-backup/compare/find-2nd.json /tmp/tmp.SOjtfW5Qlc/find-2nd
+ run_restore backup-azure-blob
+ local backup_name=backup-azure-blob
+ local restore_file=/mnt/jenkins/workspace/cloud-psmdb-operator_PR-2283/e2e-tests/demand-backup/conf/restore.yml
+ log 'running restore restore-backup-azure-blob'
+ set +o xtrace
[2026-03-27T11:34:03+0000] running restore restore-backup-azure-blob
+ /usr/sbin/sed -e 's/name:/name: restore-backup-azure-blob/'
+ cat /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2283/e2e-tests/demand-backup/conf/restore.yml
+ /usr/sbin/sed -e 's/backupName:/backupName: backup-azure-blob/'
+ kubectl_bin apply -f -
++ mktemp
+ local LAST_OUT=/tmp/tmp.Y1Z6zZAJS2
++ mktemp
+ local LAST_ERR=/tmp/tmp.z6TsdbIrh2
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl apply -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.Y1Z6zZAJS2
perconaservermongodbrestore.psmdb.percona.com/restore-backup-azure-blob created
+ cat /tmp/tmp.z6TsdbIrh2
+ rm /tmp/tmp.Y1Z6zZAJS2 /tmp/tmp.z6TsdbIrh2
+ return 0
+ wait_restore backup-azure-blob some-name
+ local backup_name=backup-azure-blob
+ local cluster_name=some-name
+ local target_state=ready
+ local wait_cluster_consistency=1
+ local wait_time=1780
+ local ok_if_ready=0
+ set +o xtrace
Waiting for the psmdb-restore/restore-backup-azure-blob object to be created.OK
Waiting psmdb-restore/restore-backup-azure-blob to reach state "ready" .OK after 0 minutes
+ [[ 1 -eq 1 ]]
+ wait_cluster_consistency some-name
+ local cluster_name=some-name
+ local wait_time=32
+ retry=0
+ sleep 7
+ echo -n 'waiting for cluster readyness'
waiting for cluster readyness++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.7HJ9pVmHG3
+++ mktemp
++ local LAST_ERR=/tmp/tmp.aOwpAwLShY
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.7HJ9pVmHG3
++ cat /tmp/tmp.aOwpAwLShY
++ rm /tmp/tmp.7HJ9pVmHG3 /tmp/tmp.aOwpAwLShY
++ return 0
+ [[ ready == \r\e\a\d\y ]]
+ echo .OK
.OK
+ simple_data_check some-name-rs0 3 0
+ local cluster_name=some-name-rs0
+ let last_pod=3-1
+ local isSharded=0
+ local cluster_pfx=
+ '[' 0 -eq 1 ']'
++ seq 0 2
+ for i in $(seq 0 $last_pod)
+ compare_mongo_cmd find myApp:myPass@some-name-rs0-0.some-name-rs0.demand-backup-15110
+ local command=find
+ local uri=myApp:myPass@some-name-rs0-0.some-name-rs0.demand-backup-15110
+ local postfix=
+ local suffix=
+ local database=myApp
+ local collection=test
+ local sort=
+ local tls=false
+ local replicaset=
+ local 'full_command=db.test.find()'
+ [[ -n '' ]]
+ log 'running db.test.find() in myApp'
+ set +o xtrace
[2026-03-27T11:34:25+0000] running db.test.find() in myApp
+ [[ false == \t\r\u\e ]]
+ mongo_command=run_mongo
+ run_mongo 'use myApp\n db.test.find()' myApp:myPass@some-name-rs0-0.some-name-rs0.demand-backup-15110 mongodb '' ''
+ local 'command=use myApp\n db.test.find()'
+ local uri=myApp:myPass@some-name-rs0-0.some-name-rs0.demand-backup-15110
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local replica_set=rs0
+ [[ myApp:myPass@some-name-rs0-0.some-name-rs0.demand-backup-15110 == *cfg* ]]
+ grep -E -v 'I NETWORK|W NETWORK|F NETWORK|"c":"NETWORK"|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|Started a new thread for the timer service'
+ /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.rOfgBNaqC6
+++ mktemp
++ local LAST_ERR=/tmp/tmp.UhjnC3FZdT
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.rOfgBNaqC6
++ cat /tmp/tmp.UhjnC3FZdT
++ rm /tmp/tmp.rOfgBNaqC6 /tmp/tmp.UhjnC3FZdT
++ return 0
+ local client_container=psmdb-client-bb8b97679-rgs8m
+ kubectl_bin exec psmdb-client-bb8b97679-rgs8m -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-0.some-name-rs0.demand-backup-15110.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
++ mktemp
+ local LAST_OUT=/tmp/tmp.ArprFZHblP
++ mktemp
+ local LAST_ERR=/tmp/tmp.MVGCPJExvd
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-bb8b97679-rgs8m -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-0.some-name-rs0.demand-backup-15110.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.ArprFZHblP
+ cat /tmp/tmp.MVGCPJExvd
+ rm /tmp/tmp.ArprFZHblP /tmp/tmp.MVGCPJExvd
+ return 0
+ [[ 0 -eq 0 ]]
+ diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2283/e2e-tests/demand-backup/compare/find.json /tmp/tmp.SOjtfW5Qlc/find
+ for i in $(seq 0 $last_pod)
+ compare_mongo_cmd find myApp:myPass@some-name-rs0-1.some-name-rs0.demand-backup-15110
+ local command=find
+ local uri=myApp:myPass@some-name-rs0-1.some-name-rs0.demand-backup-15110
+ local postfix=
+ local suffix=
+ local database=myApp
+ local collection=test
+ local sort=
+ local tls=false
+ local replicaset=
+ local 'full_command=db.test.find()'
+ [[ -n '' ]]
+ log 'running db.test.find() in myApp'
+ set +o xtrace
[2026-03-27T11:34:27+0000] running db.test.find() in myApp
+ [[ false == \t\r\u\e ]]
+ mongo_command=run_mongo
+ run_mongo 'use myApp\n db.test.find()' myApp:myPass@some-name-rs0-1.some-name-rs0.demand-backup-15110 mongodb '' ''
+ local 'command=use myApp\n db.test.find()'
+ local uri=myApp:myPass@some-name-rs0-1.some-name-rs0.demand-backup-15110
+ grep -E -v 'I NETWORK|W NETWORK|F NETWORK|"c":"NETWORK"|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|Started a new thread for the timer service'
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local replica_set=rs0
+ [[ myApp:myPass@some-name-rs0-1.some-name-rs0.demand-backup-15110 == *cfg* ]]
+ /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.Vmoi7SIA7A
+++ mktemp
++ local LAST_ERR=/tmp/tmp.8zljImhfmF
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.Vmoi7SIA7A
++ cat /tmp/tmp.8zljImhfmF
++ rm /tmp/tmp.Vmoi7SIA7A /tmp/tmp.8zljImhfmF
++ return 0
+ local client_container=psmdb-client-bb8b97679-rgs8m
+ kubectl_bin exec psmdb-client-bb8b97679-rgs8m -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-1.some-name-rs0.demand-backup-15110.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
++ mktemp
+ local LAST_OUT=/tmp/tmp.yyMsxiqzjf
++ mktemp
+ local LAST_ERR=/tmp/tmp.iMKxsBDHSC
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-bb8b97679-rgs8m -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-1.some-name-rs0.demand-backup-15110.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.yyMsxiqzjf
+ cat /tmp/tmp.iMKxsBDHSC
+ rm /tmp/tmp.yyMsxiqzjf /tmp/tmp.iMKxsBDHSC
+ return 0
+ [[ 0 -eq 0 ]]
+ diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2283/e2e-tests/demand-backup/compare/find.json /tmp/tmp.SOjtfW5Qlc/find
+ for i in $(seq 0 $last_pod)
+ compare_mongo_cmd find myApp:myPass@some-name-rs0-2.some-name-rs0.demand-backup-15110
+ local command=find
+ local uri=myApp:myPass@some-name-rs0-2.some-name-rs0.demand-backup-15110
+ local postfix=
+ local suffix=
+ local database=myApp
+ local collection=test
+ local sort=
+ local tls=false
+ local replicaset=
+ local 'full_command=db.test.find()'
+ [[ -n '' ]]
+ log 'running db.test.find() in myApp'
+ set +o xtrace
[2026-03-27T11:34:30+0000] running db.test.find() in myApp
+ [[ false == \t\r\u\e ]]
+ mongo_command=run_mongo
+ grep -E -v 'I NETWORK|W NETWORK|F NETWORK|"c":"NETWORK"|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|Started a new thread for the timer service'
+ run_mongo 'use myApp\n db.test.find()' myApp:myPass@some-name-rs0-2.some-name-rs0.demand-backup-15110 mongodb '' ''
+ /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
+ local 'command=use myApp\n db.test.find()'
+ local uri=myApp:myPass@some-name-rs0-2.some-name-rs0.demand-backup-15110
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local replica_set=rs0
+ [[ myApp:myPass@some-name-rs0-2.some-name-rs0.demand-backup-15110 == *cfg* ]]
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.N3IFroJNnI
+++ mktemp
++ local LAST_ERR=/tmp/tmp.TRFtiyVEej
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.N3IFroJNnI
++ cat /tmp/tmp.TRFtiyVEej
++ rm /tmp/tmp.N3IFroJNnI /tmp/tmp.TRFtiyVEej
++ return 0
+ local client_container=psmdb-client-bb8b97679-rgs8m
+ kubectl_bin exec psmdb-client-bb8b97679-rgs8m -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-2.some-name-rs0.demand-backup-15110.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
++ mktemp
+ local LAST_OUT=/tmp/tmp.0AEuZH6YPW
++ mktemp
+ local LAST_ERR=/tmp/tmp.zN1OobatEO
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-bb8b97679-rgs8m -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-2.some-name-rs0.demand-backup-15110.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.0AEuZH6YPW
+ cat /tmp/tmp.zN1OobatEO
+ rm /tmp/tmp.0AEuZH6YPW /tmp/tmp.zN1OobatEO
+ return 0
+ [[ 0 -eq 0 ]]
+ diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2283/e2e-tests/demand-backup/compare/find.json /tmp/tmp.SOjtfW5Qlc/find
+ desc 'check backup and restore -- minio'
+ set +o xtrace


-----------------------------------------------------------------------------------
check backup and restore -- minio
-----------------------------------------------------------------------------------

+ check_backup_in_storage backup-minio minio rs0 myApp.test.gz
+ local backup=backup-minio
+ local storage_type=minio
+ local replset=rs0
+ local file=myApp.test.gz
+ local protocol=http
+ local endpoint
++ get_backup_dest backup-minio
++ /usr/sbin/sed 's|https://engk8soperators.blob.core.windows.net/||'
++ local backup_name=backup-minio
++ kubectl_bin get psmdb-backup backup-minio -o 'jsonpath={.status.destination}'
++ sed -e 's/.json$//'
++ sed 's|s3://||'
+++ mktemp
++ sed 's|azure://||'
++ sed 's|gs://||'
++ local LAST_OUT=/tmp/tmp.g2kLuqpzfS
+++ mktemp
++ local LAST_ERR=/tmp/tmp.DjloDZfuey
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb-backup backup-minio -o 'jsonpath={.status.destination}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.g2kLuqpzfS
++ cat /tmp/tmp.DjloDZfuey
++ rm /tmp/tmp.g2kLuqpzfS /tmp/tmp.DjloDZfuey
++ return 0
+ backup_dest=operator-testing/2026-03-27T11:10:41Z
+ case ${storage_type} in
+ endpoint=minio-service
+ kubectl_bin run -i --rm aws-cli --image=docker.io/perconalab/awscli --restart=Never -- /usr/bin/env AWS_ACCESS_KEY_ID=some-access-key AWS_SECRET_ACCESS_KEY=some-secret-key AWS_DEFAULT_REGION=us-east-1 /usr/bin/aws --no-verify-ssl --endpoint-url http://minio-service:9000 s3 ls s3://operator-testing/2026-03-27T11:10:41Z/rs0/myApp.test.gz
+ grep myApp.test.gz
++ mktemp
+ local LAST_OUT=/tmp/tmp.MQXmFJVsCH
++ mktemp
+ local LAST_ERR=/tmp/tmp.NBcjReU9RG
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl run -i --rm aws-cli --image=docker.io/perconalab/awscli --restart=Never -- /usr/bin/env AWS_ACCESS_KEY_ID=some-access-key AWS_SECRET_ACCESS_KEY=some-secret-key AWS_DEFAULT_REGION=us-east-1 /usr/bin/aws --no-verify-ssl --endpoint-url http://minio-service:9000 s3 ls s3://operator-testing/2026-03-27T11:10:41Z/rs0/myApp.test.gz
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.MQXmFJVsCH
+ cat /tmp/tmp.NBcjReU9RG
All commands and output from this session will be recorded in container logs, including credentials and sensitive information passed through the command prompt.
If you don't see a command prompt, try pressing enter.
+ rm /tmp/tmp.MQXmFJVsCH /tmp/tmp.NBcjReU9RG
+ return 0
+ sleep 1
+ let retry+=1
+ '[' 1 -ge 60 ']'
+ kubectl_bin run -i --rm aws-cli --image=docker.io/perconalab/awscli --restart=Never -- /usr/bin/env AWS_ACCESS_KEY_ID=some-access-key AWS_SECRET_ACCESS_KEY=some-secret-key AWS_DEFAULT_REGION=us-east-1 /usr/bin/aws --no-verify-ssl --endpoint-url http://minio-service:9000 s3 ls s3://operator-testing/2026-03-27T11:10:41Z/rs0/myApp.test.gz
+ grep myApp.test.gz
++ mktemp
+ local LAST_OUT=/tmp/tmp.q5d1b3LDd7
++ mktemp
+ local LAST_ERR=/tmp/tmp.Zh5TLkAp9B
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl run -i --rm aws-cli --image=docker.io/perconalab/awscli --restart=Never -- /usr/bin/env AWS_ACCESS_KEY_ID=some-access-key AWS_SECRET_ACCESS_KEY=some-secret-key AWS_DEFAULT_REGION=us-east-1 /usr/bin/aws --no-verify-ssl --endpoint-url http://minio-service:9000 s3 ls s3://operator-testing/2026-03-27T11:10:41Z/rs0/myApp.test.gz
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.q5d1b3LDd7
+ cat /tmp/tmp.Zh5TLkAp9B
+ rm /tmp/tmp.q5d1b3LDd7 /tmp/tmp.Zh5TLkAp9B
+ return 0
2026-03-27 11:10:44         56 myApp.test.gz
+ run_recovery_check backup-minio some-name-rs0
+ local backup=backup-minio
+ local cluster=some-name-rs0
+ run_mongo 'use myApp\n db.test.insert({ x: 100501 })' myApp:myPass@some-name-rs0.demand-backup-15110
+ local 'command=use myApp\n db.test.insert({ x: 100501 })'
+ local uri=myApp:myPass@some-name-rs0.demand-backup-15110
+ local driver=mongodb+srv
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local replica_set=rs0
+ [[ myApp:myPass@some-name-rs0.demand-backup-15110 == *cfg* ]]
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.B82rVbnnjQ
+++ mktemp
++ local LAST_ERR=/tmp/tmp.EnnDB3ms8w
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.B82rVbnnjQ
++ cat /tmp/tmp.EnnDB3ms8w
++ rm /tmp/tmp.B82rVbnnjQ /tmp/tmp.EnnDB3ms8w
++ return 0
+ local client_container=psmdb-client-bb8b97679-rgs8m
+ kubectl_bin exec psmdb-client-bb8b97679-rgs8m -- bash -c 'printf '\''use myApp\n db.test.insert({ x: 100501 })\n'\'' | mongo mongodb+srv://myApp:myPass@some-name-rs0.demand-backup-15110.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
++ mktemp
+ local LAST_OUT=/tmp/tmp.jvpWorpd4h
++ mktemp
+ local LAST_ERR=/tmp/tmp.Dk7Ojrp5gF
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-bb8b97679-rgs8m -- bash -c 'printf '\''use myApp\n db.test.insert({ x: 100501 })\n'\'' | mongo mongodb+srv://myApp:myPass@some-name-rs0.demand-backup-15110.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.jvpWorpd4h
Percona Server for MongoDB shell version v4.4.29-28
connecting to: mongodb://some-name-rs0-0.some-name-rs0.demand-backup-15110.svc.cluster.local:27017,some-name-rs0-2.some-name-rs0.demand-backup-15110.svc.cluster.local:27017,some-name-rs0-1.some-name-rs0.demand-backup-15110.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb&replicaSet=rs0&ssl=false
Implicit session: session { "id" : UUID("2b67dc23-d76e-4c6e-ad56-b9221995170e") }
Percona Server for MongoDB server version: v8.0.19-7
WARNING: shell and server versions do not match
switched to db myApp
WriteResult({ "nInserted" : 1 })
bye
+ cat /tmp/tmp.Dk7Ojrp5gF
+ rm /tmp/tmp.jvpWorpd4h /tmp/tmp.Dk7Ojrp5gF
+ return 0
+ compare_mongo_cmd find myApp:myPass@some-name-rs0.demand-backup-15110 -2nd .svc.cluster.local myApp test
+ local command=find
+ local uri=myApp:myPass@some-name-rs0.demand-backup-15110
+ local postfix=-2nd
+ local suffix=.svc.cluster.local
+ local database=myApp
+ local collection=test
+ local sort=
+ local tls=false
+ local replicaset=
+ local 'full_command=db.test.find()'
+ [[ -n '' ]]
+ log 'running db.test.find() in myApp'
+ set +o xtrace
[2026-03-27T11:34:51+0000] running db.test.find() in myApp
+ [[ false == \t\r\u\e ]]
+ mongo_command=run_mongo
+ run_mongo 'use myApp\n db.test.find()' myApp:myPass@some-name-rs0.demand-backup-15110 mongodb .svc.cluster.local ''
+ local 'command=use myApp\n db.test.find()'
+ local uri=myApp:myPass@some-name-rs0.demand-backup-15110
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local replica_set=rs0
+ [[ myApp:myPass@some-name-rs0.demand-backup-15110 == *cfg* ]]
+ grep -E -v 'I NETWORK|W NETWORK|F NETWORK|"c":"NETWORK"|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|Started a new thread for the timer service'
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+ /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.tOWgOHg25i
+++ mktemp
++ local LAST_ERR=/tmp/tmp.3v0XED56te
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.tOWgOHg25i
++ cat /tmp/tmp.3v0XED56te
++ rm /tmp/tmp.tOWgOHg25i /tmp/tmp.3v0XED56te
++ return 0
+ local client_container=psmdb-client-bb8b97679-rgs8m
+ kubectl_bin exec psmdb-client-bb8b97679-rgs8m -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0.demand-backup-15110.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
++ mktemp
+ local LAST_OUT=/tmp/tmp.ZWdu2lJfVK
++ mktemp
+ local LAST_ERR=/tmp/tmp.n7mlEbBwDO
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-bb8b97679-rgs8m -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0.demand-backup-15110.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.ZWdu2lJfVK
+ cat /tmp/tmp.n7mlEbBwDO
+ rm /tmp/tmp.ZWdu2lJfVK /tmp/tmp.n7mlEbBwDO
+ return 0
+ [[ 0 -eq 0 ]]
+ diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2283/e2e-tests/demand-backup/compare/find-2nd.json /tmp/tmp.SOjtfW5Qlc/find-2nd
+ run_restore backup-minio
+ local backup_name=backup-minio
+ local restore_file=/mnt/jenkins/workspace/cloud-psmdb-operator_PR-2283/e2e-tests/demand-backup/conf/restore.yml
+ log 'running restore restore-backup-minio'
+ set +o xtrace
[2026-03-27T11:34:53+0000] running restore restore-backup-minio
+ cat /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2283/e2e-tests/demand-backup/conf/restore.yml
+ /usr/sbin/sed -e 's/name:/name: restore-backup-minio/'
+ /usr/sbin/sed -e 's/backupName:/backupName: backup-minio/'
+ kubectl_bin apply -f -
++ mktemp
+ local LAST_OUT=/tmp/tmp.xykpYzsLwY
++ mktemp
+ local LAST_ERR=/tmp/tmp.oso20kdmMU
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl apply -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.xykpYzsLwY
perconaservermongodbrestore.psmdb.percona.com/restore-backup-minio created
+ cat /tmp/tmp.oso20kdmMU
+ rm /tmp/tmp.xykpYzsLwY /tmp/tmp.oso20kdmMU
+ return 0
+ wait_restore backup-minio some-name
+ local backup_name=backup-minio
+ local cluster_name=some-name
+ local target_state=ready
+ local wait_cluster_consistency=1
+ local wait_time=1780
+ local ok_if_ready=0
+ set +o xtrace
Waiting for the psmdb-restore/restore-backup-minio object to be created.OK
Waiting psmdb-restore/restore-backup-minio to reach state "ready" .OK after 0 minutes
+ [[ 1 -eq 1 ]]
+ wait_cluster_consistency some-name
+ local cluster_name=some-name
+ local wait_time=32
+ retry=0
+ sleep 7
+ echo -n 'waiting for cluster readyness'
waiting for cluster readyness++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.CwjTTSWEJ8
+++ mktemp
++ local LAST_ERR=/tmp/tmp.3gxIKIlHoD
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.CwjTTSWEJ8
++ cat /tmp/tmp.3gxIKIlHoD
++ rm /tmp/tmp.CwjTTSWEJ8 /tmp/tmp.3gxIKIlHoD
++ return 0
+ [[ ready == \r\e\a\d\y ]]
+ echo .OK
.OK
+ simple_data_check some-name-rs0 3 0
+ local cluster_name=some-name-rs0
+ let last_pod=3-1
+ local isSharded=0
+ local cluster_pfx=
+ '[' 0 -eq 1 ']'
++ seq 0 2
+ for i in $(seq 0 $last_pod)
+ compare_mongo_cmd find myApp:myPass@some-name-rs0-0.some-name-rs0.demand-backup-15110
+ local command=find
+ local uri=myApp:myPass@some-name-rs0-0.some-name-rs0.demand-backup-15110
+ local postfix=
+ local suffix=
+ local database=myApp
+ local collection=test
+ local sort=
+ local tls=false
+ local replicaset=
+ local 'full_command=db.test.find()'
+ [[ -n '' ]]
+ log 'running db.test.find() in myApp'
+ set +o xtrace
[2026-03-27T11:35:15+0000] running db.test.find() in myApp
+ [[ false == \t\r\u\e ]]
+ mongo_command=run_mongo
+ run_mongo 'use myApp\n db.test.find()' myApp:myPass@some-name-rs0-0.some-name-rs0.demand-backup-15110 mongodb '' ''
+ local 'command=use myApp\n db.test.find()'
+ local uri=myApp:myPass@some-name-rs0-0.some-name-rs0.demand-backup-15110
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local replica_set=rs0
+ [[ myApp:myPass@some-name-rs0-0.some-name-rs0.demand-backup-15110 == *cfg* ]]
+ grep -E -v 'I NETWORK|W NETWORK|F NETWORK|"c":"NETWORK"|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|Started a new thread for the timer service'
+ /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.qRZwp1Jp7h
+++ mktemp
++ local LAST_ERR=/tmp/tmp.iQbSMQgxsy
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.qRZwp1Jp7h
++ cat /tmp/tmp.iQbSMQgxsy
++ rm /tmp/tmp.qRZwp1Jp7h /tmp/tmp.iQbSMQgxsy
++ return 0
+ local client_container=psmdb-client-bb8b97679-rgs8m
+ kubectl_bin exec psmdb-client-bb8b97679-rgs8m -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-0.some-name-rs0.demand-backup-15110.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
++ mktemp
+ local LAST_OUT=/tmp/tmp.2ZTCxSrEuq
++ mktemp
+ local LAST_ERR=/tmp/tmp.rlzHu2TSL4
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-bb8b97679-rgs8m -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-0.some-name-rs0.demand-backup-15110.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.2ZTCxSrEuq
+ cat /tmp/tmp.rlzHu2TSL4
+ rm /tmp/tmp.2ZTCxSrEuq /tmp/tmp.rlzHu2TSL4
+ return 0
+ [[ 0 -eq 0 ]]
+ diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2283/e2e-tests/demand-backup/compare/find.json /tmp/tmp.SOjtfW5Qlc/find
+ for i in $(seq 0 $last_pod)
+ compare_mongo_cmd find myApp:myPass@some-name-rs0-1.some-name-rs0.demand-backup-15110
+ local command=find
+ local uri=myApp:myPass@some-name-rs0-1.some-name-rs0.demand-backup-15110
+ local postfix=
+ local suffix=
+ local database=myApp
+ local collection=test
+ local sort=
+ local tls=false
+ local replicaset=
+ local 'full_command=db.test.find()'
+ [[ -n '' ]]
+ log 'running db.test.find() in myApp'
+ set +o xtrace
[2026-03-27T11:35:17+0000] running db.test.find() in myApp
+ [[ false == \t\r\u\e ]]
+ mongo_command=run_mongo
+ run_mongo 'use myApp\n db.test.find()' myApp:myPass@some-name-rs0-1.some-name-rs0.demand-backup-15110 mongodb '' ''
+ local 'command=use myApp\n db.test.find()'
+ local uri=myApp:myPass@some-name-rs0-1.some-name-rs0.demand-backup-15110
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local replica_set=rs0
+ [[ myApp:myPass@some-name-rs0-1.some-name-rs0.demand-backup-15110 == *cfg* ]]
+ /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
+ grep -E -v 'I NETWORK|W NETWORK|F NETWORK|"c":"NETWORK"|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|Started a new thread for the timer service'
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.QxeaRQlnqf
+++ mktemp
++ local LAST_ERR=/tmp/tmp.X7bgkw0eaF
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.QxeaRQlnqf
++ cat /tmp/tmp.X7bgkw0eaF
++ rm /tmp/tmp.QxeaRQlnqf /tmp/tmp.X7bgkw0eaF
++ return 0
+ local client_container=psmdb-client-bb8b97679-rgs8m
+ kubectl_bin exec psmdb-client-bb8b97679-rgs8m -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-1.some-name-rs0.demand-backup-15110.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
++ mktemp
+ local LAST_OUT=/tmp/tmp.3NYEWa8NHr
++ mktemp
+ local LAST_ERR=/tmp/tmp.bfBOnUfcUW
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-bb8b97679-rgs8m -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-1.some-name-rs0.demand-backup-15110.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.3NYEWa8NHr
+ cat /tmp/tmp.bfBOnUfcUW
+ rm /tmp/tmp.3NYEWa8NHr /tmp/tmp.bfBOnUfcUW
+ return 0
+ [[ 0 -eq 0 ]]
+ diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2283/e2e-tests/demand-backup/compare/find.json /tmp/tmp.SOjtfW5Qlc/find
+ for i in $(seq 0 $last_pod)
+ compare_mongo_cmd find myApp:myPass@some-name-rs0-2.some-name-rs0.demand-backup-15110
+ local command=find
+ local uri=myApp:myPass@some-name-rs0-2.some-name-rs0.demand-backup-15110
+ local postfix=
+ local suffix=
+ local database=myApp
+ local collection=test
+ local sort=
+ local tls=false
+ local replicaset=
+ local 'full_command=db.test.find()'
+ [[ -n '' ]]
+ log 'running db.test.find() in myApp'
+ set +o xtrace
[2026-03-27T11:35:19+0000] running db.test.find() in myApp
+ [[ false == \t\r\u\e ]]
+ mongo_command=run_mongo
+ run_mongo 'use myApp\n db.test.find()' myApp:myPass@some-name-rs0-2.some-name-rs0.demand-backup-15110 mongodb '' ''
+ local 'command=use myApp\n db.test.find()'
+ local uri=myApp:myPass@some-name-rs0-2.some-name-rs0.demand-backup-15110
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local replica_set=rs0
+ [[ myApp:myPass@some-name-rs0-2.some-name-rs0.demand-backup-15110 == *cfg* ]]
+ grep -E -v 'I NETWORK|W NETWORK|F NETWORK|"c":"NETWORK"|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|Started a new thread for the timer service'
+ /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.BIA047U2Ro
+++ mktemp
++ local LAST_ERR=/tmp/tmp.MqjTBvL2KU
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.BIA047U2Ro
++ cat /tmp/tmp.MqjTBvL2KU
++ rm /tmp/tmp.BIA047U2Ro /tmp/tmp.MqjTBvL2KU
++ return 0
+ local client_container=psmdb-client-bb8b97679-rgs8m
+ kubectl_bin exec psmdb-client-bb8b97679-rgs8m -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-2.some-name-rs0.demand-backup-15110.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
++ mktemp
+ local LAST_OUT=/tmp/tmp.dYFBSXPbM1
++ mktemp
+ local LAST_ERR=/tmp/tmp.JeqaWStoWo
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-bb8b97679-rgs8m -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-2.some-name-rs0.demand-backup-15110.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.dYFBSXPbM1
+ cat /tmp/tmp.JeqaWStoWo
+ rm /tmp/tmp.dYFBSXPbM1 /tmp/tmp.JeqaWStoWo
+ return 0
+ [[ 0 -eq 0 ]]
+ diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2283/e2e-tests/demand-backup/compare/find.json /tmp/tmp.SOjtfW5Qlc/find
+ run_mongo 'use myApp\n db.dropUser("test1user")' userAdmin:userAdmin123456@some-name-rs0.demand-backup-15110
+ local 'command=use myApp\n db.dropUser("test1user")'
+ local uri=userAdmin:userAdmin123456@some-name-rs0.demand-backup-15110
+ local driver=mongodb+srv
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local replica_set=rs0
+ [[ userAdmin:userAdmin123456@some-name-rs0.demand-backup-15110 == *cfg* ]]
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.PcOTTqvUTt
+++ mktemp
++ local LAST_ERR=/tmp/tmp.JOVIdgX84v
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.PcOTTqvUTt
++ cat /tmp/tmp.JOVIdgX84v
++ rm /tmp/tmp.PcOTTqvUTt /tmp/tmp.JOVIdgX84v
++ return 0
+ local client_container=psmdb-client-bb8b97679-rgs8m
+ kubectl_bin exec psmdb-client-bb8b97679-rgs8m -- bash -c 'printf '\''use myApp\n db.dropUser("test1user")\n'\'' | mongo mongodb+srv://userAdmin:userAdmin123456@some-name-rs0.demand-backup-15110.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
++ mktemp
+ local LAST_OUT=/tmp/tmp.VPo8EOZRbd
++ mktemp
+ local LAST_ERR=/tmp/tmp.qEpa6wo1Uj
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-bb8b97679-rgs8m -- bash -c 'printf '\''use myApp\n db.dropUser("test1user")\n'\'' | mongo mongodb+srv://userAdmin:userAdmin123456@some-name-rs0.demand-backup-15110.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.VPo8EOZRbd
Percona Server for MongoDB shell version v4.4.29-28
connecting to: mongodb://some-name-rs0-2.some-name-rs0.demand-backup-15110.svc.cluster.local:27017,some-name-rs0-1.some-name-rs0.demand-backup-15110.svc.cluster.local:27017,some-name-rs0-0.some-name-rs0.demand-backup-15110.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb&replicaSet=rs0&ssl=false
Implicit session: session { "id" : UUID("99040e0d-a2da-4d75-b529-98711447cdd5") }
Percona Server for MongoDB server version: v8.0.19-7
WARNING: shell and server versions do not match
switched to db myApp
true
bye
+ cat /tmp/tmp.qEpa6wo1Uj
+ rm /tmp/tmp.VPo8EOZRbd /tmp/tmp.qEpa6wo1Uj
+ return 0
+ run_mongo 'use myApp\n db.dropRole("test1role")' userAdmin:userAdmin123456@some-name-rs0.demand-backup-15110
+ local 'command=use myApp\n db.dropRole("test1role")'
+ local uri=userAdmin:userAdmin123456@some-name-rs0.demand-backup-15110
+ local driver=mongodb+srv
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local replica_set=rs0
+ [[ userAdmin:userAdmin123456@some-name-rs0.demand-backup-15110 == *cfg* ]]
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.3RQypOkLYc
+++ mktemp
++ local LAST_ERR=/tmp/tmp.dgtrSVipid
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.3RQypOkLYc
++ cat /tmp/tmp.dgtrSVipid
++ rm /tmp/tmp.3RQypOkLYc /tmp/tmp.dgtrSVipid
++ return 0
+ local client_container=psmdb-client-bb8b97679-rgs8m
+ kubectl_bin exec psmdb-client-bb8b97679-rgs8m -- bash -c 'printf '\''use myApp\n db.dropRole("test1role")\n'\'' | mongo mongodb+srv://userAdmin:userAdmin123456@some-name-rs0.demand-backup-15110.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
++ mktemp
+ local LAST_OUT=/tmp/tmp.vbV5Eapoxi
++ mktemp
+ local LAST_ERR=/tmp/tmp.sMZA2r3YMN
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-bb8b97679-rgs8m -- bash -c 'printf '\''use myApp\n db.dropRole("test1role")\n'\'' | mongo mongodb+srv://userAdmin:userAdmin123456@some-name-rs0.demand-backup-15110.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.vbV5Eapoxi
Percona Server for MongoDB shell version v4.4.29-28
connecting to: mongodb://some-name-rs0-0.some-name-rs0.demand-backup-15110.svc.cluster.local:27017,some-name-rs0-2.some-name-rs0.demand-backup-15110.svc.cluster.local:27017,some-name-rs0-1.some-name-rs0.demand-backup-15110.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb&replicaSet=rs0&ssl=false
Implicit session: session { "id" : UUID("1a08f385-5c0d-425e-a928-09b0faf867d7") }
Percona Server for MongoDB server version: v8.0.19-7
WARNING: shell and server versions do not match
switched to db myApp
true
bye
+ cat /tmp/tmp.sMZA2r3YMN
+ rm /tmp/tmp.vbV5Eapoxi /tmp/tmp.sMZA2r3YMN
+ return 0
+ desc 'selective restore -- minio'
+ set +o xtrace


-----------------------------------------------------------------------------------
selective restore -- minio
-----------------------------------------------------------------------------------

+ run_recovery_check_selective backup-minio some-name-rs0 myApp test2 false test1user test1role
+ local backup=backup-minio
+ local cluster=some-name-rs0
+ local database=myApp
+ local collection=test2
+ local with_users_and_roles=false
+ local username=test1user
+ local role=test1role
+ restore_name=restore-backup-minio-selective
+ run_mongo 'use myApp\n db.test2.drop()' myApp:myPass@some-name-rs0.demand-backup-15110
+ local 'command=use myApp\n db.test2.drop()'
+ local uri=myApp:myPass@some-name-rs0.demand-backup-15110
+ local driver=mongodb+srv
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local replica_set=rs0
+ [[ myApp:myPass@some-name-rs0.demand-backup-15110 == *cfg* ]]
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.a6lmKAAubj
+++ mktemp
++ local LAST_ERR=/tmp/tmp.yXDl4wEpRJ
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.a6lmKAAubj
++ cat /tmp/tmp.yXDl4wEpRJ
++ rm /tmp/tmp.a6lmKAAubj /tmp/tmp.yXDl4wEpRJ
++ return 0
+ local client_container=psmdb-client-bb8b97679-rgs8m
+ kubectl_bin exec psmdb-client-bb8b97679-rgs8m -- bash -c 'printf '\''use myApp\n db.test2.drop()\n'\'' | mongo mongodb+srv://myApp:myPass@some-name-rs0.demand-backup-15110.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
++ mktemp
+ local LAST_OUT=/tmp/tmp.fQ0hgGDQ6o
++ mktemp
+ local LAST_ERR=/tmp/tmp.UZ47N18LkK
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-bb8b97679-rgs8m -- bash -c 'printf '\''use myApp\n db.test2.drop()\n'\'' | mongo mongodb+srv://myApp:myPass@some-name-rs0.demand-backup-15110.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.fQ0hgGDQ6o
Percona Server for MongoDB shell version v4.4.29-28
connecting to: mongodb://some-name-rs0-2.some-name-rs0.demand-backup-15110.svc.cluster.local:27017,some-name-rs0-1.some-name-rs0.demand-backup-15110.svc.cluster.local:27017,some-name-rs0-0.some-name-rs0.demand-backup-15110.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb&replicaSet=rs0&ssl=false
Implicit session: session { "id" : UUID("a39e1573-444c-47b3-a26d-19f9195711c6") }
Percona Server for MongoDB server version: v8.0.19-7
WARNING: shell and server versions do not match
switched to db myApp
true
bye
+ cat /tmp/tmp.UZ47N18LkK
+ rm /tmp/tmp.fQ0hgGDQ6o /tmp/tmp.UZ47N18LkK
+ return 0
++ collection_exists test2
++ local collection=test2
++ run_mongo 'use myApp\n JSON.stringify(db.getCollectionNames())' myApp:myPass@some-name-rs0.demand-backup-15110 '' '' --quiet
++ local 'command=use myApp\n JSON.stringify(db.getCollectionNames())'
++ local uri=myApp:myPass@some-name-rs0.demand-backup-15110
++ local driver=mongodb+srv
++ local suffix=.svc.cluster.local
++ local mongo_flag=--quiet
++ local replica_set=rs0
++ grep -v 'switched to'
++ [[ myApp:myPass@some-name-rs0.demand-backup-15110 == *cfg* ]]
++ jq 'index("test2") != null'
+++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.lGptMRNOrd
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.PDm2U4eOkM
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in $(seq 0 2)
+++ set +e
+++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.lGptMRNOrd
+++ cat /tmp/tmp.PDm2U4eOkM
+++ rm /tmp/tmp.lGptMRNOrd /tmp/tmp.PDm2U4eOkM
+++ return 0
++ local client_container=psmdb-client-bb8b97679-rgs8m
++ kubectl_bin exec psmdb-client-bb8b97679-rgs8m -- bash -c 'printf '\''use myApp\n JSON.stringify(db.getCollectionNames())\n'\'' | mongo mongodb+srv://myApp:myPass@some-name-rs0.demand-backup-15110.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 --quiet'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.t24gG379H6
+++ mktemp
++ local LAST_ERR=/tmp/tmp.0LoPNN5GMZ
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl exec psmdb-client-bb8b97679-rgs8m -- bash -c 'printf '\''use myApp\n JSON.stringify(db.getCollectionNames())\n'\'' | mongo mongodb+srv://myApp:myPass@some-name-rs0.demand-backup-15110.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 --quiet'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.t24gG379H6
++ cat /tmp/tmp.0LoPNN5GMZ
++ rm /tmp/tmp.t24gG379H6 /tmp/tmp.0LoPNN5GMZ
++ return 0
+ [[ false == \t\r\u\e ]]
+ yq /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2283/e2e-tests/demand-backup/conf/restore.yml
+ yq '.metadata.name="restore-backup-minio-selective"'
+ yq '.spec.backupName="backup-minio"'
+ yq '.spec.selective.namespaces[0]="myApp.test"'
+ yq .spec.selective.withUsersAndRoles=false
+ kubectl_bin apply -f -
++ mktemp
+ local LAST_OUT=/tmp/tmp.IRZADJ2qqs
++ mktemp
+ local LAST_ERR=/tmp/tmp.r9VOfcFiXU
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl apply -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.IRZADJ2qqs
perconaservermongodbrestore.psmdb.percona.com/restore-backup-minio-selective created
+ cat /tmp/tmp.r9VOfcFiXU
+ rm /tmp/tmp.IRZADJ2qqs /tmp/tmp.r9VOfcFiXU
+ return 0
+ wait_restore backup-minio-selective some-name
+ local backup_name=backup-minio-selective
+ local cluster_name=some-name
+ local target_state=ready
+ local wait_cluster_consistency=1
+ local wait_time=1780
+ local ok_if_ready=0
+ set +o xtrace
Waiting for the psmdb-restore/restore-backup-minio-selective object to be created.OK
Waiting psmdb-restore/restore-backup-minio-selective to reach state "ready" .OK after 0 minutes
+ [[ 1 -eq 1 ]]
+ wait_cluster_consistency some-name
+ local cluster_name=some-name
+ local wait_time=32
+ retry=0
+ sleep 7
+ echo -n 'waiting for cluster readyness'
waiting for cluster readyness++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.k0b3H9iYsN
+++ mktemp
++ local LAST_ERR=/tmp/tmp.KkycfwZF0A
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.k0b3H9iYsN
++ cat /tmp/tmp.KkycfwZF0A
++ rm /tmp/tmp.k0b3H9iYsN /tmp/tmp.KkycfwZF0A
++ return 0
+ [[ ready == \r\e\a\d\y ]]
+ echo .OK
.OK
++ collection_exists test2
++ local collection=test2
++ run_mongo 'use myApp\n JSON.stringify(db.getCollectionNames())' myApp:myPass@some-name-rs0.demand-backup-15110 '' '' --quiet
++ grep -v 'switched to'
++ local 'command=use myApp\n JSON.stringify(db.getCollectionNames())'
++ local uri=myApp:myPass@some-name-rs0.demand-backup-15110
++ local driver=mongodb+srv
++ local suffix=.svc.cluster.local
++ local mongo_flag=--quiet
++ local replica_set=rs0
++ [[ myApp:myPass@some-name-rs0.demand-backup-15110 == *cfg* ]]
++ jq 'index("test2") != null'
+++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.FMACZ6VgGC
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.UbbbT8AAEY
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in $(seq 0 2)
+++ set +e
+++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.FMACZ6VgGC
+++ cat /tmp/tmp.UbbbT8AAEY
+++ rm /tmp/tmp.FMACZ6VgGC /tmp/tmp.UbbbT8AAEY
+++ return 0
++ local client_container=psmdb-client-bb8b97679-rgs8m
++ kubectl_bin exec psmdb-client-bb8b97679-rgs8m -- bash -c 'printf '\''use myApp\n JSON.stringify(db.getCollectionNames())\n'\'' | mongo mongodb+srv://myApp:myPass@some-name-rs0.demand-backup-15110.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 --quiet'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.00OQza8uQ8
+++ mktemp
++ local LAST_ERR=/tmp/tmp.Fe3rJ6gVbd
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl exec psmdb-client-bb8b97679-rgs8m -- bash -c 'printf '\''use myApp\n JSON.stringify(db.getCollectionNames())\n'\'' | mongo mongodb+srv://myApp:myPass@some-name-rs0.demand-backup-15110.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 --quiet'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.00OQza8uQ8
++ cat /tmp/tmp.Fe3rJ6gVbd
++ rm /tmp/tmp.00OQza8uQ8 /tmp/tmp.Fe3rJ6gVbd
++ return 0
+ [[ false == \t\r\u\e ]]
+ [[ false == \t\r\u\e ]]
+ [[ false == \f\a\l\s\e ]]
++ user_exists test1user
++ local username=test1user
++ run_mongo 'use myApp\n JSON.stringify(db.getUsers())' userAdmin:userAdmin123456@some-name-rs0.demand-backup-15110 '' '' --quiet
++ local 'command=use myApp\n JSON.stringify(db.getUsers())'
++ local uri=userAdmin:userAdmin123456@some-name-rs0.demand-backup-15110
++ local driver=mongodb+srv
++ local suffix=.svc.cluster.local
++ local mongo_flag=--quiet
++ local replica_set=rs0
++ [[ userAdmin:userAdmin123456@some-name-rs0.demand-backup-15110 == *cfg* ]]
++ grep -v 'switched to'
++ jq 'any(.[]; ._id==myApp.test1user)'
+++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.zkCJBWxniH
jq: error: myApp/0 is not defined at <top-level>, line 1:
any(.[]; ._id==myApp.test1user)               
jq: 1 compile error
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.Kj6LiX7LRA
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in $(seq 0 2)
+++ set +e
+++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.zkCJBWxniH
+++ cat /tmp/tmp.Kj6LiX7LRA
+++ rm /tmp/tmp.zkCJBWxniH /tmp/tmp.Kj6LiX7LRA
+++ return 0
++ local client_container=psmdb-client-bb8b97679-rgs8m
++ kubectl_bin exec psmdb-client-bb8b97679-rgs8m -- bash -c 'printf '\''use myApp\n JSON.stringify(db.getUsers())\n'\'' | mongo mongodb+srv://userAdmin:userAdmin123456@some-name-rs0.demand-backup-15110.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 --quiet'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.k2FQpX0itG
+++ mktemp
++ local LAST_ERR=/tmp/tmp.Vu1MrABlGG
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl exec psmdb-client-bb8b97679-rgs8m -- bash -c 'printf '\''use myApp\n JSON.stringify(db.getUsers())\n'\'' | mongo mongodb+srv://userAdmin:userAdmin123456@some-name-rs0.demand-backup-15110.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 --quiet'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.k2FQpX0itG
++ cat /tmp/tmp.Vu1MrABlGG
++ rm /tmp/tmp.k2FQpX0itG /tmp/tmp.Vu1MrABlGG
++ return 0
grep: write error: Broken pipe
+ [[ '' == \t\r\u\e ]]
++ role_exists test1role
++ local role=test1role
++ run_mongo 'use myApp\n JSON.stringify(db.getRoles())' userAdmin:userAdmin123456@some-name-rs0.demand-backup-15110 '' '' --quiet
++ grep -v 'switched to'
++ local 'command=use myApp\n JSON.stringify(db.getRoles())'
++ local uri=userAdmin:userAdmin123456@some-name-rs0.demand-backup-15110
++ local driver=mongodb+srv
++ jq 'any(.[]; ._id==myApp.test1role)'
++ local suffix=.svc.cluster.local
++ local mongo_flag=--quiet
++ local replica_set=rs0
++ [[ userAdmin:userAdmin123456@some-name-rs0.demand-backup-15110 == *cfg* ]]
+++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++++ mktemp
jq: error: myApp/0 is not defined at <top-level>, line 1:
any(.[]; ._id==myApp.test1role)               
jq: 1 compile error
+++ local LAST_OUT=/tmp/tmp.fKiDmOlChZ
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.0KxHS2A4vo
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in $(seq 0 2)
+++ set +e
+++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.fKiDmOlChZ
+++ cat /tmp/tmp.0KxHS2A4vo
+++ rm /tmp/tmp.fKiDmOlChZ /tmp/tmp.0KxHS2A4vo
+++ return 0
++ local client_container=psmdb-client-bb8b97679-rgs8m
++ kubectl_bin exec psmdb-client-bb8b97679-rgs8m -- bash -c 'printf '\''use myApp\n JSON.stringify(db.getRoles())\n'\'' | mongo mongodb+srv://userAdmin:userAdmin123456@some-name-rs0.demand-backup-15110.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 --quiet'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.PwR31yoX6t
+++ mktemp
++ local LAST_ERR=/tmp/tmp.GGTUJ1OuIK
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl exec psmdb-client-bb8b97679-rgs8m -- bash -c 'printf '\''use myApp\n JSON.stringify(db.getRoles())\n'\'' | mongo mongodb+srv://userAdmin:userAdmin123456@some-name-rs0.demand-backup-15110.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 --quiet'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.PwR31yoX6t
++ cat /tmp/tmp.GGTUJ1OuIK
++ rm /tmp/tmp.PwR31yoX6t /tmp/tmp.GGTUJ1OuIK
++ return 0
grep: write error: Broken pipe
+ [[ '' == \t\r\u\e ]]
+ kubectl_bin delete psmdb-restore restore-backup-minio-selective
++ mktemp
+ local LAST_OUT=/tmp/tmp.hAL8qHyGr5
++ mktemp
+ local LAST_ERR=/tmp/tmp.9ggr6gxZAD
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl delete psmdb-restore restore-backup-minio-selective
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.hAL8qHyGr5
perconaservermongodbrestore.psmdb.percona.com "restore-backup-minio-selective" deleted from demand-backup-15110 namespace
+ cat /tmp/tmp.9ggr6gxZAD
+ rm /tmp/tmp.hAL8qHyGr5 /tmp/tmp.9ggr6gxZAD
+ return 0
+ run_mongo 'use myApp\n db.test2.insert({ x: 100501 })' myApp:myPass@some-name-rs0.demand-backup-15110
+ local 'command=use myApp\n db.test2.insert({ x: 100501 })'
+ local uri=myApp:myPass@some-name-rs0.demand-backup-15110
+ local driver=mongodb+srv
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local replica_set=rs0
+ [[ myApp:myPass@some-name-rs0.demand-backup-15110 == *cfg* ]]
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.ydWQVnclMd
+++ mktemp
++ local LAST_ERR=/tmp/tmp.LX1Fmmb1O8
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.ydWQVnclMd
++ cat /tmp/tmp.LX1Fmmb1O8
++ rm /tmp/tmp.ydWQVnclMd /tmp/tmp.LX1Fmmb1O8
++ return 0
+ local client_container=psmdb-client-bb8b97679-rgs8m
+ kubectl_bin exec psmdb-client-bb8b97679-rgs8m -- bash -c 'printf '\''use myApp\n db.test2.insert({ x: 100501 })\n'\'' | mongo mongodb+srv://myApp:myPass@some-name-rs0.demand-backup-15110.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
++ mktemp
+ local LAST_OUT=/tmp/tmp.0Icqb9h2Zn
++ mktemp
+ local LAST_ERR=/tmp/tmp.uy5c7YJuOm
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-bb8b97679-rgs8m -- bash -c 'printf '\''use myApp\n db.test2.insert({ x: 100501 })\n'\'' | mongo mongodb+srv://myApp:myPass@some-name-rs0.demand-backup-15110.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.0Icqb9h2Zn
Percona Server for MongoDB shell version v4.4.29-28
connecting to: mongodb://some-name-rs0-1.some-name-rs0.demand-backup-15110.svc.cluster.local:27017,some-name-rs0-0.some-name-rs0.demand-backup-15110.svc.cluster.local:27017,some-name-rs0-2.some-name-rs0.demand-backup-15110.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb&replicaSet=rs0&ssl=false
Implicit session: session { "id" : UUID("e2d1ee43-d9b7-46c2-9134-ba353dc93008") }
Percona Server for MongoDB server version: v8.0.19-7
WARNING: shell and server versions do not match
switched to db myApp
WriteResult({ "nInserted" : 1 })
bye
+ cat /tmp/tmp.uy5c7YJuOm
+ rm /tmp/tmp.0Icqb9h2Zn /tmp/tmp.uy5c7YJuOm
+ return 0
+ desc 'selective restore with users and roles -- minio'
+ set +o xtrace


-----------------------------------------------------------------------------------
selective restore with users and roles -- minio
-----------------------------------------------------------------------------------

+ run_recovery_check_selective backup-minio some-name-rs0 myApp test2 true test1user test1role
+ local backup=backup-minio
+ local cluster=some-name-rs0
+ local database=myApp
+ local collection=test2
+ local with_users_and_roles=true
+ local username=test1user
+ local role=test1role
+ restore_name=restore-backup-minio-selective
+ run_mongo 'use myApp\n db.test2.drop()' myApp:myPass@some-name-rs0.demand-backup-15110
+ local 'command=use myApp\n db.test2.drop()'
+ local uri=myApp:myPass@some-name-rs0.demand-backup-15110
+ local driver=mongodb+srv
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local replica_set=rs0
+ [[ myApp:myPass@some-name-rs0.demand-backup-15110 == *cfg* ]]
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.RheoTyATIm
+++ mktemp
++ local LAST_ERR=/tmp/tmp.hA2V0bxDgC
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.RheoTyATIm
++ cat /tmp/tmp.hA2V0bxDgC
++ rm /tmp/tmp.RheoTyATIm /tmp/tmp.hA2V0bxDgC
++ return 0
+ local client_container=psmdb-client-bb8b97679-rgs8m
+ kubectl_bin exec psmdb-client-bb8b97679-rgs8m -- bash -c 'printf '\''use myApp\n db.test2.drop()\n'\'' | mongo mongodb+srv://myApp:myPass@some-name-rs0.demand-backup-15110.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
++ mktemp
+ local LAST_OUT=/tmp/tmp.wkNNl2N2eT
++ mktemp
+ local LAST_ERR=/tmp/tmp.8cp3XhnxHj
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-bb8b97679-rgs8m -- bash -c 'printf '\''use myApp\n db.test2.drop()\n'\'' | mongo mongodb+srv://myApp:myPass@some-name-rs0.demand-backup-15110.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.wkNNl2N2eT
Percona Server for MongoDB shell version v4.4.29-28
connecting to: mongodb://some-name-rs0-1.some-name-rs0.demand-backup-15110.svc.cluster.local:27017,some-name-rs0-0.some-name-rs0.demand-backup-15110.svc.cluster.local:27017,some-name-rs0-2.some-name-rs0.demand-backup-15110.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb&replicaSet=rs0&ssl=false
Implicit session: session { "id" : UUID("50837f85-4673-4809-a75f-8d72a8fd85ea") }
Percona Server for MongoDB server version: v8.0.19-7
WARNING: shell and server versions do not match
switched to db myApp
true
bye
+ cat /tmp/tmp.8cp3XhnxHj
+ rm /tmp/tmp.wkNNl2N2eT /tmp/tmp.8cp3XhnxHj
+ return 0
++ collection_exists test2
++ local collection=test2
++ run_mongo 'use myApp\n JSON.stringify(db.getCollectionNames())' myApp:myPass@some-name-rs0.demand-backup-15110 '' '' --quiet
++ local 'command=use myApp\n JSON.stringify(db.getCollectionNames())'
++ local uri=myApp:myPass@some-name-rs0.demand-backup-15110
++ local driver=mongodb+srv
++ local suffix=.svc.cluster.local
++ local mongo_flag=--quiet
++ local replica_set=rs0
++ [[ myApp:myPass@some-name-rs0.demand-backup-15110 == *cfg* ]]
++ grep -v 'switched to'
+++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ jq 'index("test2") != null'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.FiAfTKpS07
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.5vSygDNqoy
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in $(seq 0 2)
+++ set +e
+++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.FiAfTKpS07
+++ cat /tmp/tmp.5vSygDNqoy
+++ rm /tmp/tmp.FiAfTKpS07 /tmp/tmp.5vSygDNqoy
+++ return 0
++ local client_container=psmdb-client-bb8b97679-rgs8m
++ kubectl_bin exec psmdb-client-bb8b97679-rgs8m -- bash -c 'printf '\''use myApp\n JSON.stringify(db.getCollectionNames())\n'\'' | mongo mongodb+srv://myApp:myPass@some-name-rs0.demand-backup-15110.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 --quiet'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.xhJ9TaoH4b
+++ mktemp
++ local LAST_ERR=/tmp/tmp.ow4QoYQh0S
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl exec psmdb-client-bb8b97679-rgs8m -- bash -c 'printf '\''use myApp\n JSON.stringify(db.getCollectionNames())\n'\'' | mongo mongodb+srv://myApp:myPass@some-name-rs0.demand-backup-15110.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 --quiet'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.xhJ9TaoH4b
++ cat /tmp/tmp.ow4QoYQh0S
++ rm /tmp/tmp.xhJ9TaoH4b /tmp/tmp.ow4QoYQh0S
++ return 0
+ [[ false == \t\r\u\e ]]
+ yq /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2283/e2e-tests/demand-backup/conf/restore.yml
+ yq '.metadata.name="restore-backup-minio-selective"'
+ yq '.spec.backupName="backup-minio"'
+ yq '.spec.selective.namespaces[0]="myApp.test"'
+ kubectl_bin apply -f -
+ yq .spec.selective.withUsersAndRoles=true
++ mktemp
+ local LAST_OUT=/tmp/tmp.La83pjKdwu
++ mktemp
+ local LAST_ERR=/tmp/tmp.OMhrs2xfWp
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl apply -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.La83pjKdwu
perconaservermongodbrestore.psmdb.percona.com/restore-backup-minio-selective created
+ cat /tmp/tmp.OMhrs2xfWp
+ rm /tmp/tmp.La83pjKdwu /tmp/tmp.OMhrs2xfWp
+ return 0
+ wait_restore backup-minio-selective some-name
+ local backup_name=backup-minio-selective
+ local cluster_name=some-name
+ local target_state=ready
+ local wait_cluster_consistency=1
+ local wait_time=1780
+ local ok_if_ready=0
+ set +o xtrace
Waiting for the psmdb-restore/restore-backup-minio-selective object to be created.OK
Waiting psmdb-restore/restore-backup-minio-selective to reach state "ready" .OK after 0 minutes
+ [[ 1 -eq 1 ]]
+ wait_cluster_consistency some-name
+ local cluster_name=some-name
+ local wait_time=32
+ retry=0
+ sleep 7
+ echo -n 'waiting for cluster readyness'
waiting for cluster readyness++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.5FehQtwjEY
+++ mktemp
++ local LAST_ERR=/tmp/tmp.n2HVUkCAhk
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.5FehQtwjEY
++ cat /tmp/tmp.n2HVUkCAhk
++ rm /tmp/tmp.5FehQtwjEY /tmp/tmp.n2HVUkCAhk
++ return 0
+ [[ ready == \r\e\a\d\y ]]
+ echo .OK
.OK
++ collection_exists test2
++ local collection=test2
++ run_mongo 'use myApp\n JSON.stringify(db.getCollectionNames())' myApp:myPass@some-name-rs0.demand-backup-15110 '' '' --quiet
++ grep -v 'switched to'
++ local 'command=use myApp\n JSON.stringify(db.getCollectionNames())'
++ local uri=myApp:myPass@some-name-rs0.demand-backup-15110
++ local driver=mongodb+srv
++ local suffix=.svc.cluster.local
++ local mongo_flag=--quiet
++ local replica_set=rs0
++ [[ myApp:myPass@some-name-rs0.demand-backup-15110 == *cfg* ]]
++ jq 'index("test2") != null'
+++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.EDLC2Jipbw
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.wdcZaZks5d
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in $(seq 0 2)
+++ set +e
+++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.EDLC2Jipbw
+++ cat /tmp/tmp.wdcZaZks5d
+++ rm /tmp/tmp.EDLC2Jipbw /tmp/tmp.wdcZaZks5d
+++ return 0
++ local client_container=psmdb-client-bb8b97679-rgs8m
++ kubectl_bin exec psmdb-client-bb8b97679-rgs8m -- bash -c 'printf '\''use myApp\n JSON.stringify(db.getCollectionNames())\n'\'' | mongo mongodb+srv://myApp:myPass@some-name-rs0.demand-backup-15110.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 --quiet'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.zv6QIFqTZB
+++ mktemp
++ local LAST_ERR=/tmp/tmp.6kRHD2pmFt
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl exec psmdb-client-bb8b97679-rgs8m -- bash -c 'printf '\''use myApp\n JSON.stringify(db.getCollectionNames())\n'\'' | mongo mongodb+srv://myApp:myPass@some-name-rs0.demand-backup-15110.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 --quiet'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.zv6QIFqTZB
++ cat /tmp/tmp.6kRHD2pmFt
++ rm /tmp/tmp.zv6QIFqTZB /tmp/tmp.6kRHD2pmFt
++ return 0
+ [[ false == \t\r\u\e ]]
+ [[ true == \t\r\u\e ]]
++ user_exists test1user
++ local username=test1user
++ run_mongo 'use myApp\n JSON.stringify(db.getUsers())' userAdmin:userAdmin123456@some-name-rs0.demand-backup-15110 '' '' --quiet
++ grep -v 'switched to'
++ local 'command=use myApp\n JSON.stringify(db.getUsers())'
++ local uri=userAdmin:userAdmin123456@some-name-rs0.demand-backup-15110
++ local driver=mongodb+srv
++ local suffix=.svc.cluster.local
++ local mongo_flag=--quiet
++ local replica_set=rs0
++ [[ userAdmin:userAdmin123456@some-name-rs0.demand-backup-15110 == *cfg* ]]
+++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ jq 'any(.[]; ._id==myApp.test1user)'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.0WSE6D90Mr
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.X2kwynAGFu
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in $(seq 0 2)
+++ set +e
+++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
jq: error: myApp/0 is not defined at <top-level>, line 1:
any(.[]; ._id==myApp.test1user)               
jq: 1 compile error
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.0WSE6D90Mr
+++ cat /tmp/tmp.X2kwynAGFu
+++ rm /tmp/tmp.0WSE6D90Mr /tmp/tmp.X2kwynAGFu
+++ return 0
++ local client_container=psmdb-client-bb8b97679-rgs8m
++ kubectl_bin exec psmdb-client-bb8b97679-rgs8m -- bash -c 'printf '\''use myApp\n JSON.stringify(db.getUsers())\n'\'' | mongo mongodb+srv://userAdmin:userAdmin123456@some-name-rs0.demand-backup-15110.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 --quiet'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.g1NTZYGP4l
+++ mktemp
++ local LAST_ERR=/tmp/tmp.hdG7OrtbFy
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl exec psmdb-client-bb8b97679-rgs8m -- bash -c 'printf '\''use myApp\n JSON.stringify(db.getUsers())\n'\'' | mongo mongodb+srv://userAdmin:userAdmin123456@some-name-rs0.demand-backup-15110.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 --quiet'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.g1NTZYGP4l
++ cat /tmp/tmp.hdG7OrtbFy
++ rm /tmp/tmp.g1NTZYGP4l /tmp/tmp.hdG7OrtbFy
++ return 0
grep: write error: Broken pipe
+ [[ '' == \f\a\l\s\e ]]
++ role_exists test1role
++ local role=test1role
++ run_mongo 'use myApp\n JSON.stringify(db.getRoles())' userAdmin:userAdmin123456@some-name-rs0.demand-backup-15110 '' '' --quiet
++ grep -v 'switched to'
++ local 'command=use myApp\n JSON.stringify(db.getRoles())'
++ local uri=userAdmin:userAdmin123456@some-name-rs0.demand-backup-15110
++ local driver=mongodb+srv
++ local suffix=.svc.cluster.local
++ local mongo_flag=--quiet
++ local replica_set=rs0
++ [[ userAdmin:userAdmin123456@some-name-rs0.demand-backup-15110 == *cfg* ]]
++ jq 'any(.[]; ._id==myApp.test1role)'
+++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.TeQ9IMPLdk
jq: error: myApp/0 is not defined at <top-level>, line 1:
any(.[]; ._id==myApp.test1role)               
jq: 1 compile error
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.DzYQp0ophl
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in $(seq 0 2)
+++ set +e
+++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.TeQ9IMPLdk
+++ cat /tmp/tmp.DzYQp0ophl
+++ rm /tmp/tmp.TeQ9IMPLdk /tmp/tmp.DzYQp0ophl
+++ return 0
++ local client_container=psmdb-client-bb8b97679-rgs8m
++ kubectl_bin exec psmdb-client-bb8b97679-rgs8m -- bash -c 'printf '\''use myApp\n JSON.stringify(db.getRoles())\n'\'' | mongo mongodb+srv://userAdmin:userAdmin123456@some-name-rs0.demand-backup-15110.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 --quiet'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.0DD6RXiaCs
+++ mktemp
++ local LAST_ERR=/tmp/tmp.jd0sHpe8P0
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl exec psmdb-client-bb8b97679-rgs8m -- bash -c 'printf '\''use myApp\n JSON.stringify(db.getRoles())\n'\'' | mongo mongodb+srv://userAdmin:userAdmin123456@some-name-rs0.demand-backup-15110.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 --quiet'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.0DD6RXiaCs
++ cat /tmp/tmp.jd0sHpe8P0
++ rm /tmp/tmp.0DD6RXiaCs /tmp/tmp.jd0sHpe8P0
++ return 0
grep: write error: Broken pipe
+ [[ '' == \f\a\l\s\e ]]
+ desc 'restore from backup source, with storageName -- minio'
+ set +o xtrace


-----------------------------------------------------------------------------------
restore from backup source, with storageName -- minio
-----------------------------------------------------------------------------------

++ get_backup_dest backup-minio
++ local backup_name=backup-minio
++ kubectl_bin get psmdb-backup backup-minio -o 'jsonpath={.status.destination}'
++ sed -e 's/.json$//'
++ sed 's|s3://||'
++ sed 's|azure://||'
+++ mktemp
++ sed 's|gs://||'
++ local LAST_OUT=/tmp/tmp.6ZvENF8TAk
+++ mktemp
++ local LAST_ERR=/tmp/tmp.dDCMIQFdc6
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb-backup backup-minio -o 'jsonpath={.status.destination}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.6ZvENF8TAk
++ cat /tmp/tmp.dDCMIQFdc6
++ rm /tmp/tmp.6ZvENF8TAk /tmp/tmp.dDCMIQFdc6
++ return 0
+ backup_dest_minio=operator-testing/2026-03-27T11:10:41Z
+ run_recovery_check_bkp_source backup-minio operator-testing/2026-03-27T11:10:41Z some-name-rs0 backup-minio-source-0
+ local backup=backup-minio
+ local backup_dest=operator-testing/2026-03-27T11:10:41Z
+ local cluster=some-name-rs0
+ local source=backup-minio-source-0
+ run_mongo 'use myApp\n db.test.insert({ x: 100501 })' myApp:myPass@some-name-rs0.demand-backup-15110
+ local 'command=use myApp\n db.test.insert({ x: 100501 })'
+ local uri=myApp:myPass@some-name-rs0.demand-backup-15110
+ local driver=mongodb+srv
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local replica_set=rs0
+ [[ myApp:myPass@some-name-rs0.demand-backup-15110 == *cfg* ]]
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.9coNgI3sbU
+++ mktemp
++ local LAST_ERR=/tmp/tmp.g5erWY3vSQ
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.9coNgI3sbU
++ cat /tmp/tmp.g5erWY3vSQ
++ rm /tmp/tmp.9coNgI3sbU /tmp/tmp.g5erWY3vSQ
++ return 0
+ local client_container=psmdb-client-bb8b97679-rgs8m
+ kubectl_bin exec psmdb-client-bb8b97679-rgs8m -- bash -c 'printf '\''use myApp\n db.test.insert({ x: 100501 })\n'\'' | mongo mongodb+srv://myApp:myPass@some-name-rs0.demand-backup-15110.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
++ mktemp
+ local LAST_OUT=/tmp/tmp.FncL8dbEVT
++ mktemp
+ local LAST_ERR=/tmp/tmp.9K10PQQXum
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-bb8b97679-rgs8m -- bash -c 'printf '\''use myApp\n db.test.insert({ x: 100501 })\n'\'' | mongo mongodb+srv://myApp:myPass@some-name-rs0.demand-backup-15110.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.FncL8dbEVT
Percona Server for MongoDB shell version v4.4.29-28
connecting to: mongodb://some-name-rs0-1.some-name-rs0.demand-backup-15110.svc.cluster.local:27017,some-name-rs0-0.some-name-rs0.demand-backup-15110.svc.cluster.local:27017,some-name-rs0-2.some-name-rs0.demand-backup-15110.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb&replicaSet=rs0&ssl=false
Implicit session: session { "id" : UUID("099209c8-834d-4744-b0ed-600c12b8b30e") }
Percona Server for MongoDB server version: v8.0.19-7
WARNING: shell and server versions do not match
switched to db myApp
WriteResult({ "nInserted" : 1 })
bye
+ cat /tmp/tmp.9K10PQQXum
+ rm /tmp/tmp.FncL8dbEVT /tmp/tmp.9K10PQQXum
+ return 0
+ compare_mongo_cmd find myApp:myPass@some-name-rs0.demand-backup-15110 -2nd .svc.cluster.local myApp test
+ local command=find
+ local uri=myApp:myPass@some-name-rs0.demand-backup-15110
+ local postfix=-2nd
+ local suffix=.svc.cluster.local
+ local database=myApp
+ local collection=test
+ local sort=
+ local tls=false
+ local replicaset=
+ local 'full_command=db.test.find()'
+ [[ -n '' ]]
+ log 'running db.test.find() in myApp'
+ set +o xtrace
[2026-03-27T11:36:40+0000] running db.test.find() in myApp
+ [[ false == \t\r\u\e ]]
+ mongo_command=run_mongo
+ run_mongo 'use myApp\n db.test.find()' myApp:myPass@some-name-rs0.demand-backup-15110 mongodb .svc.cluster.local ''
+ grep -E -v 'I NETWORK|W NETWORK|F NETWORK|"c":"NETWORK"|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|Started a new thread for the timer service'
+ local 'command=use myApp\n db.test.find()'
+ local uri=myApp:myPass@some-name-rs0.demand-backup-15110
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
+ local replica_set=rs0
+ [[ myApp:myPass@some-name-rs0.demand-backup-15110 == *cfg* ]]
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.vmVX5Qzhse
+++ mktemp
++ local LAST_ERR=/tmp/tmp.rSdMiZSDzz
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.vmVX5Qzhse
++ cat /tmp/tmp.rSdMiZSDzz
++ rm /tmp/tmp.vmVX5Qzhse /tmp/tmp.rSdMiZSDzz
++ return 0
+ local client_container=psmdb-client-bb8b97679-rgs8m
+ kubectl_bin exec psmdb-client-bb8b97679-rgs8m -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0.demand-backup-15110.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
++ mktemp
+ local LAST_OUT=/tmp/tmp.AgWNbEYYdb
++ mktemp
+ local LAST_ERR=/tmp/tmp.k143RDXk3H
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-bb8b97679-rgs8m -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0.demand-backup-15110.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.AgWNbEYYdb
+ cat /tmp/tmp.k143RDXk3H
+ rm /tmp/tmp.AgWNbEYYdb /tmp/tmp.k143RDXk3H
+ return 0
+ [[ 0 -eq 0 ]]
+ diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2283/e2e-tests/demand-backup/compare/find-2nd.json /tmp/tmp.SOjtfW5Qlc/find-2nd
+ run_restore_backupsource backup-minio-source-0 operator-testing/2026-03-27T11:10:41Z
+ local backupName=backup-minio-source-0
+ local backupDest=operator-testing/2026-03-27T11:10:41Z
+ local storageName=
+ desc 'run restore restore-backup-minio-source-0 from backup backup-minio-source-0 destination is operator-testing/2026-03-27T11:10:41Z'
+ set +o xtrace


-----------------------------------------------------------------------------------
run restore restore-backup-minio-source-0 from backup backup-minio-source-0 destination is operator-testing/2026-03-27T11:10:41Z
-----------------------------------------------------------------------------------

+ '[' -z '' ']'
+ cat /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2283/e2e-tests/demand-backup/conf/restore-backupsource.yml
+ /usr/sbin/sed -e 's/name:/name: restore-backup-minio-source-0/'
+ /usr/sbin/sed -e 's|BACKUP-NAME|operator-testing/2026-03-27T11:10:41Z|'
+ /usr/sbin/sed -e /storageName/d
+ kubectl_bin apply -f -
++ mktemp
+ local LAST_OUT=/tmp/tmp.lNbbZUPCN7
++ mktemp
+ local LAST_ERR=/tmp/tmp.yiYVDfYABv
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl apply -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.lNbbZUPCN7
perconaservermongodbrestore.psmdb.percona.com/restore-backup-minio-source-0 created
+ cat /tmp/tmp.yiYVDfYABv
+ rm /tmp/tmp.lNbbZUPCN7 /tmp/tmp.yiYVDfYABv
+ return 0
+ return
+ wait_restore backup-minio-source-0 some-name
+ local backup_name=backup-minio-source-0
+ local cluster_name=some-name
+ local target_state=ready
+ local wait_cluster_consistency=1
+ local wait_time=1780
+ local ok_if_ready=0
+ set +o xtrace
Waiting for the psmdb-restore/restore-backup-minio-source-0 object to be created.OK
Waiting psmdb-restore/restore-backup-minio-source-0 to reach state "ready" .OK after 0 minutes
+ [[ 1 -eq 1 ]]
+ wait_cluster_consistency some-name
+ local cluster_name=some-name
+ local wait_time=32
+ retry=0
+ sleep 7
+ echo -n 'waiting for cluster readyness'
waiting for cluster readyness++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.wr9cQyiKkW
+++ mktemp
++ local LAST_ERR=/tmp/tmp.lb5oD0UIvP
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.wr9cQyiKkW
++ cat /tmp/tmp.lb5oD0UIvP
++ rm /tmp/tmp.wr9cQyiKkW /tmp/tmp.lb5oD0UIvP
++ return 0
+ [[ ready == \r\e\a\d\y ]]
+ echo .OK
.OK
+ simple_data_check some-name-rs0 3 0
+ local cluster_name=some-name-rs0
+ let last_pod=3-1
+ local isSharded=0
+ local cluster_pfx=
+ '[' 0 -eq 1 ']'
++ seq 0 2
+ for i in $(seq 0 $last_pod)
+ compare_mongo_cmd find myApp:myPass@some-name-rs0-0.some-name-rs0.demand-backup-15110
+ local command=find
+ local uri=myApp:myPass@some-name-rs0-0.some-name-rs0.demand-backup-15110
+ local postfix=
+ local suffix=
+ local database=myApp
+ local collection=test
+ local sort=
+ local tls=false
+ local replicaset=
+ local 'full_command=db.test.find()'
+ [[ -n '' ]]
+ log 'running db.test.find() in myApp'
+ set +o xtrace
[2026-03-27T11:37:03+0000] running db.test.find() in myApp
+ [[ false == \t\r\u\e ]]
+ mongo_command=run_mongo
+ run_mongo 'use myApp\n db.test.find()' myApp:myPass@some-name-rs0-0.some-name-rs0.demand-backup-15110 mongodb '' ''
+ local 'command=use myApp\n db.test.find()'
+ local uri=myApp:myPass@some-name-rs0-0.some-name-rs0.demand-backup-15110
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local replica_set=rs0
+ [[ myApp:myPass@some-name-rs0-0.some-name-rs0.demand-backup-15110 == *cfg* ]]
+ grep -E -v 'I NETWORK|W NETWORK|F NETWORK|"c":"NETWORK"|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|Started a new thread for the timer service'
+ /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.Xz9Z0MSGq6
+++ mktemp
++ local LAST_ERR=/tmp/tmp.LIyAX1QpTA
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.Xz9Z0MSGq6
++ cat /tmp/tmp.LIyAX1QpTA
++ rm /tmp/tmp.Xz9Z0MSGq6 /tmp/tmp.LIyAX1QpTA
++ return 0
+ local client_container=psmdb-client-bb8b97679-rgs8m
+ kubectl_bin exec psmdb-client-bb8b97679-rgs8m -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-0.some-name-rs0.demand-backup-15110.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
++ mktemp
+ local LAST_OUT=/tmp/tmp.Xs7qGAltzx
++ mktemp
+ local LAST_ERR=/tmp/tmp.i7lGmnqVyZ
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-bb8b97679-rgs8m -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-0.some-name-rs0.demand-backup-15110.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.Xs7qGAltzx
+ cat /tmp/tmp.i7lGmnqVyZ
+ rm /tmp/tmp.Xs7qGAltzx /tmp/tmp.i7lGmnqVyZ
+ return 0
+ [[ 0 -eq 0 ]]
+ diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2283/e2e-tests/demand-backup/compare/find.json /tmp/tmp.SOjtfW5Qlc/find
+ for i in $(seq 0 $last_pod)
+ compare_mongo_cmd find myApp:myPass@some-name-rs0-1.some-name-rs0.demand-backup-15110
+ local command=find
+ local uri=myApp:myPass@some-name-rs0-1.some-name-rs0.demand-backup-15110
+ local postfix=
+ local suffix=
+ local database=myApp
+ local collection=test
+ local sort=
+ local tls=false
+ local replicaset=
+ local 'full_command=db.test.find()'
+ [[ -n '' ]]
+ log 'running db.test.find() in myApp'
+ set +o xtrace
[2026-03-27T11:37:06+0000] running db.test.find() in myApp
+ [[ false == \t\r\u\e ]]
+ mongo_command=run_mongo
+ run_mongo 'use myApp\n db.test.find()' myApp:myPass@some-name-rs0-1.some-name-rs0.demand-backup-15110 mongodb '' ''
+ grep -E -v 'I NETWORK|W NETWORK|F NETWORK|"c":"NETWORK"|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|Started a new thread for the timer service'
+ local 'command=use myApp\n db.test.find()'
+ /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
+ local uri=myApp:myPass@some-name-rs0-1.some-name-rs0.demand-backup-15110
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local replica_set=rs0
+ [[ myApp:myPass@some-name-rs0-1.some-name-rs0.demand-backup-15110 == *cfg* ]]
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.HX7PjHQmga
+++ mktemp
++ local LAST_ERR=/tmp/tmp.RE9LjTK42R
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.HX7PjHQmga
++ cat /tmp/tmp.RE9LjTK42R
++ rm /tmp/tmp.HX7PjHQmga /tmp/tmp.RE9LjTK42R
++ return 0
+ local client_container=psmdb-client-bb8b97679-rgs8m
+ kubectl_bin exec psmdb-client-bb8b97679-rgs8m -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-1.some-name-rs0.demand-backup-15110.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
++ mktemp
+ local LAST_OUT=/tmp/tmp.Ru1TwQk6eq
++ mktemp
+ local LAST_ERR=/tmp/tmp.ul26HdOGdo
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-bb8b97679-rgs8m -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-1.some-name-rs0.demand-backup-15110.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.Ru1TwQk6eq
+ cat /tmp/tmp.ul26HdOGdo
+ rm /tmp/tmp.Ru1TwQk6eq /tmp/tmp.ul26HdOGdo
+ return 0
+ [[ 0 -eq 0 ]]
+ diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2283/e2e-tests/demand-backup/compare/find.json /tmp/tmp.SOjtfW5Qlc/find
+ for i in $(seq 0 $last_pod)
+ compare_mongo_cmd find myApp:myPass@some-name-rs0-2.some-name-rs0.demand-backup-15110
+ local command=find
+ local uri=myApp:myPass@some-name-rs0-2.some-name-rs0.demand-backup-15110
+ local postfix=
+ local suffix=
+ local database=myApp
+ local collection=test
+ local sort=
+ local tls=false
+ local replicaset=
+ local 'full_command=db.test.find()'
+ [[ -n '' ]]
+ log 'running db.test.find() in myApp'
+ set +o xtrace
[2026-03-27T11:37:08+0000] running db.test.find() in myApp
+ [[ false == \t\r\u\e ]]
+ mongo_command=run_mongo
+ run_mongo 'use myApp\n db.test.find()' myApp:myPass@some-name-rs0-2.some-name-rs0.demand-backup-15110 mongodb '' ''
+ local 'command=use myApp\n db.test.find()'
+ local uri=myApp:myPass@some-name-rs0-2.some-name-rs0.demand-backup-15110
+ grep -E -v 'I NETWORK|W NETWORK|F NETWORK|"c":"NETWORK"|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|Started a new thread for the timer service'
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local replica_set=rs0
+ [[ myApp:myPass@some-name-rs0-2.some-name-rs0.demand-backup-15110 == *cfg* ]]
+ /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.FKF687iEDO
+++ mktemp
++ local LAST_ERR=/tmp/tmp.7yJv9D06ho
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.FKF687iEDO
++ cat /tmp/tmp.7yJv9D06ho
++ rm /tmp/tmp.FKF687iEDO /tmp/tmp.7yJv9D06ho
++ return 0
+ local client_container=psmdb-client-bb8b97679-rgs8m
+ kubectl_bin exec psmdb-client-bb8b97679-rgs8m -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-2.some-name-rs0.demand-backup-15110.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
++ mktemp
+ local LAST_OUT=/tmp/tmp.9Y7De5WobY
++ mktemp
+ local LAST_ERR=/tmp/tmp.xDiJmTzJGL
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-bb8b97679-rgs8m -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-2.some-name-rs0.demand-backup-15110.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.9Y7De5WobY
+ cat /tmp/tmp.xDiJmTzJGL
+ rm /tmp/tmp.9Y7De5WobY /tmp/tmp.xDiJmTzJGL
+ return 0
+ [[ 0 -eq 0 ]]
+ diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2283/e2e-tests/demand-backup/compare/find.json /tmp/tmp.SOjtfW5Qlc/find
+ desc 'restore from backup source, no storageName -- minio'
+ set +o xtrace


-----------------------------------------------------------------------------------
restore from backup source, no storageName -- minio
-----------------------------------------------------------------------------------

++ get_backup_dest backup-minio
++ local backup_name=backup-minio
++ kubectl_bin get psmdb-backup backup-minio -o 'jsonpath={.status.destination}'
++ sed -e 's/.json$//'
++ sed 's|s3://||'
++ sed 's|azure://||'
++ sed 's|gs://||'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.hIqUjYnc8F
+++ mktemp
++ local LAST_ERR=/tmp/tmp.xKu5Z4uSvg
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb-backup backup-minio -o 'jsonpath={.status.destination}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.hIqUjYnc8F
++ cat /tmp/tmp.xKu5Z4uSvg
++ rm /tmp/tmp.hIqUjYnc8F /tmp/tmp.xKu5Z4uSvg
++ return 0
+ backup_dest_minio=operator-testing/2026-03-27T11:10:41Z
+ run_recovery_check_bkp_source backup-minio operator-testing/2026-03-27T11:10:41Z some-name-rs0 backup-minio-source-1
+ local backup=backup-minio
+ local backup_dest=operator-testing/2026-03-27T11:10:41Z
+ local cluster=some-name-rs0
+ local source=backup-minio-source-1
+ run_mongo 'use myApp\n db.test.insert({ x: 100501 })' myApp:myPass@some-name-rs0.demand-backup-15110
+ local 'command=use myApp\n db.test.insert({ x: 100501 })'
+ local uri=myApp:myPass@some-name-rs0.demand-backup-15110
+ local driver=mongodb+srv
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local replica_set=rs0
+ [[ myApp:myPass@some-name-rs0.demand-backup-15110 == *cfg* ]]
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.NLMHVecIf4
+++ mktemp
++ local LAST_ERR=/tmp/tmp.C5mwaLNNTx
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.NLMHVecIf4
++ cat /tmp/tmp.C5mwaLNNTx
++ rm /tmp/tmp.NLMHVecIf4 /tmp/tmp.C5mwaLNNTx
++ return 0
+ local client_container=psmdb-client-bb8b97679-rgs8m
+ kubectl_bin exec psmdb-client-bb8b97679-rgs8m -- bash -c 'printf '\''use myApp\n db.test.insert({ x: 100501 })\n'\'' | mongo mongodb+srv://myApp:myPass@some-name-rs0.demand-backup-15110.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
++ mktemp
+ local LAST_OUT=/tmp/tmp.T8QxFtuzY7
++ mktemp
+ local LAST_ERR=/tmp/tmp.TTF0sriXXt
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-bb8b97679-rgs8m -- bash -c 'printf '\''use myApp\n db.test.insert({ x: 100501 })\n'\'' | mongo mongodb+srv://myApp:myPass@some-name-rs0.demand-backup-15110.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.T8QxFtuzY7
Percona Server for MongoDB shell version v4.4.29-28
connecting to: mongodb://some-name-rs0-0.some-name-rs0.demand-backup-15110.svc.cluster.local:27017,some-name-rs0-2.some-name-rs0.demand-backup-15110.svc.cluster.local:27017,some-name-rs0-1.some-name-rs0.demand-backup-15110.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb&replicaSet=rs0&ssl=false
Implicit session: session { "id" : UUID("a17ceb4d-5591-48d8-926b-26ed9547f8c0") }
Percona Server for MongoDB server version: v8.0.19-7
WARNING: shell and server versions do not match
switched to db myApp
WriteResult({ "nInserted" : 1 })
bye
+ cat /tmp/tmp.TTF0sriXXt
+ rm /tmp/tmp.T8QxFtuzY7 /tmp/tmp.TTF0sriXXt
+ return 0
+ compare_mongo_cmd find myApp:myPass@some-name-rs0.demand-backup-15110 -2nd .svc.cluster.local myApp test
+ local command=find
+ local uri=myApp:myPass@some-name-rs0.demand-backup-15110
+ local postfix=-2nd
+ local suffix=.svc.cluster.local
+ local database=myApp
+ local collection=test
+ local sort=
+ local tls=false
+ local replicaset=
+ local 'full_command=db.test.find()'
+ [[ -n '' ]]
+ log 'running db.test.find() in myApp'
+ set +o xtrace
[2026-03-27T11:37:12+0000] running db.test.find() in myApp
+ [[ false == \t\r\u\e ]]
+ mongo_command=run_mongo
+ /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
+ run_mongo 'use myApp\n db.test.find()' myApp:myPass@some-name-rs0.demand-backup-15110 mongodb .svc.cluster.local ''
+ grep -E -v 'I NETWORK|W NETWORK|F NETWORK|"c":"NETWORK"|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|Started a new thread for the timer service'
+ local 'command=use myApp\n db.test.find()'
+ local uri=myApp:myPass@some-name-rs0.demand-backup-15110
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local replica_set=rs0
+ [[ myApp:myPass@some-name-rs0.demand-backup-15110 == *cfg* ]]
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.ZLg7oMBC4p
+++ mktemp
++ local LAST_ERR=/tmp/tmp.yChgebJzQ1
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.ZLg7oMBC4p
++ cat /tmp/tmp.yChgebJzQ1
++ rm /tmp/tmp.ZLg7oMBC4p /tmp/tmp.yChgebJzQ1
++ return 0
+ local client_container=psmdb-client-bb8b97679-rgs8m
+ kubectl_bin exec psmdb-client-bb8b97679-rgs8m -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0.demand-backup-15110.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
++ mktemp
+ local LAST_OUT=/tmp/tmp.Y0lqeEWlUh
++ mktemp
+ local LAST_ERR=/tmp/tmp.OU3IVSUPK2
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-bb8b97679-rgs8m -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0.demand-backup-15110.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.Y0lqeEWlUh
+ cat /tmp/tmp.OU3IVSUPK2
+ rm /tmp/tmp.Y0lqeEWlUh /tmp/tmp.OU3IVSUPK2
+ return 0
+ [[ 0 -eq 0 ]]
+ diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2283/e2e-tests/demand-backup/compare/find-2nd.json /tmp/tmp.SOjtfW5Qlc/find-2nd
+ run_restore_backupsource backup-minio-source-1 operator-testing/2026-03-27T11:10:41Z
+ local backupName=backup-minio-source-1
+ local backupDest=operator-testing/2026-03-27T11:10:41Z
+ local storageName=
+ desc 'run restore restore-backup-minio-source-1 from backup backup-minio-source-1 destination is operator-testing/2026-03-27T11:10:41Z'
+ set +o xtrace


-----------------------------------------------------------------------------------
run restore restore-backup-minio-source-1 from backup backup-minio-source-1 destination is operator-testing/2026-03-27T11:10:41Z
-----------------------------------------------------------------------------------

+ '[' -z '' ']'
+ cat /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2283/e2e-tests/demand-backup/conf/restore-backupsource.yml
+ /usr/sbin/sed -e 's/name:/name: restore-backup-minio-source-1/'
+ /usr/sbin/sed -e 's|BACKUP-NAME|operator-testing/2026-03-27T11:10:41Z|'
+ /usr/sbin/sed -e /storageName/d
+ kubectl_bin apply -f -
++ mktemp
+ local LAST_OUT=/tmp/tmp.hz1e6Nz7Tu
++ mktemp
+ local LAST_ERR=/tmp/tmp.WGvLt8wgiJ
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl apply -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.hz1e6Nz7Tu
perconaservermongodbrestore.psmdb.percona.com/restore-backup-minio-source-1 created
+ cat /tmp/tmp.WGvLt8wgiJ
+ rm /tmp/tmp.hz1e6Nz7Tu /tmp/tmp.WGvLt8wgiJ
+ return 0
+ return
+ wait_restore backup-minio-source-1 some-name
+ local backup_name=backup-minio-source-1
+ local cluster_name=some-name
+ local target_state=ready
+ local wait_cluster_consistency=1
+ local wait_time=1780
+ local ok_if_ready=0
+ set +o xtrace
Waiting for the psmdb-restore/restore-backup-minio-source-1 object to be created.OK
Waiting psmdb-restore/restore-backup-minio-source-1 to reach state "ready" .OK after 0 minutes
+ [[ 1 -eq 1 ]]
+ wait_cluster_consistency some-name
+ local cluster_name=some-name
+ local wait_time=32
+ retry=0
+ sleep 7
+ echo -n 'waiting for cluster readyness'
waiting for cluster readyness++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.mMsFh7vfZx
+++ mktemp
++ local LAST_ERR=/tmp/tmp.dWOE88gFIK
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.mMsFh7vfZx
++ cat /tmp/tmp.dWOE88gFIK
++ rm /tmp/tmp.mMsFh7vfZx /tmp/tmp.dWOE88gFIK
++ return 0
+ [[ ready == \r\e\a\d\y ]]
+ echo .OK
.OK
+ simple_data_check some-name-rs0 3 0
+ local cluster_name=some-name-rs0
+ let last_pod=3-1
+ local isSharded=0
+ local cluster_pfx=
+ '[' 0 -eq 1 ']'
++ seq 0 2
+ for i in $(seq 0 $last_pod)
+ compare_mongo_cmd find myApp:myPass@some-name-rs0-0.some-name-rs0.demand-backup-15110
+ local command=find
+ local uri=myApp:myPass@some-name-rs0-0.some-name-rs0.demand-backup-15110
+ local postfix=
+ local suffix=
+ local database=myApp
+ local collection=test
+ local sort=
+ local tls=false
+ local replicaset=
+ local 'full_command=db.test.find()'
+ [[ -n '' ]]
+ log 'running db.test.find() in myApp'
+ set +o xtrace
[2026-03-27T11:37:35+0000] running db.test.find() in myApp
+ [[ false == \t\r\u\e ]]
+ mongo_command=run_mongo
+ run_mongo 'use myApp\n db.test.find()' myApp:myPass@some-name-rs0-0.some-name-rs0.demand-backup-15110 mongodb '' ''
+ local 'command=use myApp\n db.test.find()'
+ local uri=myApp:myPass@some-name-rs0-0.some-name-rs0.demand-backup-15110
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local replica_set=rs0
+ grep -E -v 'I NETWORK|W NETWORK|F NETWORK|"c":"NETWORK"|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|Started a new thread for the timer service'
+ [[ myApp:myPass@some-name-rs0-0.some-name-rs0.demand-backup-15110 == *cfg* ]]
+ /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.6YFNcWneLW
+++ mktemp
++ local LAST_ERR=/tmp/tmp.lbpeK8IY5w
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.6YFNcWneLW
++ cat /tmp/tmp.lbpeK8IY5w
++ rm /tmp/tmp.6YFNcWneLW /tmp/tmp.lbpeK8IY5w
++ return 0
+ local client_container=psmdb-client-bb8b97679-rgs8m
+ kubectl_bin exec psmdb-client-bb8b97679-rgs8m -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-0.some-name-rs0.demand-backup-15110.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
++ mktemp
+ local LAST_OUT=/tmp/tmp.h0zXU4yQNp
++ mktemp
+ local LAST_ERR=/tmp/tmp.NMyP0yJguY
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-bb8b97679-rgs8m -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-0.some-name-rs0.demand-backup-15110.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.h0zXU4yQNp
+ cat /tmp/tmp.NMyP0yJguY
+ rm /tmp/tmp.h0zXU4yQNp /tmp/tmp.NMyP0yJguY
+ return 0
+ [[ 0 -eq 0 ]]
+ diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2283/e2e-tests/demand-backup/compare/find.json /tmp/tmp.SOjtfW5Qlc/find
+ for i in $(seq 0 $last_pod)
+ compare_mongo_cmd find myApp:myPass@some-name-rs0-1.some-name-rs0.demand-backup-15110
+ local command=find
+ local uri=myApp:myPass@some-name-rs0-1.some-name-rs0.demand-backup-15110
+ local postfix=
+ local suffix=
+ local database=myApp
+ local collection=test
+ local sort=
+ local tls=false
+ local replicaset=
+ local 'full_command=db.test.find()'
+ [[ -n '' ]]
+ log 'running db.test.find() in myApp'
+ set +o xtrace
[2026-03-27T11:37:37+0000] running db.test.find() in myApp
+ [[ false == \t\r\u\e ]]
+ mongo_command=run_mongo
+ run_mongo 'use myApp\n db.test.find()' myApp:myPass@some-name-rs0-1.some-name-rs0.demand-backup-15110 mongodb '' ''
+ local 'command=use myApp\n db.test.find()'
+ local uri=myApp:myPass@some-name-rs0-1.some-name-rs0.demand-backup-15110
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local replica_set=rs0
+ [[ myApp:myPass@some-name-rs0-1.some-name-rs0.demand-backup-15110 == *cfg* ]]
+ grep -E -v 'I NETWORK|W NETWORK|F NETWORK|"c":"NETWORK"|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|Started a new thread for the timer service'
+ /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.mkcDaNAMIq
+++ mktemp
++ local LAST_ERR=/tmp/tmp.RtOZzaHn66
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.mkcDaNAMIq
++ cat /tmp/tmp.RtOZzaHn66
++ rm /tmp/tmp.mkcDaNAMIq /tmp/tmp.RtOZzaHn66
++ return 0
+ local client_container=psmdb-client-bb8b97679-rgs8m
+ kubectl_bin exec psmdb-client-bb8b97679-rgs8m -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-1.some-name-rs0.demand-backup-15110.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
++ mktemp
+ local LAST_OUT=/tmp/tmp.Hb1RHXxFi3
++ mktemp
+ local LAST_ERR=/tmp/tmp.PL4v7dMqE6
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-bb8b97679-rgs8m -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-1.some-name-rs0.demand-backup-15110.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.Hb1RHXxFi3
+ cat /tmp/tmp.PL4v7dMqE6
+ rm /tmp/tmp.Hb1RHXxFi3 /tmp/tmp.PL4v7dMqE6
+ return 0
+ [[ 0 -eq 0 ]]
+ diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2283/e2e-tests/demand-backup/compare/find.json /tmp/tmp.SOjtfW5Qlc/find
+ for i in $(seq 0 $last_pod)
+ compare_mongo_cmd find myApp:myPass@some-name-rs0-2.some-name-rs0.demand-backup-15110
+ local command=find
+ local uri=myApp:myPass@some-name-rs0-2.some-name-rs0.demand-backup-15110
+ local postfix=
+ local suffix=
+ local database=myApp
+ local collection=test
+ local sort=
+ local tls=false
+ local replicaset=
+ local 'full_command=db.test.find()'
+ [[ -n '' ]]
+ log 'running db.test.find() in myApp'
+ set +o xtrace
[2026-03-27T11:37:40+0000] running db.test.find() in myApp
+ [[ false == \t\r\u\e ]]
+ mongo_command=run_mongo
+ run_mongo 'use myApp\n db.test.find()' myApp:myPass@some-name-rs0-2.some-name-rs0.demand-backup-15110 mongodb '' ''
+ local 'command=use myApp\n db.test.find()'
+ local uri=myApp:myPass@some-name-rs0-2.some-name-rs0.demand-backup-15110
+ local driver=mongodb
+ grep -E -v 'I NETWORK|W NETWORK|F NETWORK|"c":"NETWORK"|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|Started a new thread for the timer service'
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local replica_set=rs0
+ [[ myApp:myPass@some-name-rs0-2.some-name-rs0.demand-backup-15110 == *cfg* ]]
+ /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.LhG3rGICDT
+++ mktemp
++ local LAST_ERR=/tmp/tmp.6ZgWPUPZa6
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.LhG3rGICDT
++ cat /tmp/tmp.6ZgWPUPZa6
++ rm /tmp/tmp.LhG3rGICDT /tmp/tmp.6ZgWPUPZa6
++ return 0
+ local client_container=psmdb-client-bb8b97679-rgs8m
+ kubectl_bin exec psmdb-client-bb8b97679-rgs8m -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-2.some-name-rs0.demand-backup-15110.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
++ mktemp
+ local LAST_OUT=/tmp/tmp.CGfOsIXQCE
++ mktemp
+ local LAST_ERR=/tmp/tmp.dADH5DkVs0
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-bb8b97679-rgs8m -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-2.some-name-rs0.demand-backup-15110.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.CGfOsIXQCE
+ cat /tmp/tmp.dADH5DkVs0
+ rm /tmp/tmp.CGfOsIXQCE /tmp/tmp.dADH5DkVs0
+ return 0
+ [[ 0 -eq 0 ]]
+ diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2283/e2e-tests/demand-backup/compare/find.json /tmp/tmp.SOjtfW5Qlc/find
+ desc 'delete backup and check if it is removed from bucket -- minio'
+ set +o xtrace


-----------------------------------------------------------------------------------
delete backup and check if it is removed from bucket -- minio
-----------------------------------------------------------------------------------

+ kubectl_bin delete psmdb-backup --all
++ mktemp
+ local LAST_OUT=/tmp/tmp.mkfkRr2i66
++ mktemp
+ local LAST_ERR=/tmp/tmp.dPg4nbSRRn
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl delete psmdb-backup --all
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.mkfkRr2i66
perconaservermongodbbackup.psmdb.percona.com "backup-aws-s3" deleted from demand-backup-15110 namespace
perconaservermongodbbackup.psmdb.percona.com "backup-azure-blob" deleted from demand-backup-15110 namespace
perconaservermongodbbackup.psmdb.percona.com "backup-gcp-cs-s3" deleted from demand-backup-15110 namespace
perconaservermongodbbackup.psmdb.percona.com "backup-gcp-cs-sa" deleted from demand-backup-15110 namespace
perconaservermongodbbackup.psmdb.percona.com "backup-minio" deleted from demand-backup-15110 namespace
+ cat /tmp/tmp.dPg4nbSRRn
+ rm /tmp/tmp.mkfkRr2i66 /tmp/tmp.dPg4nbSRRn
+ return 0
++ kubectl_bin run -i --rm aws-cli --image=docker.io/perconalab/awscli --restart=Never -- /usr/bin/env AWS_ACCESS_KEY_ID=some-access-key AWS_SECRET_ACCESS_KEY=some-secret-key AWS_DEFAULT_REGION=us-east-1 /usr/bin/aws --endpoint-url http://minio-service:9000 s3 ls s3://operator-testing/
++ cat
++ grep -c operator-testing/2026-03-27T11:10:41Z
+++ mktemp
++ local LAST_OUT=/tmp/tmp.NVUWwlOQgx
+++ mktemp
++ local LAST_ERR=/tmp/tmp.VEmWsfh9iX
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl run -i --rm aws-cli --image=docker.io/perconalab/awscli --restart=Never -- /usr/bin/env AWS_ACCESS_KEY_ID=some-access-key AWS_SECRET_ACCESS_KEY=some-secret-key AWS_DEFAULT_REGION=us-east-1 /usr/bin/aws --endpoint-url http://minio-service:9000 s3 ls s3://operator-testing/
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.NVUWwlOQgx
++ cat /tmp/tmp.VEmWsfh9iX
All commands and output from this session will be recorded in container logs, including credentials and sensitive information passed through the command prompt.
If you don't see a command prompt, try pressing enter.
warning: couldn't attach to pod/aws-cli, falling back to streaming logs: Internal error occurred: unable to upgrade connection: container aws-cli not found in pod aws-cli_demand-backup-15110
++ rm /tmp/tmp.NVUWwlOQgx /tmp/tmp.VEmWsfh9iX
++ return 0
+ backup_exists=0
+ [[ 0 -eq 1 ]]
+ '[' -z '' ']'
+ check_backup_deletion_aws operator-testing/psmdb-demand-backup/2026-03-27T11:10:16Z /rs0/myApp.test.gz
++ echo operator-testing/psmdb-demand-backup/2026-03-27T11:10:16Z
++ cut -d/ -f1
+ bucket=operator-testing
++ cut -d/ -f2-
++ echo operator-testing/psmdb-demand-backup/2026-03-27T11:10:16Z
+ key_prefix=psmdb-demand-backup/2026-03-27T11:10:16Z
+ key=/rs0/myApp.test.gz
+ storage_name=aws-s3
+ retry=0
+ aws s3api head-object --bucket operator-testing --key psmdb-demand-backup/2026-03-27T11:10:16Z/rs0/myApp.test.gz
+ echo 'Backup psmdb-demand-backup/2026-03-27T11:10:16Z/rs0/myApp.test.gz in bucket operator-testing not found in aws-s3'
Backup psmdb-demand-backup/2026-03-27T11:10:16Z/rs0/myApp.test.gz in bucket operator-testing not found in aws-s3
+ check_backup_deletion_gcs operator-testing/psmdb-demand-backup/2026-03-27T11:10:54Z /rs0/myApp.test.gz
+ backup_dest_gcp=operator-testing/psmdb-demand-backup/2026-03-27T11:10:54Z
+ obj=/rs0/myApp.test.gz
+ storage_name=gcp-cs
+ retry=0
+ gcs_path=gs://operator-testing/psmdb-demand-backup/2026-03-27T11:10:54Z/rs0/myApp.test.gz
+ gsutil ls gs://operator-testing/psmdb-demand-backup/2026-03-27T11:10:54Z/rs0/myApp.test.gz
+ echo 'Backup gs://operator-testing/psmdb-demand-backup/2026-03-27T11:10:54Z/rs0/myApp.test.gz not found in gcp-cs'
Backup gs://operator-testing/psmdb-demand-backup/2026-03-27T11:10:54Z/rs0/myApp.test.gz not found in gcp-cs
+ check_backup_deletion_gcs operator-testing/psmdb-demand-backup/2026-03-27T11:11:07Z /rs0/myApp.test.gz
+ backup_dest_gcp=operator-testing/psmdb-demand-backup/2026-03-27T11:11:07Z
+ obj=/rs0/myApp.test.gz
+ storage_name=gcp-cs
+ retry=0
+ gcs_path=gs://operator-testing/psmdb-demand-backup/2026-03-27T11:11:07Z/rs0/myApp.test.gz
+ gsutil ls gs://operator-testing/psmdb-demand-backup/2026-03-27T11:11:07Z/rs0/myApp.test.gz
+ echo 'Backup gs://operator-testing/psmdb-demand-backup/2026-03-27T11:11:07Z/rs0/myApp.test.gz not found in gcp-cs'
Backup gs://operator-testing/psmdb-demand-backup/2026-03-27T11:11:07Z/rs0/myApp.test.gz not found in gcp-cs
+ check_backup_deletion_azure https://engk8soperators.blob.core.windows.net/operator-testing/psmdb-demand-backup/2026-03-27T11:10:28Z /rs0/myApp.test.gz
++ echo https://engk8soperators.blob.core.windows.net/operator-testing/psmdb-demand-backup/2026-03-27T11:10:28Z
++ sed 's|https://[^/]*\.blob\.core\.windows\.net/||'
+ url_path=operator-testing/psmdb-demand-backup/2026-03-27T11:10:28Z
++ echo operator-testing/psmdb-demand-backup/2026-03-27T11:10:28Z
++ cut -d/ -f1
+ container=operator-testing
++ echo operator-testing/psmdb-demand-backup/2026-03-27T11:10:28Z
++ cut -d/ -f2-
+ blob_prefix=psmdb-demand-backup/2026-03-27T11:10:28Z
+ blob=/rs0/myApp.test.gz
+ storage_name=azure-blob
+ retry=0
+ blob_path=psmdb-demand-backup/2026-03-27T11:10:28Z/rs0/myApp.test.gz
+ az storage blob show --container-name operator-testing --name psmdb-demand-backup/2026-03-27T11:10:28Z/rs0/myApp.test.gz
+ echo 'Backup psmdb-demand-backup/2026-03-27T11:10:28Z/rs0/myApp.test.gz in container operator-testing not found in azure-blob'
Backup psmdb-demand-backup/2026-03-27T11:10:28Z/rs0/myApp.test.gz in container operator-testing not found in azure-blob
+ desc 'checking backup deletion without cr'
+ set +o xtrace


-----------------------------------------------------------------------------------
checking backup deletion without cr
-----------------------------------------------------------------------------------

+ run_backup minio
+ local storage=minio
+ local backup_name=backup-minio
+ local type=logical
+ log 'running backup backup-minio'
+ set +o xtrace
[2026-03-27T11:37:58+0000] running backup backup-minio
+ yq eval '.metadata.name = "backup-minio"
			| .spec.storageName = "minio"
			| .spec.type = "logical"' /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2283/e2e-tests/demand-backup/conf/backup-minio.yml
+ kubectl_bin apply -f -
++ mktemp
+ local LAST_OUT=/tmp/tmp.Z5dzhxdzjF
++ mktemp
+ local LAST_ERR=/tmp/tmp.rUnhFRq8di
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl apply -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.Z5dzhxdzjF
perconaservermongodbbackup.psmdb.percona.com/backup-minio created
+ cat /tmp/tmp.rUnhFRq8di
+ rm /tmp/tmp.Z5dzhxdzjF /tmp/tmp.rUnhFRq8di
+ return 0
+ '[' -z '' ']'
+ run_backup aws-s3
+ local storage=aws-s3
+ local backup_name=backup-aws-s3
+ local type=logical
+ log 'running backup backup-aws-s3'
+ set +o xtrace
[2026-03-27T11:38:00+0000] running backup backup-aws-s3
+ yq eval '.metadata.name = "backup-aws-s3"
			| .spec.storageName = "aws-s3"
			| .spec.type = "logical"' /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2283/e2e-tests/demand-backup/conf/backup-aws-s3.yml
+ kubectl_bin apply -f -
++ mktemp
+ local LAST_OUT=/tmp/tmp.hauxCLyQqj
++ mktemp
+ local LAST_ERR=/tmp/tmp.nY4vjbwNCI
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl apply -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.hauxCLyQqj
perconaservermongodbbackup.psmdb.percona.com/backup-aws-s3 created
+ cat /tmp/tmp.nY4vjbwNCI
+ rm /tmp/tmp.hauxCLyQqj /tmp/tmp.nY4vjbwNCI
+ return 0
+ run_backup gcp-cs-s3
+ local storage=gcp-cs-s3
+ local backup_name=backup-gcp-cs-s3
+ local type=logical
+ log 'running backup backup-gcp-cs-s3'
+ set +o xtrace
[2026-03-27T11:38:02+0000] running backup backup-gcp-cs-s3
+ yq eval '.metadata.name = "backup-gcp-cs-s3"
			| .spec.storageName = "gcp-cs-s3"
			| .spec.type = "logical"' /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2283/e2e-tests/demand-backup/conf/backup-gcp-cs-s3.yml
+ kubectl_bin apply -f -
++ mktemp
+ local LAST_OUT=/tmp/tmp.c19u3htY9e
++ mktemp
+ local LAST_ERR=/tmp/tmp.6hM7OJ5Jwc
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl apply -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.c19u3htY9e
perconaservermongodbbackup.psmdb.percona.com/backup-gcp-cs-s3 created
+ cat /tmp/tmp.6hM7OJ5Jwc
+ rm /tmp/tmp.c19u3htY9e /tmp/tmp.6hM7OJ5Jwc
+ return 0
+ run_backup gcp-cs-sa
+ local storage=gcp-cs-sa
+ local backup_name=backup-gcp-cs-sa
+ local type=logical
+ log 'running backup backup-gcp-cs-sa'
+ set +o xtrace
[2026-03-27T11:38:04+0000] running backup backup-gcp-cs-sa
+ yq eval '.metadata.name = "backup-gcp-cs-sa"
			| .spec.storageName = "gcp-cs-sa"
			| .spec.type = "logical"' /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2283/e2e-tests/demand-backup/conf/backup-gcp-cs-sa.yml
+ kubectl_bin apply -f -
++ mktemp
+ local LAST_OUT=/tmp/tmp.IlXgzkDzof
++ mktemp
+ local LAST_ERR=/tmp/tmp.ufD44Pw4S0
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl apply -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.IlXgzkDzof
perconaservermongodbbackup.psmdb.percona.com/backup-gcp-cs-sa created
+ cat /tmp/tmp.ufD44Pw4S0
+ rm /tmp/tmp.IlXgzkDzof /tmp/tmp.ufD44Pw4S0
+ return 0
+ run_backup azure-blob
+ local storage=azure-blob
+ local backup_name=backup-azure-blob
+ local type=logical
+ log 'running backup backup-azure-blob'
+ set +o xtrace
[2026-03-27T11:38:05+0000] running backup backup-azure-blob
+ yq eval '.metadata.name = "backup-azure-blob"
			| .spec.storageName = "azure-blob"
			| .spec.type = "logical"' /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2283/e2e-tests/demand-backup/conf/backup-azure-blob.yml
+ kubectl_bin apply -f -
++ mktemp
+ local LAST_OUT=/tmp/tmp.NlPOOLq16a
++ mktemp
+ local LAST_ERR=/tmp/tmp.mfpZm4vqCf
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl apply -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.NlPOOLq16a
perconaservermongodbbackup.psmdb.percona.com/backup-azure-blob created
+ cat /tmp/tmp.mfpZm4vqCf
+ rm /tmp/tmp.NlPOOLq16a /tmp/tmp.mfpZm4vqCf
+ return 0
+ wait_backup backup-minio
+ local backup_name=backup-minio
+ local target_state=ready
+ set +o xtrace
waiting for backup-minio to reach ready state...OK
+ '[' -z '' ']'
+ wait_backup backup-aws-s3
+ local backup_name=backup-aws-s3
+ local target_state=ready
+ set +o xtrace
waiting for backup-aws-s3 to reach ready state........OK
+ wait_backup backup-gcp-cs-s3
+ local backup_name=backup-gcp-cs-s3
+ local target_state=ready
+ set +o xtrace
waiting for backup-gcp-cs-s3 to reach ready state............OK
+ wait_backup backup-gcp-cs-sa
+ local backup_name=backup-gcp-cs-sa
+ local target_state=ready
+ set +o xtrace
waiting for backup-gcp-cs-sa to reach ready state.......OK
+ wait_backup backup-azure-blob
+ local backup_name=backup-azure-blob
+ local target_state=ready
+ set +o xtrace
waiting for backup-azure-blob to reach ready state..OK
++ get_backup_dest backup-aws-s3
++ local backup_name=backup-aws-s3
++ kubectl_bin get psmdb-backup backup-aws-s3 -o 'jsonpath={.status.destination}'
++ sed -e 's/.json$//'
+++ mktemp
++ sed 's|s3://||'
++ sed 's|azure://||'
++ sed 's|gs://||'
++ local LAST_OUT=/tmp/tmp.BHnoz6tl1i
+++ mktemp
++ local LAST_ERR=/tmp/tmp.qISbXkrA7M
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb-backup backup-aws-s3 -o 'jsonpath={.status.destination}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.BHnoz6tl1i
++ cat /tmp/tmp.qISbXkrA7M
++ rm /tmp/tmp.BHnoz6tl1i /tmp/tmp.qISbXkrA7M
++ return 0
+ backup_dest_aws=operator-testing/psmdb-demand-backup/2026-03-27T11:38:12Z
++ get_backup_dest backup-gcp-cs-s3
++ local backup_name=backup-gcp-cs-s3
++ kubectl_bin get psmdb-backup backup-gcp-cs-s3 -o 'jsonpath={.status.destination}'
++ sed -e 's/.json$//'
++ sed 's|s3://||'
+++ mktemp
++ sed 's|azure://||'
++ sed 's|gs://||'
++ local LAST_OUT=/tmp/tmp.W8Iu22arKu
+++ mktemp
++ local LAST_ERR=/tmp/tmp.oMTFeIOJVg
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb-backup backup-gcp-cs-s3 -o 'jsonpath={.status.destination}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.W8Iu22arKu
++ cat /tmp/tmp.oMTFeIOJVg
++ rm /tmp/tmp.W8Iu22arKu /tmp/tmp.oMTFeIOJVg
++ return 0
+ backup_dest_gcp_s3=operator-testing/psmdb-demand-backup/2026-03-27T11:38:35Z
++ get_backup_dest backup-gcp-cs-sa
++ local backup_name=backup-gcp-cs-sa
++ kubectl_bin get psmdb-backup backup-gcp-cs-sa -o 'jsonpath={.status.destination}'
++ sed -e 's/.json$//'
++ sed 's|s3://||'
+++ mktemp
++ sed 's|azure://||'
++ sed 's|gs://||'
++ local LAST_OUT=/tmp/tmp.2dCBsvMU4P
+++ mktemp
++ local LAST_ERR=/tmp/tmp.WWPLmNQLTO
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb-backup backup-gcp-cs-sa -o 'jsonpath={.status.destination}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.2dCBsvMU4P
++ cat /tmp/tmp.WWPLmNQLTO
++ rm /tmp/tmp.2dCBsvMU4P /tmp/tmp.WWPLmNQLTO
++ return 0
+ backup_dest_gcp_sa=operator-testing/psmdb-demand-backup/2026-03-27T11:38:46Z
++ get_backup_dest backup-azure-blob
++ local backup_name=backup-azure-blob
++ kubectl_bin get psmdb-backup backup-azure-blob -o 'jsonpath={.status.destination}'
++ sed -e 's/.json$//'
++ sed 's|s3://||'
++ sed 's|azure://||'
+++ mktemp
++ sed 's|gs://||'
++ local LAST_OUT=/tmp/tmp.bOVDnWbULK
+++ mktemp
++ local LAST_ERR=/tmp/tmp.12AHBmXSct
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb-backup backup-azure-blob -o 'jsonpath={.status.destination}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.bOVDnWbULK
++ cat /tmp/tmp.12AHBmXSct
++ rm /tmp/tmp.bOVDnWbULK /tmp/tmp.12AHBmXSct
++ return 0
+ backup_dest_azure=https://engk8soperators.blob.core.windows.net/operator-testing/psmdb-demand-backup/2026-03-27T11:38:23Z
+ kubectl_bin delete psmdb --all
++ mktemp
+ local LAST_OUT=/tmp/tmp.HQKrejzOf8
++ mktemp
+ local LAST_ERR=/tmp/tmp.WlOVEe92dS
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl delete psmdb --all
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.HQKrejzOf8
perconaservermongodb.psmdb.percona.com "some-name" deleted from demand-backup-15110 namespace
+ cat /tmp/tmp.WlOVEe92dS
+ rm /tmp/tmp.HQKrejzOf8 /tmp/tmp.WlOVEe92dS
+ return 0
+ sleep 60
+ desc 'delete backup and check if it is removed from bucket -- minio'
+ set +o xtrace


-----------------------------------------------------------------------------------
delete backup and check if it is removed from bucket -- minio
-----------------------------------------------------------------------------------

+ kubectl_bin delete psmdb-backup --all
++ mktemp
+ local LAST_OUT=/tmp/tmp.weXGGQDOQG
++ mktemp
+ local LAST_ERR=/tmp/tmp.XKZ2KISLE6
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl delete psmdb-backup --all
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.weXGGQDOQG
perconaservermongodbbackup.psmdb.percona.com "backup-aws-s3" deleted from demand-backup-15110 namespace
perconaservermongodbbackup.psmdb.percona.com "backup-azure-blob" deleted from demand-backup-15110 namespace
perconaservermongodbbackup.psmdb.percona.com "backup-gcp-cs-s3" deleted from demand-backup-15110 namespace
perconaservermongodbbackup.psmdb.percona.com "backup-gcp-cs-sa" deleted from demand-backup-15110 namespace
perconaservermongodbbackup.psmdb.percona.com "backup-minio" deleted from demand-backup-15110 namespace
+ cat /tmp/tmp.XKZ2KISLE6
+ rm /tmp/tmp.weXGGQDOQG /tmp/tmp.XKZ2KISLE6
+ return 0
++ kubectl_bin run -i --rm aws-cli --image=docker.io/perconalab/awscli --restart=Never -- /usr/bin/env AWS_ACCESS_KEY_ID=some-access-key AWS_SECRET_ACCESS_KEY=some-secret-key AWS_DEFAULT_REGION=us-east-1 /usr/bin/aws --endpoint-url http://minio-service:9000 s3 ls s3://operator-testing/
++ grep -c operator-testing/2026-03-27T11:10:41Z
++ cat
+++ mktemp
++ local LAST_OUT=/tmp/tmp.VE0o7AIv9v
+++ mktemp
++ local LAST_ERR=/tmp/tmp.eYnIVk6i6T
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl run -i --rm aws-cli --image=docker.io/perconalab/awscli --restart=Never -- /usr/bin/env AWS_ACCESS_KEY_ID=some-access-key AWS_SECRET_ACCESS_KEY=some-secret-key AWS_DEFAULT_REGION=us-east-1 /usr/bin/aws --endpoint-url http://minio-service:9000 s3 ls s3://operator-testing/
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.VE0o7AIv9v
++ cat /tmp/tmp.eYnIVk6i6T
++ rm /tmp/tmp.VE0o7AIv9v /tmp/tmp.eYnIVk6i6T
++ return 0
+ backup_exists=0
+ [[ 0 -eq 1 ]]
+ '[' -z '' ']'
+ check_backup_deletion_aws operator-testing/psmdb-demand-backup/2026-03-27T11:38:12Z /rs0/myApp.test.gz
++ echo operator-testing/psmdb-demand-backup/2026-03-27T11:38:12Z
++ cut -d/ -f1
+ bucket=operator-testing
++ echo operator-testing/psmdb-demand-backup/2026-03-27T11:38:12Z
++ cut -d/ -f2-
+ key_prefix=psmdb-demand-backup/2026-03-27T11:38:12Z
+ key=/rs0/myApp.test.gz
+ storage_name=aws-s3
+ retry=0
+ aws s3api head-object --bucket operator-testing --key psmdb-demand-backup/2026-03-27T11:38:12Z/rs0/myApp.test.gz
+ echo 'Backup psmdb-demand-backup/2026-03-27T11:38:12Z/rs0/myApp.test.gz in bucket operator-testing not found in aws-s3'
Backup psmdb-demand-backup/2026-03-27T11:38:12Z/rs0/myApp.test.gz in bucket operator-testing not found in aws-s3
+ check_backup_deletion_gcs operator-testing/psmdb-demand-backup/2026-03-27T11:38:35Z /rs0/myApp.test.gz
+ backup_dest_gcp=operator-testing/psmdb-demand-backup/2026-03-27T11:38:35Z
+ obj=/rs0/myApp.test.gz
+ storage_name=gcp-cs
+ retry=0
+ gcs_path=gs://operator-testing/psmdb-demand-backup/2026-03-27T11:38:35Z/rs0/myApp.test.gz
+ gsutil ls gs://operator-testing/psmdb-demand-backup/2026-03-27T11:38:35Z/rs0/myApp.test.gz
+ echo 'Backup gs://operator-testing/psmdb-demand-backup/2026-03-27T11:38:35Z/rs0/myApp.test.gz not found in gcp-cs'
Backup gs://operator-testing/psmdb-demand-backup/2026-03-27T11:38:35Z/rs0/myApp.test.gz not found in gcp-cs
+ check_backup_deletion_gcs operator-testing/psmdb-demand-backup/2026-03-27T11:38:46Z /rs0/myApp.test.gz
+ backup_dest_gcp=operator-testing/psmdb-demand-backup/2026-03-27T11:38:46Z
+ obj=/rs0/myApp.test.gz
+ storage_name=gcp-cs
+ retry=0
+ gcs_path=gs://operator-testing/psmdb-demand-backup/2026-03-27T11:38:46Z/rs0/myApp.test.gz
+ gsutil ls gs://operator-testing/psmdb-demand-backup/2026-03-27T11:38:46Z/rs0/myApp.test.gz
+ echo 'Backup gs://operator-testing/psmdb-demand-backup/2026-03-27T11:38:46Z/rs0/myApp.test.gz not found in gcp-cs'
Backup gs://operator-testing/psmdb-demand-backup/2026-03-27T11:38:46Z/rs0/myApp.test.gz not found in gcp-cs
+ check_backup_deletion_azure https://engk8soperators.blob.core.windows.net/operator-testing/psmdb-demand-backup/2026-03-27T11:38:23Z /rs0/myApp.test.gz
++ echo https://engk8soperators.blob.core.windows.net/operator-testing/psmdb-demand-backup/2026-03-27T11:38:23Z
++ sed 's|https://[^/]*\.blob\.core\.windows\.net/||'
+ url_path=operator-testing/psmdb-demand-backup/2026-03-27T11:38:23Z
++ echo operator-testing/psmdb-demand-backup/2026-03-27T11:38:23Z
++ cut -d/ -f1
+ container=operator-testing
++ echo operator-testing/psmdb-demand-backup/2026-03-27T11:38:23Z
++ cut -d/ -f2-
+ blob_prefix=psmdb-demand-backup/2026-03-27T11:38:23Z
+ blob=/rs0/myApp.test.gz
+ storage_name=azure-blob
+ retry=0
+ blob_path=psmdb-demand-backup/2026-03-27T11:38:23Z/rs0/myApp.test.gz
+ az storage blob show --container-name operator-testing --name psmdb-demand-backup/2026-03-27T11:38:23Z/rs0/myApp.test.gz
+ echo 'Backup psmdb-demand-backup/2026-03-27T11:38:23Z/rs0/myApp.test.gz in container operator-testing not found in azure-blob'
Backup psmdb-demand-backup/2026-03-27T11:38:23Z/rs0/myApp.test.gz in container operator-testing not found in azure-blob
+ destroy demand-backup-15110
+ local namespace=demand-backup-15110
+ local ignore_logs=true
+ [[ 0 == 1 ]]
+ desc 'destroy cluster/operator and all other resources'
+ set +o xtrace


-----------------------------------------------------------------------------------
destroy cluster/operator and all other resources
-----------------------------------------------------------------------------------

+ '[' true == false ']'
+ delete_backups
+ desc 'Delete psmdb-backup'
+ set +o xtrace


-----------------------------------------------------------------------------------
Delete psmdb-backup
-----------------------------------------------------------------------------------

++ kubectl_bin get psmdb-backup --no-headers
++ wc -l
+++ mktemp
++ local LAST_OUT=/tmp/tmp.yowrbTjwnZ
+++ mktemp
++ local LAST_ERR=/tmp/tmp.Vg7Bgx4LF5
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb-backup --no-headers
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.yowrbTjwnZ
++ cat /tmp/tmp.Vg7Bgx4LF5
No resources found in demand-backup-15110 namespace.
++ rm /tmp/tmp.yowrbTjwnZ /tmp/tmp.Vg7Bgx4LF5
++ return 0
+ '[' 0 '!=' 0 ']'
+ delete_crd
+ desc 'get and delete old CRDs and RBAC'
+ set +o xtrace


-----------------------------------------------------------------------------------
get and delete old CRDs and RBAC
-----------------------------------------------------------------------------------

+ kubectl_bin delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2283/deploy/crd.yaml --ignore-not-found --wait=false
++ mktemp
+ local LAST_OUT=/tmp/tmp.1QXMk7m5zv
++ mktemp
+ local LAST_ERR=/tmp/tmp.HIvVTt5ina
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2283/deploy/crd.yaml --ignore-not-found --wait=false
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.1QXMk7m5zv
customresourcedefinition.apiextensions.k8s.io "perconaservermongodbbackups.psmdb.percona.com" deleted
customresourcedefinition.apiextensions.k8s.io "perconaservermongodbrestores.psmdb.percona.com" deleted
customresourcedefinition.apiextensions.k8s.io "perconaservermongodbs.psmdb.percona.com" deleted
+ cat /tmp/tmp.HIvVTt5ina
+ rm /tmp/tmp.1QXMk7m5zv /tmp/tmp.HIvVTt5ina
+ return 0
++ yq eval .metadata.name /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2283/deploy/crd.yaml
++ grep -v '\-\-\-'
grep: warning: stray \ before -
grep: warning: stray \ before -
+ for crd_name in $(yq eval '.metadata.name' "${src_dir}/deploy/crd.yaml" | grep -v '\-\-\-')
+ kubectl get perconaservermongodbbackups.psmdb.percona.com --all-namespaces -o wide
+ grep -v NAMESPACE
+ xargs -L 1 sh -xc 'kubectl patch perconaservermongodbbackups.psmdb.percona.com -n $0 $1 --type=merge -p "{\"metadata\":{\"finalizers\":[]}}"'
error: the server doesn't have a resource type "perconaservermongodbbackups"
+ kubectl patch perconaservermongodbbackups.psmdb.percona.com -n sh --type=merge -p '{"metadata":{"finalizers":[]}}'
error: the server doesn't have a resource type "perconaservermongodbbackups"
+ :
+ kubectl_bin wait --for=delete crd perconaservermongodbbackups.psmdb.percona.com
++ mktemp
+ local LAST_OUT=/tmp/tmp.1eZZddoa8E
++ mktemp
+ local LAST_ERR=/tmp/tmp.3K0WUOYxnJ
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl wait --for=delete crd perconaservermongodbbackups.psmdb.percona.com
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.1eZZddoa8E
+ cat /tmp/tmp.3K0WUOYxnJ
+ rm /tmp/tmp.1eZZddoa8E /tmp/tmp.3K0WUOYxnJ
+ return 0
+ for crd_name in $(yq eval '.metadata.name' "${src_dir}/deploy/crd.yaml" | grep -v '\-\-\-')
+ kubectl get perconaservermongodbrestores.psmdb.percona.com --all-namespaces -o wide
+ grep -v NAMESPACE
+ xargs -L 1 sh -xc 'kubectl patch perconaservermongodbrestores.psmdb.percona.com -n $0 $1 --type=merge -p "{\"metadata\":{\"finalizers\":[]}}"'
error: the server doesn't have a resource type "perconaservermongodbrestores"
+ kubectl patch perconaservermongodbrestores.psmdb.percona.com -n sh --type=merge -p '{"metadata":{"finalizers":[]}}'
error: the server doesn't have a resource type "perconaservermongodbrestores"
+ :
+ kubectl_bin wait --for=delete crd perconaservermongodbrestores.psmdb.percona.com
++ mktemp
+ local LAST_OUT=/tmp/tmp.DHR1C8uiyP
++ mktemp
+ local LAST_ERR=/tmp/tmp.8ERN2pyqmT
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl wait --for=delete crd perconaservermongodbrestores.psmdb.percona.com
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.DHR1C8uiyP
+ cat /tmp/tmp.8ERN2pyqmT
+ rm /tmp/tmp.DHR1C8uiyP /tmp/tmp.8ERN2pyqmT
+ return 0
+ for crd_name in $(yq eval '.metadata.name' "${src_dir}/deploy/crd.yaml" | grep -v '\-\-\-')
+ kubectl get perconaservermongodbs.psmdb.percona.com --all-namespaces -o wide
+ grep -v NAMESPACE
+ xargs -L 1 sh -xc 'kubectl patch perconaservermongodbs.psmdb.percona.com -n $0 $1 --type=merge -p "{\"metadata\":{\"finalizers\":[]}}"'
error: the server doesn't have a resource type "perconaservermongodbs"
+ kubectl patch perconaservermongodbs.psmdb.percona.com -n sh --type=merge -p '{"metadata":{"finalizers":[]}}'
error: the server doesn't have a resource type "perconaservermongodbs"
+ :
+ kubectl_bin wait --for=delete crd perconaservermongodbs.psmdb.percona.com
++ mktemp
+ local LAST_OUT=/tmp/tmp.H72WGHO1wf
++ mktemp
+ local LAST_ERR=/tmp/tmp.nlnF83zY40
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl wait --for=delete crd perconaservermongodbs.psmdb.percona.com
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.H72WGHO1wf
+ cat /tmp/tmp.nlnF83zY40
+ rm /tmp/tmp.H72WGHO1wf /tmp/tmp.nlnF83zY40
+ return 0
+ local rbac_yaml=rbac.yaml
+ '[' -n psmdb-operator ']'
+ rbac_yaml=cw-rbac.yaml
+ kubectl_bin delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2283/deploy/cw-rbac.yaml --ignore-not-found
++ mktemp
+ local LAST_OUT=/tmp/tmp.DHBhNwUug0
++ mktemp
+ local LAST_ERR=/tmp/tmp.tx5T9OTQ8w
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2283/deploy/cw-rbac.yaml --ignore-not-found
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.DHBhNwUug0
clusterrole.rbac.authorization.k8s.io "percona-server-mongodb-operator" deleted
clusterrolebinding.rbac.authorization.k8s.io "service-account-percona-server-mongodb-operator" deleted
+ cat /tmp/tmp.tx5T9OTQ8w
+ rm /tmp/tmp.DHBhNwUug0 /tmp/tmp.tx5T9OTQ8w
+ return 0
+ destroy_cert_manager
+ kubectl_bin delete -f https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml
++ mktemp
+ local LAST_OUT=/tmp/tmp.XdJdvflRRV
++ mktemp
+ local LAST_ERR=/tmp/tmp.UFMmU0sqEX
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl delete -f https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml
+ exit_status=1
+ set -e
+ '[' 1 '!=' 0 -a -n 1 ']'
+ cat /tmp/tmp.XdJdvflRRV
+ cat /tmp/tmp.UFMmU0sqEX
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": namespaces "cert-manager" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "challenges.acme.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "orders.acme.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "certificaterequests.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "certificates.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "clusterissuers.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "issuers.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": serviceaccounts "cert-manager-cainjector" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": serviceaccounts "cert-manager" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": serviceaccounts "cert-manager-webhook" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-cainjector" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-issuers" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-clusterissuers" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-certificates" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-orders" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-challenges" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-ingress-shim" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-cluster-view" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-view" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-edit" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-approve:cert-manager-io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-certificatesigningrequests" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-webhook:subjectaccessreviews" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-cainjector" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-issuers" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-clusterissuers" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-certificates" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-orders" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-challenges" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-ingress-shim" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-approve:cert-manager-io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-certificatesigningrequests" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-webhook:subjectaccessreviews" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager-cainjector:leaderelection" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager:leaderelection" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager-tokenrequest" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager-webhook:dynamic-serving" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager-cainjector:leaderelection" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager:leaderelection" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager-tokenrequest" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager-webhook:dynamic-serving" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": services "cert-manager-cainjector" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": services "cert-manager" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": services "cert-manager-webhook" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": deployments.apps "cert-manager-cainjector" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": deployments.apps "cert-manager" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": deployments.apps "cert-manager-webhook" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": mutatingwebhookconfigurations.admissionregistration.k8s.io "cert-manager-webhook" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": validatingwebhookconfigurations.admissionregistration.k8s.io "cert-manager-webhook" not found
+ sleep 0
+ for i in $(seq 0 2)
+ set +e
+ kubectl delete -f https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml
+ exit_status=1
+ set -e
+ '[' 1 '!=' 0 -a -n 1 ']'
+ cat /tmp/tmp.XdJdvflRRV
+ cat /tmp/tmp.UFMmU0sqEX
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": namespaces "cert-manager" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "challenges.acme.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "orders.acme.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "certificaterequests.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "certificates.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "clusterissuers.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "issuers.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": serviceaccounts "cert-manager-cainjector" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": serviceaccounts "cert-manager" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": serviceaccounts "cert-manager-webhook" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-cainjector" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-issuers" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-clusterissuers" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-certificates" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-orders" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-challenges" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-ingress-shim" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-cluster-view" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-view" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-edit" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-approve:cert-manager-io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-certificatesigningrequests" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-webhook:subjectaccessreviews" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-cainjector" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-issuers" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-clusterissuers" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-certificates" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-orders" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-challenges" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-ingress-shim" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-approve:cert-manager-io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-certificatesigningrequests" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-webhook:subjectaccessreviews" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager-cainjector:leaderelection" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager:leaderelection" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager-tokenrequest" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager-webhook:dynamic-serving" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager-cainjector:leaderelection" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager:leaderelection" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager-tokenrequest" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager-webhook:dynamic-serving" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": services "cert-manager-cainjector" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": services "cert-manager" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": services "cert-manager-webhook" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": deployments.apps "cert-manager-cainjector" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": deployments.apps "cert-manager" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": deployments.apps "cert-manager-webhook" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": mutatingwebhookconfigurations.admissionregistration.k8s.io "cert-manager-webhook" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": validatingwebhookconfigurations.admissionregistration.k8s.io "cert-manager-webhook" not found
+ sleep 4
+ for i in $(seq 0 2)
+ set +e
+ kubectl delete -f https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml
+ exit_status=1
+ set -e
+ '[' 1 '!=' 0 -a -n 1 ']'
+ cat /tmp/tmp.XdJdvflRRV
+ cat /tmp/tmp.UFMmU0sqEX
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": namespaces "cert-manager" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "challenges.acme.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "orders.acme.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "certificaterequests.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "certificates.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "clusterissuers.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "issuers.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": serviceaccounts "cert-manager-cainjector" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": serviceaccounts "cert-manager" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": serviceaccounts "cert-manager-webhook" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-cainjector" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-issuers" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-clusterissuers" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-certificates" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-orders" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-challenges" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-ingress-shim" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-cluster-view" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-view" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-edit" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-approve:cert-manager-io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-certificatesigningrequests" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-webhook:subjectaccessreviews" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-cainjector" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-issuers" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-clusterissuers" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-certificates" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-orders" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-challenges" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-ingress-shim" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-approve:cert-manager-io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-certificatesigningrequests" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-webhook:subjectaccessreviews" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager-cainjector:leaderelection" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager:leaderelection" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager-tokenrequest" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager-webhook:dynamic-serving" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager-cainjector:leaderelection" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager:leaderelection" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager-tokenrequest" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager-webhook:dynamic-serving" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": services "cert-manager-cainjector" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": services "cert-manager" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": services "cert-manager-webhook" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": deployments.apps "cert-manager-cainjector" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": deployments.apps "cert-manager" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": deployments.apps "cert-manager-webhook" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": mutatingwebhookconfigurations.admissionregistration.k8s.io "cert-manager-webhook" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": validatingwebhookconfigurations.admissionregistration.k8s.io "cert-manager-webhook" not found
+ sleep 8
+ cat /tmp/tmp.XdJdvflRRV
+ cat /tmp/tmp.UFMmU0sqEX
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": namespaces "cert-manager" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "challenges.acme.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "orders.acme.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "certificaterequests.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "certificates.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "clusterissuers.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "issuers.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": serviceaccounts "cert-manager-cainjector" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": serviceaccounts "cert-manager" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": serviceaccounts "cert-manager-webhook" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-cainjector" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-issuers" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-clusterissuers" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-certificates" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-orders" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-challenges" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-ingress-shim" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-cluster-view" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-view" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-edit" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-approve:cert-manager-io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-certificatesigningrequests" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-webhook:subjectaccessreviews" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-cainjector" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-issuers" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-clusterissuers" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-certificates" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-orders" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-challenges" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-ingress-shim" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-approve:cert-manager-io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-certificatesigningrequests" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-webhook:subjectaccessreviews" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager-cainjector:leaderelection" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager:leaderelection" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager-tokenrequest" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager-webhook:dynamic-serving" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager-cainjector:leaderelection" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager:leaderelection" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager-tokenrequest" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager-webhook:dynamic-serving" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": services "cert-manager-cainjector" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": services "cert-manager" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": services "cert-manager-webhook" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": deployments.apps "cert-manager-cainjector" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": deployments.apps "cert-manager" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": deployments.apps "cert-manager-webhook" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": mutatingwebhookconfigurations.admissionregistration.k8s.io "cert-manager-webhook" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": validatingwebhookconfigurations.admissionregistration.k8s.io "cert-manager-webhook" not found
+ rm /tmp/tmp.XdJdvflRRV /tmp/tmp.UFMmU0sqEX
+ return 1
+ true
+ '[' -n '' ']'
+ '[' -n psmdb-operator ']'
+ kubectl_bin delete --grace-period=0 --force=true namespace demand-backup-15110
+ rm -rf /tmp/tmp.SOjtfW5Qlc
+ kubectl_bin delete --grace-period=0 --force=true namespace psmdb-operator
++ mktemp
++ mktemp
+ local LAST_OUT=/tmp/tmp.yVtK8YdHAv
++ mktemp
+ desc 'test passed'
+ set +o xtrace


-----------------------------------------------------------------------------------
test passed
-----------------------------------------------------------------------------------

+ local LAST_ERR=/tmp/tmp.FflwtXkMzk
+ local exit_status=0
+ local timeout=4
+ local LAST_OUT=/tmp/tmp.TFEyacOmrI
++ seq 0 2
++ mktemp
+ for i in $(seq 0 2)
+ set +e
+ kubectl delete --grace-period=0 --force=true namespace demand-backup-15110
+ local LAST_ERR=/tmp/tmp.vI7lJlShId
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl delete --grace-period=0 --force=true namespace psmdb-operator