Log: /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2101/e2e-tests/logs/demand-backup.log
grep: warning: stray \ before -
Warning: version difference between client (1.34) and server (1.31) exceeds the supported minor version skew of +/-1
Warning: version difference between client (1.34) and server (1.31) exceeds the supported minor version skew of +/-1
Warning: version difference between client (1.34) and server (1.31) exceeds the supported minor version skew of +/-1
+ create_infra demand-backup-3000
+ local ns=demand-backup-3000
+ [[ 1 == 1 ]]
+ delete_crd
+ desc 'get and delete old CRDs and RBAC'
+ set +o xtrace


-----------------------------------------------------------------------------------
get and delete old CRDs and RBAC
-----------------------------------------------------------------------------------

+ kubectl_bin delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2101/deploy/crd.yaml --ignore-not-found --wait=false
++ mktemp
+ local LAST_OUT=/tmp/tmp.vuq5MJ6e2u
++ mktemp
+ local LAST_ERR=/tmp/tmp.iuIKBIxCaP
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2101/deploy/crd.yaml --ignore-not-found --wait=false
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.vuq5MJ6e2u
customresourcedefinition.apiextensions.k8s.io "perconaservermongodbbackups.psmdb.percona.com" deleted
customresourcedefinition.apiextensions.k8s.io "perconaservermongodbrestores.psmdb.percona.com" deleted
customresourcedefinition.apiextensions.k8s.io "perconaservermongodbs.psmdb.percona.com" deleted
+ cat /tmp/tmp.iuIKBIxCaP
+ rm /tmp/tmp.vuq5MJ6e2u /tmp/tmp.iuIKBIxCaP
+ return 0
++ yq eval .metadata.name /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2101/deploy/crd.yaml
++ grep -v '\-\-\-'
grep: warning: stray \ before -
grep: warning: stray \ before -
+ for crd_name in $(yq eval '.metadata.name' "${src_dir}/deploy/crd.yaml" | grep -v '\-\-\-')
+ kubectl get perconaservermongodbbackups.psmdb.percona.com --all-namespaces -o wide
+ grep -v NAMESPACE
+ xargs -L 1 sh -xc 'kubectl patch perconaservermongodbbackups.psmdb.percona.com -n $0 $1 --type=merge -p "{\"metadata\":{\"finalizers\":[]}}"'
+ kubectl patch perconaservermongodbbackups.psmdb.percona.com -n demand-backup-12747 backup-gcp-cs-s3 --type=merge -p '{"metadata":{"finalizers":[]}}'
Error from server (NotFound): perconaservermongodbbackups.psmdb.percona.com "backup-gcp-cs-s3" not found
+ kubectl patch perconaservermongodbbackups.psmdb.percona.com -n demand-backup-12747 backup-gcp-cs-sa --type=merge -p '{"metadata":{"finalizers":[]}}'
Error from server (NotFound): perconaservermongodbbackups.psmdb.percona.com "backup-gcp-cs-sa" not found
+ kubectl patch perconaservermongodbbackups.psmdb.percona.com -n demand-backup-12747 backup-minio --type=merge -p '{"metadata":{"finalizers":[]}}'
Error from server (NotFound): perconaservermongodbbackups.psmdb.percona.com "backup-minio" not found
+ :
+ kubectl_bin wait --for=delete crd perconaservermongodbbackups.psmdb.percona.com
++ mktemp
+ local LAST_OUT=/tmp/tmp.8u163Dqb48
++ mktemp
+ local LAST_ERR=/tmp/tmp.xSo6XZ15pP
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl wait --for=delete crd perconaservermongodbbackups.psmdb.percona.com
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.8u163Dqb48
customresourcedefinition.apiextensions.k8s.io/perconaservermongodbbackups.psmdb.percona.com condition met
+ cat /tmp/tmp.xSo6XZ15pP
+ rm /tmp/tmp.8u163Dqb48 /tmp/tmp.xSo6XZ15pP
+ return 0
+ for crd_name in $(yq eval '.metadata.name' "${src_dir}/deploy/crd.yaml" | grep -v '\-\-\-')
+ kubectl get perconaservermongodbrestores.psmdb.percona.com --all-namespaces -o wide
+ grep -v NAMESPACE
+ xargs -L 1 sh -xc 'kubectl patch perconaservermongodbrestores.psmdb.percona.com -n $0 $1 --type=merge -p "{\"metadata\":{\"finalizers\":[]}}"'
error: the server doesn't have a resource type "perconaservermongodbrestores"
+ kubectl patch perconaservermongodbrestores.psmdb.percona.com -n sh --type=merge -p '{"metadata":{"finalizers":[]}}'
error: the server doesn't have a resource type "perconaservermongodbrestores"
+ :
+ kubectl_bin wait --for=delete crd perconaservermongodbrestores.psmdb.percona.com
++ mktemp
+ local LAST_OUT=/tmp/tmp.VLVpjDzkfK
++ mktemp
+ local LAST_ERR=/tmp/tmp.pB9KkpQoot
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl wait --for=delete crd perconaservermongodbrestores.psmdb.percona.com
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.VLVpjDzkfK
+ cat /tmp/tmp.pB9KkpQoot
+ rm /tmp/tmp.VLVpjDzkfK /tmp/tmp.pB9KkpQoot
+ return 0
+ for crd_name in $(yq eval '.metadata.name' "${src_dir}/deploy/crd.yaml" | grep -v '\-\-\-')
+ kubectl get perconaservermongodbs.psmdb.percona.com --all-namespaces -o wide
+ grep -v NAMESPACE
+ xargs -L 1 sh -xc 'kubectl patch perconaservermongodbs.psmdb.percona.com -n $0 $1 --type=merge -p "{\"metadata\":{\"finalizers\":[]}}"'
error: the server doesn't have a resource type "perconaservermongodbs"
+ kubectl patch perconaservermongodbs.psmdb.percona.com -n sh --type=merge -p '{"metadata":{"finalizers":[]}}'
error: the server doesn't have a resource type "perconaservermongodbs"
+ :
+ kubectl_bin wait --for=delete crd perconaservermongodbs.psmdb.percona.com
++ mktemp
+ local LAST_OUT=/tmp/tmp.HZql2Ax2iM
++ mktemp
+ local LAST_ERR=/tmp/tmp.8Q9VOKxAPF
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl wait --for=delete crd perconaservermongodbs.psmdb.percona.com
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.HZql2Ax2iM
+ cat /tmp/tmp.8Q9VOKxAPF
+ rm /tmp/tmp.HZql2Ax2iM /tmp/tmp.8Q9VOKxAPF
+ return 0
+ local rbac_yaml=rbac.yaml
+ '[' -n psmdb-operator ']'
+ rbac_yaml=cw-rbac.yaml
+ kubectl_bin delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2101/deploy/cw-rbac.yaml --ignore-not-found
++ mktemp
+ local LAST_OUT=/tmp/tmp.t4jZtFFKeA
++ mktemp
+ local LAST_ERR=/tmp/tmp.MZHWyu3nN9
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2101/deploy/cw-rbac.yaml --ignore-not-found
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.t4jZtFFKeA
clusterrole.rbac.authorization.k8s.io "percona-server-mongodb-operator" deleted
clusterrolebinding.rbac.authorization.k8s.io "service-account-percona-server-mongodb-operator" deleted
+ cat /tmp/tmp.MZHWyu3nN9
+ rm /tmp/tmp.t4jZtFFKeA /tmp/tmp.MZHWyu3nN9
+ return 0
+ check_crd_for_deletion PR-2101-0c183d95
+ local git_tag=PR-2101-0c183d95
++ curl -s https://raw.githubusercontent.com/percona/percona-server-mongodb-operator/PR-2101-0c183d95/deploy/crd.yaml
++ yq eval .metadata.name
++ /usr/sbin/sed s/---//g
++ /usr/sbin/sed ':a;N;$!ba;s/\n/ /g'
+ for crd_name in $(curl -s https://raw.githubusercontent.com/percona/percona-server-mongodb-operator/${git_tag}/deploy/crd.yaml | yq eval '.metadata.name' | $sed 's/---//g' | $sed ':a;N;$!ba;s/\n/ /g')
++ kubectl_bin get crd/null -o 'jsonpath={.status.conditions[-1].type}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.ei8C7mAqNj
+++ mktemp
++ local LAST_ERR=/tmp/tmp.s5fa6P014J
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get crd/null -o 'jsonpath={.status.conditions[-1].type}'
++ exit_status=1
++ set -e
++ '[' 1 '!=' 0 -a -n 1 ']'
++ cat /tmp/tmp.ei8C7mAqNj
++ cat /tmp/tmp.s5fa6P014J
Error from server (NotFound): customresourcedefinitions.apiextensions.k8s.io "null" not found
++ sleep 0
++ for i in $(seq 0 2)
++ set +e
++ kubectl get crd/null -o 'jsonpath={.status.conditions[-1].type}'
++ exit_status=1
++ set -e
++ '[' 1 '!=' 0 -a -n 1 ']'
++ cat /tmp/tmp.ei8C7mAqNj
++ cat /tmp/tmp.s5fa6P014J
Error from server (NotFound): customresourcedefinitions.apiextensions.k8s.io "null" not found
++ sleep 4
++ for i in $(seq 0 2)
++ set +e
++ kubectl get crd/null -o 'jsonpath={.status.conditions[-1].type}'
++ exit_status=1
++ set -e
++ '[' 1 '!=' 0 -a -n 1 ']'
++ cat /tmp/tmp.ei8C7mAqNj
++ cat /tmp/tmp.s5fa6P014J
Error from server (NotFound): customresourcedefinitions.apiextensions.k8s.io "null" not found
++ sleep 8
++ cat /tmp/tmp.ei8C7mAqNj
++ cat /tmp/tmp.s5fa6P014J
Error from server (NotFound): customresourcedefinitions.apiextensions.k8s.io "null" not found
++ rm /tmp/tmp.ei8C7mAqNj /tmp/tmp.s5fa6P014J
++ return 1
+ [[ '' == \T\e\r\m\i\n\a\t\i\n\g ]]
+ '[' -n psmdb-operator ']'
+ create_namespace psmdb-operator
+ local namespace=psmdb-operator
+ local skip_clean_namespace=
+ [[ 1 == 1 ]]
+ [[ -z '' ]]
+ destroy_chaos_mesh
++ helm list --all-namespaces --filter chaos-mesh
++ tail -n1
++ awk '-F ' '{print $2}'
++ sed s/NAMESPACE//
+ local chaos_mesh_ns=
+ desc 'destroy chaos-mesh'
+ set +o xtrace


-----------------------------------------------------------------------------------
destroy chaos-mesh
-----------------------------------------------------------------------------------

+ '[' -n '' ']'
++ kubectl get MutatingWebhookConfiguration
++ grep chaos-mesh
++ awk '{print $1}'
+ timeout 30 kubectl delete MutatingWebhookConfiguration
error: resource(s) were provided, but no name was specified
+ :
++ kubectl get ValidatingWebhookConfiguration
++ grep chaos-mesh
++ awk '{print $1}'
+ timeout 30 kubectl delete ValidatingWebhookConfiguration
error: resource(s) were provided, but no name was specified
+ :
++ kubectl get ValidatingWebhookConfiguration
++ grep validate-auth
++ awk '{print $1}'
+ timeout 30 kubectl delete ValidatingWebhookConfiguration
error: resource(s) were provided, but no name was specified
+ :
++ kubectl api-resources
++ grep chaos-mesh
++ awk '{print $1}'
++ kubectl get crd
++ grep chaos-mesh.org
++ awk '{print $1}'
+ timeout 30 kubectl delete crd
error: resource(s) were provided, but no name was specified
+ :
++ kubectl get clusterrolebinding
++ grep chaos-mesh
++ awk '{print $1}'
+ timeout 30 kubectl delete clusterrolebinding
error: resource(s) were provided, but no name was specified
+ :
++ kubectl get clusterrole
++ grep chaos-mesh
++ awk '{print $1}'
+ timeout 30 kubectl delete clusterrole
error: resource(s) were provided, but no name was specified
+ :
+ desc 'cleaned up all old namespaces'
+ set +o xtrace


-----------------------------------------------------------------------------------
cleaned up all old namespaces
-----------------------------------------------------------------------------------

+ kubectl_bin get ns
+ egrep -v '^kube-|^default|Terminating|psmdb-operator|openshift|^gke-|^gmp-|^NAME'
+ awk '{print$1}'
+ '[' -n '' ']'
+ desc 'cleaned up old namespaces psmdb-operator'
+ set +o xtrace
++ mktemp


-----------------------------------------------------------------------------------
cleaned up old namespaces psmdb-operator
-----------------------------------------------------------------------------------

+ kubectl_bin delete namespace psmdb-operator --ignore-not-found
+ xargs kubectl delete ns
++ mktemp
egrep: warning: egrep is obsolescent; using grep -E
+ local LAST_OUT=/tmp/tmp.jHce5a9zXO
++ mktemp
+ local LAST_OUT=/tmp/tmp.WIcvmPU5wG
+ local LAST_ERR=/tmp/tmp.hRV9sTnvnf
+ local exit_status=0
+ local timeout=4
++ mktemp
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl get ns
+ local LAST_ERR=/tmp/tmp.UlmzNRLqGQ
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl delete namespace psmdb-operator --ignore-not-found
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.jHce5a9zXO
+ cat /tmp/tmp.hRV9sTnvnf
+ rm /tmp/tmp.jHce5a9zXO /tmp/tmp.hRV9sTnvnf
+ return 0
namespace "demand-backup-12747" deleted
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.WIcvmPU5wG
namespace "psmdb-operator" deleted
+ cat /tmp/tmp.UlmzNRLqGQ
+ rm /tmp/tmp.WIcvmPU5wG /tmp/tmp.UlmzNRLqGQ
+ return 0
+ kubectl_bin wait --for=delete namespace psmdb-operator
++ mktemp
+ local LAST_OUT=/tmp/tmp.kREB53FuqE
++ mktemp
+ local LAST_ERR=/tmp/tmp.pst3Y2IAKU
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl wait --for=delete namespace psmdb-operator
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.kREB53FuqE
+ cat /tmp/tmp.pst3Y2IAKU
+ rm /tmp/tmp.kREB53FuqE /tmp/tmp.pst3Y2IAKU
+ return 0
+ desc 'create namespace psmdb-operator'
+ set +o xtrace


-----------------------------------------------------------------------------------
create namespace psmdb-operator
-----------------------------------------------------------------------------------

+ kubectl_bin create namespace psmdb-operator
++ mktemp
+ local LAST_OUT=/tmp/tmp.y3nUzLRczU
++ mktemp
+ local LAST_ERR=/tmp/tmp.zIpQoj1P0j
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl create namespace psmdb-operator
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.y3nUzLRczU
namespace/psmdb-operator created
+ cat /tmp/tmp.zIpQoj1P0j
+ rm /tmp/tmp.y3nUzLRczU /tmp/tmp.zIpQoj1P0j
+ return 0
+ set_kube_ctx psmdb-operator
+ local namespace=psmdb-operator
++ kubectl_bin config current-context
+++ mktemp
++ local LAST_OUT=/tmp/tmp.OR4jSzv0El
+++ mktemp
++ local LAST_ERR=/tmp/tmp.renQGC0B9n
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl config current-context
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.OR4jSzv0El
++ cat /tmp/tmp.renQGC0B9n
++ rm /tmp/tmp.OR4jSzv0El /tmp/tmp.renQGC0B9n
++ return 0
+ kubectl_bin config set-context gke_cloud-dev-112233_us-central1-a_jen-psmdb-2101-0c183d95-1-cluster10 --namespace=psmdb-operator
++ mktemp
+ local LAST_OUT=/tmp/tmp.3jZ2ytkdIr
++ mktemp
+ local LAST_ERR=/tmp/tmp.0kHzeQWIIW
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl config set-context gke_cloud-dev-112233_us-central1-a_jen-psmdb-2101-0c183d95-1-cluster10 --namespace=psmdb-operator
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.3jZ2ytkdIr
Context "gke_cloud-dev-112233_us-central1-a_jen-psmdb-2101-0c183d95-1-cluster10" modified.
+ cat /tmp/tmp.0kHzeQWIIW
+ rm /tmp/tmp.3jZ2ytkdIr /tmp/tmp.0kHzeQWIIW
+ return 0
+ deploy_operator
+ desc 'start PSMDB operator: perconalab/percona-server-mongodb-operator:PR-2101-0c183d95'
+ set +o xtrace


-----------------------------------------------------------------------------------
start PSMDB operator: perconalab/percona-server-mongodb-operator:PR-2101-0c183d95
-----------------------------------------------------------------------------------

+ local cr_file
+ '[' -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2101/e2e-tests/demand-backup/conf/crd.yaml ']'
+ cr_file=/mnt/jenkins/workspace/cloud-psmdb-operator_PR-2101/deploy/crd.yaml
+ kubectl_bin apply --server-side --force-conflicts -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2101/deploy/crd.yaml
++ mktemp
+ local LAST_OUT=/tmp/tmp.TtlAVXaTpx
++ mktemp
+ local LAST_ERR=/tmp/tmp.gR709qiHHT
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl apply --server-side --force-conflicts -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2101/deploy/crd.yaml
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.TtlAVXaTpx
customresourcedefinition.apiextensions.k8s.io/perconaservermongodbbackups.psmdb.percona.com serverside-applied
customresourcedefinition.apiextensions.k8s.io/perconaservermongodbrestores.psmdb.percona.com serverside-applied
customresourcedefinition.apiextensions.k8s.io/perconaservermongodbs.psmdb.percona.com serverside-applied
+ cat /tmp/tmp.gR709qiHHT
+ rm /tmp/tmp.TtlAVXaTpx /tmp/tmp.gR709qiHHT
+ return 0
+ '[' -n psmdb-operator ']'
+ apply_rbac cw-rbac
+ local operator_namespace=psmdb-operator
+ local rbac=cw-rbac
+ cat /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2101/deploy/cw-rbac.yaml
+ sed -e 's^namespace: .*^namespace: psmdb-operator^'
+ kubectl_bin apply -n psmdb-operator -f -
++ mktemp
+ local LAST_OUT=/tmp/tmp.ISZUv0GcSl
++ mktemp
+ local LAST_ERR=/tmp/tmp.hSZ551ndYK
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl apply -n psmdb-operator -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.ISZUv0GcSl
clusterrole.rbac.authorization.k8s.io/percona-server-mongodb-operator created
serviceaccount/percona-server-mongodb-operator created
clusterrolebinding.rbac.authorization.k8s.io/service-account-percona-server-mongodb-operator created
+ cat /tmp/tmp.hSZ551ndYK
+ rm /tmp/tmp.ISZUv0GcSl /tmp/tmp.hSZ551ndYK
+ return 0
+ yq eval '
			(.spec.template.spec.containers[].image = "perconalab/percona-server-mongodb-operator:PR-2101-0c183d95") |
			((.. | select(.[] == "DISABLE_TELEMETRY")) |= .value="true") |
			((.. | select(.[] == "LOG_LEVEL")) |= .value="DEBUG")' /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2101/deploy/cw-operator.yaml
+ kubectl_bin apply -n psmdb-operator -f -
++ mktemp
+ local LAST_OUT=/tmp/tmp.mSz2n4WNlx
++ mktemp
+ local LAST_ERR=/tmp/tmp.MZbY3zFDnE
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl apply -n psmdb-operator -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.mSz2n4WNlx
deployment.apps/percona-server-mongodb-operator created
+ cat /tmp/tmp.MZbY3zFDnE
+ rm /tmp/tmp.mSz2n4WNlx /tmp/tmp.MZbY3zFDnE
+ return 0
+ sleep 20
++ get_operator_pod
++ kubectl_bin get pods --selector=name=percona-server-mongodb-operator -o 'jsonpath={.items[].metadata.name}' -n psmdb-operator
+++ mktemp
++ local LAST_OUT=/tmp/tmp.beNzuYWnZG
+++ mktemp
++ local LAST_ERR=/tmp/tmp.Uu0d7TrwBB
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=percona-server-mongodb-operator -o 'jsonpath={.items[].metadata.name}' -n psmdb-operator
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.beNzuYWnZG
++ cat /tmp/tmp.Uu0d7TrwBB
++ rm /tmp/tmp.beNzuYWnZG /tmp/tmp.Uu0d7TrwBB
++ return 0
+ wait_operator_pod percona-server-mongodb-operator-6f4db4cfcd-gjcgs
+ local pod=percona-server-mongodb-operator-6f4db4cfcd-gjcgs
+ set +o xtrace
waiting for pod/percona-server-mongodb-operator-6f4db4cfcd-gjcgs to be ready.OK
+ echo 'Print operator info from log'
Print operator info from log
+ grep 'Manager starting up'
++ get_operator_pod
++ kubectl_bin get pods --selector=name=percona-server-mongodb-operator -o 'jsonpath={.items[].metadata.name}' -n psmdb-operator
+++ mktemp
++ local LAST_OUT=/tmp/tmp.iC8c42GJ9q
+++ mktemp
++ local LAST_ERR=/tmp/tmp.agI2BNJPl6
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=percona-server-mongodb-operator -o 'jsonpath={.items[].metadata.name}' -n psmdb-operator
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.iC8c42GJ9q
++ cat /tmp/tmp.agI2BNJPl6
++ rm /tmp/tmp.iC8c42GJ9q /tmp/tmp.agI2BNJPl6
++ return 0
+ kubectl_bin logs -n psmdb-operator percona-server-mongodb-operator-6f4db4cfcd-gjcgs
++ mktemp
+ local LAST_OUT=/tmp/tmp.ggSDzfwz50
++ mktemp
+ local LAST_ERR=/tmp/tmp.LrvCFDuQih
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl logs -n psmdb-operator percona-server-mongodb-operator-6f4db4cfcd-gjcgs
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.ggSDzfwz50
+ cat /tmp/tmp.LrvCFDuQih
+ rm /tmp/tmp.ggSDzfwz50 /tmp/tmp.LrvCFDuQih
+ return 0
2025-11-03T02:10:13.897Z	INFO	setup	Manager starting up	{"gitCommit": "0c183d95497b68631f5898a1b8037ed67898e3e8", "gitBranch": "PR-2101-0c183d95", "buildTime": "", "goVersion": "go1.25.3", "os": "linux", "arch": "amd64"}
+ create_namespace demand-backup-3000
+ local namespace=demand-backup-3000
+ local skip_clean_namespace=
+ [[ 1 == 1 ]]
+ [[ -z '' ]]
+ destroy_chaos_mesh
++ helm list --all-namespaces --filter chaos-mesh
++ tail -n1
++ awk '-F ' '{print $2}'
++ sed s/NAMESPACE//
+ local chaos_mesh_ns=
+ desc 'destroy chaos-mesh'
+ set +o xtrace


-----------------------------------------------------------------------------------
destroy chaos-mesh
-----------------------------------------------------------------------------------

+ '[' -n '' ']'
++ kubectl get MutatingWebhookConfiguration
++ grep chaos-mesh
++ awk '{print $1}'
+ timeout 30 kubectl delete MutatingWebhookConfiguration
error: resource(s) were provided, but no name was specified
+ :
++ kubectl get ValidatingWebhookConfiguration
++ grep chaos-mesh
++ awk '{print $1}'
+ timeout 30 kubectl delete ValidatingWebhookConfiguration
error: resource(s) were provided, but no name was specified
+ :
++ kubectl get ValidatingWebhookConfiguration
++ grep validate-auth
++ awk '{print $1}'
+ timeout 30 kubectl delete ValidatingWebhookConfiguration
error: resource(s) were provided, but no name was specified
+ :
++ kubectl api-resources
++ grep chaos-mesh
++ awk '{print $1}'
++ kubectl get crd
++ grep chaos-mesh.org
++ awk '{print $1}'
+ timeout 30 kubectl delete crd
error: resource(s) were provided, but no name was specified
+ :
++ kubectl get clusterrolebinding
++ grep chaos-mesh
++ awk '{print $1}'
+ timeout 30 kubectl delete clusterrolebinding
error: resource(s) were provided, but no name was specified
+ :
++ kubectl get clusterrole
++ grep chaos-mesh
++ awk '{print $1}'
+ timeout 30 kubectl delete clusterrole
error: resource(s) were provided, but no name was specified
+ :
+ desc 'cleaned up all old namespaces'
+ set +o xtrace


-----------------------------------------------------------------------------------
cleaned up all old namespaces
-----------------------------------------------------------------------------------

+ kubectl_bin get ns
+ egrep -v '^kube-|^default|Terminating|psmdb-operator|openshift|^gke-|^gmp-|^NAME'
+ awk '{print$1}'
+ '[' -n '' ']'
+ desc 'cleaned up old namespaces demand-backup-3000'
+ set +o xtrace


-----------------------------------------------------------------------------------
cleaned up old namespaces demand-backup-3000++ mktemp

-----------------------------------------------------------------------------------

+ xargs kubectl delete ns
+ kubectl_bin delete namespace demand-backup-3000 --ignore-not-found
++ mktemp
egrep: warning: egrep is obsolescent; using grep -E
+ local LAST_OUT=/tmp/tmp.IBIrqt7cSI
++ mktemp
+ local LAST_OUT=/tmp/tmp.SGJhszOXvG
+ local LAST_ERR=/tmp/tmp.m3QhIhVRSg
+ local exit_status=0
+ local timeout=4
++ mktemp
++ seq 0 2
+ local LAST_ERR=/tmp/tmp.NxDyFlgTnW
+ local exit_status=0
+ local timeout=4
+ for i in $(seq 0 2)
+ set +e
+ kubectl get ns
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl delete namespace demand-backup-3000 --ignore-not-found
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.IBIrqt7cSI
+ cat /tmp/tmp.m3QhIhVRSg
+ rm /tmp/tmp.IBIrqt7cSI /tmp/tmp.m3QhIhVRSg
+ return 0
error: resource(s) were provided, but no name was specified
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.SGJhszOXvG
+ cat /tmp/tmp.NxDyFlgTnW
+ rm /tmp/tmp.SGJhszOXvG /tmp/tmp.NxDyFlgTnW
+ return 0
+ kubectl_bin wait --for=delete namespace demand-backup-3000
++ mktemp
+ local LAST_OUT=/tmp/tmp.56YTFwlarN
++ mktemp
+ local LAST_ERR=/tmp/tmp.J7WRD2pMgL
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl wait --for=delete namespace demand-backup-3000
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.56YTFwlarN
+ cat /tmp/tmp.J7WRD2pMgL
+ rm /tmp/tmp.56YTFwlarN /tmp/tmp.J7WRD2pMgL
+ return 0
+ desc 'create namespace demand-backup-3000'
+ set +o xtrace


-----------------------------------------------------------------------------------
create namespace demand-backup-3000
-----------------------------------------------------------------------------------

+ kubectl_bin create namespace demand-backup-3000
++ mktemp
+ local LAST_OUT=/tmp/tmp.TtQ5u11qyB
++ mktemp
+ local LAST_ERR=/tmp/tmp.c52WUh1Npg
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl create namespace demand-backup-3000
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.TtQ5u11qyB
namespace/demand-backup-3000 created
+ cat /tmp/tmp.c52WUh1Npg
+ rm /tmp/tmp.TtQ5u11qyB /tmp/tmp.c52WUh1Npg
+ return 0
+ set_kube_ctx demand-backup-3000
+ local namespace=demand-backup-3000
++ kubectl_bin config current-context
+++ mktemp
++ local LAST_OUT=/tmp/tmp.eqzMlEUsDT
+++ mktemp
++ local LAST_ERR=/tmp/tmp.jUtOLInDTM
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl config current-context
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.eqzMlEUsDT
++ cat /tmp/tmp.jUtOLInDTM
++ rm /tmp/tmp.eqzMlEUsDT /tmp/tmp.jUtOLInDTM
++ return 0
+ kubectl_bin config set-context gke_cloud-dev-112233_us-central1-a_jen-psmdb-2101-0c183d95-1-cluster10 --namespace=demand-backup-3000
++ mktemp
+ local LAST_OUT=/tmp/tmp.UCm77GHv53
++ mktemp
+ local LAST_ERR=/tmp/tmp.Tl7ys9Jq1I
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl config set-context gke_cloud-dev-112233_us-central1-a_jen-psmdb-2101-0c183d95-1-cluster10 --namespace=demand-backup-3000
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.UCm77GHv53
Context "gke_cloud-dev-112233_us-central1-a_jen-psmdb-2101-0c183d95-1-cluster10" modified.
+ cat /tmp/tmp.Tl7ys9Jq1I
+ rm /tmp/tmp.UCm77GHv53 /tmp/tmp.Tl7ys9Jq1I
+ return 0
+ deploy_minio
+ desc 'install Minio'
+ set +o xtrace


-----------------------------------------------------------------------------------
install Minio
-----------------------------------------------------------------------------------

+ helm uninstall minio-service
Error: uninstall: Release not loaded: minio-service: release: not found
+ :
+ helm repo remove minio
"minio" has been removed from your repositories
+ helm repo add minio https://charts.min.io/
"minio" has been added to your repositories
+ retry 10 60 helm install minio-service --version 5.4.0 --set replicas=1 --set mode=standalone --set resources.requests.memory=256Mi --set rootUser=rootuser --set rootPassword=rootpass123 --set 'users[0].accessKey=some-access-key' --set 'users[0].secretKey=some-secret-key' --set 'users[0].policy=consoleAdmin' --set service.type=ClusterIP --set configPathmc=/tmp/.minio/ --set persistence.size=2G --set securityContext.enabled=false minio/minio
+ local max=10
+ local delay=60
+ shift 2
+ local n=1
+ helm install minio-service --version 5.4.0 --set replicas=1 --set mode=standalone --set resources.requests.memory=256Mi --set rootUser=rootuser --set rootPassword=rootpass123 --set 'users[0].accessKey=some-access-key' --set 'users[0].secretKey=some-secret-key' --set 'users[0].policy=consoleAdmin' --set service.type=ClusterIP --set configPathmc=/tmp/.minio/ --set persistence.size=2G --set securityContext.enabled=false minio/minio
NAME: minio-service
LAST DEPLOYED: Mon Nov  3 02:10:50 2025
NAMESPACE: demand-backup-3000
STATUS: deployed
REVISION: 1
TEST SUITE: None
NOTES:
MinIO can be accessed via port 9000 on the following DNS name from within your cluster:
minio-service.demand-backup-3000.cluster.local

To access MinIO from localhost, run the below commands:

  1. export POD_NAME=$(kubectl get pods --namespace demand-backup-3000 -l "release=minio-service" -o jsonpath="{.items[0].metadata.name}")

  2. kubectl port-forward $POD_NAME 9000 --namespace demand-backup-3000

Read more about port forwarding here: http://kubernetes.io/docs/user-guide/kubectl/kubectl_port-forward/

You can now access MinIO server on http://localhost:9000. Follow the below steps to connect to MinIO server with mc client:

  1. Download the MinIO mc client - https://min.io/docs/minio/linux/reference/minio-mc.html#quickstart

  2. export MC_HOST_minio-service-local=http://$(kubectl get secret --namespace demand-backup-3000 minio-service -o jsonpath="{.data.rootUser}" | base64 --decode):$(kubectl get secret --namespace demand-backup-3000 minio-service -o jsonpath="{.data.rootPassword}" | base64 --decode)@localhost:9000

  3. mc ls minio-service-local
++ kubectl_bin get pods --selector=release=minio-service -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.gAuDrbU2XP
+++ mktemp
++ local LAST_ERR=/tmp/tmp.EZmBV1LL2r
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=release=minio-service -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.gAuDrbU2XP
++ cat /tmp/tmp.EZmBV1LL2r
++ rm /tmp/tmp.gAuDrbU2XP /tmp/tmp.EZmBV1LL2r
++ return 0
+ MINIO_POD=minio-service-d9589b474-tqkrq
+ wait_pod minio-service-d9589b474-tqkrq
+ local pod=minio-service-d9589b474-tqkrq
+ set +o xtrace
waiting for pod/minio-service-d9589b474-tqkrq to be ready.OK
+ '[' -n psmdb-operator ']'
+ kubectl_bin create svc -n psmdb-operator externalname minio-service --external-name=minio-service.demand-backup-3000.svc.cluster.local --tcp=9000
++ mktemp
+ local LAST_OUT=/tmp/tmp.Q5CynCCiwM
++ mktemp
+ local LAST_ERR=/tmp/tmp.rR8hDaxmLR
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl create svc -n psmdb-operator externalname minio-service --external-name=minio-service.demand-backup-3000.svc.cluster.local --tcp=9000
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.Q5CynCCiwM
service/minio-service created
+ cat /tmp/tmp.rR8hDaxmLR
+ rm /tmp/tmp.Q5CynCCiwM /tmp/tmp.rR8hDaxmLR
+ return 0
+ create_minio_bucket operator-testing
+ local bucket=operator-testing
+ kubectl_bin run -i --rm aws-cli --image=perconalab/awscli --restart=Never -- bash -c 'AWS_ACCESS_KEY_ID=some-access-key 		AWS_SECRET_ACCESS_KEY=some-secret-key 		AWS_DEFAULT_REGION=us-east-1         /usr/bin/aws --endpoint-url http://minio-service:9000 s3 mb s3://operator-testing'
++ mktemp
+ local LAST_OUT=/tmp/tmp.TwyxRo6u07
++ mktemp
+ local LAST_ERR=/tmp/tmp.dWduzzmecK
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl run -i --rm aws-cli --image=perconalab/awscli --restart=Never -- bash -c 'AWS_ACCESS_KEY_ID=some-access-key 		AWS_SECRET_ACCESS_KEY=some-secret-key 		AWS_DEFAULT_REGION=us-east-1         /usr/bin/aws --endpoint-url http://minio-service:9000 s3 mb s3://operator-testing'
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.TwyxRo6u07
make_bucket: operator-testing
pod "aws-cli" deleted from demand-backup-3000 namespace
+ cat /tmp/tmp.dWduzzmecK
All commands and output from this session will be recorded in container logs, including credentials and sensitive information passed through the command prompt.
If you don't see a command prompt, try pressing enter.
warning: couldn't attach to pod/aws-cli, falling back to streaming logs: Internal error occurred: unable to upgrade connection: container aws-cli not found in pod aws-cli_demand-backup-3000
+ rm /tmp/tmp.TwyxRo6u07 /tmp/tmp.dWduzzmecK
+ return 0
+ desc 'create secrets and start client'
+ set +o xtrace


-----------------------------------------------------------------------------------
create secrets and start client
-----------------------------------------------------------------------------------

+ cluster=some-name-rs0
+ kubectl_bin apply -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2101/e2e-tests/demand-backup/conf/secrets.yml -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2101/e2e-tests/conf/client.yml
++ mktemp
+ local LAST_OUT=/tmp/tmp.A1H4azbE6F
++ mktemp
+ local LAST_ERR=/tmp/tmp.iVdZKKTjpQ
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl apply -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2101/e2e-tests/demand-backup/conf/secrets.yml -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2101/e2e-tests/conf/client.yml
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.A1H4azbE6F
secret/some-users created
deployment.apps/psmdb-client created
+ cat /tmp/tmp.iVdZKKTjpQ
+ rm /tmp/tmp.A1H4azbE6F /tmp/tmp.iVdZKKTjpQ
+ return 0
+ apply_s3_storage_secrets
+ desc 'create secrets for cloud storages'
+ set +o xtrace


-----------------------------------------------------------------------------------
create secrets for cloud storages
-----------------------------------------------------------------------------------

+ '[' -z '' ']'
+ kubectl_bin apply -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2101/e2e-tests/conf/minio-secret.yml -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2101/e2e-tests/conf/cloud-secret.yml
++ mktemp
+ local LAST_OUT=/tmp/tmp.Xjp1Tzi3yr
++ mktemp
+ local LAST_ERR=/tmp/tmp.fdLHeuaIES
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl apply -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2101/e2e-tests/conf/minio-secret.yml -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2101/e2e-tests/conf/cloud-secret.yml
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.Xjp1Tzi3yr
secret/minio-secret created
secret/aws-s3-secret created
secret/gcp-cs-secret created
secret/azure-secret created
secret/gcp-cs-sa-key-secret created
+ cat /tmp/tmp.fdLHeuaIES
+ rm /tmp/tmp.Xjp1Tzi3yr /tmp/tmp.fdLHeuaIES
+ return 0
+ desc 'create first PSMDB cluster some-name-rs0'
+ set +o xtrace


-----------------------------------------------------------------------------------
create first PSMDB cluster some-name-rs0
-----------------------------------------------------------------------------------

+ apply_cluster /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2101/e2e-tests/demand-backup/conf/some-name-rs0.yml
+ '[' -z '' ']'
+ cat_config /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2101/e2e-tests/demand-backup/conf/some-name-rs0.yml
+ kubectl_bin apply -f -
+ cat /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2101/e2e-tests/demand-backup/conf/some-name-rs0.yml
+ yq eval '(.spec | select(.image == null)).image = "perconalab/percona-server-mongodb-operator:main-mongod7.0"'
++ mktemp
+ yq eval '(.spec | select(has("pmm"))).pmm.image = "percona/pmm-client:2.44.1-1"'
+ yq eval '(.spec | select(has("initImage"))).initImage = "perconalab/percona-server-mongodb-operator:PR-2101-0c183d95"'
+ yq eval '(.spec | select(has("backup"))).backup.image = "perconalab/percona-server-mongodb-operator:main-backup"'
+ yq eval '.spec.upgradeOptions.apply="Never"'
+ local LAST_OUT=/tmp/tmp.9Rjdml2PAf
++ mktemp
+ local LAST_ERR=/tmp/tmp.Xp5qjNn9Lq
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl apply -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.9Rjdml2PAf
perconaservermongodb.psmdb.percona.com/some-name created
+ cat /tmp/tmp.Xp5qjNn9Lq
+ rm /tmp/tmp.9Rjdml2PAf /tmp/tmp.Xp5qjNn9Lq
+ return 0
+ desc 'check if all 3 Pods started'
+ set +o xtrace


-----------------------------------------------------------------------------------
check if all 3 Pods started
-----------------------------------------------------------------------------------

+ wait_for_running some-name-rs0 3
+ local name=some-name-rs0
+ let last_pod=2
+ local check_cluster_readyness=true
+ set_debug
+ [[ 1 == 1 ]]
+ set -o xtrace
+ local rs_name=rs0
+ local cluster_name=some-name
++ seq 0 2
+ for i in $(seq 0 $last_pod)
+ [[ 0 -eq 2 ]]
+ wait_pod some-name-rs0-0
+ local pod=some-name-rs0-0
+ set +o xtrace
waiting for pod/some-name-rs0-0 to be ready...............OK
+ for i in $(seq 0 $last_pod)
+ [[ 1 -eq 2 ]]
+ wait_pod some-name-rs0-1
+ local pod=some-name-rs0-1
+ set +o xtrace
waiting for pod/some-name-rs0-1 to be ready.......OK
+ for i in $(seq 0 $last_pod)
+ [[ 2 -eq 2 ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].arbiter.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.NhVmOgT6gh
+++ mktemp
++ local LAST_ERR=/tmp/tmp.dvJslpaGkv
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].arbiter.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.NhVmOgT6gh
++ cat /tmp/tmp.dvJslpaGkv
++ rm /tmp/tmp.NhVmOgT6gh /tmp/tmp.dvJslpaGkv
++ return 0
+ [[ '' == \t\r\u\e ]]
+ wait_pod some-name-rs0-2
+ local pod=some-name-rs0-2
+ set +o xtrace
waiting for pod/some-name-rs0-2 to be ready.......OK
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].nonvoting.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.wgoc6Q1sce
+++ mktemp
++ local LAST_ERR=/tmp/tmp.yW5nH1yz2c
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].nonvoting.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.wgoc6Q1sce
++ cat /tmp/tmp.yW5nH1yz2c
++ rm /tmp/tmp.wgoc6Q1sce /tmp/tmp.yW5nH1yz2c
++ return 0
+ [[ '' == \t\r\u\e ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].hidden.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.SaqQHdo5cr
+++ mktemp
++ local LAST_ERR=/tmp/tmp.9dVpDlr7t1
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].hidden.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.SaqQHdo5cr
++ cat /tmp/tmp.9dVpDlr7t1
++ rm /tmp/tmp.SaqQHdo5cr /tmp/tmp.9dVpDlr7t1
++ return 0
+ [[ '' == \t\r\u\e ]]
+ sleep 10
+ [[ true == \t\r\u\e ]]
+ set +x
Waiting for cluster readyness...
+ desc 'check if service and statefulset created with expected config'
+ set +o xtrace


-----------------------------------------------------------------------------------
check if service and statefulset created with expected config
-----------------------------------------------------------------------------------

+ compare_kubectl statefulset/some-name-rs0
+ local resource=statefulset/some-name-rs0
+ local postfix=
+ local skip_generation_check=
+ local expected_result=/mnt/jenkins/workspace/cloud-psmdb-operator_PR-2101/e2e-tests/demand-backup/compare/statefulset_some-name-rs0.yml
+ local new_result=/tmp/tmp.NAG1atx8KS/statefulset_some-name-rs0.yml
+ '[' -n '' -a -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2101/e2e-tests/demand-backup/compare/statefulset_some-name-rs0-oc.yml ']'
+ kubectl_bin get -o yaml statefulset/some-name-rs0
+ yq eval '
			del(.metadata.ownerReferences[].apiVersion) |
			del(.metadata.managedFields) |
			del(.. | select(has("creationTimestamp")).creationTimestamp) |
			del(.. | select(has("namespace")).namespace) |
			del(.. | select(has("uid")).uid) |
			del(.metadata.resourceVersion) |
			del(.spec.template.spec.containers[].env[] | select(.name == "NAMESPACE")) |
			del(.metadata.selfLink) |
			del(.metadata.annotations."cloud.google.com/neg") |
			del(.metadata.annotations."kubectl.kubernetes.io/last-applied-configuration") |
			del(.. | select(has("image")).image) |
			del(.. | select(has("clusterIP")).clusterIP) |
			del(.. | select(has("clusterIPs")).clusterIPs) |
			del(.. | select(has("dataSource")).dataSource) |
			del(.. | select(has("procMount")).procMount) |
			del(.. | select(has("storageClassName")).storageClassName) |
			del(.. | select(has("finalizers")).finalizers) |
			del(.. | select(has("kubernetes.io/pvc-protection"))."kubernetes.io/pvc-protection") |
			del(.. | select(has("volumeName")).volumeName) |
			del(.. | select(has("volume.beta.kubernetes.io/storage-provisioner"))."volume.beta.kubernetes.io/storage-provisioner") |
			del(.. | select(has("volume.kubernetes.io/storage-provisioner"))."volume.kubernetes.io/storage-provisioner") |
			del(.spec.volumeMode) |
			del(.. | select(has("volume.kubernetes.io/selected-node"))."volume.kubernetes.io/selected-node") |
			del(.. | select(has("percona.com/last-config-hash"))."percona.com/last-config-hash") |
			del(.. | select(has("percona.com/configuration-hash"))."percona.com/configuration-hash") |
			del(.. | select(has("percona.com/ssl-hash"))."percona.com/ssl-hash") |
			del(.. | select(has("percona.com/ssl-internal-hash"))."percona.com/ssl-internal-hash") |
			del(.spec.volumeClaimTemplates[].spec.volumeMode | select(. == "Filesystem")) |
			del(.. | select(has("healthCheckNodePort")).healthCheckNodePort) |
			del(.. | select(has("nodePort")).nodePort) |
			del(.status) |
			(.. | select(tag == "!!str")) |= sub("demand-backup-3000", "NAME_SPACE") |
			del(.spec.volumeClaimTemplates[].apiVersion) |
			del(.spec.volumeClaimTemplates[].kind) |
			del(.spec.ipFamilies) |
			del(.spec.ipFamilyPolicy) |
			(.. | select(. == "extensions/v1beta1")) = "apps/v1" |
			(.. | select(. == "batch/v1beta1")) = "batch/v1" ' -
++ mktemp
+ local LAST_OUT=/tmp/tmp.9ehbuLTVPN
++ mktemp
+ local LAST_ERR=/tmp/tmp.0rIU9fomQK
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl get -o yaml statefulset/some-name-rs0
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.9ehbuLTVPN
+ cat /tmp/tmp.0rIU9fomQK
+ rm /tmp/tmp.9ehbuLTVPN /tmp/tmp.0rIU9fomQK
+ return 0
+ yq -i eval 'del(.spec.persistentVolumeClaimRetentionPolicy)' /tmp/tmp.NAG1atx8KS/statefulset_some-name-rs0.yml
+ version_gt 1.22
++ echo '1.31 >= 1.22'
++ bc -l
+ '[' 1 -eq 1 ']'
+ return 0
+ yq -i eval 'del(.spec.internalTrafficPolicy)' /tmp/tmp.NAG1atx8KS/statefulset_some-name-rs0.yml
+ yq -i eval 'del(.spec.allocateLoadBalancerNodePorts)' /tmp/tmp.NAG1atx8KS/statefulset_some-name-rs0.yml
+ [[ /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2101/e2e-tests/demand-backup/compare/statefulset_some-name-rs0.yml == */cronjob* ]]
+ '[' -n '' ']'
+ [[ 0 -eq 0 ]]
+ diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2101/e2e-tests/demand-backup/compare/statefulset_some-name-rs0.yml /tmp/tmp.NAG1atx8KS/statefulset_some-name-rs0.yml
+ log 'compare_kubectl: statefulset/some-name-rs0 OK'
+ set +o xtrace
[2025-11-03T02:13:22+0000] compare_kubectl: statefulset/some-name-rs0 OK
+ desc 'create user'
+ set +o xtrace


-----------------------------------------------------------------------------------
create user
-----------------------------------------------------------------------------------

+ run_mongo 'db.createUser({user:"myApp",pwd:"myPass",roles:[{db:"myApp",role:"readWrite"}]})' userAdmin:userAdmin123456@some-name-rs0.demand-backup-3000
+ local 'command=db.createUser({user:"myApp",pwd:"myPass",roles:[{db:"myApp",role:"readWrite"}]})'
+ local uri=userAdmin:userAdmin123456@some-name-rs0.demand-backup-3000
+ local driver=mongodb+srv
+ local suffix=.svc.cluster.local
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.IsGNPGjXni
+++ mktemp
++ local LAST_ERR=/tmp/tmp.ENkBsIYq2v
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.IsGNPGjXni
++ cat /tmp/tmp.ENkBsIYq2v
++ rm /tmp/tmp.IsGNPGjXni /tmp/tmp.ENkBsIYq2v
++ return 0
+ local client_container=psmdb-client-696897d69b-7ct8v
+ local mongo_flag=
+ [[ userAdmin:userAdmin123456@some-name-rs0.demand-backup-3000 == *cfg* ]]
+ replica_set=rs0
+ kubectl_bin exec psmdb-client-696897d69b-7ct8v -- bash -c 'printf '\''db.createUser({user:"myApp",pwd:"myPass",roles:[{db:"myApp",role:"readWrite"}]})\n'\'' | mongo mongodb+srv://userAdmin:userAdmin123456@some-name-rs0.demand-backup-3000.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
++ mktemp
+ local LAST_OUT=/tmp/tmp.AWy4llO9BN
++ mktemp
+ local LAST_ERR=/tmp/tmp.oDItszDEhD
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-696897d69b-7ct8v -- bash -c 'printf '\''db.createUser({user:"myApp",pwd:"myPass",roles:[{db:"myApp",role:"readWrite"}]})\n'\'' | mongo mongodb+srv://userAdmin:userAdmin123456@some-name-rs0.demand-backup-3000.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.AWy4llO9BN
Percona Server for MongoDB shell version v4.4.29-28
connecting to: mongodb://some-name-rs0-2.some-name-rs0.demand-backup-3000.svc.cluster.local:27017,some-name-rs0-0.some-name-rs0.demand-backup-3000.svc.cluster.local:27017,some-name-rs0-1.some-name-rs0.demand-backup-3000.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb&replicaSet=rs0&ssl=false
Implicit session: session { "id" : UUID("2f8bd998-44fe-459e-9393-c0d4e2224de8") }
Percona Server for MongoDB server version: v7.0.24-13
WARNING: shell and server versions do not match
Successfully added user: {
	"user" : "myApp",
	"roles" : [
		{
			"db" : "myApp",
			"role" : "readWrite"
		}
	]
}
bye
+ cat /tmp/tmp.oDItszDEhD
+ rm /tmp/tmp.AWy4llO9BN /tmp/tmp.oDItszDEhD
+ return 0
+ sleep 2
+ desc 'write data, read from all'
+ set +o xtrace


-----------------------------------------------------------------------------------
write data, read from all
-----------------------------------------------------------------------------------

+ run_mongo 'use myApp\n db.test.insert({ x: 100500 })' myApp:myPass@some-name-rs0.demand-backup-3000
+ local 'command=use myApp\n db.test.insert({ x: 100500 })'
+ local uri=myApp:myPass@some-name-rs0.demand-backup-3000
+ local driver=mongodb+srv
+ local suffix=.svc.cluster.local
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.C29Oa26aAR
+++ mktemp
++ local LAST_ERR=/tmp/tmp.gTaYDFUuLl
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.C29Oa26aAR
++ cat /tmp/tmp.gTaYDFUuLl
++ rm /tmp/tmp.C29Oa26aAR /tmp/tmp.gTaYDFUuLl
++ return 0
+ local client_container=psmdb-client-696897d69b-7ct8v
+ local mongo_flag=
+ [[ myApp:myPass@some-name-rs0.demand-backup-3000 == *cfg* ]]
+ replica_set=rs0
+ kubectl_bin exec psmdb-client-696897d69b-7ct8v -- bash -c 'printf '\''use myApp\n db.test.insert({ x: 100500 })\n'\'' | mongo mongodb+srv://myApp:myPass@some-name-rs0.demand-backup-3000.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
++ mktemp
+ local LAST_OUT=/tmp/tmp.t80BB8a5Cl
++ mktemp
+ local LAST_ERR=/tmp/tmp.dvq8Qokwni
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-696897d69b-7ct8v -- bash -c 'printf '\''use myApp\n db.test.insert({ x: 100500 })\n'\'' | mongo mongodb+srv://myApp:myPass@some-name-rs0.demand-backup-3000.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.t80BB8a5Cl
Percona Server for MongoDB shell version v4.4.29-28
connecting to: mongodb://some-name-rs0-2.some-name-rs0.demand-backup-3000.svc.cluster.local:27017,some-name-rs0-0.some-name-rs0.demand-backup-3000.svc.cluster.local:27017,some-name-rs0-1.some-name-rs0.demand-backup-3000.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb&replicaSet=rs0&ssl=false
Implicit session: session { "id" : UUID("e31af98c-cba8-4057-9fa4-a9cd7188d136") }
Percona Server for MongoDB server version: v7.0.24-13
WARNING: shell and server versions do not match
switched to db myApp
WriteResult({ "nInserted" : 1 })
bye
+ cat /tmp/tmp.dvq8Qokwni
+ rm /tmp/tmp.t80BB8a5Cl /tmp/tmp.dvq8Qokwni
+ return 0
+ run_mongo 'use myApp\n db.test2.insert({ x: 100501 })' myApp:myPass@some-name-rs0.demand-backup-3000
+ local 'command=use myApp\n db.test2.insert({ x: 100501 })'
+ local uri=myApp:myPass@some-name-rs0.demand-backup-3000
+ local driver=mongodb+srv
+ local suffix=.svc.cluster.local
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.OkOHgWAfqL
+++ mktemp
++ local LAST_ERR=/tmp/tmp.r8YORNNB0H
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.OkOHgWAfqL
++ cat /tmp/tmp.r8YORNNB0H
++ rm /tmp/tmp.OkOHgWAfqL /tmp/tmp.r8YORNNB0H
++ return 0
+ local client_container=psmdb-client-696897d69b-7ct8v
+ local mongo_flag=
+ [[ myApp:myPass@some-name-rs0.demand-backup-3000 == *cfg* ]]
+ replica_set=rs0
+ kubectl_bin exec psmdb-client-696897d69b-7ct8v -- bash -c 'printf '\''use myApp\n db.test2.insert({ x: 100501 })\n'\'' | mongo mongodb+srv://myApp:myPass@some-name-rs0.demand-backup-3000.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
++ mktemp
+ local LAST_OUT=/tmp/tmp.LPcLTEMtNG
++ mktemp
+ local LAST_ERR=/tmp/tmp.RqG6Hxb8Dr
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-696897d69b-7ct8v -- bash -c 'printf '\''use myApp\n db.test2.insert({ x: 100501 })\n'\'' | mongo mongodb+srv://myApp:myPass@some-name-rs0.demand-backup-3000.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.LPcLTEMtNG
Percona Server for MongoDB shell version v4.4.29-28
connecting to: mongodb://some-name-rs0-1.some-name-rs0.demand-backup-3000.svc.cluster.local:27017,some-name-rs0-2.some-name-rs0.demand-backup-3000.svc.cluster.local:27017,some-name-rs0-0.some-name-rs0.demand-backup-3000.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb&replicaSet=rs0&ssl=false
Implicit session: session { "id" : UUID("29f50279-2e17-4644-9bff-ab02b22225b5") }
Percona Server for MongoDB server version: v7.0.24-13
WARNING: shell and server versions do not match
switched to db myApp
WriteResult({ "nInserted" : 1 })
bye
+ cat /tmp/tmp.RqG6Hxb8Dr
+ rm /tmp/tmp.LPcLTEMtNG /tmp/tmp.RqG6Hxb8Dr
+ return 0
+ run_mongo 'use myApp\n db.test3.insert({ x: 100502 })' myApp:myPass@some-name-rs0.demand-backup-3000
+ local 'command=use myApp\n db.test3.insert({ x: 100502 })'
+ local uri=myApp:myPass@some-name-rs0.demand-backup-3000
+ local driver=mongodb+srv
+ local suffix=.svc.cluster.local
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.MA1IwyVKSx
+++ mktemp
++ local LAST_ERR=/tmp/tmp.HySfNugcMS
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.MA1IwyVKSx
++ cat /tmp/tmp.HySfNugcMS
++ rm /tmp/tmp.MA1IwyVKSx /tmp/tmp.HySfNugcMS
++ return 0
+ local client_container=psmdb-client-696897d69b-7ct8v
+ local mongo_flag=
+ [[ myApp:myPass@some-name-rs0.demand-backup-3000 == *cfg* ]]
+ replica_set=rs0
+ kubectl_bin exec psmdb-client-696897d69b-7ct8v -- bash -c 'printf '\''use myApp\n db.test3.insert({ x: 100502 })\n'\'' | mongo mongodb+srv://myApp:myPass@some-name-rs0.demand-backup-3000.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
++ mktemp
+ local LAST_OUT=/tmp/tmp.olA55KLmxk
++ mktemp
+ local LAST_ERR=/tmp/tmp.kvEqrXpOU7
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-696897d69b-7ct8v -- bash -c 'printf '\''use myApp\n db.test3.insert({ x: 100502 })\n'\'' | mongo mongodb+srv://myApp:myPass@some-name-rs0.demand-backup-3000.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.olA55KLmxk
Percona Server for MongoDB shell version v4.4.29-28
connecting to: mongodb://some-name-rs0-0.some-name-rs0.demand-backup-3000.svc.cluster.local:27017,some-name-rs0-1.some-name-rs0.demand-backup-3000.svc.cluster.local:27017,some-name-rs0-2.some-name-rs0.demand-backup-3000.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb&replicaSet=rs0&ssl=false
Implicit session: session { "id" : UUID("5b5cabcd-d253-42c6-bf12-7bbc92e746fb") }
Percona Server for MongoDB server version: v7.0.24-13
WARNING: shell and server versions do not match
switched to db myApp
WriteResult({ "nInserted" : 1 })
bye
+ cat /tmp/tmp.kvEqrXpOU7
+ rm /tmp/tmp.olA55KLmxk /tmp/tmp.kvEqrXpOU7
+ return 0
+ custom_user_name=test1user
+ custom_role_name=test1role
+ run_mongo 'use myApp\n db.createUser({user: "test1user",pwd:"test1pass",roles:[]})' userAdmin:userAdmin123456@some-name-rs0.demand-backup-3000
+ local 'command=use myApp\n db.createUser({user: "test1user",pwd:"test1pass",roles:[]})'
+ local uri=userAdmin:userAdmin123456@some-name-rs0.demand-backup-3000
+ local driver=mongodb+srv
+ local suffix=.svc.cluster.local
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.0zcZA9pgSh
+++ mktemp
++ local LAST_ERR=/tmp/tmp.5gafP45Hqf
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.0zcZA9pgSh
++ cat /tmp/tmp.5gafP45Hqf
++ rm /tmp/tmp.0zcZA9pgSh /tmp/tmp.5gafP45Hqf
++ return 0
+ local client_container=psmdb-client-696897d69b-7ct8v
+ local mongo_flag=
+ [[ userAdmin:userAdmin123456@some-name-rs0.demand-backup-3000 == *cfg* ]]
+ replica_set=rs0
+ kubectl_bin exec psmdb-client-696897d69b-7ct8v -- bash -c 'printf '\''use myApp\n db.createUser({user: "test1user",pwd:"test1pass",roles:[]})\n'\'' | mongo mongodb+srv://userAdmin:userAdmin123456@some-name-rs0.demand-backup-3000.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
++ mktemp
+ local LAST_OUT=/tmp/tmp.W4iDb56taR
++ mktemp
+ local LAST_ERR=/tmp/tmp.WYmELI8mRe
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-696897d69b-7ct8v -- bash -c 'printf '\''use myApp\n db.createUser({user: "test1user",pwd:"test1pass",roles:[]})\n'\'' | mongo mongodb+srv://userAdmin:userAdmin123456@some-name-rs0.demand-backup-3000.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.W4iDb56taR
Percona Server for MongoDB shell version v4.4.29-28
connecting to: mongodb://some-name-rs0-1.some-name-rs0.demand-backup-3000.svc.cluster.local:27017,some-name-rs0-2.some-name-rs0.demand-backup-3000.svc.cluster.local:27017,some-name-rs0-0.some-name-rs0.demand-backup-3000.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb&replicaSet=rs0&ssl=false
Implicit session: session { "id" : UUID("9d65b12c-6764-4d67-b821-910ac505a64a") }
Percona Server for MongoDB server version: v7.0.24-13
WARNING: shell and server versions do not match
switched to db myApp
Successfully added user: { "user" : "test1user", "roles" : [ ] }
bye
+ cat /tmp/tmp.WYmELI8mRe
+ rm /tmp/tmp.W4iDb56taR /tmp/tmp.WYmELI8mRe
+ return 0
+ run_mongo 'use myApp\n db.createRole({"role":"test1role", privileges:[],roles:[]})' userAdmin:userAdmin123456@some-name-rs0.demand-backup-3000
+ local 'command=use myApp\n db.createRole({"role":"test1role", privileges:[],roles:[]})'
+ local uri=userAdmin:userAdmin123456@some-name-rs0.demand-backup-3000
+ local driver=mongodb+srv
+ local suffix=.svc.cluster.local
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.fIuXM8Q1Qa
+++ mktemp
++ local LAST_ERR=/tmp/tmp.aFsT8XCbJK
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.fIuXM8Q1Qa
++ cat /tmp/tmp.aFsT8XCbJK
++ rm /tmp/tmp.fIuXM8Q1Qa /tmp/tmp.aFsT8XCbJK
++ return 0
+ local client_container=psmdb-client-696897d69b-7ct8v
+ local mongo_flag=
+ [[ userAdmin:userAdmin123456@some-name-rs0.demand-backup-3000 == *cfg* ]]
+ replica_set=rs0
+ kubectl_bin exec psmdb-client-696897d69b-7ct8v -- bash -c 'printf '\''use myApp\n db.createRole({"role":"test1role", privileges:[],roles:[]})\n'\'' | mongo mongodb+srv://userAdmin:userAdmin123456@some-name-rs0.demand-backup-3000.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
++ mktemp
+ local LAST_OUT=/tmp/tmp.jFPJOn6OH4
++ mktemp
+ local LAST_ERR=/tmp/tmp.vphjmChebb
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-696897d69b-7ct8v -- bash -c 'printf '\''use myApp\n db.createRole({"role":"test1role", privileges:[],roles:[]})\n'\'' | mongo mongodb+srv://userAdmin:userAdmin123456@some-name-rs0.demand-backup-3000.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.jFPJOn6OH4
Percona Server for MongoDB shell version v4.4.29-28
connecting to: mongodb://some-name-rs0-0.some-name-rs0.demand-backup-3000.svc.cluster.local:27017,some-name-rs0-1.some-name-rs0.demand-backup-3000.svc.cluster.local:27017,some-name-rs0-2.some-name-rs0.demand-backup-3000.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb&replicaSet=rs0&ssl=false
Implicit session: session { "id" : UUID("7cd3a60b-4532-4e03-8834-cdada2d5af44") }
Percona Server for MongoDB server version: v7.0.24-13
WARNING: shell and server versions do not match
switched to db myApp
{ "role" : "test1role", "privileges" : [ ], "roles" : [ ] }
bye
+ cat /tmp/tmp.vphjmChebb
+ rm /tmp/tmp.jFPJOn6OH4 /tmp/tmp.vphjmChebb
+ return 0
+ minikube_sleep
+ sleep_time=10
+ [[ '' == 1 ]]
+ compare_mongo_cmd find myApp:myPass@some-name-rs0-0.some-name-rs0.demand-backup-3000
+ local command=find
+ local uri=myApp:myPass@some-name-rs0-0.some-name-rs0.demand-backup-3000
+ local postfix=
+ local suffix=
+ local database=myApp
+ local collection=test
+ local sort=
+ local tls=false
+ local 'full_command=db.test.find()'
+ [[ -n '' ]]
+ log 'running db.test.find() in myApp'
+ set +o xtrace
[2025-11-03T02:13:37+0000] running db.test.find() in myApp
+ [[ false == \t\r\u\e ]]
+ mongo_command=run_mongo
+ run_mongo 'use myApp\n db.test.find()' myApp:myPass@some-name-rs0-0.some-name-rs0.demand-backup-3000 mongodb ''
+ local 'command=use myApp\n db.test.find()'
+ local uri=myApp:myPass@some-name-rs0-0.some-name-rs0.demand-backup-3000
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ egrep -v 'I NETWORK|W NETWORK|F NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|Started a new thread for the timer service'
+ /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
egrep: warning: egrep is obsolescent; using grep -E
++ local LAST_OUT=/tmp/tmp.V0AjUj2SlR
+++ mktemp
++ local LAST_ERR=/tmp/tmp.qz0eYqjUNs
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.V0AjUj2SlR
++ cat /tmp/tmp.qz0eYqjUNs
++ rm /tmp/tmp.V0AjUj2SlR /tmp/tmp.qz0eYqjUNs
++ return 0
+ local client_container=psmdb-client-696897d69b-7ct8v
+ local mongo_flag=
+ [[ myApp:myPass@some-name-rs0-0.some-name-rs0.demand-backup-3000 == *cfg* ]]
+ replica_set=rs0
+ kubectl_bin exec psmdb-client-696897d69b-7ct8v -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-0.some-name-rs0.demand-backup-3000.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
++ mktemp
+ local LAST_OUT=/tmp/tmp.QaKc0YxdWn
++ mktemp
+ local LAST_ERR=/tmp/tmp.3g5yvD6fi1
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-696897d69b-7ct8v -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-0.some-name-rs0.demand-backup-3000.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.QaKc0YxdWn
+ cat /tmp/tmp.3g5yvD6fi1
+ rm /tmp/tmp.QaKc0YxdWn /tmp/tmp.3g5yvD6fi1
+ return 0
+ [[ 0 -eq 0 ]]
+ diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2101/e2e-tests/demand-backup/compare/find.json /tmp/tmp.NAG1atx8KS/find
+ compare_mongo_cmd find myApp:myPass@some-name-rs0-1.some-name-rs0.demand-backup-3000
+ local command=find
+ local uri=myApp:myPass@some-name-rs0-1.some-name-rs0.demand-backup-3000
+ local postfix=
+ local suffix=
+ local database=myApp
+ local collection=test
+ local sort=
+ local tls=false
+ local 'full_command=db.test.find()'
+ [[ -n '' ]]
+ log 'running db.test.find() in myApp'
+ set +o xtrace
[2025-11-03T02:13:39+0000] running db.test.find() in myApp
+ [[ false == \t\r\u\e ]]
+ mongo_command=run_mongo
+ run_mongo 'use myApp\n db.test.find()' myApp:myPass@some-name-rs0-1.some-name-rs0.demand-backup-3000 mongodb ''
+ local 'command=use myApp\n db.test.find()'
+ local uri=myApp:myPass@some-name-rs0-1.some-name-rs0.demand-backup-3000
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ egrep -v 'I NETWORK|W NETWORK|F NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|Started a new thread for the timer service'
+ /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
egrep: warning: egrep is obsolescent; using grep -E
++ local LAST_OUT=/tmp/tmp.0DHJwWtzLU
+++ mktemp
++ local LAST_ERR=/tmp/tmp.7ttlgtQRqP
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.0DHJwWtzLU
++ cat /tmp/tmp.7ttlgtQRqP
++ rm /tmp/tmp.0DHJwWtzLU /tmp/tmp.7ttlgtQRqP
++ return 0
+ local client_container=psmdb-client-696897d69b-7ct8v
+ local mongo_flag=
+ [[ myApp:myPass@some-name-rs0-1.some-name-rs0.demand-backup-3000 == *cfg* ]]
+ replica_set=rs0
+ kubectl_bin exec psmdb-client-696897d69b-7ct8v -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-1.some-name-rs0.demand-backup-3000.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
++ mktemp
+ local LAST_OUT=/tmp/tmp.Ax4TMkKIST
++ mktemp
+ local LAST_ERR=/tmp/tmp.DHCCD0C6fp
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-696897d69b-7ct8v -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-1.some-name-rs0.demand-backup-3000.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.Ax4TMkKIST
+ cat /tmp/tmp.DHCCD0C6fp
+ rm /tmp/tmp.Ax4TMkKIST /tmp/tmp.DHCCD0C6fp
+ return 0
+ [[ 0 -eq 0 ]]
+ diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2101/e2e-tests/demand-backup/compare/find.json /tmp/tmp.NAG1atx8KS/find
+ compare_mongo_cmd find myApp:myPass@some-name-rs0-2.some-name-rs0.demand-backup-3000
+ local command=find
+ local uri=myApp:myPass@some-name-rs0-2.some-name-rs0.demand-backup-3000
+ local postfix=
+ local suffix=
+ local database=myApp
+ local collection=test
+ local sort=
+ local tls=false
+ local 'full_command=db.test.find()'
+ [[ -n '' ]]
+ log 'running db.test.find() in myApp'
+ set +o xtrace
[2025-11-03T02:13:42+0000] running db.test.find() in myApp
+ [[ false == \t\r\u\e ]]
+ mongo_command=run_mongo
+ egrep -v 'I NETWORK|W NETWORK|F NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|Started a new thread for the timer service'
+ run_mongo 'use myApp\n db.test.find()' myApp:myPass@some-name-rs0-2.some-name-rs0.demand-backup-3000 mongodb ''
+ /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
+ local 'command=use myApp\n db.test.find()'
+ local uri=myApp:myPass@some-name-rs0-2.some-name-rs0.demand-backup-3000
+ local driver=mongodb
+ local suffix=.svc.cluster.local
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
egrep: warning: egrep is obsolescent; using grep -E
+++ mktemp
++ local LAST_OUT=/tmp/tmp.q1eAg9pDrc
+++ mktemp
++ local LAST_ERR=/tmp/tmp.a2DNB6axo0
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.q1eAg9pDrc
++ cat /tmp/tmp.a2DNB6axo0
++ rm /tmp/tmp.q1eAg9pDrc /tmp/tmp.a2DNB6axo0
++ return 0
+ local client_container=psmdb-client-696897d69b-7ct8v
+ local mongo_flag=
+ [[ myApp:myPass@some-name-rs0-2.some-name-rs0.demand-backup-3000 == *cfg* ]]
+ replica_set=rs0
+ kubectl_bin exec psmdb-client-696897d69b-7ct8v -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-2.some-name-rs0.demand-backup-3000.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
++ mktemp
+ local LAST_OUT=/tmp/tmp.d4Utxl2GTY
++ mktemp
+ local LAST_ERR=/tmp/tmp.cyvw5cNfvV
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-696897d69b-7ct8v -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-2.some-name-rs0.demand-backup-3000.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.d4Utxl2GTY
+ cat /tmp/tmp.cyvw5cNfvV
+ rm /tmp/tmp.d4Utxl2GTY /tmp/tmp.cyvw5cNfvV
+ return 0
+ [[ 0 -eq 0 ]]
+ diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2101/e2e-tests/demand-backup/compare/find.json /tmp/tmp.NAG1atx8KS/find
+ wait_backup_agent some-name-rs0-0
+ local agent_pod=some-name-rs0-0
+ set +o xtrace
waiting for pbm-agent to be ready in some-name-rs0-0...2025-11-03T02:13:13.000+0000 I listening for the commands

+ wait_backup_agent some-name-rs0-1
+ local agent_pod=some-name-rs0-1
+ set +o xtrace
waiting for pbm-agent to be ready in some-name-rs0-1...2025-11-03T02:13:18.000+0000 I listening for the commands

+ wait_backup_agent some-name-rs0-2
+ local agent_pod=some-name-rs0-2
+ set +o xtrace
waiting for pbm-agent to be ready in some-name-rs0-2...2025-11-03T02:13:20.000+0000 I listening for the commands

+ '[' -z '' ']'
+ desc 'Check GCS pfofiles'
+ set +o xtrace


-----------------------------------------------------------------------------------
Check GCS pfofiles
-----------------------------------------------------------------------------------

+ compare_pbm_profile_setup some-name gcp-cs-s3
+ local cluster=some-name
+ local profile=gcp-cs-s3
+ local container=backup-agent
+ local pbm_binary=pbm
+ [[ backup-agent == \m\o\n\g\o\d ]]
+ kubectl_bin exec some-name-rs0-0 -c backup-agent -- pbm profile show gcp-cs-s3
++ mktemp
+ local LAST_OUT=/tmp/tmp.tvBX7qdFJR
++ mktemp
+ local LAST_ERR=/tmp/tmp.Gu5h1GZDlD
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec some-name-rs0-0 -c backup-agent -- pbm profile show gcp-cs-s3
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.tvBX7qdFJR
+ cat /tmp/tmp.Gu5h1GZDlD
+ rm /tmp/tmp.tvBX7qdFJR /tmp/tmp.Gu5h1GZDlD
+ return 0
+ diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2101/e2e-tests/demand-backup/compare/pbm_profile_gcp-cs-s3.yml /tmp/tmp.NAG1atx8KS/pbm_profile_gcp-cs-s3.yml
+ compare_pbm_profile_setup some-name gcp-cs-sa
+ local cluster=some-name
+ local profile=gcp-cs-sa
+ local container=backup-agent
+ local pbm_binary=pbm
+ [[ backup-agent == \m\o\n\g\o\d ]]
+ kubectl_bin exec some-name-rs0-0 -c backup-agent -- pbm profile show gcp-cs-sa
++ mktemp
+ local LAST_OUT=/tmp/tmp.UtwBjrtD7W
++ mktemp
+ local LAST_ERR=/tmp/tmp.UjDks2BPJ9
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec some-name-rs0-0 -c backup-agent -- pbm profile show gcp-cs-sa
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.UtwBjrtD7W
+ cat /tmp/tmp.UjDks2BPJ9
+ rm /tmp/tmp.UtwBjrtD7W /tmp/tmp.UjDks2BPJ9
+ return 0
+ diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2101/e2e-tests/demand-backup/compare/pbm_profile_gcp-cs-sa.yml /tmp/tmp.NAG1atx8KS/pbm_profile_gcp-cs-sa.yml
+ backup_name_minio=backup-minio
+ '[' -z '' ']'
+ backup_name_aws=backup-aws-s3
+ backup_name_gcp_s3=backup-gcp-cs-s3
+ backup_name_gcp_sa=backup-gcp-cs-sa
+ backup_name_azure=backup-azure-blob
+ desc 'run backups'
+ set +o xtrace


-----------------------------------------------------------------------------------
run backups
-----------------------------------------------------------------------------------

+ '[' -z '' ']'
+ run_backup aws-s3
+ local storage=aws-s3
+ local backup_name=backup-aws-s3
+ local type=logical
+ desc 'run backup backup-aws-s3'
+ set +o xtrace


-----------------------------------------------------------------------------------
run backup backup-aws-s3
-----------------------------------------------------------------------------------

+ yq eval '.metadata.name = "backup-aws-s3"
			| .spec.storageName = "aws-s3"
			| .spec.type = "logical"' /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2101/e2e-tests/demand-backup/conf/backup-aws-s3.yml
+ kubectl_bin apply -f -
++ mktemp
+ local LAST_OUT=/tmp/tmp.Mjw31lJHCF
++ mktemp
+ local LAST_ERR=/tmp/tmp.XkapCsP19Y
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl apply -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.Mjw31lJHCF
perconaservermongodbbackup.psmdb.percona.com/backup-aws-s3 created
+ cat /tmp/tmp.XkapCsP19Y
+ rm /tmp/tmp.Mjw31lJHCF /tmp/tmp.XkapCsP19Y
+ return 0
+ run_backup gcp-cs-s3
+ local storage=gcp-cs-s3
+ local backup_name=backup-gcp-cs-s3
+ local type=logical
+ desc 'run backup backup-gcp-cs-s3'
+ set +o xtrace


-----------------------------------------------------------------------------------
run backup backup-gcp-cs-s3
-----------------------------------------------------------------------------------

+ yq eval '.metadata.name = "backup-gcp-cs-s3"
			| .spec.storageName = "gcp-cs-s3"
			| .spec.type = "logical"' /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2101/e2e-tests/demand-backup/conf/backup-gcp-cs-s3.yml
+ kubectl_bin apply -f -
++ mktemp
+ local LAST_OUT=/tmp/tmp.OgLbI6K3vH
++ mktemp
+ local LAST_ERR=/tmp/tmp.PYEZzLQLCc
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl apply -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.OgLbI6K3vH
perconaservermongodbbackup.psmdb.percona.com/backup-gcp-cs-s3 created
+ cat /tmp/tmp.PYEZzLQLCc
+ rm /tmp/tmp.OgLbI6K3vH /tmp/tmp.PYEZzLQLCc
+ return 0
+ run_backup gcp-cs-sa
+ local storage=gcp-cs-sa
+ local backup_name=backup-gcp-cs-sa
+ local type=logical
+ desc 'run backup backup-gcp-cs-sa'
+ set +o xtrace


-----------------------------------------------------------------------------------
run backup backup-gcp-cs-sa
-----------------------------------------------------------------------------------

+ yq eval '.metadata.name = "backup-gcp-cs-sa"
			| .spec.storageName = "gcp-cs-sa"
			| .spec.type = "logical"' /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2101/e2e-tests/demand-backup/conf/backup-gcp-cs-sa.yml
+ kubectl_bin apply -f -
++ mktemp
+ local LAST_OUT=/tmp/tmp.kLybFkoQOW
++ mktemp
+ local LAST_ERR=/tmp/tmp.h26VaN0TJY
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl apply -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.kLybFkoQOW
perconaservermongodbbackup.psmdb.percona.com/backup-gcp-cs-sa created
+ cat /tmp/tmp.h26VaN0TJY
+ rm /tmp/tmp.kLybFkoQOW /tmp/tmp.h26VaN0TJY
+ return 0
+ run_backup azure-blob
+ local storage=azure-blob
+ local backup_name=backup-azure-blob
+ local type=logical
+ desc 'run backup backup-azure-blob'
+ set +o xtrace


-----------------------------------------------------------------------------------
run backup backup-azure-blob
-----------------------------------------------------------------------------------

+ yq eval '.metadata.name = "backup-azure-blob"
			| .spec.storageName = "azure-blob"
			| .spec.type = "logical"' /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2101/e2e-tests/demand-backup/conf/backup-azure-blob.yml
+ kubectl_bin apply -f -
++ mktemp
+ local LAST_OUT=/tmp/tmp.JD25DOopyk
++ mktemp
+ local LAST_ERR=/tmp/tmp.8frOE0rj81
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl apply -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.JD25DOopyk
perconaservermongodbbackup.psmdb.percona.com/backup-azure-blob created
+ cat /tmp/tmp.8frOE0rj81
+ rm /tmp/tmp.JD25DOopyk /tmp/tmp.8frOE0rj81
+ return 0
+ run_backup minio
+ local storage=minio
+ local backup_name=backup-minio
+ local type=logical
+ desc 'run backup backup-minio'
+ set +o xtrace


-----------------------------------------------------------------------------------
run backup backup-minio
-----------------------------------------------------------------------------------

+ yq eval '.metadata.name = "backup-minio"
			| .spec.storageName = "minio"
			| .spec.type = "logical"' /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2101/e2e-tests/demand-backup/conf/backup-minio.yml
+ kubectl_bin apply -f -
++ mktemp
+ local LAST_OUT=/tmp/tmp.03qjMIiZck
++ mktemp
+ local LAST_ERR=/tmp/tmp.HRmuhwPDYa
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl apply -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.03qjMIiZck
perconaservermongodbbackup.psmdb.percona.com/backup-minio created
+ cat /tmp/tmp.HRmuhwPDYa
+ rm /tmp/tmp.03qjMIiZck /tmp/tmp.HRmuhwPDYa
+ return 0
+ '[' -z '' ']'
+ wait_backup backup-aws-s3
+ local backup_name=backup-aws-s3
+ local target_state=ready
+ set +o xtrace
waiting for backup-aws-s3 to reach ready state...OK
+ wait_backup backup-gcp-cs-s3
+ local backup_name=backup-gcp-cs-s3
+ local target_state=ready
+ set +o xtrace
waiting for backup-gcp-cs-s3 to reach ready state.............OK
+ wait_backup backup-gcp-cs-sa
+ local backup_name=backup-gcp-cs-sa
+ local target_state=ready
+ set +o xtrace
waiting for backup-gcp-cs-sa to reach ready state............OK
+ wait_backup backup-azure-blob
+ local backup_name=backup-azure-blob
+ local target_state=ready
+ set +o xtrace
waiting for backup-azure-blob to reach ready state..OK
+ wait_backup backup-minio
+ local backup_name=backup-minio
+ local target_state=ready
+ set +o xtrace
waiting for backup-minio to reach ready state..OK
+ sleep 5
+ '[' -z '' ']'
+ desc 'check backup and restore -- aws-s3'
+ set +o xtrace


-----------------------------------------------------------------------------------
check backup and restore -- aws-s3
-----------------------------------------------------------------------------------

++ get_backup_dest backup-aws-s3
++ local backup_name=backup-aws-s3
++ kubectl_bin get psmdb-backup backup-aws-s3 -o 'jsonpath={.status.destination}'
++ sed -e 's/.json$//'
++ sed 's|s3://||'
++ sed 's|azure://||'
++ sed 's|gs://||'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.3WtN4W9qov
+++ mktemp
++ local LAST_ERR=/tmp/tmp.0mKIp87UiH
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb-backup backup-aws-s3 -o 'jsonpath={.status.destination}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.3WtN4W9qov
++ cat /tmp/tmp.0mKIp87UiH
++ rm /tmp/tmp.3WtN4W9qov /tmp/tmp.0mKIp87UiH
++ return 0
+ backup_dest_aws=operator-testing/psmdb-demand-backup/2025-11-03T02:13:52Z
+ check_backup_in_storage backup-aws-s3 s3 rs0 myApp.test.gz
+ local backup=backup-aws-s3
+ local storage_type=s3
+ local replset=rs0
+ local file=myApp.test.gz
+ local endpoint
+ case ${storage_type} in
+ endpoint=s3.amazonaws.com
++ get_backup_dest backup-aws-s3
++ local backup_name=backup-aws-s3
++ /usr/sbin/sed 's|https://engk8soperators.blob.core.windows.net/||'
++ kubectl_bin get psmdb-backup backup-aws-s3 -o 'jsonpath={.status.destination}'
++ sed -e 's/.json$//'
++ sed 's|s3://||'
++ sed 's|azure://||'
++ sed 's|gs://||'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.IojbcA477i
+++ mktemp
++ local LAST_ERR=/tmp/tmp.4xPqgkXIEf
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb-backup backup-aws-s3 -o 'jsonpath={.status.destination}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.IojbcA477i
++ cat /tmp/tmp.4xPqgkXIEf
++ rm /tmp/tmp.IojbcA477i /tmp/tmp.4xPqgkXIEf
++ return 0
+ backup_dest=operator-testing/psmdb-demand-backup/2025-11-03T02:13:52Z
+ [[ s3 == \m\i\n\i\o ]]
+ local url=https://s3.amazonaws.com/operator-testing/psmdb-demand-backup/2025-11-03T02:13:52Z/rs0/myApp.test.gz
+ log 'checking if https://s3.amazonaws.com/operator-testing/psmdb-demand-backup/2025-11-03T02:13:52Z/rs0/myApp.test.gz exists'
+ set +o xtrace
[2025-11-03T02:15:00+0000] checking if https://s3.amazonaws.com/operator-testing/psmdb-demand-backup/2025-11-03T02:13:52Z/rs0/myApp.test.gz exists
+ curl --fail --head https://s3.amazonaws.com/operator-testing/psmdb-demand-backup/2025-11-03T02:13:52Z/rs0/myApp.test.gz
  % Total    % Received % Xferd  Average Speed   Time    Time     Time  Current
                                 Dload  Upload   Total   Spent    Left  Speed

  0     0    0     0    0     0      0      0 --:--:-- --:--:-- --:--:--     0
  0    55    0     0    0     0      0      0 --:--:-- --:--:-- --:--:--     0
HTTP/1.1 200 OK
x-amz-id-2: Ye+E+ndd41YcUqYJPauQa91AkhI3zUI0AFwpDgjNlD4veLsObGipOEJN+1Fap74IIfQ/QjZqTF0=
x-amz-request-id: 984ZR63YK7QD6SP3
Date: Mon, 03 Nov 2025 02:15:01 GMT
Last-Modified: Mon, 03 Nov 2025 02:13:58 GMT
x-amz-expiration: expiry-date="Wed, 05 Nov 2025 00:00:00 GMT", rule-id="1 Days Cleanup"
ETag: "86c5247e16bafc8b05827bbd193d5a45"
x-amz-server-side-encryption: AES256
Accept-Ranges: bytes
Content-Type: application/octet-stream
Content-Length: 55
Server: AmazonS3

+ run_recovery_check backup-aws-s3 some-name-rs0
+ local backup=backup-aws-s3
+ local cluster=some-name-rs0
+ run_mongo 'use myApp\n db.test.insert({ x: 100501 })' myApp:myPass@some-name-rs0.demand-backup-3000
+ local 'command=use myApp\n db.test.insert({ x: 100501 })'
+ local uri=myApp:myPass@some-name-rs0.demand-backup-3000
+ local driver=mongodb+srv
+ local suffix=.svc.cluster.local
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.HOGryGt5ba
+++ mktemp
++ local LAST_ERR=/tmp/tmp.vt7dwtitvW
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.HOGryGt5ba
++ cat /tmp/tmp.vt7dwtitvW
++ rm /tmp/tmp.HOGryGt5ba /tmp/tmp.vt7dwtitvW
++ return 0
+ local client_container=psmdb-client-696897d69b-7ct8v
+ local mongo_flag=
+ [[ myApp:myPass@some-name-rs0.demand-backup-3000 == *cfg* ]]
+ replica_set=rs0
+ kubectl_bin exec psmdb-client-696897d69b-7ct8v -- bash -c 'printf '\''use myApp\n db.test.insert({ x: 100501 })\n'\'' | mongo mongodb+srv://myApp:myPass@some-name-rs0.demand-backup-3000.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
++ mktemp
+ local LAST_OUT=/tmp/tmp.mPUKCsWkJG
++ mktemp
+ local LAST_ERR=/tmp/tmp.MgSwXI5ONU
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-696897d69b-7ct8v -- bash -c 'printf '\''use myApp\n db.test.insert({ x: 100501 })\n'\'' | mongo mongodb+srv://myApp:myPass@some-name-rs0.demand-backup-3000.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.mPUKCsWkJG
Percona Server for MongoDB shell version v4.4.29-28
connecting to: mongodb://some-name-rs0-0.some-name-rs0.demand-backup-3000.svc.cluster.local:27017,some-name-rs0-1.some-name-rs0.demand-backup-3000.svc.cluster.local:27017,some-name-rs0-2.some-name-rs0.demand-backup-3000.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb&replicaSet=rs0&ssl=false
Implicit session: session { "id" : UUID("8c6d0067-4415-4c65-b18a-3bd6f596587c") }
Percona Server for MongoDB server version: v7.0.24-13
WARNING: shell and server versions do not match
switched to db myApp
WriteResult({ "nInserted" : 1 })
bye
+ cat /tmp/tmp.MgSwXI5ONU
+ rm /tmp/tmp.mPUKCsWkJG /tmp/tmp.MgSwXI5ONU
+ return 0
+ compare_mongo_cmd find myApp:myPass@some-name-rs0.demand-backup-3000 -2nd .svc.cluster.local myApp test
+ local command=find
+ local uri=myApp:myPass@some-name-rs0.demand-backup-3000
+ local postfix=-2nd
+ local suffix=.svc.cluster.local
+ local database=myApp
+ local collection=test
+ local sort=
+ local tls=false
+ local 'full_command=db.test.find()'
+ [[ -n '' ]]
+ log 'running db.test.find() in myApp'
+ set +o xtrace
[2025-11-03T02:15:03+0000] running db.test.find() in myApp
+ [[ false == \t\r\u\e ]]
+ mongo_command=run_mongo
+ run_mongo 'use myApp\n db.test.find()' myApp:myPass@some-name-rs0.demand-backup-3000 mongodb .svc.cluster.local
+ local 'command=use myApp\n db.test.find()'
+ local uri=myApp:myPass@some-name-rs0.demand-backup-3000
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ egrep -v 'I NETWORK|W NETWORK|F NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|Started a new thread for the timer service'
+ /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
egrep: warning: egrep is obsolescent; using grep -E
++ local LAST_OUT=/tmp/tmp.6BySwYvLNu
+++ mktemp
++ local LAST_ERR=/tmp/tmp.RG9Zk8Q0kc
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.6BySwYvLNu
++ cat /tmp/tmp.RG9Zk8Q0kc
++ rm /tmp/tmp.6BySwYvLNu /tmp/tmp.RG9Zk8Q0kc
++ return 0
+ local client_container=psmdb-client-696897d69b-7ct8v
+ local mongo_flag=
+ [[ myApp:myPass@some-name-rs0.demand-backup-3000 == *cfg* ]]
+ replica_set=rs0
+ kubectl_bin exec psmdb-client-696897d69b-7ct8v -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0.demand-backup-3000.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
++ mktemp
+ local LAST_OUT=/tmp/tmp.RLH9Rknxlp
++ mktemp
+ local LAST_ERR=/tmp/tmp.gSZTMkliPz
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-696897d69b-7ct8v -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0.demand-backup-3000.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.RLH9Rknxlp
+ cat /tmp/tmp.gSZTMkliPz
+ rm /tmp/tmp.RLH9Rknxlp /tmp/tmp.gSZTMkliPz
+ return 0
+ [[ 0 -eq 0 ]]
+ diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2101/e2e-tests/demand-backup/compare/find-2nd.json /tmp/tmp.NAG1atx8KS/find-2nd
+ run_restore backup-aws-s3
+ local backup_name=backup-aws-s3
+ cat /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2101/e2e-tests/demand-backup/conf/restore.yml
+ /usr/sbin/sed -e 's/name:/name: restore-backup-aws-s3/'
+ /usr/sbin/sed -e 's/backupName:/backupName: backup-aws-s3/'
+ kubectl_bin apply -f -
++ mktemp
+ local LAST_OUT=/tmp/tmp.jIABmGNLpz
++ mktemp
+ local LAST_ERR=/tmp/tmp.P0jVMOuiV2
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl apply -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.jIABmGNLpz
perconaservermongodbrestore.psmdb.percona.com/restore-backup-aws-s3 created
+ cat /tmp/tmp.P0jVMOuiV2
+ rm /tmp/tmp.jIABmGNLpz /tmp/tmp.P0jVMOuiV2
+ return 0
+ wait_restore backup-aws-s3 some-name
+ local backup_name=backup-aws-s3
+ local cluster_name=some-name
+ local target_state=ready
+ local wait_cluster_consistency=1
+ local wait_time=1780
+ local ok_if_ready=0
+ set +o xtrace
Waiting for the psmdb-restore/restore-backup-aws-s3 object to be created.OK
Waiting psmdb-restore/restore-backup-aws-s3 to reach state "ready" .OK after 0 minutes
+ [[ 1 -eq 1 ]]
+ wait_cluster_consistency some-name
+ local cluster_name=some-name
+ local wait_time=32
+ retry=0
+ sleep 7
+ echo -n 'waiting for cluster readyness'
waiting for cluster readyness++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.1ImFpmA9l8
+++ mktemp
++ local LAST_ERR=/tmp/tmp.hCqyRZBpSo
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.1ImFpmA9l8
++ cat /tmp/tmp.hCqyRZBpSo
++ rm /tmp/tmp.1ImFpmA9l8 /tmp/tmp.hCqyRZBpSo
++ return 0
+ [[ ready == \r\e\a\d\y ]]
+ echo .OK
.OK
+ simple_data_check some-name-rs0 3 0
+ local cluster_name=some-name-rs0
+ let last_pod=3-1
+ local isSharded=0
+ local cluster_pfx=
+ '[' 0 -eq 1 ']'
++ seq 0 2
+ for i in $(seq 0 $last_pod)
+ compare_mongo_cmd find myApp:myPass@some-name-rs0-0.some-name-rs0.demand-backup-3000
+ local command=find
+ local uri=myApp:myPass@some-name-rs0-0.some-name-rs0.demand-backup-3000
+ local postfix=
+ local suffix=
+ local database=myApp
+ local collection=test
+ local sort=
+ local tls=false
+ local 'full_command=db.test.find()'
+ [[ -n '' ]]
+ log 'running db.test.find() in myApp'
+ set +o xtrace
[2025-11-03T02:15:32+0000] running db.test.find() in myApp
+ [[ false == \t\r\u\e ]]
+ mongo_command=run_mongo
+ run_mongo 'use myApp\n db.test.find()' myApp:myPass@some-name-rs0-0.some-name-rs0.demand-backup-3000 mongodb ''
+ local 'command=use myApp\n db.test.find()'
+ local uri=myApp:myPass@some-name-rs0-0.some-name-rs0.demand-backup-3000
+ egrep -v 'I NETWORK|W NETWORK|F NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|Started a new thread for the timer service'
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
egrep: warning: egrep is obsolescent; using grep -E
++ local LAST_OUT=/tmp/tmp.8fZxPnZLF4
+++ mktemp
++ local LAST_ERR=/tmp/tmp.0Oc9606vk2
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.8fZxPnZLF4
++ cat /tmp/tmp.0Oc9606vk2
++ rm /tmp/tmp.8fZxPnZLF4 /tmp/tmp.0Oc9606vk2
++ return 0
+ local client_container=psmdb-client-696897d69b-7ct8v
+ local mongo_flag=
+ [[ myApp:myPass@some-name-rs0-0.some-name-rs0.demand-backup-3000 == *cfg* ]]
+ replica_set=rs0
+ kubectl_bin exec psmdb-client-696897d69b-7ct8v -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-0.some-name-rs0.demand-backup-3000.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
++ mktemp
+ local LAST_OUT=/tmp/tmp.9tptCHf4sA
++ mktemp
+ local LAST_ERR=/tmp/tmp.7Noem0xL62
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-696897d69b-7ct8v -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-0.some-name-rs0.demand-backup-3000.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.9tptCHf4sA
+ cat /tmp/tmp.7Noem0xL62
+ rm /tmp/tmp.9tptCHf4sA /tmp/tmp.7Noem0xL62
+ return 0
+ [[ 0 -eq 0 ]]
+ diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2101/e2e-tests/demand-backup/compare/find.json /tmp/tmp.NAG1atx8KS/find
+ for i in $(seq 0 $last_pod)
+ compare_mongo_cmd find myApp:myPass@some-name-rs0-1.some-name-rs0.demand-backup-3000
+ local command=find
+ local uri=myApp:myPass@some-name-rs0-1.some-name-rs0.demand-backup-3000
+ local postfix=
+ local suffix=
+ local database=myApp
+ local collection=test
+ local sort=
+ local tls=false
+ local 'full_command=db.test.find()'
+ [[ -n '' ]]
+ log 'running db.test.find() in myApp'
+ set +o xtrace
[2025-11-03T02:15:34+0000] running db.test.find() in myApp
+ [[ false == \t\r\u\e ]]
+ mongo_command=run_mongo
+ run_mongo 'use myApp\n db.test.find()' myApp:myPass@some-name-rs0-1.some-name-rs0.demand-backup-3000 mongodb ''
+ local 'command=use myApp\n db.test.find()'
+ local uri=myApp:myPass@some-name-rs0-1.some-name-rs0.demand-backup-3000
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ egrep -v 'I NETWORK|W NETWORK|F NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|Started a new thread for the timer service'
+ /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
egrep: warning: egrep is obsolescent; using grep -E
++ local LAST_OUT=/tmp/tmp.7aV839nECk
+++ mktemp
++ local LAST_ERR=/tmp/tmp.hOAh9C9hoO
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.7aV839nECk
++ cat /tmp/tmp.hOAh9C9hoO
++ rm /tmp/tmp.7aV839nECk /tmp/tmp.hOAh9C9hoO
++ return 0
+ local client_container=psmdb-client-696897d69b-7ct8v
+ local mongo_flag=
+ [[ myApp:myPass@some-name-rs0-1.some-name-rs0.demand-backup-3000 == *cfg* ]]
+ replica_set=rs0
+ kubectl_bin exec psmdb-client-696897d69b-7ct8v -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-1.some-name-rs0.demand-backup-3000.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
++ mktemp
+ local LAST_OUT=/tmp/tmp.JHWSyuzfxN
++ mktemp
+ local LAST_ERR=/tmp/tmp.y7Vshwes6D
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-696897d69b-7ct8v -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-1.some-name-rs0.demand-backup-3000.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.JHWSyuzfxN
+ cat /tmp/tmp.y7Vshwes6D
+ rm /tmp/tmp.JHWSyuzfxN /tmp/tmp.y7Vshwes6D
+ return 0
+ [[ 0 -eq 0 ]]
+ diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2101/e2e-tests/demand-backup/compare/find.json /tmp/tmp.NAG1atx8KS/find
+ for i in $(seq 0 $last_pod)
+ compare_mongo_cmd find myApp:myPass@some-name-rs0-2.some-name-rs0.demand-backup-3000
+ local command=find
+ local uri=myApp:myPass@some-name-rs0-2.some-name-rs0.demand-backup-3000
+ local postfix=
+ local suffix=
+ local database=myApp
+ local collection=test
+ local sort=
+ local tls=false
+ local 'full_command=db.test.find()'
+ [[ -n '' ]]
+ log 'running db.test.find() in myApp'
+ set +o xtrace
[2025-11-03T02:15:36+0000] running db.test.find() in myApp
+ [[ false == \t\r\u\e ]]
+ mongo_command=run_mongo
+ run_mongo 'use myApp\n db.test.find()' myApp:myPass@some-name-rs0-2.some-name-rs0.demand-backup-3000 mongodb ''
+ local 'command=use myApp\n db.test.find()'
+ local uri=myApp:myPass@some-name-rs0-2.some-name-rs0.demand-backup-3000
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ egrep -v 'I NETWORK|W NETWORK|F NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|Started a new thread for the timer service'
+ /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
egrep: warning: egrep is obsolescent; using grep -E
++ local LAST_OUT=/tmp/tmp.X8qQ3J07Rh
+++ mktemp
++ local LAST_ERR=/tmp/tmp.Gpc2sFHwE4
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.X8qQ3J07Rh
++ cat /tmp/tmp.Gpc2sFHwE4
++ rm /tmp/tmp.X8qQ3J07Rh /tmp/tmp.Gpc2sFHwE4
++ return 0
+ local client_container=psmdb-client-696897d69b-7ct8v
+ local mongo_flag=
+ [[ myApp:myPass@some-name-rs0-2.some-name-rs0.demand-backup-3000 == *cfg* ]]
+ replica_set=rs0
+ kubectl_bin exec psmdb-client-696897d69b-7ct8v -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-2.some-name-rs0.demand-backup-3000.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
++ mktemp
+ local LAST_OUT=/tmp/tmp.A2jpS6kHE2
++ mktemp
+ local LAST_ERR=/tmp/tmp.4J6zsIS0Er
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-696897d69b-7ct8v -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-2.some-name-rs0.demand-backup-3000.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.A2jpS6kHE2
+ cat /tmp/tmp.4J6zsIS0Er
+ rm /tmp/tmp.A2jpS6kHE2 /tmp/tmp.4J6zsIS0Er
+ return 0
+ [[ 0 -eq 0 ]]
+ diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2101/e2e-tests/demand-backup/compare/find.json /tmp/tmp.NAG1atx8KS/find
+ desc 'check backup and restore -- gcp-cs-s3'
+ set +o xtrace


-----------------------------------------------------------------------------------
check backup and restore -- gcp-cs-s3
-----------------------------------------------------------------------------------

++ get_backup_dest backup-gcp-cs-s3
++ local backup_name=backup-gcp-cs-s3
++ kubectl_bin get psmdb-backup backup-gcp-cs-s3 -o 'jsonpath={.status.destination}'
++ sed -e 's/.json$//'
++ sed 's|s3://||'
++ sed 's|azure://||'
++ sed 's|gs://||'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.XtS84Da6tm
+++ mktemp
++ local LAST_ERR=/tmp/tmp.5mZSX3gPwr
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb-backup backup-gcp-cs-s3 -o 'jsonpath={.status.destination}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.XtS84Da6tm
++ cat /tmp/tmp.5mZSX3gPwr
++ rm /tmp/tmp.XtS84Da6tm /tmp/tmp.5mZSX3gPwr
++ return 0
+ backup_dest_gcp_s3=operator-testing/psmdb-demand-backup/2025-11-03T02:14:15Z
+ check_backup_in_storage backup-gcp-cs-s3 gcs rs0 myApp.test.gz
+ local backup=backup-gcp-cs-s3
+ local storage_type=gcs
+ local replset=rs0
+ local file=myApp.test.gz
+ local endpoint
+ case ${storage_type} in
+ endpoint=storage.googleapis.com
++ get_backup_dest backup-gcp-cs-s3
++ local backup_name=backup-gcp-cs-s3
++ /usr/sbin/sed 's|https://engk8soperators.blob.core.windows.net/||'
++ kubectl_bin get psmdb-backup backup-gcp-cs-s3 -o 'jsonpath={.status.destination}'
++ sed -e 's/.json$//'
++ sed 's|s3://||'
+++ mktemp
++ sed 's|azure://||'
++ sed 's|gs://||'
++ local LAST_OUT=/tmp/tmp.vYiBiWBtwJ
+++ mktemp
++ local LAST_ERR=/tmp/tmp.4gKak63wCQ
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb-backup backup-gcp-cs-s3 -o 'jsonpath={.status.destination}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.vYiBiWBtwJ
++ cat /tmp/tmp.4gKak63wCQ
++ rm /tmp/tmp.vYiBiWBtwJ /tmp/tmp.4gKak63wCQ
++ return 0
+ backup_dest=operator-testing/psmdb-demand-backup/2025-11-03T02:14:15Z
+ [[ gcs == \m\i\n\i\o ]]
+ local url=https://storage.googleapis.com/operator-testing/psmdb-demand-backup/2025-11-03T02:14:15Z/rs0/myApp.test.gz
+ log 'checking if https://storage.googleapis.com/operator-testing/psmdb-demand-backup/2025-11-03T02:14:15Z/rs0/myApp.test.gz exists'
+ set +o xtrace
[2025-11-03T02:15:40+0000] checking if https://storage.googleapis.com/operator-testing/psmdb-demand-backup/2025-11-03T02:14:15Z/rs0/myApp.test.gz exists
+ curl --fail --head https://storage.googleapis.com/operator-testing/psmdb-demand-backup/2025-11-03T02:14:15Z/rs0/myApp.test.gz
  % Total    % Received % Xferd  Average Speed   Time    Time     Time  Current
                                 Dload  Upload   Total   Spent    Left  Speed

  0     0    0     0    0     0      0      0 --:--:-- --:--:-- --:--:--     0
  0    55    0     0    0     0      0      0 --:--:-- --:--:-- --:--:--     0
HTTP/2 200 
content-type: application/octet-stream
x-guploader-uploadid: AOCedOEGEsMI5H0EnWlZm97IrYnd-eIzazKT3buI-6Gj9KPmOqNZp80Vv4E7N_Yvsk-0nITDfNRgn4k
expires: Mon, 03 Nov 2025 03:15:40 GMT
date: Mon, 03 Nov 2025 02:15:40 GMT
cache-control: public, max-age=3600
last-modified: Mon, 03 Nov 2025 02:14:20 GMT
etag: "86c5247e16bafc8b05827bbd193d5a45"
x-goog-generation: 1762136060076026
x-goog-metageneration: 1
x-goog-stored-content-encoding: identity
x-goog-stored-content-length: 55
x-goog-hash: crc32c=Wl0hzA==
x-goog-hash: md5=hsUkfha6/IsFgnu9GT1aRQ==
x-goog-expiration: Tue, 04 Nov 2025 02:14:20 GMT
x-goog-storage-class: REGIONAL
accept-ranges: bytes
content-length: 55
server: UploadServer
alt-svc: h3=":443"; ma=2592000,h3-29=":443"; ma=2592000

+ run_recovery_check backup-gcp-cs-s3 some-name-rs0
+ local backup=backup-gcp-cs-s3
+ local cluster=some-name-rs0
+ run_mongo 'use myApp\n db.test.insert({ x: 100501 })' myApp:myPass@some-name-rs0.demand-backup-3000
+ local 'command=use myApp\n db.test.insert({ x: 100501 })'
+ local uri=myApp:myPass@some-name-rs0.demand-backup-3000
+ local driver=mongodb+srv
+ local suffix=.svc.cluster.local
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.Z1obX3eYFz
+++ mktemp
++ local LAST_ERR=/tmp/tmp.dzCsV0Ip8R
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.Z1obX3eYFz
++ cat /tmp/tmp.dzCsV0Ip8R
++ rm /tmp/tmp.Z1obX3eYFz /tmp/tmp.dzCsV0Ip8R
++ return 0
+ local client_container=psmdb-client-696897d69b-7ct8v
+ local mongo_flag=
+ [[ myApp:myPass@some-name-rs0.demand-backup-3000 == *cfg* ]]
+ replica_set=rs0
+ kubectl_bin exec psmdb-client-696897d69b-7ct8v -- bash -c 'printf '\''use myApp\n db.test.insert({ x: 100501 })\n'\'' | mongo mongodb+srv://myApp:myPass@some-name-rs0.demand-backup-3000.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
++ mktemp
+ local LAST_OUT=/tmp/tmp.DECMIsOk1z
++ mktemp
+ local LAST_ERR=/tmp/tmp.pCYbJmfv4z
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-696897d69b-7ct8v -- bash -c 'printf '\''use myApp\n db.test.insert({ x: 100501 })\n'\'' | mongo mongodb+srv://myApp:myPass@some-name-rs0.demand-backup-3000.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.DECMIsOk1z
Percona Server for MongoDB shell version v4.4.29-28
connecting to: mongodb://some-name-rs0-0.some-name-rs0.demand-backup-3000.svc.cluster.local:27017,some-name-rs0-1.some-name-rs0.demand-backup-3000.svc.cluster.local:27017,some-name-rs0-2.some-name-rs0.demand-backup-3000.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb&replicaSet=rs0&ssl=false
Implicit session: session { "id" : UUID("473a3526-8210-44a3-8ea9-5fb1c2992b58") }
Percona Server for MongoDB server version: v7.0.24-13
WARNING: shell and server versions do not match
switched to db myApp
WriteResult({ "nInserted" : 1 })
bye
+ cat /tmp/tmp.pCYbJmfv4z
+ rm /tmp/tmp.DECMIsOk1z /tmp/tmp.pCYbJmfv4z
+ return 0
+ compare_mongo_cmd find myApp:myPass@some-name-rs0.demand-backup-3000 -2nd .svc.cluster.local myApp test
+ local command=find
+ local uri=myApp:myPass@some-name-rs0.demand-backup-3000
+ local postfix=-2nd
+ local suffix=.svc.cluster.local
+ local database=myApp
+ local collection=test
+ local sort=
+ local tls=false
+ local 'full_command=db.test.find()'
+ [[ -n '' ]]
+ log 'running db.test.find() in myApp'
+ set +o xtrace
[2025-11-03T02:15:43+0000] running db.test.find() in myApp
+ [[ false == \t\r\u\e ]]
+ mongo_command=run_mongo
+ run_mongo 'use myApp\n db.test.find()' myApp:myPass@some-name-rs0.demand-backup-3000 mongodb .svc.cluster.local
+ local 'command=use myApp\n db.test.find()'
+ local uri=myApp:myPass@some-name-rs0.demand-backup-3000
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ egrep -v 'I NETWORK|W NETWORK|F NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|Started a new thread for the timer service'
+ /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
egrep: warning: egrep is obsolescent; using grep -E
++ local LAST_OUT=/tmp/tmp.Nt0iYP0BRa
+++ mktemp
++ local LAST_ERR=/tmp/tmp.wgZZTt6mqd
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.Nt0iYP0BRa
++ cat /tmp/tmp.wgZZTt6mqd
++ rm /tmp/tmp.Nt0iYP0BRa /tmp/tmp.wgZZTt6mqd
++ return 0
+ local client_container=psmdb-client-696897d69b-7ct8v
+ local mongo_flag=
+ [[ myApp:myPass@some-name-rs0.demand-backup-3000 == *cfg* ]]
+ replica_set=rs0
+ kubectl_bin exec psmdb-client-696897d69b-7ct8v -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0.demand-backup-3000.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
++ mktemp
+ local LAST_OUT=/tmp/tmp.WSeR5mRULH
++ mktemp
+ local LAST_ERR=/tmp/tmp.g1SzsEyUpS
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-696897d69b-7ct8v -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0.demand-backup-3000.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.WSeR5mRULH
+ cat /tmp/tmp.g1SzsEyUpS
+ rm /tmp/tmp.WSeR5mRULH /tmp/tmp.g1SzsEyUpS
+ return 0
+ [[ 0 -eq 0 ]]
+ diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2101/e2e-tests/demand-backup/compare/find-2nd.json /tmp/tmp.NAG1atx8KS/find-2nd
+ run_restore backup-gcp-cs-s3
+ local backup_name=backup-gcp-cs-s3
+ cat /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2101/e2e-tests/demand-backup/conf/restore.yml
+ /usr/sbin/sed -e 's/name:/name: restore-backup-gcp-cs-s3/'
+ /usr/sbin/sed -e 's/backupName:/backupName: backup-gcp-cs-s3/'
+ kubectl_bin apply -f -
++ mktemp
+ local LAST_OUT=/tmp/tmp.99exQxRIhD
++ mktemp
+ local LAST_ERR=/tmp/tmp.KFP7Y1mD9N
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl apply -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.99exQxRIhD
perconaservermongodbrestore.psmdb.percona.com/restore-backup-gcp-cs-s3 created
+ cat /tmp/tmp.KFP7Y1mD9N
+ rm /tmp/tmp.99exQxRIhD /tmp/tmp.KFP7Y1mD9N
+ return 0
+ wait_restore backup-gcp-cs-s3 some-name
+ local backup_name=backup-gcp-cs-s3
+ local cluster_name=some-name
+ local target_state=ready
+ local wait_cluster_consistency=1
+ local wait_time=1780
+ local ok_if_ready=0
+ set +o xtrace
Waiting for the psmdb-restore/restore-backup-gcp-cs-s3 object to be created.OK
Waiting psmdb-restore/restore-backup-gcp-cs-s3 to reach state "ready" .OK after 0 minutes
+ [[ 1 -eq 1 ]]
+ wait_cluster_consistency some-name
+ local cluster_name=some-name
+ local wait_time=32
+ retry=0
+ sleep 7
+ echo -n 'waiting for cluster readyness'
waiting for cluster readyness++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.bf0OZZljqN
+++ mktemp
++ local LAST_ERR=/tmp/tmp.iPWkFLDMID
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.bf0OZZljqN
++ cat /tmp/tmp.iPWkFLDMID
++ rm /tmp/tmp.bf0OZZljqN /tmp/tmp.iPWkFLDMID
++ return 0
+ [[ ready == \r\e\a\d\y ]]
+ echo .OK
.OK
+ simple_data_check some-name-rs0 3 0
+ local cluster_name=some-name-rs0
+ let last_pod=3-1
+ local isSharded=0
+ local cluster_pfx=
+ '[' 0 -eq 1 ']'
++ seq 0 2
+ for i in $(seq 0 $last_pod)
+ compare_mongo_cmd find myApp:myPass@some-name-rs0-0.some-name-rs0.demand-backup-3000
+ local command=find
+ local uri=myApp:myPass@some-name-rs0-0.some-name-rs0.demand-backup-3000
+ local postfix=
+ local suffix=
+ local database=myApp
+ local collection=test
+ local sort=
+ local tls=false
+ local 'full_command=db.test.find()'
+ [[ -n '' ]]
+ log 'running db.test.find() in myApp'
+ set +o xtrace
[2025-11-03T02:16:13+0000] running db.test.find() in myApp
+ [[ false == \t\r\u\e ]]
+ mongo_command=run_mongo
+ run_mongo 'use myApp\n db.test.find()' myApp:myPass@some-name-rs0-0.some-name-rs0.demand-backup-3000 mongodb ''
+ local 'command=use myApp\n db.test.find()'
+ local uri=myApp:myPass@some-name-rs0-0.some-name-rs0.demand-backup-3000
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ egrep -v 'I NETWORK|W NETWORK|F NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|Started a new thread for the timer service'
+ /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
egrep: warning: egrep is obsolescent; using grep -E
++ local LAST_OUT=/tmp/tmp.ebfabbJOT3
+++ mktemp
++ local LAST_ERR=/tmp/tmp.cq9MwLAkXF
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.ebfabbJOT3
++ cat /tmp/tmp.cq9MwLAkXF
++ rm /tmp/tmp.ebfabbJOT3 /tmp/tmp.cq9MwLAkXF
++ return 0
+ local client_container=psmdb-client-696897d69b-7ct8v
+ local mongo_flag=
+ [[ myApp:myPass@some-name-rs0-0.some-name-rs0.demand-backup-3000 == *cfg* ]]
+ replica_set=rs0
+ kubectl_bin exec psmdb-client-696897d69b-7ct8v -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-0.some-name-rs0.demand-backup-3000.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
++ mktemp
+ local LAST_OUT=/tmp/tmp.PclZ05lUmo
++ mktemp
+ local LAST_ERR=/tmp/tmp.Z2j346twzn
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-696897d69b-7ct8v -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-0.some-name-rs0.demand-backup-3000.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.PclZ05lUmo
+ cat /tmp/tmp.Z2j346twzn
+ rm /tmp/tmp.PclZ05lUmo /tmp/tmp.Z2j346twzn
+ return 0
+ [[ 0 -eq 0 ]]
+ diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2101/e2e-tests/demand-backup/compare/find.json /tmp/tmp.NAG1atx8KS/find
+ for i in $(seq 0 $last_pod)
+ compare_mongo_cmd find myApp:myPass@some-name-rs0-1.some-name-rs0.demand-backup-3000
+ local command=find
+ local uri=myApp:myPass@some-name-rs0-1.some-name-rs0.demand-backup-3000
+ local postfix=
+ local suffix=
+ local database=myApp
+ local collection=test
+ local sort=
+ local tls=false
+ local 'full_command=db.test.find()'
+ [[ -n '' ]]
+ log 'running db.test.find() in myApp'
+ set +o xtrace
[2025-11-03T02:16:15+0000] running db.test.find() in myApp
+ [[ false == \t\r\u\e ]]
+ mongo_command=run_mongo
+ run_mongo 'use myApp\n db.test.find()' myApp:myPass@some-name-rs0-1.some-name-rs0.demand-backup-3000 mongodb ''
+ local 'command=use myApp\n db.test.find()'
+ local uri=myApp:myPass@some-name-rs0-1.some-name-rs0.demand-backup-3000
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ egrep -v 'I NETWORK|W NETWORK|F NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|Started a new thread for the timer service'
+ /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
egrep: warning: egrep is obsolescent; using grep -E
++ local LAST_OUT=/tmp/tmp.qz4vrttyJv
+++ mktemp
++ local LAST_ERR=/tmp/tmp.nkYT1AOZvp
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.qz4vrttyJv
++ cat /tmp/tmp.nkYT1AOZvp
++ rm /tmp/tmp.qz4vrttyJv /tmp/tmp.nkYT1AOZvp
++ return 0
+ local client_container=psmdb-client-696897d69b-7ct8v
+ local mongo_flag=
+ [[ myApp:myPass@some-name-rs0-1.some-name-rs0.demand-backup-3000 == *cfg* ]]
+ replica_set=rs0
+ kubectl_bin exec psmdb-client-696897d69b-7ct8v -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-1.some-name-rs0.demand-backup-3000.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
++ mktemp
+ local LAST_OUT=/tmp/tmp.FcGYvmTPon
++ mktemp
+ local LAST_ERR=/tmp/tmp.qObSNCAT8Q
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-696897d69b-7ct8v -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-1.some-name-rs0.demand-backup-3000.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.FcGYvmTPon
+ cat /tmp/tmp.qObSNCAT8Q
+ rm /tmp/tmp.FcGYvmTPon /tmp/tmp.qObSNCAT8Q
+ return 0
+ [[ 0 -eq 0 ]]
+ diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2101/e2e-tests/demand-backup/compare/find.json /tmp/tmp.NAG1atx8KS/find
+ for i in $(seq 0 $last_pod)
+ compare_mongo_cmd find myApp:myPass@some-name-rs0-2.some-name-rs0.demand-backup-3000
+ local command=find
+ local uri=myApp:myPass@some-name-rs0-2.some-name-rs0.demand-backup-3000
+ local postfix=
+ local suffix=
+ local database=myApp
+ local collection=test
+ local sort=
+ local tls=false
+ local 'full_command=db.test.find()'
+ [[ -n '' ]]
+ log 'running db.test.find() in myApp'
+ set +o xtrace
[2025-11-03T02:16:18+0000] running db.test.find() in myApp
+ [[ false == \t\r\u\e ]]
+ mongo_command=run_mongo
+ run_mongo 'use myApp\n db.test.find()' myApp:myPass@some-name-rs0-2.some-name-rs0.demand-backup-3000 mongodb ''
+ local 'command=use myApp\n db.test.find()'
+ local uri=myApp:myPass@some-name-rs0-2.some-name-rs0.demand-backup-3000
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ egrep -v 'I NETWORK|W NETWORK|F NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|Started a new thread for the timer service'
+ /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
egrep: warning: egrep is obsolescent; using grep -E
++ local LAST_OUT=/tmp/tmp.enHeUKqelE
+++ mktemp
++ local LAST_ERR=/tmp/tmp.ufKDN4bhYI
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.enHeUKqelE
++ cat /tmp/tmp.ufKDN4bhYI
++ rm /tmp/tmp.enHeUKqelE /tmp/tmp.ufKDN4bhYI
++ return 0
+ local client_container=psmdb-client-696897d69b-7ct8v
+ local mongo_flag=
+ [[ myApp:myPass@some-name-rs0-2.some-name-rs0.demand-backup-3000 == *cfg* ]]
+ replica_set=rs0
+ kubectl_bin exec psmdb-client-696897d69b-7ct8v -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-2.some-name-rs0.demand-backup-3000.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
++ mktemp
+ local LAST_OUT=/tmp/tmp.8itvuSXRbf
++ mktemp
+ local LAST_ERR=/tmp/tmp.2PL57rd97q
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-696897d69b-7ct8v -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-2.some-name-rs0.demand-backup-3000.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.8itvuSXRbf
+ cat /tmp/tmp.2PL57rd97q
+ rm /tmp/tmp.8itvuSXRbf /tmp/tmp.2PL57rd97q
+ return 0
+ [[ 0 -eq 0 ]]
+ diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2101/e2e-tests/demand-backup/compare/find.json /tmp/tmp.NAG1atx8KS/find
+ desc 'check backup and restore -- gcp-cs-sa'
+ set +o xtrace


-----------------------------------------------------------------------------------
check backup and restore -- gcp-cs-sa
-----------------------------------------------------------------------------------

++ get_backup_dest backup-gcp-cs-sa
++ local backup_name=backup-gcp-cs-sa
++ kubectl_bin get psmdb-backup backup-gcp-cs-sa -o 'jsonpath={.status.destination}'
++ sed -e 's/.json$//'
++ sed 's|s3://||'
++ sed 's|azure://||'
+++ mktemp
++ sed 's|gs://||'
++ local LAST_OUT=/tmp/tmp.arbwIsyJJK
+++ mktemp
++ local LAST_ERR=/tmp/tmp.gcvYDG6b9D
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb-backup backup-gcp-cs-sa -o 'jsonpath={.status.destination}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.arbwIsyJJK
++ cat /tmp/tmp.gcvYDG6b9D
++ rm /tmp/tmp.arbwIsyJJK /tmp/tmp.gcvYDG6b9D
++ return 0
+ backup_dest_gcp_sa=operator-testing/psmdb-demand-backup/2025-11-03T02:14:38Z
+ check_backup_in_storage backup-gcp-cs-sa gcs rs0 myApp.test.gz
+ local backup=backup-gcp-cs-sa
+ local storage_type=gcs
+ local replset=rs0
+ local file=myApp.test.gz
+ local endpoint
+ case ${storage_type} in
+ endpoint=storage.googleapis.com
++ get_backup_dest backup-gcp-cs-sa
++ local backup_name=backup-gcp-cs-sa
++ /usr/sbin/sed 's|https://engk8soperators.blob.core.windows.net/||'
++ kubectl_bin get psmdb-backup backup-gcp-cs-sa -o 'jsonpath={.status.destination}'
++ sed -e 's/.json$//'
++ sed 's|s3://||'
+++ mktemp
++ sed 's|azure://||'
++ sed 's|gs://||'
++ local LAST_OUT=/tmp/tmp.7HHHSgtmiF
+++ mktemp
++ local LAST_ERR=/tmp/tmp.NWmyvrdBRs
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb-backup backup-gcp-cs-sa -o 'jsonpath={.status.destination}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.7HHHSgtmiF
++ cat /tmp/tmp.NWmyvrdBRs
++ rm /tmp/tmp.7HHHSgtmiF /tmp/tmp.NWmyvrdBRs
++ return 0
+ backup_dest=operator-testing/psmdb-demand-backup/2025-11-03T02:14:38Z
+ [[ gcs == \m\i\n\i\o ]]
+ local url=https://storage.googleapis.com/operator-testing/psmdb-demand-backup/2025-11-03T02:14:38Z/rs0/myApp.test.gz
+ log 'checking if https://storage.googleapis.com/operator-testing/psmdb-demand-backup/2025-11-03T02:14:38Z/rs0/myApp.test.gz exists'
+ set +o xtrace
[2025-11-03T02:16:21+0000] checking if https://storage.googleapis.com/operator-testing/psmdb-demand-backup/2025-11-03T02:14:38Z/rs0/myApp.test.gz exists
+ curl --fail --head https://storage.googleapis.com/operator-testing/psmdb-demand-backup/2025-11-03T02:14:38Z/rs0/myApp.test.gz
  % Total    % Received % Xferd  Average Speed   Time    Time     Time  Current
                                 Dload  Upload   Total   Spent    Left  Speed

  0     0    0     0    0     0      0      0 --:--:-- --:--:-- --:--:--     0
  0    55    0     0    0     0      0      0 --:--:-- --:--:-- --:--:--     0
HTTP/2 200 
content-type: application/x-gzip
x-guploader-uploadid: AOCedOEWyRV3SopLYF9y0P1on7NwMRRkdCuh1cFkdU1flhaCqhLhYczSD1hotD6RYmfsKDcTJ5m7Xu0
expires: Mon, 03 Nov 2025 03:16:21 GMT
date: Mon, 03 Nov 2025 02:16:21 GMT
cache-control: public, max-age=3600
last-modified: Mon, 03 Nov 2025 02:14:42 GMT
etag: "86c5247e16bafc8b05827bbd193d5a45"
x-goog-generation: 1762136082113516
x-goog-metageneration: 1
x-goog-stored-content-encoding: identity
x-goog-stored-content-length: 55
x-goog-hash: crc32c=Wl0hzA==
x-goog-hash: md5=hsUkfha6/IsFgnu9GT1aRQ==
x-goog-expiration: Tue, 04 Nov 2025 02:14:42 GMT
x-goog-storage-class: REGIONAL
accept-ranges: bytes
content-length: 55
server: UploadServer
alt-svc: h3=":443"; ma=2592000,h3-29=":443"; ma=2592000

+ run_recovery_check backup-gcp-cs-sa some-name-rs0
+ local backup=backup-gcp-cs-sa
+ local cluster=some-name-rs0
+ run_mongo 'use myApp\n db.test.insert({ x: 100501 })' myApp:myPass@some-name-rs0.demand-backup-3000
+ local 'command=use myApp\n db.test.insert({ x: 100501 })'
+ local uri=myApp:myPass@some-name-rs0.demand-backup-3000
+ local driver=mongodb+srv
+ local suffix=.svc.cluster.local
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.ierIuqDyXW
+++ mktemp
++ local LAST_ERR=/tmp/tmp.mF9XapYTS6
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.ierIuqDyXW
++ cat /tmp/tmp.mF9XapYTS6
++ rm /tmp/tmp.ierIuqDyXW /tmp/tmp.mF9XapYTS6
++ return 0
+ local client_container=psmdb-client-696897d69b-7ct8v
+ local mongo_flag=
+ [[ myApp:myPass@some-name-rs0.demand-backup-3000 == *cfg* ]]
+ replica_set=rs0
+ kubectl_bin exec psmdb-client-696897d69b-7ct8v -- bash -c 'printf '\''use myApp\n db.test.insert({ x: 100501 })\n'\'' | mongo mongodb+srv://myApp:myPass@some-name-rs0.demand-backup-3000.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
++ mktemp
+ local LAST_OUT=/tmp/tmp.kplqt3qB2p
++ mktemp
+ local LAST_ERR=/tmp/tmp.8ZOcjXTv84
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-696897d69b-7ct8v -- bash -c 'printf '\''use myApp\n db.test.insert({ x: 100501 })\n'\'' | mongo mongodb+srv://myApp:myPass@some-name-rs0.demand-backup-3000.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.kplqt3qB2p
Percona Server for MongoDB shell version v4.4.29-28
connecting to: mongodb://some-name-rs0-2.some-name-rs0.demand-backup-3000.svc.cluster.local:27017,some-name-rs0-0.some-name-rs0.demand-backup-3000.svc.cluster.local:27017,some-name-rs0-1.some-name-rs0.demand-backup-3000.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb&replicaSet=rs0&ssl=false
Implicit session: session { "id" : UUID("463f0537-1717-4e56-8ca5-44a797a836b6") }
Percona Server for MongoDB server version: v7.0.24-13
WARNING: shell and server versions do not match
switched to db myApp
WriteResult({ "nInserted" : 1 })
bye
+ cat /tmp/tmp.8ZOcjXTv84
+ rm /tmp/tmp.kplqt3qB2p /tmp/tmp.8ZOcjXTv84
+ return 0
+ compare_mongo_cmd find myApp:myPass@some-name-rs0.demand-backup-3000 -2nd .svc.cluster.local myApp test
+ local command=find
+ local uri=myApp:myPass@some-name-rs0.demand-backup-3000
+ local postfix=-2nd
+ local suffix=.svc.cluster.local
+ local database=myApp
+ local collection=test
+ local sort=
+ local tls=false
+ local 'full_command=db.test.find()'
+ [[ -n '' ]]
+ log 'running db.test.find() in myApp'
+ set +o xtrace
[2025-11-03T02:16:24+0000] running db.test.find() in myApp
+ [[ false == \t\r\u\e ]]
+ mongo_command=run_mongo
+ run_mongo 'use myApp\n db.test.find()' myApp:myPass@some-name-rs0.demand-backup-3000 mongodb .svc.cluster.local
+ local 'command=use myApp\n db.test.find()'
+ local uri=myApp:myPass@some-name-rs0.demand-backup-3000
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ egrep -v 'I NETWORK|W NETWORK|F NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|Started a new thread for the timer service'
+ /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.kNYHXUDHte
egrep: warning: egrep is obsolescent; using grep -E
+++ mktemp
++ local LAST_ERR=/tmp/tmp.yYCQY8caBR
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.kNYHXUDHte
++ cat /tmp/tmp.yYCQY8caBR
++ rm /tmp/tmp.kNYHXUDHte /tmp/tmp.yYCQY8caBR
++ return 0
+ local client_container=psmdb-client-696897d69b-7ct8v
+ local mongo_flag=
+ [[ myApp:myPass@some-name-rs0.demand-backup-3000 == *cfg* ]]
+ replica_set=rs0
+ kubectl_bin exec psmdb-client-696897d69b-7ct8v -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0.demand-backup-3000.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
++ mktemp
+ local LAST_OUT=/tmp/tmp.HZSh7bw7bF
++ mktemp
+ local LAST_ERR=/tmp/tmp.gxBeQO699k
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-696897d69b-7ct8v -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0.demand-backup-3000.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.HZSh7bw7bF
+ cat /tmp/tmp.gxBeQO699k
+ rm /tmp/tmp.HZSh7bw7bF /tmp/tmp.gxBeQO699k
+ return 0
+ [[ 0 -eq 0 ]]
+ diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2101/e2e-tests/demand-backup/compare/find-2nd.json /tmp/tmp.NAG1atx8KS/find-2nd
+ run_restore backup-gcp-cs-sa
+ local backup_name=backup-gcp-cs-sa
+ cat /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2101/e2e-tests/demand-backup/conf/restore.yml
+ /usr/sbin/sed -e 's/backupName:/backupName: backup-gcp-cs-sa/'
+ /usr/sbin/sed -e 's/name:/name: restore-backup-gcp-cs-sa/'
+ kubectl_bin apply -f -
++ mktemp
+ local LAST_OUT=/tmp/tmp.PEUUEGBqz6
++ mktemp
+ local LAST_ERR=/tmp/tmp.gMlhSAaRPv
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl apply -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.PEUUEGBqz6
perconaservermongodbrestore.psmdb.percona.com/restore-backup-gcp-cs-sa created
+ cat /tmp/tmp.gMlhSAaRPv
+ rm /tmp/tmp.PEUUEGBqz6 /tmp/tmp.gMlhSAaRPv
+ return 0
+ wait_restore backup-gcp-cs-sa some-name
+ local backup_name=backup-gcp-cs-sa
+ local cluster_name=some-name
+ local target_state=ready
+ local wait_cluster_consistency=1
+ local wait_time=1780
+ local ok_if_ready=0
+ set +o xtrace
Waiting for the psmdb-restore/restore-backup-gcp-cs-sa object to be created.OK
Waiting psmdb-restore/restore-backup-gcp-cs-sa to reach state "ready" .OK after 0 minutes
+ [[ 1 -eq 1 ]]
+ wait_cluster_consistency some-name
+ local cluster_name=some-name
+ local wait_time=32
+ retry=0
+ sleep 7
+ echo -n 'waiting for cluster readyness'
waiting for cluster readyness++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.cFUzKQS0OE
+++ mktemp
++ local LAST_ERR=/tmp/tmp.N16cSCGz78
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.cFUzKQS0OE
++ cat /tmp/tmp.N16cSCGz78
++ rm /tmp/tmp.cFUzKQS0OE /tmp/tmp.N16cSCGz78
++ return 0
+ [[ ready == \r\e\a\d\y ]]
+ echo .OK
.OK
+ simple_data_check some-name-rs0 3 0
+ local cluster_name=some-name-rs0
+ let last_pod=3-1
+ local isSharded=0
+ local cluster_pfx=
+ '[' 0 -eq 1 ']'
++ seq 0 2
+ for i in $(seq 0 $last_pod)
+ compare_mongo_cmd find myApp:myPass@some-name-rs0-0.some-name-rs0.demand-backup-3000
+ local command=find
+ local uri=myApp:myPass@some-name-rs0-0.some-name-rs0.demand-backup-3000
+ local postfix=
+ local suffix=
+ local database=myApp
+ local collection=test
+ local sort=
+ local tls=false
+ local 'full_command=db.test.find()'
+ [[ -n '' ]]
+ log 'running db.test.find() in myApp'
+ set +o xtrace
[2025-11-03T02:16:54+0000] running db.test.find() in myApp
+ [[ false == \t\r\u\e ]]
+ mongo_command=run_mongo
+ run_mongo 'use myApp\n db.test.find()' myApp:myPass@some-name-rs0-0.some-name-rs0.demand-backup-3000 mongodb ''
+ egrep -v 'I NETWORK|W NETWORK|F NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|Started a new thread for the timer service'
+ local 'command=use myApp\n db.test.find()'
+ local uri=myApp:myPass@some-name-rs0-0.some-name-rs0.demand-backup-3000
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
egrep: warning: egrep is obsolescent; using grep -E
++ local LAST_OUT=/tmp/tmp.y8gVUc4rN4
+++ mktemp
++ local LAST_ERR=/tmp/tmp.k94F2lNIV7
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.y8gVUc4rN4
++ cat /tmp/tmp.k94F2lNIV7
++ rm /tmp/tmp.y8gVUc4rN4 /tmp/tmp.k94F2lNIV7
++ return 0
+ local client_container=psmdb-client-696897d69b-7ct8v
+ local mongo_flag=
+ [[ myApp:myPass@some-name-rs0-0.some-name-rs0.demand-backup-3000 == *cfg* ]]
+ replica_set=rs0
+ kubectl_bin exec psmdb-client-696897d69b-7ct8v -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-0.some-name-rs0.demand-backup-3000.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
++ mktemp
+ local LAST_OUT=/tmp/tmp.vN4xPHh6v7
++ mktemp
+ local LAST_ERR=/tmp/tmp.eTeAhvciVw
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-696897d69b-7ct8v -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-0.some-name-rs0.demand-backup-3000.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.vN4xPHh6v7
+ cat /tmp/tmp.eTeAhvciVw
+ rm /tmp/tmp.vN4xPHh6v7 /tmp/tmp.eTeAhvciVw
+ return 0
+ [[ 0 -eq 0 ]]
+ diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2101/e2e-tests/demand-backup/compare/find.json /tmp/tmp.NAG1atx8KS/find
+ for i in $(seq 0 $last_pod)
+ compare_mongo_cmd find myApp:myPass@some-name-rs0-1.some-name-rs0.demand-backup-3000
+ local command=find
+ local uri=myApp:myPass@some-name-rs0-1.some-name-rs0.demand-backup-3000
+ local postfix=
+ local suffix=
+ local database=myApp
+ local collection=test
+ local sort=
+ local tls=false
+ local 'full_command=db.test.find()'
+ [[ -n '' ]]
+ log 'running db.test.find() in myApp'
+ set +o xtrace
[2025-11-03T02:16:57+0000] running db.test.find() in myApp
+ [[ false == \t\r\u\e ]]
+ mongo_command=run_mongo
+ run_mongo 'use myApp\n db.test.find()' myApp:myPass@some-name-rs0-1.some-name-rs0.demand-backup-3000 mongodb ''
+ local 'command=use myApp\n db.test.find()'
+ egrep -v 'I NETWORK|W NETWORK|F NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|Started a new thread for the timer service'
+ local uri=myApp:myPass@some-name-rs0-1.some-name-rs0.demand-backup-3000
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
egrep: warning: egrep is obsolescent; using grep -E
++ local LAST_OUT=/tmp/tmp.QFbEBzkbEP
+++ mktemp
++ local LAST_ERR=/tmp/tmp.Sl7P0yVXeT
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.QFbEBzkbEP
++ cat /tmp/tmp.Sl7P0yVXeT
++ rm /tmp/tmp.QFbEBzkbEP /tmp/tmp.Sl7P0yVXeT
++ return 0
+ local client_container=psmdb-client-696897d69b-7ct8v
+ local mongo_flag=
+ [[ myApp:myPass@some-name-rs0-1.some-name-rs0.demand-backup-3000 == *cfg* ]]
+ replica_set=rs0
+ kubectl_bin exec psmdb-client-696897d69b-7ct8v -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-1.some-name-rs0.demand-backup-3000.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
++ mktemp
+ local LAST_OUT=/tmp/tmp.OK9zVdH8WP
++ mktemp
+ local LAST_ERR=/tmp/tmp.jMe4ocaTu6
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-696897d69b-7ct8v -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-1.some-name-rs0.demand-backup-3000.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.OK9zVdH8WP
+ cat /tmp/tmp.jMe4ocaTu6
+ rm /tmp/tmp.OK9zVdH8WP /tmp/tmp.jMe4ocaTu6
+ return 0
+ [[ 0 -eq 0 ]]
+ diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2101/e2e-tests/demand-backup/compare/find.json /tmp/tmp.NAG1atx8KS/find
+ for i in $(seq 0 $last_pod)
+ compare_mongo_cmd find myApp:myPass@some-name-rs0-2.some-name-rs0.demand-backup-3000
+ local command=find
+ local uri=myApp:myPass@some-name-rs0-2.some-name-rs0.demand-backup-3000
+ local postfix=
+ local suffix=
+ local database=myApp
+ local collection=test
+ local sort=
+ local tls=false
+ local 'full_command=db.test.find()'
+ [[ -n '' ]]
+ log 'running db.test.find() in myApp'
+ set +o xtrace
[2025-11-03T02:16:59+0000] running db.test.find() in myApp
+ [[ false == \t\r\u\e ]]
+ mongo_command=run_mongo
+ run_mongo 'use myApp\n db.test.find()' myApp:myPass@some-name-rs0-2.some-name-rs0.demand-backup-3000 mongodb ''
+ local 'command=use myApp\n db.test.find()'
+ local uri=myApp:myPass@some-name-rs0-2.some-name-rs0.demand-backup-3000
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ egrep -v 'I NETWORK|W NETWORK|F NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|Started a new thread for the timer service'
+ /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
egrep: warning: egrep is obsolescent; using grep -E
++ local LAST_OUT=/tmp/tmp.0eqeWRMVwA
+++ mktemp
++ local LAST_ERR=/tmp/tmp.tj6j9GmGqn
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.0eqeWRMVwA
++ cat /tmp/tmp.tj6j9GmGqn
++ rm /tmp/tmp.0eqeWRMVwA /tmp/tmp.tj6j9GmGqn
++ return 0
+ local client_container=psmdb-client-696897d69b-7ct8v
+ local mongo_flag=
+ [[ myApp:myPass@some-name-rs0-2.some-name-rs0.demand-backup-3000 == *cfg* ]]
+ replica_set=rs0
+ kubectl_bin exec psmdb-client-696897d69b-7ct8v -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-2.some-name-rs0.demand-backup-3000.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
++ mktemp
+ local LAST_OUT=/tmp/tmp.MqOyoamLaC
++ mktemp
+ local LAST_ERR=/tmp/tmp.rY0T7qP6ac
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-696897d69b-7ct8v -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-2.some-name-rs0.demand-backup-3000.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.MqOyoamLaC
+ cat /tmp/tmp.rY0T7qP6ac
+ rm /tmp/tmp.MqOyoamLaC /tmp/tmp.rY0T7qP6ac
+ return 0
+ [[ 0 -eq 0 ]]
+ diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2101/e2e-tests/demand-backup/compare/find.json /tmp/tmp.NAG1atx8KS/find
+ desc 'check backup and restore -- azure-blob'
+ set +o xtrace


-----------------------------------------------------------------------------------
check backup and restore -- azure-blob
-----------------------------------------------------------------------------------

++ get_backup_dest backup-azure-blob
++ local backup_name=backup-azure-blob
++ kubectl_bin get psmdb-backup backup-azure-blob -o 'jsonpath={.status.destination}'
++ sed -e 's/.json$//'
++ sed 's|s3://||'
+++ mktemp
++ sed 's|azure://||'
++ sed 's|gs://||'
++ local LAST_OUT=/tmp/tmp.i5svhPbdiB
+++ mktemp
++ local LAST_ERR=/tmp/tmp.8be1rHQXud
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb-backup backup-azure-blob -o 'jsonpath={.status.destination}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.i5svhPbdiB
++ cat /tmp/tmp.8be1rHQXud
++ rm /tmp/tmp.i5svhPbdiB /tmp/tmp.8be1rHQXud
++ return 0
+ backup_dest_azure=https://engk8soperators.blob.core.windows.net/operator-testing/psmdb-demand-backup/2025-11-03T02:14:04Z
+ check_backup_in_storage backup-azure-blob azure rs0 myApp.test.gz
+ local backup=backup-azure-blob
+ local storage_type=azure
+ local replset=rs0
+ local file=myApp.test.gz
+ local endpoint
+ case ${storage_type} in
+ endpoint=engk8soperators.blob.core.windows.net
++ get_backup_dest backup-azure-blob
++ local backup_name=backup-azure-blob
++ /usr/sbin/sed 's|https://engk8soperators.blob.core.windows.net/||'
++ kubectl_bin get psmdb-backup backup-azure-blob -o 'jsonpath={.status.destination}'
++ sed -e 's/.json$//'
++ sed 's|s3://||'
+++ mktemp
++ sed 's|azure://||'
++ sed 's|gs://||'
++ local LAST_OUT=/tmp/tmp.LlBPpjhkrW
+++ mktemp
++ local LAST_ERR=/tmp/tmp.ohfUqUQr6X
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb-backup backup-azure-blob -o 'jsonpath={.status.destination}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.LlBPpjhkrW
++ cat /tmp/tmp.ohfUqUQr6X
++ rm /tmp/tmp.LlBPpjhkrW /tmp/tmp.ohfUqUQr6X
++ return 0
+ backup_dest=operator-testing/psmdb-demand-backup/2025-11-03T02:14:04Z
+ [[ azure == \m\i\n\i\o ]]
+ local url=https://engk8soperators.blob.core.windows.net/operator-testing/psmdb-demand-backup/2025-11-03T02:14:04Z/rs0/myApp.test.gz
+ log 'checking if https://engk8soperators.blob.core.windows.net/operator-testing/psmdb-demand-backup/2025-11-03T02:14:04Z/rs0/myApp.test.gz exists'
+ set +o xtrace
[2025-11-03T02:17:03+0000] checking if https://engk8soperators.blob.core.windows.net/operator-testing/psmdb-demand-backup/2025-11-03T02:14:04Z/rs0/myApp.test.gz exists
+ curl --fail --head https://engk8soperators.blob.core.windows.net/operator-testing/psmdb-demand-backup/2025-11-03T02:14:04Z/rs0/myApp.test.gz
  % Total    % Received % Xferd  Average Speed   Time    Time     Time  Current
                                 Dload  Upload   Total   Spent    Left  Speed

  0     0    0     0    0     0      0      0 --:--:-- --:--:-- --:--:--     0
  0    55    0     0    0     0      0      0 --:--:-- --:--:-- --:--:--     0
HTTP/1.1 200 OK
Content-Length: 55
Content-Type: application/octet-stream
Content-MD5: hsUkfha6/IsFgnu9GT1aRQ==
Last-Modified: Mon, 03 Nov 2025 02:14:08 GMT
ETag: 0x8DE1A7EABAE1D01
Server: Windows-Azure-Blob/1.0 Microsoft-HTTPAPI/2.0
x-ms-request-id: a87a947c-c01e-0006-2d67-4cecd0000000
x-ms-version: 2009-09-19
x-ms-lease-status: unlocked
x-ms-blob-type: BlockBlob
Date: Mon, 03 Nov 2025 02:17:03 GMT

+ run_recovery_check backup-azure-blob some-name-rs0
+ local backup=backup-azure-blob
+ local cluster=some-name-rs0
+ run_mongo 'use myApp\n db.test.insert({ x: 100501 })' myApp:myPass@some-name-rs0.demand-backup-3000
+ local 'command=use myApp\n db.test.insert({ x: 100501 })'
+ local uri=myApp:myPass@some-name-rs0.demand-backup-3000
+ local driver=mongodb+srv
+ local suffix=.svc.cluster.local
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.jmbkYcPCaM
+++ mktemp
++ local LAST_ERR=/tmp/tmp.lkrryBq9Ev
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.jmbkYcPCaM
++ cat /tmp/tmp.lkrryBq9Ev
++ rm /tmp/tmp.jmbkYcPCaM /tmp/tmp.lkrryBq9Ev
++ return 0
+ local client_container=psmdb-client-696897d69b-7ct8v
+ local mongo_flag=
+ [[ myApp:myPass@some-name-rs0.demand-backup-3000 == *cfg* ]]
+ replica_set=rs0
+ kubectl_bin exec psmdb-client-696897d69b-7ct8v -- bash -c 'printf '\''use myApp\n db.test.insert({ x: 100501 })\n'\'' | mongo mongodb+srv://myApp:myPass@some-name-rs0.demand-backup-3000.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
++ mktemp
+ local LAST_OUT=/tmp/tmp.wBIl4NUsZ3
++ mktemp
+ local LAST_ERR=/tmp/tmp.CHL6bwHtF9
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-696897d69b-7ct8v -- bash -c 'printf '\''use myApp\n db.test.insert({ x: 100501 })\n'\'' | mongo mongodb+srv://myApp:myPass@some-name-rs0.demand-backup-3000.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.wBIl4NUsZ3
Percona Server for MongoDB shell version v4.4.29-28
connecting to: mongodb://some-name-rs0-0.some-name-rs0.demand-backup-3000.svc.cluster.local:27017,some-name-rs0-1.some-name-rs0.demand-backup-3000.svc.cluster.local:27017,some-name-rs0-2.some-name-rs0.demand-backup-3000.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb&replicaSet=rs0&ssl=false
Implicit session: session { "id" : UUID("54e48161-2846-4201-bbc6-2f7a288eff45") }
Percona Server for MongoDB server version: v7.0.24-13
WARNING: shell and server versions do not match
switched to db myApp
WriteResult({ "nInserted" : 1 })
bye
+ cat /tmp/tmp.CHL6bwHtF9
+ rm /tmp/tmp.wBIl4NUsZ3 /tmp/tmp.CHL6bwHtF9
+ return 0
+ compare_mongo_cmd find myApp:myPass@some-name-rs0.demand-backup-3000 -2nd .svc.cluster.local myApp test
+ local command=find
+ local uri=myApp:myPass@some-name-rs0.demand-backup-3000
+ local postfix=-2nd
+ local suffix=.svc.cluster.local
+ local database=myApp
+ local collection=test
+ local sort=
+ local tls=false
+ local 'full_command=db.test.find()'
+ [[ -n '' ]]
+ log 'running db.test.find() in myApp'
+ set +o xtrace
[2025-11-03T02:17:06+0000] running db.test.find() in myApp
+ [[ false == \t\r\u\e ]]
+ mongo_command=run_mongo
+ run_mongo 'use myApp\n db.test.find()' myApp:myPass@some-name-rs0.demand-backup-3000 mongodb .svc.cluster.local
+ local 'command=use myApp\n db.test.find()'
+ local uri=myApp:myPass@some-name-rs0.demand-backup-3000
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
+ egrep -v 'I NETWORK|W NETWORK|F NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|Started a new thread for the timer service'
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.4EMGBOlFxC
egrep: warning: egrep is obsolescent; using grep -E
+++ mktemp
++ local LAST_ERR=/tmp/tmp.UssRXi4VE9
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.4EMGBOlFxC
++ cat /tmp/tmp.UssRXi4VE9
++ rm /tmp/tmp.4EMGBOlFxC /tmp/tmp.UssRXi4VE9
++ return 0
+ local client_container=psmdb-client-696897d69b-7ct8v
+ local mongo_flag=
+ [[ myApp:myPass@some-name-rs0.demand-backup-3000 == *cfg* ]]
+ replica_set=rs0
+ kubectl_bin exec psmdb-client-696897d69b-7ct8v -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0.demand-backup-3000.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
++ mktemp
+ local LAST_OUT=/tmp/tmp.dgwfZ94XlH
++ mktemp
+ local LAST_ERR=/tmp/tmp.85cPQyZQib
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-696897d69b-7ct8v -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0.demand-backup-3000.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.dgwfZ94XlH
+ cat /tmp/tmp.85cPQyZQib
+ rm /tmp/tmp.dgwfZ94XlH /tmp/tmp.85cPQyZQib
+ return 0
+ [[ 0 -eq 0 ]]
+ diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2101/e2e-tests/demand-backup/compare/find-2nd.json /tmp/tmp.NAG1atx8KS/find-2nd
+ run_restore backup-azure-blob
+ local backup_name=backup-azure-blob
+ cat /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2101/e2e-tests/demand-backup/conf/restore.yml
+ /usr/sbin/sed -e 's/name:/name: restore-backup-azure-blob/'
+ /usr/sbin/sed -e 's/backupName:/backupName: backup-azure-blob/'
+ kubectl_bin apply -f -
++ mktemp
+ local LAST_OUT=/tmp/tmp.nOdMQIIfa7
++ mktemp
+ local LAST_ERR=/tmp/tmp.404yU4soqw
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl apply -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.nOdMQIIfa7
perconaservermongodbrestore.psmdb.percona.com/restore-backup-azure-blob created
+ cat /tmp/tmp.404yU4soqw
+ rm /tmp/tmp.nOdMQIIfa7 /tmp/tmp.404yU4soqw
+ return 0
+ wait_restore backup-azure-blob some-name
+ local backup_name=backup-azure-blob
+ local cluster_name=some-name
+ local target_state=ready
+ local wait_cluster_consistency=1
+ local wait_time=1780
+ local ok_if_ready=0
+ set +o xtrace
Waiting for the psmdb-restore/restore-backup-azure-blob object to be created.OK
Waiting psmdb-restore/restore-backup-azure-blob to reach state "ready" .OK after 0 minutes
+ [[ 1 -eq 1 ]]
+ wait_cluster_consistency some-name
+ local cluster_name=some-name
+ local wait_time=32
+ retry=0
+ sleep 7
+ echo -n 'waiting for cluster readyness'
waiting for cluster readyness++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.t9vrtlmFQV
+++ mktemp
++ local LAST_ERR=/tmp/tmp.kTRIMnXEjX
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.t9vrtlmFQV
++ cat /tmp/tmp.kTRIMnXEjX
++ rm /tmp/tmp.t9vrtlmFQV /tmp/tmp.kTRIMnXEjX
++ return 0
+ [[ ready == \r\e\a\d\y ]]
+ echo .OK
.OK
+ simple_data_check some-name-rs0 3 0
+ local cluster_name=some-name-rs0
+ let last_pod=3-1
+ local isSharded=0
+ local cluster_pfx=
+ '[' 0 -eq 1 ']'
++ seq 0 2
+ for i in $(seq 0 $last_pod)
+ compare_mongo_cmd find myApp:myPass@some-name-rs0-0.some-name-rs0.demand-backup-3000
+ local command=find
+ local uri=myApp:myPass@some-name-rs0-0.some-name-rs0.demand-backup-3000
+ local postfix=
+ local suffix=
+ local database=myApp
+ local collection=test
+ local sort=
+ local tls=false
+ local 'full_command=db.test.find()'
+ [[ -n '' ]]
+ log 'running db.test.find() in myApp'
+ set +o xtrace
[2025-11-03T02:17:36+0000] running db.test.find() in myApp
+ [[ false == \t\r\u\e ]]
+ mongo_command=run_mongo
+ run_mongo 'use myApp\n db.test.find()' myApp:myPass@some-name-rs0-0.some-name-rs0.demand-backup-3000 mongodb ''
+ local 'command=use myApp\n db.test.find()'
+ local uri=myApp:myPass@some-name-rs0-0.some-name-rs0.demand-backup-3000
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ egrep -v 'I NETWORK|W NETWORK|F NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|Started a new thread for the timer service'
+ /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
egrep: warning: egrep is obsolescent; using grep -E
++ local LAST_OUT=/tmp/tmp.e84YCjHFms
+++ mktemp
++ local LAST_ERR=/tmp/tmp.E2BLa2MHnj
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.e84YCjHFms
++ cat /tmp/tmp.E2BLa2MHnj
++ rm /tmp/tmp.e84YCjHFms /tmp/tmp.E2BLa2MHnj
++ return 0
+ local client_container=psmdb-client-696897d69b-7ct8v
+ local mongo_flag=
+ [[ myApp:myPass@some-name-rs0-0.some-name-rs0.demand-backup-3000 == *cfg* ]]
+ replica_set=rs0
+ kubectl_bin exec psmdb-client-696897d69b-7ct8v -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-0.some-name-rs0.demand-backup-3000.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
++ mktemp
+ local LAST_OUT=/tmp/tmp.oM5d0N4MB0
++ mktemp
+ local LAST_ERR=/tmp/tmp.TQHlDfy0At
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-696897d69b-7ct8v -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-0.some-name-rs0.demand-backup-3000.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.oM5d0N4MB0
+ cat /tmp/tmp.TQHlDfy0At
+ rm /tmp/tmp.oM5d0N4MB0 /tmp/tmp.TQHlDfy0At
+ return 0
+ [[ 0 -eq 0 ]]
+ diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2101/e2e-tests/demand-backup/compare/find.json /tmp/tmp.NAG1atx8KS/find
+ for i in $(seq 0 $last_pod)
+ compare_mongo_cmd find myApp:myPass@some-name-rs0-1.some-name-rs0.demand-backup-3000
+ local command=find
+ local uri=myApp:myPass@some-name-rs0-1.some-name-rs0.demand-backup-3000
+ local postfix=
+ local suffix=
+ local database=myApp
+ local collection=test
+ local sort=
+ local tls=false
+ local 'full_command=db.test.find()'
+ [[ -n '' ]]
+ log 'running db.test.find() in myApp'
+ set +o xtrace
[2025-11-03T02:17:38+0000] running db.test.find() in myApp
+ [[ false == \t\r\u\e ]]
+ mongo_command=run_mongo
+ run_mongo 'use myApp\n db.test.find()' myApp:myPass@some-name-rs0-1.some-name-rs0.demand-backup-3000 mongodb ''
+ local 'command=use myApp\n db.test.find()'
+ local uri=myApp:myPass@some-name-rs0-1.some-name-rs0.demand-backup-3000
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ egrep -v 'I NETWORK|W NETWORK|F NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|Started a new thread for the timer service'
+ /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
egrep: warning: egrep is obsolescent; using grep -E
++ local LAST_OUT=/tmp/tmp.3YVr0x5jlZ
+++ mktemp
++ local LAST_ERR=/tmp/tmp.RXQihP1VCN
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.3YVr0x5jlZ
++ cat /tmp/tmp.RXQihP1VCN
++ rm /tmp/tmp.3YVr0x5jlZ /tmp/tmp.RXQihP1VCN
++ return 0
+ local client_container=psmdb-client-696897d69b-7ct8v
+ local mongo_flag=
+ [[ myApp:myPass@some-name-rs0-1.some-name-rs0.demand-backup-3000 == *cfg* ]]
+ replica_set=rs0
+ kubectl_bin exec psmdb-client-696897d69b-7ct8v -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-1.some-name-rs0.demand-backup-3000.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
++ mktemp
+ local LAST_OUT=/tmp/tmp.iATDm0hcSJ
++ mktemp
+ local LAST_ERR=/tmp/tmp.FGo2MojMAv
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-696897d69b-7ct8v -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-1.some-name-rs0.demand-backup-3000.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.iATDm0hcSJ
+ cat /tmp/tmp.FGo2MojMAv
+ rm /tmp/tmp.iATDm0hcSJ /tmp/tmp.FGo2MojMAv
+ return 0
+ [[ 0 -eq 0 ]]
+ diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2101/e2e-tests/demand-backup/compare/find.json /tmp/tmp.NAG1atx8KS/find
+ for i in $(seq 0 $last_pod)
+ compare_mongo_cmd find myApp:myPass@some-name-rs0-2.some-name-rs0.demand-backup-3000
+ local command=find
+ local uri=myApp:myPass@some-name-rs0-2.some-name-rs0.demand-backup-3000
+ local postfix=
+ local suffix=
+ local database=myApp
+ local collection=test
+ local sort=
+ local tls=false
+ local 'full_command=db.test.find()'
+ [[ -n '' ]]
+ log 'running db.test.find() in myApp'
+ set +o xtrace
[2025-11-03T02:17:40+0000] running db.test.find() in myApp
+ [[ false == \t\r\u\e ]]
+ mongo_command=run_mongo
+ run_mongo 'use myApp\n db.test.find()' myApp:myPass@some-name-rs0-2.some-name-rs0.demand-backup-3000 mongodb ''
+ local 'command=use myApp\n db.test.find()'
+ local uri=myApp:myPass@some-name-rs0-2.some-name-rs0.demand-backup-3000
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ egrep -v 'I NETWORK|W NETWORK|F NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|Started a new thread for the timer service'
+ /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
egrep: warning: egrep is obsolescent; using grep -E
++ local LAST_OUT=/tmp/tmp.23EOwOU8FV
+++ mktemp
++ local LAST_ERR=/tmp/tmp.RRmeXRUuKE
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.23EOwOU8FV
++ cat /tmp/tmp.RRmeXRUuKE
++ rm /tmp/tmp.23EOwOU8FV /tmp/tmp.RRmeXRUuKE
++ return 0
+ local client_container=psmdb-client-696897d69b-7ct8v
+ local mongo_flag=
+ [[ myApp:myPass@some-name-rs0-2.some-name-rs0.demand-backup-3000 == *cfg* ]]
+ replica_set=rs0
+ kubectl_bin exec psmdb-client-696897d69b-7ct8v -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-2.some-name-rs0.demand-backup-3000.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
++ mktemp
+ local LAST_OUT=/tmp/tmp.mIW76uUjHH
++ mktemp
+ local LAST_ERR=/tmp/tmp.ohN0XpgLDj
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-696897d69b-7ct8v -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-2.some-name-rs0.demand-backup-3000.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.mIW76uUjHH
+ cat /tmp/tmp.ohN0XpgLDj
+ rm /tmp/tmp.mIW76uUjHH /tmp/tmp.ohN0XpgLDj
+ return 0
+ [[ 0 -eq 0 ]]
+ diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2101/e2e-tests/demand-backup/compare/find.json /tmp/tmp.NAG1atx8KS/find
+ desc 'check backup and restore -- minio'
+ set +o xtrace


-----------------------------------------------------------------------------------
check backup and restore -- minio
-----------------------------------------------------------------------------------

+ check_backup_in_storage backup-minio minio rs0 myApp.test.gz
+ local backup=backup-minio
+ local storage_type=minio
+ local replset=rs0
+ local file=myApp.test.gz
+ local endpoint
+ case ${storage_type} in
+ endpoint=minio-service
++ get_backup_dest backup-minio
++ local backup_name=backup-minio
++ /usr/sbin/sed 's|https://engk8soperators.blob.core.windows.net/||'
++ kubectl_bin get psmdb-backup backup-minio -o 'jsonpath={.status.destination}'
++ sed -e 's/.json$//'
++ sed 's|s3://||'
+++ mktemp
++ sed 's|azure://||'
++ sed 's|gs://||'
++ local LAST_OUT=/tmp/tmp.JWT21RkhSs
+++ mktemp
++ local LAST_ERR=/tmp/tmp.HcTzVNuClM
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb-backup backup-minio -o 'jsonpath={.status.destination}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.JWT21RkhSs
++ cat /tmp/tmp.HcTzVNuClM
++ rm /tmp/tmp.JWT21RkhSs /tmp/tmp.HcTzVNuClM
++ return 0
+ backup_dest=operator-testing/2025-11-03T02:14:26Z
+ [[ minio == \m\i\n\i\o ]]
+ kubectl_bin run -i --rm aws-cli --image=perconalab/awscli --restart=Never -- /usr/bin/env AWS_ACCESS_KEY_ID=some-access-key AWS_SECRET_ACCESS_KEY=some-secret-key AWS_DEFAULT_REGION=us-east-1 /usr/bin/aws --endpoint-url http://minio-service:9000 s3 ls s3://operator-testing/2025-11-03T02:14:26Z/rs0/myApp.test.gz
+ grep myApp.test.gz
++ mktemp
+ local LAST_OUT=/tmp/tmp.1yop2feaMY
++ mktemp
+ local LAST_ERR=/tmp/tmp.dH0pQ0Nd5i
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl run -i --rm aws-cli --image=perconalab/awscli --restart=Never -- /usr/bin/env AWS_ACCESS_KEY_ID=some-access-key AWS_SECRET_ACCESS_KEY=some-secret-key AWS_DEFAULT_REGION=us-east-1 /usr/bin/aws --endpoint-url http://minio-service:9000 s3 ls s3://operator-testing/2025-11-03T02:14:26Z/rs0/myApp.test.gz
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.1yop2feaMY
+ cat /tmp/tmp.dH0pQ0Nd5i
+ rm /tmp/tmp.1yop2feaMY /tmp/tmp.dH0pQ0Nd5i
+ return 0
2025-11-03 02:14:30         55 myApp.test.gz
+ run_recovery_check backup-minio some-name-rs0
+ local backup=backup-minio
+ local cluster=some-name-rs0
+ run_mongo 'use myApp\n db.test.insert({ x: 100501 })' myApp:myPass@some-name-rs0.demand-backup-3000
+ local 'command=use myApp\n db.test.insert({ x: 100501 })'
+ local uri=myApp:myPass@some-name-rs0.demand-backup-3000
+ local driver=mongodb+srv
+ local suffix=.svc.cluster.local
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.txFj8qMVmY
+++ mktemp
++ local LAST_ERR=/tmp/tmp.79AFzSkEqR
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.txFj8qMVmY
++ cat /tmp/tmp.79AFzSkEqR
++ rm /tmp/tmp.txFj8qMVmY /tmp/tmp.79AFzSkEqR
++ return 0
+ local client_container=psmdb-client-696897d69b-7ct8v
+ local mongo_flag=
+ [[ myApp:myPass@some-name-rs0.demand-backup-3000 == *cfg* ]]
+ replica_set=rs0
+ kubectl_bin exec psmdb-client-696897d69b-7ct8v -- bash -c 'printf '\''use myApp\n db.test.insert({ x: 100501 })\n'\'' | mongo mongodb+srv://myApp:myPass@some-name-rs0.demand-backup-3000.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
++ mktemp
+ local LAST_OUT=/tmp/tmp.mwmX2YB594
++ mktemp
+ local LAST_ERR=/tmp/tmp.RebJmHfIzr
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-696897d69b-7ct8v -- bash -c 'printf '\''use myApp\n db.test.insert({ x: 100501 })\n'\'' | mongo mongodb+srv://myApp:myPass@some-name-rs0.demand-backup-3000.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.mwmX2YB594
Percona Server for MongoDB shell version v4.4.29-28
connecting to: mongodb://some-name-rs0-2.some-name-rs0.demand-backup-3000.svc.cluster.local:27017,some-name-rs0-0.some-name-rs0.demand-backup-3000.svc.cluster.local:27017,some-name-rs0-1.some-name-rs0.demand-backup-3000.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb&replicaSet=rs0&ssl=false
Implicit session: session { "id" : UUID("ec347eeb-cc57-4c9c-8e08-1f773e8d23c5") }
Percona Server for MongoDB server version: v7.0.24-13
WARNING: shell and server versions do not match
switched to db myApp
WriteResult({ "nInserted" : 1 })
bye
+ cat /tmp/tmp.RebJmHfIzr
+ rm /tmp/tmp.mwmX2YB594 /tmp/tmp.RebJmHfIzr
+ return 0
+ compare_mongo_cmd find myApp:myPass@some-name-rs0.demand-backup-3000 -2nd .svc.cluster.local myApp test
+ local command=find
+ local uri=myApp:myPass@some-name-rs0.demand-backup-3000
+ local postfix=-2nd
+ local suffix=.svc.cluster.local
+ local database=myApp
+ local collection=test
+ local sort=
+ local tls=false
+ local 'full_command=db.test.find()'
+ [[ -n '' ]]
+ log 'running db.test.find() in myApp'
+ set +o xtrace
[2025-11-03T02:17:50+0000] running db.test.find() in myApp
+ [[ false == \t\r\u\e ]]
+ mongo_command=run_mongo
+ run_mongo 'use myApp\n db.test.find()' myApp:myPass@some-name-rs0.demand-backup-3000 mongodb .svc.cluster.local
+ local 'command=use myApp\n db.test.find()'
+ local uri=myApp:myPass@some-name-rs0.demand-backup-3000
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ egrep -v 'I NETWORK|W NETWORK|F NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|Started a new thread for the timer service'
+ /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
egrep: warning: egrep is obsolescent; using grep -E
++ local LAST_OUT=/tmp/tmp.93nzSI6z9U
+++ mktemp
++ local LAST_ERR=/tmp/tmp.pp8lQmmBLD
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.93nzSI6z9U
++ cat /tmp/tmp.pp8lQmmBLD
++ rm /tmp/tmp.93nzSI6z9U /tmp/tmp.pp8lQmmBLD
++ return 0
+ local client_container=psmdb-client-696897d69b-7ct8v
+ local mongo_flag=
+ [[ myApp:myPass@some-name-rs0.demand-backup-3000 == *cfg* ]]
+ replica_set=rs0
+ kubectl_bin exec psmdb-client-696897d69b-7ct8v -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0.demand-backup-3000.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
++ mktemp
+ local LAST_OUT=/tmp/tmp.Lt1PydOYzf
++ mktemp
+ local LAST_ERR=/tmp/tmp.TtK5WnG5pf
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-696897d69b-7ct8v -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0.demand-backup-3000.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.Lt1PydOYzf
+ cat /tmp/tmp.TtK5WnG5pf
+ rm /tmp/tmp.Lt1PydOYzf /tmp/tmp.TtK5WnG5pf
+ return 0
+ [[ 0 -eq 0 ]]
+ diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2101/e2e-tests/demand-backup/compare/find-2nd.json /tmp/tmp.NAG1atx8KS/find-2nd
+ run_restore backup-minio
+ local backup_name=backup-minio
+ cat /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2101/e2e-tests/demand-backup/conf/restore.yml
+ /usr/sbin/sed -e 's/name:/name: restore-backup-minio/'
+ /usr/sbin/sed -e 's/backupName:/backupName: backup-minio/'
+ kubectl_bin apply -f -
++ mktemp
+ local LAST_OUT=/tmp/tmp.U7VUjSDCyv
++ mktemp
+ local LAST_ERR=/tmp/tmp.gVwhiHLdYr
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl apply -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.U7VUjSDCyv
perconaservermongodbrestore.psmdb.percona.com/restore-backup-minio created
+ cat /tmp/tmp.gVwhiHLdYr
+ rm /tmp/tmp.U7VUjSDCyv /tmp/tmp.gVwhiHLdYr
+ return 0
+ wait_restore backup-minio some-name
+ local backup_name=backup-minio
+ local cluster_name=some-name
+ local target_state=ready
+ local wait_cluster_consistency=1
+ local wait_time=1780
+ local ok_if_ready=0
+ set +o xtrace
Waiting for the psmdb-restore/restore-backup-minio object to be created.OK
Waiting psmdb-restore/restore-backup-minio to reach state "ready" .OK after 0 minutes
+ [[ 1 -eq 1 ]]
+ wait_cluster_consistency some-name
+ local cluster_name=some-name
+ local wait_time=32
+ retry=0
+ sleep 7
+ echo -n 'waiting for cluster readyness'
waiting for cluster readyness++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.OozyxeZlhT
+++ mktemp
++ local LAST_ERR=/tmp/tmp.ygSTbVjCIz
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.OozyxeZlhT
++ cat /tmp/tmp.ygSTbVjCIz
++ rm /tmp/tmp.OozyxeZlhT /tmp/tmp.ygSTbVjCIz
++ return 0
+ [[ ready == \r\e\a\d\y ]]
+ echo .OK
.OK
+ simple_data_check some-name-rs0 3 0
+ local cluster_name=some-name-rs0
+ let last_pod=3-1
+ local isSharded=0
+ local cluster_pfx=
+ '[' 0 -eq 1 ']'
++ seq 0 2
+ for i in $(seq 0 $last_pod)
+ compare_mongo_cmd find myApp:myPass@some-name-rs0-0.some-name-rs0.demand-backup-3000
+ local command=find
+ local uri=myApp:myPass@some-name-rs0-0.some-name-rs0.demand-backup-3000
+ local postfix=
+ local suffix=
+ local database=myApp
+ local collection=test
+ local sort=
+ local tls=false
+ local 'full_command=db.test.find()'
+ [[ -n '' ]]
+ log 'running db.test.find() in myApp'
+ set +o xtrace
[2025-11-03T02:18:14+0000] running db.test.find() in myApp
+ [[ false == \t\r\u\e ]]
+ mongo_command=run_mongo
+ run_mongo 'use myApp\n db.test.find()' myApp:myPass@some-name-rs0-0.some-name-rs0.demand-backup-3000 mongodb ''
+ local 'command=use myApp\n db.test.find()'
+ local uri=myApp:myPass@some-name-rs0-0.some-name-rs0.demand-backup-3000
+ egrep -v 'I NETWORK|W NETWORK|F NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|Started a new thread for the timer service'
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
egrep: warning: egrep is obsolescent; using grep -E
++ local LAST_OUT=/tmp/tmp.dDXlK9Qx0V
+++ mktemp
++ local LAST_ERR=/tmp/tmp.TczJY6glOQ
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.dDXlK9Qx0V
++ cat /tmp/tmp.TczJY6glOQ
++ rm /tmp/tmp.dDXlK9Qx0V /tmp/tmp.TczJY6glOQ
++ return 0
+ local client_container=psmdb-client-696897d69b-7ct8v
+ local mongo_flag=
+ [[ myApp:myPass@some-name-rs0-0.some-name-rs0.demand-backup-3000 == *cfg* ]]
+ replica_set=rs0
+ kubectl_bin exec psmdb-client-696897d69b-7ct8v -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-0.some-name-rs0.demand-backup-3000.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
++ mktemp
+ local LAST_OUT=/tmp/tmp.eAjhKcr9gt
++ mktemp
+ local LAST_ERR=/tmp/tmp.HgTaqAXdzK
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-696897d69b-7ct8v -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-0.some-name-rs0.demand-backup-3000.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.eAjhKcr9gt
+ cat /tmp/tmp.HgTaqAXdzK
+ rm /tmp/tmp.eAjhKcr9gt /tmp/tmp.HgTaqAXdzK
+ return 0
+ [[ 0 -eq 0 ]]
+ diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2101/e2e-tests/demand-backup/compare/find.json /tmp/tmp.NAG1atx8KS/find
+ for i in $(seq 0 $last_pod)
+ compare_mongo_cmd find myApp:myPass@some-name-rs0-1.some-name-rs0.demand-backup-3000
+ local command=find
+ local uri=myApp:myPass@some-name-rs0-1.some-name-rs0.demand-backup-3000
+ local postfix=
+ local suffix=
+ local database=myApp
+ local collection=test
+ local sort=
+ local tls=false
+ local 'full_command=db.test.find()'
+ [[ -n '' ]]
+ log 'running db.test.find() in myApp'
+ set +o xtrace
[2025-11-03T02:18:16+0000] running db.test.find() in myApp
+ [[ false == \t\r\u\e ]]
+ mongo_command=run_mongo
+ run_mongo 'use myApp\n db.test.find()' myApp:myPass@some-name-rs0-1.some-name-rs0.demand-backup-3000 mongodb ''
+ local 'command=use myApp\n db.test.find()'
+ local uri=myApp:myPass@some-name-rs0-1.some-name-rs0.demand-backup-3000
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ egrep -v 'I NETWORK|W NETWORK|F NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|Started a new thread for the timer service'
+ /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
egrep: warning: egrep is obsolescent; using grep -E
++ local LAST_OUT=/tmp/tmp.yrVK5YoH5D
+++ mktemp
++ local LAST_ERR=/tmp/tmp.zXMWg03iEc
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.yrVK5YoH5D
++ cat /tmp/tmp.zXMWg03iEc
++ rm /tmp/tmp.yrVK5YoH5D /tmp/tmp.zXMWg03iEc
++ return 0
+ local client_container=psmdb-client-696897d69b-7ct8v
+ local mongo_flag=
+ [[ myApp:myPass@some-name-rs0-1.some-name-rs0.demand-backup-3000 == *cfg* ]]
+ replica_set=rs0
+ kubectl_bin exec psmdb-client-696897d69b-7ct8v -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-1.some-name-rs0.demand-backup-3000.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
++ mktemp
+ local LAST_OUT=/tmp/tmp.rahrh3Rk63
++ mktemp
+ local LAST_ERR=/tmp/tmp.RHUykkZB13
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-696897d69b-7ct8v -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-1.some-name-rs0.demand-backup-3000.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.rahrh3Rk63
+ cat /tmp/tmp.RHUykkZB13
+ rm /tmp/tmp.rahrh3Rk63 /tmp/tmp.RHUykkZB13
+ return 0
+ [[ 0 -eq 0 ]]
+ diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2101/e2e-tests/demand-backup/compare/find.json /tmp/tmp.NAG1atx8KS/find
+ for i in $(seq 0 $last_pod)
+ compare_mongo_cmd find myApp:myPass@some-name-rs0-2.some-name-rs0.demand-backup-3000
+ local command=find
+ local uri=myApp:myPass@some-name-rs0-2.some-name-rs0.demand-backup-3000
+ local postfix=
+ local suffix=
+ local database=myApp
+ local collection=test
+ local sort=
+ local tls=false
+ local 'full_command=db.test.find()'
+ [[ -n '' ]]
+ log 'running db.test.find() in myApp'
+ set +o xtrace
[2025-11-03T02:18:18+0000] running db.test.find() in myApp
+ [[ false == \t\r\u\e ]]
+ mongo_command=run_mongo
+ /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
+ run_mongo 'use myApp\n db.test.find()' myApp:myPass@some-name-rs0-2.some-name-rs0.demand-backup-3000 mongodb ''
+ local 'command=use myApp\n db.test.find()'
+ local uri=myApp:myPass@some-name-rs0-2.some-name-rs0.demand-backup-3000
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ egrep -v 'I NETWORK|W NETWORK|F NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|Started a new thread for the timer service'
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
egrep: warning: egrep is obsolescent; using grep -E
++ local LAST_OUT=/tmp/tmp.7BZSxBvv0T
+++ mktemp
++ local LAST_ERR=/tmp/tmp.cvL7VXACL6
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.7BZSxBvv0T
++ cat /tmp/tmp.cvL7VXACL6
++ rm /tmp/tmp.7BZSxBvv0T /tmp/tmp.cvL7VXACL6
++ return 0
+ local client_container=psmdb-client-696897d69b-7ct8v
+ local mongo_flag=
+ [[ myApp:myPass@some-name-rs0-2.some-name-rs0.demand-backup-3000 == *cfg* ]]
+ replica_set=rs0
+ kubectl_bin exec psmdb-client-696897d69b-7ct8v -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-2.some-name-rs0.demand-backup-3000.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
++ mktemp
+ local LAST_OUT=/tmp/tmp.SAQTb1VXeC
++ mktemp
+ local LAST_ERR=/tmp/tmp.ZdzcuWrsoD
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-696897d69b-7ct8v -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-2.some-name-rs0.demand-backup-3000.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.SAQTb1VXeC
+ cat /tmp/tmp.ZdzcuWrsoD
+ rm /tmp/tmp.SAQTb1VXeC /tmp/tmp.ZdzcuWrsoD
+ return 0
+ [[ 0 -eq 0 ]]
+ diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2101/e2e-tests/demand-backup/compare/find.json /tmp/tmp.NAG1atx8KS/find
+ run_mongo 'use myApp\n db.dropUser("test1user")' userAdmin:userAdmin123456@some-name-rs0.demand-backup-3000
+ local 'command=use myApp\n db.dropUser("test1user")'
+ local uri=userAdmin:userAdmin123456@some-name-rs0.demand-backup-3000
+ local driver=mongodb+srv
+ local suffix=.svc.cluster.local
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.xIXAWNup9d
+++ mktemp
++ local LAST_ERR=/tmp/tmp.H1r3adTzLd
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.xIXAWNup9d
++ cat /tmp/tmp.H1r3adTzLd
++ rm /tmp/tmp.xIXAWNup9d /tmp/tmp.H1r3adTzLd
++ return 0
+ local client_container=psmdb-client-696897d69b-7ct8v
+ local mongo_flag=
+ [[ userAdmin:userAdmin123456@some-name-rs0.demand-backup-3000 == *cfg* ]]
+ replica_set=rs0
+ kubectl_bin exec psmdb-client-696897d69b-7ct8v -- bash -c 'printf '\''use myApp\n db.dropUser("test1user")\n'\'' | mongo mongodb+srv://userAdmin:userAdmin123456@some-name-rs0.demand-backup-3000.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
++ mktemp
+ local LAST_OUT=/tmp/tmp.7RbM5jZV1l
++ mktemp
+ local LAST_ERR=/tmp/tmp.qcRVh85kI8
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-696897d69b-7ct8v -- bash -c 'printf '\''use myApp\n db.dropUser("test1user")\n'\'' | mongo mongodb+srv://userAdmin:userAdmin123456@some-name-rs0.demand-backup-3000.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.7RbM5jZV1l
Percona Server for MongoDB shell version v4.4.29-28
connecting to: mongodb://some-name-rs0-2.some-name-rs0.demand-backup-3000.svc.cluster.local:27017,some-name-rs0-0.some-name-rs0.demand-backup-3000.svc.cluster.local:27017,some-name-rs0-1.some-name-rs0.demand-backup-3000.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb&replicaSet=rs0&ssl=false
Implicit session: session { "id" : UUID("7b504e09-d0ff-4f4f-8175-7a95874c441b") }
Percona Server for MongoDB server version: v7.0.24-13
WARNING: shell and server versions do not match
switched to db myApp
true
bye
+ cat /tmp/tmp.qcRVh85kI8
+ rm /tmp/tmp.7RbM5jZV1l /tmp/tmp.qcRVh85kI8
+ return 0
+ run_mongo 'use myApp\n db.dropRole("test1role")' userAdmin:userAdmin123456@some-name-rs0.demand-backup-3000
+ local 'command=use myApp\n db.dropRole("test1role")'
+ local uri=userAdmin:userAdmin123456@some-name-rs0.demand-backup-3000
+ local driver=mongodb+srv
+ local suffix=.svc.cluster.local
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.7nWiUgoPt8
+++ mktemp
++ local LAST_ERR=/tmp/tmp.wZToS9zKBH
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.7nWiUgoPt8
++ cat /tmp/tmp.wZToS9zKBH
++ rm /tmp/tmp.7nWiUgoPt8 /tmp/tmp.wZToS9zKBH
++ return 0
+ local client_container=psmdb-client-696897d69b-7ct8v
+ local mongo_flag=
+ [[ userAdmin:userAdmin123456@some-name-rs0.demand-backup-3000 == *cfg* ]]
+ replica_set=rs0
+ kubectl_bin exec psmdb-client-696897d69b-7ct8v -- bash -c 'printf '\''use myApp\n db.dropRole("test1role")\n'\'' | mongo mongodb+srv://userAdmin:userAdmin123456@some-name-rs0.demand-backup-3000.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
++ mktemp
+ local LAST_OUT=/tmp/tmp.BCgw2oZUcd
++ mktemp
+ local LAST_ERR=/tmp/tmp.GYRhPOXzuU
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-696897d69b-7ct8v -- bash -c 'printf '\''use myApp\n db.dropRole("test1role")\n'\'' | mongo mongodb+srv://userAdmin:userAdmin123456@some-name-rs0.demand-backup-3000.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.BCgw2oZUcd
Percona Server for MongoDB shell version v4.4.29-28
connecting to: mongodb://some-name-rs0-1.some-name-rs0.demand-backup-3000.svc.cluster.local:27017,some-name-rs0-2.some-name-rs0.demand-backup-3000.svc.cluster.local:27017,some-name-rs0-0.some-name-rs0.demand-backup-3000.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb&replicaSet=rs0&ssl=false
Implicit session: session { "id" : UUID("d5ec6cac-02f5-4a0d-bf09-cf9230c163de") }
Percona Server for MongoDB server version: v7.0.24-13
WARNING: shell and server versions do not match
switched to db myApp
true
bye
+ cat /tmp/tmp.GYRhPOXzuU
+ rm /tmp/tmp.BCgw2oZUcd /tmp/tmp.GYRhPOXzuU
+ return 0
+ desc 'selective restore -- minio'
+ set +o xtrace


-----------------------------------------------------------------------------------
selective restore -- minio
-----------------------------------------------------------------------------------

+ run_recovery_check_selective backup-minio some-name-rs0 myApp test2 false test1user test1role
+ local backup=backup-minio
+ local cluster=some-name-rs0
+ local database=myApp
+ local collection=test2
+ local with_users_and_roles=false
+ local username=test1user
+ local role=test1role
+ restore_name=restore-backup-minio-selective
+ run_mongo 'use myApp\n db.test2.drop()' myApp:myPass@some-name-rs0.demand-backup-3000
+ local 'command=use myApp\n db.test2.drop()'
+ local uri=myApp:myPass@some-name-rs0.demand-backup-3000
+ local driver=mongodb+srv
+ local suffix=.svc.cluster.local
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.2Gnz69Fz4L
+++ mktemp
++ local LAST_ERR=/tmp/tmp.jmcu6L7NwG
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.2Gnz69Fz4L
++ cat /tmp/tmp.jmcu6L7NwG
++ rm /tmp/tmp.2Gnz69Fz4L /tmp/tmp.jmcu6L7NwG
++ return 0
+ local client_container=psmdb-client-696897d69b-7ct8v
+ local mongo_flag=
+ [[ myApp:myPass@some-name-rs0.demand-backup-3000 == *cfg* ]]
+ replica_set=rs0
+ kubectl_bin exec psmdb-client-696897d69b-7ct8v -- bash -c 'printf '\''use myApp\n db.test2.drop()\n'\'' | mongo mongodb+srv://myApp:myPass@some-name-rs0.demand-backup-3000.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
++ mktemp
+ local LAST_OUT=/tmp/tmp.8C8LFAeaYw
++ mktemp
+ local LAST_ERR=/tmp/tmp.srMnAP26G6
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-696897d69b-7ct8v -- bash -c 'printf '\''use myApp\n db.test2.drop()\n'\'' | mongo mongodb+srv://myApp:myPass@some-name-rs0.demand-backup-3000.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.8C8LFAeaYw
Percona Server for MongoDB shell version v4.4.29-28
connecting to: mongodb://some-name-rs0-2.some-name-rs0.demand-backup-3000.svc.cluster.local:27017,some-name-rs0-0.some-name-rs0.demand-backup-3000.svc.cluster.local:27017,some-name-rs0-1.some-name-rs0.demand-backup-3000.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb&replicaSet=rs0&ssl=false
Implicit session: session { "id" : UUID("4c8e5077-2243-4815-9531-e7129d841e95") }
Percona Server for MongoDB server version: v7.0.24-13
WARNING: shell and server versions do not match
switched to db myApp
true
bye
+ cat /tmp/tmp.srMnAP26G6
+ rm /tmp/tmp.8C8LFAeaYw /tmp/tmp.srMnAP26G6
+ return 0
++ collection_exists test2
++ local collection=test2
++ run_mongo 'use myApp\n JSON.stringify(db.getCollectionNames())' myApp:myPass@some-name-rs0.demand-backup-3000 '' '' --quiet
++ grep -v 'switched to'
++ local 'command=use myApp\n JSON.stringify(db.getCollectionNames())'
++ local uri=myApp:myPass@some-name-rs0.demand-backup-3000
++ local driver=mongodb+srv
++ local suffix=.svc.cluster.local
++ jq 'index("test2") != null'
+++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.h8NrWOibj9
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.FffmiHLQaV
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in $(seq 0 2)
+++ set +e
+++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.h8NrWOibj9
+++ cat /tmp/tmp.FffmiHLQaV
+++ rm /tmp/tmp.h8NrWOibj9 /tmp/tmp.FffmiHLQaV
+++ return 0
++ local client_container=psmdb-client-696897d69b-7ct8v
++ local mongo_flag=--quiet
++ [[ myApp:myPass@some-name-rs0.demand-backup-3000 == *cfg* ]]
++ replica_set=rs0
++ kubectl_bin exec psmdb-client-696897d69b-7ct8v -- bash -c 'printf '\''use myApp\n JSON.stringify(db.getCollectionNames())\n'\'' | mongo mongodb+srv://myApp:myPass@some-name-rs0.demand-backup-3000.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 --quiet'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.5Cv6Gb1b0m
+++ mktemp
++ local LAST_ERR=/tmp/tmp.HSX3dhxeNo
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl exec psmdb-client-696897d69b-7ct8v -- bash -c 'printf '\''use myApp\n JSON.stringify(db.getCollectionNames())\n'\'' | mongo mongodb+srv://myApp:myPass@some-name-rs0.demand-backup-3000.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 --quiet'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.5Cv6Gb1b0m
++ cat /tmp/tmp.HSX3dhxeNo
++ rm /tmp/tmp.5Cv6Gb1b0m /tmp/tmp.HSX3dhxeNo
++ return 0
+ [[ false == \t\r\u\e ]]
+ yq /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2101/e2e-tests/demand-backup/conf/restore.yml
+ yq '.metadata.name="restore-backup-minio-selective"'
+ yq '.spec.backupName="backup-minio"'
+ yq '.spec.selective.namespaces[0]="myApp.test"'
+ yq .spec.selective.withUsersAndRoles=false
+ kubectl_bin apply -f -
++ mktemp
+ local LAST_OUT=/tmp/tmp.tVcSWSq5b3
++ mktemp
+ local LAST_ERR=/tmp/tmp.0sVaLg15ar
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl apply -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.tVcSWSq5b3
perconaservermongodbrestore.psmdb.percona.com/restore-backup-minio-selective created
+ cat /tmp/tmp.0sVaLg15ar
+ rm /tmp/tmp.tVcSWSq5b3 /tmp/tmp.0sVaLg15ar
+ return 0
+ wait_restore backup-minio-selective some-name
+ local backup_name=backup-minio-selective
+ local cluster_name=some-name
+ local target_state=ready
+ local wait_cluster_consistency=1
+ local wait_time=1780
+ local ok_if_ready=0
+ set +o xtrace
Waiting for the psmdb-restore/restore-backup-minio-selective object to be created.OK
Waiting psmdb-restore/restore-backup-minio-selective to reach state "ready" .OK after 0 minutes
+ [[ 1 -eq 1 ]]
+ wait_cluster_consistency some-name
+ local cluster_name=some-name
+ local wait_time=32
+ retry=0
+ sleep 7
+ echo -n 'waiting for cluster readyness'
waiting for cluster readyness++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.pjfU2xxaKG
+++ mktemp
++ local LAST_ERR=/tmp/tmp.8AXHcjyebr
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.pjfU2xxaKG
++ cat /tmp/tmp.8AXHcjyebr
++ rm /tmp/tmp.pjfU2xxaKG /tmp/tmp.8AXHcjyebr
++ return 0
+ [[ ready == \r\e\a\d\y ]]
+ echo .OK
.OK
++ collection_exists test2
++ local collection=test2
++ run_mongo 'use myApp\n JSON.stringify(db.getCollectionNames())' myApp:myPass@some-name-rs0.demand-backup-3000 '' '' --quiet
++ grep -v 'switched to'
++ local 'command=use myApp\n JSON.stringify(db.getCollectionNames())'
++ local uri=myApp:myPass@some-name-rs0.demand-backup-3000
++ local driver=mongodb+srv
++ local suffix=.svc.cluster.local
++ jq 'index("test2") != null'
+++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.i3MdmxlhiG
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.D5sxJjxt30
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in $(seq 0 2)
+++ set +e
+++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.i3MdmxlhiG
+++ cat /tmp/tmp.D5sxJjxt30
+++ rm /tmp/tmp.i3MdmxlhiG /tmp/tmp.D5sxJjxt30
+++ return 0
++ local client_container=psmdb-client-696897d69b-7ct8v
++ local mongo_flag=--quiet
++ [[ myApp:myPass@some-name-rs0.demand-backup-3000 == *cfg* ]]
++ replica_set=rs0
++ kubectl_bin exec psmdb-client-696897d69b-7ct8v -- bash -c 'printf '\''use myApp\n JSON.stringify(db.getCollectionNames())\n'\'' | mongo mongodb+srv://myApp:myPass@some-name-rs0.demand-backup-3000.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 --quiet'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.gjv72fLcHZ
+++ mktemp
++ local LAST_ERR=/tmp/tmp.9AG3qvEmkK
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl exec psmdb-client-696897d69b-7ct8v -- bash -c 'printf '\''use myApp\n JSON.stringify(db.getCollectionNames())\n'\'' | mongo mongodb+srv://myApp:myPass@some-name-rs0.demand-backup-3000.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 --quiet'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.gjv72fLcHZ
++ cat /tmp/tmp.9AG3qvEmkK
++ rm /tmp/tmp.gjv72fLcHZ /tmp/tmp.9AG3qvEmkK
++ return 0
+ [[ false == \t\r\u\e ]]
+ [[ false == \t\r\u\e ]]
+ [[ false == \f\a\l\s\e ]]
++ user_exists test1user
++ local username=test1user
++ run_mongo 'use myApp\n JSON.stringify(db.getUsers())' userAdmin:userAdmin123456@some-name-rs0.demand-backup-3000 '' '' --quiet
++ grep -v 'switched to'
++ local 'command=use myApp\n JSON.stringify(db.getUsers())'
++ jq 'any(.[]; ._id==myApp.test1user)'
++ local uri=userAdmin:userAdmin123456@some-name-rs0.demand-backup-3000
++ local driver=mongodb+srv
++ local suffix=.svc.cluster.local
+++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++++ mktemp
jq: error: myApp/0 is not defined at <top-level>, line 1:
any(.[]; ._id==myApp.test1user)               
jq: 1 compile error
+++ local LAST_OUT=/tmp/tmp.OrJmH6Griv
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.Ao2umW5N1E
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in $(seq 0 2)
+++ set +e
+++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.OrJmH6Griv
+++ cat /tmp/tmp.Ao2umW5N1E
+++ rm /tmp/tmp.OrJmH6Griv /tmp/tmp.Ao2umW5N1E
+++ return 0
++ local client_container=psmdb-client-696897d69b-7ct8v
++ local mongo_flag=--quiet
++ [[ userAdmin:userAdmin123456@some-name-rs0.demand-backup-3000 == *cfg* ]]
++ replica_set=rs0
++ kubectl_bin exec psmdb-client-696897d69b-7ct8v -- bash -c 'printf '\''use myApp\n JSON.stringify(db.getUsers())\n'\'' | mongo mongodb+srv://userAdmin:userAdmin123456@some-name-rs0.demand-backup-3000.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 --quiet'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.gXGPv7Wj3u
+++ mktemp
++ local LAST_ERR=/tmp/tmp.dHL4z6vwPC
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl exec psmdb-client-696897d69b-7ct8v -- bash -c 'printf '\''use myApp\n JSON.stringify(db.getUsers())\n'\'' | mongo mongodb+srv://userAdmin:userAdmin123456@some-name-rs0.demand-backup-3000.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 --quiet'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.gXGPv7Wj3u
++ cat /tmp/tmp.dHL4z6vwPC
++ rm /tmp/tmp.gXGPv7Wj3u /tmp/tmp.dHL4z6vwPC
++ return 0
grep: write error: Broken pipe
+ [[ '' == \t\r\u\e ]]
++ role_exists test1role
++ local role=test1role
++ run_mongo 'use myApp\n JSON.stringify(db.getRoles())' userAdmin:userAdmin123456@some-name-rs0.demand-backup-3000 '' '' --quiet
++ grep -v 'switched to'
++ local 'command=use myApp\n JSON.stringify(db.getRoles())'
++ local uri=userAdmin:userAdmin123456@some-name-rs0.demand-backup-3000
++ local driver=mongodb+srv
++ local suffix=.svc.cluster.local
+++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++++ mktemp
++ jq 'any(.[]; ._id==myApp.test1role)'
+++ local LAST_OUT=/tmp/tmp.hjFfXcAcCd
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.AuOVYbClDh
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in $(seq 0 2)
+++ set +e
+++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
jq: error: myApp/0 is not defined at <top-level>, line 1:
any(.[]; ._id==myApp.test1role)               
jq: 1 compile error
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.hjFfXcAcCd
+++ cat /tmp/tmp.AuOVYbClDh
+++ rm /tmp/tmp.hjFfXcAcCd /tmp/tmp.AuOVYbClDh
+++ return 0
++ local client_container=psmdb-client-696897d69b-7ct8v
++ local mongo_flag=--quiet
++ [[ userAdmin:userAdmin123456@some-name-rs0.demand-backup-3000 == *cfg* ]]
++ replica_set=rs0
++ kubectl_bin exec psmdb-client-696897d69b-7ct8v -- bash -c 'printf '\''use myApp\n JSON.stringify(db.getRoles())\n'\'' | mongo mongodb+srv://userAdmin:userAdmin123456@some-name-rs0.demand-backup-3000.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 --quiet'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.9OCrMy2CUy
+++ mktemp
++ local LAST_ERR=/tmp/tmp.7H9s92fQKp
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl exec psmdb-client-696897d69b-7ct8v -- bash -c 'printf '\''use myApp\n JSON.stringify(db.getRoles())\n'\'' | mongo mongodb+srv://userAdmin:userAdmin123456@some-name-rs0.demand-backup-3000.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 --quiet'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.9OCrMy2CUy
++ cat /tmp/tmp.7H9s92fQKp
++ rm /tmp/tmp.9OCrMy2CUy /tmp/tmp.7H9s92fQKp
++ return 0
grep: write error: Broken pipe
+ [[ '' == \t\r\u\e ]]
+ kubectl_bin delete psmdb-restore restore-backup-minio-selective
++ mktemp
+ local LAST_OUT=/tmp/tmp.Z4mHoc1eV7
++ mktemp
+ local LAST_ERR=/tmp/tmp.MODEt8wfRo
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl delete psmdb-restore restore-backup-minio-selective
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.Z4mHoc1eV7
perconaservermongodbrestore.psmdb.percona.com "restore-backup-minio-selective" deleted from demand-backup-3000 namespace
+ cat /tmp/tmp.MODEt8wfRo
+ rm /tmp/tmp.Z4mHoc1eV7 /tmp/tmp.MODEt8wfRo
+ return 0
+ run_mongo 'use myApp\n db.test2.insert({ x: 100501 })' myApp:myPass@some-name-rs0.demand-backup-3000
+ local 'command=use myApp\n db.test2.insert({ x: 100501 })'
+ local uri=myApp:myPass@some-name-rs0.demand-backup-3000
+ local driver=mongodb+srv
+ local suffix=.svc.cluster.local
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.PuWt9c0mkA
+++ mktemp
++ local LAST_ERR=/tmp/tmp.Iv9bsBUimc
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.PuWt9c0mkA
++ cat /tmp/tmp.Iv9bsBUimc
++ rm /tmp/tmp.PuWt9c0mkA /tmp/tmp.Iv9bsBUimc
++ return 0
+ local client_container=psmdb-client-696897d69b-7ct8v
+ local mongo_flag=
+ [[ myApp:myPass@some-name-rs0.demand-backup-3000 == *cfg* ]]
+ replica_set=rs0
+ kubectl_bin exec psmdb-client-696897d69b-7ct8v -- bash -c 'printf '\''use myApp\n db.test2.insert({ x: 100501 })\n'\'' | mongo mongodb+srv://myApp:myPass@some-name-rs0.demand-backup-3000.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
++ mktemp
+ local LAST_OUT=/tmp/tmp.Q3Rl69MumH
++ mktemp
+ local LAST_ERR=/tmp/tmp.r7fXD0O95p
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-696897d69b-7ct8v -- bash -c 'printf '\''use myApp\n db.test2.insert({ x: 100501 })\n'\'' | mongo mongodb+srv://myApp:myPass@some-name-rs0.demand-backup-3000.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.Q3Rl69MumH
Percona Server for MongoDB shell version v4.4.29-28
connecting to: mongodb://some-name-rs0-2.some-name-rs0.demand-backup-3000.svc.cluster.local:27017,some-name-rs0-0.some-name-rs0.demand-backup-3000.svc.cluster.local:27017,some-name-rs0-1.some-name-rs0.demand-backup-3000.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb&replicaSet=rs0&ssl=false
Implicit session: session { "id" : UUID("abfb83c9-e808-4dd2-ae79-bbd1db7e1c20") }
Percona Server for MongoDB server version: v7.0.24-13
WARNING: shell and server versions do not match
switched to db myApp
WriteResult({ "nInserted" : 1 })
bye
+ cat /tmp/tmp.r7fXD0O95p
+ rm /tmp/tmp.Q3Rl69MumH /tmp/tmp.r7fXD0O95p
+ return 0
+ desc 'selective restore with users and roles -- minio'
+ set +o xtrace


-----------------------------------------------------------------------------------
selective restore with users and roles -- minio
-----------------------------------------------------------------------------------

+ run_recovery_check_selective backup-minio some-name-rs0 myApp test2 true test1user test1role
+ local backup=backup-minio
+ local cluster=some-name-rs0
+ local database=myApp
+ local collection=test2
+ local with_users_and_roles=true
+ local username=test1user
+ local role=test1role
+ restore_name=restore-backup-minio-selective
+ run_mongo 'use myApp\n db.test2.drop()' myApp:myPass@some-name-rs0.demand-backup-3000
+ local 'command=use myApp\n db.test2.drop()'
+ local uri=myApp:myPass@some-name-rs0.demand-backup-3000
+ local driver=mongodb+srv
+ local suffix=.svc.cluster.local
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.gUSARN2dLW
+++ mktemp
++ local LAST_ERR=/tmp/tmp.uQ3APCkzFY
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.gUSARN2dLW
++ cat /tmp/tmp.uQ3APCkzFY
++ rm /tmp/tmp.gUSARN2dLW /tmp/tmp.uQ3APCkzFY
++ return 0
+ local client_container=psmdb-client-696897d69b-7ct8v
+ local mongo_flag=
+ [[ myApp:myPass@some-name-rs0.demand-backup-3000 == *cfg* ]]
+ replica_set=rs0
+ kubectl_bin exec psmdb-client-696897d69b-7ct8v -- bash -c 'printf '\''use myApp\n db.test2.drop()\n'\'' | mongo mongodb+srv://myApp:myPass@some-name-rs0.demand-backup-3000.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
++ mktemp
+ local LAST_OUT=/tmp/tmp.sQI5DTzs7m
++ mktemp
+ local LAST_ERR=/tmp/tmp.KHRDDbxNQa
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-696897d69b-7ct8v -- bash -c 'printf '\''use myApp\n db.test2.drop()\n'\'' | mongo mongodb+srv://myApp:myPass@some-name-rs0.demand-backup-3000.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.sQI5DTzs7m
Percona Server for MongoDB shell version v4.4.29-28
connecting to: mongodb://some-name-rs0-0.some-name-rs0.demand-backup-3000.svc.cluster.local:27017,some-name-rs0-1.some-name-rs0.demand-backup-3000.svc.cluster.local:27017,some-name-rs0-2.some-name-rs0.demand-backup-3000.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb&replicaSet=rs0&ssl=false
Implicit session: session { "id" : UUID("60d46dd6-aa2b-4884-8f54-7bf0767b85c5") }
Percona Server for MongoDB server version: v7.0.24-13
WARNING: shell and server versions do not match
switched to db myApp
true
bye
+ cat /tmp/tmp.KHRDDbxNQa
+ rm /tmp/tmp.sQI5DTzs7m /tmp/tmp.KHRDDbxNQa
+ return 0
++ collection_exists test2
++ local collection=test2
++ run_mongo 'use myApp\n JSON.stringify(db.getCollectionNames())' myApp:myPass@some-name-rs0.demand-backup-3000 '' '' --quiet
++ local 'command=use myApp\n JSON.stringify(db.getCollectionNames())'
++ local uri=myApp:myPass@some-name-rs0.demand-backup-3000
++ local driver=mongodb+srv
++ local suffix=.svc.cluster.local
++ grep -v 'switched to'
++ jq 'index("test2") != null'
+++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.0R3Xn5RFdc
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.TnLLlvmJu5
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in $(seq 0 2)
+++ set +e
+++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.0R3Xn5RFdc
+++ cat /tmp/tmp.TnLLlvmJu5
+++ rm /tmp/tmp.0R3Xn5RFdc /tmp/tmp.TnLLlvmJu5
+++ return 0
++ local client_container=psmdb-client-696897d69b-7ct8v
++ local mongo_flag=--quiet
++ [[ myApp:myPass@some-name-rs0.demand-backup-3000 == *cfg* ]]
++ replica_set=rs0
++ kubectl_bin exec psmdb-client-696897d69b-7ct8v -- bash -c 'printf '\''use myApp\n JSON.stringify(db.getCollectionNames())\n'\'' | mongo mongodb+srv://myApp:myPass@some-name-rs0.demand-backup-3000.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 --quiet'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.ExryTbXnm7
+++ mktemp
++ local LAST_ERR=/tmp/tmp.pBZgRvwvgP
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl exec psmdb-client-696897d69b-7ct8v -- bash -c 'printf '\''use myApp\n JSON.stringify(db.getCollectionNames())\n'\'' | mongo mongodb+srv://myApp:myPass@some-name-rs0.demand-backup-3000.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 --quiet'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.ExryTbXnm7
++ cat /tmp/tmp.pBZgRvwvgP
++ rm /tmp/tmp.ExryTbXnm7 /tmp/tmp.pBZgRvwvgP
++ return 0
+ [[ false == \t\r\u\e ]]
+ yq /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2101/e2e-tests/demand-backup/conf/restore.yml
+ yq '.metadata.name="restore-backup-minio-selective"'
+ yq '.spec.backupName="backup-minio"'
+ yq '.spec.selective.namespaces[0]="myApp.test"'
+ yq .spec.selective.withUsersAndRoles=true
+ kubectl_bin apply -f -
++ mktemp
+ local LAST_OUT=/tmp/tmp.ua8V9dQPxg
++ mktemp
+ local LAST_ERR=/tmp/tmp.WTufDsgQU4
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl apply -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.ua8V9dQPxg
perconaservermongodbrestore.psmdb.percona.com/restore-backup-minio-selective created
+ cat /tmp/tmp.WTufDsgQU4
+ rm /tmp/tmp.ua8V9dQPxg /tmp/tmp.WTufDsgQU4
+ return 0
+ wait_restore backup-minio-selective some-name
+ local backup_name=backup-minio-selective
+ local cluster_name=some-name
+ local target_state=ready
+ local wait_cluster_consistency=1
+ local wait_time=1780
+ local ok_if_ready=0
+ set +o xtrace
Waiting for the psmdb-restore/restore-backup-minio-selective object to be created.OK
Waiting psmdb-restore/restore-backup-minio-selective to reach state "ready" .OK after 0 minutes
+ [[ 1 -eq 1 ]]
+ wait_cluster_consistency some-name
+ local cluster_name=some-name
+ local wait_time=32
+ retry=0
+ sleep 7
+ echo -n 'waiting for cluster readyness'
waiting for cluster readyness++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.ylTTUWYeX8
+++ mktemp
++ local LAST_ERR=/tmp/tmp.2hdUYxi50x
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.ylTTUWYeX8
++ cat /tmp/tmp.2hdUYxi50x
++ rm /tmp/tmp.ylTTUWYeX8 /tmp/tmp.2hdUYxi50x
++ return 0
+ [[ ready == \r\e\a\d\y ]]
+ echo .OK
.OK
++ collection_exists test2
++ local collection=test2
++ run_mongo 'use myApp\n JSON.stringify(db.getCollectionNames())' myApp:myPass@some-name-rs0.demand-backup-3000 '' '' --quiet
++ grep -v 'switched to'
++ local 'command=use myApp\n JSON.stringify(db.getCollectionNames())'
++ local uri=myApp:myPass@some-name-rs0.demand-backup-3000
++ local driver=mongodb+srv
++ local suffix=.svc.cluster.local
++ jq 'index("test2") != null'
+++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.70s9dtF979
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.clIHroHYad
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in $(seq 0 2)
+++ set +e
+++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.70s9dtF979
+++ cat /tmp/tmp.clIHroHYad
+++ rm /tmp/tmp.70s9dtF979 /tmp/tmp.clIHroHYad
+++ return 0
++ local client_container=psmdb-client-696897d69b-7ct8v
++ local mongo_flag=--quiet
++ [[ myApp:myPass@some-name-rs0.demand-backup-3000 == *cfg* ]]
++ replica_set=rs0
++ kubectl_bin exec psmdb-client-696897d69b-7ct8v -- bash -c 'printf '\''use myApp\n JSON.stringify(db.getCollectionNames())\n'\'' | mongo mongodb+srv://myApp:myPass@some-name-rs0.demand-backup-3000.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 --quiet'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.xSvhYWjIWe
+++ mktemp
++ local LAST_ERR=/tmp/tmp.QFlTC77Kdp
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl exec psmdb-client-696897d69b-7ct8v -- bash -c 'printf '\''use myApp\n JSON.stringify(db.getCollectionNames())\n'\'' | mongo mongodb+srv://myApp:myPass@some-name-rs0.demand-backup-3000.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 --quiet'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.xSvhYWjIWe
++ cat /tmp/tmp.QFlTC77Kdp
++ rm /tmp/tmp.xSvhYWjIWe /tmp/tmp.QFlTC77Kdp
++ return 0
+ [[ false == \t\r\u\e ]]
+ [[ true == \t\r\u\e ]]
++ user_exists test1user
++ local username=test1user
++ run_mongo 'use myApp\n JSON.stringify(db.getUsers())' userAdmin:userAdmin123456@some-name-rs0.demand-backup-3000 '' '' --quiet
++ grep -v 'switched to'
++ local 'command=use myApp\n JSON.stringify(db.getUsers())'
++ local uri=userAdmin:userAdmin123456@some-name-rs0.demand-backup-3000
++ local driver=mongodb+srv
++ local suffix=.svc.cluster.local
++ jq 'any(.[]; ._id==myApp.test1user)'
+++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.tRUlXhbK06
jq: error: myApp/0 is not defined at <top-level>, line 1:
any(.[]; ._id==myApp.test1user)               
jq: 1 compile error
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.EyhyEj660t
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in $(seq 0 2)
+++ set +e
+++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.tRUlXhbK06
+++ cat /tmp/tmp.EyhyEj660t
+++ rm /tmp/tmp.tRUlXhbK06 /tmp/tmp.EyhyEj660t
+++ return 0
++ local client_container=psmdb-client-696897d69b-7ct8v
++ local mongo_flag=--quiet
++ [[ userAdmin:userAdmin123456@some-name-rs0.demand-backup-3000 == *cfg* ]]
++ replica_set=rs0
++ kubectl_bin exec psmdb-client-696897d69b-7ct8v -- bash -c 'printf '\''use myApp\n JSON.stringify(db.getUsers())\n'\'' | mongo mongodb+srv://userAdmin:userAdmin123456@some-name-rs0.demand-backup-3000.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 --quiet'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.MrhpNdYGRc
+++ mktemp
++ local LAST_ERR=/tmp/tmp.PUAnLjLsBr
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl exec psmdb-client-696897d69b-7ct8v -- bash -c 'printf '\''use myApp\n JSON.stringify(db.getUsers())\n'\'' | mongo mongodb+srv://userAdmin:userAdmin123456@some-name-rs0.demand-backup-3000.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 --quiet'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.MrhpNdYGRc
++ cat /tmp/tmp.PUAnLjLsBr
++ rm /tmp/tmp.MrhpNdYGRc /tmp/tmp.PUAnLjLsBr
++ return 0
grep: write error: Broken pipe
+ [[ '' == \f\a\l\s\e ]]
++ role_exists test1role
++ local role=test1role
++ run_mongo 'use myApp\n JSON.stringify(db.getRoles())' userAdmin:userAdmin123456@some-name-rs0.demand-backup-3000 '' '' --quiet
++ grep -v 'switched to'
++ local 'command=use myApp\n JSON.stringify(db.getRoles())'
++ local uri=userAdmin:userAdmin123456@some-name-rs0.demand-backup-3000
++ local driver=mongodb+srv
++ local suffix=.svc.cluster.local
++ jq 'any(.[]; ._id==myApp.test1role)'
+++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.sOxIqVTQ40
jq: error: myApp/0 is not defined at <top-level>, line 1:
any(.[]; ._id==myApp.test1role)               
jq: 1 compile error
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.5fS6jBVRxZ
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in $(seq 0 2)
+++ set +e
+++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.sOxIqVTQ40
+++ cat /tmp/tmp.5fS6jBVRxZ
+++ rm /tmp/tmp.sOxIqVTQ40 /tmp/tmp.5fS6jBVRxZ
+++ return 0
++ local client_container=psmdb-client-696897d69b-7ct8v
++ local mongo_flag=--quiet
++ [[ userAdmin:userAdmin123456@some-name-rs0.demand-backup-3000 == *cfg* ]]
++ replica_set=rs0
++ kubectl_bin exec psmdb-client-696897d69b-7ct8v -- bash -c 'printf '\''use myApp\n JSON.stringify(db.getRoles())\n'\'' | mongo mongodb+srv://userAdmin:userAdmin123456@some-name-rs0.demand-backup-3000.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 --quiet'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.kZfrzi0Kfx
+++ mktemp
++ local LAST_ERR=/tmp/tmp.lmE0SUVhvF
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl exec psmdb-client-696897d69b-7ct8v -- bash -c 'printf '\''use myApp\n JSON.stringify(db.getRoles())\n'\'' | mongo mongodb+srv://userAdmin:userAdmin123456@some-name-rs0.demand-backup-3000.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 --quiet'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.kZfrzi0Kfx
++ cat /tmp/tmp.lmE0SUVhvF
++ rm /tmp/tmp.kZfrzi0Kfx /tmp/tmp.lmE0SUVhvF
++ return 0
grep: write error: Broken pipe
+ [[ '' == \f\a\l\s\e ]]
+ desc 'restore from backup source, with storageName -- minio'
+ set +o xtrace


-----------------------------------------------------------------------------------
restore from backup source, with storageName -- minio
-----------------------------------------------------------------------------------

++ get_backup_dest backup-minio
++ local backup_name=backup-minio
++ kubectl_bin get psmdb-backup backup-minio -o 'jsonpath={.status.destination}'
++ sed -e 's/.json$//'
++ sed 's|s3://||'
++ sed 's|azure://||'
+++ mktemp
++ sed 's|gs://||'
++ local LAST_OUT=/tmp/tmp.THsc3UwyPh
+++ mktemp
++ local LAST_ERR=/tmp/tmp.8akxWiQQqi
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb-backup backup-minio -o 'jsonpath={.status.destination}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.THsc3UwyPh
++ cat /tmp/tmp.8akxWiQQqi
++ rm /tmp/tmp.THsc3UwyPh /tmp/tmp.8akxWiQQqi
++ return 0
+ backup_dest_minio=operator-testing/2025-11-03T02:14:26Z
+ run_recovery_check_bkp_source backup-minio operator-testing/2025-11-03T02:14:26Z some-name-rs0 backup-minio-source-0
+ local backup=backup-minio
+ local backup_dest=operator-testing/2025-11-03T02:14:26Z
+ local cluster=some-name-rs0
+ local source=backup-minio-source-0
+ run_mongo 'use myApp\n db.test.insert({ x: 100501 })' myApp:myPass@some-name-rs0.demand-backup-3000
+ local 'command=use myApp\n db.test.insert({ x: 100501 })'
+ local uri=myApp:myPass@some-name-rs0.demand-backup-3000
+ local driver=mongodb+srv
+ local suffix=.svc.cluster.local
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.i0KBptsb8y
+++ mktemp
++ local LAST_ERR=/tmp/tmp.Jxp0aB7MgS
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.i0KBptsb8y
++ cat /tmp/tmp.Jxp0aB7MgS
++ rm /tmp/tmp.i0KBptsb8y /tmp/tmp.Jxp0aB7MgS
++ return 0
+ local client_container=psmdb-client-696897d69b-7ct8v
+ local mongo_flag=
+ [[ myApp:myPass@some-name-rs0.demand-backup-3000 == *cfg* ]]
+ replica_set=rs0
+ kubectl_bin exec psmdb-client-696897d69b-7ct8v -- bash -c 'printf '\''use myApp\n db.test.insert({ x: 100501 })\n'\'' | mongo mongodb+srv://myApp:myPass@some-name-rs0.demand-backup-3000.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
++ mktemp
+ local LAST_OUT=/tmp/tmp.xeGzsjNaON
++ mktemp
+ local LAST_ERR=/tmp/tmp.f5lANI0MuD
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-696897d69b-7ct8v -- bash -c 'printf '\''use myApp\n db.test.insert({ x: 100501 })\n'\'' | mongo mongodb+srv://myApp:myPass@some-name-rs0.demand-backup-3000.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.xeGzsjNaON
Percona Server for MongoDB shell version v4.4.29-28
connecting to: mongodb://some-name-rs0-1.some-name-rs0.demand-backup-3000.svc.cluster.local:27017,some-name-rs0-2.some-name-rs0.demand-backup-3000.svc.cluster.local:27017,some-name-rs0-0.some-name-rs0.demand-backup-3000.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb&replicaSet=rs0&ssl=false
Implicit session: session { "id" : UUID("a8cc05f6-b710-4adc-bb68-a7a0167cf155") }
Percona Server for MongoDB server version: v7.0.24-13
WARNING: shell and server versions do not match
switched to db myApp
WriteResult({ "nInserted" : 1 })
bye
+ cat /tmp/tmp.f5lANI0MuD
+ rm /tmp/tmp.xeGzsjNaON /tmp/tmp.f5lANI0MuD
+ return 0
+ compare_mongo_cmd find myApp:myPass@some-name-rs0.demand-backup-3000 -2nd .svc.cluster.local myApp test
+ local command=find
+ local uri=myApp:myPass@some-name-rs0.demand-backup-3000
+ local postfix=-2nd
+ local suffix=.svc.cluster.local
+ local database=myApp
+ local collection=test
+ local sort=
+ local tls=false
+ local 'full_command=db.test.find()'
+ [[ -n '' ]]
+ log 'running db.test.find() in myApp'
+ set +o xtrace
[2025-11-03T02:19:41+0000] running db.test.find() in myApp
+ [[ false == \t\r\u\e ]]
+ mongo_command=run_mongo
+ run_mongo 'use myApp\n db.test.find()' myApp:myPass@some-name-rs0.demand-backup-3000 mongodb .svc.cluster.local
+ local 'command=use myApp\n db.test.find()'
+ local uri=myApp:myPass@some-name-rs0.demand-backup-3000
+ egrep -v 'I NETWORK|W NETWORK|F NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|Started a new thread for the timer service'
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
egrep: warning: egrep is obsolescent; using grep -E
++ local LAST_OUT=/tmp/tmp.UREzYtzOpD
+++ mktemp
++ local LAST_ERR=/tmp/tmp.Bb9VcEJrR6
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.UREzYtzOpD
++ cat /tmp/tmp.Bb9VcEJrR6
++ rm /tmp/tmp.UREzYtzOpD /tmp/tmp.Bb9VcEJrR6
++ return 0
+ local client_container=psmdb-client-696897d69b-7ct8v
+ local mongo_flag=
+ [[ myApp:myPass@some-name-rs0.demand-backup-3000 == *cfg* ]]
+ replica_set=rs0
+ kubectl_bin exec psmdb-client-696897d69b-7ct8v -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0.demand-backup-3000.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
++ mktemp
+ local LAST_OUT=/tmp/tmp.vMpmosOmpC
++ mktemp
+ local LAST_ERR=/tmp/tmp.TPu6TGnO13
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-696897d69b-7ct8v -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0.demand-backup-3000.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.vMpmosOmpC
+ cat /tmp/tmp.TPu6TGnO13
+ rm /tmp/tmp.vMpmosOmpC /tmp/tmp.TPu6TGnO13
+ return 0
+ [[ 0 -eq 0 ]]
+ diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2101/e2e-tests/demand-backup/compare/find-2nd.json /tmp/tmp.NAG1atx8KS/find-2nd
+ run_restore_backupsource backup-minio-source-0 operator-testing/2025-11-03T02:14:26Z
+ local backupName=backup-minio-source-0
+ local backupDest=operator-testing/2025-11-03T02:14:26Z
+ local storageName=
+ desc 'run restore restore-backup-minio-source-0 from backup backup-minio-source-0 destination is operator-testing/2025-11-03T02:14:26Z'
+ set +o xtrace


-----------------------------------------------------------------------------------
run restore restore-backup-minio-source-0 from backup backup-minio-source-0 destination is operator-testing/2025-11-03T02:14:26Z
-----------------------------------------------------------------------------------

+ '[' -z '' ']'
+ cat /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2101/e2e-tests/demand-backup/conf/restore-backupsource.yml
+ /usr/sbin/sed -e 's/name:/name: restore-backup-minio-source-0/'
+ /usr/sbin/sed -e 's|BACKUP-NAME|operator-testing/2025-11-03T02:14:26Z|'
+ /usr/sbin/sed -e /storageName/d
+ kubectl_bin apply -f -
++ mktemp
+ local LAST_OUT=/tmp/tmp.d22rn0BRaP
++ mktemp
+ local LAST_ERR=/tmp/tmp.0ZcJbcYx29
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl apply -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.d22rn0BRaP
perconaservermongodbrestore.psmdb.percona.com/restore-backup-minio-source-0 created
+ cat /tmp/tmp.0ZcJbcYx29
+ rm /tmp/tmp.d22rn0BRaP /tmp/tmp.0ZcJbcYx29
+ return 0
+ return
+ wait_restore backup-minio-source-0 some-name
+ local backup_name=backup-minio-source-0
+ local cluster_name=some-name
+ local target_state=ready
+ local wait_cluster_consistency=1
+ local wait_time=1780
+ local ok_if_ready=0
+ set +o xtrace
Waiting for the psmdb-restore/restore-backup-minio-source-0 object to be created.OK
Waiting psmdb-restore/restore-backup-minio-source-0 to reach state "ready" .OK after 0 minutes
+ [[ 1 -eq 1 ]]
+ wait_cluster_consistency some-name
+ local cluster_name=some-name
+ local wait_time=32
+ retry=0
+ sleep 7
+ echo -n 'waiting for cluster readyness'
waiting for cluster readyness++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.bRWAXBaZXL
+++ mktemp
++ local LAST_ERR=/tmp/tmp.TAEM8BAmon
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.bRWAXBaZXL
++ cat /tmp/tmp.TAEM8BAmon
++ rm /tmp/tmp.bRWAXBaZXL /tmp/tmp.TAEM8BAmon
++ return 0
+ [[ ready == \r\e\a\d\y ]]
+ echo .OK
.OK
+ simple_data_check some-name-rs0 3 0
+ local cluster_name=some-name-rs0
+ let last_pod=3-1
+ local isSharded=0
+ local cluster_pfx=
+ '[' 0 -eq 1 ']'
++ seq 0 2
+ for i in $(seq 0 $last_pod)
+ compare_mongo_cmd find myApp:myPass@some-name-rs0-0.some-name-rs0.demand-backup-3000
+ local command=find
+ local uri=myApp:myPass@some-name-rs0-0.some-name-rs0.demand-backup-3000
+ local postfix=
+ local suffix=
+ local database=myApp
+ local collection=test
+ local sort=
+ local tls=false
+ local 'full_command=db.test.find()'
+ [[ -n '' ]]
+ log 'running db.test.find() in myApp'
+ set +o xtrace
[2025-11-03T02:20:05+0000] running db.test.find() in myApp
+ [[ false == \t\r\u\e ]]
+ mongo_command=run_mongo
+ run_mongo 'use myApp\n db.test.find()' myApp:myPass@some-name-rs0-0.some-name-rs0.demand-backup-3000 mongodb ''
+ local 'command=use myApp\n db.test.find()'
+ local uri=myApp:myPass@some-name-rs0-0.some-name-rs0.demand-backup-3000
+ egrep -v 'I NETWORK|W NETWORK|F NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|Started a new thread for the timer service'
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
egrep: warning: egrep is obsolescent; using grep -E
++ local LAST_OUT=/tmp/tmp.AH4YL1qWwF
+++ mktemp
++ local LAST_ERR=/tmp/tmp.JiHWX11iQf
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.AH4YL1qWwF
++ cat /tmp/tmp.JiHWX11iQf
++ rm /tmp/tmp.AH4YL1qWwF /tmp/tmp.JiHWX11iQf
++ return 0
+ local client_container=psmdb-client-696897d69b-7ct8v
+ local mongo_flag=
+ [[ myApp:myPass@some-name-rs0-0.some-name-rs0.demand-backup-3000 == *cfg* ]]
+ replica_set=rs0
+ kubectl_bin exec psmdb-client-696897d69b-7ct8v -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-0.some-name-rs0.demand-backup-3000.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
++ mktemp
+ local LAST_OUT=/tmp/tmp.lrpikuoGk6
++ mktemp
+ local LAST_ERR=/tmp/tmp.ONWrhd255Z
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-696897d69b-7ct8v -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-0.some-name-rs0.demand-backup-3000.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.lrpikuoGk6
+ cat /tmp/tmp.ONWrhd255Z
+ rm /tmp/tmp.lrpikuoGk6 /tmp/tmp.ONWrhd255Z
+ return 0
+ [[ 0 -eq 0 ]]
+ diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2101/e2e-tests/demand-backup/compare/find.json /tmp/tmp.NAG1atx8KS/find
+ for i in $(seq 0 $last_pod)
+ compare_mongo_cmd find myApp:myPass@some-name-rs0-1.some-name-rs0.demand-backup-3000
+ local command=find
+ local uri=myApp:myPass@some-name-rs0-1.some-name-rs0.demand-backup-3000
+ local postfix=
+ local suffix=
+ local database=myApp
+ local collection=test
+ local sort=
+ local tls=false
+ local 'full_command=db.test.find()'
+ [[ -n '' ]]
+ log 'running db.test.find() in myApp'
+ set +o xtrace
[2025-11-03T02:20:08+0000] running db.test.find() in myApp
+ [[ false == \t\r\u\e ]]
+ mongo_command=run_mongo
+ run_mongo 'use myApp\n db.test.find()' myApp:myPass@some-name-rs0-1.some-name-rs0.demand-backup-3000 mongodb ''
+ local 'command=use myApp\n db.test.find()'
+ local uri=myApp:myPass@some-name-rs0-1.some-name-rs0.demand-backup-3000
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ egrep -v 'I NETWORK|W NETWORK|F NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|Started a new thread for the timer service'
+ /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
egrep: warning: egrep is obsolescent; using grep -E
++ local LAST_OUT=/tmp/tmp.RObuMPUB7E
+++ mktemp
++ local LAST_ERR=/tmp/tmp.aSGJO7RQ37
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.RObuMPUB7E
++ cat /tmp/tmp.aSGJO7RQ37
++ rm /tmp/tmp.RObuMPUB7E /tmp/tmp.aSGJO7RQ37
++ return 0
+ local client_container=psmdb-client-696897d69b-7ct8v
+ local mongo_flag=
+ [[ myApp:myPass@some-name-rs0-1.some-name-rs0.demand-backup-3000 == *cfg* ]]
+ replica_set=rs0
+ kubectl_bin exec psmdb-client-696897d69b-7ct8v -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-1.some-name-rs0.demand-backup-3000.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
++ mktemp
+ local LAST_OUT=/tmp/tmp.13BcIyqFTO
++ mktemp
+ local LAST_ERR=/tmp/tmp.HsrxnuGlT5
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-696897d69b-7ct8v -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-1.some-name-rs0.demand-backup-3000.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.13BcIyqFTO
+ cat /tmp/tmp.HsrxnuGlT5
+ rm /tmp/tmp.13BcIyqFTO /tmp/tmp.HsrxnuGlT5
+ return 0
+ [[ 0 -eq 0 ]]
+ diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2101/e2e-tests/demand-backup/compare/find.json /tmp/tmp.NAG1atx8KS/find
+ for i in $(seq 0 $last_pod)
+ compare_mongo_cmd find myApp:myPass@some-name-rs0-2.some-name-rs0.demand-backup-3000
+ local command=find
+ local uri=myApp:myPass@some-name-rs0-2.some-name-rs0.demand-backup-3000
+ local postfix=
+ local suffix=
+ local database=myApp
+ local collection=test
+ local sort=
+ local tls=false
+ local 'full_command=db.test.find()'
+ [[ -n '' ]]
+ log 'running db.test.find() in myApp'
+ set +o xtrace
[2025-11-03T02:20:10+0000] running db.test.find() in myApp
+ [[ false == \t\r\u\e ]]
+ mongo_command=run_mongo
+ run_mongo 'use myApp\n db.test.find()' myApp:myPass@some-name-rs0-2.some-name-rs0.demand-backup-3000 mongodb ''
+ local 'command=use myApp\n db.test.find()'
+ local uri=myApp:myPass@some-name-rs0-2.some-name-rs0.demand-backup-3000
+ egrep -v 'I NETWORK|W NETWORK|F NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|Started a new thread for the timer service'
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
egrep: warning: egrep is obsolescent; using grep -E
++ local LAST_OUT=/tmp/tmp.uoZ71O4mdf
+++ mktemp
++ local LAST_ERR=/tmp/tmp.atgUzGpj9w
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.uoZ71O4mdf
++ cat /tmp/tmp.atgUzGpj9w
++ rm /tmp/tmp.uoZ71O4mdf /tmp/tmp.atgUzGpj9w
++ return 0
+ local client_container=psmdb-client-696897d69b-7ct8v
+ local mongo_flag=
+ [[ myApp:myPass@some-name-rs0-2.some-name-rs0.demand-backup-3000 == *cfg* ]]
+ replica_set=rs0
+ kubectl_bin exec psmdb-client-696897d69b-7ct8v -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-2.some-name-rs0.demand-backup-3000.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
++ mktemp
+ local LAST_OUT=/tmp/tmp.jSwbygpA1V
++ mktemp
+ local LAST_ERR=/tmp/tmp.2ezyDdL2C7
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-696897d69b-7ct8v -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-2.some-name-rs0.demand-backup-3000.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.jSwbygpA1V
+ cat /tmp/tmp.2ezyDdL2C7
+ rm /tmp/tmp.jSwbygpA1V /tmp/tmp.2ezyDdL2C7
+ return 0
+ [[ 0 -eq 0 ]]
+ diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2101/e2e-tests/demand-backup/compare/find.json /tmp/tmp.NAG1atx8KS/find
+ desc 'restore from backup source, no storageName -- minio'
+ set +o xtrace


-----------------------------------------------------------------------------------
restore from backup source, no storageName -- minio
-----------------------------------------------------------------------------------

++ get_backup_dest backup-minio
++ local backup_name=backup-minio
++ kubectl_bin get psmdb-backup backup-minio -o 'jsonpath={.status.destination}'
++ sed -e 's/.json$//'
++ sed 's|s3://||'
+++ mktemp
++ sed 's|azure://||'
++ sed 's|gs://||'
++ local LAST_OUT=/tmp/tmp.wlet7XWlxV
+++ mktemp
++ local LAST_ERR=/tmp/tmp.kqXvHSU82J
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb-backup backup-minio -o 'jsonpath={.status.destination}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.wlet7XWlxV
++ cat /tmp/tmp.kqXvHSU82J
++ rm /tmp/tmp.wlet7XWlxV /tmp/tmp.kqXvHSU82J
++ return 0
+ backup_dest_minio=operator-testing/2025-11-03T02:14:26Z
+ run_recovery_check_bkp_source backup-minio operator-testing/2025-11-03T02:14:26Z some-name-rs0 backup-minio-source-1
+ local backup=backup-minio
+ local backup_dest=operator-testing/2025-11-03T02:14:26Z
+ local cluster=some-name-rs0
+ local source=backup-minio-source-1
+ run_mongo 'use myApp\n db.test.insert({ x: 100501 })' myApp:myPass@some-name-rs0.demand-backup-3000
+ local 'command=use myApp\n db.test.insert({ x: 100501 })'
+ local uri=myApp:myPass@some-name-rs0.demand-backup-3000
+ local driver=mongodb+srv
+ local suffix=.svc.cluster.local
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.i652jjhyCJ
+++ mktemp
++ local LAST_ERR=/tmp/tmp.Gui9KYsrqD
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.i652jjhyCJ
++ cat /tmp/tmp.Gui9KYsrqD
++ rm /tmp/tmp.i652jjhyCJ /tmp/tmp.Gui9KYsrqD
++ return 0
+ local client_container=psmdb-client-696897d69b-7ct8v
+ local mongo_flag=
+ [[ myApp:myPass@some-name-rs0.demand-backup-3000 == *cfg* ]]
+ replica_set=rs0
+ kubectl_bin exec psmdb-client-696897d69b-7ct8v -- bash -c 'printf '\''use myApp\n db.test.insert({ x: 100501 })\n'\'' | mongo mongodb+srv://myApp:myPass@some-name-rs0.demand-backup-3000.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
++ mktemp
+ local LAST_OUT=/tmp/tmp.bphWdEvMiG
++ mktemp
+ local LAST_ERR=/tmp/tmp.xmWm5Y6JFM
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-696897d69b-7ct8v -- bash -c 'printf '\''use myApp\n db.test.insert({ x: 100501 })\n'\'' | mongo mongodb+srv://myApp:myPass@some-name-rs0.demand-backup-3000.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.bphWdEvMiG
Percona Server for MongoDB shell version v4.4.29-28
connecting to: mongodb://some-name-rs0-2.some-name-rs0.demand-backup-3000.svc.cluster.local:27017,some-name-rs0-0.some-name-rs0.demand-backup-3000.svc.cluster.local:27017,some-name-rs0-1.some-name-rs0.demand-backup-3000.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb&replicaSet=rs0&ssl=false
Implicit session: session { "id" : UUID("5c1e660c-2e12-4f3d-80a4-f5e7700c035e") }
Percona Server for MongoDB server version: v7.0.24-13
WARNING: shell and server versions do not match
switched to db myApp
WriteResult({ "nInserted" : 1 })
bye
+ cat /tmp/tmp.xmWm5Y6JFM
+ rm /tmp/tmp.bphWdEvMiG /tmp/tmp.xmWm5Y6JFM
+ return 0
+ compare_mongo_cmd find myApp:myPass@some-name-rs0.demand-backup-3000 -2nd .svc.cluster.local myApp test
+ local command=find
+ local uri=myApp:myPass@some-name-rs0.demand-backup-3000
+ local postfix=-2nd
+ local suffix=.svc.cluster.local
+ local database=myApp
+ local collection=test
+ local sort=
+ local tls=false
+ local 'full_command=db.test.find()'
+ [[ -n '' ]]
+ log 'running db.test.find() in myApp'
+ set +o xtrace
[2025-11-03T02:20:16+0000] running db.test.find() in myApp
+ [[ false == \t\r\u\e ]]
+ mongo_command=run_mongo
+ run_mongo 'use myApp\n db.test.find()' myApp:myPass@some-name-rs0.demand-backup-3000 mongodb .svc.cluster.local
+ local 'command=use myApp\n db.test.find()'
+ local uri=myApp:myPass@some-name-rs0.demand-backup-3000
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ egrep -v 'I NETWORK|W NETWORK|F NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|Started a new thread for the timer service'
+ /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
egrep: warning: egrep is obsolescent; using grep -E
++ local LAST_OUT=/tmp/tmp.qDK0ORbM6P
+++ mktemp
++ local LAST_ERR=/tmp/tmp.szAgEKPqAY
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.qDK0ORbM6P
++ cat /tmp/tmp.szAgEKPqAY
++ rm /tmp/tmp.qDK0ORbM6P /tmp/tmp.szAgEKPqAY
++ return 0
+ local client_container=psmdb-client-696897d69b-7ct8v
+ local mongo_flag=
+ [[ myApp:myPass@some-name-rs0.demand-backup-3000 == *cfg* ]]
+ replica_set=rs0
+ kubectl_bin exec psmdb-client-696897d69b-7ct8v -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0.demand-backup-3000.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
++ mktemp
+ local LAST_OUT=/tmp/tmp.w2gCn3hgzS
++ mktemp
+ local LAST_ERR=/tmp/tmp.QrfOH4r5sY
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-696897d69b-7ct8v -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0.demand-backup-3000.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.w2gCn3hgzS
+ cat /tmp/tmp.QrfOH4r5sY
+ rm /tmp/tmp.w2gCn3hgzS /tmp/tmp.QrfOH4r5sY
+ return 0
+ [[ 0 -eq 0 ]]
+ diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2101/e2e-tests/demand-backup/compare/find-2nd.json /tmp/tmp.NAG1atx8KS/find-2nd
+ run_restore_backupsource backup-minio-source-1 operator-testing/2025-11-03T02:14:26Z
+ local backupName=backup-minio-source-1
+ local backupDest=operator-testing/2025-11-03T02:14:26Z
+ local storageName=
+ desc 'run restore restore-backup-minio-source-1 from backup backup-minio-source-1 destination is operator-testing/2025-11-03T02:14:26Z'
+ set +o xtrace


-----------------------------------------------------------------------------------
run restore restore-backup-minio-source-1 from backup backup-minio-source-1 destination is operator-testing/2025-11-03T02:14:26Z
-----------------------------------------------------------------------------------

+ '[' -z '' ']'
+ cat /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2101/e2e-tests/demand-backup/conf/restore-backupsource.yml
+ /usr/sbin/sed -e 's/name:/name: restore-backup-minio-source-1/'
+ /usr/sbin/sed -e /storageName/d
+ kubectl_bin apply -f -
+ /usr/sbin/sed -e 's|BACKUP-NAME|operator-testing/2025-11-03T02:14:26Z|'
++ mktemp
+ local LAST_OUT=/tmp/tmp.yt8xsqKkl3
++ mktemp
+ local LAST_ERR=/tmp/tmp.qmPadMFyRA
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl apply -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.yt8xsqKkl3
perconaservermongodbrestore.psmdb.percona.com/restore-backup-minio-source-1 created
+ cat /tmp/tmp.qmPadMFyRA
+ rm /tmp/tmp.yt8xsqKkl3 /tmp/tmp.qmPadMFyRA
+ return 0
+ return
+ wait_restore backup-minio-source-1 some-name
+ local backup_name=backup-minio-source-1
+ local cluster_name=some-name
+ local target_state=ready
+ local wait_cluster_consistency=1
+ local wait_time=1780
+ local ok_if_ready=0
+ set +o xtrace
Waiting for the psmdb-restore/restore-backup-minio-source-1 object to be created.OK
Waiting psmdb-restore/restore-backup-minio-source-1 to reach state "ready" .OK after 0 minutes
+ [[ 1 -eq 1 ]]
+ wait_cluster_consistency some-name
+ local cluster_name=some-name
+ local wait_time=32
+ retry=0
+ sleep 7
+ echo -n 'waiting for cluster readyness'
waiting for cluster readyness++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.FsKYv6PcBi
+++ mktemp
++ local LAST_ERR=/tmp/tmp.A0EHTjmWKC
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.FsKYv6PcBi
++ cat /tmp/tmp.A0EHTjmWKC
++ rm /tmp/tmp.FsKYv6PcBi /tmp/tmp.A0EHTjmWKC
++ return 0
+ [[ ready == \r\e\a\d\y ]]
+ echo .OK
.OK
+ simple_data_check some-name-rs0 3 0
+ local cluster_name=some-name-rs0
+ let last_pod=3-1
+ local isSharded=0
+ local cluster_pfx=
+ '[' 0 -eq 1 ']'
++ seq 0 2
+ for i in $(seq 0 $last_pod)
+ compare_mongo_cmd find myApp:myPass@some-name-rs0-0.some-name-rs0.demand-backup-3000
+ local command=find
+ local uri=myApp:myPass@some-name-rs0-0.some-name-rs0.demand-backup-3000
+ local postfix=
+ local suffix=
+ local database=myApp
+ local collection=test
+ local sort=
+ local tls=false
+ local 'full_command=db.test.find()'
+ [[ -n '' ]]
+ log 'running db.test.find() in myApp'
+ set +o xtrace
[2025-11-03T02:20:41+0000] running db.test.find() in myApp
+ [[ false == \t\r\u\e ]]
+ mongo_command=run_mongo
+ run_mongo 'use myApp\n db.test.find()' myApp:myPass@some-name-rs0-0.some-name-rs0.demand-backup-3000 mongodb ''
+ local 'command=use myApp\n db.test.find()'
+ local uri=myApp:myPass@some-name-rs0-0.some-name-rs0.demand-backup-3000
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ egrep -v 'I NETWORK|W NETWORK|F NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|Started a new thread for the timer service'
+ /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
egrep: warning: egrep is obsolescent; using grep -E
++ local LAST_OUT=/tmp/tmp.zuJNowlNWS
+++ mktemp
++ local LAST_ERR=/tmp/tmp.1ICHeagcqt
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.zuJNowlNWS
++ cat /tmp/tmp.1ICHeagcqt
++ rm /tmp/tmp.zuJNowlNWS /tmp/tmp.1ICHeagcqt
++ return 0
+ local client_container=psmdb-client-696897d69b-7ct8v
+ local mongo_flag=
+ [[ myApp:myPass@some-name-rs0-0.some-name-rs0.demand-backup-3000 == *cfg* ]]
+ replica_set=rs0
+ kubectl_bin exec psmdb-client-696897d69b-7ct8v -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-0.some-name-rs0.demand-backup-3000.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
++ mktemp
+ local LAST_OUT=/tmp/tmp.up7YgiG0fM
++ mktemp
+ local LAST_ERR=/tmp/tmp.j3BFd8sSF3
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-696897d69b-7ct8v -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-0.some-name-rs0.demand-backup-3000.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.up7YgiG0fM
+ cat /tmp/tmp.j3BFd8sSF3
+ rm /tmp/tmp.up7YgiG0fM /tmp/tmp.j3BFd8sSF3
+ return 0
+ [[ 0 -eq 0 ]]
+ diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2101/e2e-tests/demand-backup/compare/find.json /tmp/tmp.NAG1atx8KS/find
+ for i in $(seq 0 $last_pod)
+ compare_mongo_cmd find myApp:myPass@some-name-rs0-1.some-name-rs0.demand-backup-3000
+ local command=find
+ local uri=myApp:myPass@some-name-rs0-1.some-name-rs0.demand-backup-3000
+ local postfix=
+ local suffix=
+ local database=myApp
+ local collection=test
+ local sort=
+ local tls=false
+ local 'full_command=db.test.find()'
+ [[ -n '' ]]
+ log 'running db.test.find() in myApp'
+ set +o xtrace
[2025-11-03T02:20:43+0000] running db.test.find() in myApp
+ [[ false == \t\r\u\e ]]
+ mongo_command=run_mongo
+ run_mongo 'use myApp\n db.test.find()' myApp:myPass@some-name-rs0-1.some-name-rs0.demand-backup-3000 mongodb ''
+ local 'command=use myApp\n db.test.find()'
+ local uri=myApp:myPass@some-name-rs0-1.some-name-rs0.demand-backup-3000
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ egrep -v 'I NETWORK|W NETWORK|F NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|Started a new thread for the timer service'
+ /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
egrep: warning: egrep is obsolescent; using grep -E
++ local LAST_OUT=/tmp/tmp.JTQW18viph
+++ mktemp
++ local LAST_ERR=/tmp/tmp.sm5xDQlY7b
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.JTQW18viph
++ cat /tmp/tmp.sm5xDQlY7b
++ rm /tmp/tmp.JTQW18viph /tmp/tmp.sm5xDQlY7b
++ return 0
+ local client_container=psmdb-client-696897d69b-7ct8v
+ local mongo_flag=
+ [[ myApp:myPass@some-name-rs0-1.some-name-rs0.demand-backup-3000 == *cfg* ]]
+ replica_set=rs0
+ kubectl_bin exec psmdb-client-696897d69b-7ct8v -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-1.some-name-rs0.demand-backup-3000.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
++ mktemp
+ local LAST_OUT=/tmp/tmp.vw2WXCTivs
++ mktemp
+ local LAST_ERR=/tmp/tmp.4UvgZGmeW4
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-696897d69b-7ct8v -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-1.some-name-rs0.demand-backup-3000.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.vw2WXCTivs
+ cat /tmp/tmp.4UvgZGmeW4
+ rm /tmp/tmp.vw2WXCTivs /tmp/tmp.4UvgZGmeW4
+ return 0
+ [[ 0 -eq 0 ]]
+ diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2101/e2e-tests/demand-backup/compare/find.json /tmp/tmp.NAG1atx8KS/find
+ for i in $(seq 0 $last_pod)
+ compare_mongo_cmd find myApp:myPass@some-name-rs0-2.some-name-rs0.demand-backup-3000
+ local command=find
+ local uri=myApp:myPass@some-name-rs0-2.some-name-rs0.demand-backup-3000
+ local postfix=
+ local suffix=
+ local database=myApp
+ local collection=test
+ local sort=
+ local tls=false
+ local 'full_command=db.test.find()'
+ [[ -n '' ]]
+ log 'running db.test.find() in myApp'
+ set +o xtrace
[2025-11-03T02:20:45+0000] running db.test.find() in myApp
+ [[ false == \t\r\u\e ]]
+ mongo_command=run_mongo
+ run_mongo 'use myApp\n db.test.find()' myApp:myPass@some-name-rs0-2.some-name-rs0.demand-backup-3000 mongodb ''
+ local 'command=use myApp\n db.test.find()'
+ egrep -v 'I NETWORK|W NETWORK|F NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|Started a new thread for the timer service'
+ local uri=myApp:myPass@some-name-rs0-2.some-name-rs0.demand-backup-3000
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
egrep: warning: egrep is obsolescent; using grep -E
++ local LAST_OUT=/tmp/tmp.KW1fXCFFrp
+++ mktemp
++ local LAST_ERR=/tmp/tmp.aC1y9A5CTO
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.KW1fXCFFrp
++ cat /tmp/tmp.aC1y9A5CTO
++ rm /tmp/tmp.KW1fXCFFrp /tmp/tmp.aC1y9A5CTO
++ return 0
+ local client_container=psmdb-client-696897d69b-7ct8v
+ local mongo_flag=
+ [[ myApp:myPass@some-name-rs0-2.some-name-rs0.demand-backup-3000 == *cfg* ]]
+ replica_set=rs0
+ kubectl_bin exec psmdb-client-696897d69b-7ct8v -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-2.some-name-rs0.demand-backup-3000.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
++ mktemp
+ local LAST_OUT=/tmp/tmp.tGywvXPCel
++ mktemp
+ local LAST_ERR=/tmp/tmp.PUCx3lcrGl
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-696897d69b-7ct8v -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-2.some-name-rs0.demand-backup-3000.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.tGywvXPCel
+ cat /tmp/tmp.PUCx3lcrGl
+ rm /tmp/tmp.tGywvXPCel /tmp/tmp.PUCx3lcrGl
+ return 0
+ [[ 0 -eq 0 ]]
+ diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2101/e2e-tests/demand-backup/compare/find.json /tmp/tmp.NAG1atx8KS/find
+ desc 'delete backup and check if it is removed from bucket -- minio'
+ set +o xtrace


-----------------------------------------------------------------------------------
delete backup and check if it is removed from bucket -- minio
-----------------------------------------------------------------------------------

+ kubectl_bin delete psmdb-backup --all
++ mktemp
+ local LAST_OUT=/tmp/tmp.NY9g8PgnMl
++ mktemp
+ local LAST_ERR=/tmp/tmp.e1nm9dBihx
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl delete psmdb-backup --all
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.NY9g8PgnMl
perconaservermongodbbackup.psmdb.percona.com "backup-aws-s3" deleted from demand-backup-3000 namespace
perconaservermongodbbackup.psmdb.percona.com "backup-azure-blob" deleted from demand-backup-3000 namespace
perconaservermongodbbackup.psmdb.percona.com "backup-gcp-cs-s3" deleted from demand-backup-3000 namespace
perconaservermongodbbackup.psmdb.percona.com "backup-gcp-cs-sa" deleted from demand-backup-3000 namespace
perconaservermongodbbackup.psmdb.percona.com "backup-minio" deleted from demand-backup-3000 namespace
+ cat /tmp/tmp.e1nm9dBihx
+ rm /tmp/tmp.NY9g8PgnMl /tmp/tmp.e1nm9dBihx
+ return 0
++ kubectl_bin run -i --rm aws-cli --image=perconalab/awscli --restart=Never -- /usr/bin/env AWS_ACCESS_KEY_ID=some-access-key AWS_SECRET_ACCESS_KEY=some-secret-key AWS_DEFAULT_REGION=us-east-1 /usr/bin/aws --endpoint-url http://minio-service:9000 s3 ls s3://operator-testing/
++ grep -c operator-testing/2025-11-03T02:14:26Z
++ cat
+++ mktemp
++ local LAST_OUT=/tmp/tmp.xzWdEuUoze
+++ mktemp
++ local LAST_ERR=/tmp/tmp.caVpEkEYzD
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl run -i --rm aws-cli --image=perconalab/awscli --restart=Never -- /usr/bin/env AWS_ACCESS_KEY_ID=some-access-key AWS_SECRET_ACCESS_KEY=some-secret-key AWS_DEFAULT_REGION=us-east-1 /usr/bin/aws --endpoint-url http://minio-service:9000 s3 ls s3://operator-testing/
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.xzWdEuUoze
++ cat /tmp/tmp.caVpEkEYzD
++ rm /tmp/tmp.xzWdEuUoze /tmp/tmp.caVpEkEYzD
++ return 0
+ backup_exists=0
+ [[ 0 -eq 1 ]]
+ '[' -z '' ']'
+ check_backup_deletion https://s3.amazonaws.com/operator-testing/psmdb-demand-backup/2025-11-03T02:13:52Z aws-s3
+ local path=https://s3.amazonaws.com/operator-testing/psmdb-demand-backup/2025-11-03T02:13:52Z
+ local storage_name=aws-s3
+ local retry=0
++ curl -sw '%{http_code}' -o /dev/null https://s3.amazonaws.com/operator-testing/psmdb-demand-backup/2025-11-03T02:13:52Z
+ [[ 403 -eq 403 ]]
+ check_backup_deletion https://storage.googleapis.com/operator-testing/psmdb-demand-backup/2025-11-03T02:14:15Z gcp-cs-s3
+ local path=https://storage.googleapis.com/operator-testing/psmdb-demand-backup/2025-11-03T02:14:15Z
+ local storage_name=gcp-cs-s3
+ local retry=0
++ curl -sw '%{http_code}' -o /dev/null https://storage.googleapis.com/operator-testing/psmdb-demand-backup/2025-11-03T02:14:15Z
+ [[ 404 -eq 403 ]]
++ curl -sw '%{http_code}' -o /dev/null https://storage.googleapis.com/operator-testing/psmdb-demand-backup/2025-11-03T02:14:15Z
+ [[ 404 -eq 404 ]]
+ check_backup_deletion https://storage.googleapis.com/operator-testing/psmdb-demand-backup/2025-11-03T02:14:38Z gcp-cs-sa
+ local path=https://storage.googleapis.com/operator-testing/psmdb-demand-backup/2025-11-03T02:14:38Z
+ local storage_name=gcp-cs-sa
+ local retry=0
++ curl -sw '%{http_code}' -o /dev/null https://storage.googleapis.com/operator-testing/psmdb-demand-backup/2025-11-03T02:14:38Z
+ [[ 404 -eq 403 ]]
++ curl -sw '%{http_code}' -o /dev/null https://storage.googleapis.com/operator-testing/psmdb-demand-backup/2025-11-03T02:14:38Z
+ [[ 404 -eq 404 ]]
+ check_backup_deletion https://engk8soperators.blob.core.windows.net/operator-testing/psmdb-demand-backup/2025-11-03T02:14:04Z azure-blob
+ local path=https://engk8soperators.blob.core.windows.net/operator-testing/psmdb-demand-backup/2025-11-03T02:14:04Z
+ local storage_name=azure-blob
+ local retry=0
++ curl -sw '%{http_code}' -o /dev/null https://engk8soperators.blob.core.windows.net/operator-testing/psmdb-demand-backup/2025-11-03T02:14:04Z
+ [[ 404 -eq 403 ]]
++ curl -sw '%{http_code}' -o /dev/null https://engk8soperators.blob.core.windows.net/operator-testing/psmdb-demand-backup/2025-11-03T02:14:04Z
+ [[ 404 -eq 404 ]]
+ desc 'checking backup deletion without cr'
+ set +o xtrace


-----------------------------------------------------------------------------------
checking backup deletion without cr
-----------------------------------------------------------------------------------

+ run_backup minio
+ local storage=minio
+ local backup_name=backup-minio
+ local type=logical
+ desc 'run backup backup-minio'
+ set +o xtrace


-----------------------------------------------------------------------------------
run backup backup-minio
-----------------------------------------------------------------------------------

+ yq eval '.metadata.name = "backup-minio"
			| .spec.storageName = "minio"
			| .spec.type = "logical"' /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2101/e2e-tests/demand-backup/conf/backup-minio.yml
+ kubectl_bin apply -f -
++ mktemp
+ local LAST_OUT=/tmp/tmp.fgBNODR9Ro
++ mktemp
+ local LAST_ERR=/tmp/tmp.zPH68Kp6Te
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl apply -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.fgBNODR9Ro
perconaservermongodbbackup.psmdb.percona.com/backup-minio created
+ cat /tmp/tmp.zPH68Kp6Te
+ rm /tmp/tmp.fgBNODR9Ro /tmp/tmp.zPH68Kp6Te
+ return 0
+ '[' -z '' ']'
+ run_backup aws-s3
+ local storage=aws-s3
+ local backup_name=backup-aws-s3
+ local type=logical
+ desc 'run backup backup-aws-s3'
+ set +o xtrace


-----------------------------------------------------------------------------------
run backup backup-aws-s3
-----------------------------------------------------------------------------------

+ yq eval '.metadata.name = "backup-aws-s3"
			| .spec.storageName = "aws-s3"
			| .spec.type = "logical"' /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2101/e2e-tests/demand-backup/conf/backup-aws-s3.yml
+ kubectl_bin apply -f -
++ mktemp
+ local LAST_OUT=/tmp/tmp.ZQdyTToyfB
++ mktemp
+ local LAST_ERR=/tmp/tmp.FRAurGs5Ww
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl apply -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.ZQdyTToyfB
perconaservermongodbbackup.psmdb.percona.com/backup-aws-s3 created
+ cat /tmp/tmp.FRAurGs5Ww
+ rm /tmp/tmp.ZQdyTToyfB /tmp/tmp.FRAurGs5Ww
+ return 0
+ run_backup gcp-cs-s3
+ local storage=gcp-cs-s3
+ local backup_name=backup-gcp-cs-s3
+ local type=logical
+ desc 'run backup backup-gcp-cs-s3'
+ set +o xtrace


-----------------------------------------------------------------------------------
run backup backup-gcp-cs-s3
-----------------------------------------------------------------------------------

+ yq eval '.metadata.name = "backup-gcp-cs-s3"
			| .spec.storageName = "gcp-cs-s3"
			| .spec.type = "logical"' /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2101/e2e-tests/demand-backup/conf/backup-gcp-cs-s3.yml
+ kubectl_bin apply -f -
++ mktemp
+ local LAST_OUT=/tmp/tmp.WEVb5IfKsX
++ mktemp
+ local LAST_ERR=/tmp/tmp.8F9u6clPvC
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl apply -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.WEVb5IfKsX
perconaservermongodbbackup.psmdb.percona.com/backup-gcp-cs-s3 created
+ cat /tmp/tmp.8F9u6clPvC
+ rm /tmp/tmp.WEVb5IfKsX /tmp/tmp.8F9u6clPvC
+ return 0
+ run_backup gcp-cs-sa
+ local storage=gcp-cs-sa
+ local backup_name=backup-gcp-cs-sa
+ local type=logical
+ desc 'run backup backup-gcp-cs-sa'
+ set +o xtrace


-----------------------------------------------------------------------------------
run backup backup-gcp-cs-sa
-----------------------------------------------------------------------------------

+ yq eval '.metadata.name = "backup-gcp-cs-sa"
			| .spec.storageName = "gcp-cs-sa"
			| .spec.type = "logical"' /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2101/e2e-tests/demand-backup/conf/backup-gcp-cs-sa.yml
+ kubectl_bin apply -f -
++ mktemp
+ local LAST_OUT=/tmp/tmp.Jyep12lj8i
++ mktemp
+ local LAST_ERR=/tmp/tmp.0TZgiWJyHP
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl apply -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.Jyep12lj8i
perconaservermongodbbackup.psmdb.percona.com/backup-gcp-cs-sa created
+ cat /tmp/tmp.0TZgiWJyHP
+ rm /tmp/tmp.Jyep12lj8i /tmp/tmp.0TZgiWJyHP
+ return 0
+ run_backup azure-blob
+ local storage=azure-blob
+ local backup_name=backup-azure-blob
+ local type=logical
+ desc 'run backup backup-azure-blob'
+ set +o xtrace


-----------------------------------------------------------------------------------
run backup backup-azure-blob
-----------------------------------------------------------------------------------

+ yq eval '.metadata.name = "backup-azure-blob"
			| .spec.storageName = "azure-blob"
			| .spec.type = "logical"' /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2101/e2e-tests/demand-backup/conf/backup-azure-blob.yml
+ kubectl_bin apply -f -
++ mktemp
+ local LAST_OUT=/tmp/tmp.jC669VMHRF
++ mktemp
+ local LAST_ERR=/tmp/tmp.1wxXZnvCuz
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl apply -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.jC669VMHRF
perconaservermongodbbackup.psmdb.percona.com/backup-azure-blob created
+ cat /tmp/tmp.1wxXZnvCuz
+ rm /tmp/tmp.jC669VMHRF /tmp/tmp.1wxXZnvCuz
+ return 0
+ wait_backup backup-minio
+ local backup_name=backup-minio
+ local target_state=ready
+ set +o xtrace
waiting for backup-minio to reach ready state...OK
+ '[' -z '' ']'
+ wait_backup backup-aws-s3
+ local backup_name=backup-aws-s3
+ local target_state=ready
+ set +o xtrace
waiting for backup-aws-s3 to reach ready state.................OK
+ wait_backup backup-gcp-cs-s3
+ local backup_name=backup-gcp-cs-s3
+ local target_state=ready
+ set +o xtrace
waiting for backup-gcp-cs-s3 to reach ready state.......OK
+ wait_backup backup-gcp-cs-sa
+ local backup_name=backup-gcp-cs-sa
+ local target_state=ready
+ set +o xtrace
waiting for backup-gcp-cs-sa to reach ready state..OK
+ wait_backup backup-azure-blob
+ local backup_name=backup-azure-blob
+ local target_state=ready
+ set +o xtrace
waiting for backup-azure-blob to reach ready state..OK
++ get_backup_dest backup-aws-s3
++ local backup_name=backup-aws-s3
++ kubectl_bin get psmdb-backup backup-aws-s3 -o 'jsonpath={.status.destination}'
++ sed -e 's/.json$//'
++ sed 's|s3://||'
++ sed 's|azure://||'
+++ mktemp
++ sed 's|gs://||'
++ local LAST_OUT=/tmp/tmp.K2wkbEAAq2
+++ mktemp
++ local LAST_ERR=/tmp/tmp.nw7ngbqexV
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb-backup backup-aws-s3 -o 'jsonpath={.status.destination}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.K2wkbEAAq2
++ cat /tmp/tmp.nw7ngbqexV
++ rm /tmp/tmp.K2wkbEAAq2 /tmp/tmp.nw7ngbqexV
++ return 0
+ backup_dest_aws=operator-testing/psmdb-demand-backup/2025-11-03T02:21:35Z
++ get_backup_dest backup-gcp-cs-s3
++ local backup_name=backup-gcp-cs-s3
++ kubectl_bin get psmdb-backup backup-gcp-cs-s3 -o 'jsonpath={.status.destination}'
++ sed -e 's/.json$//'
+++ mktemp
++ sed 's|azure://||'
++ sed 's|s3://||'
++ local LAST_OUT=/tmp/tmp.Otd9Bi3UWQ
+++ mktemp
++ sed 's|gs://||'
++ local LAST_ERR=/tmp/tmp.Tsnyi8Wk11
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb-backup backup-gcp-cs-s3 -o 'jsonpath={.status.destination}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.Otd9Bi3UWQ
++ cat /tmp/tmp.Tsnyi8Wk11
++ rm /tmp/tmp.Otd9Bi3UWQ /tmp/tmp.Tsnyi8Wk11
++ return 0
+ backup_dest_gcp_s3=operator-testing/psmdb-demand-backup/2025-11-03T02:21:47Z
++ get_backup_dest backup-gcp-cs-sa
++ local backup_name=backup-gcp-cs-sa
++ kubectl_bin get psmdb-backup backup-gcp-cs-sa -o 'jsonpath={.status.destination}'
++ sed -e 's/.json$//'
+++ mktemp
++ sed 's|s3://||'
++ local LAST_OUT=/tmp/tmp.kLA3jMBGvf
+++ mktemp
++ sed 's|azure://||'
++ sed 's|gs://||'
++ local LAST_ERR=/tmp/tmp.s79jTbumw4
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb-backup backup-gcp-cs-sa -o 'jsonpath={.status.destination}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.kLA3jMBGvf
++ cat /tmp/tmp.s79jTbumw4
++ rm /tmp/tmp.kLA3jMBGvf /tmp/tmp.s79jTbumw4
++ return 0
+ backup_dest_gcp_sa=operator-testing/psmdb-demand-backup/2025-11-03T02:21:13Z
++ get_backup_dest backup-azure-blob
++ local backup_name=backup-azure-blob
++ kubectl_bin get psmdb-backup backup-azure-blob -o 'jsonpath={.status.destination}'
++ sed -e 's/.json$//'
++ sed 's|s3://||'
++ sed 's|azure://||'
+++ mktemp
++ sed 's|gs://||'
++ local LAST_OUT=/tmp/tmp.kORo1FlmMu
+++ mktemp
++ local LAST_ERR=/tmp/tmp.B7vEQ24C3e
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb-backup backup-azure-blob -o 'jsonpath={.status.destination}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.kORo1FlmMu
++ cat /tmp/tmp.B7vEQ24C3e
++ rm /tmp/tmp.kORo1FlmMu /tmp/tmp.B7vEQ24C3e
++ return 0
+ backup_dest_azure=https://engk8soperators.blob.core.windows.net/operator-testing/psmdb-demand-backup/2025-11-03T02:21:24Z
+ kubectl_bin delete psmdb --all
++ mktemp
+ local LAST_OUT=/tmp/tmp.Ijpxs19zcC
++ mktemp
+ local LAST_ERR=/tmp/tmp.lny5D0M76v
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl delete psmdb --all
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.Ijpxs19zcC
perconaservermongodb.psmdb.percona.com "some-name" deleted from demand-backup-3000 namespace
+ cat /tmp/tmp.lny5D0M76v
+ rm /tmp/tmp.Ijpxs19zcC /tmp/tmp.lny5D0M76v
+ return 0
+ sleep 60
+ desc 'delete backup and check if it is removed from bucket -- minio'
+ set +o xtrace


-----------------------------------------------------------------------------------
delete backup and check if it is removed from bucket -- minio
-----------------------------------------------------------------------------------

+ kubectl_bin delete psmdb-backup --all
++ mktemp
+ local LAST_OUT=/tmp/tmp.Bl7ToO7SdD
++ mktemp
+ local LAST_ERR=/tmp/tmp.duVh8Fp8C8
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl delete psmdb-backup --all
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.Bl7ToO7SdD
perconaservermongodbbackup.psmdb.percona.com "backup-aws-s3" deleted from demand-backup-3000 namespace
perconaservermongodbbackup.psmdb.percona.com "backup-azure-blob" deleted from demand-backup-3000 namespace
perconaservermongodbbackup.psmdb.percona.com "backup-gcp-cs-s3" deleted from demand-backup-3000 namespace
perconaservermongodbbackup.psmdb.percona.com "backup-gcp-cs-sa" deleted from demand-backup-3000 namespace
perconaservermongodbbackup.psmdb.percona.com "backup-minio" deleted from demand-backup-3000 namespace
+ cat /tmp/tmp.duVh8Fp8C8
+ rm /tmp/tmp.Bl7ToO7SdD /tmp/tmp.duVh8Fp8C8
+ return 0
++ kubectl_bin run -i --rm aws-cli --image=perconalab/awscli --restart=Never -- /usr/bin/env AWS_ACCESS_KEY_ID=some-access-key AWS_SECRET_ACCESS_KEY=some-secret-key AWS_DEFAULT_REGION=us-east-1 /usr/bin/aws --endpoint-url http://minio-service:9000 s3 ls s3://operator-testing/
++ grep -c operator-testing/2025-11-03T02:14:26Z
++ cat
+++ mktemp
++ local LAST_OUT=/tmp/tmp.9kNRkNdfRe
+++ mktemp
++ local LAST_ERR=/tmp/tmp.7zRl0d46kX
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl run -i --rm aws-cli --image=perconalab/awscli --restart=Never -- /usr/bin/env AWS_ACCESS_KEY_ID=some-access-key AWS_SECRET_ACCESS_KEY=some-secret-key AWS_DEFAULT_REGION=us-east-1 /usr/bin/aws --endpoint-url http://minio-service:9000 s3 ls s3://operator-testing/
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.9kNRkNdfRe
++ cat /tmp/tmp.7zRl0d46kX
All commands and output from this session will be recorded in container logs, including credentials and sensitive information passed through the command prompt.
If you don't see a command prompt, try pressing enter.
warning: couldn't attach to pod/aws-cli, falling back to streaming logs: Internal error occurred: unable to upgrade connection: container aws-cli not found in pod aws-cli_demand-backup-3000
++ rm /tmp/tmp.9kNRkNdfRe /tmp/tmp.7zRl0d46kX
++ return 0
+ backup_exists=0
+ [[ 0 -eq 1 ]]
+ '[' -z '' ']'
+ check_backup_deletion https://s3.amazonaws.com/operator-testing/psmdb-demand-backup/2025-11-03T02:21:35Z aws-s3
+ local path=https://s3.amazonaws.com/operator-testing/psmdb-demand-backup/2025-11-03T02:21:35Z
+ local storage_name=aws-s3
+ local retry=0
++ curl -sw '%{http_code}' -o /dev/null https://s3.amazonaws.com/operator-testing/psmdb-demand-backup/2025-11-03T02:21:35Z
+ [[ 403 -eq 403 ]]
+ check_backup_deletion https://storage.googleapis.com/operator-testing/psmdb-demand-backup/2025-11-03T02:21:47Z gcp-cs-s3
+ local path=https://storage.googleapis.com/operator-testing/psmdb-demand-backup/2025-11-03T02:21:47Z
+ local storage_name=gcp-cs-s3
+ local retry=0
++ curl -sw '%{http_code}' -o /dev/null https://storage.googleapis.com/operator-testing/psmdb-demand-backup/2025-11-03T02:21:47Z
+ [[ 404 -eq 403 ]]
++ curl -sw '%{http_code}' -o /dev/null https://storage.googleapis.com/operator-testing/psmdb-demand-backup/2025-11-03T02:21:47Z
+ [[ 404 -eq 404 ]]
+ check_backup_deletion https://storage.googleapis.com/operator-testing/psmdb-demand-backup/2025-11-03T02:21:13Z gcp-cs-sa
+ local path=https://storage.googleapis.com/operator-testing/psmdb-demand-backup/2025-11-03T02:21:13Z
+ local storage_name=gcp-cs-sa
+ local retry=0
++ curl -sw '%{http_code}' -o /dev/null https://storage.googleapis.com/operator-testing/psmdb-demand-backup/2025-11-03T02:21:13Z
+ [[ 404 -eq 403 ]]
++ curl -sw '%{http_code}' -o /dev/null https://storage.googleapis.com/operator-testing/psmdb-demand-backup/2025-11-03T02:21:13Z
+ [[ 404 -eq 404 ]]
+ check_backup_deletion https://engk8soperators.blob.core.windows.net/operator-testing/psmdb-demand-backup/2025-11-03T02:21:24Z azure-blob
+ local path=https://engk8soperators.blob.core.windows.net/operator-testing/psmdb-demand-backup/2025-11-03T02:21:24Z
+ local storage_name=azure-blob
+ local retry=0
++ curl -sw '%{http_code}' -o /dev/null https://engk8soperators.blob.core.windows.net/operator-testing/psmdb-demand-backup/2025-11-03T02:21:24Z
+ [[ 404 -eq 403 ]]
++ curl -sw '%{http_code}' -o /dev/null https://engk8soperators.blob.core.windows.net/operator-testing/psmdb-demand-backup/2025-11-03T02:21:24Z
+ [[ 404 -eq 404 ]]
+ destroy demand-backup-3000
+ local namespace=demand-backup-3000
+ local ignore_logs=true
+ [[ 0 == 1 ]]
+ desc 'destroy cluster/operator and all other resources'
+ set +o xtrace


-----------------------------------------------------------------------------------
destroy cluster/operator and all other resources
-----------------------------------------------------------------------------------

+ '[' true == false ']'
+ delete_backups
+ desc 'Delete psmdb-backup'
+ set +o xtrace


-----------------------------------------------------------------------------------
Delete psmdb-backup
-----------------------------------------------------------------------------------

++ kubectl_bin get psmdb-backup --no-headers
++ wc -l
+++ mktemp
++ local LAST_OUT=/tmp/tmp.5xs4Ohb2fr
+++ mktemp
++ local LAST_ERR=/tmp/tmp.HBjMWoP34R
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb-backup --no-headers
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.5xs4Ohb2fr
++ cat /tmp/tmp.HBjMWoP34R
No resources found in demand-backup-3000 namespace.
++ rm /tmp/tmp.5xs4Ohb2fr /tmp/tmp.HBjMWoP34R
++ return 0
+ '[' 0 '!=' 0 ']'
+ delete_crd
+ desc 'get and delete old CRDs and RBAC'
+ set +o xtrace


-----------------------------------------------------------------------------------
get and delete old CRDs and RBAC
-----------------------------------------------------------------------------------

+ kubectl_bin delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2101/deploy/crd.yaml --ignore-not-found --wait=false
++ mktemp
+ local LAST_OUT=/tmp/tmp.EM7xJiT3kO
++ mktemp
+ local LAST_ERR=/tmp/tmp.s1pXFNof1T
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2101/deploy/crd.yaml --ignore-not-found --wait=false
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.EM7xJiT3kO
customresourcedefinition.apiextensions.k8s.io "perconaservermongodbbackups.psmdb.percona.com" deleted
customresourcedefinition.apiextensions.k8s.io "perconaservermongodbrestores.psmdb.percona.com" deleted
customresourcedefinition.apiextensions.k8s.io "perconaservermongodbs.psmdb.percona.com" deleted
+ cat /tmp/tmp.s1pXFNof1T
+ rm /tmp/tmp.EM7xJiT3kO /tmp/tmp.s1pXFNof1T
+ return 0
++ yq eval .metadata.name /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2101/deploy/crd.yaml
++ grep -v '\-\-\-'
grep: warning: stray \ before -
grep: warning: stray \ before -
+ for crd_name in $(yq eval '.metadata.name' "${src_dir}/deploy/crd.yaml" | grep -v '\-\-\-')
+ kubectl get perconaservermongodbbackups.psmdb.percona.com --all-namespaces -o wide
+ grep -v NAMESPACE
+ xargs -L 1 sh -xc 'kubectl patch perconaservermongodbbackups.psmdb.percona.com -n $0 $1 --type=merge -p "{\"metadata\":{\"finalizers\":[]}}"'
error: the server doesn't have a resource type "perconaservermongodbbackups"
+ kubectl patch perconaservermongodbbackups.psmdb.percona.com -n sh --type=merge -p '{"metadata":{"finalizers":[]}}'
error: the server doesn't have a resource type "perconaservermongodbbackups"
+ :
+ kubectl_bin wait --for=delete crd perconaservermongodbbackups.psmdb.percona.com
++ mktemp
+ local LAST_OUT=/tmp/tmp.pkX6hmUh0c
++ mktemp
+ local LAST_ERR=/tmp/tmp.bq4tNfGZtw
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl wait --for=delete crd perconaservermongodbbackups.psmdb.percona.com
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.pkX6hmUh0c
+ cat /tmp/tmp.bq4tNfGZtw
+ rm /tmp/tmp.pkX6hmUh0c /tmp/tmp.bq4tNfGZtw
+ return 0
+ for crd_name in $(yq eval '.metadata.name' "${src_dir}/deploy/crd.yaml" | grep -v '\-\-\-')
+ kubectl get perconaservermongodbrestores.psmdb.percona.com --all-namespaces -o wide
+ grep -v NAMESPACE
+ xargs -L 1 sh -xc 'kubectl patch perconaservermongodbrestores.psmdb.percona.com -n $0 $1 --type=merge -p "{\"metadata\":{\"finalizers\":[]}}"'
error: the server doesn't have a resource type "perconaservermongodbrestores"
+ kubectl patch perconaservermongodbrestores.psmdb.percona.com -n sh --type=merge -p '{"metadata":{"finalizers":[]}}'
error: the server doesn't have a resource type "perconaservermongodbrestores"
+ :
+ kubectl_bin wait --for=delete crd perconaservermongodbrestores.psmdb.percona.com
++ mktemp
+ local LAST_OUT=/tmp/tmp.tgdzNYv7ER
++ mktemp
+ local LAST_ERR=/tmp/tmp.GrGvEWuLsz
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl wait --for=delete crd perconaservermongodbrestores.psmdb.percona.com
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.tgdzNYv7ER
+ cat /tmp/tmp.GrGvEWuLsz
+ rm /tmp/tmp.tgdzNYv7ER /tmp/tmp.GrGvEWuLsz
+ return 0
+ for crd_name in $(yq eval '.metadata.name' "${src_dir}/deploy/crd.yaml" | grep -v '\-\-\-')
+ kubectl get perconaservermongodbs.psmdb.percona.com --all-namespaces -o wide
+ grep -v NAMESPACE
+ xargs -L 1 sh -xc 'kubectl patch perconaservermongodbs.psmdb.percona.com -n $0 $1 --type=merge -p "{\"metadata\":{\"finalizers\":[]}}"'
No resources found
+ kubectl patch perconaservermongodbs.psmdb.percona.com -n sh --type=merge -p '{"metadata":{"finalizers":[]}}'
error: the server doesn't have a resource type "perconaservermongodbs"
+ :
+ kubectl_bin wait --for=delete crd perconaservermongodbs.psmdb.percona.com
++ mktemp
+ local LAST_OUT=/tmp/tmp.XkGfciOu4d
++ mktemp
+ local LAST_ERR=/tmp/tmp.oLo10dhnkk
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl wait --for=delete crd perconaservermongodbs.psmdb.percona.com
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.XkGfciOu4d
+ cat /tmp/tmp.oLo10dhnkk
+ rm /tmp/tmp.XkGfciOu4d /tmp/tmp.oLo10dhnkk
+ return 0
+ local rbac_yaml=rbac.yaml
+ '[' -n psmdb-operator ']'
+ rbac_yaml=cw-rbac.yaml
+ kubectl_bin delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2101/deploy/cw-rbac.yaml --ignore-not-found
++ mktemp
+ local LAST_OUT=/tmp/tmp.hW374oBwNB
++ mktemp
+ local LAST_ERR=/tmp/tmp.et8cOqmVay
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2101/deploy/cw-rbac.yaml --ignore-not-found
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.hW374oBwNB
clusterrole.rbac.authorization.k8s.io "percona-server-mongodb-operator" deleted
clusterrolebinding.rbac.authorization.k8s.io "service-account-percona-server-mongodb-operator" deleted
+ cat /tmp/tmp.et8cOqmVay
+ rm /tmp/tmp.hW374oBwNB /tmp/tmp.et8cOqmVay
+ return 0
+ destroy_cert_manager
+ kubectl_bin delete -f https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml
++ mktemp
+ local LAST_OUT=/tmp/tmp.bGxlXBcNM7
++ mktemp
+ local LAST_ERR=/tmp/tmp.rn0eywABbG
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl delete -f https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml
+ exit_status=1
+ set -e
+ '[' 1 '!=' 0 -a -n 1 ']'
+ cat /tmp/tmp.bGxlXBcNM7
+ cat /tmp/tmp.rn0eywABbG
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml": namespaces "cert-manager" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "certificaterequests.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "certificates.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "challenges.acme.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "clusterissuers.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "issuers.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "orders.acme.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml": serviceaccounts "cert-manager-cainjector" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml": serviceaccounts "cert-manager" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml": serviceaccounts "cert-manager-webhook" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-cainjector" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-issuers" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-clusterissuers" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-certificates" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-orders" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-challenges" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-ingress-shim" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-cluster-view" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-view" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-edit" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-approve:cert-manager-io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-certificatesigningrequests" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-webhook:subjectaccessreviews" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-cainjector" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-issuers" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-clusterissuers" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-certificates" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-orders" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-challenges" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-ingress-shim" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-approve:cert-manager-io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-certificatesigningrequests" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-webhook:subjectaccessreviews" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager-cainjector:leaderelection" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager:leaderelection" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager-tokenrequest" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager-webhook:dynamic-serving" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager-cainjector:leaderelection" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager:leaderelection" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager-cert-manager-tokenrequest" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager-webhook:dynamic-serving" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml": services "cert-manager-cainjector" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml": services "cert-manager" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml": services "cert-manager-webhook" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml": deployments.apps "cert-manager-cainjector" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml": deployments.apps "cert-manager" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml": deployments.apps "cert-manager-webhook" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml": mutatingwebhookconfigurations.admissionregistration.k8s.io "cert-manager-webhook" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml": validatingwebhookconfigurations.admissionregistration.k8s.io "cert-manager-webhook" not found
+ sleep 0
+ for i in $(seq 0 2)
+ set +e
+ kubectl delete -f https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml
+ exit_status=1
+ set -e
+ '[' 1 '!=' 0 -a -n 1 ']'
+ cat /tmp/tmp.bGxlXBcNM7
+ cat /tmp/tmp.rn0eywABbG
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml": namespaces "cert-manager" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "certificaterequests.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "certificates.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "challenges.acme.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "clusterissuers.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "issuers.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "orders.acme.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml": serviceaccounts "cert-manager-cainjector" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml": serviceaccounts "cert-manager" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml": serviceaccounts "cert-manager-webhook" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-cainjector" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-issuers" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-clusterissuers" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-certificates" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-orders" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-challenges" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-ingress-shim" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-cluster-view" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-view" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-edit" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-approve:cert-manager-io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-certificatesigningrequests" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-webhook:subjectaccessreviews" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-cainjector" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-issuers" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-clusterissuers" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-certificates" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-orders" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-challenges" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-ingress-shim" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-approve:cert-manager-io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-certificatesigningrequests" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-webhook:subjectaccessreviews" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager-cainjector:leaderelection" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager:leaderelection" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager-tokenrequest" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager-webhook:dynamic-serving" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager-cainjector:leaderelection" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager:leaderelection" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager-cert-manager-tokenrequest" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager-webhook:dynamic-serving" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml": services "cert-manager-cainjector" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml": services "cert-manager" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml": services "cert-manager-webhook" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml": deployments.apps "cert-manager-cainjector" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml": deployments.apps "cert-manager" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml": deployments.apps "cert-manager-webhook" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml": mutatingwebhookconfigurations.admissionregistration.k8s.io "cert-manager-webhook" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml": validatingwebhookconfigurations.admissionregistration.k8s.io "cert-manager-webhook" not found
+ sleep 4
+ for i in $(seq 0 2)
+ set +e
+ kubectl delete -f https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml
+ exit_status=1
+ set -e
+ '[' 1 '!=' 0 -a -n 1 ']'
+ cat /tmp/tmp.bGxlXBcNM7
+ cat /tmp/tmp.rn0eywABbG
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml": namespaces "cert-manager" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "certificaterequests.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "certificates.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "challenges.acme.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "clusterissuers.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "issuers.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "orders.acme.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml": serviceaccounts "cert-manager-cainjector" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml": serviceaccounts "cert-manager" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml": serviceaccounts "cert-manager-webhook" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-cainjector" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-issuers" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-clusterissuers" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-certificates" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-orders" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-challenges" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-ingress-shim" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-cluster-view" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-view" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-edit" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-approve:cert-manager-io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-certificatesigningrequests" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-webhook:subjectaccessreviews" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-cainjector" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-issuers" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-clusterissuers" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-certificates" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-orders" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-challenges" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-ingress-shim" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-approve:cert-manager-io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-certificatesigningrequests" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-webhook:subjectaccessreviews" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager-cainjector:leaderelection" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager:leaderelection" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager-tokenrequest" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager-webhook:dynamic-serving" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager-cainjector:leaderelection" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager:leaderelection" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager-cert-manager-tokenrequest" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager-webhook:dynamic-serving" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml": services "cert-manager-cainjector" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml": services "cert-manager" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml": services "cert-manager-webhook" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml": deployments.apps "cert-manager-cainjector" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml": deployments.apps "cert-manager" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml": deployments.apps "cert-manager-webhook" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml": mutatingwebhookconfigurations.admissionregistration.k8s.io "cert-manager-webhook" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml": validatingwebhookconfigurations.admissionregistration.k8s.io "cert-manager-webhook" not found
+ sleep 8
+ cat /tmp/tmp.bGxlXBcNM7
+ cat /tmp/tmp.rn0eywABbG
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml": namespaces "cert-manager" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "certificaterequests.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "certificates.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "challenges.acme.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "clusterissuers.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "issuers.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "orders.acme.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml": serviceaccounts "cert-manager-cainjector" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml": serviceaccounts "cert-manager" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml": serviceaccounts "cert-manager-webhook" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-cainjector" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-issuers" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-clusterissuers" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-certificates" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-orders" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-challenges" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-ingress-shim" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-cluster-view" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-view" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-edit" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-approve:cert-manager-io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-certificatesigningrequests" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-webhook:subjectaccessreviews" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-cainjector" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-issuers" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-clusterissuers" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-certificates" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-orders" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-challenges" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-ingress-shim" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-approve:cert-manager-io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-certificatesigningrequests" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-webhook:subjectaccessreviews" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager-cainjector:leaderelection" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager:leaderelection" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager-tokenrequest" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager-webhook:dynamic-serving" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager-cainjector:leaderelection" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager:leaderelection" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager-cert-manager-tokenrequest" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager-webhook:dynamic-serving" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml": services "cert-manager-cainjector" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml": services "cert-manager" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml": services "cert-manager-webhook" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml": deployments.apps "cert-manager-cainjector" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml": deployments.apps "cert-manager" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml": deployments.apps "cert-manager-webhook" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml": mutatingwebhookconfigurations.admissionregistration.k8s.io "cert-manager-webhook" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml": validatingwebhookconfigurations.admissionregistration.k8s.io "cert-manager-webhook" not found
+ rm /tmp/tmp.bGxlXBcNM7 /tmp/tmp.rn0eywABbG
+ return 1
+ true
+ '[' -n '' ']'
+ '[' -n psmdb-operator ']'
+ kubectl_bin delete --grace-period=0 --force=true namespace demand-backup-3000
+ kubectl_bin delete --grace-period=0 --force=true namespace psmdb-operator
+ rm -rf /tmp/tmp.NAG1atx8KS
++ mktemp
++ mktemp
+ local LAST_OUT=/tmp/tmp.PH7MgEEKWy
+ desc 'test passed'
+ set +o xtrace


-----------------------------------------------------------------------------------
test passed
-----------------------------------------------------------------------------------

+ local LAST_OUT=/tmp/tmp.4izVdeXwt5
++ mktemp
++ mktemp
+ local LAST_ERR=/tmp/tmp.ZPO28Utdvb
+ local exit_status=0
+ local timeout=4
+ local LAST_ERR=/tmp/tmp.pBcecRl1JB
+ local exit_status=0
+ local timeout=4
++ seq 0 2
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl delete --grace-period=0 --force=true namespace demand-backup-3000
+ for i in $(seq 0 2)
+ set +e
+ kubectl delete --grace-period=0 --force=true namespace psmdb-operator